diff --git a/asap-tools/data-sources/paced-feeder/src/table.rs b/asap-tools/data-sources/paced-feeder/src/table.rs index 97cd6546..dd33aa6b 100644 --- a/asap-tools/data-sources/paced-feeder/src/table.rs +++ b/asap-tools/data-sources/paced-feeder/src/table.rs @@ -57,6 +57,15 @@ struct GaugeValues { dist: ParetoDist, } +/// A recording to replay instead of generated values: `steps` values per +/// series, series-major in [`series_labels`] order. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ReplayValues { + steps: usize, + values: Vec, +} + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct ParetoDist { @@ -161,6 +170,31 @@ fn metric(name: &str, spec: &Value, seed: u64) -> Result<(Metric, u32)> { ); } let series = series_labels(&spec.labels)?; + if let Some(replay) = spec.values.get("replay") { + ensure!(spec.kind == "gauge", "`replay` values apply to gauges only"); + let replay: ReplayValues = + serde_json::from_value(replay.clone()).context("gauge `values.replay`")?; + ensure!(replay.steps > 0, "`replay.steps` must be positive"); + ensure!( + replay.values.len() == series.len() * replay.steps, + "`replay.values` has {} values, not {} series x {} steps", + replay.values.len(), + series.len(), + replay.steps + ); + let metric = Metric { + name: name.to_string(), + labels: spec.labels, + scales: vec![1.0; series.len()], + series, + kind: Kind::Replay { + steps: replay.steps, + values: Arc::new(replay.values), + }, + seed: splitmix64(seed ^ fnv1a(name)), + }; + return Ok((metric, spec.samples_per_sec_per_series)); + } let (skew, kind) = if spec.kind == "gauge" { let values: GaugeValues = serde_json::from_value(spec.values).context("gauge `values`")?; ensure!( @@ -415,6 +449,24 @@ mod tests { assert_eq!(series[11], ["1", "2", "2.1"]); } + #[test] + fn replay_values_loop_per_series() { + let spec = json!({ + "type": "gauge", + "samples_per_sec_per_series": 10, + "labels": [{"name": "job", "cardinality": 2}], + "values": {"replay": {"steps": 3, "values": [1.0, 2.0, 3.0, 10.0, 20.0, 30.0]}}, + }); + let (m, rate) = metric("data", &spec, 7).unwrap(); + assert_eq!(rate, 10); + let seen: Vec = (0..4).map(|n| m.value(1, n)).collect(); + assert_eq!(seen, [10.0, 20.0, 30.0, 10.0]); + // One value per series and step, no more and no fewer. + let mut short = spec.clone(); + short["values"]["replay"]["values"] = json!([1.0, 2.0]); + assert!(metric("data", &short, 7).is_err()); + } + #[test] fn rejects_a_child_before_its_parent() { let labels = vec![label("label_0", 3, Some("job")), label("job", 2, None)]; diff --git a/asap-tools/data-sources/paced-feeder/src/workload.rs b/asap-tools/data-sources/paced-feeder/src/workload.rs index 81acc6d6..a25c8a4b 100644 --- a/asap-tools/data-sources/paced-feeder/src/workload.rs +++ b/asap-tools/data-sources/paced-feeder/src/workload.rs @@ -59,6 +59,9 @@ pub enum Kind { /// `(n + 1) ยท scale` at sample index `n`: starts at the first sample, grows /// by the series' scale per sample and never resets. Counter, + /// Recorded values, `steps` per series, series-major: sample index `n` + /// takes step `n mod steps`, so the recording loops. + Replay { steps: usize, values: Arc> }, } #[derive(Debug, Clone)] @@ -213,6 +216,9 @@ impl Metric { scale * pareto_scale * u.powf(-1.0 / pareto_shape) } Kind::Counter => (sample_index + 1) as f64 * scale, + Kind::Replay { steps, ref values } => { + values[series * steps + (sample_index % steps as u64) as usize] + } } } } diff --git a/asap-tools/experiments/datasets/alibaba/make_replay.py b/asap-tools/experiments/datasets/alibaba/make_replay.py new file mode 100644 index 00000000..1f5b469a --- /dev/null +++ b/asap-tools/experiments/datasets/alibaba/make_replay.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""A replay table for paced-feeder from Alibaba v2022 MSMetrics: pod CPU +utilization in the random workloads' label shape (4 region x 5 service x 10 +job x 50 label_0), so their plans apply unchanged. + +Each job is one microservice and each label_0 one of its pods; region and +service only group microservices. Label values follow the feeder's table +convention: a child's value is its parent's, a `.`, and its index (`job="3.1.7"`). +Microservices are drawn by `--seed` among those with at least 50 pods that +report at every step of the input; pods are their first 50 by name. + +Writes `.csv` (`offset_ms,region,service,job,label_0,value`, sorted by +offset, offsets from the first step) and `_mapping.csv` (labels to +msname/msinstanceid). With `--table`, also `_table.json`: that +workload table with its `data` metric's values replaced by the replay, which +paced-feeder loops (one trace step per sample). + +Usage: python3 make_replay.py MSMetricsUpdate_0.csv MSMetricsUpdate_1.csv \ + --out replay [--table eval_table.json] +""" + +import argparse +import csv +import json +import random +from collections import defaultdict + +REGIONS, SERVICES, JOBS, PODS = 4, 5, 10, 50 + + +def rows(paths): + for path in paths: + with open(path, newline="") as f: + for r in csv.DictReader(f): + yield r + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("csv", nargs="+") + parser.add_argument("--out", required=True) + parser.add_argument("--seed", type=int, default=862) + parser.add_argument("--table", help="workload table whose `data` gets the replay") + args = parser.parse_args() + + steps, seen = set(), defaultdict(set) # pod -> steps + pods = defaultdict(set) # msname -> pods + for r in rows(args.csv): + steps.add(r["timestamp"]) + seen[r["msinstanceid"]].add(r["timestamp"]) + pods[r["msname"]].add(r["msinstanceid"]) + full = {p for p, s in seen.items() if len(s) == len(steps)} + eligible = sorted(m for m, ps in pods.items() if len(ps & full) >= PODS) + jobs = REGIONS * SERVICES * JOBS + print(f"{len(steps)} steps; {len(eligible)} microservices eligible") + chosen = random.Random(args.seed).sample(eligible, jobs) + + labels = {} # pod -> (region, service, job, label_0) + with open(f"{args.out}_mapping.csv", "w", newline="") as f: + w = csv.writer(f) + w.writerow(["region", "service", "job", "label_0", "msname", "msinstanceid"]) + for i, msname in enumerate(chosen): + region = str(i // (SERVICES * JOBS)) + service = f"{region}.{i // JOBS % SERVICES}" + job = f"{service}.{i % JOBS}" + ordered = sorted( + pods[msname] & full, key=lambda p: int(p.rsplit("_", 1)[1]) + ) + for k, pod in enumerate(ordered[:PODS]): + labels[pod] = (region, service, job, f"{job}.{k}") + w.writerow([*labels[pod], msname, pod]) + + start = min(int(s) for s in steps) + out = sorted( + (int(r["timestamp"]) - start, *labels[r["msinstanceid"]], r["cpu_utilization"]) + for r in rows(args.csv) + if r["msinstanceid"] in labels + ) + with open(f"{args.out}.csv", "w", newline="") as f: + w = csv.writer(f) + w.writerow(["offset_ms", "region", "service", "job", "label_0", "value"]) + w.writerows(out) + print(f"{len(out)} rows, {len(labels)} series") + if args.table: + write_table(args.table, f"{args.out}_table.json", out, len(steps)) + + +def write_table(table_path, out_path, rows, steps): + """`table_path` with `data`'s values replaced by `rows`, series-major in + paced-feeder's series order (mixed radix, label_0 fastest).""" + by_series = {} + for offset, *_, label_0, value in rows: # sorted by offset + by_series.setdefault(label_0, []).append(round(float(value), 6)) + values = [] + for r in range(REGIONS): + for s in range(SERVICES): + for j in range(JOBS): + for k in range(PODS): + values += by_series[f"{r}.{s}.{j}.{k}"] + assert len(values) == len(by_series) * steps + table = json.load(open(table_path)) + data = table["workloads"][0]["e2e"]["metrics"]["data"] + data["values"] = {"replay": {"steps": steps, "values": values}} + with open(out_path, "w") as f: + json.dump(table, f) + print(out_path) + + +if __name__ == "__main__": + main() diff --git a/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py b/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py index 289236a8..5a1fb248 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py +++ b/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py @@ -133,7 +133,24 @@ def workload_set(workload): return f"seed-{workload['seed']}" + (f"-{variant}" if variant else "") -def pilot_configs(workload, pilot_dir): +def as_dashboards(groups, dashboards, key): + """`groups` repeated once per dashboard, each copy its own client thread + and output group; `key` ("id" or "name") stays unique per copy.""" + if dashboards == 1: + return groups + copies = [] + for d in range(1, dashboards + 1): + for group in groups: + group = copy.deepcopy(group) + if key == "id": + group["id"] = len(copies) + 1 + else: + group[key] = f"{group[key]}_d{d}" + copies.append(group) + return copies + + +def pilot_configs(workload, pilot_dir, dashboards=1): """The five-system pilot's run configs, {path under EXPERIMENTS_DIR: content}: ASAPQuery (the saved plan in `pilot_dir`) and Prometheus in one experiment_run_e2e.py config, VictoriaMetrics in another, and raw @@ -144,7 +161,10 @@ def pilot_configs(workload, pilot_dir): freshness delay, then evaluates each query at multiples of its interval for PILOT_DURATION_MS: the same windows relative to its data start. Raw ClickHouse runs the manifest's clickhouse_sql, and ClickHouse+MV - the same RQEs' Query over exact views (sketch_bench_workloads).""" + the same RQEs' Query over exact views (sketch_bench_workloads). + + With `dashboards` K > 1, every query group runs K times concurrently, + as K dashboards over the same data, from `-dashK/`.""" queries = [q for q in workload["queries"] if q["support"] == PILOT_SUPPORT] lookback_secs = max(q["lookback_ms"] for q in queries) // 1000 starting_delay = lookback_secs + QUERY_FRESHNESS_DELAY_SECS @@ -152,7 +172,8 @@ def pilot_configs(workload, pilot_dir): feed_secs = starting_delay + FEED_BOUND_QUERY_PHASES * duration seed = workload["data_facts"]["feeder_seed"] table = pilot_dir / "eval_table.json" - out_dir = Path(CONFIG_DIR) / workload_set(workload) / workload["id"] + variant = f"-dash{dashboards}" if dashboards > 1 else "" + out_dir = Path(CONFIG_DIR) / workload_set(workload) / (workload["id"] + variant) e2e, _ = build_config( str(table), @@ -163,6 +184,10 @@ def pilot_configs(workload, pilot_dir): feeder_duration_secs=feed_secs, prebuilt_output_dir=str(pilot_dir), ) + e2e_params = e2e["experiment_params"] + e2e_params["query_groups"] = as_dashboards( + e2e_params["query_groups"], dashboards, "id" + ) # One monitoring tool per run, so VictoriaMetrics is its own baseline run. vm = copy.deepcopy(e2e) del vm["controller"] @@ -232,7 +257,7 @@ def pilot_configs(workload, pilot_dir): "paced_feeder": feeder, "clickhouse_image_tag": CLICKHOUSE_IMAGE_TAG, }, - "query_groups": groups, + "query_groups": as_dashboards(groups, dashboards, "name"), "data_ingestion_interval_ms": workload["data_facts"]["scrape_interval_ms"], } files[name] = "# @package experiment_params\n" + yaml.safe_dump( @@ -277,7 +302,16 @@ def main(): help="the workload's archived pilot plan (eval_table.json and planner " f"configs): also write the pilot's run configs under {CONFIG_DIR}", ) + parser.add_argument( + "--dashboards", + type=int, + default=1, + help="with --pilot-dir: issue every query group this many times " + "concurrently, into -dashK/", + ) args = parser.parse_args() + if args.dashboards < 1: + parser.error("--dashboards must be at least 1") workload = json.loads(args.manifest.read_text()) rendered = render(workload) args.out.mkdir(parents=True, exist_ok=True) @@ -305,7 +339,9 @@ def main(): ) print(table) if args.pilot_dir: - for path, content in pilot_configs(workload, args.pilot_dir).items(): + for path, content in pilot_configs( + workload, args.pilot_dir, args.dashboards + ).items(): path = Path(EXPERIMENTS_DIR) / path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content) diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads.py b/asap-tools/experiments/datasets/fake_metric/random_workloads.py index 24834873..5bb2a4d6 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads.py +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads.py @@ -69,6 +69,8 @@ "base": (1, 50), # Lookbacks x3: 1m -> 3m, ..., 20m -> 60m; quantiles 100s -> 300s. "long": (3, 50), + # x2 series: 20,000 at 10/s, 200,000 samples/s. + "x2": (1, 100), # x5 series: 50,000 at 10/s, 500,000 samples/s. "x5": (1, 250), } diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/eval_table.json new file mode 100644 index 00000000..a83c16b5 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/eval_table.json @@ -0,0 +1,574 @@ +{ + "inputs": { + "workload-01.json": "e77d8bca3a39966f67be06c554d6866be939c444dac5f87be2c677bb74acefd7" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-01", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-01/q01-sum-fine/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 20, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-01/q01-sum-fine/keys/600s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/q02-sum-coarse/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-01/q02-sum-coarse/keys/600s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 1, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-01/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/topk4_q07-topk-sum/keys/1200s", + "k": 4, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 30, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-01/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-01/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 600000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 12000000.0, + "covered_min_N": 12000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 12000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 12000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/inference_config.yaml new file mode 100644 index 00000000..9a005085 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/inference_config.yaml @@ -0,0 +1,80 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[10m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[10m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[10m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[10m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 59 + query: topk by (region, service) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 40 + query: topk by (region, service) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/plans.json.gz new file mode 100644 index 00000000..27c551e8 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/streaming_config.yaml new file mode 100644 index 00000000..c0e3e9f7 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 25000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 20000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: sliding +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01.json new file mode 100644 index 00000000..23c5686a --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01.json @@ -0,0 +1,363 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-01", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 20) * 20 - 600) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 20) * 20 - 600)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 20) * 20 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 20) * 20 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 20000, + "k": 8, + "lookback_ms": 600000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q10-cardinality", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY region, service", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/eval_table.json new file mode 100644 index 00000000..d344d026 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/eval_table.json @@ -0,0 +1,573 @@ +{ + "inputs": { + "workload-02.json": "8e17c6d061acf0722bc5a7b67e163d17ac64486022c6119d45dfc2011148f32c" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-02", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-02/q01-sum-fine/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 5, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-02/q01-sum-fine/keys/300s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-02/q02-sum-coarse/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 5, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-02/q02-sum-coarse/keys/300s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-02/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-02/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-02/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 1, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-02/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/topk4_q07-topk-sum/keys/1200s", + "k": 4, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 60, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-02/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-02/q01-sum-fine/keys/300s", + "instant": false, + "interval_secs": 5, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-02/q02-sum-coarse/keys/300s", + "instant": false, + "interval_secs": 5, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-02/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 12000000.0, + "covered_min_N": 12000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-02/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 12000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 12000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/inference_config.yaml new file mode 100644 index 00000000..3d590028 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/inference_config.yaml @@ -0,0 +1,79 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service) (sum_over_time(data[5m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[5m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service, job) (sum_over_time(data[5m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 59 + query: topk by (region, service) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 20 + query: topk by (region, service) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/plans.json.gz new file mode 100644 index 00000000..4047bab0 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/streaming_config.yaml new file mode 100644 index 00000000..984fbe74 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 25000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 5000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 5000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: sliding +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 80 + width: 4096 + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02.json new file mode 100644 index 00000000..4c04d6cd --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02.json @@ -0,0 +1,361 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-02", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 5) * 5 - 300) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 5) * 5 - 300)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 5) * 5 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 5) * 5 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q08-topk-count", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, increase(data_total[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 5000, + "k": 4, + "lookback_ms": 300000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/eval_table.json new file mode 100644 index 00000000..a5910fbd --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/eval_table.json @@ -0,0 +1,520 @@ +{ + "inputs": { + "workload-03.json": "ac913568d60bae42366e422f000fd20d9bdfbe37d32cb8b0a4feca21ab9d175f" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-03", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-03/q01-sum-fine/keys/60s", + "lookback_ms": 60000, + "metric": "data" + } + }, + "interval_secs": 1, + "promql": "sum by (region, service, job) (sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-03/q01-sum-fine/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-03/q02-sum-coarse/keys/60s", + "lookback_ms": 60000, + "metric": "data" + } + }, + "interval_secs": 1, + "promql": "sum by (region, service) (sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-03/q02-sum-coarse/keys/60s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-03/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-03/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-03/q04-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 1, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-03/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/topk4_q07-topk-sum/keys/1200s", + "k": 4, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 30, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-03/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 60000.0, + "covered_min_N": 60000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-03/q01-sum-fine/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 60.0, + "max_N": 60000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 60000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-03/q02-sum-coarse/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-03/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 60.0, + "max_N": 3000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 60000000.0, + "covered_min_N": 60000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-03/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 1200.0, + "max_N": 60000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 60000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/inference_config.yaml new file mode 100644 index 00000000..3e9e32f5 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/inference_config.yaml @@ -0,0 +1,80 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service) (sum_over_time(data[1m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[1m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service, job) (sum_over_time(data[1m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 60 + query: topk by (region) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 40 + query: topk by (region) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service, job) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/plans.json.gz new file mode 100644 index 00000000..11d056d5 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/streaming_config.yaml new file mode 100644 index 00000000..d1b13eb9 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 240 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03.json new file mode 100644 index 00000000..1573d1f5 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03.json @@ -0,0 +1,361 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-03", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 1000, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 1000, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 1) * 1 - 60) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 1) * 1 - 60)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 1) * 1 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 1) * 1 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 1000, + "k": 8, + "lookback_ms": 60000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 1000, + "lookback_ms": 60000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -60, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/eval_table.json new file mode 100644 index 00000000..40ae391e --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/eval_table.json @@ -0,0 +1,519 @@ +{ + "inputs": { + "workload-04.json": "11e77e8ab7e51caf1cb2a9af62f6589fd6a8cc8908cb2bc77e122d4a33cb93e1" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-04", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-04/q01-sum-fine/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 10, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-04/q01-sum-fine/keys/600s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-04/q02-sum-coarse/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 10, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-04/q02-sum-coarse/keys/600s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-04/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-04/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-04/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 2, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-04/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/topk4_q07-topk-sum/keys/1200s", + "k": 4, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 30, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-04/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-04/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 600000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-04/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-04/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 60000000.0, + "covered_min_N": 60000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-04/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 1200.0, + "max_N": 60000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 60000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/inference_config.yaml new file mode 100644 index 00000000..ec06c999 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/inference_config.yaml @@ -0,0 +1,77 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service) (sum_over_time(data[10m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[10m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service, job) (sum_over_time(data[10m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[10m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 40 + query: topk by (region) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 30 + query: topk by (region, service) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/plans.json.gz new file mode 100644 index 00000000..dfa83e4b Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/streaming_config.yaml new file mode 100644 index 00000000..636436db --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 10000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 10000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 2000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04.json new file mode 100644 index 00000000..e72247ed --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04.json @@ -0,0 +1,357 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-04", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 10000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 10000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 10000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 10000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 2000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 10) * 10 - 600) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 10) * 10 - 600)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 10) * 10 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 10) * 10 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 10000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, increase(data_total[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 10000, + "k": 4, + "lookback_ms": 600000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region", + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 32, + "promql": "topk by (region) (8, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 10) * 10 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 10000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 10000, + "lookback_ms": 600000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/eval_table.json new file mode 100644 index 00000000..1a032558 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/eval_table.json @@ -0,0 +1,571 @@ +{ + "inputs": { + "workload-05.json": "6141edf0c371ca529b4867f2383a0c5e0cf6ee4a744ab3e678b8731ddd3f6942" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-05", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-05/q01-sum-fine/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 10, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-05/q01-sum-fine/keys/300s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-05/q02-sum-coarse/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 10, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-05/q02-sum-coarse/keys/300s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-05/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + "random/seed=862/workload-05/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-05/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/topk8_q06-topk-sum/keys/60s", + "k": 8, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 1, + "promql": "topk by (region, service) (8, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-05/topk8_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/topk4_q07-topk-sum/keys/1200s", + "k": 4, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 60, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-05/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-05/q01-sum-fine/keys/300s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-05/q02-sum-coarse/keys/300s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-05/topk8_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 60000000.0, + "covered_min_N": 60000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-05/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 1200.0, + "max_N": 60000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 60000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/inference_config.yaml new file mode 100644 index 00000000..fd47a6c3 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/inference_config.yaml @@ -0,0 +1,79 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[5m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[5m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[5m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 20 + query: topk by (region) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 59 + query: topk by (region, service) (8, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.50, data[100s])) + planned_subquery: quantile_over_time(0.5, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/plans.json.gz new file mode 100644 index 00000000..dcc186b6 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/streaming_config.yaml new file mode 100644 index 00000000..617dd9e1 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 25000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 10000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 10000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 80 + width: 4096 + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 240 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: sliding +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05.json new file mode 100644 index 00000000..9bb2f75d --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05.json @@ -0,0 +1,361 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-05", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.50)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 8, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 10) * 10 - 300) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 10) * 10 - 300)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 10) * 10 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 10) * 10 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 10000, + "k": 8, + "lookback_ms": 300000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region", + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 32, + "promql": "topk by (region) (8, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/eval_table.json new file mode 100644 index 00000000..eb948fe1 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/eval_table.json @@ -0,0 +1,523 @@ +{ + "inputs": { + "workload-06.json": "bdf7ab7c3dd9d8313a2c252a2ef6daca6f482993b489d319c011e3cd0ffed374" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-06", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-06/q01-sum-fine/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 5, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-06/q01-sum-fine/keys/300s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-06/q02-sum-coarse/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 5, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-06/q02-sum-coarse/keys/300s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + "random/seed=862/workload-06/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + "random/seed=862/workload-06/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-06/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 2, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-06/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/topk8_q07-topk-sum/keys/1200s", + "k": 8, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 60, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-06/topk8_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-06/q01-sum-fine/keys/300s", + "instant": false, + "interval_secs": 5, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-06/q02-sum-coarse/keys/300s", + "instant": false, + "interval_secs": 5, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-06/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 12000000.0, + "covered_min_N": 12000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-06/topk8_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 12000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 12000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/inference_config.yaml new file mode 100644 index 00000000..d31f2ec0 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/inference_config.yaml @@ -0,0 +1,79 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service) (sum_over_time(data[5m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[5m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service, job) (sum_over_time(data[5m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 30 + query: topk by (region, service) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 20 + query: topk by (region, service) (8, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service, job) (quantile_over_time(0.50, data[100s])) + planned_subquery: quantile_over_time(0.5, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.50, data[100s])) + planned_subquery: quantile_over_time(0.5, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/plans.json.gz new file mode 100644 index 00000000..eb814cbd Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/streaming_config.yaml new file mode 100644 index 00000000..38466937 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 5000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 5000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 2000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06.json new file mode 100644 index 00000000..417d8b6c --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06.json @@ -0,0 +1,365 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-06", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.50)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.50)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 2000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 5) * 5 - 300) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 5) * 5 - 300)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 5) * 5 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 5) * 5 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 5000, + "k": 8, + "lookback_ms": 300000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region", + "service" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/eval_table.json new file mode 100644 index 00000000..63641254 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/eval_table.json @@ -0,0 +1,572 @@ +{ + "inputs": { + "workload-07.json": "1046b2b6cdc6b38b5b54f5b291d5763b913a878db2dc2755bc289dc0114060cc" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-07", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-07/q01-sum-fine/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 20, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-07/q01-sum-fine/keys/600s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-07/q02-sum-coarse/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-07/q02-sum-coarse/keys/600s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-07/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-07/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + "random/seed=862/workload-07/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 2, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-07/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/topk8_q07-topk-sum/keys/1200s", + "k": 8, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 60, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-07/topk8_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-07/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 600000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-07/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-07/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 60.0, + "max_N": 3000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 12000000.0, + "covered_min_N": 12000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-07/topk8_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 12000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 12000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/inference_config.yaml new file mode 100644 index 00000000..7832e815 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/inference_config.yaml @@ -0,0 +1,80 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[10m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[10m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[10m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[10m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 30 + query: topk by (region) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 20 + query: topk by (region, service) (8, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.50, data[100s])) + planned_subquery: quantile_over_time(0.5, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/plans.json.gz new file mode 100644 index 00000000..5f5c73c2 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/streaming_config.yaml new file mode 100644 index 00000000..17f3458b --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 25000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 20000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 2000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07.json new file mode 100644 index 00000000..d9f016bc --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07.json @@ -0,0 +1,359 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-07", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.50)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q06-topk-sum", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 2000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 20) * 20 - 600) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 20) * 20 - 600)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 20) * 20 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 20) * 20 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q08-topk-count", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, increase(data_total[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 20000, + "k": 4, + "lookback_ms": 600000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region", + "service" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q10-cardinality", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY region", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/eval_table.json new file mode 100644 index 00000000..e59dcf2b --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/eval_table.json @@ -0,0 +1,518 @@ +{ + "inputs": { + "workload-08.json": "ee2864b9ecf924c78c7c5d2a0aa3afcca8d38fb9d84f9364da129da66483ee8d" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-08", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-08/q01-sum-fine/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 10, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-08/q01-sum-fine/keys/300s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-08/q02-sum-coarse/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 10, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-08/q02-sum-coarse/keys/300s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-08/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-08/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-08/q04-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 1, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-08/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/topk4_q07-topk-sum/keys/1200s", + "k": 4, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 30, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-08/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-08/q01-sum-fine/keys/300s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-08/q02-sum-coarse/keys/300s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-08/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 60.0, + "max_N": 3000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 60000000.0, + "covered_min_N": 60000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-08/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 1200.0, + "max_N": 60000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 60000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/inference_config.yaml new file mode 100644 index 00000000..76fac1df --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/inference_config.yaml @@ -0,0 +1,78 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[5m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[5m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[5m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 60 + query: topk by (region) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 40 + query: topk by (region) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/plans.json.gz new file mode 100644 index 00000000..727ad1bb Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/streaming_config.yaml new file mode 100644 index 00000000..c1479350 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 10000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 10000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 240 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08.json new file mode 100644 index 00000000..313fac22 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08.json @@ -0,0 +1,351 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-08", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 10) * 10 - 300) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 10) * 10 - 300)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 10) * 10 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 10) * 10 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region", + "grouping": [ + "region" + ], + "id": "q08-topk-count", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 4, + "output_series_upper_bound": 32, + "promql": "topk by (region) (8, increase(data_total[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 10000, + "k": 8, + "lookback_ms": 300000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q10-cardinality", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/eval_table.json new file mode 100644 index 00000000..44ab9b24 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/eval_table.json @@ -0,0 +1,525 @@ +{ + "inputs": { + "workload-09.json": "803866a2b4df14f80ed4bd0a6f9d16508ae66a3a853dd1842904dd3df32904ac" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-09", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-09/q01-sum-fine/keys/60s", + "lookback_ms": 60000, + "metric": "data" + } + }, + "interval_secs": 2, + "promql": "sum by (region, service, job) (sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-09/q01-sum-fine/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-09/q02-sum-coarse/keys/60s", + "lookback_ms": 60000, + "metric": "data" + } + }, + "interval_secs": 2, + "promql": "sum by (region, service) (sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-09/q02-sum-coarse/keys/60s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-09/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-09/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-09/q03-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/topk8_q06-topk-sum/keys/60s", + "k": 8, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 1, + "promql": "topk by (region, service) (8, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-09/topk8_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/topk8_q07-topk-sum/keys/1200s", + "k": 8, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 30, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-09/topk8_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 60000.0, + "covered_min_N": 60000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-09/q01-sum-fine/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 60.0, + "max_N": 60000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 60000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-09/q02-sum-coarse/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-09/topk8_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 12000000.0, + "covered_min_N": 12000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-09/topk8_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 12000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 12000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/inference_config.yaml new file mode 100644 index 00000000..a21787b7 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/inference_config.yaml @@ -0,0 +1,81 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[1m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[1m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[1m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 59 + query: topk by (region, service) (8, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 40 + query: topk by (region, service) (8, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service, job) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service, job) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/plans.json.gz new file mode 100644 index 00000000..a315ba97 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/streaming_config.yaml new file mode 100644 index 00000000..ca977a18 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 2000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 240 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: sliding +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 320 + width: 4096 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09.json new file mode 100644 index 00000000..fd0e4de9 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09.json @@ -0,0 +1,365 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-09", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 2000, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 2000, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 8, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 2) * 2 - 60) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 2) * 2 - 60)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 2) * 2 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 2) * 2 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q08-topk-count", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, increase(data_total[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 2000, + "k": 4, + "lookback_ms": 60000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region", + "service" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q10-cardinality", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 2000, + "lookback_ms": 60000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -60, NOW()) AND NOW() GROUP BY region, service", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/eval_table.json new file mode 100644 index 00000000..2373612d --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/eval_table.json @@ -0,0 +1,569 @@ +{ + "inputs": { + "workload-10.json": "9c023862533163e359fcbd2af0b847ce17adbe214640103a16b8bc97665cbd9c" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-10", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-10/q01-sum-fine/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 20, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-10/q01-sum-fine/keys/600s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-10/q02-sum-coarse/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-10/q02-sum-coarse/keys/600s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-10/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-10/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-10/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/topk8_q06-topk-sum/keys/60s", + "k": 8, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 2, + "promql": "topk by (region) (8, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-10/topk8_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/topk8_q07-topk-sum/keys/1200s", + "k": 8, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 60, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-10/topk8_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-10/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 600000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-10/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-10/topk8_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 60.0, + "max_N": 3000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk8_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 12000000.0, + "covered_min_N": 12000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-10/topk8_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 12000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 12000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/inference_config.yaml new file mode 100644 index 00000000..f9dc1204 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/inference_config.yaml @@ -0,0 +1,77 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[10m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[10m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[10m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[10m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 30 + query: topk by (region) (8, sum_over_time(data[1m])) + planned_subquery: topk by (region) (8, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 20 + query: topk by (region, service) (8, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/plans.json.gz new file mode 100644 index 00000000..5cf3b152 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/streaming_config.yaml new file mode 100644 index 00000000..6f3d2736 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 25000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 20000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 240 + width: 4096 + slideIntervalMs: 2000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10.json new file mode 100644 index 00000000..43c739b6 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10.json @@ -0,0 +1,355 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-10", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region", + "grouping": [ + "region" + ], + "id": "q06-topk-sum", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 32, + "promql": "topk by (region) (8, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 2000, + "k": 8, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 20) * 20 - 600) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 20) * 20 - 600)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 20) * 20 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 20) * 20 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 20000, + "k": 8, + "lookback_ms": 600000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region", + "service" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q10-cardinality", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY region", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/sketch_bench_workloads.py b/asap-tools/experiments/datasets/fake_metric/sketch_bench_workloads.py index 01fd972f..cb3655d1 100644 --- a/asap-tools/experiments/datasets/fake_metric/sketch_bench_workloads.py +++ b/asap-tools/experiments/datasets/fake_metric/sketch_bench_workloads.py @@ -581,10 +581,14 @@ def init_sql(metrics: dict, views: List[View], create_views: bool) -> str: columns = [f"{name} String" for name in names] + [ f"{value} Float64" for value in value_columns(desc) ] + # Raw queries read a time window across all series, so only a time + # order lets ClickHouse skip what's outside it. Views read their own + # tables; their source keeps its label order. + order = f"({', '.join(names)}, ts)" if create_views else "ts" statements += [ f"DROP TABLE IF EXISTS {table}", f"CREATE TABLE {table} (ts DateTime64(3), {', '.join(columns)}) " - f"ENGINE = MergeTree ORDER BY ({', '.join(names)}, ts)", + f"ENGINE = MergeTree ORDER BY {order}", ] if create_views: statements += [view_ddl(view) for view in views] diff --git a/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py b/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py new file mode 100644 index 00000000..abb675e2 --- /dev/null +++ b/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Collate a random_workload_batch.py batch: one row per (set, workload, arm) +with total CPU-seconds (warm-up + queries), peak memory, the median, p95 and +sum of every answered query's latency, and whether the run is valid: + +- no query failed or timed out; +- its answers agree with the reference arm (Prometheus, which keeps up, or + else ClickHouse+MV): exact arms to EXACT_REL_ERR, ASAPQuery's sketches to + their bounds. + +Separately, whether ingest kept up: during the query phase the feeder never +ran more than MAX_INGEST_LAG_SECS behind (from the feeder's +/paced_feeder_output/*_stats.json). A lagging system answers on-time +queries from incomplete windows, which the agreement check catches; one that +answers late but correctly is slow, not wrong, and stays valid. + +Then each baseline's ratio to ASAPQuery (baseline / asap, > 1 means ASAPQuery +is better) over workloads where both runs are valid. Only workloads whose +batch arms all finished, one of them a reference, are collated. + +Usage: + python3 collate_batch.py / [--csv out.csv] +""" + +import argparse +import csv +import glob +import json +import statistics +import sys +from collections import defaultdict +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from compare_arms import Arm, agreement, percentile # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from random_workload_batch import ARMS, OUTPUTS_DIR # noqa: E402 + +REFERENCE = "prom" +# The reference of a batch without Prometheus: exact, and answers every window. +FALLBACK_REFERENCE = "chmv" +# The queries' freshness delay: a feed further behind leaves on-time +# queries' windows incomplete. +MAX_INGEST_LAG_SECS = 10 +EXACT_REL_ERR = 1e-3 +# VictoriaMetrics may count a window-boundary sample Prometheus doesn't. +VM_REL_ERR = 5e-3 +EXACT_TOPK_PRECISION = 0.99 +# ASAPQuery's sketches: quantiles at the 0.95 accuracy SLA; sums are exact +# but for a window-boundary sample; top-k at its precision target. +ASAP_QUANTILE_REL_ERR = 0.05 +ASAP_SUM_REL_ERR = 5e-3 +ASAP_TOPK_PRECISION = 0.95 + +FIELDS = [ + "set", + "workload", + "arm", + "valid", + "why_invalid", + "ingest_kept_up", + "cpu_secs", + "warmup_cpu_secs", + "query_cpu_secs", + "peak_memory_gib", + "latency_p50", + "latency_p95", + "latency_sum", + "queries", + "failed_queries", + "max_ingest_lag_secs", +] + + +def max_ingest_lag(arm, mode_dir): + """The feeder's largest start lag (s) during the query phase, or None + without its stats.""" + stats = glob.glob(str(mode_dir / "paced_feeder_output" / "*_stats.json")) + if not stats: + return None + ticks = json.loads(Path(stats[0]).read_text())["ticks"] + query_ticks = [t for t in ticks if t["tick"] >= arm.warmup_secs] or ticks + return max(t["start_lag_ms"] for t in query_ticks) / 1000 + + +def disagreements(arm_name, arm, ref): + """The queries whose answers fall outside `arm_name`'s bounds against + the reference arm `ref`.""" + bad = [] + for gq, name in arm.names.items(): + topk = name.startswith("topk") + quantile = "quantile" in name + values = [] + for key, want in ref.results.items(): + if key[:2] != gq or key not in arm.results or not want: + continue + value, _ = agreement(arm.results[key], want, topk) + if value is not None: + values.append(value) + if not values: + bad.append(f"{name}: no comparable windows") + continue + mean = statistics.mean(values) + if arm_name == "asap": + ok = ( + mean >= ASAP_TOPK_PRECISION + if topk + else mean <= (ASAP_QUANTILE_REL_ERR if quantile else ASAP_SUM_REL_ERR) + ) + else: + rel_err = VM_REL_ERR if arm_name == "vm" else EXACT_REL_ERR + ok = mean >= EXACT_TOPK_PRECISION if topk else mean <= rel_err + if not ok: + bad.append(f"{name}: {'precision' if topk else 'rel err'} {mean:.3g}") + return bad + + +def rows(batch_dir): + jobs = json.loads((Path(batch_dir) / "state.json").read_text()) + by_workload = defaultdict(dict) + for j in jobs: + by_workload[(j["set"], j["workload"])][j["arm"]] = j + out = [] + for (set_name, workload), jobs_by_arm in sorted(by_workload.items()): + if any(j["status"] != "done" for j in jobs_by_arm.values()): + continue + reference = REFERENCE if REFERENCE in jobs_by_arm else FALLBACK_REFERENCE + if reference not in jobs_by_arm: + continue + dirs = { + a: OUTPUTS_DIR / j["names"][-1] / ARMS[a][3] for a, j in jobs_by_arm.items() + } + arms = {a: Arm(d) for a, d in dirs.items()} + for arm_name, arm in arms.items(): + r = arm.resources() + recorded = [x for xs in arm.latencies.values() for x in xs] + # A query that failed or timed out records no latency. + latencies = [x for x in recorded if x is not None] + lag = max_ingest_lag(arm, dirs[arm_name]) + why = [] + if len(recorded) != len(latencies): + why.append(f"{len(recorded) - len(latencies)} failed queries") + if arm_name != reference: + why += disagreements(arm_name, arm, arms[reference]) + out.append( + { + "set": set_name, + "workload": workload, + "arm": arm_name, + "valid": not why, + "why_invalid": "; ".join(why), + "ingest_kept_up": ( + None if lag is None else lag <= MAX_INGEST_LAG_SECS + ), + "cpu_secs": round(r["cpu_secs"]), + "warmup_cpu_secs": round(r["warmup_cpu_secs"]), + "query_cpu_secs": round(r["query_cpu_secs"]), + "peak_memory_gib": round(r["peak_memory_bytes"] / 2**30, 2), + "latency_p50": round(statistics.median(latencies), 3), + "latency_p95": round(percentile(latencies, 95), 3), + "latency_sum": round(sum(latencies), 1), + "queries": len(recorded), + "failed_queries": len(recorded) - len(latencies), + "max_ingest_lag_secs": None if lag is None else round(lag, 1), + } + ) + return out + + +def ratios(table): + """Per set and baseline arm: the geometric mean, over workloads where + both it and ASAPQuery are valid, of baseline / asap.""" + by = {(r["set"], r["workload"], r["arm"]): r for r in table} + result = defaultdict(lambda: defaultdict(list)) + for (set_name, workload, arm), r in by.items(): + asap = by.get((set_name, workload, "asap")) + if arm == "asap" or not (asap and r["valid"] and asap["valid"]): + continue + for key in ("cpu_secs", "latency_p50", "latency_p95", "latency_sum"): + result[(set_name, arm)][key].append(r[key] / asap[key]) + return { + k: { + **{m: statistics.geometric_mean(v) for m, v in metrics.items()}, + "n": len(metrics["cpu_secs"]), + } + for k, metrics in result.items() + } + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("batch_dir") + parser.add_argument("--csv") + args = parser.parse_args() + table = rows(args.batch_dir) + if args.csv: + with open(args.csv, "w", newline="") as f: + w = csv.DictWriter(f, FIELDS) + w.writeheader() + w.writerows(table) + print("Invalid runs:") + for r in table: + if not r["valid"]: + print(f" {r['set']} {r['workload']} {r['arm']}: {r['why_invalid']}") + print( + "\nIngest fell behind (feeder lag > {}s in the query phase):".format( + MAX_INGEST_LAG_SECS + ) + ) + for r in table: + if r["ingest_kept_up"] is False: + print( + f" {r['set']} {r['workload']} {r['arm']}: {r['max_ingest_lag_secs']}s" + ) + print( + "\nASAPQuery improvement = baseline / asap, geometric mean over workloads " + "where both runs are valid" + ) + print( + f"{'set':14s} {'baseline':9s} {'n':>2s} {'CPU':>7s} {'p50':>7s} {'p95':>7s} {'sum':>7s}" + ) + for (set_name, arm), m in sorted(ratios(table).items()): + print( + f"{set_name:14s} {arm:9s} {m['n']:2d} {m['cpu_secs']:6.1f}x " + f"{m['latency_p50']:6.1f}x {m['latency_p95']:6.1f}x {m['latency_sum']:6.1f}x" + ) + + +if __name__ == "__main__": + main() diff --git a/asap-tools/experiments/post_experiment/multi_experiment/compare_arms.py b/asap-tools/experiments/post_experiment/multi_experiment/compare_arms.py index 7578016f..b8f1b707 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/compare_arms.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/compare_arms.py @@ -137,13 +137,23 @@ def report(arms, reference): interval = arms[reference].interval_secs[gq[0]] print(f"\n[{gq[0]}.{gq[1]}] every {interval:g}s: {name}") for arm_name, arm in arms.items(): - xs = arm.latencies.get(gq) - if not xs: + recorded = arm.latencies.get(gq) + if not recorded: continue - within = sum(x <= interval for x in xs) / len(xs) + # A failed or timed-out query records no latency; it never counts + # as answered within the interval. + xs = [x for x in recorded if x is not None] + failed = len(recorded) - len(xs) + within = sum(x <= interval for x in xs) / len(recorded) + stats = ( + f"{percentile(xs, 50):8.3f} {percentile(xs, 95):8.3f} " + f"{percentile(xs, 99):8.3f}" + if xs + else f"{'-':>8s} {'-':>8s} {'-':>8s}" + ) print( - f" {arm_name:16s} n={len(xs):5d} {percentile(xs, 50):8.3f} " - f"{percentile(xs, 95):8.3f} {percentile(xs, 99):8.3f} {within:6.1%}" + f" {arm_name:16s} n={len(recorded):5d} {stats} {within:6.1%}" + + (f" FAILED {failed}" if failed else "") ) print(f"\n== Agreement with {reference}") diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py new file mode 100644 index 00000000..9a998526 --- /dev/null +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Cost-latency scatter from collate_batch.py's CSV: one panel per workload +set, one point per (workload, system); x = p95 query latency, y = total +CPU-seconds, both log. Lower left is better. A run collate_batch.py marks +invalid (failed queries, or answers that disagree with the reference) is +drawn hollow: its latency and CPU don't stand for a correct answer. + +Two arrows from LiteOlly's geometric-mean point show its improvement in CPU +(vertical) and p95 latency (horizontal) over the worst baseline for that +metric: collate_batch.py's geometric mean of baseline / LiteOlly over +workloads where both are valid. Arrow lengths are those ratios; labels give +the ratios against Prometheus and ClickHouse. + +An oval encloses each baseline's valid runs (all its runs if none is valid). + +Writes /cost_latency__.png; never overwrites. + +Usage: python3 plot_cost_latency.py collated.csv out_dir +""" + +import csv +import datetime +import statistics +import sys +from pathlib import Path +from collections import defaultdict + +import matplotlib +import numpy as np + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib import ticker # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from collate_batch import ratios # noqa: E402 + +# Validated categorical slots 1-5 (fixed order) and a marker per system, so +# identity never rests on color alone. +SYSTEMS = [ + ("asap", "LiteOlly", "#2a78d6", "o"), + ("prom", "Prometheus", "#eb6834", "s"), + ("vm", "VictoriaMetrics", "#1baf7a", "^"), + ("ch", "ClickHouse", "#eda100", "D"), + ("chmv", "ClickHouse+MV", "#e87ba4", "v"), +] +SURFACE, INK, MUTED = "#fcfcfb", "#0b0b0b", "#52514e" + + +def oval(ax, xs, ys, color): + """Ellipse in log space along the points' covariance, scaled to enclose + them all with a fixed margin (in decades) for the markers.""" + pts = np.log10(np.column_stack([xs, ys])) + center = pts.mean(axis=0) + # The small diagonal keeps one or collinear points drawable. + cov = np.cov(pts.T, bias=True) + np.eye(2) * 1e-4 + inv = np.linalg.inv(cov) + d = pts - center + reach = np.sqrt(np.einsum("ij,jk,ik->i", d, inv, d).max()) + vals, vecs = np.linalg.eigh(cov) + t = np.linspace(0, 2 * np.pi, 200) + ring = (vecs * (np.sqrt(vals) * reach + 0.05)) @ np.vstack([np.cos(t), np.sin(t)]) + ax.plot(*10 ** (ring + center[:, None]), color=color, linewidth=1.5, zorder=2) + + +def arrows(ax, rows, improvement, set_name): + """LiteOlly's improvement over the worst baseline per metric, drawn from + LiteOlly's geometric-mean point (over its valid runs, else all) and + labelled with its ratios against Prometheus and ClickHouse.""" + names = {key: label for key, label, *_ in SYSTEMS} + lite = [r for r in rows if r["arm"] == "asap"] + leftmost = min(float(r["latency_p95"]) for r in lite) + lite = [r for r in lite if r["valid"] == "True"] or lite + lx = statistics.geometric_mean(float(r["latency_p95"]) for r in lite) + ly = statistics.geometric_mean(float(r["cpu_secs"]) for r in lite) + worst = {} + for metric in ("cpu_secs", "latency_p95"): + worst[metric] = max( + ((arm, m[metric]) for (s, arm), m in improvement.items() if s == set_name), + key=lambda am: am[1], + ) + + def vs(metric, word): + """e.g. "6.2x cheaper vs. ClickHouse,\n4.9x vs. Prometheus", largest first.""" + (a, ra), (b, rb) = sorted( + ((a, improvement[(set_name, a)][metric]) for a in ("prom", "ch")), + key=lambda ar: -ar[1], + ) + return f"{ra:.1f}x {word} vs. {names[a]},\n{rb:.1f}x vs. {names[b]}" + + style = dict(arrowstyle="<->", color=INK, linewidth=2, mutation_scale=22) + arm, cpu = worst["cpu_secs"] + # Left of every LiteOlly point; its label sits above the arrow's top. + x = leftmost / 1.3 + ax.annotate("", (x, ly), xytext=(x, ly * cpu), arrowprops=style, zorder=4) + ax.text( + x / 1.15, + ly * cpu * 1.12, + vs("cpu_secs", "cheaper"), + color=INK, + fontsize=13, + ha="left", + va="bottom", + ) + arm, lat = worst["latency_p95"] + y = ly / 1.4 + ax.annotate("", (lx, y), xytext=(lx * lat, y), arrowprops=style, zorder=4) + ax.text( + lx * lat**0.5, + y / 1.08, + vs("latency_p95", "faster"), + color=INK, + fontsize=13, + ha="center", + va="top", + ) + left, right = ax.get_xlim() + ax.set_xlim(min(left, x / 2), right) + bottom, top = ax.get_ylim() + ax.set_ylim(min(bottom, ly / 2.6), max(top, ly * cpu * 2.2)) + + +def main(csv_path, out_dir): + rows = list(csv.DictReader(open(csv_path))) + by_set = defaultdict(list) + for r in rows: + by_set[r["set"]].append(r) + sets = sorted(by_set) + numeric = ("cpu_secs", "latency_p50", "latency_p95", "latency_sum") + improvement = ratios( + [ + {**r, "valid": r["valid"] == "True", **{k: float(r[k]) for k in numeric}} + for r in rows + ] + ) + fig, axes = plt.subplots( + 1, len(sets), figsize=(7 * len(sets), 6), facecolor=SURFACE, squeeze=False + ) + for ax, set_name in zip(axes[0], sets): + ax.set_facecolor(SURFACE) + for key, label, color, marker in SYSTEMS: + pts = [r for r in by_set[set_name] if r["arm"] == key] + xs = [float(r["latency_p95"]) for r in pts] + ys = [float(r["cpu_secs"]) for r in pts] + valid = [r["valid"] == "True" for r in pts] + ax.scatter( + [x for x, v in zip(xs, valid) if v], + [y for y, v in zip(ys, valid) if v], + s=128, + color=color, + marker=marker, + label=label, + edgecolors=SURFACE, + linewidths=1, + zorder=3, + ) + ax.scatter( + [x for x, v in zip(xs, valid) if not v], + [y for y, v in zip(ys, valid) if not v], + s=112, + facecolors="none", + edgecolors=color, + marker=marker, + linewidths=2.5, + zorder=3, + ) + # Invalid runs don't stand for the system, so they don't stretch its oval. + fit = [(x, y) for x, y, v in zip(xs, ys, valid) if v] or list(zip(xs, ys)) + if key != "asap" and fit: + oval(ax, *zip(*fit), color) + ax.set_xscale("log") + ax.set_yscale("log") + ax.set_xlabel("p95 Query Latency (s)", color=MUTED, fontsize=14) + ax.set_ylabel("Total Compute Cost (CPU-sec)", color=MUTED, fontsize=14) + for axis in (ax.xaxis, ax.yaxis): + axis.set_major_locator(ticker.LogLocator(subs=(1, 2, 5))) + axis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f"{v:g}")) + axis.set_minor_formatter(ticker.NullFormatter()) + ax.grid(True, which="major", color="#e6e5e1", linewidth=0.8) + ax.tick_params(colors=MUTED, labelsize=13, width=2) + for spine in ax.spines.values(): + spine.set_color(INK) + spine.set_linewidth(2) + arrows(ax, by_set[set_name], improvement, set_name) + fig.suptitle( + "Cost-latency tradeoffs of LiteOlly vs. baselines", color=INK, fontsize=16 + ) + fig.legend( + *axes[0][0].get_legend_handles_labels(), + frameon=False, + fontsize=13, + loc="lower center", + ncol=min(len(SYSTEMS), 3 * len(sets)), + ) + # A one-panel figure fits three legend entries a row, so two rows. + fig.tight_layout(rect=(0, 0.07 if len(sets) > 1 else 0.12, 1, 1)) + stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + out_path = Path(out_dir) / f"cost_latency_{'+'.join(sets)}_{stamp}.png" + with open(out_path, "xb") as f: + fig.savefig(f, dpi=150, facecolor=SURFACE, format="png") + print(out_path) + + +if __name__ == "__main__": + main(*sys.argv[1:3]) diff --git a/asap-tools/experiments/random_workload_batch.py b/asap-tools/experiments/random_workload_batch.py new file mode 100644 index 00000000..b4a35ff6 --- /dev/null +++ b/asap-tools/experiments/random_workload_batch.py @@ -0,0 +1,394 @@ +#!/usr/bin/env python3 +"""Run a batch of random-workload arms on a cluster, many at once. + +Each job is one arm (asap, prom, vm, ch, chmv) of one workload, an +experiment_run_e2e.py or experiment_run_clickhouse.py run on its own slot: +two nodes, node_offset = 2 * slot, num_nodes = 1. Jobs run longest first. A +failed job retries once under a new experiment name. Before each job the +slot's nodes are cleaned of anything an earlier, killed job left running. +When every arm of a workload is done, compare_arms.py writes its summary. + +State is kept in /state.json after every change, so a restarted +driver resumes: finished jobs are skipped and jobs that were running rerun. + +Configs come from random_workload_artifacts.py --pilot-dir, under +config/experiment_type/random_workload///. + +Usage (from asap-tools/experiments, in tmux): + python3 random_workload_batch.py --batch overnight-1015 \\ + --username milindsr --hostname-suffix --slots 21 \\ + --set seed-862:01-10 --set seed-862-long:01-10 --set seed-862-x5:01 \\ + [--dry-run] [--stop-launching-after-hours 7] + +A rerun of some arms with changed configs: --arm ch --tag ts names the runs +s862_w01_ch_ts, ...; --config-variant sla10 reads -sla10/ configs. +""" + +import argparse +import gzip +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + +import yaml + +EXPERIMENTS_DIR = Path(__file__).resolve().parent +OUTPUTS_DIR = EXPERIMENTS_DIR.parent.parent / "experiment_outputs" +CONFIG_DIR = EXPERIMENTS_DIR / "config" / "experiment_type" / "random_workload" +COMPARE_ARMS = ( + EXPERIMENTS_DIR / "post_experiment" / "multi_experiment" / "compare_arms.py" +) +POLL_SECS = 30 +MAX_ATTEMPTS = 2 +REFERENCE_ARM = "chmv" # exact, and the fastest exact arm: answers every window + +# arm -> (runner, config, extra overrides, mode directory) +ARMS = { + "asap": ( + "experiment_run_e2e.py", + "e2e", + ["experiment_params.experiment=[{mode:sketchdb,server:sketchdb}]"], + "sketchdb", + ), + "prom": ( + "experiment_run_e2e.py", + "e2e", + ["experiment_params.experiment=[{mode:baseline,server:prometheus}]"], + "baseline", + ), + "vm": ("experiment_run_e2e.py", "victoriametrics", [], "baseline"), + "ch": ("experiment_run_clickhouse.py", "clickhouse", [], "baseline"), + "chmv": ("experiment_run_clickhouse.py", "clickhouse_mv", [], "baseline_mv"), +} +# Arms that fall behind 1 s queries in the pilot, and so take longest. +SLOW_ARMS = {"prom", "ch"} +# Anything a job leaves running on its nodes: feeders, bare-metal +# Prometheus, monitors and query clients, and every system's container. +CLEANUP = ( + "pkill -f '[t]arget/release/paced_feeder'; pkill -x prometheus; " + "pkill -f '[r]emote_monitor.py'; pkill -f '[q]uery_engine_rust'; " + "docker rm -f clickhouse-server victoriametrics-single victoriametrics-agent " + "sketchdb-queryengine-rust sketchdb-prometheusclient prometheus-container " + ">/dev/null 2>&1; true" +) + + +def parse_set(spec): + """`seed-862-long:01-10` or `seed-862-x5:01,03` -> (set, [workload ids]).""" + name, _, ranges = spec.partition(":") + ids = [] + for part in ranges.split(","): + lo, _, hi = part.partition("-") + ids += [f"workload-{n:02d}" for n in range(int(lo), int(hi or lo) + 1)] + return name, ids + + +def warmup_secs(set_name, workload): + config = CONFIG_DIR / set_name / workload / "clickhouse.yaml" + groups = yaml.safe_load(config.read_text())["query_groups"] + return min(g["client_options"]["starting_delay"] for g in groups) + + +def make_jobs(sets, arms=tuple(ARMS), tag="", config_variant=""): + jobs = [] + for set_name, workloads in sets: + for workload in workloads: + config_dir = f"{workload}-{config_variant}" if config_variant else workload + for arm in arms: + config = CONFIG_DIR / set_name / config_dir / f"{ARMS[arm][1]}.yaml" + if not config.exists(): + raise SystemExit(f"missing {config}; render it first") + variant = set_name.count("-") > 1 # seed-862-long, not seed-862 + # Only for ordering: slow arms of stretched sets take longest. + estimate = warmup_secs(set_name, workload) + ( + (6000 if variant else 2400) if arm in SLOW_ARMS else 900 + ) + jobs.append( + { + "key": f"{set_name}/{workload}/{arm}", + "set": set_name, + "workload": workload, + "arm": arm, + "tag": tag, + "config_dir": config_dir, + "estimate_secs": estimate, + "attempts": 0, + "status": "queued", + "names": [], + } + ) + jobs.sort(key=lambda j: -j["estimate_secs"]) + return jobs + + +def experiment_name(job): + tag = job["set"].replace("seed-", "s").replace("-", "") + name = f"{tag}_{job['workload'].replace('workload-', 'w')}_{job['arm']}" + if job.get("tag"): + name += f"_{job['tag']}" + return name if job["attempts"] == 1 else f"{name}_r{job['attempts']}" + + +def command(job, name, slot, args): + runner, config, extra, _ = ARMS[job["arm"]] + cmd = [ + sys.executable, + runner, + "experiment_type=random_workload/" + f"{job['set']}/{job.get('config_dir', job['workload'])}/{config}", + f"experiment.name={name}", + "providers.cloudlab.num_nodes=1", + f"providers.cloudlab.node_offset={2 * slot}", + f"providers.cloudlab.username={args.username}", + f"providers.cloudlab.hostname_suffix={args.hostname_suffix}", + "controller.punting=false", + *extra, + ] + if runner == "experiment_run_e2e.py": + # The Prometheus TSDB copy is large for long and x5, and unused. + cmd.append("flow.skip_copy_prometheus_data=true") + return cmd + + +def succeeded(job, name): + """The run exited 0 and recorded latencies and results.""" + out = OUTPUTS_DIR / name / ARMS[job["arm"]][3] / "prometheus_client_output" + for f in ("query_latencies.jsonl.gz", "query_results.jsonl.gz"): + path = out / f + if not path.exists() or path.stat().st_size == 0: + return False + try: + with gzip.open(out / "query_latencies.jsonl.gz", "rt") as f: + return bool(f.readline()) + except (OSError, EOFError): + return False + + +def clean_slot(slot, args): + for node in (2 * slot, 2 * slot + 1): + host = f"{args.username}@node{node}.{args.hostname_suffix}" + subprocess.run( + [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "ConnectTimeout=20", + host, + CLEANUP, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=120, + ) + + +def summarize(batch_dir, jobs, set_name, workload): + done = { + j["arm"]: j["names"][-1] + for j in jobs + if j["set"] == set_name and j["workload"] == workload and j["status"] == "done" + } + reference = REFERENCE_ARM if REFERENCE_ARM in done else "prom" + if reference not in done: + return + cmd = [sys.executable, str(COMPARE_ARMS), "--reference", reference] + for arm, name in done.items(): + cmd += ["--arm", f"{arm}={OUTPUTS_DIR / name / ARMS[arm][3]}"] + out = batch_dir / "summaries" / f"{set_name}_{workload}.txt" + out.parent.mkdir(exist_ok=True) + with open(out, "w") as f: + f.write(f"arms: {done}\n\n") + f.flush() + subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT, cwd=EXPERIMENTS_DIR) + + +def alive(name): + """Whether a runner for experiment `name` runs on this machine.""" + pattern = f"experiment.name={name}( |$)" + return ( + subprocess.run(["pgrep", "-f", pattern], stdout=subprocess.DEVNULL).returncode + == 0 + ) + + +def settle_orphans(jobs, is_alive=alive): + """The slots of orphaned jobs (runs an earlier driver left going) that + still run, so nothing launches over them. A finished orphan becomes done, + or queued for its retry.""" + busy = set() + for j in jobs: + if j["status"] != "orphaned": + continue + if "slot" not in j: + raise SystemExit(f"{j['names'][-1]} has no recorded slot") + if is_alive(j["names"][-1]): + busy.add(j["slot"]) + elif succeeded(j, j["names"][-1]): + j["status"] = "done" + else: + j["status"] = "queued" if j["attempts"] < MAX_ATTEMPTS else "failed" + return busy + + +def save(batch_dir, jobs): + tmp = batch_dir / "state.json.tmp" + tmp.write_text(json.dumps(jobs, indent=1)) + os.replace(tmp, batch_dir / "state.json") + + +def log(batch_dir, message): + line = f"{time.strftime('%Y-%m-%d %H:%M:%S')} {message}" + print(line, flush=True) + with open(batch_dir / "driver.log", "a") as f: + f.write(line + "\n") + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--batch", required=True, help="name; state under experiment_outputs/" + ) + parser.add_argument("--username", required=True) + parser.add_argument("--hostname-suffix", required=True) + parser.add_argument( + "--slots", type=int, required=True, help="two nodes each, from node0" + ) + parser.add_argument( + "--set", action="append", required=True, metavar="SET:WORKLOADS" + ) + parser.add_argument("--stop-launching-after-hours", type=float) + parser.add_argument( + "--skip-slots", + type=lambda s: {int(x) for x in s.split(",")}, + default=set(), + help="slots to leave alone, e.g. 3,15,16 (their nodes are busy)", + ) + parser.add_argument( + "--arm", action="append", choices=ARMS, help="only these arms (default all)" + ) + parser.add_argument("--tag", default="", help="suffix for experiment names") + parser.add_argument( + "--config-variant", default="", help="read configs from -VARIANT/" + ) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + batch_dir = OUTPUTS_DIR / args.batch + batch_dir.mkdir(parents=True, exist_ok=True) + state = batch_dir / "state.json" + if state.exists(): + jobs = json.loads(state.read_text()) + for j in jobs: + if j["status"] == "running": + # A run the earlier driver left behind keeps going on its own + # and keeps its slot until it ends. + j["status"] = "orphaned" if alive(j["names"][-1]) else "queued" + else: + jobs = make_jobs( + [parse_set(s) for s in args.set], + args.arm or tuple(ARMS), + args.tag, + args.config_variant, + ) + save(batch_dir, jobs) + + if args.dry_run: + for j in jobs: + j["attempts"] += 1 + print( + f"{j['estimate_secs'] / 3600:4.1f} h " + + shlex.join(command(j, experiment_name(j), 0, args)) + ) + j["attempts"] -= 1 + print( + f"{len(jobs)} jobs, ~{sum(j['estimate_secs'] for j in jobs) / 3600:.0f} " + f"slot-hours on {args.slots} slots" + ) + return + + started = time.time() + running = {} # slot -> (job, process) + while True: + for slot, (job, proc) in list(running.items()): + if proc.poll() is None: + continue + del running[slot] + name = job["names"][-1] + if proc.returncode == 0 and succeeded(job, name): + job["status"] = "done" + elif job["attempts"] < MAX_ATTEMPTS: + job["status"] = "queued" + else: + job["status"] = "failed" + log( + batch_dir, + f"slot {slot}: {name} exit {proc.returncode} -> {job['status']}", + ) + save(batch_dir, jobs) + peers = [ + j + for j in jobs + if (j["set"], j["workload"]) == (job["set"], job["workload"]) + ] + if all(j["status"] in ("done", "failed") for j in peers): + summarize(batch_dir, jobs, job["set"], job["workload"]) + log(batch_dir, f"summary: {job['set']} {job['workload']}") + + launching = args.stop_launching_after_hours is None or ( + time.time() - started < args.stop_launching_after_hours * 3600 + ) + orphan_slots = settle_orphans(jobs) + for slot in range(args.slots): + if ( + slot in running + or slot in args.skip_slots + or slot in orphan_slots + or not launching + ): + continue + # Retries first, then the longest queued job. + queued = [j for j in jobs if j["status"] == "queued"] + if not queued: + break + job = max(queued, key=lambda j: (j["attempts"], j["estimate_secs"])) + clean_slot(slot, args) + job["attempts"] += 1 + name = experiment_name(job) + job["names"].append(name) + job["status"] = "running" + job["slot"] = slot + save(batch_dir, jobs) + logs = batch_dir / "logs" + logs.mkdir(exist_ok=True) + out = open(logs / f"{name}.log", "w") + proc = subprocess.Popen( + command(job, name, slot, args), + cwd=EXPERIMENTS_DIR, + stdout=out, + stderr=subprocess.STDOUT, + env={**os.environ, "PYTHONUNBUFFERED": "1"}, + ) + running[slot] = (job, proc) + log(batch_dir, f"slot {slot} (node{2 * slot}-{2 * slot + 1}): start {name}") + + if not running and ( + not launching or not any(j["status"] == "queued" for j in jobs) + ): + break + time.sleep(POLL_SECS) + + counts = { + s: sum(j["status"] == s for j in jobs) for s in ("done", "failed", "queued") + } + log(batch_dir, f"batch finished: {counts}") + + +if __name__ == "__main__": + main() diff --git a/asap-tools/experiments/tests/test_random_workload_artifacts.py b/asap-tools/experiments/tests/test_random_workload_artifacts.py index 47ba9ddd..cd158a2f 100644 --- a/asap-tools/experiments/tests/test_random_workload_artifacts.py +++ b/asap-tools/experiments/tests/test_random_workload_artifacts.py @@ -194,6 +194,42 @@ def test_every_arm_feeds_the_same_data_and_schedule(self): raw_sql = "".join(files[f"interval_{t}s_baseline.sql"] for t in (1, 20, 30)) self.assertTrue(all(sql in raw_sql for sql in pilot_sql)) + def test_two_dashboards_repeat_every_group_on_the_same_plan(self): + workload = workloads.generate_workloads(seed=862)[0] + + def configs(dashboards): + files = artifacts.pilot_configs(workload, PILOT_DIR, dashboards) + # The SQL files are the same, under the variant's directory. + return { + p.name: yaml.safe_load(c.replace("workload-01-dash2/", "workload-01/")) + for p, c in files.items() + if p.suffix == ".yaml" + }, {p.parent.name for p in files} + + one, _ = configs(1) + two, dirs = configs(2) + self.assertEqual(dirs, {"workload-01-dash2"}) + for name, key in ( + ("e2e.yaml", "id"), + ("clickhouse.yaml", "name"), + ("clickhouse_mv.yaml", "name"), + ): + with self.subTest(config=name): + single, double = one[name], two[name] + params = ( + (single["experiment_params"], double["experiment_params"]) + if name == "e2e.yaml" + else (single, double) + ) + groups, copies = (p.pop("query_groups") for p in params) + # Same plan, feeder and data; only the groups repeat. + self.assertEqual(double, single) + # Each copy is its own group, so the client keeps its latencies apart. + self.assertEqual(len({g[key] for g in copies}), 2 * len(groups)) + for g in groups + copies: + del g[key] + self.assertEqual(copies, 2 * groups) + class PilotConfigsPreflightTest(unittest.TestCase): """Each config through Hydra and the checks its runner makes before it diff --git a/asap-tools/experiments/tests/test_random_workload_batch.py b/asap-tools/experiments/tests/test_random_workload_batch.py new file mode 100644 index 00000000..97213359 --- /dev/null +++ b/asap-tools/experiments/tests/test_random_workload_batch.py @@ -0,0 +1,82 @@ +"""Tests for random_workload_batch.py's job planning.""" + +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import random_workload_batch as batch + + +class BatchPlanningTest(unittest.TestCase): + def test_sets_name_their_workloads(self): + self.assertEqual( + batch.parse_set("seed-862-x5:01,03-04"), + ("seed-862-x5", ["workload-01", "workload-03", "workload-04"]), + ) + + def test_retries_get_a_new_experiment_name(self): + # A rerun under the old name hits the earlier run's ClickHouse data + # directory, which the runner can't chmod. + job = {"set": "seed-862-long", "workload": "workload-07", "arm": "ch"} + names = [] + for attempts in (1, 2): + names.append(batch.experiment_name({**job, "attempts": attempts})) + self.assertEqual(names, ["s862long_w07_ch", "s862long_w07_ch_r2"]) + + def test_a_tagged_variant_rerun_names_and_configs_apart(self): + # Reruns must not reuse the original runs' names, whose outputs they + # would overwrite. + with tempfile.TemporaryDirectory() as tmp: + sets = Path(tmp) / "seed-862-long" + (sets / "workload-04").mkdir(parents=True) + (sets / "workload-04" / "clickhouse.yaml").write_text( + "query_groups: [{client_options: {starting_delay: 600}}]" + ) + (sets / "workload-04-sla10").mkdir() + (sets / "workload-04-sla10" / "e2e.yaml").write_text("") + with mock.patch.object(batch, "CONFIG_DIR", Path(tmp)): + [job] = batch.make_jobs( + [("seed-862-long", ["workload-04"])], ("asap",), "sla10", "sla10" + ) + job["attempts"] = 1 + self.assertEqual(batch.experiment_name(job), "s862long_w04_asap_sla10") + + class Args: + username, hostname_suffix = "u", "s" + + self.assertIn( + "experiment_type=random_workload/seed-862-long/workload-04-sla10/e2e", + batch.command(job, "n", 0, Args), + ) + + def test_a_live_orphan_keeps_its_slot(self): + # A resumed driver once cleaned and reused the slots of runs it + # had left going, wrecking them. + jobs = [ + {"status": "orphaned", "names": ["a"], "slot": 9, "attempts": 1}, + {"status": "orphaned", "names": ["b"], "slot": 10, "attempts": 1}, + {"status": "queued", "names": [], "attempts": 0}, + ] + with mock.patch.object(batch, "succeeded", return_value=False): + busy = batch.settle_orphans(jobs, is_alive=lambda name: name == "a") + self.assertEqual(busy, {9}) + self.assertEqual([j["status"] for j in jobs], ["orphaned", "queued", "queued"]) + + def test_each_job_runs_on_its_slots_two_nodes(self): + job = {"set": "seed-862", "workload": "workload-01", "arm": "prom"} + + class Args: + username, hostname_suffix = "u", "s" + + cmd = batch.command(job, "s862_w01_prom", 3, Args) + self.assertIn("providers.cloudlab.num_nodes=1", cmd) + self.assertIn("providers.cloudlab.node_offset=6", cmd) + self.assertIn("flow.skip_copy_prometheus_data=true", cmd) + self.assertIn( + "experiment_params.experiment=[{mode:baseline,server:prometheus}]", cmd + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/asap-tools/experiments/tests/test_sketch_bench_workloads.py b/asap-tools/experiments/tests/test_sketch_bench_workloads.py index 1330a381..9275e7e5 100644 --- a/asap-tools/experiments/tests/test_sketch_bench_workloads.py +++ b/asap-tools/experiments/tests/test_sketch_bench_workloads.py @@ -186,6 +186,11 @@ def test_metric_without_labels_raises(self): with self.assertRaises(ValueError): sbw.init_sql({"data": {"type": "gauge", "labels": []}}, [], False) + def test_raw_table_is_ordered_by_time_only(self): + metrics = {"data": {"type": "gauge", "labels": [{"name": "job"}]}} + self.assertIn("ORDER BY ts;", sbw.init_sql(metrics, [], False)) + self.assertIn("ORDER BY (job, ts);", sbw.init_sql(metrics, [], True)) + def test_describe_is_one_line(self): entry = {"promql": "sum by (job) (\n data\n)", "interval_secs": 1} self.assertEqual(sbw.describe(entry), "sum by (job) ( data )") diff --git a/asap-tools/queriers/prometheus-client/main_prometheus_client.py b/asap-tools/queriers/prometheus-client/main_prometheus_client.py index 1382c7c3..44512f40 100644 --- a/asap-tools/queriers/prometheus-client/main_prometheus_client.py +++ b/asap-tools/queriers/prometheus-client/main_prometheus_client.py @@ -652,6 +652,9 @@ def main(args: Any) -> None: global_query_idx += len(query_group.queries) server_url_for_alignment = None + # Every query of every group can be in flight at once; a smaller pool + # opens a new connection per excess request, inside its latency. + pool_size = max(requests.adapters.DEFAULT_POOLSIZE, global_query_idx) servers: Dict[str, QueryClient] = {} for server in config.servers: @@ -676,7 +679,7 @@ def main(args: Any) -> None: ) # Mount debug adapter for HTTP request logging - debug_adapter = DebugHTTPAdapter(server.name) + debug_adapter = DebugHTTPAdapter(server.name, pool_maxsize=pool_size) client.session.mount("http://", debug_adapter) client.session.mount("https://", debug_adapter) else: @@ -691,7 +694,7 @@ def main(args: Any) -> None: ) # Mount debug adapter for HTTP request logging - debug_adapter = DebugHTTPAdapter(server.name) + debug_adapter = DebugHTTPAdapter(server.name, pool_maxsize=pool_size) client.session.mount("http://", debug_adapter) client.session.mount("https://", debug_adapter)