From 4fd18db6cf8e49fb2a4fa84b5e8a794eaeb65df6 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 01:29:33 -0400 Subject: [PATCH 01/20] feat(asap-tools): batch driver for random-workload arms random_workload_batch.py runs every arm of the given workload sets on a cluster, one job per two-node slot, longest first; cleans a slot's nodes before each job, retries a failed job once under a new name, writes a compare_arms summary per workload (ClickHouse+MV as the reference), and keeps its state on disk so a restarted driver resumes. Co-Authored-By: Claude Opus 5.5 --- .../experiments/random_workload_batch.py | 330 ++++++++++++++++++ .../tests/test_random_workload_batch.py | 40 +++ 2 files changed, 370 insertions(+) create mode 100644 asap-tools/experiments/random_workload_batch.py create mode 100644 asap-tools/experiments/tests/test_random_workload_batch.py diff --git a/asap-tools/experiments/random_workload_batch.py b/asap-tools/experiments/random_workload_batch.py new file mode 100644 index 00000000..bc874980 --- /dev/null +++ b/asap-tools/experiments/random_workload_batch.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +"""Run a batch of random-workload arms on a cluster, many at once. + +Each job is one arm (asap, prom, vm, ch, chmv) of one workload, an +experiment_run_e2e.py or experiment_run_clickhouse.py run on its own slot: +two nodes, node_offset = 2 * slot, num_nodes = 1. Jobs run longest first. A +failed job retries once under a new experiment name. Before each job the +slot's nodes are cleaned of anything an earlier, killed job left running. +When every arm of a workload is done, compare_arms.py writes its summary. + +State is kept in /state.json after every change, so a restarted +driver resumes: finished jobs are skipped and jobs that were running rerun. + +Configs come from random_workload_artifacts.py --pilot-dir, under +config/experiment_type/random_workload///. + +Usage (from asap-tools/experiments, in tmux): + python3 random_workload_batch.py --batch overnight-1015 \\ + --username milindsr --hostname-suffix --slots 21 \\ + --set seed-862:01-10 --set seed-862-long:01-10 --set seed-862-x5:01 \\ + [--dry-run] [--stop-launching-after-hours 7] +""" + +import argparse +import gzip +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + +import yaml + +EXPERIMENTS_DIR = Path(__file__).resolve().parent +OUTPUTS_DIR = EXPERIMENTS_DIR.parent.parent / "experiment_outputs" +CONFIG_DIR = EXPERIMENTS_DIR / "config" / "experiment_type" / "random_workload" +COMPARE_ARMS = ( + EXPERIMENTS_DIR / "post_experiment" / "multi_experiment" / "compare_arms.py" +) +POLL_SECS = 30 +MAX_ATTEMPTS = 2 +REFERENCE_ARM = "chmv" # exact, and the fastest exact arm: answers every window + +# arm -> (runner, config, extra overrides, mode directory) +ARMS = { + "asap": ( + "experiment_run_e2e.py", + "e2e", + ["experiment_params.experiment=[{mode:sketchdb,server:sketchdb}]"], + "sketchdb", + ), + "prom": ( + "experiment_run_e2e.py", + "e2e", + ["experiment_params.experiment=[{mode:baseline,server:prometheus}]"], + "baseline", + ), + "vm": ("experiment_run_e2e.py", "victoriametrics", [], "baseline"), + "ch": ("experiment_run_clickhouse.py", "clickhouse", [], "baseline"), + "chmv": ("experiment_run_clickhouse.py", "clickhouse_mv", [], "baseline_mv"), +} +# Arms that fall behind 1 s queries in the pilot, and so take longest. +SLOW_ARMS = {"prom", "ch"} +# Anything a job leaves running on its nodes: feeders, bare-metal +# Prometheus, monitors and query clients, and every system's container. +CLEANUP = ( + "pkill -f '[t]arget/release/paced_feeder'; pkill -x prometheus; " + "pkill -f '[r]emote_monitor.py'; pkill -f '[q]uery_engine_rust'; " + "docker rm -f clickhouse-server victoriametrics-single victoriametrics-agent " + "sketchdb-queryengine-rust sketchdb-prometheusclient prometheus-container " + ">/dev/null 2>&1; true" +) + + +def parse_set(spec): + """`seed-862-long:01-10` or `seed-862-x5:01,03` -> (set, [workload ids]).""" + name, _, ranges = spec.partition(":") + ids = [] + for part in ranges.split(","): + lo, _, hi = part.partition("-") + ids += [f"workload-{n:02d}" for n in range(int(lo), int(hi or lo) + 1)] + return name, ids + + +def warmup_secs(set_name, workload): + config = CONFIG_DIR / set_name / workload / "clickhouse.yaml" + groups = yaml.safe_load(config.read_text())["query_groups"] + return min(g["client_options"]["starting_delay"] for g in groups) + + +def make_jobs(sets): + jobs = [] + for set_name, workloads in sets: + for workload in workloads: + for arm in ARMS: + config = CONFIG_DIR / set_name / workload / f"{ARMS[arm][1]}.yaml" + if not config.exists(): + raise SystemExit(f"missing {config}; render it first") + variant = set_name.count("-") > 1 # seed-862-long, not seed-862 + # Only for ordering: slow arms of stretched sets take longest. + estimate = warmup_secs(set_name, workload) + ( + (6000 if variant else 2400) if arm in SLOW_ARMS else 900 + ) + jobs.append( + { + "key": f"{set_name}/{workload}/{arm}", + "set": set_name, + "workload": workload, + "arm": arm, + "estimate_secs": estimate, + "attempts": 0, + "status": "queued", + "names": [], + } + ) + jobs.sort(key=lambda j: -j["estimate_secs"]) + return jobs + + +def experiment_name(job): + tag = job["set"].replace("seed-", "s").replace("-", "") + name = f"{tag}_{job['workload'].replace('workload-', 'w')}_{job['arm']}" + return name if job["attempts"] == 1 else f"{name}_r{job['attempts']}" + + +def command(job, name, slot, args): + runner, config, extra, _ = ARMS[job["arm"]] + cmd = [ + sys.executable, + runner, + f"experiment_type=random_workload/{job['set']}/{job['workload']}/{config}", + f"experiment.name={name}", + "providers.cloudlab.num_nodes=1", + f"providers.cloudlab.node_offset={2 * slot}", + f"providers.cloudlab.username={args.username}", + f"providers.cloudlab.hostname_suffix={args.hostname_suffix}", + "controller.punting=false", + *extra, + ] + if runner == "experiment_run_e2e.py": + # The Prometheus TSDB copy is large for long and x5, and unused. + cmd.append("flow.skip_copy_prometheus_data=true") + return cmd + + +def succeeded(job, name): + """The run exited 0 and recorded latencies and results.""" + out = OUTPUTS_DIR / name / ARMS[job["arm"]][3] / "prometheus_client_output" + for f in ("query_latencies.jsonl.gz", "query_results.jsonl.gz"): + path = out / f + if not path.exists() or path.stat().st_size == 0: + return False + try: + with gzip.open(out / "query_latencies.jsonl.gz", "rt") as f: + return bool(f.readline()) + except (OSError, EOFError): + return False + + +def clean_slot(slot, args): + for node in (2 * slot, 2 * slot + 1): + host = f"{args.username}@node{node}.{args.hostname_suffix}" + subprocess.run( + [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "ConnectTimeout=20", + host, + CLEANUP, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=120, + ) + + +def summarize(batch_dir, jobs, set_name, workload): + done = { + j["arm"]: j["names"][-1] + for j in jobs + if j["set"] == set_name and j["workload"] == workload and j["status"] == "done" + } + reference = REFERENCE_ARM if REFERENCE_ARM in done else "prom" + if reference not in done: + return + cmd = [sys.executable, str(COMPARE_ARMS), "--reference", reference] + for arm, name in done.items(): + cmd += ["--arm", f"{arm}={OUTPUTS_DIR / name / ARMS[arm][3]}"] + out = batch_dir / "summaries" / f"{set_name}_{workload}.txt" + out.parent.mkdir(exist_ok=True) + with open(out, "w") as f: + f.write(f"arms: {done}\n\n") + f.flush() + subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT, cwd=EXPERIMENTS_DIR) + + +def save(batch_dir, jobs): + tmp = batch_dir / "state.json.tmp" + tmp.write_text(json.dumps(jobs, indent=1)) + os.replace(tmp, batch_dir / "state.json") + + +def log(batch_dir, message): + line = f"{time.strftime('%Y-%m-%d %H:%M:%S')} {message}" + print(line, flush=True) + with open(batch_dir / "driver.log", "a") as f: + f.write(line + "\n") + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--batch", required=True, help="name; state under experiment_outputs/" + ) + parser.add_argument("--username", required=True) + parser.add_argument("--hostname-suffix", required=True) + parser.add_argument( + "--slots", type=int, required=True, help="two nodes each, from node0" + ) + parser.add_argument( + "--set", action="append", required=True, metavar="SET:WORKLOADS" + ) + parser.add_argument("--stop-launching-after-hours", type=float) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + batch_dir = OUTPUTS_DIR / args.batch + batch_dir.mkdir(parents=True, exist_ok=True) + state = batch_dir / "state.json" + if state.exists(): + jobs = json.loads(state.read_text()) + for j in jobs: + if j["status"] == "running": # the driver died under it + j["status"] = "queued" + else: + jobs = make_jobs([parse_set(s) for s in args.set]) + save(batch_dir, jobs) + + if args.dry_run: + for j in jobs: + j["attempts"] += 1 + print( + f"{j['estimate_secs'] / 3600:4.1f} h " + + shlex.join(command(j, experiment_name(j), 0, args)) + ) + j["attempts"] -= 1 + print( + f"{len(jobs)} jobs, ~{sum(j['estimate_secs'] for j in jobs) / 3600:.0f} " + f"slot-hours on {args.slots} slots" + ) + return + + started = time.time() + running = {} # slot -> (job, process) + while True: + for slot, (job, proc) in list(running.items()): + if proc.poll() is None: + continue + del running[slot] + name = job["names"][-1] + if proc.returncode == 0 and succeeded(job, name): + job["status"] = "done" + elif job["attempts"] < MAX_ATTEMPTS: + job["status"] = "queued" + else: + job["status"] = "failed" + log( + batch_dir, + f"slot {slot}: {name} exit {proc.returncode} -> {job['status']}", + ) + save(batch_dir, jobs) + peers = [ + j + for j in jobs + if (j["set"], j["workload"]) == (job["set"], job["workload"]) + ] + if all(j["status"] in ("done", "failed") for j in peers): + summarize(batch_dir, jobs, job["set"], job["workload"]) + log(batch_dir, f"summary: {job['set']} {job['workload']}") + + launching = args.stop_launching_after_hours is None or ( + time.time() - started < args.stop_launching_after_hours * 3600 + ) + for slot in range(args.slots): + if slot in running or not launching: + continue + # Retries first, then the longest queued job. + queued = [j for j in jobs if j["status"] == "queued"] + if not queued: + break + job = max(queued, key=lambda j: (j["attempts"], j["estimate_secs"])) + clean_slot(slot, args) + job["attempts"] += 1 + name = experiment_name(job) + job["names"].append(name) + job["status"] = "running" + save(batch_dir, jobs) + logs = batch_dir / "logs" + logs.mkdir(exist_ok=True) + out = open(logs / f"{name}.log", "w") + proc = subprocess.Popen( + command(job, name, slot, args), + cwd=EXPERIMENTS_DIR, + stdout=out, + stderr=subprocess.STDOUT, + env={**os.environ, "PYTHONUNBUFFERED": "1"}, + ) + running[slot] = (job, proc) + log(batch_dir, f"slot {slot} (node{2 * slot}-{2 * slot + 1}): start {name}") + + if not running and ( + not launching or not any(j["status"] == "queued" for j in jobs) + ): + break + time.sleep(POLL_SECS) + + counts = { + s: sum(j["status"] == s for j in jobs) for s in ("done", "failed", "queued") + } + log(batch_dir, f"batch finished: {counts}") + + +if __name__ == "__main__": + main() diff --git a/asap-tools/experiments/tests/test_random_workload_batch.py b/asap-tools/experiments/tests/test_random_workload_batch.py new file mode 100644 index 00000000..c9b1f1e0 --- /dev/null +++ b/asap-tools/experiments/tests/test_random_workload_batch.py @@ -0,0 +1,40 @@ +"""Tests for random_workload_batch.py's job planning.""" + +import unittest + +import random_workload_batch as batch + + +class BatchPlanningTest(unittest.TestCase): + def test_sets_name_their_workloads(self): + self.assertEqual( + batch.parse_set("seed-862-x5:01,03-04"), + ("seed-862-x5", ["workload-01", "workload-03", "workload-04"]), + ) + + def test_retries_get_a_new_experiment_name(self): + # A rerun under the old name hits the earlier run's ClickHouse data + # directory, which the runner can't chmod. + job = {"set": "seed-862-long", "workload": "workload-07", "arm": "ch"} + names = [] + for attempts in (1, 2): + names.append(batch.experiment_name({**job, "attempts": attempts})) + self.assertEqual(names, ["s862long_w07_ch", "s862long_w07_ch_r2"]) + + def test_each_job_runs_on_its_slots_two_nodes(self): + job = {"set": "seed-862", "workload": "workload-01", "arm": "prom"} + + class Args: + username, hostname_suffix = "u", "s" + + cmd = batch.command(job, "s862_w01_prom", 3, Args) + self.assertIn("providers.cloudlab.num_nodes=1", cmd) + self.assertIn("providers.cloudlab.node_offset=6", cmd) + self.assertIn("flow.skip_copy_prometheus_data=true", cmd) + self.assertIn( + "experiment_params.experiment=[{mode:baseline,server:prometheus}]", cmd + ) + + +if __name__ == "__main__": + unittest.main() From c449330c6ff27907d788c0550eecb3491718fb23 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 08:27:14 -0400 Subject: [PATCH 02/20] feat(asap-tools): resume a batch around runs still in flight On resume, a job whose run is still alive is left alone (orphaned) instead of rerun, and --skip-slots keeps the driver off busy nodes. Co-Authored-By: Claude Opus 5.5 --- .../experiments/random_workload_batch.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/asap-tools/experiments/random_workload_batch.py b/asap-tools/experiments/random_workload_batch.py index bc874980..9eed2484 100644 --- a/asap-tools/experiments/random_workload_batch.py +++ b/asap-tools/experiments/random_workload_batch.py @@ -227,6 +227,12 @@ def main(): "--set", action="append", required=True, metavar="SET:WORKLOADS" ) parser.add_argument("--stop-launching-after-hours", type=float) + parser.add_argument( + "--skip-slots", + type=lambda s: {int(x) for x in s.split(",")}, + default=set(), + help="slots to leave alone, e.g. 3,15,16 (their nodes are busy)", + ) parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() @@ -236,8 +242,17 @@ def main(): if state.exists(): jobs = json.loads(state.read_text()) for j in jobs: - if j["status"] == "running": # the driver died under it - j["status"] = "queued" + if j["status"] == "running": + # A run the earlier driver left behind keeps going on its own; + # this driver neither reruns nor watches it. + alive = ( + subprocess.run( + ["pgrep", "-f", f"experiment.name={j['names'][-1]}( |$)"], + stdout=subprocess.DEVNULL, + ).returncode + == 0 + ) + j["status"] = "orphaned" if alive else "queued" else: jobs = make_jobs([parse_set(s) for s in args.set]) save(batch_dir, jobs) @@ -288,7 +303,7 @@ def main(): time.time() - started < args.stop_launching_after_hours * 3600 ) for slot in range(args.slots): - if slot in running or not launching: + if slot in running or slot in args.skip_slots or not launching: continue # Retries first, then the longest queued job. queued = [j for j in jobs if j["status"] == "queued"] From 7c01819c1932f366e5349e36664d18ae0dea4016 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 09:29:12 -0400 Subject: [PATCH 03/20] feat(asap-tools): collate and plot a random-workload batch collate_batch.py writes one row per (set, workload, arm): CPU, memory, latency p50/p95/sum, failed queries, the feeder's ingest lag, and whether the run is valid (no failed queries, answers agreeing with Prometheus). It reports each baseline's ratio to ASAPQuery over valid pairs. plot_cost_latency.py draws p95 latency against total CPU per set, invalid runs hollow. compare_arms now reports failed queries instead of crashing. Co-Authored-By: Claude Opus 5.5 --- .../multi_experiment/collate_batch.py | 226 ++++++++++++++++++ .../multi_experiment/compare_arms.py | 20 +- .../multi_experiment/plot_cost_latency.py | 100 ++++++++ 3 files changed, 341 insertions(+), 5 deletions(-) create mode 100644 asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py create mode 100644 asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py diff --git a/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py b/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py new file mode 100644 index 00000000..95db09a9 --- /dev/null +++ b/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Collate a random_workload_batch.py batch: one row per (set, workload, arm) +with total CPU-seconds (warm-up + queries), peak memory, the median, p95 and +sum of every answered query's latency, and whether the run is valid: + +- no query failed or timed out; +- its answers agree with the reference arm (Prometheus, which keeps up): + exact arms to EXACT_REL_ERR, ASAPQuery's sketches to their bounds. + +Separately, whether ingest kept up: during the query phase the feeder never +ran more than MAX_INGEST_LAG_SECS behind (from the feeder's +/paced_feeder_output/*_stats.json). A lagging system answers on-time +queries from incomplete windows, which the agreement check catches; one that +answers late but correctly is slow, not wrong, and stays valid. + +Then each baseline's ratio to ASAPQuery (baseline / asap, > 1 means ASAPQuery +is better) over workloads where both runs are valid. Only workloads whose +arms all finished are collated. + +Usage: + python3 collate_batch.py / [--csv out.csv] +""" + +import argparse +import csv +import glob +import json +import statistics +import sys +from collections import defaultdict +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from compare_arms import Arm, agreement, percentile # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from random_workload_batch import ARMS, OUTPUTS_DIR # noqa: E402 + +REFERENCE = "prom" +# The queries' freshness delay: a feed further behind leaves on-time +# queries' windows incomplete. +MAX_INGEST_LAG_SECS = 10 +EXACT_REL_ERR = 1e-3 +EXACT_TOPK_PRECISION = 0.99 +# ASAPQuery's sketches: quantiles at the 0.95 accuracy SLA; sums are exact +# but for a window-boundary sample; top-k at its precision target. +ASAP_QUANTILE_REL_ERR = 0.05 +ASAP_SUM_REL_ERR = 5e-3 +ASAP_TOPK_PRECISION = 0.95 + +FIELDS = [ + "set", + "workload", + "arm", + "valid", + "why_invalid", + "ingest_kept_up", + "cpu_secs", + "warmup_cpu_secs", + "query_cpu_secs", + "peak_memory_gib", + "latency_p50", + "latency_p95", + "latency_sum", + "queries", + "failed_queries", + "max_ingest_lag_secs", +] + + +def max_ingest_lag(arm, mode_dir): + """The feeder's largest start lag (s) during the query phase, or None + without its stats.""" + stats = glob.glob(str(mode_dir / "paced_feeder_output" / "*_stats.json")) + if not stats: + return None + ticks = json.loads(Path(stats[0]).read_text())["ticks"] + query_ticks = [t for t in ticks if t["tick"] >= arm.warmup_secs] or ticks + return max(t["start_lag_ms"] for t in query_ticks) / 1000 + + +def disagreements(arm_name, arm, ref): + """The queries whose answers fall outside `arm_name`'s bounds against + the reference arm `ref`.""" + bad = [] + for gq, name in arm.names.items(): + topk = name.startswith("topk") + quantile = "quantile" in name + values = [] + for key, want in ref.results.items(): + if key[:2] != gq or key not in arm.results or not want: + continue + value, _ = agreement(arm.results[key], want, topk) + if value is not None: + values.append(value) + if not values: + bad.append(f"{name}: no comparable windows") + continue + mean = statistics.mean(values) + if arm_name == "asap": + ok = ( + mean >= ASAP_TOPK_PRECISION + if topk + else mean <= (ASAP_QUANTILE_REL_ERR if quantile else ASAP_SUM_REL_ERR) + ) + else: + ok = mean >= EXACT_TOPK_PRECISION if topk else mean <= EXACT_REL_ERR + if not ok: + bad.append(f"{name}: {'precision' if topk else 'rel err'} {mean:.3g}") + return bad + + +def rows(batch_dir): + jobs = json.loads((Path(batch_dir) / "state.json").read_text()) + by_workload = defaultdict(dict) + for j in jobs: + by_workload[(j["set"], j["workload"])][j["arm"]] = j + out = [] + for (set_name, workload), jobs_by_arm in sorted(by_workload.items()): + if set(jobs_by_arm) != set(ARMS) or any( + j["status"] != "done" for j in jobs_by_arm.values() + ): + continue + dirs = { + a: OUTPUTS_DIR / j["names"][-1] / ARMS[a][3] for a, j in jobs_by_arm.items() + } + arms = {a: Arm(d) for a, d in dirs.items()} + for arm_name, arm in arms.items(): + r = arm.resources() + recorded = [x for xs in arm.latencies.values() for x in xs] + # A query that failed or timed out records no latency. + latencies = [x for x in recorded if x is not None] + lag = max_ingest_lag(arm, dirs[arm_name]) + why = [] + if len(recorded) != len(latencies): + why.append(f"{len(recorded) - len(latencies)} failed queries") + if arm_name != REFERENCE: + why += disagreements(arm_name, arm, arms[REFERENCE]) + out.append( + { + "set": set_name, + "workload": workload, + "arm": arm_name, + "valid": not why, + "why_invalid": "; ".join(why), + "ingest_kept_up": ( + None if lag is None else lag <= MAX_INGEST_LAG_SECS + ), + "cpu_secs": round(r["cpu_secs"]), + "warmup_cpu_secs": round(r["warmup_cpu_secs"]), + "query_cpu_secs": round(r["query_cpu_secs"]), + "peak_memory_gib": round(r["peak_memory_bytes"] / 2**30, 2), + "latency_p50": round(statistics.median(latencies), 3), + "latency_p95": round(percentile(latencies, 95), 3), + "latency_sum": round(sum(latencies), 1), + "queries": len(recorded), + "failed_queries": len(recorded) - len(latencies), + "max_ingest_lag_secs": None if lag is None else round(lag, 1), + } + ) + return out + + +def ratios(table): + """Per set and baseline arm: the geometric mean, over workloads where + both it and ASAPQuery are valid, of baseline / asap.""" + by = {(r["set"], r["workload"], r["arm"]): r for r in table} + result = defaultdict(lambda: defaultdict(list)) + for (set_name, workload, arm), r in by.items(): + asap = by[(set_name, workload, "asap")] + if arm == "asap" or not (r["valid"] and asap["valid"]): + continue + for key in ("cpu_secs", "latency_p50", "latency_p95", "latency_sum"): + result[(set_name, arm)][key].append(r[key] / asap[key]) + return { + k: { + **{m: statistics.geometric_mean(v) for m, v in metrics.items()}, + "n": len(metrics["cpu_secs"]), + } + for k, metrics in result.items() + } + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("batch_dir") + parser.add_argument("--csv") + args = parser.parse_args() + table = rows(args.batch_dir) + if args.csv: + with open(args.csv, "w", newline="") as f: + w = csv.DictWriter(f, FIELDS) + w.writeheader() + w.writerows(table) + print("Invalid runs:") + for r in table: + if not r["valid"]: + print(f" {r['set']} {r['workload']} {r['arm']}: {r['why_invalid']}") + print( + "\nIngest fell behind (feeder lag > {}s in the query phase):".format( + MAX_INGEST_LAG_SECS + ) + ) + for r in table: + if r["ingest_kept_up"] is False: + print( + f" {r['set']} {r['workload']} {r['arm']}: {r['max_ingest_lag_secs']}s" + ) + print( + "\nASAPQuery improvement = baseline / asap, geometric mean over workloads " + "where both runs are valid" + ) + print( + f"{'set':14s} {'baseline':9s} {'n':>2s} {'CPU':>7s} {'p50':>7s} {'p95':>7s} {'sum':>7s}" + ) + for (set_name, arm), m in sorted(ratios(table).items()): + print( + f"{set_name:14s} {arm:9s} {m['n']:2d} {m['cpu_secs']:6.1f}x " + f"{m['latency_p50']:6.1f}x {m['latency_p95']:6.1f}x {m['latency_sum']:6.1f}x" + ) + + +if __name__ == "__main__": + main() diff --git a/asap-tools/experiments/post_experiment/multi_experiment/compare_arms.py b/asap-tools/experiments/post_experiment/multi_experiment/compare_arms.py index 7578016f..b8f1b707 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/compare_arms.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/compare_arms.py @@ -137,13 +137,23 @@ def report(arms, reference): interval = arms[reference].interval_secs[gq[0]] print(f"\n[{gq[0]}.{gq[1]}] every {interval:g}s: {name}") for arm_name, arm in arms.items(): - xs = arm.latencies.get(gq) - if not xs: + recorded = arm.latencies.get(gq) + if not recorded: continue - within = sum(x <= interval for x in xs) / len(xs) + # A failed or timed-out query records no latency; it never counts + # as answered within the interval. + xs = [x for x in recorded if x is not None] + failed = len(recorded) - len(xs) + within = sum(x <= interval for x in xs) / len(recorded) + stats = ( + f"{percentile(xs, 50):8.3f} {percentile(xs, 95):8.3f} " + f"{percentile(xs, 99):8.3f}" + if xs + else f"{'-':>8s} {'-':>8s} {'-':>8s}" + ) print( - f" {arm_name:16s} n={len(xs):5d} {percentile(xs, 50):8.3f} " - f"{percentile(xs, 95):8.3f} {percentile(xs, 99):8.3f} {within:6.1%}" + f" {arm_name:16s} n={len(recorded):5d} {stats} {within:6.1%}" + + (f" FAILED {failed}" if failed else "") ) print(f"\n== Agreement with {reference}") diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py new file mode 100644 index 00000000..f83a2c18 --- /dev/null +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Cost-latency scatter from collate_batch.py's CSV: one panel per workload +set, one point per (workload, system); x = p95 query latency, y = total +CPU-seconds, both log. Lower left is better. A run collate_batch.py marks +invalid (failed queries, or answers that disagree with the reference) is +drawn hollow: its latency and CPU don't stand for a correct answer. + +Usage: python3 plot_cost_latency.py collated.csv out.png +""" + +import csv +import sys +from collections import defaultdict + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +# Validated categorical slots 1-5 (fixed order) and a marker per system, so +# identity never rests on color alone. +SYSTEMS = [ + ("asap", "ASAPQuery", "#2a78d6", "o", (8, 6)), + ("prom", "Prometheus", "#eb6834", "s", (8, 6)), + ("vm", "VictoriaMetrics", "#1baf7a", "^", (8, -14)), + ("ch", "ClickHouse", "#eda100", "D", (8, 6)), + ("chmv", "ClickHouse+MV", "#e87ba4", "v", (-30, 12)), +] +SURFACE, INK, MUTED = "#fcfcfb", "#0b0b0b", "#52514e" + + +def main(csv_path, out_path): + rows = list(csv.DictReader(open(csv_path))) + by_set = defaultdict(list) + for r in rows: + by_set[r["set"]].append(r) + sets = sorted(by_set) + fig, axes = plt.subplots( + 1, len(sets), figsize=(6 * len(sets), 5), facecolor=SURFACE, squeeze=False + ) + for ax, set_name in zip(axes[0], sets): + ax.set_facecolor(SURFACE) + workloads = sorted({r["workload"] for r in by_set[set_name]}) + for key, label, color, marker, offset in SYSTEMS: + pts = [r for r in by_set[set_name] if r["arm"] == key] + xs = [float(r["latency_p95"]) for r in pts] + ys = [float(r["cpu_secs"]) for r in pts] + valid = [r["valid"] == "True" for r in pts] + ax.scatter( + [x for x, v in zip(xs, valid) if v], + [y for y, v in zip(ys, valid) if v], + s=64, + color=color, + marker=marker, + label=label, + edgecolors=SURFACE, + linewidths=2, + zorder=3, + ) + ax.scatter( + [x for x, v in zip(xs, valid) if not v], + [y for y, v in zip(ys, valid) if not v], + s=56, + facecolors="none", + edgecolors=color, + marker=marker, + linewidths=1.5, + zorder=3, + ) + if xs: + # Direct label at the system's median point. + mx, my = sorted(xs)[len(xs) // 2], sorted(ys)[len(ys) // 2] + ax.annotate( + label, + (mx, my), + xytext=offset, + textcoords="offset points", + fontsize=9, + color=INK, + ) + ax.set_xscale("log") + ax.set_yscale("log") + ax.set_xlabel("p95 query latency (s)", color=MUTED) + ax.set_ylabel("total CPU (CPU-seconds, warm-up + queries)", color=MUTED) + title = ( + f"{set_name}: {len(workloads)} workload(s), lower left is better" + "\nhollow: invalid run (failed queries or wrong answers)" + ) + ax.set_title(title, color=INK, fontsize=10, loc="left") + ax.grid(True, which="major", color="#e6e5e1", linewidth=0.8) + ax.tick_params(colors=MUTED) + for spine in ax.spines.values(): + spine.set_color("#d4d3cf") + axes[0][0].legend(frameon=False, fontsize=9, loc="upper left") + fig.tight_layout() + fig.savefig(out_path, dpi=150, facecolor=SURFACE) + + +if __name__ == "__main__": + main(*sys.argv[1:3]) From d424e2cca53cf2fb1d0ae2e906b8bfcb10c94fc4 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 09:54:58 -0400 Subject: [PATCH 04/20] feat(asap-tools): VictoriaMetrics boundary tolerance; timestamped plots Co-Authored-By: Claude Opus 5.5 --- .../multi_experiment/collate_batch.py | 5 ++++- .../multi_experiment/plot_cost_latency.py | 14 +++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py b/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py index 95db09a9..144b1478 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py @@ -41,6 +41,8 @@ # queries' windows incomplete. MAX_INGEST_LAG_SECS = 10 EXACT_REL_ERR = 1e-3 +# VictoriaMetrics may count a window-boundary sample Prometheus doesn't. +VM_REL_ERR = 5e-3 EXACT_TOPK_PRECISION = 0.99 # ASAPQuery's sketches: quantiles at the 0.95 accuracy SLA; sums are exact # but for a window-boundary sample; top-k at its precision target. @@ -104,7 +106,8 @@ def disagreements(arm_name, arm, ref): else mean <= (ASAP_QUANTILE_REL_ERR if quantile else ASAP_SUM_REL_ERR) ) else: - ok = mean >= EXACT_TOPK_PRECISION if topk else mean <= EXACT_REL_ERR + rel_err = VM_REL_ERR if arm_name == "vm" else EXACT_REL_ERR + ok = mean >= EXACT_TOPK_PRECISION if topk else mean <= rel_err if not ok: bad.append(f"{name}: {'precision' if topk else 'rel err'} {mean:.3g}") return bad diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py index f83a2c18..c0831440 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -5,11 +5,15 @@ invalid (failed queries, or answers that disagree with the reference) is drawn hollow: its latency and CPU don't stand for a correct answer. -Usage: python3 plot_cost_latency.py collated.csv out.png +Writes /cost_latency__.png; never overwrites. + +Usage: python3 plot_cost_latency.py collated.csv out_dir """ import csv +import datetime import sys +from pathlib import Path from collections import defaultdict import matplotlib @@ -29,7 +33,7 @@ SURFACE, INK, MUTED = "#fcfcfb", "#0b0b0b", "#52514e" -def main(csv_path, out_path): +def main(csv_path, out_dir): rows = list(csv.DictReader(open(csv_path))) by_set = defaultdict(list) for r in rows: @@ -93,7 +97,11 @@ def main(csv_path, out_path): spine.set_color("#d4d3cf") axes[0][0].legend(frameon=False, fontsize=9, loc="upper left") fig.tight_layout() - fig.savefig(out_path, dpi=150, facecolor=SURFACE) + stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + out_path = Path(out_dir) / f"cost_latency_{'+'.join(sets)}_{stamp}.png" + with open(out_path, "xb") as f: + fig.savefig(f, dpi=150, facecolor=SURFACE, format="png") + print(out_path) if __name__ == "__main__": From e6922abb1480ae0bd0fd92f171c0555cb05d3352 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 10:10:50 -0400 Subject: [PATCH 05/20] feat(asap-tools): larger, unlabeled cost-latency plots; LiteOlly name Co-Authored-By: Claude Opus 5.5 --- .../multi_experiment/plot_cost_latency.py | 45 ++++++++----------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py index c0831440..41ff20e2 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -24,11 +24,11 @@ # Validated categorical slots 1-5 (fixed order) and a marker per system, so # identity never rests on color alone. SYSTEMS = [ - ("asap", "ASAPQuery", "#2a78d6", "o", (8, 6)), - ("prom", "Prometheus", "#eb6834", "s", (8, 6)), - ("vm", "VictoriaMetrics", "#1baf7a", "^", (8, -14)), - ("ch", "ClickHouse", "#eda100", "D", (8, 6)), - ("chmv", "ClickHouse+MV", "#e87ba4", "v", (-30, 12)), + ("asap", "LiteOlly", "#2a78d6", "o"), + ("prom", "Prometheus", "#eb6834", "s"), + ("vm", "VictoriaMetrics", "#1baf7a", "^"), + ("ch", "ClickHouse", "#eda100", "D"), + ("chmv", "ClickHouse+MV", "#e87ba4", "v"), ] SURFACE, INK, MUTED = "#fcfcfb", "#0b0b0b", "#52514e" @@ -40,12 +40,12 @@ def main(csv_path, out_dir): by_set[r["set"]].append(r) sets = sorted(by_set) fig, axes = plt.subplots( - 1, len(sets), figsize=(6 * len(sets), 5), facecolor=SURFACE, squeeze=False + 1, len(sets), figsize=(7 * len(sets), 6), facecolor=SURFACE, squeeze=False ) for ax, set_name in zip(axes[0], sets): ax.set_facecolor(SURFACE) workloads = sorted({r["workload"] for r in by_set[set_name]}) - for key, label, color, marker, offset in SYSTEMS: + for key, label, color, marker in SYSTEMS: pts = [r for r in by_set[set_name] if r["arm"] == key] xs = [float(r["latency_p95"]) for r in pts] ys = [float(r["cpu_secs"]) for r in pts] @@ -53,49 +53,40 @@ def main(csv_path, out_dir): ax.scatter( [x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], - s=64, + s=128, color=color, marker=marker, label=label, edgecolors=SURFACE, - linewidths=2, + linewidths=1, zorder=3, ) ax.scatter( [x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v], - s=56, + s=112, facecolors="none", edgecolors=color, marker=marker, - linewidths=1.5, + linewidths=2.5, zorder=3, ) - if xs: - # Direct label at the system's median point. - mx, my = sorted(xs)[len(xs) // 2], sorted(ys)[len(ys) // 2] - ax.annotate( - label, - (mx, my), - xytext=offset, - textcoords="offset points", - fontsize=9, - color=INK, - ) ax.set_xscale("log") ax.set_yscale("log") - ax.set_xlabel("p95 query latency (s)", color=MUTED) - ax.set_ylabel("total CPU (CPU-seconds, warm-up + queries)", color=MUTED) + ax.set_xlabel("p95 query latency (s)", color=MUTED, fontsize=14) + ax.set_ylabel( + "total CPU (CPU-seconds, warm-up + queries)", color=MUTED, fontsize=14 + ) title = ( f"{set_name}: {len(workloads)} workload(s), lower left is better" "\nhollow: invalid run (failed queries or wrong answers)" ) - ax.set_title(title, color=INK, fontsize=10, loc="left") + ax.set_title(title, color=INK, fontsize=14, loc="left") ax.grid(True, which="major", color="#e6e5e1", linewidth=0.8) - ax.tick_params(colors=MUTED) + ax.tick_params(colors=MUTED, labelsize=13) for spine in ax.spines.values(): spine.set_color("#d4d3cf") - axes[0][0].legend(frameon=False, fontsize=9, loc="upper left") + axes[0][0].legend(frameon=False, fontsize=13, loc="best") fig.tight_layout() stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") out_path = Path(out_dir) / f"cost_latency_{'+'.join(sets)}_{stamp}.png" From 944bf684d56cf328128fdae59b08abfc6a327715 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 10:17:09 -0400 Subject: [PATCH 06/20] feat(asap-tools): LiteOlly improvement over the worst baseline beside each scatter Co-Authored-By: Claude Opus 5.5 --- .../multi_experiment/plot_cost_latency.py | 59 ++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py index 41ff20e2..bef561f6 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -5,6 +5,10 @@ invalid (failed queries, or answers that disagree with the reference) is drawn hollow: its latency and CPU don't stand for a correct answer. +Beside each scatter, two bars: LiteOlly's improvement in CPU and in p95 +latency over the worst baseline for that metric, as collate_batch.py's +geometric mean of baseline / LiteOlly over workloads where both are valid. + Writes /cost_latency__.png; never overwrites. Usage: python3 plot_cost_latency.py collated.csv out_dir @@ -21,6 +25,9 @@ matplotlib.use("Agg") import matplotlib.pyplot as plt # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from collate_batch import ratios # noqa: E402 + # Validated categorical slots 1-5 (fixed order) and a marker per system, so # identity never rests on color alone. SYSTEMS = [ @@ -39,10 +46,23 @@ def main(csv_path, out_dir): for r in rows: by_set[r["set"]].append(r) sets = sorted(by_set) + numeric = ("cpu_secs", "latency_p50", "latency_p95", "latency_sum") + improvement = ratios( + [ + {**r, "valid": r["valid"] == "True", **{k: float(r[k]) for k in numeric}} + for r in rows + ] + ) + names = {key: label for key, label, *_ in SYSTEMS} fig, axes = plt.subplots( - 1, len(sets), figsize=(7 * len(sets), 6), facecolor=SURFACE, squeeze=False + 1, + 2 * len(sets), + figsize=(9.5 * len(sets), 6), + facecolor=SURFACE, + squeeze=False, + gridspec_kw={"width_ratios": [3, 1.3] * len(sets)}, ) - for ax, set_name in zip(axes[0], sets): + for ax, bar_ax, set_name in zip(axes[0][::2], axes[0][1::2], sets): ax.set_facecolor(SURFACE) workloads = sorted({r["workload"] for r in by_set[set_name]}) for key, label, color, marker in SYSTEMS: @@ -86,6 +106,41 @@ def main(csv_path, out_dir): ax.tick_params(colors=MUTED, labelsize=13) for spine in ax.spines.values(): spine.set_color("#d4d3cf") + bars = [] + for metric, what in (("cpu_secs", "CPU"), ("latency_p95", "p95 latency")): + arm, m = max( + ((arm, m) for (s, arm), m in improvement.items() if s == set_name), + key=lambda am: am[1][metric], + ) + bars.append((what, names[arm], m[metric])) + whats, worst, heights = zip(*bars) + # One worst baseline goes in the title; two go under their bars. + one = len(set(worst)) == 1 + labels = whats if one else [f"{w}\n{b}" for w, b in zip(whats, worst)] + bar_ax.set_facecolor(SURFACE) + bar_ax.bar(labels, heights, width=0.5, color=SYSTEMS[0][2], zorder=3) + for i, h in enumerate(heights): + bar_ax.annotate( + f"{h:.1f}x", + (i, h), + xytext=(0, 4), + textcoords="offset points", + ha="center", + color=INK, + fontsize=13, + ) + bar_ax.set_ylim(0, max(heights) * 1.15) + bar_ax.set_ylabel("LiteOlly improvement, geomean (x)", color=MUTED, fontsize=14) + bar_ax.set_title( + "vs worst baseline" + (f":\n{worst[0]}" if one else ""), + color=INK, + fontsize=14, + loc="left", + ) + bar_ax.grid(True, axis="y", color="#e6e5e1", linewidth=0.8) + bar_ax.tick_params(colors=MUTED, labelsize=12) + for spine in bar_ax.spines.values(): + spine.set_color("#d4d3cf") axes[0][0].legend(frameon=False, fontsize=13, loc="best") fig.tight_layout() stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") From defa76eea51a5be44a111fc6e635c4a8d7a9d6de Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 10:20:16 -0400 Subject: [PATCH 07/20] feat(asap-tools): improvement arrows in the scatter instead of bars Co-Authored-By: Claude Opus 5.5 --- .../multi_experiment/plot_cost_latency.py | 108 ++++++++++-------- 1 file changed, 60 insertions(+), 48 deletions(-) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py index bef561f6..331f21b0 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -5,9 +5,10 @@ invalid (failed queries, or answers that disagree with the reference) is drawn hollow: its latency and CPU don't stand for a correct answer. -Beside each scatter, two bars: LiteOlly's improvement in CPU and in p95 -latency over the worst baseline for that metric, as collate_batch.py's -geometric mean of baseline / LiteOlly over workloads where both are valid. +Two arrows from LiteOlly's geometric-mean point show its improvement in CPU +(vertical) and p95 latency (horizontal) over the worst baseline for that +metric: collate_batch.py's geometric mean of baseline / LiteOlly over +workloads where both are valid. Arrow lengths are those ratios. Writes /cost_latency__.png; never overwrites. @@ -16,6 +17,7 @@ import csv import datetime +import statistics import sys from pathlib import Path from collections import defaultdict @@ -40,6 +42,50 @@ SURFACE, INK, MUTED = "#fcfcfb", "#0b0b0b", "#52514e" +def arrows(ax, rows, improvement, set_name): + """LiteOlly's improvement over the worst baseline per metric, drawn from + LiteOlly's geometric-mean point (over its valid runs, else all).""" + names = {key: label for key, label, *_ in SYSTEMS} + lite = [r for r in rows if r["arm"] == "asap"] + lite = [r for r in lite if r["valid"] == "True"] or lite + lx = statistics.geometric_mean(float(r["latency_p95"]) for r in lite) + ly = statistics.geometric_mean(float(r["cpu_secs"]) for r in lite) + worst = {} + for metric in ("cpu_secs", "latency_p95"): + worst[metric] = max( + ((arm, m[metric]) for (s, arm), m in improvement.items() if s == set_name), + key=lambda am: am[1], + ) + style = dict(arrowstyle="->", color=INK, linewidth=2, mutation_scale=22) + arm, cpu = worst["cpu_secs"] + x = lx / 1.5 + ax.annotate("", (x, ly), xytext=(x, ly * cpu), arrowprops=style, zorder=4) + ax.text( + x * 1.08, + ly * cpu**0.75, + f"{cpu:.1f}x less CPU\nvs {names[arm]}", + color=INK, + fontsize=13, + va="center", + ) + arm, lat = worst["latency_p95"] + y = ly / 1.4 + ax.annotate("", (lx, y), xytext=(lx * lat, y), arrowprops=style, zorder=4) + ax.text( + lx * lat**0.5, + y / 1.08, + f"{lat:.1f}x lower p95 latency vs {names[arm]}", + color=INK, + fontsize=13, + ha="center", + va="top", + ) + left, right = ax.get_xlim() + ax.set_xlim(min(left, lx / 2), right) + bottom, top = ax.get_ylim() + ax.set_ylim(min(bottom, ly / 2), top) + + def main(csv_path, out_dir): rows = list(csv.DictReader(open(csv_path))) by_set = defaultdict(list) @@ -53,16 +99,10 @@ def main(csv_path, out_dir): for r in rows ] ) - names = {key: label for key, label, *_ in SYSTEMS} fig, axes = plt.subplots( - 1, - 2 * len(sets), - figsize=(9.5 * len(sets), 6), - facecolor=SURFACE, - squeeze=False, - gridspec_kw={"width_ratios": [3, 1.3] * len(sets)}, + 1, len(sets), figsize=(7 * len(sets), 6), facecolor=SURFACE, squeeze=False ) - for ax, bar_ax, set_name in zip(axes[0][::2], axes[0][1::2], sets): + for ax, set_name in zip(axes[0], sets): ax.set_facecolor(SURFACE) workloads = sorted({r["workload"] for r in by_set[set_name]}) for key, label, color, marker in SYSTEMS: @@ -106,43 +146,15 @@ def main(csv_path, out_dir): ax.tick_params(colors=MUTED, labelsize=13) for spine in ax.spines.values(): spine.set_color("#d4d3cf") - bars = [] - for metric, what in (("cpu_secs", "CPU"), ("latency_p95", "p95 latency")): - arm, m = max( - ((arm, m) for (s, arm), m in improvement.items() if s == set_name), - key=lambda am: am[1][metric], - ) - bars.append((what, names[arm], m[metric])) - whats, worst, heights = zip(*bars) - # One worst baseline goes in the title; two go under their bars. - one = len(set(worst)) == 1 - labels = whats if one else [f"{w}\n{b}" for w, b in zip(whats, worst)] - bar_ax.set_facecolor(SURFACE) - bar_ax.bar(labels, heights, width=0.5, color=SYSTEMS[0][2], zorder=3) - for i, h in enumerate(heights): - bar_ax.annotate( - f"{h:.1f}x", - (i, h), - xytext=(0, 4), - textcoords="offset points", - ha="center", - color=INK, - fontsize=13, - ) - bar_ax.set_ylim(0, max(heights) * 1.15) - bar_ax.set_ylabel("LiteOlly improvement, geomean (x)", color=MUTED, fontsize=14) - bar_ax.set_title( - "vs worst baseline" + (f":\n{worst[0]}" if one else ""), - color=INK, - fontsize=14, - loc="left", - ) - bar_ax.grid(True, axis="y", color="#e6e5e1", linewidth=0.8) - bar_ax.tick_params(colors=MUTED, labelsize=12) - for spine in bar_ax.spines.values(): - spine.set_color("#d4d3cf") - axes[0][0].legend(frameon=False, fontsize=13, loc="best") - fig.tight_layout() + arrows(ax, by_set[set_name], improvement, set_name) + fig.legend( + *axes[0][0].get_legend_handles_labels(), + frameon=False, + fontsize=13, + loc="lower center", + ncol=len(SYSTEMS), + ) + fig.tight_layout(rect=(0, 0.07, 1, 1)) stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") out_path = Path(out_dir) / f"cost_latency_{'+'.join(sets)}_{stamp}.png" with open(out_path, "xb") as f: From 7a2e5828c2b53fa96b823e2a14c8e0063ecb8b5f Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 11:06:47 -0400 Subject: [PATCH 08/20] feat(asap-tools): time-ordered raw ClickHouse table; batch reruns of chosen arms The raw table's label-first order made every query read the whole table. The batch driver can rerun some arms under a name tag and from a config variant directory. Co-Authored-By: Claude Opus 5.5 --- .../fake_metric/sketch_bench_workloads.py | 6 +++- .../experiments/random_workload_batch.py | 31 ++++++++++++++++--- .../tests/test_random_workload_batch.py | 29 +++++++++++++++++ .../tests/test_sketch_bench_workloads.py | 5 +++ 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/asap-tools/experiments/datasets/fake_metric/sketch_bench_workloads.py b/asap-tools/experiments/datasets/fake_metric/sketch_bench_workloads.py index 01fd972f..cb3655d1 100644 --- a/asap-tools/experiments/datasets/fake_metric/sketch_bench_workloads.py +++ b/asap-tools/experiments/datasets/fake_metric/sketch_bench_workloads.py @@ -581,10 +581,14 @@ def init_sql(metrics: dict, views: List[View], create_views: bool) -> str: columns = [f"{name} String" for name in names] + [ f"{value} Float64" for value in value_columns(desc) ] + # Raw queries read a time window across all series, so only a time + # order lets ClickHouse skip what's outside it. Views read their own + # tables; their source keeps its label order. + order = f"({', '.join(names)}, ts)" if create_views else "ts" statements += [ f"DROP TABLE IF EXISTS {table}", f"CREATE TABLE {table} (ts DateTime64(3), {', '.join(columns)}) " - f"ENGINE = MergeTree ORDER BY ({', '.join(names)}, ts)", + f"ENGINE = MergeTree ORDER BY {order}", ] if create_views: statements += [view_ddl(view) for view in views] diff --git a/asap-tools/experiments/random_workload_batch.py b/asap-tools/experiments/random_workload_batch.py index 9eed2484..06bfa370 100644 --- a/asap-tools/experiments/random_workload_batch.py +++ b/asap-tools/experiments/random_workload_batch.py @@ -19,6 +19,9 @@ --username milindsr --hostname-suffix --slots 21 \\ --set seed-862:01-10 --set seed-862-long:01-10 --set seed-862-x5:01 \\ [--dry-run] [--stop-launching-after-hours 7] + +A rerun of some arms with changed configs: --arm ch --tag ts names the runs +s862_w01_ch_ts, ...; --config-variant sla10 reads -sla10/ configs. """ import argparse @@ -90,12 +93,13 @@ def warmup_secs(set_name, workload): return min(g["client_options"]["starting_delay"] for g in groups) -def make_jobs(sets): +def make_jobs(sets, arms=tuple(ARMS), tag="", config_variant=""): jobs = [] for set_name, workloads in sets: for workload in workloads: - for arm in ARMS: - config = CONFIG_DIR / set_name / workload / f"{ARMS[arm][1]}.yaml" + config_dir = f"{workload}-{config_variant}" if config_variant else workload + for arm in arms: + config = CONFIG_DIR / set_name / config_dir / f"{ARMS[arm][1]}.yaml" if not config.exists(): raise SystemExit(f"missing {config}; render it first") variant = set_name.count("-") > 1 # seed-862-long, not seed-862 @@ -109,6 +113,8 @@ def make_jobs(sets): "set": set_name, "workload": workload, "arm": arm, + "tag": tag, + "config_dir": config_dir, "estimate_secs": estimate, "attempts": 0, "status": "queued", @@ -122,6 +128,8 @@ def make_jobs(sets): def experiment_name(job): tag = job["set"].replace("seed-", "s").replace("-", "") name = f"{tag}_{job['workload'].replace('workload-', 'w')}_{job['arm']}" + if job.get("tag"): + name += f"_{job['tag']}" return name if job["attempts"] == 1 else f"{name}_r{job['attempts']}" @@ -130,7 +138,8 @@ def command(job, name, slot, args): cmd = [ sys.executable, runner, - f"experiment_type=random_workload/{job['set']}/{job['workload']}/{config}", + "experiment_type=random_workload/" + f"{job['set']}/{job.get('config_dir', job['workload'])}/{config}", f"experiment.name={name}", "providers.cloudlab.num_nodes=1", f"providers.cloudlab.node_offset={2 * slot}", @@ -233,6 +242,13 @@ def main(): default=set(), help="slots to leave alone, e.g. 3,15,16 (their nodes are busy)", ) + parser.add_argument( + "--arm", action="append", choices=ARMS, help="only these arms (default all)" + ) + parser.add_argument("--tag", default="", help="suffix for experiment names") + parser.add_argument( + "--config-variant", default="", help="read configs from -VARIANT/" + ) parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() @@ -254,7 +270,12 @@ def main(): ) j["status"] = "orphaned" if alive else "queued" else: - jobs = make_jobs([parse_set(s) for s in args.set]) + jobs = make_jobs( + [parse_set(s) for s in args.set], + args.arm or tuple(ARMS), + args.tag, + args.config_variant, + ) save(batch_dir, jobs) if args.dry_run: diff --git a/asap-tools/experiments/tests/test_random_workload_batch.py b/asap-tools/experiments/tests/test_random_workload_batch.py index c9b1f1e0..940aacaf 100644 --- a/asap-tools/experiments/tests/test_random_workload_batch.py +++ b/asap-tools/experiments/tests/test_random_workload_batch.py @@ -1,6 +1,9 @@ """Tests for random_workload_batch.py's job planning.""" +import tempfile import unittest +from pathlib import Path +from unittest import mock import random_workload_batch as batch @@ -21,6 +24,32 @@ def test_retries_get_a_new_experiment_name(self): names.append(batch.experiment_name({**job, "attempts": attempts})) self.assertEqual(names, ["s862long_w07_ch", "s862long_w07_ch_r2"]) + def test_a_tagged_variant_rerun_names_and_configs_apart(self): + # Reruns must not reuse the original runs' names, whose outputs they + # would overwrite. + with tempfile.TemporaryDirectory() as tmp: + sets = Path(tmp) / "seed-862-long" + (sets / "workload-04").mkdir(parents=True) + (sets / "workload-04" / "clickhouse.yaml").write_text( + "query_groups: [{client_options: {starting_delay: 600}}]" + ) + (sets / "workload-04-sla10").mkdir() + (sets / "workload-04-sla10" / "e2e.yaml").write_text("") + with mock.patch.object(batch, "CONFIG_DIR", Path(tmp)): + [job] = batch.make_jobs( + [("seed-862-long", ["workload-04"])], ("asap",), "sla10", "sla10" + ) + job["attempts"] = 1 + self.assertEqual(batch.experiment_name(job), "s862long_w04_asap_sla10") + + class Args: + username, hostname_suffix = "u", "s" + + self.assertIn( + "experiment_type=random_workload/seed-862-long/workload-04-sla10/e2e", + batch.command(job, "n", 0, Args), + ) + def test_each_job_runs_on_its_slots_two_nodes(self): job = {"set": "seed-862", "workload": "workload-01", "arm": "prom"} diff --git a/asap-tools/experiments/tests/test_sketch_bench_workloads.py b/asap-tools/experiments/tests/test_sketch_bench_workloads.py index 1330a381..9275e7e5 100644 --- a/asap-tools/experiments/tests/test_sketch_bench_workloads.py +++ b/asap-tools/experiments/tests/test_sketch_bench_workloads.py @@ -186,6 +186,11 @@ def test_metric_without_labels_raises(self): with self.assertRaises(ValueError): sbw.init_sql({"data": {"type": "gauge", "labels": []}}, [], False) + def test_raw_table_is_ordered_by_time_only(self): + metrics = {"data": {"type": "gauge", "labels": [{"name": "job"}]}} + self.assertIn("ORDER BY ts;", sbw.init_sql(metrics, [], False)) + self.assertIn("ORDER BY (job, ts);", sbw.init_sql(metrics, [], True)) + def test_describe_is_one_line(self): entry = {"promql": "sum by (job) (\n data\n)", "interval_secs": 1} self.assertEqual(sbw.describe(entry), "sum by (job) ( data )") From 0a716635c815280496a295bfa961cb7aeb796a3c Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 11:23:40 -0400 Subject: [PATCH 09/20] fix(asap-tools): a resumed batch keeps its orphaned runs' slots It cleaned and reused them, wrecking the runs still going there. Co-Authored-By: Claude Opus 5.5 --- .../experiments/random_workload_batch.py | 50 +++++++++++++++---- .../tests/test_random_workload_batch.py | 13 +++++ 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/asap-tools/experiments/random_workload_batch.py b/asap-tools/experiments/random_workload_batch.py index 06bfa370..b4a35ff6 100644 --- a/asap-tools/experiments/random_workload_batch.py +++ b/asap-tools/experiments/random_workload_batch.py @@ -207,6 +207,34 @@ def summarize(batch_dir, jobs, set_name, workload): subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT, cwd=EXPERIMENTS_DIR) +def alive(name): + """Whether a runner for experiment `name` runs on this machine.""" + pattern = f"experiment.name={name}( |$)" + return ( + subprocess.run(["pgrep", "-f", pattern], stdout=subprocess.DEVNULL).returncode + == 0 + ) + + +def settle_orphans(jobs, is_alive=alive): + """The slots of orphaned jobs (runs an earlier driver left going) that + still run, so nothing launches over them. A finished orphan becomes done, + or queued for its retry.""" + busy = set() + for j in jobs: + if j["status"] != "orphaned": + continue + if "slot" not in j: + raise SystemExit(f"{j['names'][-1]} has no recorded slot") + if is_alive(j["names"][-1]): + busy.add(j["slot"]) + elif succeeded(j, j["names"][-1]): + j["status"] = "done" + else: + j["status"] = "queued" if j["attempts"] < MAX_ATTEMPTS else "failed" + return busy + + def save(batch_dir, jobs): tmp = batch_dir / "state.json.tmp" tmp.write_text(json.dumps(jobs, indent=1)) @@ -259,16 +287,9 @@ def main(): jobs = json.loads(state.read_text()) for j in jobs: if j["status"] == "running": - # A run the earlier driver left behind keeps going on its own; - # this driver neither reruns nor watches it. - alive = ( - subprocess.run( - ["pgrep", "-f", f"experiment.name={j['names'][-1]}( |$)"], - stdout=subprocess.DEVNULL, - ).returncode - == 0 - ) - j["status"] = "orphaned" if alive else "queued" + # A run the earlier driver left behind keeps going on its own + # and keeps its slot until it ends. + j["status"] = "orphaned" if alive(j["names"][-1]) else "queued" else: jobs = make_jobs( [parse_set(s) for s in args.set], @@ -323,8 +344,14 @@ def main(): launching = args.stop_launching_after_hours is None or ( time.time() - started < args.stop_launching_after_hours * 3600 ) + orphan_slots = settle_orphans(jobs) for slot in range(args.slots): - if slot in running or slot in args.skip_slots or not launching: + if ( + slot in running + or slot in args.skip_slots + or slot in orphan_slots + or not launching + ): continue # Retries first, then the longest queued job. queued = [j for j in jobs if j["status"] == "queued"] @@ -336,6 +363,7 @@ def main(): name = experiment_name(job) job["names"].append(name) job["status"] = "running" + job["slot"] = slot save(batch_dir, jobs) logs = batch_dir / "logs" logs.mkdir(exist_ok=True) diff --git a/asap-tools/experiments/tests/test_random_workload_batch.py b/asap-tools/experiments/tests/test_random_workload_batch.py index 940aacaf..97213359 100644 --- a/asap-tools/experiments/tests/test_random_workload_batch.py +++ b/asap-tools/experiments/tests/test_random_workload_batch.py @@ -50,6 +50,19 @@ class Args: batch.command(job, "n", 0, Args), ) + def test_a_live_orphan_keeps_its_slot(self): + # A resumed driver once cleaned and reused the slots of runs it + # had left going, wrecking them. + jobs = [ + {"status": "orphaned", "names": ["a"], "slot": 9, "attempts": 1}, + {"status": "orphaned", "names": ["b"], "slot": 10, "attempts": 1}, + {"status": "queued", "names": [], "attempts": 0}, + ] + with mock.patch.object(batch, "succeeded", return_value=False): + busy = batch.settle_orphans(jobs, is_alive=lambda name: name == "a") + self.assertEqual(busy, {9}) + self.assertEqual([j["status"] for j in jobs], ["orphaned", "queued", "queued"]) + def test_each_job_runs_on_its_slots_two_nodes(self): job = {"set": "seed-862", "workload": "workload-01", "arm": "prom"} From 828236784aee915959eb8ac34f11d09a9a8d0eef Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 12:05:12 -0400 Subject: [PATCH 10/20] fix(asap-tools): fit a one-panel cost-latency legend Co-Authored-By: Claude Opus 5.5 --- .../post_experiment/multi_experiment/plot_cost_latency.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py index 331f21b0..20e9a4e5 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -152,9 +152,10 @@ def main(csv_path, out_dir): frameon=False, fontsize=13, loc="lower center", - ncol=len(SYSTEMS), + ncol=min(len(SYSTEMS), 3 * len(sets)), ) - fig.tight_layout(rect=(0, 0.07, 1, 1)) + # A one-panel figure fits three legend entries a row, so two rows. + fig.tight_layout(rect=(0, 0.07 if len(sets) > 1 else 0.12, 1, 1)) stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") out_path = Path(out_dir) / f"cost_latency_{'+'.join(sets)}_{stamp}.png" with open(out_path, "xb") as f: From 1fea0bcc0c7f2e17d8541f4cefe372efbb984a4f Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 12:38:23 -0400 Subject: [PATCH 11/20] feat(asap-tools): baseline ovals, double arrows, denser log ticks in cost-latency plot Also a figure title, thicker axes, and "total compute cost" on the y axis. Co-Authored-By: Claude Opus 5.5 --- .../multi_experiment/plot_cost_latency.py | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py index 20e9a4e5..f9e68191 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -10,6 +10,8 @@ metric: collate_batch.py's geometric mean of baseline / LiteOlly over workloads where both are valid. Arrow lengths are those ratios. +An oval encloses each baseline's valid runs (all its runs if none is valid). + Writes /cost_latency__.png; never overwrites. Usage: python3 plot_cost_latency.py collated.csv out_dir @@ -23,9 +25,11 @@ from collections import defaultdict import matplotlib +import numpy as np matplotlib.use("Agg") import matplotlib.pyplot as plt # noqa: E402 +from matplotlib import ticker # noqa: E402 sys.path.insert(0, str(Path(__file__).resolve().parent)) from collate_batch import ratios # noqa: E402 @@ -42,6 +46,22 @@ SURFACE, INK, MUTED = "#fcfcfb", "#0b0b0b", "#52514e" +def oval(ax, xs, ys, color): + """Ellipse in log space along the points' covariance, scaled to enclose + them all with a fixed margin (in decades) for the markers.""" + pts = np.log10(np.column_stack([xs, ys])) + center = pts.mean(axis=0) + # The small diagonal keeps one or collinear points drawable. + cov = np.cov(pts.T, bias=True) + np.eye(2) * 1e-4 + inv = np.linalg.inv(cov) + d = pts - center + reach = np.sqrt(np.einsum("ij,jk,ik->i", d, inv, d).max()) + vals, vecs = np.linalg.eigh(cov) + t = np.linspace(0, 2 * np.pi, 200) + ring = (vecs * (np.sqrt(vals) * reach + 0.05)) @ np.vstack([np.cos(t), np.sin(t)]) + ax.plot(*10 ** (ring + center[:, None]), color=color, linewidth=1.5, zorder=2) + + def arrows(ax, rows, improvement, set_name): """LiteOlly's improvement over the worst baseline per metric, drawn from LiteOlly's geometric-mean point (over its valid runs, else all).""" @@ -56,7 +76,7 @@ def arrows(ax, rows, improvement, set_name): ((arm, m[metric]) for (s, arm), m in improvement.items() if s == set_name), key=lambda am: am[1], ) - style = dict(arrowstyle="->", color=INK, linewidth=2, mutation_scale=22) + style = dict(arrowstyle="<->", color=INK, linewidth=2, mutation_scale=22) arm, cpu = worst["cpu_secs"] x = lx / 1.5 ax.annotate("", (x, ly), xytext=(x, ly * cpu), arrowprops=style, zorder=4) @@ -131,22 +151,32 @@ def main(csv_path, out_dir): linewidths=2.5, zorder=3, ) + # Invalid runs don't stand for the system, so they don't stretch its oval. + fit = [(x, y) for x, y, v in zip(xs, ys, valid) if v] or list(zip(xs, ys)) + if key != "asap" and fit: + oval(ax, *zip(*fit), color) ax.set_xscale("log") ax.set_yscale("log") ax.set_xlabel("p95 query latency (s)", color=MUTED, fontsize=14) - ax.set_ylabel( - "total CPU (CPU-seconds, warm-up + queries)", color=MUTED, fontsize=14 - ) + ax.set_ylabel("total compute cost", color=MUTED, fontsize=14) + for axis in (ax.xaxis, ax.yaxis): + axis.set_major_locator(ticker.LogLocator(subs=(1, 2, 5))) + axis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f"{v:g}")) + axis.set_minor_formatter(ticker.NullFormatter()) title = ( f"{set_name}: {len(workloads)} workload(s), lower left is better" "\nhollow: invalid run (failed queries or wrong answers)" ) ax.set_title(title, color=INK, fontsize=14, loc="left") ax.grid(True, which="major", color="#e6e5e1", linewidth=0.8) - ax.tick_params(colors=MUTED, labelsize=13) + ax.tick_params(colors=MUTED, labelsize=13, width=2) for spine in ax.spines.values(): - spine.set_color("#d4d3cf") + spine.set_color(INK) + spine.set_linewidth(2) arrows(ax, by_set[set_name], improvement, set_name) + fig.suptitle( + "Cost-latency tradeoffs of LiteOlly vs. baselines", color=INK, fontsize=16 + ) fig.legend( *axes[0][0].get_legend_handles_labels(), frameon=False, From d54efa34b3852bc0fa50009e237b0ef3ab72b9e2 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 12:49:37 -0400 Subject: [PATCH 12/20] feat(asap-tools): multiple-dashboard random-workload configs random_workload_artifacts.py --dashboards K writes -dashK/ run configs with every query group repeated K times (e2e, VictoriaMetrics, ClickHouse and ClickHouse+MV), on the same plan, feeder and data. Each copy is its own client group and thread, so copies are issued concurrently and their latencies recorded apart. The query client's connection pool now fits every query in flight; the default 10 would open a new connection per excess request at K >= 2. collate_batch.py collates batches of an arm subset, against ClickHouse+MV when Prometheus didn't run. Co-Authored-By: Claude Opus 5.5 --- .../fake_metric/random_workload_artifacts.py | 46 +++++++++++++++++-- .../multi_experiment/collate_batch.py | 24 ++++++---- .../tests/test_random_workload_artifacts.py | 36 +++++++++++++++ .../main_prometheus_client.py | 7 ++- 4 files changed, 96 insertions(+), 17 deletions(-) diff --git a/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py b/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py index 289236a8..5a1fb248 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py +++ b/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py @@ -133,7 +133,24 @@ def workload_set(workload): return f"seed-{workload['seed']}" + (f"-{variant}" if variant else "") -def pilot_configs(workload, pilot_dir): +def as_dashboards(groups, dashboards, key): + """`groups` repeated once per dashboard, each copy its own client thread + and output group; `key` ("id" or "name") stays unique per copy.""" + if dashboards == 1: + return groups + copies = [] + for d in range(1, dashboards + 1): + for group in groups: + group = copy.deepcopy(group) + if key == "id": + group["id"] = len(copies) + 1 + else: + group[key] = f"{group[key]}_d{d}" + copies.append(group) + return copies + + +def pilot_configs(workload, pilot_dir, dashboards=1): """The five-system pilot's run configs, {path under EXPERIMENTS_DIR: content}: ASAPQuery (the saved plan in `pilot_dir`) and Prometheus in one experiment_run_e2e.py config, VictoriaMetrics in another, and raw @@ -144,7 +161,10 @@ def pilot_configs(workload, pilot_dir): freshness delay, then evaluates each query at multiples of its interval for PILOT_DURATION_MS: the same windows relative to its data start. Raw ClickHouse runs the manifest's clickhouse_sql, and ClickHouse+MV - the same RQEs' Query over exact views (sketch_bench_workloads).""" + the same RQEs' Query over exact views (sketch_bench_workloads). + + With `dashboards` K > 1, every query group runs K times concurrently, + as K dashboards over the same data, from `-dashK/`.""" queries = [q for q in workload["queries"] if q["support"] == PILOT_SUPPORT] lookback_secs = max(q["lookback_ms"] for q in queries) // 1000 starting_delay = lookback_secs + QUERY_FRESHNESS_DELAY_SECS @@ -152,7 +172,8 @@ def pilot_configs(workload, pilot_dir): feed_secs = starting_delay + FEED_BOUND_QUERY_PHASES * duration seed = workload["data_facts"]["feeder_seed"] table = pilot_dir / "eval_table.json" - out_dir = Path(CONFIG_DIR) / workload_set(workload) / workload["id"] + variant = f"-dash{dashboards}" if dashboards > 1 else "" + out_dir = Path(CONFIG_DIR) / workload_set(workload) / (workload["id"] + variant) e2e, _ = build_config( str(table), @@ -163,6 +184,10 @@ def pilot_configs(workload, pilot_dir): feeder_duration_secs=feed_secs, prebuilt_output_dir=str(pilot_dir), ) + e2e_params = e2e["experiment_params"] + e2e_params["query_groups"] = as_dashboards( + e2e_params["query_groups"], dashboards, "id" + ) # One monitoring tool per run, so VictoriaMetrics is its own baseline run. vm = copy.deepcopy(e2e) del vm["controller"] @@ -232,7 +257,7 @@ def pilot_configs(workload, pilot_dir): "paced_feeder": feeder, "clickhouse_image_tag": CLICKHOUSE_IMAGE_TAG, }, - "query_groups": groups, + "query_groups": as_dashboards(groups, dashboards, "name"), "data_ingestion_interval_ms": workload["data_facts"]["scrape_interval_ms"], } files[name] = "# @package experiment_params\n" + yaml.safe_dump( @@ -277,7 +302,16 @@ def main(): help="the workload's archived pilot plan (eval_table.json and planner " f"configs): also write the pilot's run configs under {CONFIG_DIR}", ) + parser.add_argument( + "--dashboards", + type=int, + default=1, + help="with --pilot-dir: issue every query group this many times " + "concurrently, into -dashK/", + ) args = parser.parse_args() + if args.dashboards < 1: + parser.error("--dashboards must be at least 1") workload = json.loads(args.manifest.read_text()) rendered = render(workload) args.out.mkdir(parents=True, exist_ok=True) @@ -305,7 +339,9 @@ def main(): ) print(table) if args.pilot_dir: - for path, content in pilot_configs(workload, args.pilot_dir).items(): + for path, content in pilot_configs( + workload, args.pilot_dir, args.dashboards + ).items(): path = Path(EXPERIMENTS_DIR) / path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py b/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py index 144b1478..abb675e2 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/collate_batch.py @@ -4,8 +4,9 @@ sum of every answered query's latency, and whether the run is valid: - no query failed or timed out; -- its answers agree with the reference arm (Prometheus, which keeps up): - exact arms to EXACT_REL_ERR, ASAPQuery's sketches to their bounds. +- its answers agree with the reference arm (Prometheus, which keeps up, or + else ClickHouse+MV): exact arms to EXACT_REL_ERR, ASAPQuery's sketches to + their bounds. Separately, whether ingest kept up: during the query phase the feeder never ran more than MAX_INGEST_LAG_SECS behind (from the feeder's @@ -15,7 +16,7 @@ Then each baseline's ratio to ASAPQuery (baseline / asap, > 1 means ASAPQuery is better) over workloads where both runs are valid. Only workloads whose -arms all finished are collated. +batch arms all finished, one of them a reference, are collated. Usage: python3 collate_batch.py / [--csv out.csv] @@ -37,6 +38,8 @@ from random_workload_batch import ARMS, OUTPUTS_DIR # noqa: E402 REFERENCE = "prom" +# The reference of a batch without Prometheus: exact, and answers every window. +FALLBACK_REFERENCE = "chmv" # The queries' freshness delay: a feed further behind leaves on-time # queries' windows incomplete. MAX_INGEST_LAG_SECS = 10 @@ -120,9 +123,10 @@ def rows(batch_dir): by_workload[(j["set"], j["workload"])][j["arm"]] = j out = [] for (set_name, workload), jobs_by_arm in sorted(by_workload.items()): - if set(jobs_by_arm) != set(ARMS) or any( - j["status"] != "done" for j in jobs_by_arm.values() - ): + if any(j["status"] != "done" for j in jobs_by_arm.values()): + continue + reference = REFERENCE if REFERENCE in jobs_by_arm else FALLBACK_REFERENCE + if reference not in jobs_by_arm: continue dirs = { a: OUTPUTS_DIR / j["names"][-1] / ARMS[a][3] for a, j in jobs_by_arm.items() @@ -137,8 +141,8 @@ def rows(batch_dir): why = [] if len(recorded) != len(latencies): why.append(f"{len(recorded) - len(latencies)} failed queries") - if arm_name != REFERENCE: - why += disagreements(arm_name, arm, arms[REFERENCE]) + if arm_name != reference: + why += disagreements(arm_name, arm, arms[reference]) out.append( { "set": set_name, @@ -170,8 +174,8 @@ def ratios(table): by = {(r["set"], r["workload"], r["arm"]): r for r in table} result = defaultdict(lambda: defaultdict(list)) for (set_name, workload, arm), r in by.items(): - asap = by[(set_name, workload, "asap")] - if arm == "asap" or not (r["valid"] and asap["valid"]): + asap = by.get((set_name, workload, "asap")) + if arm == "asap" or not (asap and r["valid"] and asap["valid"]): continue for key in ("cpu_secs", "latency_p50", "latency_p95", "latency_sum"): result[(set_name, arm)][key].append(r[key] / asap[key]) diff --git a/asap-tools/experiments/tests/test_random_workload_artifacts.py b/asap-tools/experiments/tests/test_random_workload_artifacts.py index 47ba9ddd..cd158a2f 100644 --- a/asap-tools/experiments/tests/test_random_workload_artifacts.py +++ b/asap-tools/experiments/tests/test_random_workload_artifacts.py @@ -194,6 +194,42 @@ def test_every_arm_feeds_the_same_data_and_schedule(self): raw_sql = "".join(files[f"interval_{t}s_baseline.sql"] for t in (1, 20, 30)) self.assertTrue(all(sql in raw_sql for sql in pilot_sql)) + def test_two_dashboards_repeat_every_group_on_the_same_plan(self): + workload = workloads.generate_workloads(seed=862)[0] + + def configs(dashboards): + files = artifacts.pilot_configs(workload, PILOT_DIR, dashboards) + # The SQL files are the same, under the variant's directory. + return { + p.name: yaml.safe_load(c.replace("workload-01-dash2/", "workload-01/")) + for p, c in files.items() + if p.suffix == ".yaml" + }, {p.parent.name for p in files} + + one, _ = configs(1) + two, dirs = configs(2) + self.assertEqual(dirs, {"workload-01-dash2"}) + for name, key in ( + ("e2e.yaml", "id"), + ("clickhouse.yaml", "name"), + ("clickhouse_mv.yaml", "name"), + ): + with self.subTest(config=name): + single, double = one[name], two[name] + params = ( + (single["experiment_params"], double["experiment_params"]) + if name == "e2e.yaml" + else (single, double) + ) + groups, copies = (p.pop("query_groups") for p in params) + # Same plan, feeder and data; only the groups repeat. + self.assertEqual(double, single) + # Each copy is its own group, so the client keeps its latencies apart. + self.assertEqual(len({g[key] for g in copies}), 2 * len(groups)) + for g in groups + copies: + del g[key] + self.assertEqual(copies, 2 * groups) + class PilotConfigsPreflightTest(unittest.TestCase): """Each config through Hydra and the checks its runner makes before it diff --git a/asap-tools/queriers/prometheus-client/main_prometheus_client.py b/asap-tools/queriers/prometheus-client/main_prometheus_client.py index 1382c7c3..44512f40 100644 --- a/asap-tools/queriers/prometheus-client/main_prometheus_client.py +++ b/asap-tools/queriers/prometheus-client/main_prometheus_client.py @@ -652,6 +652,9 @@ def main(args: Any) -> None: global_query_idx += len(query_group.queries) server_url_for_alignment = None + # Every query of every group can be in flight at once; a smaller pool + # opens a new connection per excess request, inside its latency. + pool_size = max(requests.adapters.DEFAULT_POOLSIZE, global_query_idx) servers: Dict[str, QueryClient] = {} for server in config.servers: @@ -676,7 +679,7 @@ def main(args: Any) -> None: ) # Mount debug adapter for HTTP request logging - debug_adapter = DebugHTTPAdapter(server.name) + debug_adapter = DebugHTTPAdapter(server.name, pool_maxsize=pool_size) client.session.mount("http://", debug_adapter) client.session.mount("https://", debug_adapter) else: @@ -691,7 +694,7 @@ def main(args: Any) -> None: ) # Mount debug adapter for HTTP request logging - debug_adapter = DebugHTTPAdapter(server.name) + debug_adapter = DebugHTTPAdapter(server.name, pool_maxsize=pool_size) client.session.mount("http://", debug_adapter) client.session.mount("https://", debug_adapter) From 19a8fa2be7440d5fe3ad8534c44247c4f6a09ec8 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 12:48:32 -0400 Subject: [PATCH 13/20] feat(asap-tools): replay recorded gauge values in the paced feeder A workload table's gauge can carry recorded values (values.replay), which the feeder loops one step per sample in place of generated ones. make_replay.py fits Alibaba v2022 pod CPU into the random workloads' label shape and writes such a table. Co-Authored-By: Claude Opus 5.5 --- .../data-sources/paced-feeder/src/table.rs | 52 +++++++++ .../data-sources/paced-feeder/src/workload.rs | 6 + .../datasets/alibaba/make_replay.py | 110 ++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 asap-tools/experiments/datasets/alibaba/make_replay.py diff --git a/asap-tools/data-sources/paced-feeder/src/table.rs b/asap-tools/data-sources/paced-feeder/src/table.rs index 97cd6546..dd33aa6b 100644 --- a/asap-tools/data-sources/paced-feeder/src/table.rs +++ b/asap-tools/data-sources/paced-feeder/src/table.rs @@ -57,6 +57,15 @@ struct GaugeValues { dist: ParetoDist, } +/// A recording to replay instead of generated values: `steps` values per +/// series, series-major in [`series_labels`] order. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ReplayValues { + steps: usize, + values: Vec, +} + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct ParetoDist { @@ -161,6 +170,31 @@ fn metric(name: &str, spec: &Value, seed: u64) -> Result<(Metric, u32)> { ); } let series = series_labels(&spec.labels)?; + if let Some(replay) = spec.values.get("replay") { + ensure!(spec.kind == "gauge", "`replay` values apply to gauges only"); + let replay: ReplayValues = + serde_json::from_value(replay.clone()).context("gauge `values.replay`")?; + ensure!(replay.steps > 0, "`replay.steps` must be positive"); + ensure!( + replay.values.len() == series.len() * replay.steps, + "`replay.values` has {} values, not {} series x {} steps", + replay.values.len(), + series.len(), + replay.steps + ); + let metric = Metric { + name: name.to_string(), + labels: spec.labels, + scales: vec![1.0; series.len()], + series, + kind: Kind::Replay { + steps: replay.steps, + values: Arc::new(replay.values), + }, + seed: splitmix64(seed ^ fnv1a(name)), + }; + return Ok((metric, spec.samples_per_sec_per_series)); + } let (skew, kind) = if spec.kind == "gauge" { let values: GaugeValues = serde_json::from_value(spec.values).context("gauge `values`")?; ensure!( @@ -415,6 +449,24 @@ mod tests { assert_eq!(series[11], ["1", "2", "2.1"]); } + #[test] + fn replay_values_loop_per_series() { + let spec = json!({ + "type": "gauge", + "samples_per_sec_per_series": 10, + "labels": [{"name": "job", "cardinality": 2}], + "values": {"replay": {"steps": 3, "values": [1.0, 2.0, 3.0, 10.0, 20.0, 30.0]}}, + }); + let (m, rate) = metric("data", &spec, 7).unwrap(); + assert_eq!(rate, 10); + let seen: Vec = (0..4).map(|n| m.value(1, n)).collect(); + assert_eq!(seen, [10.0, 20.0, 30.0, 10.0]); + // One value per series and step, no more and no fewer. + let mut short = spec.clone(); + short["values"]["replay"]["values"] = json!([1.0, 2.0]); + assert!(metric("data", &short, 7).is_err()); + } + #[test] fn rejects_a_child_before_its_parent() { let labels = vec![label("label_0", 3, Some("job")), label("job", 2, None)]; diff --git a/asap-tools/data-sources/paced-feeder/src/workload.rs b/asap-tools/data-sources/paced-feeder/src/workload.rs index 81acc6d6..a25c8a4b 100644 --- a/asap-tools/data-sources/paced-feeder/src/workload.rs +++ b/asap-tools/data-sources/paced-feeder/src/workload.rs @@ -59,6 +59,9 @@ pub enum Kind { /// `(n + 1) · scale` at sample index `n`: starts at the first sample, grows /// by the series' scale per sample and never resets. Counter, + /// Recorded values, `steps` per series, series-major: sample index `n` + /// takes step `n mod steps`, so the recording loops. + Replay { steps: usize, values: Arc> }, } #[derive(Debug, Clone)] @@ -213,6 +216,9 @@ impl Metric { scale * pareto_scale * u.powf(-1.0 / pareto_shape) } Kind::Counter => (sample_index + 1) as f64 * scale, + Kind::Replay { steps, ref values } => { + values[series * steps + (sample_index % steps as u64) as usize] + } } } } diff --git a/asap-tools/experiments/datasets/alibaba/make_replay.py b/asap-tools/experiments/datasets/alibaba/make_replay.py new file mode 100644 index 00000000..1f5b469a --- /dev/null +++ b/asap-tools/experiments/datasets/alibaba/make_replay.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""A replay table for paced-feeder from Alibaba v2022 MSMetrics: pod CPU +utilization in the random workloads' label shape (4 region x 5 service x 10 +job x 50 label_0), so their plans apply unchanged. + +Each job is one microservice and each label_0 one of its pods; region and +service only group microservices. Label values follow the feeder's table +convention: a child's value is its parent's, a `.`, and its index (`job="3.1.7"`). +Microservices are drawn by `--seed` among those with at least 50 pods that +report at every step of the input; pods are their first 50 by name. + +Writes `.csv` (`offset_ms,region,service,job,label_0,value`, sorted by +offset, offsets from the first step) and `_mapping.csv` (labels to +msname/msinstanceid). With `--table`, also `_table.json`: that +workload table with its `data` metric's values replaced by the replay, which +paced-feeder loops (one trace step per sample). + +Usage: python3 make_replay.py MSMetricsUpdate_0.csv MSMetricsUpdate_1.csv \ + --out replay [--table eval_table.json] +""" + +import argparse +import csv +import json +import random +from collections import defaultdict + +REGIONS, SERVICES, JOBS, PODS = 4, 5, 10, 50 + + +def rows(paths): + for path in paths: + with open(path, newline="") as f: + for r in csv.DictReader(f): + yield r + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("csv", nargs="+") + parser.add_argument("--out", required=True) + parser.add_argument("--seed", type=int, default=862) + parser.add_argument("--table", help="workload table whose `data` gets the replay") + args = parser.parse_args() + + steps, seen = set(), defaultdict(set) # pod -> steps + pods = defaultdict(set) # msname -> pods + for r in rows(args.csv): + steps.add(r["timestamp"]) + seen[r["msinstanceid"]].add(r["timestamp"]) + pods[r["msname"]].add(r["msinstanceid"]) + full = {p for p, s in seen.items() if len(s) == len(steps)} + eligible = sorted(m for m, ps in pods.items() if len(ps & full) >= PODS) + jobs = REGIONS * SERVICES * JOBS + print(f"{len(steps)} steps; {len(eligible)} microservices eligible") + chosen = random.Random(args.seed).sample(eligible, jobs) + + labels = {} # pod -> (region, service, job, label_0) + with open(f"{args.out}_mapping.csv", "w", newline="") as f: + w = csv.writer(f) + w.writerow(["region", "service", "job", "label_0", "msname", "msinstanceid"]) + for i, msname in enumerate(chosen): + region = str(i // (SERVICES * JOBS)) + service = f"{region}.{i // JOBS % SERVICES}" + job = f"{service}.{i % JOBS}" + ordered = sorted( + pods[msname] & full, key=lambda p: int(p.rsplit("_", 1)[1]) + ) + for k, pod in enumerate(ordered[:PODS]): + labels[pod] = (region, service, job, f"{job}.{k}") + w.writerow([*labels[pod], msname, pod]) + + start = min(int(s) for s in steps) + out = sorted( + (int(r["timestamp"]) - start, *labels[r["msinstanceid"]], r["cpu_utilization"]) + for r in rows(args.csv) + if r["msinstanceid"] in labels + ) + with open(f"{args.out}.csv", "w", newline="") as f: + w = csv.writer(f) + w.writerow(["offset_ms", "region", "service", "job", "label_0", "value"]) + w.writerows(out) + print(f"{len(out)} rows, {len(labels)} series") + if args.table: + write_table(args.table, f"{args.out}_table.json", out, len(steps)) + + +def write_table(table_path, out_path, rows, steps): + """`table_path` with `data`'s values replaced by `rows`, series-major in + paced-feeder's series order (mixed radix, label_0 fastest).""" + by_series = {} + for offset, *_, label_0, value in rows: # sorted by offset + by_series.setdefault(label_0, []).append(round(float(value), 6)) + values = [] + for r in range(REGIONS): + for s in range(SERVICES): + for j in range(JOBS): + for k in range(PODS): + values += by_series[f"{r}.{s}.{j}.{k}"] + assert len(values) == len(by_series) * steps + table = json.load(open(table_path)) + data = table["workloads"][0]["e2e"]["metrics"]["data"] + data["values"] = {"replay": {"steps": steps, "values": values}} + with open(out_path, "w") as f: + json.dump(table, f) + print(out_path) + + +if __name__ == "__main__": + main() From aa454fc73538a826086ef6590b6e553ce8fdcf2b Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 13:03:19 -0400 Subject: [PATCH 14/20] feat(asap-tools): cost on x, p95 latency on y in cost-latency plot Panel subtitles drop the workload set name. Co-Authored-By: Claude Opus 5.5 --- .../multi_experiment/plot_cost_latency.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py index f9e68191..3cb749df 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 """Cost-latency scatter from collate_batch.py's CSV: one panel per workload -set, one point per (workload, system); x = p95 query latency, y = total -CPU-seconds, both log. Lower left is better. A run collate_batch.py marks +set, one point per (workload, system); x = total CPU-seconds, y = p95 query +latency, both log. Lower left is better. A run collate_batch.py marks invalid (failed queries, or answers that disagree with the reference) is drawn hollow: its latency and CPU don't stand for a correct answer. Two arrows from LiteOlly's geometric-mean point show its improvement in CPU -(vertical) and p95 latency (horizontal) over the worst baseline for that +(horizontal) and p95 latency (vertical) over the worst baseline for that metric: collate_batch.py's geometric mean of baseline / LiteOlly over workloads where both are valid. Arrow lengths are those ratios. @@ -68,8 +68,8 @@ def arrows(ax, rows, improvement, set_name): names = {key: label for key, label, *_ in SYSTEMS} lite = [r for r in rows if r["arm"] == "asap"] lite = [r for r in lite if r["valid"] == "True"] or lite - lx = statistics.geometric_mean(float(r["latency_p95"]) for r in lite) - ly = statistics.geometric_mean(float(r["cpu_secs"]) for r in lite) + lx = statistics.geometric_mean(float(r["cpu_secs"]) for r in lite) + ly = statistics.geometric_mean(float(r["latency_p95"]) for r in lite) worst = {} for metric in ("cpu_secs", "latency_p95"): worst[metric] = max( @@ -77,24 +77,24 @@ def arrows(ax, rows, improvement, set_name): key=lambda am: am[1], ) style = dict(arrowstyle="<->", color=INK, linewidth=2, mutation_scale=22) - arm, cpu = worst["cpu_secs"] + arm, lat = worst["latency_p95"] x = lx / 1.5 - ax.annotate("", (x, ly), xytext=(x, ly * cpu), arrowprops=style, zorder=4) + ax.annotate("", (x, ly), xytext=(x, ly * lat), arrowprops=style, zorder=4) ax.text( x * 1.08, - ly * cpu**0.75, - f"{cpu:.1f}x less CPU\nvs {names[arm]}", + ly * lat**0.75, + f"{lat:.1f}x lower p95\nlatency vs\n{names[arm]}", color=INK, fontsize=13, va="center", ) - arm, lat = worst["latency_p95"] + arm, cpu = worst["cpu_secs"] y = ly / 1.4 - ax.annotate("", (lx, y), xytext=(lx * lat, y), arrowprops=style, zorder=4) + ax.annotate("", (lx, y), xytext=(lx * cpu, y), arrowprops=style, zorder=4) ax.text( - lx * lat**0.5, + lx * cpu**0.5, y / 1.08, - f"{lat:.1f}x lower p95 latency vs {names[arm]}", + f"{cpu:.1f}x less CPU vs {names[arm]}", color=INK, fontsize=13, ha="center", @@ -127,8 +127,8 @@ def main(csv_path, out_dir): workloads = sorted({r["workload"] for r in by_set[set_name]}) for key, label, color, marker in SYSTEMS: pts = [r for r in by_set[set_name] if r["arm"] == key] - xs = [float(r["latency_p95"]) for r in pts] - ys = [float(r["cpu_secs"]) for r in pts] + xs = [float(r["cpu_secs"]) for r in pts] + ys = [float(r["latency_p95"]) for r in pts] valid = [r["valid"] == "True" for r in pts] ax.scatter( [x for x, v in zip(xs, valid) if v], @@ -157,14 +157,14 @@ def main(csv_path, out_dir): oval(ax, *zip(*fit), color) ax.set_xscale("log") ax.set_yscale("log") - ax.set_xlabel("p95 query latency (s)", color=MUTED, fontsize=14) - ax.set_ylabel("total compute cost", color=MUTED, fontsize=14) + ax.set_xlabel("Total Compute Cost", color=MUTED, fontsize=14) + ax.set_ylabel("p95 Query Latency", color=MUTED, fontsize=14) for axis in (ax.xaxis, ax.yaxis): axis.set_major_locator(ticker.LogLocator(subs=(1, 2, 5))) axis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f"{v:g}")) axis.set_minor_formatter(ticker.NullFormatter()) title = ( - f"{set_name}: {len(workloads)} workload(s), lower left is better" + f"{len(workloads)} workload(s), lower left is better" "\nhollow: invalid run (failed queries or wrong answers)" ) ax.set_title(title, color=INK, fontsize=14, loc="left") From 8a9e9b01f2cf076b82939e89ee79da082431e286 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 13:05:55 -0400 Subject: [PATCH 15/20] feat(asap-tools): drop per-panel subtitle from cost-latency plot Co-Authored-By: Claude Opus 5.5 --- .../post_experiment/multi_experiment/plot_cost_latency.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py index 3cb749df..fb96f562 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -124,7 +124,6 @@ def main(csv_path, out_dir): ) for ax, set_name in zip(axes[0], sets): ax.set_facecolor(SURFACE) - workloads = sorted({r["workload"] for r in by_set[set_name]}) for key, label, color, marker in SYSTEMS: pts = [r for r in by_set[set_name] if r["arm"] == key] xs = [float(r["cpu_secs"]) for r in pts] @@ -163,11 +162,6 @@ def main(csv_path, out_dir): axis.set_major_locator(ticker.LogLocator(subs=(1, 2, 5))) axis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f"{v:g}")) axis.set_minor_formatter(ticker.NullFormatter()) - title = ( - f"{len(workloads)} workload(s), lower left is better" - "\nhollow: invalid run (failed queries or wrong answers)" - ) - ax.set_title(title, color=INK, fontsize=14, loc="left") ax.grid(True, which="major", color="#e6e5e1", linewidth=0.8) ax.tick_params(colors=MUTED, labelsize=13, width=2) for spine in ax.spines.values(): From 458b8c525097cbc3b7568596d2cd1c79712237b3 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 13:07:51 -0400 Subject: [PATCH 16/20] feat(asap-tools): cost back on y, units on cost-latency axis labels Co-Authored-By: Claude Opus 5.5 --- .../multi_experiment/plot_cost_latency.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py index fb96f562..edf8ae7a 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 """Cost-latency scatter from collate_batch.py's CSV: one panel per workload -set, one point per (workload, system); x = total CPU-seconds, y = p95 query -latency, both log. Lower left is better. A run collate_batch.py marks +set, one point per (workload, system); x = p95 query latency, y = total +CPU-seconds, both log. Lower left is better. A run collate_batch.py marks invalid (failed queries, or answers that disagree with the reference) is drawn hollow: its latency and CPU don't stand for a correct answer. Two arrows from LiteOlly's geometric-mean point show its improvement in CPU -(horizontal) and p95 latency (vertical) over the worst baseline for that +(vertical) and p95 latency (horizontal) over the worst baseline for that metric: collate_batch.py's geometric mean of baseline / LiteOlly over workloads where both are valid. Arrow lengths are those ratios. @@ -68,8 +68,8 @@ def arrows(ax, rows, improvement, set_name): names = {key: label for key, label, *_ in SYSTEMS} lite = [r for r in rows if r["arm"] == "asap"] lite = [r for r in lite if r["valid"] == "True"] or lite - lx = statistics.geometric_mean(float(r["cpu_secs"]) for r in lite) - ly = statistics.geometric_mean(float(r["latency_p95"]) for r in lite) + lx = statistics.geometric_mean(float(r["latency_p95"]) for r in lite) + ly = statistics.geometric_mean(float(r["cpu_secs"]) for r in lite) worst = {} for metric in ("cpu_secs", "latency_p95"): worst[metric] = max( @@ -77,24 +77,24 @@ def arrows(ax, rows, improvement, set_name): key=lambda am: am[1], ) style = dict(arrowstyle="<->", color=INK, linewidth=2, mutation_scale=22) - arm, lat = worst["latency_p95"] + arm, cpu = worst["cpu_secs"] x = lx / 1.5 - ax.annotate("", (x, ly), xytext=(x, ly * lat), arrowprops=style, zorder=4) + ax.annotate("", (x, ly), xytext=(x, ly * cpu), arrowprops=style, zorder=4) ax.text( x * 1.08, - ly * lat**0.75, - f"{lat:.1f}x lower p95\nlatency vs\n{names[arm]}", + ly * cpu**0.75, + f"{cpu:.1f}x less CPU\nvs {names[arm]}", color=INK, fontsize=13, va="center", ) - arm, cpu = worst["cpu_secs"] + arm, lat = worst["latency_p95"] y = ly / 1.4 - ax.annotate("", (lx, y), xytext=(lx * cpu, y), arrowprops=style, zorder=4) + ax.annotate("", (lx, y), xytext=(lx * lat, y), arrowprops=style, zorder=4) ax.text( - lx * cpu**0.5, + lx * lat**0.5, y / 1.08, - f"{cpu:.1f}x less CPU vs {names[arm]}", + f"{lat:.1f}x lower p95 latency vs {names[arm]}", color=INK, fontsize=13, ha="center", @@ -126,8 +126,8 @@ def main(csv_path, out_dir): ax.set_facecolor(SURFACE) for key, label, color, marker in SYSTEMS: pts = [r for r in by_set[set_name] if r["arm"] == key] - xs = [float(r["cpu_secs"]) for r in pts] - ys = [float(r["latency_p95"]) for r in pts] + xs = [float(r["latency_p95"]) for r in pts] + ys = [float(r["cpu_secs"]) for r in pts] valid = [r["valid"] == "True" for r in pts] ax.scatter( [x for x, v in zip(xs, valid) if v], @@ -156,8 +156,8 @@ def main(csv_path, out_dir): oval(ax, *zip(*fit), color) ax.set_xscale("log") ax.set_yscale("log") - ax.set_xlabel("Total Compute Cost", color=MUTED, fontsize=14) - ax.set_ylabel("p95 Query Latency", color=MUTED, fontsize=14) + ax.set_xlabel("p95 Query Latency (s)", color=MUTED, fontsize=14) + ax.set_ylabel("Total Compute Cost (CPU-sec)", color=MUTED, fontsize=14) for axis in (ax.xaxis, ax.yaxis): axis.set_major_locator(ticker.LogLocator(subs=(1, 2, 5))) axis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f"{v:g}")) From f0e58efa7defdf57e71693c68b272e2625faab80 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 13:14:46 -0400 Subject: [PATCH 17/20] feat(asap-tools): cost-latency arrows labelled faster/cheaper vs Prometheus and ClickHouse Co-Authored-By: Claude Opus 5.5 --- .../multi_experiment/plot_cost_latency.py | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py index edf8ae7a..c3cc63f6 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -8,7 +8,8 @@ Two arrows from LiteOlly's geometric-mean point show its improvement in CPU (vertical) and p95 latency (horizontal) over the worst baseline for that metric: collate_batch.py's geometric mean of baseline / LiteOlly over -workloads where both are valid. Arrow lengths are those ratios. +workloads where both are valid. Arrow lengths are those ratios; labels give +the ratios against Prometheus and ClickHouse. An oval encloses each baseline's valid runs (all its runs if none is valid). @@ -64,9 +65,11 @@ def oval(ax, xs, ys, color): def arrows(ax, rows, improvement, set_name): """LiteOlly's improvement over the worst baseline per metric, drawn from - LiteOlly's geometric-mean point (over its valid runs, else all).""" + LiteOlly's geometric-mean point (over its valid runs, else all) and + labelled with its ratios against Prometheus and ClickHouse.""" names = {key: label for key, label, *_ in SYSTEMS} lite = [r for r in rows if r["arm"] == "asap"] + leftmost = min(float(r["latency_p95"]) for r in lite) lite = [r for r in lite if r["valid"] == "True"] or lite lx = statistics.geometric_mean(float(r["latency_p95"]) for r in lite) ly = statistics.geometric_mean(float(r["cpu_secs"]) for r in lite) @@ -76,17 +79,28 @@ def arrows(ax, rows, improvement, set_name): ((arm, m[metric]) for (s, arm), m in improvement.items() if s == set_name), key=lambda am: am[1], ) + + def vs(metric, word): + return "\n".join( + f"{improvement[(set_name, a)][metric]:.1f}x {word} than {names[a]}" + for a in ("prom", "ch") + ) + style = dict(arrowstyle="<->", color=INK, linewidth=2, mutation_scale=22) arm, cpu = worst["cpu_secs"] - x = lx / 1.5 + # Left of every LiteOlly point, so the label beside it clears them. + x = leftmost / 1.3 ax.annotate("", (x, ly), xytext=(x, ly * cpu), arrowprops=style, zorder=4) ax.text( - x * 1.08, - ly * cpu**0.75, - f"{cpu:.1f}x less CPU\nvs {names[arm]}", + x / 1.1, + ly * cpu**0.5, + vs("cpu_secs", "cheaper"), color=INK, fontsize=13, + rotation=90, + ha="right", va="center", + multialignment="center", ) arm, lat = worst["latency_p95"] y = ly / 1.4 @@ -94,16 +108,16 @@ def arrows(ax, rows, improvement, set_name): ax.text( lx * lat**0.5, y / 1.08, - f"{lat:.1f}x lower p95 latency vs {names[arm]}", + vs("latency_p95", "faster"), color=INK, fontsize=13, ha="center", va="top", ) left, right = ax.get_xlim() - ax.set_xlim(min(left, lx / 2), right) + ax.set_xlim(min(left, x / 2), right) bottom, top = ax.get_ylim() - ax.set_ylim(min(bottom, ly / 2), top) + ax.set_ylim(min(bottom, ly / 2.6), top) def main(csv_path, out_dir): From f474ee6229e63f601be76f1277a622bad3607eb3 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 13:17:30 -0400 Subject: [PATCH 18/20] feat(asap-tools): horizontal cost-latency arrow labels, largest ratio first Co-Authored-By: Claude Opus 5.5 --- .../multi_experiment/plot_cost_latency.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py index c3cc63f6..c5b474cc 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -81,26 +81,26 @@ def arrows(ax, rows, improvement, set_name): ) def vs(metric, word): - return "\n".join( - f"{improvement[(set_name, a)][metric]:.1f}x {word} than {names[a]}" - for a in ("prom", "ch") + """e.g. "6.2x cheaper vs. ClickHouse,\n4.9x vs. Prometheus", largest first.""" + (a, ra), (b, rb) = sorted( + ((a, improvement[(set_name, a)][metric]) for a in ("prom", "ch")), + key=lambda ar: -ar[1], ) + return f"{ra:.1f}x {word} vs. {names[a]},\n{rb:.1f}x vs. {names[b]}" style = dict(arrowstyle="<->", color=INK, linewidth=2, mutation_scale=22) arm, cpu = worst["cpu_secs"] - # Left of every LiteOlly point, so the label beside it clears them. + # Left of every LiteOlly point; its label sits above the arrow's top. x = leftmost / 1.3 ax.annotate("", (x, ly), xytext=(x, ly * cpu), arrowprops=style, zorder=4) ax.text( - x / 1.1, - ly * cpu**0.5, + x, + ly * cpu * 1.12, vs("cpu_secs", "cheaper"), color=INK, fontsize=13, - rotation=90, - ha="right", - va="center", - multialignment="center", + ha="center", + va="bottom", ) arm, lat = worst["latency_p95"] y = ly / 1.4 @@ -115,9 +115,9 @@ def vs(metric, word): va="top", ) left, right = ax.get_xlim() - ax.set_xlim(min(left, x / 2), right) + ax.set_xlim(min(left, x / 2.5), right) bottom, top = ax.get_ylim() - ax.set_ylim(min(bottom, ly / 2.6), top) + ax.set_ylim(min(bottom, ly / 2.6), max(top, ly * cpu * 2.2)) def main(csv_path, out_dir): From cff51d9a995ff8d93bbc75baffa846f28e41763a Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 13:18:33 -0400 Subject: [PATCH 19/20] fix(asap-tools): cheaper label starts at its arrow, clear of the y axis Co-Authored-By: Claude Opus 5.5 --- .../post_experiment/multi_experiment/plot_cost_latency.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py index c5b474cc..9a998526 100644 --- a/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py +++ b/asap-tools/experiments/post_experiment/multi_experiment/plot_cost_latency.py @@ -94,12 +94,12 @@ def vs(metric, word): x = leftmost / 1.3 ax.annotate("", (x, ly), xytext=(x, ly * cpu), arrowprops=style, zorder=4) ax.text( - x, + x / 1.15, ly * cpu * 1.12, vs("cpu_secs", "cheaper"), color=INK, fontsize=13, - ha="center", + ha="left", va="bottom", ) arm, lat = worst["latency_p95"] @@ -115,7 +115,7 @@ def vs(metric, word): va="top", ) left, right = ax.get_xlim() - ax.set_xlim(min(left, x / 2.5), right) + ax.set_xlim(min(left, x / 2), right) bottom, top = ax.get_ylim() ax.set_ylim(min(bottom, ly / 2.6), max(top, ly * cpu * 2.2)) From 22b3e263aa9926c527de74f932f8001f9540e1d8 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 13:34:46 -0400 Subject: [PATCH 20/20] feat(asap-tools): x2 random-workload variant (20,000 series) Frozen for seed 862 with saved plans for all ten workloads, for a cardinality point between base and x5. Co-Authored-By: Claude Opus 5.5 --- .../datasets/fake_metric/random_workloads.py | 2 + .../workload-01-pilot/eval_table.json | 574 ++++++++++++++++++ .../workload-01-pilot/inference_config.yaml | 80 +++ .../workload-01-pilot/plans.json.gz | Bin 0 -> 2492 bytes .../workload-01-pilot/streaming_config.yaml | 89 +++ .../seed-862-x2/workload-01.json | 363 +++++++++++ .../workload-02-pilot/eval_table.json | 573 +++++++++++++++++ .../workload-02-pilot/inference_config.yaml | 79 +++ .../workload-02-pilot/plans.json.gz | Bin 0 -> 2485 bytes .../workload-02-pilot/streaming_config.yaml | 89 +++ .../seed-862-x2/workload-02.json | 361 +++++++++++ .../workload-03-pilot/eval_table.json | 520 ++++++++++++++++ .../workload-03-pilot/inference_config.yaml | 80 +++ .../workload-03-pilot/plans.json.gz | Bin 0 -> 2310 bytes .../workload-03-pilot/streaming_config.yaml | 89 +++ .../seed-862-x2/workload-03.json | 361 +++++++++++ .../workload-04-pilot/eval_table.json | 519 ++++++++++++++++ .../workload-04-pilot/inference_config.yaml | 77 +++ .../workload-04-pilot/plans.json.gz | Bin 0 -> 2323 bytes .../workload-04-pilot/streaming_config.yaml | 89 +++ .../seed-862-x2/workload-04.json | 357 +++++++++++ .../workload-05-pilot/eval_table.json | 571 +++++++++++++++++ .../workload-05-pilot/inference_config.yaml | 79 +++ .../workload-05-pilot/plans.json.gz | Bin 0 -> 2560 bytes .../workload-05-pilot/streaming_config.yaml | 89 +++ .../seed-862-x2/workload-05.json | 361 +++++++++++ .../workload-06-pilot/eval_table.json | 523 ++++++++++++++++ .../workload-06-pilot/inference_config.yaml | 79 +++ .../workload-06-pilot/plans.json.gz | Bin 0 -> 2333 bytes .../workload-06-pilot/streaming_config.yaml | 89 +++ .../seed-862-x2/workload-06.json | 365 +++++++++++ .../workload-07-pilot/eval_table.json | 572 +++++++++++++++++ .../workload-07-pilot/inference_config.yaml | 80 +++ .../workload-07-pilot/plans.json.gz | Bin 0 -> 2501 bytes .../workload-07-pilot/streaming_config.yaml | 89 +++ .../seed-862-x2/workload-07.json | 359 +++++++++++ .../workload-08-pilot/eval_table.json | 518 ++++++++++++++++ .../workload-08-pilot/inference_config.yaml | 78 +++ .../workload-08-pilot/plans.json.gz | Bin 0 -> 2304 bytes .../workload-08-pilot/streaming_config.yaml | 89 +++ .../seed-862-x2/workload-08.json | 351 +++++++++++ .../workload-09-pilot/eval_table.json | 525 ++++++++++++++++ .../workload-09-pilot/inference_config.yaml | 81 +++ .../workload-09-pilot/plans.json.gz | Bin 0 -> 2393 bytes .../workload-09-pilot/streaming_config.yaml | 89 +++ .../seed-862-x2/workload-09.json | 365 +++++++++++ .../workload-10-pilot/eval_table.json | 569 +++++++++++++++++ .../workload-10-pilot/inference_config.yaml | 77 +++ .../workload-10-pilot/plans.json.gz | Bin 0 -> 2553 bytes .../workload-10-pilot/streaming_config.yaml | 89 +++ .../seed-862-x2/workload-10.json | 355 +++++++++++ 51 files changed, 10744 insertions(+) create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/eval_table.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/inference_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/plans.json.gz create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/streaming_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/eval_table.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/inference_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/plans.json.gz create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/streaming_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/eval_table.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/inference_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/plans.json.gz create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/streaming_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/eval_table.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/inference_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/plans.json.gz create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/streaming_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/eval_table.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/inference_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/plans.json.gz create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/streaming_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/eval_table.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/inference_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/plans.json.gz create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/streaming_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/eval_table.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/inference_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/plans.json.gz create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/streaming_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/eval_table.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/inference_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/plans.json.gz create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/streaming_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/eval_table.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/inference_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/plans.json.gz create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/streaming_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/eval_table.json create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/inference_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/plans.json.gz create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/streaming_config.yaml create mode 100644 asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10.json diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads.py b/asap-tools/experiments/datasets/fake_metric/random_workloads.py index 24834873..5bb2a4d6 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads.py +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads.py @@ -69,6 +69,8 @@ "base": (1, 50), # Lookbacks x3: 1m -> 3m, ..., 20m -> 60m; quantiles 100s -> 300s. "long": (3, 50), + # x2 series: 20,000 at 10/s, 200,000 samples/s. + "x2": (1, 100), # x5 series: 50,000 at 10/s, 500,000 samples/s. "x5": (1, 250), } diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/eval_table.json new file mode 100644 index 00000000..a83c16b5 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/eval_table.json @@ -0,0 +1,574 @@ +{ + "inputs": { + "workload-01.json": "e77d8bca3a39966f67be06c554d6866be939c444dac5f87be2c677bb74acefd7" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-01", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-01/q01-sum-fine/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 20, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-01/q01-sum-fine/keys/600s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/q02-sum-coarse/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-01/q02-sum-coarse/keys/600s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 1, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-01/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/topk4_q07-topk-sum/keys/1200s", + "k": 4, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 30, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-01/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-01/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 600000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 12000000.0, + "covered_min_N": 12000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 12000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 12000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/inference_config.yaml new file mode 100644 index 00000000..9a005085 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/inference_config.yaml @@ -0,0 +1,80 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[10m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[10m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[10m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[10m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 59 + query: topk by (region, service) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 40 + query: topk by (region, service) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/plans.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..27c551e8cb7ba8ec237fb434659035fc5e2e2f46 GIT binary patch literal 2492 zcmZY7c{~&TAHeZS4@ckfZH})Dx#p~~6)V)_ishV4j*xy2vbhQ)cS$m=*>cRW6w5VN zawJD3=M*`H(Bxaw58t_G5$#v{dHjBlc|8AkJ>H)`KA+DYZ~0_l;RMls%s$?AKhJ<* zElhA=z@C37&bFVOfC;<{X|(w)rQGNlq({m|1hr*`=BRN3_s0O}MwPgaLn#mbmHfn5 z_^xy7z?tGy`}=rG-}oCYyfccOhvF>NXm)IQL~MDvrO>@!YE6H})J>+bzKwhu`oTCJ zX}o60+29~PP#dBn^h&c?*B$XaX!lLO(=KHZyxA=-VKQce%{7df-)anl<)%ODLCl~~ zdA!Y%Fi4h7&y9soP4yT$Z?&+!I+xHJ`Esq0v03Gp(cF)OX2s4jSox)ytkabtQ1Enk zd29?@-gcTjeE&ZED$%zH18x@Uwu!+2P zX{9DdZ7e#-W%yl{p5|;*Ta+%Puw-)NVPU(D=RigTnd_f@wzA1>qoW9L=SaB(-O``( zr~Zq&A&%$R?B+^mxGF`;RRY_uwagiB2axJHLY@6TaH)~+Fm+t76WUxt&x-e?NdFgG zhHPd}KhnKr(;!`|k74uBrh`?bVeX%`qTe@g!;TW2mp0rqPKwThoGGI*l+qy!ML#?k zMv?Am_a0!F0lv<8HUo6Su*zpxdPhhVPCW(H=X{MZ9PS{AU7@GL;7N~a?N3lMD_p1C zI)MQOD=MgHTY zK0CATFHmW2l@xY5&GI8vmQ0?985haeRJGpyUU>&CP{^(}W|6#AyLi5pu2}4_7SUac z*1oiOdZhVZ;Vb*RZz@f=$D~){FG=;`4-mZHxx3G`M|mW9H>hWrM;769OI@&W&XsAmZRC`&M(P87T%x{D%*TE24hnSl z!nk6Mt-v7%JE%#(qBxRoVgsuj+jh{X$aeOL?tmc>%7(Q`C``MWy$!t-M#~!Y?EhqT z6W|^%HT5e|2ed0P7_5keyJjaBhl_m%-%Z?P6}TZBv^;Q=Em`>tUT*=y!DjGwvP}m!5aZR9Fxmvj#Q3dd$ zCS6e;6^6CMjxX7GI~ZR8ub|FJ=L9T^eg*JZx}!%3tCvC<2Lw`vI!uf1PA(oTXqwj& ztQSpo3@3&orM2<)iG)N#h!jBnH#DT>);Av?2)yR?s8w<@E2&4|;^hf$c$e~HoegC@ z*bZ{HNo1&X)|o}XMBD9nIZyLZKt?TH6h_D@agcAE&pcN*>;<~t;*zdcypXfAyrmPR* z_ht0yT%ss$mj-B@pgidoP@ZI!v!fT3wI(WvQ8sPO5hlJ;UKBpFkFXs$@F-M?pHXK; zw0WAvE?sz}jylnFNS#O>+fk#6s11WS$*71iO>z5=-ZuXPxVfFB{>4Nw93K(=P_{w7 z>23j{1+Q_Sf({&LSoOaOPa(>hlc&Zkhr9a@m8-?n-zl>tAN`g4;p)SsI?N3|VM_8rN`NW}#WsOQ-+$#pkU zWN<$rURY~czOepjYgj++@!UsuFLB$`lK2&-ZvmG37wbHScNDkd7f>fN7fG+lY6 z7Vle5KW+h>!s{Aklh;g|5EK_RO*U-7BRLrJ1iEzUvW~!m^o$6v?gex zhFsNZq5bHoQ?lLal~d5?9O}5HcF<^fO6GYD5CVkw1VD)W78A7>G4P6D;A6yOSuhc) z7c5G=e&Fq;bLQ7YFN*P5mR~>Lm7ijlP@N)Fm7ifL ZbbWoCw|CDT9#3G;*K^b_UF^Mk{sV+?qWk~= literal 0 HcmV?d00001 diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/streaming_config.yaml new file mode 100644 index 00000000..c0e3e9f7 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 25000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 20000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: sliding +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01.json new file mode 100644 index 00000000..23c5686a --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-01.json @@ -0,0 +1,363 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-01", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 20) * 20 - 600) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 20) * 20 - 600)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 20) * 20 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 20) * 20 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 20000, + "k": 8, + "lookback_ms": 600000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q10-cardinality", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY region, service", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/eval_table.json new file mode 100644 index 00000000..d344d026 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/eval_table.json @@ -0,0 +1,573 @@ +{ + "inputs": { + "workload-02.json": "8e17c6d061acf0722bc5a7b67e163d17ac64486022c6119d45dfc2011148f32c" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-02", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-02/q01-sum-fine/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 5, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-02/q01-sum-fine/keys/300s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-02/q02-sum-coarse/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 5, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-02/q02-sum-coarse/keys/300s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-02/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-02/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-02/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 1, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-02/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/topk4_q07-topk-sum/keys/1200s", + "k": 4, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 60, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-02/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-02/q01-sum-fine/keys/300s", + "instant": false, + "interval_secs": 5, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-02/q02-sum-coarse/keys/300s", + "instant": false, + "interval_secs": 5, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-02/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 12000000.0, + "covered_min_N": 12000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-02/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 12000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 12000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/inference_config.yaml new file mode 100644 index 00000000..3d590028 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/inference_config.yaml @@ -0,0 +1,79 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service) (sum_over_time(data[5m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[5m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service, job) (sum_over_time(data[5m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 59 + query: topk by (region, service) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 20 + query: topk by (region, service) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/plans.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..4047bab02ea10ad6ff62318ddb9ce8d810a47f1c GIT binary patch literal 2485 zcmZY7c{~&T9|v$sLg6t-VnRrcBG+WDhuBOO-{H&8U2;U`J}OJ!kSirrWQ5F7uCSa# za-_{@?lZB8$=tO$vLDLt;PHHX-mk~!pZDkU$4mSn2M30SvhdLK+!Y6$D3s? z!@+0RN;JBWYtmDRTXSLhR*zzygV`{#B#%q;E4#@1mM9lQ-$FwGJnhIOk(5gYX-1y8 zy@O?IfVq6;`1R%NPzNiaWEJt?d(=RozjbA2d*m~@2~-aUf#&gWi~LM0O~9Ltyq;pk zhu{(#IVGapaW#MHZ7POXx)HHfzG5*=Z)7qi5*oL*%2ooVYxU0ORXv=vb`4q~qdSLg zsmMiTRG>S{6~k!oWr#9xG%)Nt96xj$p#+XV=)SC{TDIbfPPAU21?qls_b6qeW*p<) zi6kB~K9y3!y+UwA8Lw9IH?T&7^3!OL4WN>_ll%dXQ ziv_K3HDi-Z(8LvaAJE#O-;cvdkAJA@LGx2Cz3e4WKh-T2U96Y&p0zk{nDO%OyNGk_ zIoP{PKJkjhedk`rM?)GF**%m%^$kn;S-1u}K6>cj`z@!>dZqX&ZT3{h-1&{=qT3b% z(`5gCQ4fT}BXZ|>rI`)e8Ek*&5hR8l3b#?0A1-B@S~XuHhqJ zg3zpK?>1}KTy*hhgXH3h-m@tLROLytxSm2(dHmVSYZ8QsMg5p%{g_$x{2MSoUb*4v zD~L*+08m$BpUq!C;dwX#TTft-??`yhclV}SEXi(m+2$C|quq^9Zzz@w5Kk--AexOc zPMA@?h1k7 z(8U3;moacFdX7gLJ0Z=+W4`GGqQeHQ8DUbJjPfw#CJ_>-bMLHf1JW@(KYQ85O2*l) z@|DSXtTosl^T>lsVSzODJbIqwx;Q)o$5bT0(<0}NMWr0VX`A9K%Uzw9xXJ*+MH~tr z;xIeo?WEt^Nq@wbdAjjmfjZMz`iGl7NL z1sBAc4Vmew1y#SQsl}gjA}T}3%7DuHu_@)N&ZFyDSvPErWI(Db8w9WM^y*MgEr$%q zNa4!!5YjY1%F+DkI0$;%UO7!jBnMn_lBW7`P{i8nJ5%M@>D(2WE%%`#KADe^3bLL* z{4bJDfo{_;B{M-!QX6h6z|)2LrpgPvhvEE>_=jWa60;ZSZLuR_b%!B^mK|R>m0ZB* zkqw5!KjRTE7^?$fXJom)#X6lxwd~+;T(2Dsj;p8XU|(eEueLb5uj3voBnL-lywxb+QfzPT!27u-8k*&mM0AG$r1bZsAZ)lH4Ab z;+1TCQ368Gc-!O^aPJ&t$KX>W(fyo~zcCmth_#rpHc#}CXC*Vi%|WZ{-sQJ41>f=d zndIj3%WpLWwgkGojiLV5?zQweFUJkbaLsAGlT=zs@VR%cb@o-)QB8Bh>ss9oJ%2Q( zpe}Z?vL^Eev8nrsO+c#w-jTrFK*a&qC}Qhg|4X;#sF#I)k-(5ir~qugw`Ig?z!AO!{K+N4qOdp* zQCO|l9=D{xp^bVaASYkHVK|#8tI>Oa?{4^=^=%2?FA`|SElQF&NMk--KLV2mjldA| z{&15uM%RyrR*)pFT~UMWt=-uka>z&xwqJEw-+KTz-vRzB{leO?S`Tg5tk<5rq~4*> z1}z|fZ_qF@QuMS@JHUT8{51SqBKKnx9eE^*IDL>t){+Nc*}*a6qndc92wI`e691>i zV9bAdjDqdAmE{*%HKu&+qrpDxSs$t48???j3j1GY!@MYB)oF3x6D4DXC7W zrq__Kn^JP-3Q%lDt`DmEkL4D(#<%RVsd&#C`J> z^AV$&-~Y*=0#y0-^Y_vW*}r$^KDwn(E=W}CD@c4gvIj0JSH-)cfb^9*Z8{2D1fbZ& zn~ZiN)mcYyzQr21fDVBcV3@5r8b+1FLP0f#k$rCVZ8aq5OAe3fEw z>;&2Rt5mS`elvr?%Q2^752*RoAhHU>d#=LPk$c>pu8th7eE`(sqqHPMicc6Z0Aaha obdv$l=i4vXODDAD$$qHfD9T6zla+;q$vn&=@spb*L1Sh4H@fH0+5i9m literal 0 HcmV?d00001 diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/streaming_config.yaml new file mode 100644 index 00000000..984fbe74 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 25000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 5000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 5000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: sliding +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 80 + width: 4096 + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02.json new file mode 100644 index 00000000..4c04d6cd --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-02.json @@ -0,0 +1,361 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-02", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 5) * 5 - 300) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 5) * 5 - 300)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 5) * 5 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 5) * 5 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q08-topk-count", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, increase(data_total[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 5000, + "k": 4, + "lookback_ms": 300000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/eval_table.json new file mode 100644 index 00000000..a5910fbd --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/eval_table.json @@ -0,0 +1,520 @@ +{ + "inputs": { + "workload-03.json": "ac913568d60bae42366e422f000fd20d9bdfbe37d32cb8b0a4feca21ab9d175f" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-03", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-03/q01-sum-fine/keys/60s", + "lookback_ms": 60000, + "metric": "data" + } + }, + "interval_secs": 1, + "promql": "sum by (region, service, job) (sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-03/q01-sum-fine/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-03/q02-sum-coarse/keys/60s", + "lookback_ms": 60000, + "metric": "data" + } + }, + "interval_secs": 1, + "promql": "sum by (region, service) (sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-03/q02-sum-coarse/keys/60s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-03/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-03/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-03/q04-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 1, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-03/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/topk4_q07-topk-sum/keys/1200s", + "k": 4, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 30, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-03/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 60000.0, + "covered_min_N": 60000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-03/q01-sum-fine/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 60.0, + "max_N": 60000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 60000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-03/q02-sum-coarse/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-03/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 60.0, + "max_N": 3000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 60000000.0, + "covered_min_N": 60000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-03/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 1200.0, + "max_N": 60000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 60000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/inference_config.yaml new file mode 100644 index 00000000..3e9e32f5 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/inference_config.yaml @@ -0,0 +1,80 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service) (sum_over_time(data[1m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[1m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service, job) (sum_over_time(data[1m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 60 + query: topk by (region) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 40 + query: topk by (region) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service, job) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/plans.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..11d056d56cdd508a710cb0a3ae7b5b095678d3fb GIT binary patch literal 2310 zcmY++2{;ps9|v$6P3doof94hDD3l{z2y^GYDj{+vcex51vaz|(mr{p%n{4m{^j}S`8?0>_dMV4^ZP+#*w}J8@NUNrygck5`%1g{dOl`a&b0O% z)8f@Sh|b3P$^AtRl(*h=z*ATe{a9hJOH_ra{#ye*zc|5zVCVLDE{RIRd_BGxax1bK zloWKqC!R!(e5O)gl)eGCHVSV#6;8?HGpFj_krt!Puy(fhNhZA_P6>yn-11DrY6I%__vnhM$u(U^WVsFVf`aP zHipOZhr$f4=*}{ihKN2vu$`weWvR}0KM2I;&w1=R^_gPp?wKc7Ir7*|1v2upI|L#t9k3f9d}*7`lXg(zxRXnauAP8oT4CO_&yu0dZ5yHK$- zT6#Y{ofVqqFRR$IkwI)ZX)d1YnDKC?cnrWJMHsAx$!f8#ujATTyXr^qIhH*X%FUY3 zAD$0-ESW~(ovznO6&VCxE*W!!`xegy)JKWUAqEE1oiet9cAAPjgAjTJYC`3CVa;&kJHF{OQDNd3;`&8xBhg zNHZ(@9{*gQeXet3B!{<#xbq;hbcUSWSeMQzxA%xbb8THHE|kMsG(Gt8ND{E)KJ35$wAh zDr?4DrheKp%SW5mv$AwH>C(HgCH?nhOM4|t`scs9(rb2RSX|bpA1MQkuj28QM%5LI z$F*g5>|zr~KiYTR+q=O>eYt%i-U0T=Yh6!VGpTlwGWBUCRD_w2p6He1*b(~C-T~{7 z$dLnBR8{z~svv}sV#*j*IqJsBaGhOE%uo|SJzcSVQ?rggOKumy6wfu8;`M|a8byuXrQh|yA!i#Y z!;%W5X+LYVgrU`kC&Ws(VBO*&#aGfGmu9c>bUgW2My zUbmNxwy;yHD~(8Z;Kc?`n99+Y!saW|#aA$9>Y{J7+0r#m83-_Zm0~D&DH9Xi>2*05 z_h%Qknw9%_@A0}tpJNE|4W(8c?`wn>eKCzSOyV}?wOx$7%NbE_lf}JA$BgTuF4MD0 zF}N_Ch6o!8X)Q%BTaEF&ky8<~(gJSWu<)UeK1 zyUsodT{qhN^u=mnt%x~Hs+r>~pbA(O5zgTRVvx$zrV1G9lV3??`q_|*%}Mc}W}NzF z;4n=K3*{Ue8O}mTA*9J+6-ytFO9=1-Wuqwonkvi$GytISDbv;@{3~UCN{o<*h`f| zGCwZnjiCy^ML#Rx)@SGd^qABD^ok6xx}zP#HFO=Aa|Kj@-UDnm!@NM_fbC6GU}DH2 zq`(*Gmn@r2M0v>q}SgKL@6uZ!>zG@4%H2FV8V@?#B)}_5rAxsf+QFT!@0r7$v#y1L1=1jk) zZwCo*q6E%C=W^w|CFRr=-zdp~kLUyK*fy7=BHx#zblP10lN8B-|6?gycUX0bQx?1o z*lSiU2QBwQwpxz2{t_Bkjw(kX`ZxX`{j3z)4o-!0Xf|RvG|y>yM(V^e44OC_wE{y^ z*HB346mW<688VC13{nRlnhFD>!2?;PAT96_eay;GHjzj)yGW!->^%4l4@1M7l??Q0 zmP__5%qq>GHNZZTN&^ZSD|o0GD8eREpkn=7^uH=wBS;IZ2R;qa0~hj42;IKSkfwoS zI-enkJR!#R4fQM=x&ZvP2%-zJ3jLy{tfQu*23y**W3@k`K}%wR8gT$%VmkOSiVS_@ JqRW|?{sq&pZe9QY literal 0 HcmV?d00001 diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/streaming_config.yaml new file mode 100644 index 00000000..d1b13eb9 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 240 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03.json new file mode 100644 index 00000000..1573d1f5 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-03.json @@ -0,0 +1,361 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-03", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 1000, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 1000, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 1) * 1 - 60) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 1) * 1 - 60)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 1) * 1 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 1) * 1 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 1000, + "k": 8, + "lookback_ms": 60000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 1000, + "lookback_ms": 60000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -60, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/eval_table.json new file mode 100644 index 00000000..40ae391e --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/eval_table.json @@ -0,0 +1,519 @@ +{ + "inputs": { + "workload-04.json": "11e77e8ab7e51caf1cb2a9af62f6589fd6a8cc8908cb2bc77e122d4a33cb93e1" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-04", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-04/q01-sum-fine/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 10, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-04/q01-sum-fine/keys/600s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-04/q02-sum-coarse/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 10, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-04/q02-sum-coarse/keys/600s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-04/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-04/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-04/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 2, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-04/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/topk4_q07-topk-sum/keys/1200s", + "k": 4, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 30, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-04/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-04/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 600000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-04/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-04/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 60000000.0, + "covered_min_N": 60000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-04/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 1200.0, + "max_N": 60000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 60000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/inference_config.yaml new file mode 100644 index 00000000..ec06c999 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/inference_config.yaml @@ -0,0 +1,77 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service) (sum_over_time(data[10m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[10m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service, job) (sum_over_time(data[10m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[10m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 40 + query: topk by (region) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 30 + query: topk by (region, service) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-04-pilot/plans.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..dfa83e4b7a20833a6f4fcc1b5b0707a806c12ec9 GIT binary patch literal 2323 zcmY+?c{~%09|!O`vPnOTa`cOBj%S{Va^x5>WJSyIAQI(_A(G^AUlYPp%EW^TW3eGI z7R`OEM3XZylE<-yT$yvf*71AtdjI$Nyk4Kz_m3|m83Za6?(yd3T=RAD3smtA^!EdN z%Drguap2%Bj#)Z!be?A*M7f4(($6xZ&h`%|sU;=g6#|}$D;?~L8uQUPewQyLFYAhc zf#jJcpl_7^$sGoK%8gS*QXzQgGdX>%W5jU#K2KYH2(GfSRteR>Pw1Y9>wt?mWrO8q zw-M%Z6CExWjsYvL+n{?tX~lhT?cc0eV?Aovo~|+2*~(iRV{|#6e?2eIX5kUE*;2yQVS4oXw$z zoY&c8nZJ_5!ElQL&$jE1n#RkB%XpYp+obCYQnsKZQ3J2XAxf#h2TczBW{z7K2=k!Q z9H(CRYr@UheU#*vA?n>aFw!DSD9aj^pw0B-6R51xxN4DVl||uH1jN3$@X6WcXwg4& zq)aWM#~;6U+9Wl3t5S}XpnX(TSg!SqfpBfdw%qHwPdaT+pnVt|Dvf1JE;Ab*YTDN5 zVkTWFca*d}`J^KRHls);IaZtLzKtVeMGK_rIF8^BOMhQ)rcS#b+@RMGv)1u*_14YBZ|gbGs6cM z)ua#g>i%`un+37k;k8oJT_rCE`upH^#PIqYbaEvty!Q6s+2__>Jzg~_y0$h>W%(!l z4tTt;WK@@CrxSj&zO+}F{Gaob2s<-sh(O1I%(6Fd+`PbSWJaWQ{__(f8iB2MINH9J|u` zo61CgbEl$ITs!e-E0Hi_9bVbMMrjb}5sRaFw-;(|LxNcb$_)(aexJqJh)SV)vTR4~Jo7T+L~?La zX?_#psW6F;)rff74YT(oA6>}UG@%C&-YOMDfCi+C3%65rHrq+C(+^9a;i5)BOlh?@ z@OoeIU(PEt)4$UKI8`41tyHlLq>HP`n#MT*}Q_vDnM7hD`f`$~2}%DjLH zIfD=8Y}Y({mdt7v^g`>g9fgP17c#6OFM3p+R&itLmSVQ?a@h3lY<9{-Y|Ukr_m|o+ zvat=~tpVkF(Uo-@heW%rSL5FfyPLaju=_c|Qt<8m^XZjJB}7}R(Q3?;7f#V;u|PqNNTk@Kqa^P6}{adI#XW5IvqaK z+7`Og&OQASli@UCvVF4?J&7_;zLo_>C9_xNR*L#Rrh;IV3|->f$G%tXLJ3xwOGMx0 zXl!5HSH^Ucul7jG)C%H*d=U3grXI#crRSU|Uv5;9`D)=`^x{k%1by%l&8@@U4Kbz| z>>6TuF|3e_xr}y$CpSJcEsU>v6@Rt2LwlV*)brn`A9I2Y1z8Yw2Mht_Y+5AA1VOO!9c-ZXE(c=_%pHYPT*L&Jh2W^Q6zeIk&GeGm$8J|=8j zmO>C3X?ZY~)SCEBw5*(J{T3tjipau=Qz58d>~!O%uTJ!Uka!0WlHWxm9@NY=BS@B0 z;D=fY@_@eY0q19V*PsB{4?HJ9NDy+=>&w5<_ll~@qx4W#3fbx)g;a345r3wc%WE&$ zNMS$JUT_cSGfy}G(g^%K%g-JMeE7leq&89;DT7&q^1ybXQfmM(9*(WBk}Xr=G>{IVFXv{7Q3B)CS7MtICBiPMA_Cy%?N; zxBBDVU5tv>Lo%T;`&Ra;VH8Pe4j^$BXqz}jd01d~9W>6)Fx0Il zmLBU2se*FN5aJKIKYrAr74=Xo?%^cF;~c42Y=HZ@Lx#JJO>y?N2+OgWh2>-mj8EyI zxZ-B^;IT6NsxH77gC9r8G($c?f0_a6f^?s_b^Q{3s~*>EShUjPKy~yuT89@n?G9HD zEL8CD2Ve}EZTMMai;vT2@Y&)48)yjnY0FU5NR;WY2{Z=nGLu?w05VRb2pK10Q)Iz{ z(p<{= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 10000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 10000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 10000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 10000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 2000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 10) * 10 - 600) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 10) * 10 - 600)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 10) * 10 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 10) * 10 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 10000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, increase(data_total[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 10000, + "k": 4, + "lookback_ms": 600000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region", + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 32, + "promql": "topk by (region) (8, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 10) * 10 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 10000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 10000, + "lookback_ms": 600000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/eval_table.json new file mode 100644 index 00000000..1a032558 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/eval_table.json @@ -0,0 +1,571 @@ +{ + "inputs": { + "workload-05.json": "6141edf0c371ca529b4867f2383a0c5e0cf6ee4a744ab3e678b8731ddd3f6942" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-05", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-05/q01-sum-fine/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 10, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-05/q01-sum-fine/keys/300s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-05/q02-sum-coarse/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 10, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-05/q02-sum-coarse/keys/300s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-05/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + "random/seed=862/workload-05/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-05/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/topk8_q06-topk-sum/keys/60s", + "k": 8, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 1, + "promql": "topk by (region, service) (8, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-05/topk8_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/topk4_q07-topk-sum/keys/1200s", + "k": 4, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 60, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-05/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-05/q01-sum-fine/keys/300s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-05/q02-sum-coarse/keys/300s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-05/topk8_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 60000000.0, + "covered_min_N": 60000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-05/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 1200.0, + "max_N": 60000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 60000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/inference_config.yaml new file mode 100644 index 00000000..fd47a6c3 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/inference_config.yaml @@ -0,0 +1,79 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[5m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[5m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[5m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 20 + query: topk by (region) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 59 + query: topk by (region, service) (8, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.50, data[100s])) + planned_subquery: quantile_over_time(0.5, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-05-pilot/plans.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..dcc186b61871b7a7308a8d84a03c8f046a2c7f72 GIT binary patch literal 2560 zcmZwFc{~&T9|v%iZ;o#z^L1r8*YvgIKBAvWv8^0ma)cp64BMJkbIegxmcm@QMoq{S zU-y{Qm${ae`^Zr!HOF#ghx&V6Bo+QMBM@m1VO^v|(AWsZtvf83r`U zy9)!^mi5_e>f?!Fqo=^+Wq)-Dl%cH}sv~vOuD)SqbGd~Rw!U#9>`T`=l;figxulmc zURYMU`R+UiuSt=}9{qR~Yb(i{F<=ByedT zAaa`6iuq>2sYiNQeOj1cj-US!ed>m7yKqj%6~$=mu;1pz0l-euONlr*4>fKSGc(qwzy6&%Q&HCiK(`wM-X z6P9cRsu#6ol+!}|9c-2#$WGZhBA=#NO!U-MwxCeu%I5h3sIlh3?)LN?PKZqBOHIhl z1#73N2o~Zzn|%axb?u8)8a$675tyVGP28w5GVZF{e4}qb+rLob(h<{O_MXueh;z>J zR$wm`vlo%afO+qtd5o?cT3TCo+heI7M(JQ*1*5SJ8frmt^QgX>uPrhdcw%c`oeU58T;e(#o_ReYXy(2;bLNsEH$aL=Po+u zn(j0GaYp*FJfVZkbP2(T-`~b`7vj+f*W)9N!XjSFILGOrr)q)qh)L%Byr;*#GHXhE z*eiD76GHHy{V7;UL=0DHd;?Z(<63Dy7WE!jn;$x0M_jx+o_>MId%(3OIWsx3W3Mu} zpJ{`?Sth)jr;3U*|uQhwJ;L?zStDgv^1D{z3#63?WjDj7t z$;$PZn9cPms4LdRF3r=)sBRxLSi`YvwM@P1cX0C&Wz>%d zt@%>#^}@kX`20KD;LceGe0@hnhUtl;Z>Oo$FTTvfV|LOVhN{^?PM#Gdh-MZYS;zB{ zIP05mjEur&J#mB__E`?q@2vmC?Lc_5%UfLa^SQT+Hed5Imhvfo;jC;V3#qkpUG~~8 zq}`#IpxZKNMsoGjY-ADLzTL~Z#rKxF2qCU2#N7sr;SK7M?%a4Xe&w8mx60hx(cvWA zkVW=woeFGzA@_c=Rdltj>7oTQrM_5ms9)86?jHO-^O0c~^k)aRV_<3t1(pzgA;J_rm#e#gx4`ucq#uW1?f^KF}LMeQT3vqL=xB zVY{br5S{io2j3#QA;~;93=ymm`oF*$5s{T0z`DwVf#LeQ zP*SkQlLvNpsmI;AuD$1xoBZBgC|P87GYLffI=$c@a*x3a>Q~-BY9{gP;}n0Ee|uy2 zx-3p{yB#eUG&BSDi`L|a+Ay+54~*P2^b=m0R4Jm4lz{-oo9^(-J%@f0C2j>yfOsZ) zyGWp+uI#+xubggmzcGooo-hIUiT%)>2^uhZ_83ivMTw~azL#!ww;cQ=ss`B3e`X+{ zDf(A_GL{iiaIM#{Z~ZUx~jv-vh^CJ5ks~WJEAeypR%54d~-i?xl&6v zQ(kR!*9o_*1?d};A~oH&1Z{Mg^!|Tz&2tRlU8w=sn8Rvwl&A;7mpR zc>Q>fk&~ePaPjLfc|a5~Z$FI@Kg^1456%MZJ5ye+aG0cjI)o;!;MrQFV+TPO+sL{CwpqyAka^9G zpX}qtpBecHuR^L7;VNT49eQg)gnzw+UvcXoenrAf7!kf5-$J1Cf2@gZXb~_p1yp02 z0-6T@kZTL?3u!$O8KEH*fHX5E{{{L($Yd*MS|(Jp?Tq7I&YT`M`xQnS6QKpi-_wE* zng3K0U$Gw7Sce1eu*`rT2Hc5Md-B*qNjgHNH^>)3Qd==8s}4Z_B9e$v24J^am= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.50)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 8, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 10) * 10 - 300) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 10) * 10 - 300)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 10) * 10 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 10) * 10 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 10000, + "k": 8, + "lookback_ms": 300000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region", + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 32, + "promql": "topk by (region) (8, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/eval_table.json new file mode 100644 index 00000000..eb948fe1 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/eval_table.json @@ -0,0 +1,523 @@ +{ + "inputs": { + "workload-06.json": "bdf7ab7c3dd9d8313a2c252a2ef6daca6f482993b489d319c011e3cd0ffed374" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-06", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-06/q01-sum-fine/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 5, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-06/q01-sum-fine/keys/300s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-06/q02-sum-coarse/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 5, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-06/q02-sum-coarse/keys/300s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + "random/seed=862/workload-06/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + "random/seed=862/workload-06/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-06/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 2, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-06/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/topk8_q07-topk-sum/keys/1200s", + "k": 8, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 60, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-06/topk8_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-06/q01-sum-fine/keys/300s", + "instant": false, + "interval_secs": 5, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-06/q02-sum-coarse/keys/300s", + "instant": false, + "interval_secs": 5, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-06/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 12000000.0, + "covered_min_N": 12000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-06/topk8_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 12000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 12000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/inference_config.yaml new file mode 100644 index 00000000..d31f2ec0 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/inference_config.yaml @@ -0,0 +1,79 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service) (sum_over_time(data[5m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[5m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service, job) (sum_over_time(data[5m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 30 + query: topk by (region, service) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 20 + query: topk by (region, service) (8, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service, job) (quantile_over_time(0.50, data[100s])) + planned_subquery: quantile_over_time(0.5, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.50, data[100s])) + planned_subquery: quantile_over_time(0.5, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/plans.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..eb814cbddb07db0cc392d4ec2dd09523c16bf8d3 GIT binary patch literal 2333 zcmY+?cTm#_7Qk^JNMKp>Ly@*Z5-fB=Iiv}p2@ymvB0^Z{9hOAtY$(BiI237$^d=gX zA|+Trl;R;hk);V)dI-H70fJ!2>CNfLeExVdZ{~gfdaxvZeiCHBkB4*n4_9mu!Y>Gi zoNS?@L&d`cVWu*O(tSu7uh@9CZXJ$ZhvBKWexn9C4 z8LdGiI*dTJg6&6o+o~i*2ECY-ZMW-|*_EuRsp!u$kDX%<>R20-!!9@b-XSu)Jww=Y z36o16y2@s62UjL55Ql?#j9xcn^E&&VyeuZU)M@wLT%D_K4+8RespqgQ+5sY0t{joh<*+ZX{@-{pU&dg(RV5wNPp+A zPJ0#XZr;G#B9bbgLvzi3FSBc&*Yc)<$emST8m-^^F}C$}fhF7Mx4BYLxqV5Q%-JSnoyBlJ zW_Y0^P#gPFI#=+)bv+Bc-3{7}eI&(3yJbA)ff>tqh-Iv?Ot9I; z<(01MGOAZ}qj&0hEr)viRx<5eX^wwmA2{m}eBV_2Eu?uN{G-(?3PhieE7OUSDliRxLIKjZu$72)|nNioPEI>_MA?yA8c z_3{*7K|}xDE0hlbxZlb8QKWIT;LPZ7Iret)c4irDqfBLVXM-^Xr7@22FZXut{-&35 z3B|0BO|jL|+-lb|Gwyf0X3>EgSUkKkm6o0=BRGI>#m&2dXG!voLn*q`&4q0uPCGsR@PW+;E!*gcmsizH)fo|}_w3n3nCeW-s zlPzQwK0#;yD?@9fNvXVV9l6I6(CWvn#c3s#Hra~|hdkzaa3KeW^v20g{}HjohpV!X zmXAH1XPT$ZndPD6&8CzRn9Z#XB?M-etY;>=ZOkZH6Wtzhf@W`_jk9YPNIywg*R>7X zW7s$qnBQYbpy2v40dAW)kQ}Q4thk=}!L&(Mq=l>0c0iFrm;48%ZgGB1`II)|S0?^R zBcj#HuYD4hNR%52@6V8t3OS^^kN+AAStz}@1S@P~^20H6bWu2O*$XoAN**-zk$|Ft zj*8Ty3HsT!Y5_F`VM}~M`&`>x$*;VtTo+p*wV>7JlujmmNnC=B6zF{I^FmKg6F7nz z=FZf+-q+orIf<9b40FnA z8t3!>%&mFViz~%Y?ILx6YQ-UFljG)z2}G0Qm*M?b1?t4+eU;_d453IvXd0N#9bs6g zkAfan6hOqKRCQf~Tc@RTUB1{JOg;|Z%P?;6`Uzvh*8g3SuIN~Trf5A-pkiSZ{BALF z;B)E8Z4IdOfuVU95)O2Mwg`G~YT;R+{)cAbDsh#}IuG@L9Z?jO9_8WTa`3UUeYa%C zRQdjhShb3S^TWG^jI+ToRe=dGBR0YXnhj=*g)!P#-*Ja;Mr}szwsa9Uk0PU{)`^=Y zvVt=S46zP)uCqpqPc&Na&)ig@=+<*_u&<&2^}MHYz%xHg8|@kGjd9KZ?MG3OQSX5d zK($gbtjYNXaWQ&;U3y@uRt|!fd}%e8GcW``g0{Kz4XKkR0Of~&6PZM25SJ&k40c43 zenxcR&1mHOa86uA%j>f^uNetpA~m=xI5G77f%!NXR!_J2W#uF80AmDE_A;L~_*AAb z34Eih>fqAQD#yR3${uQzl>PYyRrn{&V*o*8YHc($_Pl@oKajY44WT36zJu&K^$64R z3v#fvH5o-;+|~Q&9GCGoBJ039@K&`IyvZHO1#4Y^fr`L=MW}4q!SV2o5X;F3?(C}^ z*^1iui0!)We1V?cEDBic_#toz^btAVfOhm|Iojn5B0c)Rm;n%&ntn0X+V}#(K^aH| z#T~ItIUE#&2t_GN!W~?1#QVe#{PIM^3|%gKc806XLfeo4OoR5gkOZm`0e}d7H$?mz z35L%@CE-UDvkw-WFB@vSOd~ADqeV-_%v6${#BEEkX>rGRm0Y=vkZAxN^w|)ZV1)EB z{LTgVP3X6eC8r`c2}kX&zxlfH37Fv?IQ|52Nh=vEKB3xUEF~=f@Y}b8?nc64UErbUDreAD%lGC~b;0=)YK zWHGP^+Lz?tFXk!vzc&4H!*WBg%~sp7_9ONWZxyFH9|Bxl91e)f(wz@4Udzq(U(W%B AL;wH) literal 0 HcmV?d00001 diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/streaming_config.yaml new file mode 100644 index 00000000..38466937 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 5000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 5000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 2000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06.json new file mode 100644 index 00000000..417d8b6c --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-06.json @@ -0,0 +1,365 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-06", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.50)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.50)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 2000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 5) * 5 - 300) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 5) * 5 - 300)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 5) * 5 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 5) * 5 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 5000, + "k": 8, + "lookback_ms": 300000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region", + "service" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/eval_table.json new file mode 100644 index 00000000..63641254 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/eval_table.json @@ -0,0 +1,572 @@ +{ + "inputs": { + "workload-07.json": "1046b2b6cdc6b38b5b54f5b291d5763b913a878db2dc2755bc289dc0114060cc" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-07", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-07/q01-sum-fine/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 20, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-07/q01-sum-fine/keys/600s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-07/q02-sum-coarse/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-07/q02-sum-coarse/keys/600s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-07/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-07/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + "random/seed=862/workload-07/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 2, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-07/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/topk8_q07-topk-sum/keys/1200s", + "k": 8, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 60, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-07/topk8_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-07/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 600000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-07/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-07/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 60.0, + "max_N": 3000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 12000000.0, + "covered_min_N": 12000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-07/topk8_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 12000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 12000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/inference_config.yaml new file mode 100644 index 00000000..7832e815 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/inference_config.yaml @@ -0,0 +1,80 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[10m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[10m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[10m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[10m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 30 + query: topk by (region) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 20 + query: topk by (region, service) (8, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.50, data[100s])) + planned_subquery: quantile_over_time(0.5, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-07-pilot/plans.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..5f5c73c2b1790db94ca6c459fdb7b1f79f385315 GIT binary patch literal 2501 zcmZY7dpHwpAHZ=WS@k9)q7#-mJPtXQ2M@ihjnEjPv8cU|Im}_=3A1uck=ZQCVVL6z z6>`XU$|;0q4Nd4Uhb6?E!gwFb^}ZL^=a27o-+%n>`;Xs=yP~2+2N+&L>;69O7_5dD z)(<1_Ez`<=;F4_Y`klM1xeGV34RdhKf>2|JaxM$re#&R}2bm6t{(f>ho`N;^{GRyw zN)*XxAML0sVYSULbe7(Inv#Thv>b?#YNR5=&X?E9q;#TPFT}D~X#4w(!;$udV(CZs z;pcKf;K)>KHoGu2ik3ECMsq*g^^Bmx5juei6lEArSqK} zaEl#rL+?F}G~(#xFNW$=*5AuXmj`NBPw;!?oJhQ9Y4ak|&7QM4PN_^<7P&F}Ew}ms zhq|`NV~lwAWKDj|{jC7at6QdJxWXrUa#ad$wE3PjY89FVpRcydzyYW$oHH;zKkS7t{o z#Ek?Y23^Udg}m9U>*R^I6x$V~l`0R4fLouf!521P(XvhDln%prp@i<0@nIC>&BV$> zP8A#9Utny^a3iHZ!z_1Nz{ZS~cr$0zaYmsohnbPE%U>hvpQCx<--D-PLKJO0DzxeL z#U(tm^L_VgO>F`2*s4$IO(hvB2=T~V~6^pMhu8JKVhnJ7;W z8r(Z=*y}%-S%o-Pe(@#M!-%fTwkKsiDo!dJYrdv0n_6hl{8tv(s>ZJKlYf3fVC9_! zP-$+?o!^Ucanq_`>AFcrLsm^NNL9o;#h?51OU%2qfvfI@w*posqT5lO2*Nt+p6m7N z15?RvT{JFgUAwRTYCP{i0bcz>Km@t9HBK<7xIOC8=$RAq5ie{lbkYx(-~E!_Be5c@>>*y!W4+@&WNSPiiN$5q(faQM=UB@!j7>G>Q>W8 zd(0Bhi*yAl_n&tV6@x6FvA>sQY4mXwb;CcBWEvrpPR>zUfLMpY_q zQ5z$e_ajtJIydtE!{^#5%P& zhsde-y*+M+ow4pHYIY|x!V6IeaSq^>A=HP+Ly0-`rHDxjX-~~54U$`Kv#kX#%d6Tl z;hZYv@9t9zhtW;R!HlqX`UPeXk+HhH`)*YLU@cb=PZVNrFK&QDTQ6BgTmJ)>oq(=o z!?ml7D(2(kGs;bF*@UzpXlvbL#wNSc!6M`-smvBxwTgth3s*(;8AYiuP(8Asv&fY& zq$E)(!WZW!A3pU>^mjXpe(~VwV|Y=1fZ`cTipY^-Z9}jkPHpj%>E~_Mf(~#-!PLMW z)sn~a;KIVtxw}evQeU4k#wili3s!Z9*vb=;`-=!ggt;Ry`Jd2w^~rnDXg$n;mrqMG zKce~H=qU<6ToNd*2L=}FZz*a^U9JOo&6W_fKrTQ5fT<8c;Ab)giJE+QuG0S*wZdn& zsSid4-wtW z>ppSd{T-I7t#?US?-V$>Dys{X@YaQ%!)&QdiT<$iup?0k5R?l&M>q%+2AFN;hhd~S zX*=N)RcO;(=uV}SD^VcmwL}oK-FizpF(u5eyhi*~j*U6=phUE=ycuLO$aum7;&$J^ zII2V7&>h+;$CrFzv-U7bp+~9y@ff(dGwBXBaQ=xW4mB}qKlcA6djwwV z7bF|<_^q~vEJ?oKyub}yl8I32-rQu>N)eO5Qz z?P*vU75Pr7Pw+cn*uDw~sEf!lDzmRqY~ptdp?D86nB*HPleG;jJ-woPPK5y>C8`I*g3-Bg5u zw!8IuGCjG7v{QGu``Ia=p#Bsv&0x!#^E#@c(+An!Sf*=eOgqm)q&Gja5VTVIM zB7=d;J8?$a48}?3{o|y(p)GT2Q2L_bfE-R@!JP8H!BEpc(8+j{?En__*~hkDZk&I- z`6jmt)%ID1diZUTz27vnk~y7d3J6!Yc#cQ_0s+v?)W{K=vbPgfMD`HObGBP8mTFHz Vu89c= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.50)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.5 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q06-topk-sum", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 2000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 20) * 20 - 600) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 20) * 20 - 600)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 20) * 20 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 20) * 20 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q08-topk-count", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, increase(data_total[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 20000, + "k": 4, + "lookback_ms": 600000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region", + "service" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q10-cardinality", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY region", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/eval_table.json new file mode 100644 index 00000000..e59dcf2b --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/eval_table.json @@ -0,0 +1,518 @@ +{ + "inputs": { + "workload-08.json": "ee2864b9ecf924c78c7c5d2a0aa3afcca8d38fb9d84f9364da129da66483ee8d" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-08", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-08/q01-sum-fine/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 10, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-08/q01-sum-fine/keys/300s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-08/q02-sum-coarse/keys/300s", + "lookback_ms": 300000, + "metric": "data" + } + }, + "interval_secs": 10, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-08/q02-sum-coarse/keys/300s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-08/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-08/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-08/q04-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/topk4_q06-topk-sum/keys/60s", + "k": 4, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 1, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-08/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/topk4_q07-topk-sum/keys/1200s", + "k": 4, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 30, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-08/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-08/q01-sum-fine/keys/300s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-08/q02-sum-coarse/keys/300s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-08/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 60.0, + "max_N": 3000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 60000000.0, + "covered_min_N": 60000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-08/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 1200.0, + "max_N": 60000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 60000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/inference_config.yaml new file mode 100644 index 00000000..76fac1df --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/inference_config.yaml @@ -0,0 +1,78 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[5m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[5m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[5m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 60 + query: topk by (region) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 40 + query: topk by (region) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/plans.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..727ad1bbe517f9d01f8ac79f2ba69e832dbb1050 GIT binary patch literal 2304 zcmY+?c{~&T9|v$|-!{JuLyi?@rGuCw_ifG`9gafGqeITQ8p|<9$YesLq=e<1h-{H7 zROoPw@1kUiVl0v7o_wwE*Z24Jc>eKzJYMg|g`e;OpFuywKlA3!kuRan+X3hs+Z}0v$9ac zDW^P&<*Mo4@44{xd`OIztIpTB9kpj&P#vcd^Fg)#p^n-WZOhaglreNV%#BeIc3r38 zo;#P%g)f&J(TGDE_O&6-}&&yRH(rlw@RwGdQO1_PNTblxlT6>wi`A-=e*NvG;)MI z!t!-Q`7jtw9#jMGFl2>C3O!1%0p<5AbbNrKD0Mg3oz%(JR5qQY4rIz2EUW&63?ibrd0>jR4uZ}E+ma=mvGn;6|;`ByK{luwp|q^7Lg z7?d~Qu1;wDx0v@@vT4D>o_}`Oj=Mnnqo0;8Ux@pW?%PXdkJ&9bzg#kSdE1sdGS419 zbxx%vCNSmGnk%llOe5@Jvk{1npV?ri{0--t{K|n zAF(|2Gd=gn`Lurrms6{pbeQiJ4f~l(;(1|bk9|Uo?ya_ERb_GgBwvq-8inH?>ZF+Y6 zfa_lI?>6Pm4XG>EbvVq@gNLo+z7p!DZZbfq>2hj~~!4lv3) zkX9X`HtBRCzetQ}C5!Z~Mq5@SEvZv{pRCKB>qp9+g(A(;Ti{;s*(92o{-I(be;a61 z%8Rs35dEn8{NS3y$RiJm9wYq&(uaqS>3HraNV(o?RCZIC^}^sNE6G-g5i!h3ox+kr zto#vpn^)F*^VH2srN|;=5iGJA^o>8dRjztcqRSYskpWfGMaF=YR&i#n%Wd04CU*21 zt>@)n?zmC2>Ap1V`a$s}Dqds80&(x>RNX$mB&v;xjR_8)8Oy_(#fil!ujhdl@wsj_ zZjXkX_vd+HpI&Mbxg)tKqAq>U0%`^A7;Rh{O89DYL7Wn-?E zVVm^dhRoGEyq+%=k3Sy}4$c3s(U!F)h|w-_Vjx7Es7vaAg-Y`f3WJDAf;#k`hCn2UhNK=?q%==@VGt<^0z`cV zv>8{6d=pq}H71-voR{BH4}yc?WjaQPK>2M7@GBmh$(Tx@yqLPXxx!f<+?57(F>P3V zk`QpS=XV_^nNXdu<&)fIM*)))vlVPj=45R`UqV-1Ujp$3OGq@8hrVkXd}P^lPRtlK zC&1-1X%(3p*u889M*jm41G@~nygIPZ25#Rb>b6zi5~1LjCE+?HvT=BC5%gr4Z}yzA zl|=UwiSzbkF6ov%lgnM>pxGo@v-llN|s5Iv9R3% zNHCR$fq{T#C}5-zG-7U|-W>L+xRPX5fwi=`QxA#_bCx88^ppmqYNFktNGNg$WDe$8 zojNb*#+sbkZHm`J zH5e2qu#lb*lDwrA;eqrhY@x+xCU3`_b=pj9nlqEZuYDy%Z}QL;qtsAHNPC=#ZgMU^mSra$#Y2py%G_i<;5j_IusU)ZT6l;KeOhN&H{AQ*w w7!bH1e0eT;%O^w(GNzEJl>oc49kW`E^W)71KOf)5h5+C6d-aK;8h*aN0PcrBv;Y7A literal 0 HcmV?d00001 diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/streaming_config.yaml new file mode 100644 index 00000000..c1479350 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 10000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 10000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 240 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08.json new file mode 100644 index 00000000..313fac22 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-08.json @@ -0,0 +1,351 @@ +{ + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 20000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, + "id": "workload-08", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 10) * 10 - 300) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 10) * 10 - 300)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 10) * 10 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 10) * 10 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region", + "grouping": [ + "region" + ], + "id": "q08-topk-count", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 4, + "output_series_upper_bound": 32, + "promql": "topk by (region) (8, increase(data_total[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 10000, + "k": 8, + "lookback_ms": 300000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q10-cardinality", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/eval_table.json new file mode 100644 index 00000000..44ab9b24 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/eval_table.json @@ -0,0 +1,525 @@ +{ + "inputs": { + "workload-09.json": "803866a2b4df14f80ed4bd0a6f9d16508ae66a3a853dd1842904dd3df32904ac" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-09", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-09/q01-sum-fine/keys/60s", + "lookback_ms": 60000, + "metric": "data" + } + }, + "interval_secs": 2, + "promql": "sum by (region, service, job) (sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-09/q01-sum-fine/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-09/q02-sum-coarse/keys/60s", + "lookback_ms": 60000, + "metric": "data" + } + }, + "interval_secs": 2, + "promql": "sum by (region, service) (sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-09/q02-sum-coarse/keys/60s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-09/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service", + "job" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-09/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-09/q03-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/topk8_q06-topk-sum/keys/60s", + "k": 8, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 1, + "promql": "topk by (region, service) (8, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-09/topk8_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/topk8_q07-topk-sum/keys/1200s", + "k": 8, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 30, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-09/topk8_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 60000.0, + "covered_min_N": 60000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-09/q01-sum-fine/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 60.0, + "max_N": 60000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 60000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-09/q02-sum-coarse/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-09/topk8_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 600000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 12000000.0, + "covered_min_N": 12000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-09/topk8_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 12000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 12000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/inference_config.yaml new file mode 100644 index 00000000..a21787b7 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/inference_config.yaml @@ -0,0 +1,81 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[1m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[1m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[1m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 59 + query: topk by (region, service) (8, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 40 + query: topk by (region, service) (8, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service, job) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service, job) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-09-pilot/plans.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..a315ba979820ac955cd7d29779c80165018006c8 GIT binary patch literal 2393 zcmZwFdpOgJ9|!QEd_`G55^~O_tM#i~ehC>v*;XnOg;u7-nfuJ04Ka=~*W3693kVF+%Uyf9H72ZNNyq8G}m>pi54ltwiJu{Y^&3I zJ`eT{KBzoSW4^cmMqe#az+D;kOpmk$qXoP{Tgv9fXD+5{eWSjO5&|xal;^Sg=nIT| zLU6FQPTIvhp1+6ZNoy@$hOv5G7Zdsg|yxcFKY$hX1 z3J4Yr09;sZ$|BK&t`;`FQq;mDGttc{9ZuL)he@Yv7p0E4_0$R{a=Qd|T+T7{!B{({ zIZW-j4}#C{EuQRPzW=fuCos_k)8u_P(qC4_m-O#1FCLqHG<;z{n8srdMK%=ZA=C)z zw^h=~W-S3P6*754yn&-H%gtH-x{9f_xxesYMtX`m)*igtz@P7(Y8qX1sF`CHrU;U3 zgA=4T@=bd?=z5t~N_;U*tIqeeHY!8jb{|#B+<8Fqc9+-h1-$U*VRvId5RefdiG>XaRf=ONV#X zOJX^LXC*aTPzbE7Fi15}*co(u-g0~L=(BLlXoIyqtN7d1w)#!&n=G|re6G@RzO2{m zj@;pGtAW7;s({;t@I0)mCEpVav zEq<-_Cgv{!epev9hcdVYqk9?La+|(7sR^yNZd%|SEojuLCUQlA5Z>vus$sfqs36rm zLhcqCnRX_sH4hvaesURZap2LdM#-o17nc->?-t2SCx&!u(_lWgNNtxU27}=K zKieKh+Ni_uJ9`<*xPw$VNDhZQP>FO z>*A{iW*&io6U=)%q-EsGt|I2g9Vr?+l)?jbA-863&Q$9bOBaYuem$+rPv9;_XmPm; zrV*D>E{8oZr@V!0Xmv?tR8F{+t_^_;yKz^mM=qv*B9J(CYC;EPULU~HjTLkfE1$=9T%>Y|>@n@pX;=0Mf4=qph4zuKSVzy#i{6!4 zm2v4_23ntd^^|4I!R2DJ)$lp#@{*?VTJ4^35@(q{iyV;Z98#JhQ-V@2z&}RpzNh#P z0c(dwBja<3B*z;=OYM_%QR5;iPQF4_(om^Kq7*QGJss*d_^UtAAL{3u=I25B!SL6Q zkdIk5s?&o@Te((o6HRxHKyT9xKBAhAu2V@CVvRz!~^-O3&F$n~@0}P7zgR(Nk zAqziQ0~J68K934t(ccPAoP`J>fmSQ}Ocq0gGehq~vZylO+Gax&=C-s6zXrdv2C9In ze`{kAHzC`g=~fbm1r|({GfN*pR#Er0AJKh1Cvsn%tPHsedfd5)ZJxL+gey2Ih@ zSUrWr-DeniIWjN6FTfwgmaFwWz8&((<${Eu?t^hMxN*(XAHGkLS3gpB8<>B4(9PHk zM|)#}daIDts%LHRjXJoy&BZV-!LW^M6elojliEY-K5yXi>>s12Z#>vSs2)-&wHO(^ zcLIDuz0rtS?HI}t&DqxpB$$O59`1ye0W+bbK}Z??>$mt5-)P-v{X?7`cw(zI)pZB8 zkI;SCGJ}O`2EY-^0Z`y*Pct>tnT^oo_z)1Z8kiFygR(0?(5asoz#6cI$298m4Yq=N zk3kopSS;Vby9Oq~*`6bXl8+8FGeDi$3{7s|(zft5_#FdS2iE_sjpe-z6+#2CD+bIO zh6G1EXBSE-I?N1n?hHsYW%nIr=v!bul&qY+t_= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 2000, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 2000, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "avg by (region, service, job) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 8, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 2) * 2 - 60) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 2) * 2 - 60)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 2) * 2 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 2) * 2 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", + "grouping": [ + "region" + ], + "id": "q08-topk-count", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, increase(data_total[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 2000, + "k": 4, + "lookback_ms": 60000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region", + "service" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q10-cardinality", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 2000, + "lookback_ms": 60000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, service, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -60, NOW()) AND NOW() GROUP BY region, service", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/eval_table.json new file mode 100644 index 00000000..2373612d --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/eval_table.json @@ -0,0 +1,569 @@ +{ + "inputs": { + "workload-10.json": "9c023862533163e359fcbd2af0b847ce17adbe214640103a16b8bc97665cbd9c" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "a3753f2e3fdbe686db450179b8c018e95f02e17f", + "workloads": [ + { + "dataset": "random/seed=862/workload-10", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 20000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-10/q01-sum-fine/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 20, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-10/q01-sum-fine/keys/600s" + ] + }, + { + "expr": { + "rqe": { + "capability": "sum", + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-10/q02-sum-coarse/keys/600s", + "lookback_ms": 600000, + "metric": "data" + } + }, + "interval_secs": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-10/q02-sum-coarse/keys/600s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region", + "service" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q03-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-10/q03-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q04-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-10/q04-quantile/values/100s" + ] + }, + { + "expr": { + "agg": "avg", + "by": [ + "region" + ], + "of": { + "rqe": { + "capability": "quantile", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q05-quantile/values/100s", + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + } + }, + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-10/q05-quantile/values/100s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/topk8_q06-topk-sum/keys/60s", + "k": 8, + "lookback_ms": 60000, + "metric": "data", + "topk_by": [ + "region" + ] + } + }, + "interval_secs": 2, + "promql": "topk by (region) (8, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-10/topk8_q06-topk-sum/keys/60s" + ] + }, + { + "expr": { + "rqe": { + "capability": "topk_by_value", + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/topk8_q07-topk-sum/keys/1200s", + "k": 8, + "lookback_ms": 1200000, + "metric": "data", + "topk_by": [ + "region", + "service" + ] + } + }, + "interval_secs": 60, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-10/topk8_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 600000.0, + "covered_min_N": 600000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-10/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 600000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 600000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-10/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 20000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 5e-05, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 5e-05, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "topk", + "grid_K": 5000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-10/topk8_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 4, + "keys_per_window": 5000 + }, + "lookback_secs": 60.0, + "max_N": 3000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk8_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 12000000.0, + "covered_min_N": 12000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-10/topk8_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 200000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 12000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 12000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 100, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/inference_config.yaml new file mode 100644 index 00000000..f9dc1204 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/inference_config.yaml @@ -0,0 +1,77 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[10m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[10m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[10m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[10m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 30 + query: topk by (region) (8, sum_over_time(data[1m])) + planned_subquery: topk by (region) (8, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 20 + query: topk by (region, service) (8, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862-x2/workload-10-pilot/plans.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..5cf3b152bad4ffe961653bf29e395b10820de6e8 GIT binary patch literal 2553 zcmZY3cQhM%8^Ce5I%vI=ic;&6RE-j96Wr^H#OM%8tF;>?#Eey|Z`G!1w5WAUs1mfL zv;@VQ(ioA%sF9daq&Cg9uX?ZZo|p4Ezvp|-^T%_3;_)0DrQBrHv0X2B2M=Ebl&_}; z+xHw3^C6cDhP!d`y{o2*Bij`7`srCetIi@TbD8uv057M@zyXJ1i=EAoJPvg`zRa&m zk1OHNfbDQdGi5CFNyn=KW=TVa1z~!+pH9XrL?L|<=c#6XL6kJcbj@hb7n*D%-T|Po*;$>ChRontuv=->_>nd-0j{oDl#&VW)C6WV zz;CbtOsw=)MYG=H^qx>Zj|7kgw2%d&m{1?d_d4};~ntweHR5=f@vNDmxsaj%PR#ah09dY~1 z62oOjY#`5^22qjp;|29LsSUF!VOoLO#v|ZD@Uy?@TeR5|ow1~aA*V(3J4mi&=AZ!W ztTKm0B)6IIgh5OaukYGxSWG7OAN-cWq6HQtL(@8Lr4uph2wwAzuFP1SsFUF4_(e%P z$Ipzj7RMcJLkH&79mrVa1fLmxr+m&P=^#=ZGxsADI*`EZpqKzdaFcC2NVPqDwXU-Coq8=`CO*JxaNq0g_}B@hgP>6vNws|HY9 zVMZ8Clc3Q1%BE8Phkb}fJT$471{q(|3%jyNQrthw&YP^L_baK%G9^UU< zfKBVWH6VH=?+M{y$EoIp*&AOV^WA+l6ZT9eMViEl>xO5y|JG^r1*GZ5P3oCK9N*kn z;3Pyg*$WXRh^uzyREzMYU~6uxbWK3UAy;x9i%7r_KS$fr{Y5c#$AZ-v5n!eGS$z6Q z7VjZTb03qkU>j*I%oseb7?C4sZu}&TtF1r`2j+ z&m%J4Q28Dto849Diih?+cPYXgGp^3X1H2tF38bk=g8bC}OMHwSxy-A(S!Ala!#ivc z`B>Be_LbLvKX1=yFaM&)Km0!-pLoO<*|q#dwU=eahmBftR1OW#oog?Us2=HL%--u6 zJkTNa`h9?I*4}PZ${6&WL%aFO8{mFO&agE2Vcl<;UwJR`9pSTPcp)2ZjLL>TKp%)o zT^$VmU?(iYM>mK&F7|KOC~x3?mIfAM0I`Q~YWs^p@}WnS(@i4;jIJgM7*$^VM<*u5 zIhcp#UI1;*a!LLKT;zZpOWKdFuozkLFo)Kz2v`;Th&I&#>L`?AJ^@9b&HtgCiyfr5 z+6=`cvt4oot#at-Cv^k&LwW`4fFCycc5|9nmG20jXTpv0hGEjPmHbn)ZDw#rj*)3@ z`J5K{r5y!mqiAS2=dNG)oO^+Z%4#zU6c4>YC247q3zS zY_V>YW(Z?9+;`pL5F&(2Ry<0~#zrnvP8l8AzR}leU&|w#SG23lEnA}qYShm)6apqJ z>&uiH1iOd@r5J;%(j-onmfb!^vd>nTkw2ff5||R0QboG~;x~%h8=TMvMBK2wvBya` z!@^S2!lG$ix;I5ZDD4jW(|6*905y&|J`sabKsBfuv=t*CyJww<{!(_9;&OJ`>ofL* z1WPeyc+$kiVi-cqi02ZU}?{1?_p^ik7exf-ea zf6-6gPnbG%-1M+VPlvcAy`_#)57E`6fM}`|ke~X2O5@r2?H#BH&ZzC28(+e4U=v(% zKWz)G@G7Ij1bvuhAo@|08`>U=Qmz$CQZo1hLY9cY1(uj6e($vB|D6`KKY9Vg^>Iy_ z&+{ms7J2oU)SmvBR2BLGQ#n^WD= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job", + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q03-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q03-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q04-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q04-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 1000, + "lookback_ms": 100000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 1000, + "lookback_ms": 100000, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region", + "grouping": [ + "region" + ], + "id": "q06-topk-sum", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 32, + "promql": "topk by (region) (8, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 2000, + "k": 8, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 20) * 20 - 600) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 20) * 20 - 600)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 20) * 20 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 20) * 20 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 20000, + "k": 8, + "lookback_ms": 600000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", + "grouping": [ + "region", + "service" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region", + "service" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "clickhouse_sql": "SELECT region, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region", + "grouping": [ + "region" + ], + "id": "q10-cardinality", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "events", + "value": "user_id" + } + ], + "sql": "SELECT region, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY region", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "variant": "x2", + "version": 1 +}