diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 09b969991..e88b81572 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -598,7 +598,7 @@ fn plan_selection_json( id.clone(), json!({ "cost_per_second": cost.cost_per_second, - "unit": cost.unit, + "cost_unit": cost.cost_unit, "source": cost.source, "usage": usage_json(&cost.usage), "per_node": per_node, diff --git a/crates/executor/src/capability.rs b/crates/executor/src/capability.rs index c5d64623d..d57afb8e0 100644 --- a/crates/executor/src/capability.rs +++ b/crates/executor/src/capability.rs @@ -125,9 +125,10 @@ pub fn validate_summary_kernel( } /// The only shared grouping with a kernel: Hydra over Count-Min. Each update -/// adds one item column's value to its group, with weight 1 (a row count) or -/// a weight column proven non-negative (a Count-Min point count only -/// overestimates when no weight is negative). +/// adds one item column's value to its group, with weight 1 (a row count, +/// `UnitCount`), a positive integer constant (`PositiveConstant`) or a weight +/// column proven non-negative (a Count-Min point count only overestimates +/// when no weight is negative). fn validate_hydra_cms( family: &SummaryFamilyType, input: &SummaryUpdate, @@ -158,12 +159,12 @@ fn validate_hydra_cms( } } -/// A HydraCms constant weight as the `i32` sketchlib's Hydra adds: a -/// positive integer within `i32`; an error otherwise. -pub(crate) fn hydra_constant_weight(weight: f64) -> Result { +/// A HydraCms constant weight as a positive integer at most `i32::MAX` +/// (asap_sketchlib's Hydra adds `i32` weights); an error otherwise. +pub(crate) fn hydra_constant_weight(weight: f64) -> Result { if weight.is_finite() && weight > 0.0 && weight.fract() == 0.0 && weight <= f64::from(i32::MAX) { - Ok(weight as i32) + Ok(std::num::NonZeroU32::new(weight as u32).expect("weight > 0")) } else { Err(format!( "HydraCms constant weight must be a positive integer within i32 (asap_sketchlib's \ diff --git a/crates/executor/src/operators/mod.rs b/crates/executor/src/operators/mod.rs index 5059582e7..1bc2a0fc8 100644 --- a/crates/executor/src/operators/mod.rs +++ b/crates/executor/src/operators/mod.rs @@ -34,14 +34,7 @@ pub(crate) mod vector_window; pub use aggregate::Reduction; pub use series_window::SubquerySteps; pub use sort::SortKey; -pub use summary::SummaryEvaluation; -/// A shared summary build's default row weight: one per row, a count. -fn unit_weight() -> i32 { - 1 -} -fn is_unit_weight(weight: &i32) -> bool { - *weight == 1 -} +pub use summary::{HydraRowWeight, SummaryEvaluation}; #[derive(Clone, serde::Serialize, serde::Deserialize)] enum Kind { #[serde(skip)] @@ -164,13 +157,8 @@ enum Kind { SharedSummaryBuild { family: SummaryFamilyType, item: usize, - /// The weight column; each row adds `constant_weight` without one. - #[serde(default, skip_serializing_if = "Option::is_none")] - weight: Option, - /// What each row adds when there is no weight column: 1 for a count, - /// or a positive integer constant weight. - #[serde(default = "unit_weight", skip_serializing_if = "is_unit_weight")] - constant_weight: i32, + /// What each row adds to its item's count. + row_weight: HydraRowWeight, groups: Vec, /// Rows for which this is not true update no state; their group is kept. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/crates/executor/src/operators/summary/mod.rs b/crates/executor/src/operators/summary/mod.rs index 44de52ecd..51b9d8dbc 100644 --- a/crates/executor/src/operators/summary/mod.rs +++ b/crates/executor/src/operators/summary/mod.rs @@ -1,5 +1,20 @@ use super::*; use planner_types::ir::schema::{FamilyCapability, MergeInputRule}; +use std::num::NonZeroU32; + +/// What each row of a shared HydraCms build adds to its item's count. +/// Produced by the physical planner from a validated `SummaryUpdate` +/// weight and read once per row by the build. +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum HydraRowWeight { + /// The value of this input column (Float64 or Int64, non-negative); a + /// row whose value is NULL adds nothing. + Column(usize), + /// This positive integer per row: 1 for a row count. At most `i32::MAX`, + /// since asap_sketchlib's Hydra adds `i32` weights; + /// [`Operator::shared_summary_build`] refuses a larger one. + Constant(NonZeroU32), +} /// A summary evaluation: a sketch query, or an exact evaluation with typed parameters. #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] pub enum SummaryEvaluation { @@ -63,20 +78,15 @@ impl Operator { }) } /// One HydraCms state shared by every group, emitted as one row per group - /// that reads it. Each row adds its `weight` column's value (or - /// `constant_weight` without one, which must be positive) to its `item` - /// value; a row whose weight is NULL adds nothing. + /// that reads it. Each row adds its `row_weight` to its `item` value; a + /// row whose weight column is NULL adds nothing. pub fn shared_summary_build( input: SchemaRef, family: SummaryFamilyType, item: usize, - weight: Option, - constant_weight: i32, + row_weight: HydraRowWeight, groups: Vec, ) -> Result { - if constant_weight <= 0 { - return Err(invalid("HydraCms constant weight must be positive")); - } crate::values::validate_family(&family)?; crate::factory::create_hydra_cms(&family).map_err(Error::Invalid)?; validate_groups(&input, &groups)?; @@ -88,12 +98,22 @@ impl Operator { "HydraCms items must be non-null Utf8, Int64 or Bool", )); } - if let Some(weight) = weight { - if !matches!( - plain(&input, weight)?.0, - DataType::Float64 | DataType::Int64 - ) { - return Err(invalid("HydraCms weights must be Float64 or Int64")); + match row_weight { + HydraRowWeight::Column(weight) => { + if !matches!( + plain(&input, weight)?.0, + DataType::Float64 | DataType::Int64 + ) { + return Err(invalid("HydraCms weights must be Float64 or Int64")); + } + } + HydraRowWeight::Constant(weight) => { + if i32::try_from(weight.get()).is_err() { + return Err(Error::Invalid(format!( + "HydraCms constant weight must be at most i32::MAX (asap_sketchlib's \ + Hydra adds i32 weights), got {weight}" + ))); + } } } let mut fields = groups @@ -110,8 +130,7 @@ impl Operator { kind: Kind::SharedSummaryBuild { family, item, - weight, - constant_weight, + row_weight, groups, filter: None, }, @@ -345,8 +364,7 @@ pub(super) fn execute<'a>( Kind::SharedSummaryBuild { family, item, - weight, - constant_weight, + row_weight, groups, filter, } => Ok(futures::stream::once(async move { @@ -356,7 +374,7 @@ pub(super) fn execute<'a>( input, family, *item, - (*weight, *constant_weight), + *row_weight, groups, filter.as_deref(), &context, @@ -800,7 +818,7 @@ async fn build_shared_summary( mut input: Input<'_, Batch>, family: &SummaryFamilyType, item: usize, - (weight, constant_weight): (Option, i32), + row_weight: HydraRowWeight, groups: &[usize], filter: Option<&Expression>, context: &RunContext, @@ -834,10 +852,14 @@ async fn build_shared_summary( if !selected(filter, row)? { continue; } - let weight = match weight.map(|i| &row[i]) { - None => constant_weight, - Some(Value::Null) => continue, - Some(value) => hydra_weight(value)?, + let weight = match row_weight { + HydraRowWeight::Constant(weight) => { + i32::try_from(weight.get()).expect("validated HydraCms constant weight") + } + HydraRowWeight::Column(i) => match &row[i] { + Value::Null => continue, + value => hydra_weight(value)?, + }, }; let item = match &row[item] { Value::Utf8(v) => v.to_string(), diff --git a/crates/executor/src/operators/unchecked.rs b/crates/executor/src/operators/unchecked.rs index 1d6980fe6..3b40dd7c8 100644 --- a/crates/executor/src/operators/unchecked.rs +++ b/crates/executor/src/operators/unchecked.rs @@ -169,19 +169,11 @@ impl TryFrom for Operator { Kind::SharedSummaryBuild { family, item, - weight, - constant_weight, + row_weight, groups, filter, } => with_filter( - Operator::shared_summary_build( - input(0)?, - family, - item, - weight, - constant_weight, - groups, - )?, + Operator::shared_summary_build(input(0)?, family, item, row_weight, groups)?, filter, )?, Kind::KeyedEvaluation { state, k } => { diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index 1dc798aa0..9a9b5e589 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -2,7 +2,7 @@ //! Compilation needs no readers; deployment resolves inputs after selection. //! A node without a native lowering fails compilation; the deployment runtime //! must execute it. -use crate::operators::SummaryEvaluation; +use crate::operators::{HydraRowWeight, SummaryEvaluation}; use crate::summary_kernels::exact::ExactEvaluation; use crate::{ operators::{Expression, Operator, Reduction, SortKey}, @@ -1027,20 +1027,20 @@ fn summary_build( }; // Validated: the unit count, a positive integer constant or a // non-negative weight column. - let (weight, constant_weight) = match &update.weight { - SummaryInputExpr::Column(weight) => (Some(named_column(input, weight)?), 1), - SummaryInputExpr::Constant(weight) => ( - None, + let row_weight = match &update.weight { + SummaryInputExpr::Column(weight) => { + HydraRowWeight::Column(named_column(input, weight)?) + } + SummaryInputExpr::Constant(weight) => HydraRowWeight::Constant( crate::capability::hydra_constant_weight(*weight).map_err(Error::Invalid)?, ), - _ => (None, 1), + _ => unreachable!("validated HydraCms weight"), }; return Operator::shared_summary_build( input.clone(), family.clone(), named_column(input, item)?, - weight, - constant_weight, + row_weight, groups(input, keys)?, ); } diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index 3a8d895b4..3fe0cbb78 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -2,7 +2,7 @@ //! planner and answers each group's item point counts from one shared grid. use asap_executor::{ dag::{ - operators::Operator, + operators::{HydraRowWeight, Operator}, values::{Batch, SchemaRef, Value}, Limits, RunContext, Scope, }, @@ -355,6 +355,55 @@ fn positive_constant_weight_adds_the_constant_per_row() { } } +// The row weight is one serialized field: `{"Constant":2}` for a constant +// weight, `{"Column":2}` for a weight column. A deserialized constant above +// `i32::MAX` (asap_sketchlib's Hydra adds `i32` weights) is refused, as is +// one passed to the constructor. +#[test] +fn row_weight_serializes_as_one_field_and_rejects_constants_above_i32() { + let constant = SummaryUpdate { + weight: SummaryInputExpr::Constant(2.0), + weight_domain: WeightDomain::NonNegative { + proof: NonNegativeWeightProof::PositiveConstant, + }, + ..unit_count() + }; + let column = weighted(non_negative()); + for (update, expected) in [ + (constant, serde_json::json!({"Constant": 2})), + (column, serde_json::json!({"Column": 2})), + ] { + let build = compile_node(&summary_agg(update, None), &[input_schema()]).unwrap(); + let wire = serde_json::to_value(&build).unwrap(); + let fields = &wire["kind"]["SharedSummaryBuild"]; + assert_eq!(fields["row_weight"], expected, "{wire}"); + assert!(fields.get("weight").is_none() && fields.get("constant_weight").is_none()); + let restored: Operator = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(&restored).unwrap(), wire); + } + let build = |weight: u32| { + Operator::shared_summary_build( + input_schema(), + family(), + 1, + HydraRowWeight::Constant(std::num::NonZeroU32::new(weight).unwrap()), + vec![0], + ) + }; + assert!(build(i32::MAX as u32).is_ok()); + let error = build(i32::MAX as u32 + 1) + .err() + .expect("constant above i32::MAX"); + assert!(error.to_string().contains("i32::MAX"), "{error}"); + let mut wire = serde_json::to_value(build(2).unwrap()).unwrap(); + wire["kind"]["SharedSummaryBuild"]["row_weight"] = + serde_json::json!({"Constant": i32::MAX as u32 + 1}); + let error = serde_json::from_value::(wire) + .err() + .expect("deserialized constant above i32::MAX"); + assert!(error.to_string().contains("i32::MAX"), "{error}"); +} + // A weight column of unknown sign is rejected at binding: a Count-Min point // count only overestimates when no weight is negative. #[test] diff --git a/crates/logical-asap-optimization/src/asap_aware_cse.rs b/crates/logical-asap-optimization/src/asap_aware_cse.rs index 9113d974b..66ad12e10 100644 --- a/crates/logical-asap-optimization/src/asap_aware_cse.rs +++ b/crates/logical-asap-optimization/src/asap_aware_cse.rs @@ -513,12 +513,15 @@ pub fn summary_capability_pass2_candidates( // Every subset of a capability sharing candidate is a Pass 2 // candidate: 2^n of them. Past MAX_CAPABILITY_SHARING_MEMBERS that is // no longer a list plan selection can price. - if candidates + let largest = candidates .iter() - .any(|candidate| candidate.members.len() > MAX_CAPABILITY_SHARING_MEMBERS) - { + .map(|candidate| candidate.members.len()) + .max() + .unwrap_or(0); + if largest > MAX_CAPABILITY_SHARING_MEMBERS { return Err(PerComputationCandidateError::TooManyCandidates { - candidates: "summary builds in one capability sharing candidate", + counted: "summary builds in one capability sharing candidate", + found: largest, max: MAX_CAPABILITY_SHARING_MEMBERS, }); } @@ -745,7 +748,8 @@ pub fn pane_candidates(roots: &[QueryRoot], interval_ms: &[Option]) -> Vec< /// The window `W` of a build the rule can split: a `SummaryAgg` whose /// family merges, over a range `TimeRange(W)` over an instant vector read -/// at the query's time or a fixed offset from it. +/// at the query's time or a fixed offset from it: no `@` anchor on any of +/// the time shifts stacked below the range. fn pane_window_ms(build: &OperatorNode) -> Option { let Some(ASAPOp::SummaryAgg { child, family, .. }) = build.asap() else { return None; @@ -759,11 +763,17 @@ fn pane_window_ms(build: &OperatorNode) -> Option { else { return None; }; - if input.result_kind != OperatorResultKind::InstantVector - || matches!(input.non_asap(), Some(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_some()) - { + if input.result_kind != OperatorResultKind::InstantVector { return None; } + // Refuse an `@` anchor on any of the shifts stacked below the range. + let mut below = input; + while let Some(NonASAPOp::TimeShift { shift, child }) = below.non_asap() { + if shift.at.is_some() { + return None; + } + below = child; + } u64::try_from(range.as_millis()).ok().filter(|&ms| ms > 0) } @@ -789,7 +799,8 @@ pub fn window_composition_pass2_candidates( ) -> Result, PerComputationCandidateError> { if candidates.len() > MAX_PANE_CANDIDATES { return Err(PerComputationCandidateError::TooManyCandidates { - candidates: "summary builds that could be built in tumbling panes", + counted: "summary builds that could be built in tumbling panes", + found: candidates.len(), max: MAX_PANE_CANDIDATES, }); } @@ -1274,10 +1285,11 @@ fn subsets( let n = candidates.len(); if n > MAX_REUSE_CANDIDATES { return Err(PerComputationCandidateError::TooManyCandidates { - candidates: match rule { + counted: match rule { "rollup" => "rollup candidates", _ => "top-k reuse candidates", }, + found: n, max: MAX_REUSE_CANDIDATES, }); } @@ -1955,10 +1967,15 @@ mod tests { .. } )); - // The message names the cap from its constant, not a copy of it. - assert!(error - .to_string() - .contains(&format!("more than {MAX_CAPABILITY_SHARING_MEMBERS} "))); + // The message names the count found and the cap from its constant. + assert_eq!( + error.to_string(), + format!( + "found {} summary builds in one capability sharing candidate, more than \ + {MAX_CAPABILITY_SHARING_MEMBERS}", + MAX_CAPABILITY_SHARING_MEMBERS + 1 + ) + ); } /// Equal targets give identical builds, which are the @@ -2076,6 +2093,82 @@ mod tests { assert!(panes(&roots, &[None]).is_empty()); } + /// Past their caps, the pane rule and the reuse subsets name how many + /// candidates they found. + #[test] + fn too_many_pane_or_reuse_candidates_name_the_count_found() { + let roots = chose( + &[("quantile_over_time(0.99, m[5m])", 0.01)], + SketchAlgorithm::Kll, + ); + let panes: Vec<_> = pane_candidates(&roots, &[Some(60_000)]) + .into_iter() + .cycle() + .take(MAX_PANE_CANDIDATES + 1) + .collect(); + let error = window_composition_pass2_candidates(&roots, &panes).unwrap_err(); + assert_eq!( + error.to_string(), + format!( + "found {} summary builds that could be built in tumbling panes, more than \ + {MAX_PANE_CANDIDATES}", + MAX_PANE_CANDIDATES + 1 + ) + ); + let error = subsets(&[0u8; MAX_REUSE_CANDIDATES + 1], "rollup").unwrap_err(); + assert_eq!( + error.to_string(), + format!( + "found {} rollup candidates, more than {MAX_REUSE_CANDIDATES}", + MAX_REUSE_CANDIDATES + 1 + ) + ); + } + + /// A range read at an `@` anchor has no panes, however many shifts + /// stack above the anchored one: `m[5m] @ 100` with an extra + /// `offset 1m` shift between the range and the anchor is still anchored. + #[test] + fn an_anchor_below_stacked_shifts_has_no_panes() { + use asap_types::ir::operator::TimeShift; + let roots = chose( + &[("quantile_over_time(0.99, m[5m] @ 100)", 0.01)], + SketchAlgorithm::Kll, + ); + let QueryRoot::Operator(root) = &roots[0] else { + panic!("operator root") + }; + let build = OperatorNode::reachable(root) + .into_iter() + .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) + .expect("a build"); + assert_eq!(pane_window_ms(&build), None, "one anchored shift"); + // Put an `offset 1m` shift between the range and its anchored shift. + let restacked = build + .with_new_children(|range| { + Rc::new( + range + .with_new_children(|anchored| { + assert!(matches!( + anchored.non_asap(), + Some(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_some() + )); + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 60_000, + at: None, + }, + child: anchored.clone(), + })) + .unwrap() + }) + .unwrap(), + ) + }) + .unwrap(); + assert_eq!(pane_window_ms(&restacked), None, "anchor below an offset"); + } + /// Exact accumulators whose states merge get panes; IRate, which reads /// the last two samples of the window, does not. #[test] diff --git a/crates/logical-asap-optimization/src/per_computation_candidates.rs b/crates/logical-asap-optimization/src/per_computation_candidates.rs index 3722ff690..cc1d02369 100644 --- a/crates/logical-asap-optimization/src/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/per_computation_candidates.rs @@ -429,10 +429,12 @@ pub enum PerComputationCandidateError { /// A Pass 2 rule found more than `max` candidates of one kind, whose /// 2^n subsets it would enumerate: the workload is too large to plan /// this way, not bad input. - #[error("more than {max} {candidates} (2^n Pass 2 candidates)")] + #[error("found {found} {counted}, more than {max}")] TooManyCandidates { /// What was counted, e.g. "rollup candidates". - candidates: &'static str, + counted: &'static str, + /// How many were found, more than `max`. + found: usize, /// The cap that was exceeded. max: usize, }, diff --git a/crates/logical-asap-optimization/src/workload_candidates.rs b/crates/logical-asap-optimization/src/workload_candidates.rs index 4f85af28b..640a78bff 100644 --- a/crates/logical-asap-optimization/src/workload_candidates.rs +++ b/crates/logical-asap-optimization/src/workload_candidates.rs @@ -284,11 +284,13 @@ impl Pass2List { let candidates = rollup_candidates(roots); // Every rollup subset is combined with every top-k subset: cap the // product, not each factor. - if candidates.len() + topk_reuse_candidates(roots).len() > MAX_REUSE_CANDIDATES { + let found = candidates.len() + topk_reuse_candidates(roots).len(); + if found > MAX_REUSE_CANDIDATES { self.out.push(Pass2::failed( rules, PerComputationCandidateError::TooManyCandidates { - candidates: "rollup and top-k reuse candidates together", + counted: "rollup and top-k reuse candidates together", + found, max: MAX_REUSE_CANDIDATES, }, )); @@ -425,10 +427,7 @@ mod tests { }; assert_eq!( error.to_string(), - format!( - "more than {MAX_REUSE_CANDIDATES} rollup and top-k reuse candidates together \ - (2^n Pass 2 candidates)" - ) + format!("found 21 rollup and top-k reuse candidates together, more than {MAX_REUSE_CANDIDATES}") ); } } diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index 4c8272704..c9d4051e5 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -496,7 +496,8 @@ pub struct CandidateCost { /// Its price per second under cost model 1 or 2 (under cost model 1, the /// sum of its nodes'); `None` under cost model 3. pub cost_per_second: Option, - pub unit: &'static str, + /// The unit of `cost_per_second`: always [`COST_UNIT`]. + pub cost_unit: &'static str, /// Where its prices came from, from its nodes' [`NodeCost::price_source`]: /// [`COST_SOURCE`] (all formulas), [`MEASURED_COST_SOURCE`] (all /// measured) or [`MIXED_COST_SOURCE`]. @@ -2658,7 +2659,7 @@ fn price( Ok(CandidateCost { usage, cost_per_second: cost, - unit: COST_UNIT, + cost_unit: COST_UNIT, source: cost_source(&per_node), per_node, }) @@ -6393,7 +6394,7 @@ mod tests { ..ResourceUsage::default() }, cost_per_second: None, - unit: COST_UNIT, + cost_unit: COST_UNIT, source: COST_SOURCE, per_node: BTreeMap::new(), } diff --git a/tools/dag-viewer/app.js b/tools/dag-viewer/app.js index a24e323f7..48f2caf0c 100644 --- a/tools/dag-viewer/app.js +++ b/tools/dag-viewer/app.js @@ -160,7 +160,7 @@ const row = rows.find((r) => rowKey(r) === `P:${id}`); const priced = rows.filter((r) => r.rank !== null).length; $('lane2-meta').innerHTML = `${esc(c.display_name)} · from ${esc(c.from_logical)} · ` + - (cost && cost.cost_per_second !== null ? `total ${fmtCost(cost.cost_per_second)} ${esc(cost.unit.replace(/_/g, ' '))} · rank ${row.rank} of ${priced}` + (cost && cost.cost_per_second !== null ? `total ${fmtCost(cost.cost_per_second)} ${esc(cost.cost_unit.replace(/_/g, ' '))} · rank ${row.rank} of ${priced}` : cost ? 'no price (cost model 3)' : 'not priced') + ` · ${statusChip(row.status)}` + (row.accepted ? ` · ${esc(row.accepted)}` : ''); @@ -203,7 +203,7 @@ (selection.selected ? ` → 1 selected (${count('rejected')} invalid, ${count('costlier')} valid but costlier)` : ' → none selected'); $('rank-meta').textContent = stageCandidateHeader(doc); const sel = selection.selected ? selection.costs[selection.selected] : null; - $('cost-unit').textContent = sel ? sel.unit.replace(/_/g, ' ') : ''; + $('cost-unit').textContent = sel ? sel.cost_unit.replace(/_/g, ' ') : ''; $('pick1').innerHTML = logical.candidates.map((c) => ``).join(''); $('pick2').innerHTML = rows.filter((r) => rowKey(r).startsWith('P')) diff --git a/tools/dag-viewer/editor.js b/tools/dag-viewer/editor.js index 6acc84ca8..ae3db30e5 100644 --- a/tools/dag-viewer/editor.js +++ b/tools/dag-viewer/editor.js @@ -128,7 +128,7 @@ const selection = result.stage3_plan_selection; const cost = selection.costs[selection.selected]; plannerStatus.textContent = selection.selected - ? `Stage 3 selected ${selection.selected} (cost ${cost.cost_per_second} ${cost.unit}) of ${result.stage1_logical_asap_optimization.combinations} candidates.` + ? `Stage 3 selected ${selection.selected} (cost ${cost.cost_per_second} ${cost.cost_unit}) of ${result.stage1_logical_asap_optimization.combinations} candidates.` : `Stage 3 selected no plan: ${selection.error || result.stage1_logical_asap_optimization.error || selection.rejected.map((r) => `${r.id}: ${r.reason}`).join('; ')}`; } catch (error) { // Without a response the page is not served by server.py. diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index 03488a2ae..db38b8620 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -103821,6 +103821,7 @@ "costs": { "P1": { "cost_per_second": 7.94021, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 2.32, @@ -103902,7 +103903,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 0.0, "cpu_ops_per_second": 4100210.0, @@ -103924,6 +103924,7 @@ }, "P2": { "cost_per_second": 7.9402099999999995, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 4.64, @@ -103999,7 +104000,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 0.0, "cpu_ops_per_second": 4100210.0, @@ -104021,6 +104021,7 @@ }, "P3": { "cost_per_second": 7.940209999999999, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 4.64, @@ -104090,7 +104091,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 0.0, "cpu_ops_per_second": 4100210.0, @@ -104112,6 +104112,7 @@ }, "P4": { "cost_per_second": 8.54021, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 2.32, @@ -104193,7 +104194,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 0.0, "cpu_ops_per_second": 4700210.0, @@ -104215,6 +104215,7 @@ }, "P4-m1": { "cost_per_second": 12.940212599999999, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 0.3866686, @@ -104296,7 +104297,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 56000000.0, "cpu_ops_per_second": 3700211.0, @@ -104318,6 +104318,7 @@ }, "P5": { "cost_per_second": 8.540209999999998, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 4.64, @@ -104393,7 +104394,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 0.0, "cpu_ops_per_second": 4700210.0, @@ -104415,6 +104415,7 @@ }, "P5-m1": { "cost_per_second": 12.940212599999999, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 0.3866686, @@ -104496,7 +104497,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 56000000.0, "cpu_ops_per_second": 3700211.0, @@ -104518,6 +104518,7 @@ }, "P6": { "cost_per_second": 8.54021, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 2.32, @@ -104599,7 +104600,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 0.0, "cpu_ops_per_second": 4700210.0, @@ -104621,6 +104621,7 @@ }, "P6-m1": { "cost_per_second": 12.940212599999999, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 2.32, @@ -104702,7 +104703,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 56000000.0, "cpu_ops_per_second": 3700211.0, @@ -104724,6 +104724,7 @@ }, "P7": { "cost_per_second": 8.540209999999998, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 4.64, @@ -104799,7 +104800,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 0.0, "cpu_ops_per_second": 4700210.0, @@ -104821,6 +104821,7 @@ }, "P7-m1": { "cost_per_second": 12.940212599999999, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 2.32, @@ -104902,7 +104903,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 56000000.0, "cpu_ops_per_second": 3700211.0, @@ -104924,6 +104924,7 @@ }, "P8": { "cost_per_second": 9.14021, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 2.32, @@ -105005,7 +105006,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 0.0, "cpu_ops_per_second": 5300210.0, @@ -105027,6 +105027,7 @@ }, "P8-m1": { "cost_per_second": 13.540212599999998, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 2.32, @@ -105108,7 +105109,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 56000000.0, "cpu_ops_per_second": 4300211.0, @@ -105130,6 +105130,7 @@ }, "P8-m2": { "cost_per_second": 13.540212599999998, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 0.3866686, @@ -105211,7 +105212,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 56000000.0, "cpu_ops_per_second": 4300211.0, @@ -105233,6 +105233,7 @@ }, "P8-m3": { "cost_per_second": 17.940215199999997, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 0.3866686, @@ -105314,7 +105315,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 112000000.0, "cpu_ops_per_second": 3300212.0, @@ -105336,6 +105336,7 @@ }, "P9": { "cost_per_second": 9.140209999999998, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 4.64, @@ -105411,7 +105412,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 0.0, "cpu_ops_per_second": 5300210.0, @@ -105433,6 +105433,7 @@ }, "P9-m1": { "cost_per_second": 13.540212599999998, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 2.32, @@ -105514,7 +105515,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 56000000.0, "cpu_ops_per_second": 4300211.0, @@ -105536,6 +105536,7 @@ }, "P9-m2": { "cost_per_second": 13.540212599999998, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 0.3866686, @@ -105617,7 +105618,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 56000000.0, "cpu_ops_per_second": 4300211.0, @@ -105639,6 +105639,7 @@ }, "P9-m3": { "cost_per_second": 17.940215199999997, + "cost_unit": "cost_per_second", "per_node": { "0": { "cost_per_second": 0.7733372, @@ -105714,7 +105715,6 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "unit": "cost_per_second", "usage": { "average_memory_bytes": 112000000.0, "cpu_ops_per_second": 3300212.0, diff --git a/tools/dag-viewer/stages.js b/tools/dag-viewer/stages.js index db5a0473b..d55e23055 100644 --- a/tools/dag-viewer/stages.js +++ b/tools/dag-viewer/stages.js @@ -204,7 +204,7 @@ function validateSelection(selection, physicalById) { if (!physicalById.has(id)) { errors.push(`${at}: not a Stage 2 candidate`); return; } // `cost_per_second` is null under cost model 3 (own cluster), which has no prices. if (!cost || (cost.cost_per_second !== null && (typeof cost.cost_per_second !== 'number' || !Number.isFinite(cost.cost_per_second)))) errors.push(`${at}: cost_per_second must be a number or null`); - if (!cost || typeof cost.unit !== 'string') errors.push(`${at}: unit must be a string`); + if (!cost || typeof cost.cost_unit !== 'string') errors.push(`${at}: cost_unit must be a string`); const nodeIds = new Set(physicalById.get(id).dag.nodes.map((node) => String(node.id))); Object.keys((cost && cost.per_node) || {}).forEach((key) => { if (!nodeIds.has(key)) errors.push(`${at}.per_node names missing node ${key}`); @@ -313,7 +313,6 @@ function stageCandidateRows(doc) { logical: candidate.from_logical, display_name: candidate.display_name, total: cost ? cost.cost_per_second : null, - unit: cost ? cost.unit : null, rank: null, status, reason: status === 'rejected' && rejection ? rejection.reason : null, @@ -321,10 +320,10 @@ function stageCandidateRows(doc) { }; }); (physical.pruned || []).forEach((p) => rows.push({ - id: p.from_logical, logical: p.from_logical, display_name: p.display_name, total: null, unit: null, rank: null, status: 'pruned', reason: p.reason, + id: p.from_logical, logical: p.from_logical, display_name: p.display_name, total: null, rank: null, status: 'pruned', reason: p.reason, })); ((doc.stage1_logical_asap_optimization || {}).candidates || []).filter((c) => 'error' in c).forEach((c) => rows.push({ - id: c.id, logical: c.id, display_name: c.display_name, total: null, unit: null, rank: null, status: 'composition error', reason: c.error, + id: c.id, logical: c.id, display_name: c.display_name, total: null, rank: null, status: 'composition error', reason: c.error, })); const order = { selected: 0, costlier: 1, rejected: 2, 'not judged': 2, pruned: 3, 'composition error': 4 }; rows.sort((a, b) => order[a.status] - order[b.status] || (a.total ?? 0) - (b.total ?? 0)); diff --git a/tools/dag-viewer/test_viewer.py b/tools/dag-viewer/test_viewer.py index d76a39a9f..a87a00f98 100644 --- a/tools/dag-viewer/test_viewer.py +++ b/tools/dag-viewer/test_viewer.py @@ -114,7 +114,7 @@ def test_contract_violations_are_reported(self): "unknown rejection cause": lambda d: selection(d)["rejected"][0].update(cause="invalid"), "per_node": lambda d: selection(d)["costs"]["P1"]["per_node"].update({"99": {"cost_per_second": 1, "multiplier": 1}}), "cost per second": lambda d: selection(d)["costs"]["P1"].pop("cost_per_second"), - "cost for unknown candidate": lambda d: selection(d)["costs"].update({"P99": {"cost_per_second": 1, "unit": "u", "per_node": {}}}), + "cost for unknown candidate": lambda d: selection(d)["costs"].update({"P99": {"cost_per_second": 1, "cost_unit": "u", "per_node": {}}}), "missing stage": lambda d: d.pop("stage2_physical_asap_optimization"), "binary operand names no node": lambda d: stage0(d)["nodes"].append( {"operator": {"NonASAP": {"BinaryOp": {"lhs": 0, "rhs": 42, "operator": {}, "return_bool": False}}}}), @@ -198,8 +198,8 @@ def test_pruned_and_composition_errors_come_last_with_their_reasons(self): self.assertEqual( rows[-2:], [ - {"id": "L2", "logical": "L2", "display_name": "Q1 exact", "total": None, "unit": None, "rank": None, "status": "pruned", "reason": "no implementation"}, - {"id": "L99", "logical": "L99", "display_name": "Q1 exact", "total": None, "unit": None, "rank": None, "status": "composition error", "reason": "cannot substitute"}, + {"id": "L2", "logical": "L2", "display_name": "Q1 exact", "total": None, "rank": None, "status": "pruned", "reason": "no implementation"}, + {"id": "L99", "logical": "L99", "display_name": "Q1 exact", "total": None, "rank": None, "status": "composition error", "reason": "cannot substitute"}, ], )