diff --git a/api/v1alpha1/reservation_types.go b/api/v1alpha1/reservation_types.go
index 2b9e25470..88b3988cf 100644
--- a/api/v1alpha1/reservation_types.go
+++ b/api/v1alpha1/reservation_types.go
@@ -179,6 +179,14 @@ type FailoverHostReservationSpec struct {
// See Spec.TargetHost for the target host of the failover reservation.
// +kubebuilder:validation:Optional
SourceHosts []string `json:"sourceHosts,omitempty"`
+
+ // PartitionDescription is a short, human-readable label describing what
+ // distinguishes the hosts of this failover partition from those of other
+ // partitions, e.g. "spec.aggregates=[hana]". It carries no scheduling
+ // semantics and exists purely to make a reservation understandable at a
+ // glance. See characterizePartitions in the failoverhost controller.
+ // +kubebuilder:validation:Optional
+ PartitionDescription string `json:"partitionDescription,omitempty"`
}
// ReservationSpec defines the desired state of Reservation.
diff --git a/cmd/manager/main.go b/cmd/manager/main.go
index 9dcd8f85d..e5858decd 100644
--- a/cmd/manager/main.go
+++ b/cmd/manager/main.go
@@ -59,6 +59,7 @@ import (
"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments"
commitmentsapi "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments/api"
"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover"
+ "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failoverhost"
"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/inflight"
"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/quota"
"github.com/cobaltcore-dev/cortex/pkg/cache"
@@ -791,6 +792,30 @@ func main() {
"maxVMsToProcess", failoverConfig.MaxVMsToProcess,
"vmSelectionRotationInterval", failoverConfig.VMSelectionRotationInterval)
}
+ if slices.Contains(mainConfig.EnabledControllers, "failover-host-reservations-controller") {
+ setupLog.Info("enabling controller", "controller", "failover-host-reservations-controller")
+ failoverHostConfig := conf.GetConfigOrDie[failoverhost.Config]().Failover.Hosts
+ failoverHostMonitor := failoverhost.NewMonitor()
+ if err := metrics.Registry.Register(failoverHostMonitor); err != nil {
+ setupLog.Error(err, "failed to register failover host monitor metrics, continuing without metrics")
+ failoverHostMonitor = nil
+ }
+ controller := &failoverhost.Controller{
+ Client: multiclusterClient,
+ Config: failoverHostConfig,
+ Monitor: failoverHostMonitor,
+ }
+ if err := controller.SetupWithManager(mgr); err != nil {
+ setupLog.Error(err, "unable to create controller",
+ "controller", "failover-host-reservations-controller")
+ os.Exit(1)
+ }
+ setupLog.Info("failover-host-reservations-controller registered",
+ "ratio", failoverHostConfig.Ratio,
+ "minimum", failoverHostConfig.Minimum,
+ "generationInterval", failoverHostConfig.GenerationInterval,
+ "externalSchedulerURL", failoverHostConfig.ExternalSchedulerURL)
+ }
if slices.Contains(mainConfig.EnabledControllers, "capacity-controller") {
setupLog.Info("enabling controller", "controller", "capacity-controller")
capacityConfig := conf.GetConfigOrDie[capacity.Config]()
diff --git a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md
index b1540706f..3d7a84958 100644
--- a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md
+++ b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md
@@ -122,7 +122,7 @@ pay-as-you-go placement and is not slotted. This is how a committed reservation
> confirmed location — and several Nova requests can be in flight at once. How Cortex reserves for its
> candidates before the next request scores, and how it reconciles against what Nova actually did, is the
> subject of
-> [Concurrency and in-flight reservations](../03-reservations-and-inventory/04-concurrency-and-in-flight-reservations.md).
+> [Concurrency and in-flight reservations](../03-reservations-and-inventory/05-concurrency-and-in-flight-reservations.md).
## Closing the loop: descheduling
diff --git a/docs/03-reservations-and-inventory/01-reservations-overview.md b/docs/03-reservations-and-inventory/01-reservations-overview.md
index d79376f9b..93b3b13ad 100644
--- a/docs/03-reservations-and-inventory/01-reservations-overview.md
+++ b/docs/03-reservations-and-inventory/01-reservations-overview.md
@@ -46,7 +46,9 @@ are checked arithmetically against headroom. This is the subject of
These reserve **headroom for evacuation**: if a host fails, its workloads need somewhere to land. Cortex
pre-clears space to absorb up to `m` simultaneous host failures, shared across the protected workloads
rather than duplicated per workload. This is the subject of
-[Failover reservations for KVM HA](03-failover-reservations.md).
+[Failover reservations for KVM HA](03-failover-reservations.md) — now **deprecated** in favour of the
+whole-host approach in [Failover host reservations](04-failover-host-reservations.md), which keeps entire
+hosts empty as gang-evacuation targets.
### In-flight reservations
@@ -59,7 +61,7 @@ in-flight reservation blocks that capacity from "decided" until "running", then
> The reasoning here is pessimistic by necessity: because concurrent requests race, Cortex assumes a slot
> it just recommended is already taken until the placement is realized or abandoned. In-flight
> reservations are the persistence layer behind the scheduler's concurrency handling —
-> [Concurrency and in-flight reservations](04-concurrency-and-in-flight-reservations.md) explains the
+> [Concurrency and in-flight reservations](05-concurrency-and-in-flight-reservations.md) explains the
> races they defend against and how the scheduler serializes around them.
## Inspecting reservations
diff --git a/docs/03-reservations-and-inventory/03-failover-reservations.md b/docs/03-reservations-and-inventory/03-failover-reservations.md
index d9e4a6bb1..a2d232d49 100644
--- a/docs/03-reservations-and-inventory/03-failover-reservations.md
+++ b/docs/03-reservations-and-inventory/03-failover-reservations.md
@@ -6,6 +6,16 @@
# Failover reservations for KVM HA
+> [!WARNING]
+> **Deprecated.** The slot-based failover reservation system described on this page (the
+> `failover-reservations-controller`, reserving a shared `FailoverReservation` pool) is being **replaced by the
+> failover *host* system** — see [Failover host reservations](04-failover-host-reservations.md). Rather than
+> reserving scattered headroom slots, the new system keeps whole hosts empty so a failed host's entire workload
+> gang-evacuates onto one reserved host. The `FailoverReservation` type is marked deprecated in
+> `api/v1alpha1/reservation_types.go` and will be removed; new deployments should enable the
+> `failover-host-reservations-controller` instead. This page is retained to document the existing system during
+> migration.
+
Where committed resources reserve space a customer paid for, **failover reservations** reserve space
*nobody is using yet* — headroom kept clear so that when a host fails, the workloads on it have somewhere
to be evacuated. This page explains the two design choices that make failover headroom affordable: it is
@@ -94,4 +104,4 @@ same slot.
## Next
-[Prev: Committed-resource reservations](02-committed-resource-reservations.md) · [Next: Concurrency and in-flight reservations »](04-concurrency-and-in-flight-reservations.md)
+[Prev: Committed-resource reservations](02-committed-resource-reservations.md) · [Next: Failover host reservations »](04-failover-host-reservations.md)
diff --git a/docs/03-reservations-and-inventory/04-failover-host-reservations.md b/docs/03-reservations-and-inventory/04-failover-host-reservations.md
new file mode 100644
index 000000000..f3936dbb7
--- /dev/null
+++ b/docs/03-reservations-and-inventory/04-failover-host-reservations.md
@@ -0,0 +1,506 @@
+
+
+# Failover host reservations
+
+The previous page reserved **headroom** — a shared pool of free capacity spread across the fleet, sized to
+absorb `m` host failures. This page describes a different, coarser-grained mechanism that is replacing it:
+instead of reserving scattered slots, Cortex keeps a fraction of **whole hosts** empty, so that when a host
+dies its *entire* workload can be gang-evacuated onto one reserved host in a single move (*gang-evacuated*:
+all the host's VMs moved together, in one placement decision, rather than one at a time).
+
+> [!IMPORTANT]
+> Failover host reservations **supersede** the slot-based failover reservations described on the
+> [previous page](03-failover-reservations.md). The older system is deprecated and will be removed. New
+> deployments should enable the failover-host reservations controller; the old slot-based controller is
+> retained only for migration. The key reason for the switch: whole-host reservations trade a little more
+> idle capacity for a pre-validated, gang-evacuable landing zone, so no placement has to be re-solved under
+> pressure the moment a host dies.
+
+## Partitions: hosts are not interchangeable
+
+Hosts cannot fail over onto *any* other host. A HANA host, a general-purpose host, and an external-customer
+host each carry scheduling constraints — flavors, image properties, aggregates, availability zone — that only
+matching hosts satisfy. A failover target is only useful if the dead host's VMs would actually be *accepted*
+there.
+
+Cortex therefore groups hosts into **failover partitions**: sets of hosts within which every host can be
+evacuated onto every other host. Crucially, these groups are **not configured by hand**. They are *discovered*
+from the scheduler's own placement decisions, so the grouping always reflects the live filters, weights and
+constraints — a partition can never contain a target the scheduler would in fact reject.
+
+One could imagine solving this differently: hand-craft the partitions from the hypervisors directly — write
+rules that read each host's flavors, image properties, aggregates and availability zone and group the
+"compatible" ones together. But those are exactly the constraints the VM scheduling pipeline *already*
+encodes. Re-expressing them here would mean re-implementing a second, parallel copy of the placement logic,
+which then has to be kept in lock-step with the real one forever; the moment a filter changes in the pipeline
+and not in the failover rules, the partitions quietly diverge from what the scheduler would actually accept,
+and a "failover target" may reject the very workload it was reserved for.
+
+So Cortex takes a **self-partitioning approach at the VM level** instead. Rather than restating the rules, it
+asks the real scheduler — per VM — where each VM could go, and lets the partitions *fall out* of those
+answers (the next section works through how). The compatibility rules are never duplicated: whatever the
+pipeline does, the partitions inherit, automatically and for free, including every future change to the
+filters and weighers.
+
+Everything on this page is a continuous background process: Cortex periodically re-derives the partitions and
+the set of reserved hosts from the current state of the fleet, and nudges the actual reservations toward that
+freshly computed target. Nothing here is a one-time setup step; the picture re-settles every few minutes as
+the fleet changes.
+
+## How a reservation is kept up to date
+
+The mechanism has two cooperating halves:
+
+- A **planner** runs on a fixed interval (default every `5m`). It looks at the current fleet, works out which
+ hosts *should* be reserved as failover targets, and publishes that as the desired picture. This is the
+ expensive part — it talks to the scheduler — so it runs on a slow cadence, never on every event.
+- A **reconciler** continuously nudges reality toward that desired picture: it creates reservations that
+ should exist, removes ones that no longer should, and updates ones that have drifted. It is cheap, and it
+ does nothing clever — it just closes the gap.
+
+```mermaid
+flowchart LR
+ PLAN["Planner (periodic)
decides which hosts to reserve"] -->|"publishes desired picture"| STORE[("desired picture")]
+ STORE -.->|"read"| RECON["Reconciler (continuous)
makes reality match"]
+ RECON -->|"create / update / remove"| REAL[("actual reservations")]
+```
+
+Because the planner republishes the **whole** desired picture each pass (rather than editing it in place), the
+reconciler gets pruning for free: any reservation that exists in the fleet but is absent from the latest
+picture is simply one the planner no longer wants, and is removed.
+
+A corollary matters for safety: the planner only ever publishes a picture it computed from a *complete*
+snapshot. If anything goes wrong while gathering the inputs — the fleet snapshot can't be collected, or a
+scheduler probe fails — the whole pass is aborted and the previously published picture is left untouched; the
+planner simply retries on the next cycle. The reconciler therefore never acts on a half-gathered view, and
+reservations are never recomputed from partial data.
+
+```mermaid
+stateDiagram-v2
+ direction LR
+ [*] --> compare
+ compare --> create: wanted, missing
+ compare --> remove: present, no longer wanted
+ compare --> update: present but drifted
+ compare --> noop: already matches
+ create --> [*]
+ remove --> [*]
+ update --> [*]
+ noop --> [*]
+```
+
+## How the planner decides: building the failover graph
+
+The planner's job — "which hosts should be reserved?" — comes down to building a graph of who can rescue whom,
+grouping that graph into partitions, and then reserving a fraction of each partition.
+
+The important thing to understand is that the graph is **narrowed in several steps**, and each step throws out
+candidates for a *different* reason. Keeping those reasons straight is the whole algorithm, because the single
+most common question — *"why does this host have no failover target?"* — has a different answer at each step:
+
+1. **Discover who can rescue whom** (Step 1 below). Ask the scheduler, per VM, which hosts would accept it,
+ intersect across all VMs on a host, then check the target is big enough for the whole host — yielding
+ one-way rescue links `source → target`. *A host drops out if its own VMs disagree (empty intersection), or
+ a target drops out if it is smaller than the source.*
+2. **Group into partitions** (Step 2). Keep only links that work *both* ways, then cluster those mutual pairs
+ into groups where everyone can rescue everyone. *A one-way link leaves no trace in the partitions.*
+3. **Make the partitions disjoint** (Step 3). A host can land in two overlapping groups; assign it to one.
+ *A group left with a single host is then discarded.*
+4. **Pick the failover hosts** (Step 4). Reserve a fraction of each partition as empty landing zones.
+
+Each emphasised clause is an edge case, worked through below. A
+[troubleshooting table](#why-does-this-host-have-no-failover-target) at the end maps each symptom back to the
+step that caused it.
+
+### Why Bron-Kerbosch for the partitioning
+
+Step 2 reduces to a classic graph question: in the undirected graph of mutual rescue pairs, a failover
+partition is a set of hosts that are *all pairwise* connected — a **maximal clique**. Cortex enumerates those
+cliques with the **Bron-Kerbosch** algorithm, and the choice is deliberate:
+
+- **The problem is literally maximal-clique enumeration, not something weaker.** "Every host can evacuate onto
+ every other host" is the definition of a clique. Cheaper alternatives solve a *different* problem and would
+ silently admit invalid partitions: connected components or a union-find grouping would lump together hosts
+ linked only transitively (`a`↔`b`, `b`↔`c` but not `a`↔`c`), producing a "partition" in which `a` cannot
+ actually rescue `c`. A greedy "grow a group while everyone stays connected" heuristic avoids that but picks
+ an arbitrary clique and misses overlapping ones, so some hosts would lose a partition they genuinely belong
+ to.
+- **We need *all maximal* cliques, including overlapping ones.** A host can legitimately belong to more than
+ one partition (it is Step 3's own job to resolve that overlap afterwards). Bron-Kerbosch is the standard,
+ well-understood algorithm that enumerates *exactly* the maximal cliques — no duplicates, nothing missed —
+ which is what makes the downstream overlap-resolution well-defined.
+- **The fleet is partitioned by construction, so the worst case never bites.** Maximal-clique enumeration is
+ exponential in the general case, which sounds alarming. But the mutual-rescue graph is not a general graph:
+ it is already almost-disconnected into compatibility islands (HANA, general-purpose, external-customer, …),
+ and cliques only form *within* an island. Each clique-search therefore runs over a small, dense subgraph,
+ not the whole fleet, which keeps it comfortably fast in practice.
+
+With that settled, the four steps in detail:
+
+### Step 1 — Discover who can rescue whom
+
+For every occupied host, Cortex finds the hosts that host's *entire* workload could be gang-evacuated onto.
+The result is a set of directed **rescue links**:
+
+> A link `source → target` means *"every VM on `source` would be accepted on `target`, and `target` is big
+> enough to hold them all at once."*
+
+Direction matters: `source → target` does **not** imply `target → source`. The target might simply be too
+small to take the source's workload even when the reverse would fit.
+
+Rather than reimplementing placement logic, Cortex **asks the real scheduler**, for each VM, where that VM
+could go. This guarantees the rescue links obey exactly the same rules as ordinary placement. A host is a valid
+target for a source only if it is accepted for **every** VM on that source — so Cortex intersects the
+per-VM answers:
+
+```mermaid
+flowchart TB
+ subgraph src["Source host S (vm1, vm2, vm3)"]
+ VM1["vm1 → scheduler says: {A, B, C}"]
+ VM2["vm2 → scheduler says: {B, C, D}"]
+ VM3["vm3 → scheduler says: {A, C}"]
+ end
+ VM1 --> INT
+ VM2 --> INT
+ VM3 --> INT["only host accepted by ALL three = C"]
+ INT --> FIT{"is C big enough
for all of S at once?"}
+ FIT -->|yes| EDGE["rescue link S → C"]
+ FIT -->|no| DROP["discard"]
+```
+
+**Edge case — the host's own VMs disagree.** If the VMs on a host pull in different directions (say a HANA VM
+and a GPU VM that no single host accepts together), the intersection is empty and the host gets **no rescue
+links at all** — it goes unprotected this pass, even though it is running work:
+
+```mermaid
+flowchart TB
+ subgraph src2["Source host S — VMs want incompatible things"]
+ V1["vm1 (HANA flavor) → {A, B}"]
+ V2["vm2 (GPU flavor) → {C, D}"]
+ end
+ V1 --> X["no host accepted by both = none"]
+ V2 --> X
+ X --> R["S has no failover target this pass"]
+ style R fill:#ffebee,stroke:#c62828
+```
+
+This is a deliberate limitation: Cortex does not (yet) split one host's workload across several targets, so a
+host whose own VMs cannot share a single landing zone simply waits until its workload mix changes. The
+failover controller will not solve this on its own — it only reserves whole-host landing zones. Spreading a
+single host's VMs across *several* targets is instead a job for Cortex's broader scheduling components, which
+will in future be able to steer placement so that a host's workload stays jointly evacuable (or to plan a
+multi-target evacuation directly), removing the "one host, one landing zone" constraint assumed here.
+
+Three things keep this probing faithful and safe:
+
+- **Each probe mirrors a real migration** — it carries the VM's owner, availability zone, flavor and image, and
+ tells the scheduler to treat it as a **live migration** (not an evacuation), so exactly the same filters apply
+ as in a genuine move. The choice of *live migration* over *evacuation* is deliberate and matters for
+ correctness. A failover target is reserved **empty**, which means that before its reservation can go active,
+ Cortex must first **drain** whatever currently runs on it — and draining a live host is done by *live*
+ migration, not evacuation. A host that a VM could only be *evacuated* onto (cold migration, VM powered off)
+ but not *live-migrated* onto therefore cannot actually be emptied, so admitting it as a target would block
+ the drain and break partitioning. Live migration is also the stricter test — on top of capacity it checks CPU
+ architecture, CPU modes, hypervisor features and emulated devices between source and target — so Cortex
+ assumes that any host which accepts a live migration also accepts the weaker cold migration (evacuation) a
+ real failover performs. Probing with the stricter requirement keeps every discovered partition genuinely
+ drainable.
+- **Candidates are judged as if empty.** The probe tells the scheduler to measure each candidate against its
+ *total* capacity, ignoring whatever VMs currently run there. This is deliberate: a failover target is reserved
+ **empty**, so the right question is "could this host hold the workload once cleared?", not "does it have room
+ right now?". Were the probe to measure live free capacity instead, a busy host would accept no one, gain no
+ inbound rescue link, form no mutual edge, and stay orphaned — even though reserving it would drain it. Whether
+ two hosts are *compatible* is thus independent of how full they happen to be; which host is cheapest to
+ actually empty is decided later, in Step 4.
+- **The probing never changes anything.** It is strictly read-only, so asking "where could this VM go?"
+ thousands of times never disturbs live scheduling. It also tells the scheduler to *ignore* existing failover
+ host reservations — otherwise a host already reserved for failover would look occupied and vanish from its own
+ partition, fragmenting the grouping.
+
+Finally, a hard **whole-host fit** check discards any target that is smaller than its source (memory first,
+then CPU). An evacuation moves *all* of the source's VMs onto the target at once, so the target must be at least
+as large as the source. This check compares *total* capacities, independent of the empty-host probing above, so
+an undersized target is dropped whether or not it currently runs anything.
+
+Two outcomes of a per-VM lookup must not be confused, because they are handled oppositely — and the
+distinction is exactly what upholds the "consistent picture or nothing" rule:
+
+- **The scheduler answers that a VM has nowhere to go** (an empty candidate set, no error). This is a valid
+ answer, not a failure. That VM simply contributes nothing to the intersection; if it empties the
+ intersection, its host goes unprotected this pass, but the pass continues normally.
+- **The lookup itself fails** (the scheduler is unreachable, times out, or returns a non-200). This is an
+ infrastructure failure, not an answer — on a partial view Cortex cannot assert anything about the failover
+ graph, so it **aborts the entire pass** and leaves the last published picture untouched, retrying next
+ cycle. Swallowing the error and skipping just that VM would collapse the affected host's rescue links and
+ prune reservations that are in fact still valid, so a single transient blip must never be allowed to erase
+ coverage.
+
+### Step 2 — Group hosts into partitions
+
+A **failover partition is a set of hosts in which every host can be evacuated onto every other host.** For that
+to hold, rescue must work in *both* directions between any two members: `a` can rescue `b` **and** `b` can
+rescue `a`. So Step 2 first keeps only the **mutual** links — the one-way links from Step 1 are discarded here.
+
+**One-way links are dropped.** This is why a link can exist in Step 1 yet leave no trace in the partitions:
+
+The reason mutuality is required — not just a convenient graph property — is that a failover must be
+*reversible*. A VM evacuated onto a target host should not be **locked in** there once the original host
+recovers. If rescue only worked one way (`source → target` but not back), the evacuated workload could land on
+the target and then have nowhere to return to, stranding it. Insisting the link work in *both* directions
+guarantees the partition can be restored: after the evacuation, the VMs can fail back off the target just as
+they failed onto it, so the partition returns to a fully failover-able state rather than degrading with each
+event.
+
+```mermaid
+flowchart LR
+ subgraph dir["rescue links from Step 1"]
+ direction LR
+ a1((a)) --> b1((b))
+ b1 --> a1
+ b1 --> c1((c))
+ c1 -. "no c→b
(c too small for b)" .-> b1
+ end
+ dir ==> mut
+ subgraph mut["only mutual pairs survive"]
+ direction LR
+ a2((a)) --- b2((b))
+ b2 -. "dropped:
works one way only" .- c2((c))
+ end
+ style c2 fill:#ffebee,stroke:#c62828
+```
+
+Within the mutual graph, a partition is a group where **everyone can rescue everyone**. Finding the largest
+such groups is a classic graph problem (finding *cliques*), and that is exactly what Cortex does — no hand
+tuning, the groups fall straight out of the mutual links.
+
+**Why groups, and why they can overlap.** Consider three hosts where `a`↔`b` and `b`↔`c` are mutual but `a`
+and `c` cannot rescue each other. Then `{a, b, c}` is *not* a valid partition — `a` and `c` don't fit the "all
+rescue all" rule. Instead there are two valid groups, `{a, b}` and `{b, c}`, and `b` sits in both. That overlap
+is expected here; Step 3 resolves it.
+
+**Empty hosts get folded in.** A host with no VMs (or one already reserved) has nothing to evacuate and so has
+no rescue links of its own — it would otherwise be a group of one. Such a host is **recruited** into a partition
+as long as **every** member of that partition can rescue onto it *and* the empty host is the same effective size
+as those members. The reachability check is one-directional on purpose: an empty host runs nothing, so it is
+assumed able to evacuate anywhere; only the inbound direction (`member → empty host`) is verified. But the empty
+host also becomes a *source* of the partition's eventual failover target, so it must itself be rescuable by its
+new peers. With no VMs it has no rescue links to prove this, so Cortex falls back to the same whole-host fit rule
+used in Step 1 and requires equal capacity (memory then CPU, rounded to GiB): an empty host larger than the
+partition could never be evacuated onto a smaller reserved target, so it is **not** recruited. If even one member
+can't reach it, or the sizes differ, the empty host stays on its own:
+
+This recruitment mechanism has a consequence worth calling out: **a partition is only ever discovered if at
+least one of its hosts runs a valid, placeable VM.** Rescue links are derived *from* running VMs (Step 1), so
+a group of hosts with no VMs at all produces no links, forms no clique, and recruits no empty hosts — it is
+invisible to the planner. Concretely, a HANA partition in which no HANA VM is currently running yields no
+rescue links, so no clique forms and no failover host is reserved for it. This is not a problem: a partition
+with no workload has nothing to evacuate, so there is no failover host to find and none is needed. The moment
+a VM lands on one of those hosts, the links reappear and the partition (with its empty peers recruited) comes
+back into existence.
+
+```mermaid
+flowchart TB
+ subgraph ok["empty host E is recruited"]
+ direction LR
+ a3((a)) -->|reaches| e1((E))
+ b3((b)) -->|reaches| e1
+ okr["every member of {a, b} reaches E
⇒ partition becomes {a, b, E}"]
+ end
+ subgraph no["empty host F stays alone"]
+ direction LR
+ a4((a)) -->|reaches| f1((F))
+ b4((b)) -. "b does NOT reach F" .-> f1
+ nor["not every member reaches F
⇒ F is left on its own"]
+ end
+ style nor fill:#ffebee,stroke:#c62828
+```
+
+The whole grouping is computed deterministically: given the same fleet, the same partitions come out every
+time, so reservations don't flap between passes for no reason.
+
+### Step 3 — Make the partitions disjoint
+
+Step 2 can leave a host in **two** partitions at once (like `b` above). But a host can only be reserved for one
+partition — reserving it for two would produce conflicting records about who it's meant to rescue, and one would
+silently overwrite the other. So before planning, Cortex forces the partitions apart: each shared host is kept
+in exactly **one** partition — the **smallest** one it belongs to (ties broken deterministically by the
+partition's first host name).
+
+Keeping the shared host in the *smallest* partition is what preserves coverage for as many partitions as
+possible. A small partition only survives if it keeps its shared hosts — losing one can drop it below the
+two-host minimum and collapse it entirely. A large partition, by contrast, usually stays well above that
+minimum even after giving a host up. So assigning the shared host to the smaller group lets *both* groups
+survive, whereas assigning it to the larger one would often strand the smaller group:
+
+```mermaid
+flowchart TB
+ subgraph before["overlapping groups from Step 2"]
+ p1["{a, b, c, d} (4 hosts)"]
+ p2["{c, e} (2 hosts)"]
+ end
+ before ==>|"shared host c stays with the SMALLER group"| after
+ subgraph after["disjoint partitions — both survive"]
+ q1["{a, b, d} ✓ shrank by one, still ≥ 2"]
+ q2["{c, e} ✓ keeps c, stays a valid pair"]
+ end
+ style q1 fill:#e8f5e9,stroke:#2e7d32
+ style q2 fill:#e8f5e9,stroke:#2e7d32
+```
+
+Had `c` instead been kept with the larger `{a, b, c, d}`, the pair `{c, e}` would have collapsed to the lone
+host `{e}` and been discarded (a lone host has no peer to fail over from), leaving `e` unprotected this pass
+for no reason other than how the overlap was resolved. Preferring the smaller partition removes that failure
+mode wherever the larger partition can spare the host — which, being larger, it usually can.
+
+**Edge case — coverage can still be lost when neither partition can spare the host.** The smallest-first rule
+does not make the problem vanish entirely: if a host is shared between two partitions that are *both* size two,
+whichever one loses it collapses to a lone host and is discarded. There is no assignment that saves both — the
+host is the only peer each has — so one partition is unavoidably stranded, and the deterministic tiebreak
+decides which. This is now the only way the disjoint step costs a host its failover target.
+
+### Step 4 — Pick the failover hosts in each partition
+
+For each partition, Cortex decides how many hosts to reserve as failover targets. It deliberately does **not**
+pick a fixed number: too few leaves no room to evacuate into, too many wastes capacity. The count scales with
+the partition size, under three rules:
+
+- **A ratio of the partition** (default `10%`), rounded **down**.
+- **A floor** (default `1`), so even a small partition keeps at least one target.
+- **Never all of them** — at least one host must be left to actually run workload, so a partition is never
+ fully reserved. And a partition of fewer than two hosts reserves nothing at all: there would be no peer for a
+ reserved host to rescue.
+
+Worked through for the defaults, the interplay is easier to see than to state:
+
+| partition size | 10 %, rounded down | after the floor of 1 | after "leave ≥ 1 running" | **reserved** |
+|---:|---:|---:|---:|---:|
+| 1 | — | — | — | **0** (no peer) |
+| 2 | 0 | 1 | 1 | **1** |
+| 10 | 1 | 1 | 9 | **1** |
+| 15 | 1 | 1 | 14 | **1** |
+| 25 | 2 | 2 | 24 | **2** |
+
+The floor dominates small partitions; the ratio only starts to matter once a partition is large enough that
+10 % of it exceeds 1 (here, around 20 hosts).
+
+**Which** hosts become the targets comes down to the *order* in which the partition's hosts are considered.
+Cortex builds one ordered slice of candidates and then simply takes the first `reserved`-many from it. The
+order is what encodes the policy — and it is built to **retain existing reservations first and only create new
+ones when the count demands it**:
+
+```
+candidate order ──────────────────────────────────────────────────────────────▶
+┌───────────────────────────────┬───────────────────────────────────────────────┐
+│ already-reserved hosts │ not-yet-reserved hosts │
+│ (this partition), by name │ by ascending workload (cheapest to empty) │
+├───────────────────────────────┼───────────────────────────────────────────────┤
+│ R1 R2 R3 … │ N1 N2 N3 N4 … │
+└───────────────────────────────┴───────────────────────────────────────────────┘
+ └──────────── take the first `reserved` of these ────────────┘
+ kept reservations are consumed before any new one is created
+```
+
+Reading the slice left to right:
+
+- **Already-reserved hosts come first, so they stay reserved.** A host being prepared as a failover target may
+ have VMs mid-way through being moved off it; flipping the designation each pass would throw that work away.
+ Because they sit at the front, the planner only reaches a *new* host once every existing reservation has
+ already been re-chosen. If there are *more* already-reserved hosts than the count needs, the surplus falls
+ past the cut-off and is released back into normal service; if there are fewer, the remainder is filled from
+ the new hosts.
+- **New targets, when needed, are the cheapest to empty.** The not-yet-reserved hosts are ordered by ascending
+ workload, so the first new host the planner reaches is the one whose VMs are cheapest to migrate away.
+
+So in the common steady state where the right number of hosts is already reserved, the cut-off lands exactly
+at the boundary between the two groups: every existing reservation is retained and no new one is created. New
+reservations only appear when the count grows or an existing target leaves the partition.
+
+Each chosen host is then reserved in full, recording the other partition members as the hosts it stands ready
+to rescue.
+
+## The reservation lifecycle
+
+A failover host reservation blocks its target host from receiving ordinary placements, so the host stays empty
+and available as a landing zone. Over its life a reservation moves through these stages:
+
+```mermaid
+stateDiagram-v2
+ [*] --> Designated: host chosen as a failover target
+ Designated --> Active: all VMs moved away, host now empty
+ Designated --> Failed: draining fails or takes too long
+ Failed --> Designated: cooldown elapses, eligible again
+ Active --> InUse: a gang evacuation lands here
+ InUse --> Used: evacuation complete → host returns to service
+ Active --> Revoked: no longer part of any partition
+ Designated --> Revoked: no longer part of any partition
+ Used --> [*]
+ Revoked --> [*]
+```
+
+- **Designated** — the host has been chosen as a target but may still have VMs that need draining off it. No
+ *new* VMs are placed here. It stays in this stage until the host is empty.
+- **Active** — the host is empty and ready to absorb an evacuation.
+- **Failed** — draining failed repeatedly or took too long; the host is held back from being re-chosen for a
+ cooldown period.
+- **In Use** — a gang evacuation onto this host is underway (and may absorb further failures until it
+ completes).
+- **Revoked** — the host no longer belongs to any partition (for example it became unschedulable), so the
+ reservation is dropped.
+- **Used** — the failover event finished; the host becomes a regular host again.
+
+> [!NOTE]
+> Treat this diagram as the *intended progression*, not a strict state machine. The stages are tracked
+> independently rather than as a single current-state field, so the arrows show the normal path rather than the
+> only possible transitions.
+
+## Configuration
+
+Enable the controller and configure it under the nested `failover.hosts` key:
+
+```yaml
+enabledControllers:
+ - failover-host-reservations-controller
+# ...
+failover:
+ hosts:
+ ratio: 0.1 # fraction of each partition to reserve (0, 1]
+ minimum: 1 # floor per partition, > 0
+ generationInterval: 5m # how often to recompute, ≥ 1m
+ externalSchedulerURL: "http://localhost:8080/scheduler/nova/external"
+```
+
+> [!NOTE]
+> The settings live under `failover.hosts` as a deliberate example of grouping a controller's config under its
+> own key rather than adding more flat top-level keys. This is independent of the (deprecated) slot-based
+> failover controller configured alongside it. Bad values for `ratio`, `minimum`, or the interval are rejected
+> at startup.
+
+Like the other reservation kinds, failover host reservations make a host look fuller to the pipeline than its
+raw capacity suggests — here by reserving the host **entirely**. The next page turns from the *kinds* of
+reserved capacity to the concurrency problem all of them share: how the scheduler keeps parallel requests from
+double-spending the same slot.
+
+## Why does this host have no failover target?
+
+Because the graph is narrowed in several steps, a host can end up unprotected for quite different reasons. This
+table maps the symptom back to the step that caused it:
+
+| Symptom | Why | Step |
+|---|---|---|
+| host produces no rescue links | its own VMs can't share any single target | 1 · Discover rescue links |
+| a target was discarded | it's smaller than the source host | 1 · Discover rescue links |
+| a link doesn't lead to a partition | it only works one way, not both | 2 · Group into partitions |
+| an empty host is left on its own | not *every* member of a partition can reach it | 2 · Group into partitions |
+| a host loses its partition | it was shared between two size-two partitions, and the one that lost it collapsed | 3 · Make partitions disjoint |
+| a partition reserves nothing | it has fewer than two hosts | 4 · Pick failover hosts |
+
+The controller logs each of these decisions as it makes them, so a host's fate through the steps can be traced
+pass by pass.
+
+## Next
+
+[Prev: Failover reservations (deprecated)](03-failover-reservations.md) · [Next: Concurrency and in-flight reservations »](05-concurrency-and-in-flight-reservations.md)
diff --git a/docs/03-reservations-and-inventory/04-concurrency-and-in-flight-reservations.md b/docs/03-reservations-and-inventory/05-concurrency-and-in-flight-reservations.md
similarity index 98%
rename from docs/03-reservations-and-inventory/04-concurrency-and-in-flight-reservations.md
rename to docs/03-reservations-and-inventory/05-concurrency-and-in-flight-reservations.md
index f71147d49..9f54deeec 100644
--- a/docs/03-reservations-and-inventory/04-concurrency-and-in-flight-reservations.md
+++ b/docs/03-reservations-and-inventory/05-concurrency-and-in-flight-reservations.md
@@ -133,4 +133,4 @@ Placement API shim.
## Next
-[Prev: Failover reservations](03-failover-reservations.md) · [Next: The Placement API shim »](05-placement-api-shim.md)
+[Prev: Failover host reservations](04-failover-host-reservations.md) · [Next: The Placement API shim »](06-placement-api-shim.md)
diff --git a/docs/03-reservations-and-inventory/05-placement-api-shim.md b/docs/03-reservations-and-inventory/06-placement-api-shim.md
similarity index 99%
rename from docs/03-reservations-and-inventory/05-placement-api-shim.md
rename to docs/03-reservations-and-inventory/06-placement-api-shim.md
index e8bde2417..2fd8d38ff 100644
--- a/docs/03-reservations-and-inventory/05-placement-api-shim.md
+++ b/docs/03-reservations-and-inventory/06-placement-api-shim.md
@@ -195,4 +195,4 @@ facts: the knowledge database.
## Next
-[Prev: Concurrency and in-flight reservations](04-concurrency-and-in-flight-reservations.md) · [Next: Chapter 4 — The knowledge database »](../04-knowledge-database/readme.md)
+[Prev: Concurrency and in-flight reservations](05-concurrency-and-in-flight-reservations.md) · [Next: Chapter 4 — The knowledge database »](../04-knowledge-database/readme.md)
diff --git a/docs/03-reservations-and-inventory/readme.md b/docs/03-reservations-and-inventory/readme.md
index 87724bb24..b1069b479 100644
--- a/docs/03-reservations-and-inventory/readme.md
+++ b/docs/03-reservations-and-inventory/readme.md
@@ -13,8 +13,9 @@ Cortex intends to become the authority for inventory itself.
The chapter opens with an overview of the three reservation kinds, then goes deep on committed
resources (customer commitments synced from Limes, plus the quota and capacity controllers that
-support them) and failover reservations (headroom for evacuating workloads when a host fails). It
-then covers the concurrency problem all reservations share — keeping parallel scheduling requests from
+support them) and failover reservations (headroom for evacuating workloads when a host fails) — both
+the deprecated slot-based system and the whole-host system that replaces it. It then covers the
+concurrency problem all reservations share — keeping parallel scheduling requests from
double-spending the same slot — and closes with the **Placement API shim**, the
OpenStack-Placement-compatible front through which Cortex will eventually serve inventory for the hosts
it understands.
@@ -23,9 +24,10 @@ it understands.
1. [Reservations overview](01-reservations-overview.md) — committed, failover, and in-flight reservations.
2. [Committed-resource reservations and Limes quota/capacity](02-committed-resource-reservations.md) — commitments, `FlavorGroupCapacity`, `ProjectQuota`, and the LIQUID API.
-3. [Failover reservations for KVM HA](03-failover-reservations.md) — shared, best-effort evacuation headroom.
-4. [Concurrency and in-flight reservations](04-concurrency-and-in-flight-reservations.md) — how parallel requests avoid double-spending capacity.
-5. [The Placement API shim](05-placement-api-shim.md) — passthrough today, KVM-backed inventory planned.
+3. [Failover reservations for KVM HA](03-failover-reservations.md) — shared, best-effort evacuation headroom (**deprecated**).
+4. [Failover host reservations](04-failover-host-reservations.md) — whole-host evacuation targets grouped into discovered failover partitions.
+5. [Concurrency and in-flight reservations](05-concurrency-and-in-flight-reservations.md) — how parallel requests avoid double-spending capacity.
+6. [The Placement API shim](06-placement-api-shim.md) — passthrough today, KVM-backed inventory planned.
## Next
diff --git a/docs/04-knowledge-database/05-infrastructure-dashboard.md b/docs/04-knowledge-database/05-infrastructure-dashboard.md
index 8cef52c1d..582a34d21 100644
--- a/docs/04-knowledge-database/05-infrastructure-dashboard.md
+++ b/docs/04-knowledge-database/05-infrastructure-dashboard.md
@@ -38,7 +38,7 @@ The dashboard definitions are development/operations tooling, deliberately kept
`cortex-compute-kvm-details.json` (the compute KVM boards),
`cortex-compute-vmware-overview.json` and `cortex-compute-vmware-details.json` (the compute VMware
boards), `cortex-global-compute-capacity.json`, `cortex-placement-shim-status.json` (the
- [shim](../03-reservations-and-inventory/05-placement-api-shim.md) status board),
+ [shim](../03-reservations-and-inventory/06-placement-api-shim.md) status board),
`cortex-project-flavor-usage.json`, `cortex-status-global.json`, `cortex-status.json`, and
`project.json`.
diff --git a/docs/06-cortex-library/09-supervisor.md b/docs/06-cortex-library/09-supervisor.md
index 55d9654e3..09af07255 100644
--- a/docs/06-cortex-library/09-supervisor.md
+++ b/docs/06-cortex-library/09-supervisor.md
@@ -6,7 +6,7 @@
# The `supervisor` package
-The [Placement API shim](../03-reservations-and-inventory/05-placement-api-shim.md) sits on a hot path:
+The [Placement API shim](../03-reservations-and-inventory/06-placement-api-shim.md) sits on a hot path:
schedulers hit it constantly, and it forwards most of that traffic to upstream OpenStack Placement. That
request path must stay up *even when the shim's Kubernetes controller-manager cannot* — for instance when
it loses connectivity to a local or remote apiserver. `pkg/shim/supervisor` is what decouples the two, so
@@ -56,7 +56,7 @@ So a lost apiserver connection becomes a *degraded* state (the manager loops, re
> by the pod going unready.
The package is deliberately shim-agnostic so future shims can reuse it; today the
-[Placement API shim](../03-reservations-and-inventory/05-placement-api-shim.md) is its user, enabled by
+[Placement API shim](../03-reservations-and-inventory/06-placement-api-shim.md) is its user, enabled by
`--self-heal` (on by default). It is the resilience half of the shim's design: the shim page explains
*what* the shim serves, this package explains *why a control-plane hiccup does not stop it serving*. That
closes the tour of `pkg/` — and the book. From here, the code and its generated CRD/godoc reference are
diff --git a/docs/readme.md b/docs/readme.md
index 510e91707..b2d959ed0 100644
--- a/docs/readme.md
+++ b/docs/readme.md
@@ -55,8 +55,8 @@ and `cmd/shim`, and the Helm defaults under `helm/`.
1. [Reservations overview](03-reservations-and-inventory/01-reservations-overview.md)
2. [Committed-resource reservations and Limes quota/capacity](03-reservations-and-inventory/02-committed-resource-reservations.md)
3. [Failover reservations for KVM HA](03-reservations-and-inventory/03-failover-reservations.md)
-4. [Concurrency and in-flight reservations](03-reservations-and-inventory/04-concurrency-and-in-flight-reservations.md)
-5. [The Placement API shim](03-reservations-and-inventory/05-placement-api-shim.md)
+4. [Concurrency and in-flight reservations](03-reservations-and-inventory/05-concurrency-and-in-flight-reservations.md)
+5. [The Placement API shim](03-reservations-and-inventory/06-placement-api-shim.md)
### [4 — The knowledge database](04-knowledge-database/readme.md)
@@ -91,7 +91,7 @@ and `cmd/shim`, and the Helm defaults under `helm/`.
chapters for your domain, and keep [Monitoring Cortex](04-knowledge-database/06-monitoring.md) nearby.
- **Integrators** wire Cortex into an existing scheduler — start with
[The scheduling engine](02-external-scheduler-api/01-the-scheduling-engine.md) and the
- [Placement API shim](03-reservations-and-inventory/05-placement-api-shim.md).
+ [Placement API shim](03-reservations-and-inventory/06-placement-api-shim.md).
- **Developers** extend Cortex — read Chapter 1, then
[The Cortex library](06-cortex-library/readme.md) and
[Extend Cortex](02-external-scheduler-api/07-extending-cortex.md).
diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml
index 1a4377d1d..e6f6f5f87 100644
--- a/helm/bundles/cortex-nova/templates/alerts.yaml
+++ b/helm/bundles/cortex-nova/templates/alerts.yaml
@@ -1029,5 +1029,107 @@ spec:
controller logs (filter on "periodic reconciliation completed" and
"processed VM failover reservations") to identify the affected VMs and the reason
their reservations cannot be created.
+ - alert: CortexNovaFailoverHostCalculationStalled
+ # The failover host controller recomputes its desired reservations on a
+ # periodic calculation loop. runs_total only increments when a pass
+ # completes, so increase()==0 over a window several times the loop interval
+ # means the loop is wedged, panicking, or every pass is aborting before it
+ # completes — in all cases the failover host reservations are no longer
+ # being maintained and will silently drift from the cluster. A pod that is
+ # down is covered by CortexNovaSchedulingDown, so require the series to
+ # still be exported (the metric is present but flat).
+ expr: |
+ increase(cortex_failover_host_calculation_runs_total{service="cortex-nova-metrics"}[30m]) == 0
+ for: 30m
+ labels:
+{{- if .Values.alerts.persesDashboardBaseURL }}
+ persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex_status?var-service=cortex-nova-metrics"
+{{- end }}
+ context: failover-hosts
+ service: cortex
+ severity: warning
+ support_group: workload-management
+ playbook: ""
+ annotations:
+ summary: "Failover host calculation loop is not completing passes"
+ description: >
+ The failover host controller has not completed a calculation-loop pass in
+ the last 30 minutes. Its failover host reservations are no longer being
+ recomputed and will drift from the actual cluster state. Check the
+ failover-host-controller logs for a stuck loop or repeated aborts.
+ - alert: CortexNovaFailoverHostCalculationAborting
+ # Each calculation pass aborts (without touching the desired state) when the
+ # cluster snapshot is inconsistent (reason="collect") or the external
+ # scheduler lookup fails (reason="pathfinding"). A sustained abort rate means
+ # no pass is producing a fresh desired state, so reservations go stale.
+ expr: rate(cortex_failover_host_calculation_aborts_total{service="cortex-nova-metrics"}[15m]) > 0
+ for: 30m
+ labels:
+{{- if .Values.alerts.persesDashboardBaseURL }}
+ persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex_status?var-service=cortex-nova-metrics"
+{{- end }}
+ context: failover-hosts
+ service: cortex
+ severity: warning
+ support_group: workload-management
+ playbook: ""
+ annotations:
+ summary: "Failover host calculation loop keeps aborting (reason=`{{ "{{" }} $labels.reason {{ "}}" }}`)"
+ description: >
+ The failover host calculation loop has been aborting before producing a
+ desired state for over 30 minutes (reason=`{{ "{{" }} $labels.reason {{ "}}" }}`).
+ reason="collect" means the cluster snapshot was inconsistent (e.g. a VM on an
+ unknown hypervisor, or a hypervisor without an availability zone);
+ reason="pathfinding" means the external scheduler lookup failed. Failover host
+ reservations are not being refreshed while this persists.
+ - alert: CortexNovaFailoverHostReconcileErrors
+ # The reconciler failed to create, update or delete failover host
+ # reservations. A sustained error rate means the desired reservations
+ # computed by the loop cannot be applied to the cluster.
+ expr: rate(cortex_failover_host_reservations_errors_total{service="cortex-nova-metrics"}[15m]) > 0
+ for: 30m
+ labels:
+{{- if .Values.alerts.persesDashboardBaseURL }}
+ persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex_status?var-service=cortex-nova-metrics"
+{{- end }}
+ context: failover-hosts
+ service: cortex
+ severity: warning
+ support_group: workload-management
+ playbook: ""
+ annotations:
+ summary: "Failover host reservations cannot be reconciled (op=`{{ "{{" }} $labels.op {{ "}}" }}`)"
+ description: >
+ The failover host reconciler has been failing to {{ "{{" }} $labels.op {{ "}}" }}
+ reservations for over 30 minutes. The desired failover host reservations computed
+ by the calculation loop are not being applied, so actual reservations drift from
+ the desired set. Check the failover-host-controller-reconcile logs and the
+ Reservation objects for the affected hosts.
+ - alert: CortexNovaFailoverHostSchedulerErrors
+ # The path finder asks the external scheduler, per VM, where each host's
+ # workload could evacuate. A high fraction of failed requests (transport
+ # errors or non-2xx) makes every calculation pass abort, so this is the root
+ # cause behind a persistent CortexNovaFailoverHostCalculationAborting with
+ # reason="pathfinding".
+ expr: |
+ (sum(rate(cortex_failover_host_scheduler_requests_total{service="cortex-nova-metrics",status!~"2.."}[15m])))
+ / (sum(rate(cortex_failover_host_scheduler_requests_total{service="cortex-nova-metrics"}[15m]))) > 0.1
+ for: 15m
+ labels:
+{{- if .Values.alerts.persesDashboardBaseURL }}
+ persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex_status?var-service=cortex-nova-metrics"
+{{- end }}
+ context: failover-hosts
+ service: cortex
+ severity: warning
+ support_group: workload-management
+ playbook: ""
+ annotations:
+ summary: "Failover host path finder scheduler error rate >10%"
+ description: >
+ More than 10% of the external scheduler requests issued by the failover host
+ path finder are failing (transport errors or non-2xx responses). The path finder
+ cannot determine valid evacuation targets, so failover partitions and reservations
+ cannot be recomputed. Check that the external scheduler is reachable and healthy.
{{- end }}
{{- end }}
diff --git a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml
index 7fc4f0223..8f2eb52fb 100644
--- a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml
+++ b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml
@@ -15,6 +15,20 @@ spec:
---
apiVersion: cortex.cloud/v1alpha1
kind: KPI
+metadata:
+ name: failover-partition-graph
+spec:
+ schedulingDomain: nova
+ impl: failover_partition_graph_kpi
+ description: |
+ This KPI exposes the failover partition graph derived from the hypervisors
+ and failover host reservations: nodes (cortex_failover_partition_graph_node)
+ are the hypervisors (value 1 when the hypervisor has a failover path, 0 when
+ it has none), edges (cortex_failover_partition_graph_edge) are the
+ source->target evacuation paths within each partition.
+---
+apiVersion: cortex.cloud/v1alpha1
+kind: KPI
metadata:
name: kvm-project-utilization
spec:
diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml
index 28b643566..3fcd58c96 100644
--- a/helm/bundles/cortex-nova/values.yaml
+++ b/helm/bundles/cortex-nova/values.yaml
@@ -185,6 +185,7 @@ cortex-scheduling-controllers:
- hypervisor-overcommit-controller
- committed-resource-reservations-controller
- failover-reservations-controller
+ - failover-host-reservations-controller
- quota-controller
- capacity-controller
enabledTasks:
@@ -358,6 +359,38 @@ cortex-scheduling-controllers:
# A zero value disables pruning.
maxReservationDeletesPerRun: 20
+ # --- Failover host reservations controller ---
+ # Keeps whole compute hosts reserved as gang-evacuation targets: it groups
+ # hosts into "failover partitions" (sets of hosts that can all be evacuated
+ # onto one another, discovered from the scheduler) and keeps a share of each
+ # partition empty and reserved, so when a host fails its entire workload can
+ # be moved onto a reserved host in one step. This is independent of the
+ # flavor-based failover controller configured above (schedulerURL,
+ # reconcileInterval, flavorFailoverRequirements, ...).
+ #
+ # Nested under `failover.hosts` as a deliberate role model for grouping a
+ # controller's settings under its own key, rather than adding yet more flat
+ # top-level keys. Other controller configs should be refactored toward this.
+ failover:
+ hosts:
+ # Fraction of each partition's hosts to keep reserved as failover
+ # targets, rounded down (e.g. 0.1 of a 10-host partition reserves 1
+ # host). Must be within (0, 1]. Combined with `minimum` below, which
+ # sets a floor so even small partitions keep some headroom.
+ ratio: 0.1
+ # Minimum number of failover hosts to reserve per partition, applied even
+ # when `ratio` would round down to fewer. Must be greater than 0. The
+ # controller never reserves so many that a partition is left with no host
+ # to run workload (it always keeps at least one source host).
+ minimum: 1
+ # How often the failover partitions are recalculated and the reservations
+ # updated to match. Must be at least 1m.
+ generationInterval: 5m
+ # URL of the nova external scheduler API, used to discover which hosts a
+ # given host's workload could be evacuated onto. Typically the same
+ # scheduler endpoint the flavor-based failover controller uses.
+ externalSchedulerURL: "http://localhost:8080/scheduler/nova/external"
+
cortex-knowledge-controllers:
<<: *cortex
namePrefix: cortex-nova-knowledge
diff --git a/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml b/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml
index a9bdceebf..2e6aa319a 100644
--- a/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml
+++ b/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml
@@ -172,6 +172,14 @@ spec:
FailoverHostReservation contains fields specific to failover host reservations.
Only used when Type is FailoverHostReservation.
properties:
+ partitionDescription:
+ description: |-
+ PartitionDescription is a short, human-readable label describing what
+ distinguishes the hosts of this failover partition from those of other
+ partitions, e.g. "spec.aggregates=[hana]". It carries no scheduling
+ semantics and exists purely to make a reservation understandable at a
+ glance. See characterizePartitions in the failoverhost controller.
+ type: string
sourceHosts:
description: |-
SourceHosts is a list of hosts inside the same failover partition from
diff --git a/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go b/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go
new file mode 100644
index 000000000..23f8aeae0
--- /dev/null
+++ b/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go
@@ -0,0 +1,152 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package compute
+
+import (
+ "context"
+ "maps"
+ "strconv"
+ "time"
+
+ "github.com/cobaltcore-dev/cortex/api/v1alpha1"
+ "github.com/cobaltcore-dev/cortex/internal/knowledge/db"
+ "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins"
+ "github.com/cobaltcore-dev/cortex/pkg/conf"
+ hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+ vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1"
+ "github.com/prometheus/client_golang/prometheus"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var failoverPartitionGraphKPILogger = ctrl.Log.WithName("failover-partition-graph-kpi")
+
+// KPI exposing the failover partition graph derived from the hypervisors and
+// failover host reservations present in the cluster. The graph has two series:
+//
+// - cortex_failover_partition_graph_node{host,az,memory,cpu,vms} — one per
+// hypervisor, whether or not it participates in any failover reservation,
+// carrying the hypervisor's availability zone, total capacity and the number
+// of vms currently running on it. The value is 1 when the hypervisor has at
+// least one failover path (it is the source or target of an edge) and 0 when
+// it has none.
+// - cortex_failover_partition_graph_edge{source,target} 1 — one per directed
+// evacuation path: every SourceHost of a failover host reservation may fail
+// over onto the reservation's TargetHost.
+type FailoverPartitionGraphKPI struct {
+ // Common base for all KPIs that provides standard functionality.
+ plugins.BaseKPI[struct{}]
+
+ node *prometheus.Desc
+ edge *prometheus.Desc
+}
+
+func (FailoverPartitionGraphKPI) GetName() string { return "failover_partition_graph_kpi" }
+
+// Initialize the KPI.
+func (k *FailoverPartitionGraphKPI) Init(db *db.DB, client client.Client, opts conf.RawOpts) error {
+ if err := k.BaseKPI.Init(db, client, opts); err != nil {
+ return err
+ }
+ k.node = prometheus.NewDesc(
+ "cortex_failover_partition_graph_node",
+ "A hypervisor in the failover partition graph",
+ []string{"host", "az", "memory", "cpu", "vms"},
+ nil,
+ )
+ k.edge = prometheus.NewDesc(
+ "cortex_failover_partition_graph_edge",
+ "A directed evacuation path (source can fail over onto target) in the failover partition graph",
+ []string{"source", "target"},
+ nil,
+ )
+ return nil
+}
+
+// Conform to the prometheus collector interface by providing the descriptors.
+func (k *FailoverPartitionGraphKPI) Describe(ch chan<- *prometheus.Desc) {
+ ch <- k.node
+ ch <- k.edge
+}
+
+// Collect the failover partition graph metrics: one node per hypervisor and one
+// edge per failover evacuation path.
+func (k *FailoverPartitionGraphKPI) Collect(ch chan<- prometheus.Metric) {
+ // Bound the list calls so a slow API server can't hang the Prometheus scrape
+ // indefinitely; if they fail we log so the disappearance of the metrics is
+ // not silent.
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ reservationList := &v1alpha1.ReservationList{}
+ if err := k.Client.List(ctx, reservationList); err != nil {
+ failoverPartitionGraphKPILogger.Error(err, "Failed to list reservations")
+ return
+ }
+ // A (source,target) pair could in principle repeat across reservations, and
+ // Prometheus rejects duplicate series within a single scrape, so collapse the
+ // edges into a set first. Track which hosts take part in any path so nodes
+ // without a failover path can be reported with value 0.
+ type edge struct{ source, target string }
+ emittedEdges := map[edge]struct{}{}
+ hasPath := map[string]struct{}{}
+ for _, r := range reservationList.Items {
+ if r.Spec.Type != v1alpha1.ReservationTypeFailoverHost {
+ continue
+ }
+ target := r.Spec.TargetHost
+ if target == "" || r.Spec.FailoverHostReservation == nil {
+ continue
+ }
+ for _, source := range r.Spec.FailoverHostReservation.SourceHosts {
+ hasPath[source] = struct{}{}
+ hasPath[target] = struct{}{}
+ e := edge{source: source, target: target}
+ if _, ok := emittedEdges[e]; ok {
+ continue
+ }
+ emittedEdges[e] = struct{}{}
+ ch <- prometheus.MustNewConstMetric(
+ k.edge, prometheus.GaugeValue, 1, source, target,
+ )
+ }
+ }
+ hypervisorList := &hv1.HypervisorList{}
+ if err := k.Client.List(ctx, hypervisorList); err != nil {
+ failoverPartitionGraphKPILogger.Error(err, "Failed to list hypervisors")
+ return
+ }
+ // Count the vms running on each host so the node series can expose it. A vm
+ // Nova hasn't placed yet has no host and is skipped (it also nil-panics on
+ // Status.Nova otherwise).
+ vmList := &vmv1alpha1.VirtualMachineList{}
+ if err := k.Client.List(ctx, vmList); err != nil {
+ failoverPartitionGraphKPILogger.Error(err, "Failed to list virtual machines")
+ return
+ }
+ vmsByHost := map[string]int{}
+ for _, vm := range vmList.Items {
+ if vm.Status.Nova == nil || vm.Status.Nova.Host == "" {
+ continue
+ }
+ vmsByHost[vm.Status.Nova.Host]++
+ }
+ for _, hv := range hypervisorList.Items {
+ capacity := map[hv1.ResourceName]resource.Quantity{}
+ maps.Copy(capacity, hv.Status.Capacity) // base capacity
+ maps.Copy(capacity, hv.Status.EffectiveCapacity)
+ memory := capacity[hv1.ResourceMemory]
+ cpu := capacity[hv1.ResourceCPU]
+ value := 0.0
+ if _, ok := hasPath[hv.Name]; ok {
+ value = 1
+ }
+ ch <- prometheus.MustNewConstMetric(
+ k.node, prometheus.GaugeValue, value,
+ hv.Name, hv.Labels[corev1.LabelTopologyZone],
+ memory.String(), cpu.String(), strconv.Itoa(vmsByHost[hv.Name]),
+ )
+ }
+}
diff --git a/internal/knowledge/kpis/plugins/compute/failover_partition_graph_test.go b/internal/knowledge/kpis/plugins/compute/failover_partition_graph_test.go
new file mode 100644
index 000000000..f95592bb9
--- /dev/null
+++ b/internal/knowledge/kpis/plugins/compute/failover_partition_graph_test.go
@@ -0,0 +1,200 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package compute
+
+import (
+ "testing"
+
+ "github.com/cobaltcore-dev/cortex/api/v1alpha1"
+ "github.com/cobaltcore-dev/cortex/pkg/conf"
+ hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+ vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1"
+ "github.com/prometheus/client_golang/prometheus"
+ dto "github.com/prometheus/client_model/go"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func failoverHostReservation(name, target string, sources ...string) v1alpha1.Reservation {
+ return v1alpha1.Reservation{
+ ObjectMeta: v1.ObjectMeta{Name: name},
+ Spec: v1alpha1.ReservationSpec{
+ Type: v1alpha1.ReservationTypeFailoverHost,
+ SchedulingDomain: "nova",
+ TargetHost: target,
+ FailoverHostReservation: &v1alpha1.FailoverHostReservationSpec{
+ SourceHosts: sources,
+ },
+ },
+ }
+}
+
+func hypervisor(name, az string) hv1.Hypervisor {
+ return hv1.Hypervisor{
+ ObjectMeta: v1.ObjectMeta{Name: name, Labels: map[string]string{corev1.LabelTopologyZone: az}},
+ Status: hv1.HypervisorStatus{
+ Capacity: map[hv1.ResourceName]resource.Quantity{
+ hv1.ResourceMemory: resource.MustParse("256Gi"),
+ hv1.ResourceCPU: resource.MustParse("64"),
+ },
+ },
+ }
+}
+
+// virtualMachine builds a vm Nova reports as placed on the given host.
+func virtualMachine(name, host string) vmv1alpha1.VirtualMachine {
+ return vmv1alpha1.VirtualMachine{
+ ObjectMeta: v1.ObjectMeta{Name: name},
+ Status: vmv1alpha1.VMStatus{Nova: &vmv1alpha1.NovaStatus{Host: host}},
+ }
+}
+
+func TestFailoverPartitionGraphKPI_GetName(t *testing.T) {
+ kpi := &FailoverPartitionGraphKPI{}
+ if name := kpi.GetName(); name != "failover_partition_graph_kpi" {
+ t.Errorf("expected name failover_partition_graph_kpi, got %q", name)
+ }
+}
+
+func TestFailoverPartitionGraphKPI_Describe(t *testing.T) {
+ kpi := &FailoverPartitionGraphKPI{}
+ if err := kpi.Init(nil, nil, conf.NewRawOpts("{}")); err != nil {
+ t.Fatalf("expected no error, got %v", err)
+ }
+ ch := make(chan *prometheus.Desc, 2)
+ kpi.Describe(ch)
+ close(ch)
+ count := 0
+ for range ch {
+ count++
+ }
+ if count != 2 {
+ t.Errorf("expected 2 descriptors, got %d", count)
+ }
+}
+
+func TestFailoverPartitionGraphKPI_Collect(t *testing.T) {
+ scheme, err := v1alpha1.SchemeBuilder.Build()
+ if err != nil {
+ t.Fatalf("expected no error, got %v", err)
+ }
+ if err := hv1.AddToScheme(scheme); err != nil {
+ t.Fatalf("expected no error, got %v", err)
+ }
+ if err := vmv1alpha1.AddToScheme(scheme); err != nil {
+ t.Fatalf("expected no error, got %v", err)
+ }
+
+ // Two failover-host reservations sharing source host h1 (so the edge for h1
+ // is emitted to both targets), plus a non-failover reservation that must be
+ // ignored.
+ resA := failoverHostReservation("failover-host-t1", "t1", "h1", "h2")
+ resB := failoverHostReservation("failover-host-t2", "t2", "h1", "h3")
+ other := v1alpha1.Reservation{
+ ObjectMeta: v1.ObjectMeta{Name: "r-other"},
+ Spec: v1alpha1.ReservationSpec{
+ Type: v1alpha1.ReservationTypeInFlight,
+ SchedulingDomain: "nova",
+ TargetHost: "ignored",
+ },
+ }
+ // The hypervisors backing the graph. h-idle takes part in no reservation, so
+ // its node is reported with value 0.
+ t1 := hypervisor("t1", "az-a")
+ t2 := hypervisor("t2", "az-b")
+ h1 := hypervisor("h1", "az-a")
+ h2 := hypervisor("h2", "az-a")
+ h3 := hypervisor("h3", "az-b")
+ idle := hypervisor("h-idle", "az-a")
+
+ // Two vms on h1 and one on t2; every other host (incl. idle) has none, and an
+ // unplaced vm must not be counted.
+ vm1 := virtualMachine("vm1", "h1")
+ vm2 := virtualMachine("vm2", "h1")
+ vm3 := virtualMachine("vm3", "t2")
+ vmUnplaced := virtualMachine("vm-unplaced", "")
+
+ client := fake.NewClientBuilder().WithScheme(scheme).
+ WithObjects(&resA, &resB, &other, &t1, &t2, &h1, &h2, &h3, &idle,
+ &vm1, &vm2, &vm3, &vmUnplaced).Build()
+
+ kpi := &FailoverPartitionGraphKPI{}
+ if err := kpi.Init(nil, client, conf.NewRawOpts("{}")); err != nil {
+ t.Fatalf("expected no error, got %v", err)
+ }
+
+ ch := make(chan prometheus.Metric, 32)
+ kpi.Collect(ch)
+ close(ch)
+
+ var nodes, edges int
+ nodeAZ := map[string]string{}
+ nodeValue := map[string]float64{}
+ nodeVMs := map[string]string{}
+ edgeSet := map[string]bool{}
+ for m := range ch {
+ var metric dto.Metric
+ if err := m.Write(&metric); err != nil {
+ t.Fatalf("failed to write metric: %v", err)
+ }
+ labels := map[string]string{}
+ for _, l := range metric.Label {
+ labels[l.GetName()] = l.GetValue()
+ }
+ switch m.Desc().String() {
+ case kpi.node.String():
+ nodes++
+ nodeAZ[labels["host"]] = labels["az"]
+ nodeValue[labels["host"]] = metric.GetGauge().GetValue()
+ nodeVMs[labels["host"]] = labels["vms"]
+ if labels["memory"] != "256Gi" || labels["cpu"] != "64" {
+ t.Errorf("node %s: unexpected resources memory=%q cpu=%q", labels["host"], labels["memory"], labels["cpu"])
+ }
+ case kpi.edge.String():
+ edges++
+ edgeSet[labels["source"]+"->"+labels["target"]] = true
+ default:
+ t.Errorf("unexpected metric descriptor %s", m.Desc().String())
+ }
+ }
+
+ // Nodes: one per hypervisor, including the idle one.
+ if nodes != 6 {
+ t.Errorf("expected 6 node metrics, got %d", nodes)
+ }
+ wantAZ := map[string]string{"t1": "az-a", "t2": "az-b", "h1": "az-a", "h2": "az-a", "h3": "az-b", "h-idle": "az-a"}
+ for host, az := range wantAZ {
+ if nodeAZ[host] != az {
+ t.Errorf("node %s: expected az %q, got %q", host, az, nodeAZ[host])
+ }
+ }
+ // Every hypervisor with a failover path reports 1; the idle one reports 0.
+ for _, host := range []string{"t1", "t2", "h1", "h2", "h3"} {
+ if nodeValue[host] != 1 {
+ t.Errorf("node %s: expected value 1, got %v", host, nodeValue[host])
+ }
+ }
+ if nodeValue["h-idle"] != 0 {
+ t.Errorf("node h-idle: expected value 0, got %v", nodeValue["h-idle"])
+ }
+ // The vms label counts placed vms per host: h1 has two, t2 has one, the rest
+ // zero. The unplaced vm is not counted anywhere.
+ wantVMs := map[string]string{"h1": "2", "t2": "1", "t1": "0", "h2": "0", "h3": "0", "h-idle": "0"}
+ for host, want := range wantVMs {
+ if nodeVMs[host] != want {
+ t.Errorf("node %s: expected vms %q, got %q", host, want, nodeVMs[host])
+ }
+ }
+ // Edges: h1->t1, h2->t1, h1->t2, h3->t2.
+ if edges != 4 {
+ t.Errorf("expected 4 edge metrics, got %d", edges)
+ }
+ for _, want := range []string{"h1->t1", "h2->t1", "h1->t2", "h3->t2"} {
+ if !edgeSet[want] {
+ t.Errorf("missing edge %s, got %v", want, edgeSet)
+ }
+ }
+}
diff --git a/internal/knowledge/kpis/supported_kpis.go b/internal/knowledge/kpis/supported_kpis.go
index 617d58008..992dd614a 100644
--- a/internal/knowledge/kpis/supported_kpis.go
+++ b/internal/knowledge/kpis/supported_kpis.go
@@ -20,6 +20,8 @@ var supportedKPIs = map[string]plugins.KPI{
"vm_commitments_kpi": &compute.VMCommitmentsKPI{},
"vm_faults_kpi": &compute.VMFaultsKPI{},
+ "failover_partition_graph_kpi": &compute.FailoverPartitionGraphKPI{},
+
"kvm_host_capacity_kpi": &infrastructure.KVMHostCapacityKPI{},
"kvm_project_utilization_kpi": &infrastructure.KVMProjectUtilizationKPI{},
"kvm_hana_stacking_kpi": &infrastructure.KVMHanaStackingKPI{},
diff --git a/internal/scheduling/reservations/failoverhost/controller.go b/internal/scheduling/reservations/failoverhost/controller.go
new file mode 100644
index 000000000..db2c46400
--- /dev/null
+++ b/internal/scheduling/reservations/failoverhost/controller.go
@@ -0,0 +1,586 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+// Package failoverhost keeps a fleet of OpenStack Nova compute hosts ready to
+// absorb the workload of a failed host.
+//
+// # What a failover host reservation is
+//
+// When a hypervisor fails, Nova has to evacuate every virtual machine running
+// on it onto other hosts. For that to succeed quickly and predictably there
+// must be somewhere to put those vms. This package keeps a fraction of every
+// group of interchangeable hosts empty and reserved, so a whole host's workload
+// can be gang-evacuated onto a reserved "failover host" in one move. A failover
+// host reservation (v1alpha1.Reservation of type FailoverHostReservation)
+// blocks a target host from receiving ordinary placements so it stays available
+// for this purpose. Its full lifecycle (Designated -> Active -> In Use, plus the
+// Failed and Revoked/Used terminal states) is documented on the
+// ReservationTypeFailoverHost constant in the v1alpha1 package.
+//
+// # Partitions
+//
+// Hosts are not interchangeable across the whole cloud: a HANA host, a
+// general-purpose host and an external-customer host can generally only ever
+// fail over within their own kind, because their workloads carry scheduling
+// constraints (flavors, image properties, aggregates) that only matching hosts
+// satisfy. We therefore group hosts into failover partitions: maximal sets of
+// hosts within which every host can be evacuated onto every other host. We do
+// not configure these groups by hand; we discover them from the scheduler's own
+// placement decisions, so the grouping always reflects the live constraints.
+//
+// # The two halves of the controller
+//
+// The controller has a producer and a consumer that communicate through the
+// desiredReservations map:
+//
+// - A periodic calculation loop (Start -> loop, see loop.go) recomputes the
+// desired set of failover host reservations from the current cluster state
+// on a fixed interval, and stores it as desiredReservations.
+// - An event-driven reconciler (Reconcile, below) brings the actual
+// reservations in the cluster in line with that desired set, creating,
+// updating and pruning Reservation objects.
+//
+// The loop never writes Reservation objects itself; it only recomputes the
+// desired state and nudges the reconciler. Keeping the expensive, external-call
+// heavy computation out of the reconcile path means reconciles stay cheap and
+// idempotent.
+//
+// # The calculation pipeline
+//
+// Each pass of the loop runs a three-stage pipeline, each stage a small
+// interface so it can be understood, tested and swapped in isolation:
+//
+// 1. PathFinder (paths.go) turns the cluster into the directed edges of a
+// failover graph: source -> target means "every vm on source can be
+// evacuated onto target, and target is big enough to hold them all at once".
+// 2. PartitionBuilder (partitions.go) clusters those edges into partitions
+// (maximal sets of mutually-evacuable hosts).
+// 3. ReservationPlanner (reservations.go) decides, per partition, which hosts
+// become failover targets, honouring the configured ratio and minimum.
+//
+// loop.go wires these together and documents the end-to-end flow.
+package failoverhost
+
+import (
+ "context"
+ "errors"
+ "maps"
+ "net/http"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/cobaltcore-dev/cortex/api/v1alpha1"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/event"
+ "sigs.k8s.io/controller-runtime/pkg/handler"
+ "sigs.k8s.io/controller-runtime/pkg/source"
+
+ hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+ vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/equality"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ ctrl "sigs.k8s.io/controller-runtime"
+ logf "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+// Config is the top-level config section this controller reads from the shared
+// cortex conf.json. The controller's own settings are nested under failover.hosts
+// so they sit alongside the (flavor-based) failover controller's settings without
+// colliding with the flat, top-level keys read from the same file.
+type Config struct {
+ Failover struct {
+ Hosts ControllerConfig `json:"hosts"`
+ } `json:"failover"`
+}
+
+// ControllerConfig contains configuration for failover host reservations.
+type ControllerConfig struct {
+ // Ratio is a global ratio that determines how many failover hosts inside
+ // a partition are created. For example, a ratio of 0.1 means that
+ // if a partition consists of 10 hosts, 1 failover host will be created.
+ // The number of resulting failover hosts in a partition will be rounded
+ // down. This means if there are not sufficient hosts inside a partition,
+ // it is possible that no failover hosts will be created.
+ Ratio float64 `json:"ratio"`
+
+ // Minimum is the minimum number of failover hosts that should be created
+ // in a partition, regardless of the ratio. This ensures that even small
+ // partitions have at least a certain number of failover hosts.
+ Minimum int `json:"minimum"`
+
+ // GenerationInterval is the interval within which failover partitions are
+ // recalculated and failover host reservations are updated accordingly.
+ GenerationInterval metav1.Duration `json:"generationInterval"`
+
+ // ExternalSchedulerURL is the URL of the external scheduler service that
+ // will be used to request failover host placements.
+ ExternalSchedulerURL string `json:"externalSchedulerURL"`
+}
+
+// Controller keeps the cluster's failover host reservations in sync with the
+// current cluster state. It has two cooperating halves (see the package doc for
+// the bigger picture):
+//
+// - a periodic calculation loop (Start -> loop) that recomputes the desired
+// reservations and stores them in desiredReservations, and
+// - the Reconcile method that drives the actual Reservation objects toward
+// that desired set.
+//
+// The loop is the single writer of desiredReservations and Reconcile the single
+// reader, both synchronised by desiredReservationsMu.
+type Controller struct {
+ client.Client // k8s client
+
+ Config ControllerConfig
+
+ // Monitor records Prometheus metrics for the calculation loop and the
+ // reconciler. It may be nil (e.g. in tests); every call site nil-guards it.
+ Monitor *Monitor
+
+ // desiredReservations is the desired state computed by the latest loop
+ // pass: the failover host reservations that should exist right now, keyed
+ // by their object reference (name only; reservations are cluster-scoped).
+ // Reconcile compares the actual cluster against this map. It is replaced
+ // wholesale each pass rather than mutated in place.
+ desiredReservations map[corev1.ObjectReference]v1alpha1.Reservation
+
+ // desiredReservationsMu guards desiredReservations against concurrent access
+ // by the calculation loop (writer) and Reconcile (reader).
+ desiredReservationsMu sync.Mutex
+
+ // reconcileCh carries a raw wake-up signal from the loop to the reconciler.
+ // After a pass the loop sends one event per reservation that is either
+ // desired or still present, so the reconciler re-examines each against the
+ // freshly computed desired state. The event payload only identifies the
+ // reservation; the reconciler reads desiredReservations itself.
+ reconcileCh chan event.GenericEvent
+
+ // The three stages of the calculation pipeline run by loop. They are
+ // defaulted to the production implementations in SetupWithManager but can be
+ // set beforehand to substitute fakes in tests. See loop for how they fit
+ // together, and paths.go / partitions.go / reservations.go for each stage.
+ pathFinder PathFinder
+ partitionBuilder PartitionBuilder
+ reservationPlanner ReservationPlanner
+}
+
+// Reconcile drives a single failover host reservation toward the desired state
+// computed by the calculation loop. It is triggered by the loop (via
+// reconcileCh) for every reservation that is either desired or still present, so
+// between them the two cover create, update, delete and no-op:
+//
+// - desired, not present -> create it.
+// - not desired, present -> prune it (it no longer matches any partition).
+// - desired and present -> update it if the spec/labels drifted.
+// - neither desired nor present, or already in sync -> nothing to do.
+//
+// It is level-triggered and idempotent: it reads the current actual and desired
+// state on every call and makes at most one write, so redundant events and
+// retries are harmless.
+func (c *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+ log := logf.FromContext(ctx).WithName("failoverhost-controller-reconcile").
+ WithValues("reservationName", req.Name)
+
+ // Fetch the actual reservation from the cluster, if it exists.
+ actual := &v1alpha1.Reservation{}
+ actualExists := true
+ if err := c.Get(ctx, req.NamespacedName, actual); err != nil {
+ if !apierrors.IsNotFound(err) {
+ log.Error(err, "failed to get reservation")
+ return ctrl.Result{}, err
+ }
+ actualExists = false
+ }
+
+ // Fetch the desired reservation produced by the partitioning loop.
+ c.desiredReservationsMu.Lock()
+ desired, desiredExists := c.desiredReservations[corev1.ObjectReference{Name: req.Name}]
+ c.desiredReservationsMu.Unlock()
+
+ switch {
+ case !desiredExists && !actualExists:
+ // Nothing to do, both the desired and actual state agree it is gone.
+ log.V(1).Info("reservation neither desired nor present, nothing to do")
+ return ctrl.Result{}, nil
+
+ case !desiredExists && actualExists:
+ // The reservation is no longer part of any failover partition, prune it.
+ if err := c.Delete(ctx, actual); err != nil {
+ if apierrors.IsNotFound(err) {
+ return ctrl.Result{}, nil
+ }
+ log.Error(err, "failed to delete stale reservation")
+ c.Monitor.RecordReservationError("delete")
+ return ctrl.Result{}, err
+ }
+ log.Info("deleted stale failover host reservation", "host", actual.Spec.TargetHost)
+ c.Monitor.RecordReservationDeleted()
+ return ctrl.Result{}, nil
+
+ case desiredExists && !actualExists:
+ // The reservation is desired but not yet present, create it.
+ toCreate := desired.DeepCopy()
+ if err := c.Create(ctx, toCreate); err != nil {
+ if apierrors.IsAlreadyExists(err) {
+ return ctrl.Result{}, nil
+ }
+ log.Error(err, "failed to create failover host reservation")
+ c.Monitor.RecordReservationError("create")
+ return ctrl.Result{}, err
+ }
+ log.Info("created failover host reservation", "host", desired.Spec.TargetHost)
+ c.Monitor.RecordReservationCreated()
+ return ctrl.Result{}, nil
+
+ default:
+ // Both exist. Update the actual spec if it drifted from the desired one.
+ // equality.Semantic handles resource.Quantity correctly, unlike
+ // reflect.DeepEqual which trips over its unexported cached state.
+ if equality.Semantic.DeepEqual(actual.Spec, desired.Spec) &&
+ maps.Equal(actual.Labels, desired.Labels) {
+ log.V(1).Info("reservation already matches desired state")
+ return ctrl.Result{}, nil
+ }
+ actual.Spec = desired.Spec
+ // Replace labels outright rather than merging, so a label that is no
+ // longer desired is removed — a merge would keep it and make every pass
+ // re-detect drift and rewrite the object forever.
+ actual.Labels = maps.Clone(desired.Labels)
+ if err := c.Update(ctx, actual); err != nil {
+ log.Error(err, "failed to update failover host reservation")
+ c.Monitor.RecordReservationError("update")
+ return ctrl.Result{}, err
+ }
+ log.Info("updated failover host reservation", "host", desired.Spec.TargetHost)
+ c.Monitor.RecordReservationUpdated()
+ return ctrl.Result{}, nil
+ }
+}
+
+// Start runs the periodic calculation loop until ctx is cancelled.
+//
+// It implements manager.Runnable so the manager owns its lifecycle (it is
+// registered via mgr.Add in SetupWithManager). Start first validates the config
+// — a misconfigured ratio, minimum or interval is a programming/deployment error
+// that should fail fast rather than silently produce no reservations — then runs
+// one pass immediately (so reservations exist without waiting a full interval)
+// and re-runs loop on every tick. On shutdown it closes reconcileCh so the
+// reconciler's event source drains cleanly.
+func (c *Controller) Start(ctx context.Context) error {
+ log := logf.FromContext(ctx).WithName("failoverhost-controller-startup")
+ log.Info("starting failover host controller")
+
+ // Validate the config
+ interval := c.Config.GenerationInterval.Duration
+ if interval <= 0 || interval < time.Minute {
+ return errors.New("generation interval too low or not set")
+ }
+ if c.Config.Ratio <= 0 || c.Config.Ratio > 1 {
+ return errors.New("failover ratio must be between 0 and 1")
+ }
+ if c.Config.Minimum <= 0 {
+ return errors.New("minimum must be greater than 0")
+ }
+
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+ defer close(c.reconcileCh)
+ // Run the routine once before entering the ticker loop.
+ c.loop(ctx)
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-ticker.C:
+ c.loop(ctx)
+ }
+ }
+}
+
+// SetupWithManager wires both halves of the controller into the manager:
+//
+// - It registers the Controller as a Runnable (mgr.Add) so the manager starts
+// and stops the periodic calculation loop (Start) alongside the controller.
+// - It registers the reconciler with a channel event source, so the loop can
+// enqueue reconcile requests by sending on reconcileCh.
+//
+// It also defaults the three pipeline stages to their production
+// implementations. Tests may set any of them beforehand to substitute a fake
+// (e.g. a path finder pointed at a stub scheduler); only the nil ones are
+// defaulted here.
+func (c *Controller) SetupWithManager(mgr ctrl.Manager) error {
+ if mgr == nil {
+ return errors.New("manager cannot be nil")
+ }
+ // Default the pipeline stages. Tests may set these before calling Setup to
+ // substitute fakes; here we wire the production implementations.
+ if c.pathFinder == nil {
+ c.pathFinder = &evacuationPathFinder{
+ // Timeout so a hung scheduler can't stall the calculation loop.
+ httpClient: &http.Client{Timeout: 30 * time.Second},
+ schedulerURL: c.Config.ExternalSchedulerURL,
+ monitor: c.Monitor,
+ }
+ }
+ if c.partitionBuilder == nil {
+ c.partitionBuilder = cliquePartitionBuilder{}
+ }
+ if c.reservationPlanner == nil {
+ c.reservationPlanner = ratioReservationPlanner{ratio: c.Config.Ratio, minimum: c.Config.Minimum}
+ }
+
+ c.reconcileCh = make(chan event.GenericEvent)
+ src := source.Channel(c.reconcileCh, &handler.EnqueueRequestForObject{})
+
+ // Register the loop so the manager starts and stops it alongside the controller.
+ if err := mgr.Add(c); err != nil {
+ return err
+ }
+
+ return ctrl.NewControllerManagedBy(mgr).
+ Named("failoverhost-controller").
+ WatchesRawSource(src).
+ Complete(c)
+}
+
+// topology is the host/vm layout a single calculation pass reasons over: which
+// vms sit on which hypervisor, and which failover host reservations exist right
+// now. It is a consistent snapshot built once per loop by collect, so every
+// pipeline stage reasons over the same view of the cluster.
+type topology struct {
+ // hypervisors indexes every known hypervisor by its name.
+ hypervisors map[string]hv1.Hypervisor
+ // vmsByHost groups the placed virtual machines by the hypervisor they run
+ // on. Unplaced vms are excluded, so every key is a populated host.
+ vmsByHost map[string][]vmv1alpha1.VirtualMachine
+ // existing holds the failover host reservations currently in the cluster,
+ // keyed by their object reference. The planner prefers to keep these stable.
+ existing map[corev1.ObjectReference]v1alpha1.Reservation
+}
+
+// loop runs one calculation pass of the failover host pipeline:
+//
+// 1. collect — snapshot the cluster into a consistent topology.
+// 2. FindPaths — for each occupied host, find the hosts its whole workload
+// could be gang-evacuated onto (the directed edges of the failover graph).
+// 3. BuildPartitions — cluster those edges into failover partitions: maximal
+// sets of hosts that can all fail over onto one another. These cliques can
+// overlap, so loop then assigns each shared host to exactly one partition
+// (the smallest) to keep the planner input disjoint.
+// 4. PlanReservations — per partition, pick which hosts become failover
+// targets, honouring the configured ratio/minimum and keeping existing
+// reservations stable.
+// 5. publish — store the result as the desired state and wake the reconciler.
+//
+// Any error or inconsistency aborts the whole pass without touching the desired
+// state: a partial recalculation is worse than a slightly stale one, as it could
+// drop reservations that are still valid.
+func (c *Controller) loop(ctx context.Context) {
+ log := logf.FromContext(ctx).WithName("failoverhost-controller-calculation-loop")
+ log.Info("recalculating failover host reservations")
+
+ start := time.Now()
+
+ topo, ok := c.collect(ctx)
+ if !ok {
+ c.Monitor.RecordCalculationAbort("collect")
+ return // collect already logged the reason.
+ }
+
+ paths, err := c.pathFinder.FindPaths(ctx, topo)
+ if err != nil {
+ log.Error(err, "failed to find failover paths")
+ c.Monitor.RecordCalculationAbort("pathfinding")
+ return
+ }
+
+ // Cluster the failover edges into partitions. See partitions.go for why a
+ // partition is a maximal clique of mutually-evacuable hosts, and how empty
+ // hosts are recruited into the partition that can reach them.
+ partitions := c.partitionBuilder.BuildPartitions(paths)
+ log.V(1).Info("clustered failover paths into partitions",
+ "paths", len(paths), "partitions", len(partitions))
+
+ // Maximal cliques overlap: a host with a mutual edge into two otherwise
+ // disjoint cliques (a—b, b—c but not a—c) appears in both. The planner keys
+ // reservations by target host name ("failover-host-"+host), so a host picked
+ // as a failover target by two partitions would yield two reservations with
+ // the same name but conflicting SourceHosts, and whichever is written last to
+ // the desired map wins — silently dropping the other partition's coverage. To
+ // keep the planner input disjoint we assign each shared host to exactly one
+ // partition: the smallest it belongs to. Assigning to the smallest keeps the
+ // most partitions alive — a small partition only survives if it retains its
+ // shared hosts, whereas a large one usually stays above the two-host minimum
+ // even after giving one up — so failover space reaches as many partitions as
+ // possible. Ties are broken by the partition's first host name so the choice
+ // is deterministic. Partitions are already sorted by their first member, so a
+ // stable sort by ascending size keeps that tiebreak order.
+ sort.SliceStable(partitions, func(i, j int) bool {
+ return len(partitions[i]) < len(partitions[j])
+ })
+ assigned := map[string]bool{}
+ deduped := make([][]hv1.Hypervisor, 0, len(partitions))
+ for _, partition := range partitions {
+ kept := make([]hv1.Hypervisor, 0, len(partition))
+ for _, hv := range partition {
+ if assigned[hv.Name] {
+ continue
+ }
+ assigned[hv.Name] = true
+ kept = append(kept, hv)
+ }
+ // Drop a partition emptied out by the dedupe, and one reduced below two
+ // hosts: a lone host offers no failover peer (see PlanReservations).
+ if len(kept) >= 2 {
+ deduped = append(deduped, kept)
+ }
+ }
+ // Restore the deterministic ordering the builder produced (by first host
+ // name), which the size sort above disturbed.
+ sort.SliceStable(deduped, func(i, j int) bool {
+ return deduped[i][0].Name < deduped[j][0].Name
+ })
+ partitions = deduped
+ labels := characterizePartitions(partitions)
+ for i, partition := range partitions {
+ // Describe what sets this partition apart from the others: the shared
+ // fields no other partition shares. See characterizePartitions.
+ log.V(1).Info("characterized failover partition",
+ "partition", i, "partitionSize", len(partition),
+ "distinctBy", labels[i])
+ }
+
+ // Plan the desired reservations per partition and collect them into one map.
+ // We then swap that map in wholesale as the new desired state, under the
+ // lock Reconcile reads it with, so Reconcile never sees a half-built state.
+ // Replacing (rather than mutating) the map is what lets Reconcile prune
+ // reservations that are no longer planned: anything absent from the new map
+ // but still present in the cluster is detected as stale.
+ desired := make(map[corev1.ObjectReference]v1alpha1.Reservation)
+ for i, partition := range partitions {
+ planned := c.reservationPlanner.PlanReservations(topo, partition, labels[i])
+ for _, reservation := range planned {
+ desired[corev1.ObjectReference{Name: reservation.Name}] = reservation
+ }
+ log.V(1).Info("planned failover hosts for partition",
+ "partitionSize", len(partition), "planned", len(planned))
+ }
+ c.desiredReservationsMu.Lock()
+ c.desiredReservations = desired
+ c.desiredReservationsMu.Unlock()
+ log.Info("generated desired failover host reservations",
+ "partitions", len(partitions), "desiredReservations", len(desired),
+ "existingReservations", len(topo.existing))
+ c.Monitor.RecordCalculation(calculationSummary{
+ duration: time.Since(start),
+ partitions: len(partitions),
+ desiredReservations: len(desired),
+ existingReservations: len(topo.existing),
+ })
+
+ c.publish(ctx, topo, desired)
+}
+
+// collect snapshots the current cluster into a topology. It returns ok=false if
+// the snapshot is inconsistent (e.g. a vm sits on an unknown hypervisor, or a
+// hypervisor lacks an availability zone), in which case the loop must abort
+// rather than compute on a partial view.
+func (c *Controller) collect(ctx context.Context) (topology, bool) {
+ log := logf.FromContext(ctx).WithName("failoverhost-controller-calculation-loop")
+
+ hvs := &hv1.HypervisorList{}
+ if err := c.List(ctx, hvs); err != nil {
+ log.Error(err, "failed to list hypervisors")
+ return topology{}, false
+ }
+ hvsByName := make(map[string]hv1.Hypervisor)
+ for i := range hvs.Items {
+ hvsByName[hvs.Items[i].Name] = hvs.Items[i]
+ }
+
+ vms := &vmv1alpha1.VirtualMachineList{}
+ if err := c.List(ctx, vms); err != nil {
+ log.Error(err, "failed to list virtual machines")
+ return topology{}, false
+ }
+ vmsByHost := make(map[string][]vmv1alpha1.VirtualMachine)
+ for i := range vms.Items {
+ vm := vms.Items[i]
+ // Skip vms that Nova hasn't placed on a host yet. They can't contribute
+ // a failover path and would otherwise nil-panic on Status.Nova below.
+ if vm.Status.Nova == nil || vm.Status.Nova.Host == "" {
+ log.V(1).Info("skipping unplaced vm", "vm", vm.Name)
+ continue
+ }
+ // Skip vms that don't have a valid hypervisor type specified in their flavor.
+ hvType, ok := vm.Spec.Nova.Flavor.ExtraSpecs["capabilities:hypervisor_type"]
+ if !ok {
+ log.V(1).Info("skipping vm with no hypervisor type", "vm", vm.Name)
+ continue
+ }
+ // Only KVM virtual machines are handled by the failoverhost controller.
+ if !strings.EqualFold(hvType, "QEMU") && !strings.EqualFold(hvType, "CH") {
+ log.V(1).Info("skipping vm with invalid hypervisor type", "vm", vm.Name, "hvType", hvType)
+ continue
+ }
+ vmsByHost[vm.Status.Nova.Host] = append(vmsByHost[vm.Status.Nova.Host], vm)
+ }
+ log.V(1).Info("listed cluster state", "hypervisors", len(hvsByName), "virtualMachines", len(vms.Items))
+
+ // Ensure that all vms are on known hypervisors with an availability zone,
+ // otherwise abort to avoid calculating on an inconsistent set of hosts.
+ for host := range vmsByHost {
+ hv, ok := hvsByName[host]
+ if !ok {
+ log.Error(nil, "vm is on unknown hypervisor, aborting", "host", host)
+ return topology{}, false
+ }
+ if _, ok := hv.Labels[corev1.LabelTopologyZone]; !ok {
+ log.Error(nil, "hypervisor has no availability zone, aborting", "host", host)
+ return topology{}, false
+ }
+ }
+
+ allReservations := &v1alpha1.ReservationList{}
+ if err := c.List(ctx, allReservations); err != nil {
+ log.Error(err, "failed to list existing failover host reservations")
+ return topology{}, false
+ }
+ existing := make(map[corev1.ObjectReference]v1alpha1.Reservation)
+ for _, reservation := range allReservations.Items {
+ if reservation.Spec.Type == v1alpha1.ReservationTypeFailoverHost {
+ existing[corev1.ObjectReference{Name: reservation.Name}] = reservation
+ }
+ }
+
+ return topology{hypervisors: hvsByName, vmsByHost: vmsByHost, existing: existing}, true
+}
+
+// publish wakes the reconciler for every reservation that is either desired now
+// or already present in the cluster. The union matters: desired-but-absent
+// reservations need creating, and present-but-no-longer-desired reservations
+// need pruning, so both sets must be re-examined. Sending is a raw signal — the
+// reconciler re-reads the desired state itself — so the event payload only needs
+// to identify the reservation. Sends block on reconcileCh; a cancelled ctx (the
+// manager shutting down) unblocks and returns.
+func (c *Controller) publish(ctx context.Context, topo topology, desired map[corev1.ObjectReference]v1alpha1.Reservation) {
+ toReconcile := make(map[corev1.ObjectReference]v1alpha1.Reservation)
+ for k, v := range topo.existing {
+ toReconcile[k] = v
+ }
+ for k, v := range desired {
+ toReconcile[k] = v
+ }
+ for _, reservation := range toReconcile {
+ evt := event.GenericEvent{Object: &reservation}
+ select {
+ case c.reconcileCh <- evt:
+ case <-ctx.Done():
+ return
+ }
+ }
+}
diff --git a/internal/scheduling/reservations/failoverhost/controller_test.go b/internal/scheduling/reservations/failoverhost/controller_test.go
new file mode 100644
index 000000000..23bd5365d
--- /dev/null
+++ b/internal/scheduling/reservations/failoverhost/controller_test.go
@@ -0,0 +1,396 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package failoverhost
+
+import (
+ "context"
+ "slices"
+ "testing"
+
+ "github.com/cobaltcore-dev/cortex/api/v1alpha1"
+ hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+ vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1"
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/types"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+// newTestClient builds a fake controller-runtime client with the schemes the
+// failover host controller needs (reservations, hypervisors, virtual machines)
+// and the Reservation status subresource registered. Shared by the Reconcile
+// tests here and the loop test in loop_test.go.
+func newTestClient(t *testing.T, objs ...client.Object) client.Client {
+ t.Helper()
+ scheme := runtime.NewScheme()
+ if err := v1alpha1.AddToScheme(scheme); err != nil {
+ t.Fatalf("add v1alpha1 scheme: %v", err)
+ }
+ if err := hv1.AddToScheme(scheme); err != nil {
+ t.Fatalf("add hv1 scheme: %v", err)
+ }
+ if err := vmv1alpha1.AddToScheme(scheme); err != nil {
+ t.Fatalf("add vmv1alpha1 scheme: %v", err)
+ }
+ return fake.NewClientBuilder().
+ WithScheme(scheme).
+ WithObjects(objs...).
+ WithStatusSubresource(&v1alpha1.Reservation{}).
+ Build()
+}
+
+// failoverReservation builds a minimal failover host reservation targeting the
+// given host, matching the shape the planner produces.
+func failoverReservation(host string) v1alpha1.Reservation {
+ return v1alpha1.Reservation{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "failover-host-" + host,
+ Labels: map[string]string{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelFailoverHost},
+ },
+ Spec: v1alpha1.ReservationSpec{
+ Type: v1alpha1.ReservationTypeFailoverHost,
+ TargetHost: host,
+ },
+ }
+}
+
+func reconcileReq(name string) ctrl.Request {
+ return ctrl.Request{NamespacedName: types.NamespacedName{Name: name}}
+}
+
+// Reconcile covers the four reconcile branches: desired-but-absent (create),
+// present-but-undesired (delete), present-and-drifted (update), and in-sync
+// (no-op). Each branch is driven by seeding desiredReservations and the fake
+// cluster independently, then asserting the resulting cluster state.
+
+func TestReconcileCreatesDesiredReservation(t *testing.T) {
+ c := &Controller{Client: newTestClient(t)}
+ desired := failoverReservation("hv1")
+ c.desiredReservations = map[corev1.ObjectReference]v1alpha1.Reservation{
+ {Name: desired.Name}: desired,
+ }
+ req := reconcileReq(desired.Name)
+ if _, err := c.Reconcile(context.Background(), req); err != nil {
+ t.Fatalf("reconcile: %v", err)
+ }
+ got := &v1alpha1.Reservation{}
+ if err := c.Get(context.Background(), req.NamespacedName, got); err != nil {
+ t.Fatalf("expected reservation to be created: %v", err)
+ }
+ if got.Spec.TargetHost != "hv1" {
+ t.Errorf("target host = %q, want hv1", got.Spec.TargetHost)
+ }
+}
+
+func TestReconcileDeletesStaleReservation(t *testing.T) {
+ stale := failoverReservation("hvX")
+ c := &Controller{Client: newTestClient(t, &stale)}
+ c.desiredReservations = map[corev1.ObjectReference]v1alpha1.Reservation{}
+ req := reconcileReq(stale.Name)
+ if _, err := c.Reconcile(context.Background(), req); err != nil {
+ t.Fatalf("reconcile: %v", err)
+ }
+ got := &v1alpha1.Reservation{}
+ if err := c.Get(context.Background(), req.NamespacedName, got); !apierrors.IsNotFound(err) {
+ t.Errorf("expected reservation to be deleted, got err=%v", err)
+ }
+}
+
+func TestReconcileUpdatesDriftedReservation(t *testing.T) {
+ actual := failoverReservation("hv1")
+ actual.Spec.AvailabilityZone = "az-old"
+ c := &Controller{Client: newTestClient(t, &actual)}
+ desired := failoverReservation("hv1")
+ desired.Spec.AvailabilityZone = "az-new"
+ c.desiredReservations = map[corev1.ObjectReference]v1alpha1.Reservation{
+ {Name: desired.Name}: desired,
+ }
+ req := reconcileReq(desired.Name)
+ if _, err := c.Reconcile(context.Background(), req); err != nil {
+ t.Fatalf("reconcile: %v", err)
+ }
+ got := &v1alpha1.Reservation{}
+ if err := c.Get(context.Background(), req.NamespacedName, got); err != nil {
+ t.Fatalf("get: %v", err)
+ }
+ if got.Spec.AvailabilityZone != "az-new" {
+ t.Errorf("availability zone = %q, want az-new", got.Spec.AvailabilityZone)
+ }
+}
+
+func TestReconcileNoopWhenInSync(t *testing.T) {
+ actual := failoverReservation("hv1")
+ c := &Controller{Client: newTestClient(t, &actual)}
+ desired := failoverReservation("hv1")
+ c.desiredReservations = map[corev1.ObjectReference]v1alpha1.Reservation{
+ {Name: desired.Name}: desired,
+ }
+ req := reconcileReq(desired.Name)
+ before := &v1alpha1.Reservation{}
+ if err := c.Get(context.Background(), req.NamespacedName, before); err != nil {
+ t.Fatalf("get: %v", err)
+ }
+ if _, err := c.Reconcile(context.Background(), req); err != nil {
+ t.Fatalf("reconcile: %v", err)
+ }
+ after := &v1alpha1.Reservation{}
+ if err := c.Get(context.Background(), req.NamespacedName, after); err != nil {
+ t.Fatalf("get: %v", err)
+ }
+ // An update bumps the resourceVersion, so an unchanged one proves the no-op.
+ if before.ResourceVersion != after.ResourceVersion {
+ t.Errorf("reservation was rewritten: %q -> %q", before.ResourceVersion, after.ResourceVersion)
+ }
+}
+
+// buildHV builds a hypervisor in az1 with an availability-zone label and the
+// given memory (GiB) and cpu capacity, so collect accepts it and the planner can
+// order hosts by cost.
+func buildHV(name string, memGi, cpu int64) *hv1.Hypervisor {
+ return &hv1.Hypervisor{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Labels: map[string]string{corev1.LabelTopologyZone: "az1"},
+ },
+ Status: hv1.HypervisorStatus{
+ Capacity: map[hv1.ResourceName]resource.Quantity{
+ hv1.ResourceMemory: *resource.NewQuantity(memGi*1024*1024*1024, resource.BinarySI),
+ hv1.ResourceCPU: *resource.NewQuantity(cpu, resource.DecimalSI),
+ },
+ },
+ }
+}
+
+// buildVM builds an az1 virtual machine that Nova reports as placed on the given
+// host, so collect indexes it under that host rather than skipping it. It carries
+// a QEMU hypervisor type, the kind collect handles, so it isn't filtered out.
+func buildVM(name, host string) *vmv1alpha1.VirtualMachine {
+ return &vmv1alpha1.VirtualMachine{
+ ObjectMeta: metav1.ObjectMeta{Name: name},
+ Spec: vmv1alpha1.VMSpec{
+ AZ: "az1",
+ Resources: map[vmv1alpha1.ResourceName]resource.Quantity{
+ vmv1alpha1.ResourceMemory: *resource.NewQuantity(2*1024*1024*1024, resource.BinarySI),
+ vmv1alpha1.ResourceCPU: *resource.NewQuantity(1, resource.DecimalSI),
+ },
+ Nova: vmv1alpha1.NovaSpec{
+ Flavor: vmv1alpha1.Flavor{
+ ExtraSpecs: map[string]string{"capabilities:hypervisor_type": "QEMU"},
+ },
+ },
+ },
+ Status: vmv1alpha1.VMStatus{
+ Nova: &vmv1alpha1.NovaStatus{Host: host},
+ },
+ }
+}
+
+// TestLoopGeneratesReservations exercises the whole pipeline end-to-end against
+// a fake cluster and a stub external scheduler: collect -> FindPaths (via HTTP)
+// -> BuildPartitions -> PlanReservations -> publish. It asserts the loop turns a
+// pair of mutually-evacuable hosts into a desired failover host reservation.
+func TestLoopGeneratesReservations(t *testing.T) {
+ // Two equally-sized hosts each running one vm. The external scheduler says
+ // each vm can evacuate onto the other host, so they form one partition.
+ hvA := buildHV("hvA", 64, 32)
+ hvB := buildHV("hvB", 64, 32)
+ vmA := buildVM("vmA", "hvA")
+ vmB := buildVM("vmB", "hvB")
+
+ // Stub scheduler: answer each evacuation query with the other host, so the
+ // path finder discovers the mutual edges hvA<->hvB.
+ finder, server := stubScheduler(t, map[string][]string{
+ "vmA": {"hvB"},
+ "vmB": {"hvA"},
+ }, nil)
+ defer server.Close()
+
+ // Wire the pipeline stages explicitly (loop is driven directly here, not via
+ // SetupWithManager), using the stub path finder.
+ c := &Controller{
+ Client: newTestClient(t, hvA, hvB, vmA, vmB),
+ Config: ControllerConfig{Ratio: 0.5, Minimum: 1},
+ reconcileCh: make(chan event.GenericEvent, 16),
+ pathFinder: finder,
+ partitionBuilder: cliquePartitionBuilder{},
+ reservationPlanner: ratioReservationPlanner{ratio: 0.5, minimum: 1},
+ }
+ c.loop(context.Background())
+
+ c.desiredReservationsMu.Lock()
+ defer c.desiredReservationsMu.Unlock()
+ if len(c.desiredReservations) == 0 {
+ t.Fatalf("expected at least one desired reservation, got none")
+ }
+ for _, r := range c.desiredReservations {
+ if r.Spec.Type != v1alpha1.ReservationTypeFailoverHost {
+ t.Errorf("unexpected reservation type %q", r.Spec.Type)
+ }
+ if r.Spec.AvailabilityZone != "az1" {
+ t.Errorf("availability zone = %q, want az1", r.Spec.AvailabilityZone)
+ }
+ }
+}
+
+// TestCollectFiltersByHypervisorType checks that collect only indexes vms whose
+// flavor declares a KVM hypervisor type (QEMU or CH, case-insensitively). A vm
+// with no hypervisor type or a non-KVM one is dropped, so the failoverhost
+// controller never plans failover paths for workloads it doesn't handle.
+func TestCollectFiltersByHypervisorType(t *testing.T) {
+ hv := buildHV("hv", 64, 32)
+ // kvm and ch are handled; the lowercase "qemu" proves the match is
+ // case-insensitive. none (no extra spec) and vmware are dropped.
+ kvm := buildVM("kvm", "hv")
+ lower := buildVM("lower", "hv")
+ lower.Spec.Nova.Flavor.ExtraSpecs["capabilities:hypervisor_type"] = "qemu"
+ ch := buildVM("ch", "hv")
+ ch.Spec.Nova.Flavor.ExtraSpecs["capabilities:hypervisor_type"] = "CH"
+ vmware := buildVM("vmware", "hv")
+ vmware.Spec.Nova.Flavor.ExtraSpecs["capabilities:hypervisor_type"] = "vmware"
+ none := buildVM("none", "hv")
+ none.Spec.Nova.Flavor.ExtraSpecs = nil
+
+ c := &Controller{Client: newTestClient(t, hv, kvm, lower, ch, vmware, none)}
+ topo, ok := c.collect(context.Background())
+ if !ok {
+ t.Fatal("collect: expected success, got ok=false")
+ }
+
+ got := make([]string, 0)
+ for _, vm := range topo.vmsByHost["hv"] {
+ got = append(got, vm.Name)
+ }
+ slices.Sort(got)
+ if want := []string{"ch", "kvm", "lower"}; !slices.Equal(got, want) {
+ t.Errorf("indexed vms = %v, want %v", got, want)
+ }
+}
+
+// fakePartitionBuilder returns a fixed set of partitions regardless of input, so
+// a test can drive loop with overlapping cliques the real builder would produce.
+type fakePartitionBuilder struct{ partitions [][]hv1.Hypervisor }
+
+func (f fakePartitionBuilder) BuildPartitions([][2]hv1.Hypervisor) [][]hv1.Hypervisor {
+ return f.partitions
+}
+
+// TestLoopDeduplicatesSharedHost checks that when overlapping maximal cliques
+// share a host (hvB in both {hvA,hvB} and {hvB,hvC}), loop assigns that host to
+// exactly one partition before planning, so planning never sees hvB in two
+// partitions and cannot emit two reservations with the same name. The two
+// partitions are equal size, so the deterministic tiebreak keeps hvB in the one
+// with the smaller first host name ({hvA,hvB}); the other shrinks to the single
+// host hvC, too small to reserve. The surviving {hvA,hvB} partition reserves one
+// target (hvA, the first equal-cost host), whose only source host is hvB — proof
+// hvB was planned once, as a source, not as a conflicting second target.
+func TestLoopDeduplicatesSharedHost(t *testing.T) {
+ hvA := buildHV("hvA", 64, 32)
+ hvB := buildHV("hvB", 64, 32)
+ hvC := buildHV("hvC", 64, 32)
+ vmA := buildVM("vmA", "hvA")
+ vmB := buildVM("vmB", "hvB")
+ vmC := buildVM("vmC", "hvC")
+
+ // The path finder's edges are irrelevant here: the fake builder ignores them
+ // and hands loop the overlapping cliques directly. The stub still needs an
+ // answer per vm so FindPaths succeeds.
+ finder, server := stubScheduler(t, map[string][]string{
+ "vmA": {"hvB"}, "vmB": {"hvA", "hvC"}, "vmC": {"hvB"},
+ }, nil)
+ defer server.Close()
+
+ c := &Controller{
+ Client: newTestClient(t, hvA, hvB, hvC, vmA, vmB, vmC),
+ Config: ControllerConfig{Ratio: 0.5, Minimum: 1},
+ reconcileCh: make(chan event.GenericEvent, 16),
+ pathFinder: finder,
+ partitionBuilder: fakePartitionBuilder{partitions: [][]hv1.Hypervisor{
+ {*hvA, *hvB}, {*hvB, *hvC},
+ }},
+ reservationPlanner: ratioReservationPlanner{ratio: 0.5, minimum: 1},
+ }
+ c.loop(context.Background())
+
+ c.desiredReservationsMu.Lock()
+ defer c.desiredReservationsMu.Unlock()
+ // Exactly one reservation survives: {hvA,hvB} reserves hvA, whose source host
+ // is the deduped hvB. {hvB,hvC} lost hvB and shrank to lone hvC, reserving
+ // nothing.
+ if len(c.desiredReservations) != 1 {
+ t.Fatalf("desired reservations = %d, want 1", len(c.desiredReservations))
+ }
+ aRes, ok := c.desiredReservations[corev1.ObjectReference{Name: "failover-host-hvA"}]
+ if !ok {
+ t.Fatalf("expected a reservation targeting hvA")
+ }
+ if got := aRes.Spec.FailoverHostReservation.SourceHosts; len(got) != 1 || got[0] != "hvB" {
+ t.Errorf("hvA source hosts = %v, want [hvB]", got)
+ }
+}
+
+// TestLoopSharedHostKeptInSmallestPartition checks that when a shared host sits
+// in partitions of different sizes, loop assigns it to the smallest one, keeping
+// the most partitions alive. Here hvC is shared between {hvA,hvB,hvC} (size 3)
+// and {hvC,hvD} (size 2). Keeping hvC in the smaller {hvC,hvD} lets that
+// partition survive, while {hvA,hvB,hvC} merely shrinks to {hvA,hvB} and stays
+// above the two-host minimum — so both partitions retain a failover target.
+// Keeping hvC in the larger partition instead would collapse {hvC,hvD} to a lone
+// host and leave hvD unprotected.
+func TestLoopSharedHostKeptInSmallestPartition(t *testing.T) {
+ hvA := buildHV("hvA", 64, 32)
+ hvB := buildHV("hvB", 64, 32)
+ hvC := buildHV("hvC", 64, 32)
+ hvD := buildHV("hvD", 64, 32)
+ vmA := buildVM("vmA", "hvA")
+ vmB := buildVM("vmB", "hvB")
+ vmC := buildVM("vmC", "hvC")
+ vmD := buildVM("vmD", "hvD")
+
+ finder, server := stubScheduler(t, map[string][]string{
+ "vmA": {"hvB"}, "vmB": {"hvA"}, "vmC": {"hvD"}, "vmD": {"hvC"},
+ }, nil)
+ defer server.Close()
+
+ c := &Controller{
+ Client: newTestClient(t, hvA, hvB, hvC, hvD, vmA, vmB, vmC, vmD),
+ Config: ControllerConfig{Ratio: 0.5, Minimum: 1},
+ reconcileCh: make(chan event.GenericEvent, 16),
+ pathFinder: finder,
+ partitionBuilder: fakePartitionBuilder{partitions: [][]hv1.Hypervisor{
+ {*hvA, *hvB, *hvC}, {*hvC, *hvD},
+ }},
+ reservationPlanner: ratioReservationPlanner{ratio: 0.5, minimum: 1},
+ }
+ c.loop(context.Background())
+
+ c.desiredReservationsMu.Lock()
+ defer c.desiredReservationsMu.Unlock()
+ // Both partitions survive: {hvC,hvD} keeps the shared hvC and {hvA,hvB,hvC}
+ // shrinks to {hvA,hvB}. Each reserves one target, so the shared host did not
+ // cost either partition its coverage.
+ if len(c.desiredReservations) != 2 {
+ t.Fatalf("desired reservations = %d, want 2", len(c.desiredReservations))
+ }
+ // {hvC,hvD} must still exist, proving hvC stayed in the smaller partition.
+ // Whichever of hvC/hvD is reserved, its sole source host is the other.
+ cdReserved := false
+ for name, r := range c.desiredReservations {
+ sources := r.Spec.FailoverHostReservation.SourceHosts
+ if name.Name == "failover-host-hvC" && len(sources) == 1 && sources[0] == "hvD" {
+ cdReserved = true
+ }
+ if name.Name == "failover-host-hvD" && len(sources) == 1 && sources[0] == "hvC" {
+ cdReserved = true
+ }
+ }
+ if !cdReserved {
+ t.Errorf("expected a {hvC,hvD} reservation, got %v", c.desiredReservations)
+ }
+}
diff --git a/internal/scheduling/reservations/failoverhost/monitor.go b/internal/scheduling/reservations/failoverhost/monitor.go
new file mode 100644
index 000000000..d2c3bc9c2
--- /dev/null
+++ b/internal/scheduling/reservations/failoverhost/monitor.go
@@ -0,0 +1,206 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package failoverhost
+
+import (
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+)
+
+// Monitor provides Prometheus metrics for the failover host controller. It
+// covers two activities: the periodic calculation loop that recomputes the
+// desired reservations, and the reconciler that drives the actual Reservation
+// objects toward that desired state.
+//
+// Unlike the sibling failover controller, the calculation loop here is
+// cluster-wide (one pass covers every partition and availability zone at once,
+// and a partition clique is not guaranteed to be single-AZ), so the loop-level
+// metrics carry no availability-zone label. The scheduler-request and reconcile
+// metrics do carry a small label each where it is actionable.
+type Monitor struct {
+ calculationRuns prometheus.Counter
+ calculationDuration prometheus.Histogram
+ calculationAborts *prometheus.CounterVec
+ partitions prometheus.Gauge
+ desiredReservations prometheus.Gauge
+ existingReservations prometheus.Gauge
+
+ schedulerRequests *prometheus.CounterVec
+ schedulerRequestDuration prometheus.Histogram
+
+ reservationsCreated prometheus.Counter
+ reservationsUpdated prometheus.Counter
+ reservationsDeleted prometheus.Counter
+ reservationsErrors *prometheus.CounterVec
+}
+
+// calculationSummary holds the statistics of a single calculation-loop pass,
+// filled across the pass and recorded once at the end via RecordCalculation.
+type calculationSummary struct {
+ duration time.Duration
+ partitions int
+ desiredReservations int
+ existingReservations int
+}
+
+// NewMonitor creates a new monitor with Prometheus metrics.
+func NewMonitor() *Monitor {
+ m := &Monitor{
+ calculationRuns: prometheus.NewCounter(prometheus.CounterOpts{
+ Name: "cortex_failover_host_calculation_runs_total",
+ Help: "Total number of completed failover host calculation-loop passes since pod restart",
+ }),
+ calculationDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
+ Name: "cortex_failover_host_calculation_duration_seconds",
+ Help: "Duration of failover host calculation-loop passes",
+ Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
+ }),
+ calculationAborts: prometheus.NewCounterVec(prometheus.CounterOpts{
+ Name: "cortex_failover_host_calculation_aborts_total",
+ Help: "Total number of failover host calculation-loop passes aborted before producing a desired state, by reason",
+ }, []string{"reason"}),
+ partitions: prometheus.NewGauge(prometheus.GaugeOpts{
+ Name: "cortex_failover_host_partitions",
+ Help: "Number of failover partitions from the last calculation-loop pass",
+ }),
+ desiredReservations: prometheus.NewGauge(prometheus.GaugeOpts{
+ Name: "cortex_failover_host_desired_reservations",
+ Help: "Number of desired failover host reservations from the last calculation-loop pass",
+ }),
+ existingReservations: prometheus.NewGauge(prometheus.GaugeOpts{
+ Name: "cortex_failover_host_existing_reservations",
+ Help: "Number of failover host reservations present in the cluster at the last calculation-loop pass",
+ }),
+ schedulerRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
+ Name: "cortex_failover_host_scheduler_requests_total",
+ Help: "Total number of external scheduler requests issued by the failover host path finder, by status (HTTP status code or 'error' on transport failure)",
+ }, []string{"status"}),
+ schedulerRequestDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
+ Name: "cortex_failover_host_scheduler_request_duration_seconds",
+ Help: "Duration of external scheduler requests issued by the failover host path finder",
+ Buckets: []float64{0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30},
+ }),
+ reservationsCreated: prometheus.NewCounter(prometheus.CounterOpts{
+ Name: "cortex_failover_host_reservations_created_total",
+ Help: "Total number of failover host reservations created since pod restart",
+ }),
+ reservationsUpdated: prometheus.NewCounter(prometheus.CounterOpts{
+ Name: "cortex_failover_host_reservations_updated_total",
+ Help: "Total number of failover host reservations updated since pod restart",
+ }),
+ reservationsDeleted: prometheus.NewCounter(prometheus.CounterOpts{
+ Name: "cortex_failover_host_reservations_deleted_total",
+ Help: "Total number of stale failover host reservations deleted since pod restart",
+ }),
+ reservationsErrors: prometheus.NewCounterVec(prometheus.CounterOpts{
+ Name: "cortex_failover_host_reservations_errors_total",
+ Help: "Total number of failed failover host reservation writes since pod restart, by operation",
+ }, []string{"op"}),
+ }
+ // Pre-initialize the labelled series so they appear at zero before the first
+ // event, which lets rate()/increase() alerts behave from pod startup.
+ m.calculationAborts.WithLabelValues("collect")
+ m.calculationAborts.WithLabelValues("pathfinding")
+ m.reservationsErrors.WithLabelValues("create")
+ m.reservationsErrors.WithLabelValues("update")
+ m.reservationsErrors.WithLabelValues("delete")
+ return m
+}
+
+// RecordCalculation records the metrics of a single completed calculation pass.
+func (m *Monitor) RecordCalculation(summary calculationSummary) {
+ if m == nil {
+ return
+ }
+ m.calculationRuns.Inc()
+ m.calculationDuration.Observe(summary.duration.Seconds())
+ m.partitions.Set(float64(summary.partitions))
+ m.desiredReservations.Set(float64(summary.desiredReservations))
+ m.existingReservations.Set(float64(summary.existingReservations))
+}
+
+// RecordCalculationAbort records a calculation pass that aborted before
+// producing a desired state. reason is "collect" or "pathfinding".
+func (m *Monitor) RecordCalculationAbort(reason string) {
+ if m == nil {
+ return
+ }
+ m.calculationAborts.WithLabelValues(reason).Inc()
+}
+
+// RecordSchedulerRequest records one external scheduler request. status is the
+// HTTP status code as a string, or "error" on a transport-level failure.
+func (m *Monitor) RecordSchedulerRequest(status string, duration time.Duration) {
+ if m == nil {
+ return
+ }
+ m.schedulerRequests.WithLabelValues(status).Inc()
+ m.schedulerRequestDuration.Observe(duration.Seconds())
+}
+
+// RecordReservationCreated records a successful reservation create.
+func (m *Monitor) RecordReservationCreated() {
+ if m == nil {
+ return
+ }
+ m.reservationsCreated.Inc()
+}
+
+// RecordReservationUpdated records a successful reservation update.
+func (m *Monitor) RecordReservationUpdated() {
+ if m == nil {
+ return
+ }
+ m.reservationsUpdated.Inc()
+}
+
+// RecordReservationDeleted records a successful stale-reservation delete.
+func (m *Monitor) RecordReservationDeleted() {
+ if m == nil {
+ return
+ }
+ m.reservationsDeleted.Inc()
+}
+
+// RecordReservationError records a failed reservation write. op is "create",
+// "update" or "delete".
+func (m *Monitor) RecordReservationError(op string) {
+ if m == nil {
+ return
+ }
+ m.reservationsErrors.WithLabelValues(op).Inc()
+}
+
+// Describe implements prometheus.Collector.
+func (m *Monitor) Describe(ch chan<- *prometheus.Desc) {
+ m.calculationRuns.Describe(ch)
+ m.calculationDuration.Describe(ch)
+ m.calculationAborts.Describe(ch)
+ m.partitions.Describe(ch)
+ m.desiredReservations.Describe(ch)
+ m.existingReservations.Describe(ch)
+ m.schedulerRequests.Describe(ch)
+ m.schedulerRequestDuration.Describe(ch)
+ m.reservationsCreated.Describe(ch)
+ m.reservationsUpdated.Describe(ch)
+ m.reservationsDeleted.Describe(ch)
+ m.reservationsErrors.Describe(ch)
+}
+
+// Collect implements prometheus.Collector.
+func (m *Monitor) Collect(ch chan<- prometheus.Metric) {
+ m.calculationRuns.Collect(ch)
+ m.calculationDuration.Collect(ch)
+ m.calculationAborts.Collect(ch)
+ m.partitions.Collect(ch)
+ m.desiredReservations.Collect(ch)
+ m.existingReservations.Collect(ch)
+ m.schedulerRequests.Collect(ch)
+ m.schedulerRequestDuration.Collect(ch)
+ m.reservationsCreated.Collect(ch)
+ m.reservationsUpdated.Collect(ch)
+ m.reservationsDeleted.Collect(ch)
+ m.reservationsErrors.Collect(ch)
+}
diff --git a/internal/scheduling/reservations/failoverhost/monitor_test.go b/internal/scheduling/reservations/failoverhost/monitor_test.go
new file mode 100644
index 000000000..6709695f1
--- /dev/null
+++ b/internal/scheduling/reservations/failoverhost/monitor_test.go
@@ -0,0 +1,89 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package failoverhost
+
+import (
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestMonitor_RecordCalculation(t *testing.T) {
+ m := NewMonitor()
+ m.RecordCalculation(calculationSummary{
+ duration: 500 * time.Millisecond,
+ partitions: 3,
+ desiredReservations: 4,
+ existingReservations: 5,
+ })
+
+ if got := testutil.ToFloat64(m.calculationRuns); got != 1 {
+ t.Errorf("calculationRuns = %v, want 1", got)
+ }
+ if got := testutil.ToFloat64(m.partitions); got != 3 {
+ t.Errorf("partitions = %v, want 3", got)
+ }
+ if got := testutil.ToFloat64(m.desiredReservations); got != 4 {
+ t.Errorf("desiredReservations = %v, want 4", got)
+ }
+ if got := testutil.ToFloat64(m.existingReservations); got != 5 {
+ t.Errorf("existingReservations = %v, want 5", got)
+ }
+}
+
+func TestMonitor_Counters(t *testing.T) {
+ m := NewMonitor()
+ m.RecordCalculationAbort("collect")
+ m.RecordCalculationAbort("pathfinding")
+ m.RecordCalculationAbort("pathfinding")
+ m.RecordReservationCreated()
+ m.RecordReservationUpdated()
+ m.RecordReservationDeleted()
+ m.RecordReservationError("create")
+ m.RecordSchedulerRequest("200", 50*time.Millisecond)
+ m.RecordSchedulerRequest("error", 100*time.Millisecond)
+
+ expected := `
+# HELP cortex_failover_host_calculation_aborts_total Total number of failover host calculation-loop passes aborted before producing a desired state, by reason
+# TYPE cortex_failover_host_calculation_aborts_total counter
+cortex_failover_host_calculation_aborts_total{reason="collect"} 1
+cortex_failover_host_calculation_aborts_total{reason="pathfinding"} 2
+`
+ if err := testutil.CollectAndCompare(m.calculationAborts, strings.NewReader(expected)); err != nil {
+ t.Error(err)
+ }
+
+ if got := testutil.ToFloat64(m.reservationsCreated); got != 1 {
+ t.Errorf("reservationsCreated = %v, want 1", got)
+ }
+ if got := testutil.ToFloat64(m.reservationsUpdated); got != 1 {
+ t.Errorf("reservationsUpdated = %v, want 1", got)
+ }
+ if got := testutil.ToFloat64(m.reservationsDeleted); got != 1 {
+ t.Errorf("reservationsDeleted = %v, want 1", got)
+ }
+ if got := testutil.ToFloat64(m.reservationsErrors.WithLabelValues("create")); got != 1 {
+ t.Errorf("reservationsErrors{create} = %v, want 1", got)
+ }
+ if got := testutil.ToFloat64(m.schedulerRequests.WithLabelValues("200")); got != 1 {
+ t.Errorf("schedulerRequests{200} = %v, want 1", got)
+ }
+ if got := testutil.ToFloat64(m.schedulerRequests.WithLabelValues("error")); got != 1 {
+ t.Errorf("schedulerRequests{error} = %v, want 1", got)
+ }
+}
+
+// A nil monitor must be safe to call so controllers and tests can run without one.
+func TestMonitor_NilSafe(t *testing.T) {
+ var m *Monitor
+ m.RecordCalculation(calculationSummary{})
+ m.RecordCalculationAbort("collect")
+ m.RecordSchedulerRequest("200", time.Second)
+ m.RecordReservationCreated()
+ m.RecordReservationUpdated()
+ m.RecordReservationDeleted()
+ m.RecordReservationError("create")
+}
diff --git a/internal/scheduling/reservations/failoverhost/partitions.go b/internal/scheduling/reservations/failoverhost/partitions.go
new file mode 100644
index 000000000..13620cc95
--- /dev/null
+++ b/internal/scheduling/reservations/failoverhost/partitions.go
@@ -0,0 +1,309 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package failoverhost
+
+import (
+ "encoding/json"
+ "fmt"
+ "maps"
+ "reflect"
+ "slices"
+ "sort"
+ "strings"
+
+ hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+)
+
+// PartitionBuilder is the second pipeline stage. It clusters the directed
+// failover edges found by the PathFinder into failover partitions.
+//
+// A failover partition is a maximal set of hypervisors in which every host can
+// be evacuated onto every other host. For example, HANA hosts, general-purpose
+// hosts and external-customer hosts typically each form their own partition,
+// because their workloads can only ever fail over within the group.
+type PartitionBuilder interface {
+ // BuildPartitions groups the [source, target] failover edges into
+ // partitions. The output is deterministic regardless of edge order.
+ BuildPartitions(paths [][2]hv1.Hypervisor) [][]hv1.Hypervisor
+}
+
+// cliquePartitionBuilder builds partitions by finding maximal cliques in the
+// undirected graph of mutual failover edges: an undirected edge a—b exists only
+// if both a→b and b→a are valid paths, since failover requires that the hosts
+// can evacuate onto each other. A failover partition is exactly a maximal clique
+// in that graph — a largest-possible set of hosts all pairwise mutually
+// evacuable.
+//
+// Empty hosts (hosts without any outgoing edge, e.g. because they host no vms)
+// fall out as single-element cliques. They are recruited into an existing
+// partition whose members all have an outgoing edge toward the empty host, as
+// we then assume the reverse direction is also possible.
+type cliquePartitionBuilder struct{}
+
+// BuildPartitions implements PartitionBuilder. It works in four steps:
+//
+// 1. Build the directed adjacency from the edges, and note which hosts have no
+// outgoing edge at all (the "empty" hosts, handled specially in step 3).
+// 2. Derive the undirected "mutual" adjacency (a—b iff both a→b and b→a), and
+// run Bron-Kerbosch over it to enumerate every maximal clique. Each clique
+// is one partition. Bron-Kerbosch recursively grows a candidate clique R by
+// trying each still-eligible vertex in P, intersecting P and the
+// already-seen set X with that vertex's neighbours; a clique is maximal when
+// both P and X are empty. Non-empty hosts with no mutual edge are emitted as
+// their own single-host partitions.
+// 3. Recruit each empty host into the first partition all of whose members can
+// reach it (member→empty for every member). An empty host that no partition
+// fully reaches stays on its own.
+// 4. Translate host names back to Hypervisor values and sort, so the result is
+// deterministic despite the map iteration used throughout.
+func (cliquePartitionBuilder) BuildPartitions(paths [][2]hv1.Hypervisor) [][]hv1.Hypervisor {
+ // Collect all hosts and the directed adjacency, keyed by host name.
+ hvsByName := map[string]hv1.Hypervisor{}
+ outgoing := map[string]map[string]bool{}
+ for _, path := range paths {
+ src, dst := path[0], path[1]
+ hvsByName[src.Name] = src
+ hvsByName[dst.Name] = dst
+ if outgoing[src.Name] == nil {
+ outgoing[src.Name] = map[string]bool{}
+ }
+ outgoing[src.Name][dst.Name] = true
+ }
+
+ // Build the undirected adjacency from mutual edges only. Hosts without any
+ // outgoing edge are empty hosts and are recruited into partitions later.
+ emptyHosts := []string{}
+ for name := range hvsByName {
+ if len(outgoing[name]) == 0 {
+ emptyHosts = append(emptyHosts, name)
+ }
+ }
+ mutual := map[string]map[string]bool{}
+ for a, dsts := range outgoing {
+ for b := range dsts {
+ if outgoing[b][a] { // b also reaches a, so the edge is mutual
+ if mutual[a] == nil {
+ mutual[a] = map[string]bool{}
+ }
+ mutual[a][b] = true
+ }
+ }
+ }
+
+ // Find all maximal cliques in the undirected mutual graph using the
+ // Bron-Kerbosch algorithm. These are our failover partitions.
+ cliques := [][]string{}
+ all := map[string]bool{}
+ for a := range mutual {
+ all[a] = true
+ }
+ var bronKerbosch func(r, p, x map[string]bool)
+ bronKerbosch = func(r, p, x map[string]bool) {
+ if len(p) == 0 && len(x) == 0 {
+ if len(r) == 0 {
+ return // Don't emit the empty clique when there are no mutual edges.
+ }
+ clique := make([]string, 0, len(r))
+ for name := range r {
+ clique = append(clique, name)
+ }
+ cliques = append(cliques, clique)
+ return
+ }
+ // Iterate over a copy of p, as p is mutated inside the loop.
+ candidates := make([]string, 0, len(p))
+ for v := range p {
+ candidates = append(candidates, v)
+ }
+ for _, v := range candidates {
+ if !p[v] {
+ continue
+ }
+ nr := map[string]bool{v: true}
+ maps.Copy(nr, r)
+ np, nx := map[string]bool{}, map[string]bool{}
+ for n := range p {
+ if mutual[v][n] {
+ np[n] = true
+ }
+ }
+ for n := range x {
+ if mutual[v][n] {
+ nx[n] = true
+ }
+ }
+ bronKerbosch(nr, np, nx)
+ delete(p, v)
+ x[v] = true
+ }
+ }
+ bronKerbosch(map[string]bool{}, all, map[string]bool{})
+
+ // Non-empty hosts that share no mutual edge with any other host are not part
+ // of any clique. They form their own single-element partition.
+ covered := map[string]bool{}
+ for _, clique := range cliques {
+ for _, name := range clique {
+ covered[name] = true
+ }
+ }
+ for name := range outgoing {
+ if !covered[name] {
+ cliques = append(cliques, []string{name})
+ }
+ }
+
+ // Sort before recruitment so the result is deterministic: Bron-Kerbosch and
+ // the loops above both iterate maps, so clique contents and order are random
+ // per run. We sort each clique's members, then order the cliques, so an empty
+ // host reachable by several cliques is always recruited into the same one.
+ for _, clique := range cliques {
+ slices.Sort(clique)
+ }
+ sort.SliceStable(cliques, func(i, j int) bool {
+ return slices.Compare(cliques[i], cliques[j]) < 0
+ })
+ slices.Sort(emptyHosts)
+
+ // effectiveCapacity returns a host's gang-fit capacity: its base capacity with
+ // effectiveCapacity overlaid on top (fallback = capacity), matching how the
+ // PathFinder sizes hosts. Copied into a fresh map so we never mutate Status.
+ effectiveCapacity := func(name string) map[hv1.ResourceName]resource.Quantity {
+ capa := map[hv1.ResourceName]resource.Quantity{}
+ maps.Copy(capa, hvsByName[name].Status.Capacity)
+ maps.Copy(capa, hvsByName[name].Status.EffectiveCapacity)
+ return capa
+ }
+ for _, empty := range emptyHosts {
+ emptyCapa := effectiveCapacity(empty)
+ recruited := false
+ for i, clique := range cliques {
+ allReach := len(clique) > 0
+ for _, member := range clique {
+ // Every member must reach the empty host (member can evacuate onto
+ // it), and the empty host must be the same effective size as the
+ // member. The empty host has no vms, so Step 1 never produced an
+ // empty->member edge to prove it can rescue its peers; we fall back
+ // to the same gang-fit rule and require equal capacity. A larger
+ // empty host would be recorded as a source of a smaller reserved
+ // target it could never actually evacuate onto.
+ if !outgoing[member][empty] || cmpResources(emptyCapa, effectiveCapacity(member)) != 0 {
+ allReach = false
+ break
+ }
+ }
+ if allReach {
+ cliques[i] = append(clique, empty)
+ recruited = true
+ break
+ }
+ }
+ if !recruited {
+ cliques = append(cliques, []string{empty})
+ }
+ }
+
+ // Translate the clique host names back into hypervisors. Sort for
+ // deterministic output, as the clustering relies on map iteration.
+ partitions := make([][]hv1.Hypervisor, 0, len(cliques))
+ for _, clique := range cliques {
+ slices.Sort(clique)
+ partition := make([]hv1.Hypervisor, 0, len(clique))
+ for _, name := range clique {
+ partition = append(partition, hvsByName[name])
+ }
+ partitions = append(partitions, partition)
+ }
+ sort.SliceStable(partitions, func(i, j int) bool {
+ return partitions[i][0].Name < partitions[j][0].Name
+ })
+ return partitions
+}
+
+// characterizePartitions returns a short, human-readable label per partition
+// describing what sets it apart from the others, e.g. "spec.aggregates=[hana]".
+// The result is index-aligned with partitions.
+//
+// For each partition we first intersect its hypervisors down to the leaf paths
+// whose value is identical across all of them — the fields that implicitly make
+// those hosts a partition (a common AZ, aggregate, or hypervisor type), without
+// hard-coding which fields matter. We then keep only the *distinguishing* leaves:
+// a field is dropped if its shared value is identical across every partition, as
+// such fleet-wide constants (default booleans, versions, ...) say nothing about
+// what makes a partition special. The surviving leaves are rendered as a compact,
+// sorted "key=value key=value" string for at-a-glance logging.
+func characterizePartitions(partitions [][]hv1.Hypervisor) []string {
+ // flatten serializes a hypervisor to its leaf paths, keyed by dotted path.
+ flatten := func(hv hv1.Hypervisor) map[string]any {
+ raw, err := json.Marshal(hv)
+ if err != nil {
+ return nil
+ }
+ var nested map[string]any
+ if err := json.Unmarshal(raw, &nested); err != nil {
+ return nil
+ }
+ flat := make(map[string]any)
+ var walk func(prefix string, v any)
+ walk = func(prefix string, v any) {
+ if m, ok := v.(map[string]any); ok {
+ for k, vv := range m {
+ key := k
+ if prefix != "" {
+ key = prefix + "." + k
+ }
+ walk(key, vv)
+ }
+ return
+ }
+ flat[prefix] = v
+ }
+ walk("", nested)
+ return flat
+ }
+ // shared[i] holds the leaves common to every hypervisor in partition i.
+ shared := make([]map[string]any, len(partitions))
+ for i, partition := range partitions {
+ if len(partition) == 0 {
+ shared[i] = map[string]any{}
+ continue
+ }
+ common := flatten(partition[0])
+ for _, hv := range partition[1:] {
+ flat := flatten(hv)
+ for key, want := range common {
+ if got, ok := flat[key]; !ok || !reflect.DeepEqual(got, want) {
+ delete(common, key)
+ }
+ }
+ }
+ shared[i] = common
+ }
+ // Count how many partitions carry each (key, value). A field common to all
+ // of them is a fleet-wide constant and tells us nothing distinguishing.
+ seen := map[string]int{}
+ for _, common := range shared {
+ for key, val := range common {
+ seen[key+"="+fmt.Sprintf("%v", val)]++
+ }
+ }
+ // Render the distinguishing leaves of each partition as a sorted label. A
+ // field is distinguishing unless every partition shares it (a fleet-wide
+ // constant); with a single partition there is nothing to contrast against,
+ // so all its shared fields survive.
+ labels := make([]string, len(partitions))
+ for i, common := range shared {
+ parts := make([]string, 0, len(common))
+ for key, val := range common {
+ pair := key + "=" + fmt.Sprintf("%v", val)
+ if len(partitions) == 1 || seen[pair] < len(partitions) {
+ parts = append(parts, pair)
+ }
+ }
+ slices.Sort(parts)
+ labels[i] = strings.Join(parts, " ")
+ }
+ return labels
+}
diff --git a/internal/scheduling/reservations/failoverhost/partitions_test.go b/internal/scheduling/reservations/failoverhost/partitions_test.go
new file mode 100644
index 000000000..26f52b81d
--- /dev/null
+++ b/internal/scheduling/reservations/failoverhost/partitions_test.go
@@ -0,0 +1,216 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package failoverhost
+
+import (
+ "slices"
+ "strings"
+ "testing"
+
+ hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// hv builds a minimal hypervisor identified only by its name.
+func hv(name string) hv1.Hypervisor {
+ return hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: name}}
+}
+
+// hvWithMemory builds a hypervisor with the given name and memory capacity in GiB.
+func hvWithMemory(name string, memGiB int64) hv1.Hypervisor {
+ h := hv(name)
+ h.Status.Capacity = map[hv1.ResourceName]resource.Quantity{
+ hv1.ResourceMemory: *resource.NewQuantity(memGiB*(1<<30), resource.BinarySI),
+ }
+ return h
+}
+
+// edgesHV turns directed hypervisor pairs into failover paths, preserving the
+// hypervisors' capacity (unlike edges, which rebuilds bare hosts from names).
+func edgesHV(pairs ...[2]hv1.Hypervisor) [][2]hv1.Hypervisor {
+ return pairs
+}
+
+// edges turns a list of directed name pairs into failover paths.
+func edges(pairs ...[2]string) [][2]hv1.Hypervisor {
+ paths := make([][2]hv1.Hypervisor, 0, len(pairs))
+ for _, p := range pairs {
+ paths = append(paths, [2]hv1.Hypervisor{hv(p[0]), hv(p[1])})
+ }
+ return paths
+}
+
+// partitionNames extracts the sorted host names of each partition.
+func partitionNames(partitions [][]hv1.Hypervisor) [][]string {
+ out := make([][]string, 0, len(partitions))
+ for _, partition := range partitions {
+ names := make([]string, 0, len(partition))
+ for _, hv := range partition {
+ names = append(names, hv.Name)
+ }
+ slices.Sort(names)
+ out = append(out, names)
+ }
+ return out
+}
+
+func TestBuildPartitions(t *testing.T) {
+ tests := []struct {
+ name string
+ paths [][2]hv1.Hypervisor
+ want [][]string
+ }{
+ {
+ name: "empty input",
+ paths: nil,
+ want: [][]string{},
+ },
+ {
+ name: "single mutual edge forms a partition",
+ paths: edges([2]string{"a", "b"}, [2]string{"b", "a"}),
+ want: [][]string{{"a", "b"}},
+ },
+ {
+ name: "one-directional edge is not a partition",
+ paths: edges([2]string{"a", "b"}),
+ // b is empty (no outgoing edge) and a can reach b, so a-b form one.
+ want: [][]string{{"a", "b"}},
+ },
+ {
+ name: "fully connected triangle",
+ paths: edges(
+ [2]string{"a", "b"}, [2]string{"b", "a"},
+ [2]string{"b", "c"}, [2]string{"c", "b"},
+ [2]string{"a", "c"}, [2]string{"c", "a"},
+ ),
+ want: [][]string{{"a", "b", "c"}},
+ },
+ {
+ name: "two separate partitions",
+ paths: edges(
+ [2]string{"a", "b"}, [2]string{"b", "a"},
+ [2]string{"c", "d"}, [2]string{"d", "c"},
+ ),
+ want: [][]string{{"a", "b"}, {"c", "d"}},
+ },
+ {
+ name: "missing mutual edge splits the clique",
+ paths: edges(
+ [2]string{"a", "b"}, [2]string{"b", "a"},
+ [2]string{"b", "c"}, [2]string{"c", "b"},
+ // a and c are not mutually reachable.
+ ),
+ // a-b and b-c are the two maximal cliques.
+ want: [][]string{{"a", "b"}, {"b", "c"}},
+ },
+ {
+ name: "empty host recruited into partition reaching it",
+ paths: edges(
+ [2]string{"a", "b"}, [2]string{"b", "a"},
+ // e has no outgoing edges but both a and b reach it.
+ [2]string{"a", "e"}, [2]string{"b", "e"},
+ ),
+ want: [][]string{{"a", "b", "e"}},
+ },
+ {
+ name: "empty host not reachable by all stays alone",
+ paths: edges(
+ [2]string{"a", "b"}, [2]string{"b", "a"},
+ // only a reaches e, so e cannot be recruited.
+ [2]string{"a", "e"},
+ ),
+ want: [][]string{{"a", "b"}, {"e"}},
+ },
+ {
+ name: "larger empty host is not recruited",
+ paths: edgesHV(
+ [2]hv1.Hypervisor{hvWithMemory("a", 1), hvWithMemory("b", 1)},
+ [2]hv1.Hypervisor{hvWithMemory("b", 1), hvWithMemory("a", 1)},
+ // a and b reach the empty host e, but e is larger than they are,
+ // so they could never evacuate onto it and it stays alone.
+ [2]hv1.Hypervisor{hvWithMemory("a", 1), hvWithMemory("e", 4)},
+ [2]hv1.Hypervisor{hvWithMemory("b", 1), hvWithMemory("e", 4)},
+ ),
+ want: [][]string{{"a", "b"}, {"e"}},
+ },
+ {
+ name: "equal-sized empty host is recruited",
+ paths: edgesHV(
+ [2]hv1.Hypervisor{hvWithMemory("a", 4), hvWithMemory("b", 4)},
+ [2]hv1.Hypervisor{hvWithMemory("b", 4), hvWithMemory("a", 4)},
+ [2]hv1.Hypervisor{hvWithMemory("a", 4), hvWithMemory("e", 4)},
+ [2]hv1.Hypervisor{hvWithMemory("b", 4), hvWithMemory("e", 4)},
+ ),
+ want: [][]string{{"a", "b", "e"}},
+ },
+ }
+
+ builder := cliquePartitionBuilder{}
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := builder.BuildPartitions(tt.paths)
+ gotNames := partitionNames(got)
+ if !slices.EqualFunc(gotNames, tt.want, slices.Equal) {
+ t.Errorf("got %v, want %v", gotNames, tt.want)
+ }
+ })
+ }
+}
+
+// hvWithAggregates builds a hypervisor with the given name and aggregates.
+func hvWithAggregates(name string, aggregates ...string) hv1.Hypervisor {
+ h := hv(name)
+ h.Spec.Aggregates = aggregates
+ return h
+}
+
+func TestCharacterizePartitions(t *testing.T) {
+ t.Run("no partitions", func(t *testing.T) {
+ if got := characterizePartitions(nil); len(got) != 0 {
+ t.Errorf("got %v, want empty", got)
+ }
+ })
+
+ t.Run("distinguishing aggregate is reported per partition", func(t *testing.T) {
+ labels := characterizePartitions([][]hv1.Hypervisor{
+ {hvWithAggregates("a", "hana"), hvWithAggregates("b", "hana")},
+ {hvWithAggregates("c", "general"), hvWithAggregates("d", "general")},
+ })
+ want := []string{"spec.aggregates=[hana]", "spec.aggregates=[general]"}
+ if !slices.Equal(labels, want) {
+ t.Errorf("got %v, want %v", labels, want)
+ }
+ })
+
+ t.Run("field shared across all partitions is dropped", func(t *testing.T) {
+ // Both partitions share the same aggregate, so it distinguishes neither.
+ labels := characterizePartitions([][]hv1.Hypervisor{
+ {hvWithAggregates("a", "hana")},
+ {hvWithAggregates("b", "hana")},
+ })
+ for i, label := range labels {
+ if strings.Contains(label, "aggregates") {
+ t.Errorf("partition %d label %q should not mention the shared aggregate", i, label)
+ }
+ }
+ })
+
+ t.Run("single partition keeps its distinguishing fields", func(t *testing.T) {
+ // With only one partition nothing is fleet-wide, so shared fields survive.
+ labels := characterizePartitions([][]hv1.Hypervisor{
+ {hvWithAggregates("a", "hana")},
+ })
+ if len(labels) != 1 || !strings.Contains(labels[0], "spec.aggregates=[hana]") {
+ t.Errorf("got %v, want a label containing spec.aggregates=[hana]", labels)
+ }
+ })
+
+ t.Run("empty partition yields an empty label", func(t *testing.T) {
+ labels := characterizePartitions([][]hv1.Hypervisor{{}})
+ if len(labels) != 1 || labels[0] != "" {
+ t.Errorf("got %v, want one empty label", labels)
+ }
+ })
+}
diff --git a/internal/scheduling/reservations/failoverhost/pathfinding.go b/internal/scheduling/reservations/failoverhost/pathfinding.go
new file mode 100644
index 000000000..055a86364
--- /dev/null
+++ b/internal/scheduling/reservations/failoverhost/pathfinding.go
@@ -0,0 +1,353 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package failoverhost
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "maps"
+ "net/http"
+ "slices"
+ "strconv"
+ "time"
+
+ api "github.com/cobaltcore-dev/cortex/api/external/nova"
+ "github.com/cobaltcore-dev/cortex/api/scheduling"
+ "github.com/cobaltcore-dev/cortex/api/v1alpha1"
+ hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+ vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ logf "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+// PathFinder is the first pipeline stage. It discovers the directed edges of the
+// failover graph: for every occupied host it reports the hosts that host's
+// entire workload could be gang-drained onto. An edge source->target means
+// "all vms on source can be live-migrated onto target, and target is large
+// enough to hold them all at once".
+//
+// We probe with live-migration intent rather than evacuation intent even though
+// a failover ultimately evacuates (cold-migrates) the workload. The reason is
+// the drain that happens *before* a reservation goes active: a failover target
+// must be reserved empty, so cortex live-migrates its current workload away
+// first. A host that can only be evacuated-to but not live-migrated-from cannot
+// be drained, so admitting it as a target would block that drain and break
+// partitioning. Live migration is the stricter requirement (it adds CPU/feature
+// compatibility checks on top of capacity), so we assume that anything which can
+// be live-migrated can also be cold-migrated/evacuated.
+//
+// The direction matters. source->target does not imply target->source: target
+// might be too small for source's workload even if the reverse fits. The
+// PartitionBuilder later keeps only the mutual edges, so this stage is free to
+// report one-directional possibilities.
+type PathFinder interface {
+ // FindPaths returns the failover edges as [source, target] hypervisor pairs.
+ // A vm that the scheduler has no target for yields an empty candidate set and
+ // is not an error; it just can't contribute a path. Any lookup error, by
+ // contrast, is an infrastructure failure (scheduler unreachable/timed out/
+ // non-200) that invalidates the pass, so it is returned to abort it rather
+ // than silently collapsing the whole failover graph.
+ FindPaths(ctx context.Context, topo topology) ([][2]hv1.Hypervisor, error)
+}
+
+// evacuationPathFinder finds failover paths by asking the external cortex
+// scheduler, per vm, where that vm could be live-migrated to, then intersecting
+// the answers across all vms on a host and filtering by gang-level fit.
+//
+// Asking the real scheduler (rather than reimplementing placement here) is
+// deliberate: it means failover paths are computed with exactly the same
+// filters, weighers and constraints that govern ordinary placement, so a
+// partition can never contain a target the scheduler would actually reject.
+type evacuationPathFinder struct {
+ // httpClient issues the scheduler requests. Give it a timeout so a hung
+ // scheduler cannot stall the whole calculation loop. Separately injected so
+ // tests can point it at a stub server.
+ httpClient *http.Client
+ // schedulerURL is the external cortex scheduler endpoint to POST to.
+ schedulerURL string
+ // monitor records scheduler request metrics. May be nil (e.g. in tests).
+ monitor *Monitor
+}
+
+// FindPaths implements PathFinder.
+//
+// For a source host S, a target host T is a valid failover target only if every
+// vm on S can be live-migrated onto T. We therefore query the scheduler per vm
+// and keep the running intersection of the candidate sets: a target survives
+// only if it appears for every vm on the host. (If the intersection comes out
+// empty — e.g. because vms on S carry conflicting scheduling constraints — then
+// S has no failover target for now. In the future we may relax this to tolerate
+// single vms that don't fit the same partition as the rest.)
+//
+// Finally we apply the hard, gang-level constraint: a drain/evacuation moves all
+// of S's vms onto T at once, so T must be at least as large as S. We drop every
+// target smaller than its source before emitting the edge. Capacity is compared
+// on a fresh copy of the hypervisor's capacity maps so we never mutate the
+// shared Status.Capacity.
+func (f *evacuationPathFinder) FindPaths(ctx context.Context, topo topology) ([][2]hv1.Hypervisor, error) {
+ log := logf.FromContext(ctx)
+
+ paths := [][2]hv1.Hypervisor{}
+ for sourceHost, vms := range topo.vmsByHost {
+ // targetHosts is the running intersection of candidate targets across
+ // all vms on this source host. initialized tracks whether we've seeded it
+ // with the first vm's candidates yet.
+ targetHosts := []hv1.Hypervisor{}
+ initialized := false
+ for _, vm := range vms {
+ vmTargetHosts, err := f.fetchLiveMigrationCandidates(ctx, vm, topo.hypervisors)
+ if err != nil {
+ // A lookup error is an infrastructure failure (the scheduler is
+ // unreachable, timed out, or returned a non-200) — not an answer
+ // that this vm has no candidates, which comes back as an empty
+ // slice with a nil error. We cannot assert anything about the
+ // failover graph on a partial view, so abort the whole pass. The
+ // caller leaves the desired state untouched and retries next
+ // cycle; swallowing the error here would instead collapse every
+ // host's paths and prune all still-valid reservations.
+ return nil, fmt.Errorf(
+ "failed to fetch live migration candidates for vm %q on host %q: %w",
+ vm.Name, sourceHost, err)
+ }
+ // Log out the target hosts for this vm.
+ vmTargetHostNames := make([]string, len(vmTargetHosts))
+ for i, host := range vmTargetHosts {
+ vmTargetHostNames[i] = host.Name
+ }
+ log.V(1).Info("fetched live migration candidates for vm",
+ "sourceHost", sourceHost, "vm", vm.Name, "targetHosts", vmTargetHostNames)
+ if !initialized {
+ targetHosts = vmTargetHosts // Seed with the first vm's candidates.
+ initialized = true
+ continue
+ }
+ if len(targetHosts) == 0 {
+ break // Intersection already empty, nothing can survive.
+ }
+ intersection := []hv1.Hypervisor{}
+ for _, vmTargetHost := range vmTargetHosts {
+ for _, targetHost := range targetHosts {
+ if vmTargetHost.Name == targetHost.Name {
+ intersection = append(intersection, targetHost)
+ }
+ }
+ }
+ targetHosts = intersection
+ }
+ // Note: if the intersection is empty because some vms on this host have
+ // conflicting scheduling constraints, then this host has no target host
+ // for now. In the future, we may tolerate this if single vms don't fit
+ // into the same failover partition.
+ for _, targetHv := range targetHosts {
+ sourceHv := topo.hypervisors[sourceHost]
+ // Apply the hard, gang-level constraint: drop results where the
+ // target host is smaller than the source host, so that once we
+ // gang-schedule all vms during an evacuation they all fit the
+ // target. We copy into fresh maps so we don't mutate the shared
+ // hypervisor Status.Capacity.
+ sourceHvCapa := map[hv1.ResourceName]resource.Quantity{}
+ maps.Copy(sourceHvCapa, sourceHv.Status.Capacity) // base capacity
+ maps.Copy(sourceHvCapa, sourceHv.Status.EffectiveCapacity)
+ targetHvCapa := map[hv1.ResourceName]resource.Quantity{}
+ maps.Copy(targetHvCapa, targetHv.Status.Capacity)
+ maps.Copy(targetHvCapa, targetHv.Status.EffectiveCapacity)
+ if cmpResources(sourceHvCapa, targetHvCapa) > 0 {
+ log.V(1).Info("dropping failover path, target smaller than source",
+ "sourceHost", sourceHv.Name, "targetHost", targetHv.Name)
+ continue
+ }
+ paths = append(paths, [2]hv1.Hypervisor{sourceHv, targetHv})
+ log.V(1).Info("added failover path",
+ "sourceHost", sourceHv.Name, "targetHost", targetHv.Name)
+ }
+ }
+ return paths, nil
+}
+
+// fetchLiveMigrationCandidates asks the external cortex scheduler where a single
+// vm could be live-migrated to, and returns the subset of known hypervisors it
+// accepts.
+//
+// We probe with live-migration intent even though a failover ultimately
+// evacuates: a failover target must first be drained via live migration before
+// its reservation goes active, so a target a vm cannot be live-migrated onto is
+// useless regardless of whether it could be evacuated onto. Live migration is
+// also the stricter check (it adds CPU/feature compatibility on top of
+// capacity), so we treat "can live-migrate" as implying "can evacuate".
+//
+// The request is shaped to mirror a real live migration as closely as possible:
+// it carries the vm's owner, availability zone, flavor and image so the same
+// filters apply, offers every known host as a candidate (Hosts) so the scheduler
+// has a host set to filter over, sets the live_migrate scheduler hint plus
+// source_host (so filter_live_migratable runs its feature-compatibility checks),
+// and ignores the vm's current host as a target. Four options matter:
+//
+// - ReadOnly, so probing placements never mutate shared scheduling state.
+// - SkipInflight and SkipCommittedResourceTracking, which ReadOnly runs must
+// set (scheduling.Options.Validate rejects a ReadOnly run otherwise, and the
+// scheduler calls Validate) so this probe neither creates in-flight
+// reservations nor writes CR allocations.
+// - IgnoredReservationTypes = FailoverHost, so existing failover host
+// reservations don't block the very hosts we are reasoning about — otherwise
+// a reserved host would be hidden from its own partition and the clustering
+// would fragment.
+// - AssumeEmptyHosts, so candidates are judged by their total capacity rather
+// than their current free capacity. A failover target is reserved *empty*, so
+// what matters for partitioning is whether a host could hold the workload once
+// cleared, not whether it has room right now. Without this a busy host never
+// accepts anyone, gets no inbound links, forms no mutual edge, and stays
+// orphaned — even though reserving it would drain it. Compatibility
+// (flavors, traits, AZ) is thus occupancy-independent; which host is actually
+// cheapest to empty is left to the planner's candidate ordering. The hard
+// whole-host fit check in FindPaths still compares *total* capacities, so an
+// undersized target is dropped regardless of this option.
+func (f *evacuationPathFinder) fetchLiveMigrationCandidates(ctx context.Context, vm vmv1alpha1.VirtualMachine, hvsByName map[string]hv1.Hypervisor) ([]hv1.Hypervisor, error) {
+ log := logf.FromContext(ctx)
+
+ resMemoryMB, resVCPUs := uint64(0), uint64(0)
+ if resMemory, ok := vm.Spec.Resources[vmv1alpha1.ResourceMemory]; ok {
+ // Flavors are specified in Mebibytes, even if they declare "MB".
+ if v := resMemory.Value(); v > 0 {
+ resMemoryMB = uint64(v) / 1024 / 1024
+ }
+ }
+ if resCPU, ok := vm.Spec.Resources[vmv1alpha1.ResourceCPU]; ok {
+ if v := resCPU.Value(); v > 0 {
+ resVCPUs = uint64(v)
+ }
+ }
+ // Build valid image properties. Note: may not include uuid or other metadata keys.
+ imageProps := map[string]any{}
+ for k, v := range vm.Spec.Nova.Image.Properties {
+ imageProps[k] = v // Note: may not include uuid or other metadata keys.
+ }
+ // We also include a proper image hypervisor type so the filter which checks
+ // that the flavor is compatible with the image can properly evaluate the
+ // hypervisor type. We set it explicitly here to ensure the filter has the
+ // necessary information. Note: since we only handle KVM virtual machines,
+ // we set it to KVM explicitly.
+ imageProps["img_hv_type"] = string(api.NovaImageMetaHVTypeKVM)
+ // Offer every known host as a candidate. The scheduler filters/weighs over
+ // exactly this host set — it does not gather the cluster itself — so leaving
+ // Hosts empty would make it return no candidates. Each host also needs a
+ // weight entry, which the scheduler requires to accompany the host set.
+ hosts := make([]api.ExternalSchedulerHost, 0, len(hvsByName))
+ weights := make(map[string]float64, len(hvsByName))
+ for name := range hvsByName {
+ hosts = append(hosts, api.ExternalSchedulerHost{ComputeHost: name})
+ weights[name] = 0
+ }
+ requestObj := api.ExternalSchedulerRequest{
+ Hosts: hosts,
+ Weights: weights,
+ Options: scheduling.Options{
+ // Read-only probe: don't write any resources. ReadOnly additionally
+ // requires SkipInflight and SkipCommittedResourceTracking, else the
+ // scheduler rejects the request in Options.Validate.
+ ReadOnly: true,
+ SkipInflight: true,
+ SkipCommittedResourceTracking: true,
+ // Judge candidates by total capacity, not current free capacity: a
+ // failover target is reserved empty, so a busy host that would fit the
+ // workload once cleared must still be a valid candidate. Without this,
+ // occupied hosts never accept anyone and stay orphaned.
+ AssumeEmptyHosts: true,
+ // Ignore existing failover host reservations, otherwise they would
+ // block these hosts, separate them from their failover partition and
+ // break the clustering.
+ IgnoredReservationTypes: []v1alpha1.ReservationType{
+ v1alpha1.ReservationTypeFailoverHost,
+ },
+ },
+ Spec: api.NovaObject[api.NovaSpec]{
+ Data: api.NovaSpec{
+ InstanceUUID: vm.Name,
+ NumInstances: 1,
+ UserID: vm.Spec.Nova.Ownership.User,
+ ProjectID: vm.Spec.Nova.Ownership.Project,
+ AvailabilityZone: vm.Spec.AZ,
+ // Avoid migrating to the same host as the current VM.
+ IgnoreHosts: new([]string{vm.Status.Nova.Host}),
+ // Probe with live-migration intent rather than evacuation. Draining a
+ // host before its reservation goes active relies on live migration, so
+ // a target that can only be evacuated-to but not live-migrated-from
+ // would break partitioning (see fetchLiveMigrationCandidates). We
+ // assume that a host which accepts a live migration also accepts the
+ // weaker cold migration (evacuation). The source_host hint lets
+ // filter_live_migratable run its feature-compatibility checks; the
+ // check type must be the literal "live_migrate" nova uses, which maps
+ // to LiveMigrationIntent in GetIntent.
+ SchedulerHints: map[string]any{
+ "_nova_check_type": "live_migrate",
+ "source_host": vm.Status.Nova.Host,
+ },
+ Flavor: api.NovaObject[api.NovaFlavor]{
+ Data: api.NovaFlavor{
+ Name: vm.Spec.Nova.Flavor.Name,
+ ExtraSpecs: vm.Spec.Nova.Flavor.ExtraSpecs,
+ MemoryMB: resMemoryMB,
+ VCPUs: resVCPUs,
+ },
+ },
+ Image: api.NovaObject[api.NovaImageMeta]{
+ Data: api.NovaImageMeta{
+ ID: vm.Spec.Nova.Image.UUID,
+ Properties: api.NovaObject[map[string]any]{
+ Data: imageProps,
+ },
+ },
+ },
+ },
+ },
+ }
+ log.V(1).Info("external scheduler request prepared", "request", requestObj)
+
+ reqBody, err := json.Marshal(requestObj)
+ if err != nil {
+ log.Error(err, "failed to marshal external scheduler request")
+ return nil, fmt.Errorf("failed to marshal external scheduler request: %w", err)
+ }
+ httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost,
+ f.schedulerURL, bytes.NewReader(reqBody))
+ if err != nil {
+ log.Error(err, "failed to create HTTP request")
+ return nil, fmt.Errorf("failed to create HTTP request: %w", err)
+ }
+ httpReq.Header.Set("Content-Type", "application/json")
+
+ startTime := time.Now()
+ response, err := f.httpClient.Do(httpReq)
+ duration := time.Since(startTime)
+ if err != nil {
+ f.monitor.RecordSchedulerRequest("error", duration)
+ log.Error(err, "failed to send external scheduler request")
+ return nil, fmt.Errorf("failed to send external scheduler request: %w", err)
+ }
+ defer response.Body.Close()
+ f.monitor.RecordSchedulerRequest(strconv.Itoa(response.StatusCode), duration)
+ log.V(1).Info("external scheduler request sent",
+ "duration", duration, "status", response.StatusCode)
+
+ if response.StatusCode != http.StatusOK {
+ log.Error(nil, "external scheduler request failed", "status", response.StatusCode)
+ return nil, fmt.Errorf("external scheduler request failed with status: %d", response.StatusCode)
+ }
+
+ var responseObj api.ExternalSchedulerResponse
+ if err := json.NewDecoder(response.Body).Decode(&responseObj); err != nil {
+ log.Error(err, "failed to decode external scheduler response")
+ return nil, fmt.Errorf("failed to decode external scheduler response: %w", err)
+ }
+ log.V(1).Info("external scheduler response received", "response", responseObj)
+
+ candidates := []hv1.Hypervisor{}
+ for _, hv := range hvsByName {
+ if slices.Contains(responseObj.Hosts, hv.Name) {
+ candidates = append(candidates, hv)
+ }
+ }
+ return candidates, nil
+}
diff --git a/internal/scheduling/reservations/failoverhost/pathfinding_test.go b/internal/scheduling/reservations/failoverhost/pathfinding_test.go
new file mode 100644
index 000000000..1d0ca0c63
--- /dev/null
+++ b/internal/scheduling/reservations/failoverhost/pathfinding_test.go
@@ -0,0 +1,174 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package failoverhost
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "slices"
+ "testing"
+
+ api "github.com/cobaltcore-dev/cortex/api/external/nova"
+ hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+ vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1"
+)
+
+// stubScheduler returns a path finder wired to an httptest server whose per-vm
+// answers come from candidatesByVM (keyed by vm name / InstanceUUID); a vm in
+// errVMs gets an HTTP 500. The handler asserts the request carries valid options,
+// a non-empty host set, and a kvm img_hv_type, mirroring what the real scheduler
+// requires — so a regression that drops any of them is caught here. The caller
+// closes the server.
+func stubScheduler(t *testing.T, candidatesByVM map[string][]string, errVMs map[string]bool) (*evacuationPathFinder, *httptest.Server) {
+ t.Helper()
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var req api.ExternalSchedulerRequest
+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+ t.Errorf("decode request: %v", err)
+ }
+ if err := req.Options.Validate(); err != nil {
+ t.Errorf("invalid scheduling options: %v", err)
+ }
+ if !req.Options.AssumeEmptyHosts {
+ // A failover target is reserved empty, so candidates must be judged by
+ // total capacity. Dropping this would orphan every busy host.
+ t.Errorf("request for vm %q must set AssumeEmptyHosts", req.Spec.Data.InstanceUUID)
+ }
+ if len(req.Hosts) == 0 {
+ t.Errorf("request for vm %q has no candidate hosts", req.Spec.Data.InstanceUUID)
+ }
+ if got := req.Spec.Data.Image.Data.Properties.Data["img_hv_type"]; got != string(api.NovaImageMetaHVTypeKVM) {
+ // The hypervisor-type filter needs img_hv_type to judge flavor/image
+ // compatibility; dropping it would make that filter misbehave.
+ t.Errorf("request for vm %q must set img_hv_type=%q, got %q", req.Spec.Data.InstanceUUID, api.NovaImageMetaHVTypeKVM, got)
+ }
+ if errVMs[req.Spec.Data.InstanceUUID] {
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ hosts := candidatesByVM[req.Spec.Data.InstanceUUID]
+ if err := json.NewEncoder(w).Encode(api.ExternalSchedulerResponse{Hosts: hosts}); err != nil {
+ t.Errorf("encode response: %v", err)
+ }
+ }))
+ finder := &evacuationPathFinder{httpClient: server.Client(), schedulerURL: server.URL}
+ return finder, server
+}
+
+// pathNames collapses failover edges into sorted "source->target" strings, for
+// order-independent comparison.
+func pathNames(paths [][2]hv1.Hypervisor) []string {
+ out := make([]string, 0, len(paths))
+ for _, p := range paths {
+ out = append(out, p[0].Name+"->"+p[1].Name)
+ }
+ slices.Sort(out)
+ return out
+}
+
+// topoWith builds a topology of the named hosts (each sized memGi GiB) and the
+// vms placed on them, so FindPaths can be exercised without a k8s client.
+func topoWith(hostMemGi map[string]int64, vmsByHost map[string][]string) topology {
+ hypervisors := map[string]hv1.Hypervisor{}
+ for name, memGi := range hostMemGi {
+ hypervisors[name] = *buildHV(name, memGi, 1)
+ }
+ vms := map[string][]vmv1alpha1.VirtualMachine{}
+ for host, vmNames := range vmsByHost {
+ for _, name := range vmNames {
+ vms[host] = append(vms[host], *buildVM(name, host))
+ }
+ }
+ return topology{hypervisors: hypervisors, vmsByHost: vms}
+}
+
+func TestFindPathsErrorsOnLookupFailure(t *testing.T) {
+ // A source host runs two vms. One vm's lookup fails with a scheduler error.
+ // That's an infrastructure failure, not an answer, so FindPaths must return
+ // an error to abort the pass rather than silently dropping paths — otherwise
+ // the caller would publish an empty desired state and prune valid
+ // reservations.
+ finder, server := stubScheduler(t,
+ map[string][]string{"vmGood": {"b"}},
+ map[string]bool{"vmBad": true},
+ )
+ defer server.Close()
+
+ // vmBad is listed first so the error hits the seeding vm, exactly the case
+ // the old index-based seeding mishandled.
+ topo := topoWith(
+ map[string]int64{"src": 32, "b": 32},
+ map[string][]string{"src": {"vmBad", "vmGood"}},
+ )
+ if _, err := finder.FindPaths(context.Background(), topo); err == nil {
+ t.Fatal("FindPaths: expected error on scheduler lookup failure, got nil")
+ }
+}
+
+func TestFindPathsIntersectsCandidatesAcrossVMs(t *testing.T) {
+ // Host "src" runs two vms. vm1 can go to {a,b}, vm2 can go to {b,c}. Only
+ // "b" is a valid target for the whole host, so only src->b is a path.
+ finder, server := stubScheduler(t, map[string][]string{
+ "vm1": {"a", "b"},
+ "vm2": {"b", "c"},
+ }, nil)
+ defer server.Close()
+
+ topo := topoWith(
+ map[string]int64{"src": 32, "a": 32, "b": 32, "c": 32},
+ map[string][]string{"src": {"vm1", "vm2"}},
+ )
+ got, err := finder.FindPaths(context.Background(), topo)
+ if err != nil {
+ t.Fatalf("FindPaths: %v", err)
+ }
+ if want := []string{"src->b"}; !slices.Equal(pathNames(got), want) {
+ t.Errorf("paths = %v, want %v", pathNames(got), want)
+ }
+}
+
+func TestFindPathsDropsTargetSmallerThanSource(t *testing.T) {
+ // "big" (64Gi) runs one vm that can evacuate to "small" (16Gi) and "equal"
+ // (64Gi). The gang-fit filter drops "small", keeping only big->equal.
+ finder, server := stubScheduler(t, map[string][]string{
+ "vm1": {"small", "equal"},
+ }, nil)
+ defer server.Close()
+
+ topo := topoWith(
+ map[string]int64{"big": 64, "small": 16, "equal": 64},
+ map[string][]string{"big": {"vm1"}},
+ )
+ got, err := finder.FindPaths(context.Background(), topo)
+ if err != nil {
+ t.Fatalf("FindPaths: %v", err)
+ }
+ if want := []string{"big->equal"}; !slices.Equal(pathNames(got), want) {
+ t.Errorf("paths = %v, want %v", pathNames(got), want)
+ }
+}
+
+func TestFindPathsAreMutualForEquallySizedPeers(t *testing.T) {
+ // Two equal hosts whose single vm can evacuate onto the other yield both
+ // directions, which the partition builder later turns into one partition.
+ finder, server := stubScheduler(t, map[string][]string{
+ "vmA": {"hvB"},
+ "vmB": {"hvA"},
+ }, nil)
+ defer server.Close()
+
+ topo := topoWith(
+ map[string]int64{"hvA": 64, "hvB": 64},
+ map[string][]string{"hvA": {"vmA"}, "hvB": {"vmB"}},
+ )
+ got, err := finder.FindPaths(context.Background(), topo)
+ if err != nil {
+ t.Fatalf("FindPaths: %v", err)
+ }
+ if want := []string{"hvA->hvB", "hvB->hvA"}; !slices.Equal(pathNames(got), want) {
+ t.Errorf("paths = %v, want %v", pathNames(got), want)
+ }
+}
diff --git a/internal/scheduling/reservations/failoverhost/planning.go b/internal/scheduling/reservations/failoverhost/planning.go
new file mode 100644
index 000000000..c734c8ba5
--- /dev/null
+++ b/internal/scheduling/reservations/failoverhost/planning.go
@@ -0,0 +1,199 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package failoverhost
+
+import (
+ "maps"
+ "slices"
+ "sort"
+
+ "github.com/cobaltcore-dev/cortex/api/v1alpha1"
+ hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ReservationPlanner is the third pipeline stage. For a single failover
+// partition it decides which hosts should become failover targets, i.e. be kept
+// empty so another host in the partition can gang-evacuate onto them.
+//
+// We do not reserve a fixed number per partition. Reserving too few leaves no
+// landing space when a host fails; reserving too many wastes capacity. So the
+// count scales with the partition: a configured ratio of the hosts, floored at a
+// configured minimum so even small partitions keep some headroom.
+type ReservationPlanner interface {
+ // PlanReservations returns the desired failover host reservations for the
+ // given partition. The topology supplies the reservations that already
+ // exist, which the planner prefers to keep so target hosts stay stable
+ // across passes rather than flapping from one host to another. description
+ // is a human-readable label of what distinguishes the partition's hosts,
+ // stored verbatim on each reservation. See characterizePartitions.
+ PlanReservations(topo topology, partition []hv1.Hypervisor, description string) []v1alpha1.Reservation
+}
+
+// ratioReservationPlanner reserves a ratio of each partition's hosts as failover
+// targets, never fewer than a configured minimum. It prefers to keep existing
+// reservations and, when it must create new ones, picks the hosts with the
+// least allocated capacity so that the eventual evacuation migrates as few
+// resources as possible.
+type ratioReservationPlanner struct {
+ // ratio is the fraction of a partition's hosts to reserve as failover
+ // targets, rounded down (e.g. 0.1 of a 10-host partition reserves 1).
+ ratio float64
+ // minimum is the floor on reservations per partition, applied even when the
+ // ratio would round down to fewer.
+ minimum int
+}
+
+// PlanReservations implements ReservationPlanner.
+//
+// It builds a freshly computed reservation for every host in the partition and
+// returns the first desiredCount of them, where desiredCount =
+// max(len(partition)*ratio, minimum), ordered [already-reserved..., cheapest-new...]:
+//
+// - Hosts that already hold a reservation for this partition come first, so a
+// host that is already a failover target stays one. This stability matters:
+// a designated failover host may have vms being live-migrated off it, and
+// flipping the designation to a different host each pass would waste that
+// work. If there are more such hosts than desiredCount, the surplus is left
+// out of the result and the reconciler prunes it, freeing the host back into
+// normal service.
+// - Not-yet-reserved hosts come next, ordered by ascending allocated capacity,
+// so when we must designate a fresh target we pick the host whose vms are
+// cheapest to migrate away.
+//
+// Crucially the spec is recomputed for every host, including already-reserved
+// ones, rather than carried over from topo.existing. A kept reservation's
+// SourceHosts, Resources and AvailabilityZone therefore always track the current
+// partition: if the partition's membership or a host's capacity changed, the
+// recomputed spec drifts from the stored one and the reconciler updates it.
+func (p ratioReservationPlanner) PlanReservations(topo topology, partition []hv1.Hypervisor, description string) []v1alpha1.Reservation {
+ // A partition of fewer than two hosts offers no failover: there is no peer
+ // for a reserved host to receive an evacuation from. Reserving the lone host
+ // would only block it from normal placement for no benefit, so skip it.
+ if len(partition) < 2 {
+ return nil
+ }
+
+ // Record which hosts already hold a failover reservation, so we can order
+ // their recomputed reservations first and keep target hosts stable. We use
+ // the spec's target host, not Status.Host: a freshly created reservation the
+ // scheduler hasn't placed yet has an empty Status.Host.
+ hostsReserved := []string{}
+ for _, reservation := range topo.existing {
+ hostsReserved = append(hostsReserved, reservation.Spec.TargetHost)
+ }
+
+ // Build a freshly computed reservation for every host in the partition and
+ // split them into already-reserved and fresh candidates.
+ existing := []v1alpha1.Reservation{}
+ fresh := []v1alpha1.Reservation{}
+ for _, hv := range partition {
+ sourceHosts := []string{} // Other hosts in the same partition.
+ for _, otherHv := range partition {
+ if otherHv.Name != hv.Name {
+ sourceHosts = append(sourceHosts, otherHv.Name)
+ }
+ }
+ reservedResources := map[hv1.ResourceName]resource.Quantity{}
+ maps.Copy(reservedResources, hv.Status.Capacity) // base capacity
+ maps.Copy(reservedResources, hv.Status.EffectiveCapacity)
+ reservation := v1alpha1.Reservation{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "failover-host-" + hv.Name,
+ Labels: map[string]string{
+ v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelFailoverHost,
+ },
+ },
+ Spec: v1alpha1.ReservationSpec{
+ Type: v1alpha1.ReservationTypeFailoverHost,
+ SchedulingDomain: v1alpha1.SchedulingDomainNova,
+ AvailabilityZone: hv.Labels[corev1.LabelTopologyZone],
+ Resources: reservedResources, // Allocate full host.
+ TargetHost: hv.Name,
+ FailoverHostReservation: &v1alpha1.FailoverHostReservationSpec{
+ SourceHosts: sourceHosts,
+ PartitionDescription: description,
+ },
+ },
+ }
+ if slices.Contains(hostsReserved, hv.Name) {
+ existing = append(existing, reservation)
+ } else {
+ fresh = append(fresh, reservation)
+ }
+ }
+ // topo.existing is a map, so sort for a stable order. Otherwise, when a
+ // partition holds more already-reserved hosts than we need, the truncation
+ // below would keep a different subset each pass and flap the target hosts.
+ sort.SliceStable(existing, func(i, j int) bool {
+ return existing[i].Spec.TargetHost < existing[j].Spec.TargetHost
+ })
+ // Order new candidates by ascending allocated capacity, so that if we have to
+ // create reservations we pick the hosts that are cheapest to evacuate onto.
+ sort.SliceStable(fresh, func(i, j int) bool {
+ return cmpResources(fresh[i].Spec.Resources, fresh[j].Spec.Resources) < 0
+ })
+
+ // Candidate order is [already-reserved..., cheapest-new...]. Reserve a ratio
+ // of the partition, never fewer than the minimum, but never so many that no
+ // host is left to run workload — always keep at least one source host.
+ available := make([]v1alpha1.Reservation, 0, len(existing)+len(fresh))
+ available = append(available, existing...)
+ available = append(available, fresh...)
+ desiredCount := int(float64(len(partition)) * p.ratio)
+ if desiredCount < p.minimum {
+ desiredCount = p.minimum
+ }
+ if maxReservable := len(partition) - 1; desiredCount > maxReservable {
+ desiredCount = maxReservable
+ }
+ if len(available) > desiredCount {
+ available = available[:desiredCount]
+ }
+ return available
+}
+
+// cmpResources compares the resources of hypervisor a with the resources
+// of hypervisor b. If the resources of a are greater then b, it returns 1.
+// If the resources of a are less than b, it returns -1.
+// If the resources are equal, it returns 0.
+func cmpResources(a, b map[hv1.ResourceName]resource.Quantity) int {
+ // Consider memory first, then consider cpu if there isn't already
+ // a decisive comparison can be made based on memory.
+ memA, ok := a[hv1.ResourceMemory]
+ if !ok {
+ memA = *resource.NewQuantity(0, resource.BinarySI)
+ }
+ memB, ok := b[hv1.ResourceMemory]
+ if !ok {
+ memB = *resource.NewQuantity(0, resource.BinarySI)
+ }
+ // Round to GiB to absorb small per-node discrepancies in reported memory
+ // capacity (Linux frees varying amounts during boot, so identical hardware
+ // may show slightly different effective capacities).
+ const gib = 1 << 30
+ memA = *resource.NewQuantity(memA.Value()/gib*gib, resource.BinarySI)
+ memB = *resource.NewQuantity(memB.Value()/gib*gib, resource.BinarySI)
+ if memA.Cmp(memB) > 0 {
+ return 1
+ } else if memA.Cmp(memB) < 0 {
+ return -1
+ }
+ cpuA, ok := a[hv1.ResourceCPU]
+ if !ok {
+ cpuA = *resource.NewQuantity(0, resource.DecimalSI)
+ }
+ cpuB, ok := b[hv1.ResourceCPU]
+ if !ok {
+ cpuB = *resource.NewQuantity(0, resource.DecimalSI)
+ }
+ if cpuA.Cmp(cpuB) > 0 {
+ return 1
+ } else if cpuA.Cmp(cpuB) < 0 {
+ return -1
+ }
+ return 0
+}
diff --git a/internal/scheduling/reservations/failoverhost/planning_test.go b/internal/scheduling/reservations/failoverhost/planning_test.go
new file mode 100644
index 000000000..419039a08
--- /dev/null
+++ b/internal/scheduling/reservations/failoverhost/planning_test.go
@@ -0,0 +1,224 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package failoverhost
+
+import (
+ "slices"
+ "testing"
+
+ "github.com/cobaltcore-dev/cortex/api/v1alpha1"
+ hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// res builds a resource map from an optional memory and cpu quantity string.
+func res(mem, cpu string) map[hv1.ResourceName]resource.Quantity {
+ out := map[hv1.ResourceName]resource.Quantity{}
+ if mem != "" {
+ out[hv1.ResourceMemory] = resource.MustParse(mem)
+ }
+ if cpu != "" {
+ out[hv1.ResourceCPU] = resource.MustParse(cpu)
+ }
+ return out
+}
+
+func TestCmpResources(t *testing.T) {
+ tests := []struct {
+ name string
+ a, b map[hv1.ResourceName]resource.Quantity
+ want int
+ }{
+ {"equal", res("8Gi", "4"), res("8Gi", "4"), 0},
+ {"memory greater dominates", res("16Gi", "1"), res("8Gi", "99"), 1},
+ {"memory less dominates", res("4Gi", "99"), res("8Gi", "1"), -1},
+ {"cpu breaks memory tie greater", res("8Gi", "8"), res("8Gi", "4"), 1},
+ {"cpu breaks memory tie less", res("8Gi", "2"), res("8Gi", "4"), -1},
+ {"missing memory treated as zero", res("", "4"), res("8Gi", "4"), -1},
+ {"both empty are equal", res("", ""), res("", ""), 0},
+ {"missing cpu treated as zero", res("8Gi", ""), res("8Gi", "4"), -1},
+ // Sub-GiB discrepancies (e.g. from different amounts of memory freed during
+ // Linux boot) must not be treated as different capacities.
+ {"sub-gib difference treated as equal", res("4227288964Ki", "4"), res("4227172876Ki", "4"), 0},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := cmpResources(tt.a, tt.b); got != tt.want {
+ t.Errorf("cmpResources() = %d, want %d", got, tt.want)
+ }
+ })
+ }
+}
+
+// partitionOf builds a partition of hosts, each with the given memory capacity
+// in GiB so the planner can order them by allocation cost.
+func partitionOf(memByHost map[string]int64) []hv1.Hypervisor {
+ partition := make([]hv1.Hypervisor, 0, len(memByHost))
+ for name, memGi := range memByHost {
+ partition = append(partition, hv1.Hypervisor{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Labels: map[string]string{corev1.LabelTopologyZone: "az1"},
+ },
+ Status: hv1.HypervisorStatus{
+ Capacity: map[hv1.ResourceName]resource.Quantity{
+ hv1.ResourceMemory: *resource.NewQuantity(memGi*1024*1024*1024, resource.BinarySI),
+ },
+ },
+ })
+ }
+ slices.SortFunc(partition, func(a, b hv1.Hypervisor) int { return cmpStrings(a.Name, b.Name) })
+ return partition
+}
+
+func cmpStrings(a, b string) int {
+ switch {
+ case a < b:
+ return -1
+ case a > b:
+ return 1
+ default:
+ return 0
+ }
+}
+
+func TestPlanReservationsRatioAndMinimum(t *testing.T) {
+ partition := partitionOf(map[string]int64{"hv1": 64, "hv2": 32, "hv3": 16, "hv4": 48})
+ empty := topology{existing: map[corev1.ObjectReference]v1alpha1.Reservation{}}
+
+ tests := []struct {
+ name string
+ ratio float64
+ minimum int
+ want int // number of planned reservations
+ }{
+ {"ratio rounds down", 0.5, 1, 2}, // 4 * 0.5 = 2
+ {"minimum floors ratio", 0.1, 2, 2}, // 4 * 0.1 = 0, floored to 2
+ {"ratio above minimum wins", 0.75, 1, 3},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ planner := ratioReservationPlanner{ratio: tt.ratio, minimum: tt.minimum}
+ got := planner.PlanReservations(empty, partition, "")
+ if len(got) != tt.want {
+ t.Fatalf("planned %d reservations, want %d", len(got), tt.want)
+ }
+ for _, r := range got {
+ if r.Spec.Type != v1alpha1.ReservationTypeFailoverHost {
+ t.Errorf("unexpected reservation type %q", r.Spec.Type)
+ }
+ if r.Spec.AvailabilityZone != "az1" {
+ t.Errorf("availability zone = %q, want az1", r.Spec.AvailabilityZone)
+ }
+ }
+ })
+ }
+}
+
+func TestPlanReservationsPrefersSmallestHosts(t *testing.T) {
+ // With no existing reservations, the planner creates new ones for the hosts
+ // that are cheapest to evacuate, i.e. the smallest capacity first.
+ partition := partitionOf(map[string]int64{"big": 64, "small": 16, "medium": 32})
+ empty := topology{existing: map[corev1.ObjectReference]v1alpha1.Reservation{}}
+ planner := ratioReservationPlanner{ratio: 0.34, minimum: 1} // 3 * 0.34 = 1
+
+ got := planner.PlanReservations(empty, partition, "")
+ if len(got) != 1 {
+ t.Fatalf("planned %d reservations, want 1", len(got))
+ }
+ if got[0].Spec.TargetHost != "small" {
+ t.Errorf("target host = %q, want smallest host 'small'", got[0].Spec.TargetHost)
+ }
+}
+
+func TestPlanReservationsKeepsExistingStable(t *testing.T) {
+ partition := partitionOf(map[string]int64{"big": 64, "small": 16})
+ existing := v1alpha1.Reservation{
+ ObjectMeta: metav1.ObjectMeta{Name: "failover-host-big"},
+ Spec: v1alpha1.ReservationSpec{
+ Type: v1alpha1.ReservationTypeFailoverHost,
+ TargetHost: "big", // already reserved, even though it's the bigger host
+ Resources: res("64Gi", ""),
+ },
+ }
+ topo := topology{existing: map[corev1.ObjectReference]v1alpha1.Reservation{
+ {Name: existing.Name}: existing,
+ }}
+ planner := ratioReservationPlanner{ratio: 0.5, minimum: 1} // reserve 1 of 2
+
+ got := planner.PlanReservations(topo, partition, "")
+ if len(got) != 1 {
+ t.Fatalf("planned %d reservations, want 1", len(got))
+ }
+ // The existing reservation wins over creating a new one on the smaller host,
+ // so target hosts don't flip between passes.
+ if got[0].Spec.TargetHost != "big" {
+ t.Errorf("target host = %q, want existing 'big'", got[0].Spec.TargetHost)
+ }
+}
+
+func TestPlanReservationsRecomputesExistingSpec(t *testing.T) {
+ // A host that is already reserved keeps its designation, but its spec is
+ // recomputed from the current partition rather than carried over stale. Here
+ // the partition grew (hv3 joined), so the kept reservation's SourceHosts must
+ // now include hv3 and its stale Resources must be refreshed.
+ partition := partitionOf(map[string]int64{"hv1": 64, "hv2": 32, "hv3": 16})
+ existing := v1alpha1.Reservation{
+ ObjectMeta: metav1.ObjectMeta{Name: "failover-host-hv1"},
+ Spec: v1alpha1.ReservationSpec{
+ Type: v1alpha1.ReservationTypeFailoverHost,
+ TargetHost: "hv1",
+ Resources: res("8Gi", ""), // stale capacity
+ FailoverHostReservation: &v1alpha1.FailoverHostReservationSpec{SourceHosts: []string{"hv2"}},
+ },
+ }
+ topo := topology{existing: map[corev1.ObjectReference]v1alpha1.Reservation{
+ {Name: existing.Name}: existing,
+ }}
+ planner := ratioReservationPlanner{ratio: 0.34, minimum: 1} // 3 * 0.34 = 1
+
+ got := planner.PlanReservations(topo, partition, "spec.aggregates=[hana]")
+ if len(got) != 1 {
+ t.Fatalf("planned %d reservations, want 1", len(got))
+ }
+ r := got[0]
+ if r.Spec.TargetHost != "hv1" {
+ t.Fatalf("target host = %q, want kept 'hv1'", r.Spec.TargetHost)
+ }
+ wantSources := []string{"hv2", "hv3"}
+ if r.Spec.FailoverHostReservation == nil ||
+ !slices.Equal(r.Spec.FailoverHostReservation.SourceHosts, wantSources) {
+ t.Errorf("source hosts = %v, want %v", r.Spec.FailoverHostReservation, wantSources)
+ }
+ if got := r.Spec.FailoverHostReservation.PartitionDescription; got != "spec.aggregates=[hana]" {
+ t.Errorf("partition description = %q, want %q", got, "spec.aggregates=[hana]")
+ }
+ wantMem := int64(64) * 1024 * 1024 * 1024
+ if mem := r.Spec.Resources[hv1.ResourceMemory]; mem.Value() != wantMem {
+ t.Errorf("memory = %d, want refreshed %d", mem.Value(), wantMem)
+ }
+}
+
+func TestPlanReservationsSkipsSingleHostPartition(t *testing.T) {
+ // even with minimum >= 1 — reserving it would just waste the host.
+ partition := partitionOf(map[string]int64{"solo": 32})
+ empty := topology{existing: map[corev1.ObjectReference]v1alpha1.Reservation{}}
+ planner := ratioReservationPlanner{ratio: 1, minimum: 1}
+ if got := planner.PlanReservations(empty, partition, ""); len(got) != 0 {
+ t.Errorf("planned %d reservations for a 1-host partition, want 0", len(got))
+ }
+}
+
+func TestPlanReservationsLeavesOneSourceHost(t *testing.T) {
+ // With minimum larger than the partition, we must still leave at least one
+ // host unreserved to run workload; cap desiredCount at len(partition)-1.
+ partition := partitionOf(map[string]int64{"hv1": 32, "hv2": 32, "hv3": 32})
+ empty := topology{existing: map[corev1.ObjectReference]v1alpha1.Reservation{}}
+ planner := ratioReservationPlanner{ratio: 1, minimum: 99}
+ if got := planner.PlanReservations(empty, partition, ""); len(got) != len(partition)-1 {
+ t.Errorf("planned %d reservations, want %d (one source host kept)", len(got), len(partition)-1)
+ }
+}
diff --git a/tools/mirror/apply-with-status.sh b/tools/mirror/apply-with-status.sh
new file mode 100755
index 000000000..7e417f365
--- /dev/null
+++ b/tools/mirror/apply-with-status.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+#
+# apply-with-status.sh — apply dumped Kubernetes resources into a cluster,
+# status subresource included.
+#
+# WHAT IT DOES
+# `kubectl apply` never restores the `.status` of an object: status lives on a
+# separate subresource that the API server owns. When you want to replicate a
+# snapshot of resources (e.g. captured with `kubectl get ... -o yaml`) into
+# another cluster *with their status intact*, a plain apply is not enough.
+#
+# This script does it in two passes:
+# 1. Sanitize and server-side apply every item, creating the objects.
+# 2. Patch each object's `.status` onto the freshly created resource via the
+# status subresource.
+#
+# Before applying, it strips metadata that binds objects to their origin
+# cluster. The important one is `ownerReferences`: those point at owner
+# objects (e.g. Nodes) that don't exist in the target cluster, so Kubernetes'
+# garbage collector would delete the resources moments after creation. The
+# other fields (uid, resourceVersion, generation, ...) are server-managed and
+# would be rejected or cause conflicts if carried over.
+#
+# INPUT
+# Each file in FILES is expected to be a Kubernetes List (the shape produced
+# by `kubectl get -o yaml`), i.e. an object with an `.items[]` array.
+# Capture such a snapshot from the source cluster like:
+#
+# kubectl get hypervisors -o yaml > hvs.yaml.out
+# kubectl get virtualmachines -o yaml > vms.yaml.out
+#
+# USAGE
+# Point your kubeconfig/context at the *target* cluster, then run from the
+# directory holding the snapshot files:
+#
+# ./apply-with-status.sh
+#
+# By default the standard kubectl config (the KUBECONFIG env var, else
+# ~/.kube/config) and its current context are used. Override them for a
+# single run via environment variables:
+#
+# MIRROR_KUBECONFIG=target.yaml MIRROR_CONTEXT=kind-cluster-a ./apply-with-status.sh
+#
+# Adjust the FILES array below to match the snapshots you want to replay.
+#
+# REQUIREMENTS
+# bash, kubectl, and yq (https://github.com/mikefarah/yq).
+#
+set -euo pipefail
+
+# Snapshot files to replay, relative to the current directory. Each must be a
+# Kubernetes List (`.items[]`) as produced by `kubectl get ... -o yaml`.
+FILES=(hvs.yaml.out vms.yaml.out)
+
+# Seconds to wait between the apply pass and the status-patch pass, giving the
+# API server a moment to persist all created objects.
+SETTLE_SECONDS=3
+
+# Optional kubeconfig path and context for the target cluster. When empty,
+# kubectl falls back to its default config (KUBECONFIG env var, else
+# ~/.kube/config) and current context.
+MIRROR_KUBECONFIG="${MIRROR_KUBECONFIG:-}"
+MIRROR_CONTEXT="${MIRROR_CONTEXT:-}"
+
+# Assemble the kubectl flags for the chosen config/context. Empty when neither
+# is set, so kubectl uses its own defaults.
+kubectl_args=()
+if [ -n "$MIRROR_KUBECONFIG" ]; then
+ kubectl_args+=(--kubeconfig "$MIRROR_KUBECONFIG")
+fi
+if [ -n "$MIRROR_CONTEXT" ]; then
+ kubectl_args+=(--context "$MIRROR_CONTEXT")
+fi
+
+for bin in kubectl yq; do
+ if ! command -v "$bin" >/dev/null 2>&1; then
+ echo "error: required command '$bin' not found in PATH" >&2
+ exit 1
+ fi
+done
+
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+# Strip fields that bind the objects to the source cluster. ownerReferences are
+# the important one: they point at owner objects that don't exist here, so the
+# garbage collector deletes the resources right after creation. The rest are
+# server-managed and would conflict with the target cluster's own bookkeeping.
+sanitize='del(.metadata.ownerReferences, .metadata.uid, .metadata.resourceVersion,
+ .metadata.generation, .metadata.creationTimestamp,
+ .metadata.managedFields, .metadata.selfLink)'
+
+# Pass 1: create every object (spec only; status is ignored by apply).
+for f in "${FILES[@]}"; do
+ clean="$tmp/$(basename "$f").clean"
+ yq "(.items[]) |= ($sanitize)" "$f" > "$clean"
+ echo ">>> applying specs from $f"
+ kubectl ${kubectl_args[@]+"${kubectl_args[@]}"} apply --server-side --force-conflicts -f "$clean"
+done
+
+# Give the API server a moment to persist all created objects.
+sleep "$SETTLE_SECONDS"
+
+# Pass 2: patch each object's status onto the now-existing resource.
+for f in "${FILES[@]}"; do
+ n=$(yq '.items | length' "$f")
+ echo ">>> patching status for $n items from $f"
+ for i in $(seq 0 $((n - 1))); do
+ obj="$tmp/obj.yaml"
+ st="$tmp/status.yaml"
+ yq ".items[$i]" "$f" > "$obj"
+ kind=$(yq '.kind' "$obj")
+ name=$(yq '.metadata.name' "$obj")
+ # Skip items that carry no status — nothing to patch.
+ if [ "$(yq '.status // "null"' "$obj")" = "null" ]; then
+ continue
+ fi
+ yq '{"status": .status}' "$obj" > "$st"
+ kubectl ${kubectl_args[@]+"${kubectl_args[@]}"} patch "$kind" "$name" --subresource=status --type=merge --patch-file="$st"
+ done
+done