diff --git a/plugins/cozystack/.claude-plugin/plugin.json b/plugins/cozystack/.claude-plugin/plugin.json index 496ead3..76e1766 100644 --- a/plugins/cozystack/.claude-plugin/plugin.json +++ b/plugins/cozystack/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "cozystack", - "version": "1.17.0", + "version": "1.18.0", "description": "Cozystack platform skills bundle. Start with cozystack:wizard — it begins with a free-form 'tell me about your setup and goal' question, parses hints, then asks Talos / Ubuntu / Existing, builds a chain, and dispatches downstream skills via a cluster config directory the operator picks (every artifact lives there: inventory.yml, kubeconfig, .state.yaml, cozystack-platform-package.yaml — operator manages git on their own; optional sops opt-in encrypts secret files in-tree). Skills, invoked as cozystack:: wizard (orchestrator + 3-route dispatcher + Phase 4.5 active research + auto-dispatches debug on any failed_at), talos-bootstrap (Talos node prep via talm — Talos-1.12-aware maintenance probe, NAT-provider cert-SAN guardrail before first talm apply, multidoc machine-config with per-node VIP-link IPv4 stubs, etcd bootstrap, kubeconfig fetch, cozystack-tuned shape verification with Phase 11.5 auto-upgrade), talos-reset (cloud-provider terminate+relaunch helper for OCI/AWS/GCP/Hetzner when nodes are unrecoverable from inside; preserves block volumes + secondary VNICs + NSG memberships), ubuntu-bootstrap (wraps cozystack/ansible-cozystack — OS prep + k3s install in one go), cluster-install (Cozystack on a ready cluster — node-readiness, ZFS pool provisioning via privileged DaemonSet on Talos with hostNetwork, extractedprism for kube-apiserver HA, OCI-tag-normalized cozy-installer chart, Platform Package, inline tenants/root ingress patch + LINSTOR pool registration during watch loop with combined HRs-Ready + pools-registered gate, Phase 8.6 default StorageClasses for v1.3.x, Phase 9.1 end-to-end reachability probe), debug (investigate a stuck or broken install — gathers symptoms, classifies operator error / config drift / upstream bug / not-yet-supported, applies fixes or workarounds, drafts upstream issues on approval; never opens PRs or files silently), cluster-upgrade (v1.x patch/minor upgrade with release-notes analysis), package-deploy (dev-loop deploy of a single package with ExternalArtifact support), package-bump (single-package version bump with changelog adaptation), external-app-create (scaffold a new external-apps package), dev-ui-bootstrap (bootstrap a UI dev sandbox — Playwright + Vite dev server for cozystack-ui pointed at a chosen kubeconfig; worktree, @playwright/test, Chromium, playwright.config.ts, dev:e2e / test:e2e scripts, kubectl proxy + Vite on a free port), comment-audit (audit a diff's code comments for design-doc prose — rationale, product motivation, before/after narrative, incident retelling — and report cuts, tightens, keeps and arguments duplicated across sites; reports only, modifies nothing), comment-trim (apply the cuts — delete the prose, tighten verbose-but-legitimate blocks, keep every comment that documents a real mechanism, then prove mechanically with the bundled comments-only.py that no code moved and a grep-based pointer-invariant check that no pointer sits on top of the prose it replaces; leaves the result in the working tree, does not commit or push). comment-audit and comment-trim are repository-agnostic and usable on any codebase. All skills match the operator's natural language detected from conversation context — code identifiers, commands, file paths, and GitHub-public text stay canonical. All skills follow the same gate-and-confirm discipline: read-only lookups run freely; any mutation needs explicit per-step approval.", "author": { "name": "Cozystack", diff --git a/plugins/cozystack/skills/cluster-install/SKILL.md b/plugins/cozystack/skills/cluster-install/SKILL.md index 65f5184..369f3a9 100644 --- a/plugins/cozystack/skills/cluster-install/SKILL.md +++ b/plugins/cozystack/skills/cluster-install/SKILL.md @@ -1,7 +1,7 @@ --- name: cluster-install description: Use when installing Cozystack on an existing Kubernetes cluster (kubeadm / k3s / RKE2 / managed). Discovers cluster facts, validates node readiness via kubectl debug, recommends an installer + platform variant, gathers values interactively, creates the ZFS pool on each storage node through kubectl debug (cozystack standardises on ZFS for LINSTOR — LVM / LVM-thin paths are not supported), installs extractedprism (per-node kube-apiserver HA proxy, generic variant default), installs the cozy-installer chart, applies the Platform Package (on current releases OIDC is enabled after convergence, not at install), patches the root Tenant host + ingress, waits until every HelmRelease is Ready, prints a NOTES-style access summary, and offers an issue-template handoff to cozystack/* on fatal failure. Not for Kubernetes bootstrap and not for upgrades — see `cozystack:cluster-upgrade` for that. Talos node-prep is out of scope — `cozystack:cluster-install` refuses if Talos nodes are missing cozystack-tuned extensions and points at `cozystack:talos-bootstrap`. -argument-hint: "[--config-dir=] [--context=] [--installer-version=] [--installer-variant=] [--platform-variant=] [--no-extractedprism] [--api-host=] [--dry-run]" +argument-hint: "[--config-dir=] [--context=] [--installer-version=] [--installer-variant=] [--platform-variant=] [--no-extractedprism] [--api-host=] [--dry-run]" --- # cozystack:cluster-install @@ -75,7 +75,7 @@ Collect: - Existing storage classes and the default. - Existing LoadBalancer-class services. - Control-plane node labels: `node-role.kubernetes.io/control-plane` key **and value** per node (see `references/node-checks.md`). -- **Storage discovery** (per node, via `kubectl debug node` in read-only mode — `lsblk`, `pvs`, `vgs`, `lvs --all`, `zpool list`, `command -v pvcreate vgcreate lvcreate lvs`; on Talos additionally `lsmod` for `drbd`/`zfs`/`openvswitch` and `/etc/lvm/lvm.conf` global_filter). Records: unmounted devices ≥ 50 GiB, existing VGs / LVs / zpools (will be reused as Phase 4 defaults), LVM-tools availability, Talos extension readiness. Skip on `isp-hosted`. +- **Storage discovery** (per node, via `kubectl debug node` in read-only mode — `lsblk`, `pvs`, `vgs`, `lvs --all`, `zpool list`, `command -v pvcreate vgcreate lvcreate lvs`; on Talos additionally `lsmod` for `drbd`/`zfs`/`openvswitch` and `/etc/lvm/lvm.conf` global_filter). Records: unmounted devices ≥ 50 GiB, existing VGs / LVs / zpools (will be reused as Phase 4 defaults), LVM-tools availability, Talos extension readiness. Skip on `isp-hosted` and `isp-hosted-slim`. Apply variant recommendation logic from `references/variants.md`. Print: @@ -106,17 +106,18 @@ recommended: Hard refusals (don't move on, surface clearly): - Cluster domain is not `cozy.local` → refuse, link to bootstrap docs for the user's distribution. +- Selected variant (recommended, overridden or passed with `--platform-variant`) is `isp-slim` / `isp-slim-generic` and any node has an empty `spec.podCIDR` (`kubectl --context $CTX get nodes --output jsonpath='{range .items[*]}{.metadata.name}={.spec.podCIDR}{"\n"}{end}'`) → refuse: Cilium takes pod ranges from that field there. Fix with `--pod-network-cidr` on kubeadm or `--allocate-node-cidrs` plus `--cluster-cidr` on kube-controller-manager, or pick the full variant. - CNI conflict (non-hosted) → refuse, list pods to remove. - Conflicting workloads (ingress-nginx, cert-manager, etc., non-hosted) → refuse, list workloads. - kubectl can't reach the cluster → refuse, suggest `kubectl auth can-i '*' '*' --all-namespaces`. ## Phase 3 — Node readiness validation -Skip on `isp-hosted` (no node access required). +Skip on `isp-hosted` and `isp-hosted-slim` (no node access required). For every other case, drive `kubectl debug node` per `references/node-checks.md`. For homogeneous clusters, check one sample CP node and one sample worker; warn that the rest are assumed identical. For heterogeneous, check every node. -**Talos-specific early-exit.** If Phase 2 detected Talos on any node, run the four Talos checks from `references/node-checks.md` (lsmod drbd / lsmod zfs / lsmod openvswitch / `/etc/lvm/lvm.conf` cozystack global_filter). If any of them fail on any Talos node, **STOP GATE 1 fails immediately** with this message: +**Talos-specific early-exit.** If Phase 2 detected Talos on any node, run the four Talos checks from `references/node-checks.md` (lsmod drbd / lsmod zfs / lsmod openvswitch / `/etc/lvm/lvm.conf` cozystack global_filter). On `isp-slim` drop the openvswitch check: there is no Kube-OVN. If any of them fail on any Talos node, **STOP GATE 1 fails immediately** with this message: ```text Talos nodes detected without cozystack-tuned extensions. @@ -224,12 +225,14 @@ options: Slot legend (every slot the operator may want to edit): 1. **Bundles** (multiSelect, defaults from variant overlay): - - system (required for `isp-full*`; off for `isp-hosted`) + - system (on for every `isp-*` variant except `isp-hosted` up to v1.6.x; `isp-hosted*` render it with a noop networking Package) - paas (databases / applications) - - iaas (Cluster API + VMs) + - iaas (Cluster API + VMs; refused on `isp-hosted*` and `isp-slim*` — do not offer it there) - naas (Network as a Service) -2. **Storage** (only when `system` bundle is on — `isp-hosted` skips). Cozystack standardises on **ZFS** for LINSTOR pools; see `references/storage-backends.md`. No backend question — only device selection and pool layout per node. + On a slim variant, also collect **enabledPackages**: the packages to opt in, as full Package names (`cozystack.postgres-operator`, `cozystack.postgres-application`, …), each with its whole `dependsOn` chain (see `references/variants.md`). Default empty. Rendered into `bundles.enabledPackages` in the Platform Package. + +2. **Storage** (only when `system` bundle is on — `isp-hosted` and `isp-hosted-slim` skip). Cozystack standardises on **ZFS** for LINSTOR pools; see `references/storage-backends.md`. No backend question — only device selection and pool layout per node. a. **Per-node disk selection** — for every storage-providing node, show Phase 2's unmounted-device list and ask which device(s) to use. Default is the largest unmounted disk ≥ 50 GiB. If a node has multiple candidates or the operator wants a mirror / RAID-Z, prompt for a vdev layout: @@ -255,8 +258,8 @@ Slot legend (every slot the operator may want to edit): Refuse to proceed if any storage node lacks ZFS tooling (`zpool` / `zfs` binaries) — the Phase 3 storage discovery would have caught it, but re-verify here in case a fix was applied since. RHEL 10 family is not supported on the storage path; see `references/known-failures.md`. -3. **podCIDR / serviceCIDR / joinCIDR** — show detected as defaults. If user picks Other, validate format (CIDR notation, no overlap with host networks the cluster sees, joinCIDR ≠ podCIDR ≠ serviceCIDR). -4. **podGateway** — auto-derive as the first IP of podCIDR, confirm. +3. **podCIDR / serviceCIDR / joinCIDR** — skip on `isp-slim` / `isp-slim-generic` (Kube-OVN only; Cilium takes pod CIDRs from `node.spec.podCIDR`). Otherwise show detected as defaults. If user picks Other, validate format (CIDR notation, no overlap with host networks the cluster sees, joinCIDR ≠ podCIDR ≠ serviceCIDR). +4. **podGateway** — skip on `isp-slim*`. Otherwise auto-derive as the first IP of podCIDR, confirm. 5. **apiServerHost** — the address Cilium / KubeOVN / cozystack-operator dial to reach kube-apiserver. The skill picks this automatically by installer variant — **do not ask the operator** unless they passed `--api-host=`. - **`talos` variant** — `localhost:7445` (KubePrism, built into Talos machine-config). Operator can't override; Cozystack's `values-isp-full.yaml` overlay hard-codes this for Cilium. @@ -266,7 +269,7 @@ Slot legend (every slot the operator may want to edit): The plan presentation in Phase 5 always shows which choice landed and how to flip it. 6. **LB / external IPs** — - - Mode: `externalIPs` (recommended for now; deprecated upstream in k8s v1.36) vs `loadBalancer` (Cilium L2/BGP or external cloud LB; needs more wiring). + - Mode: `externalIPs` (recommended for now; deprecated upstream in k8s v1.36) vs `loadBalancer` (Cilium L2/BGP or external cloud LB; needs more wiring). On `isp-slim*` MetalLB is off unless `cozystack.metallb` is opted in: `loadBalancer` mode needs an admin-created `CiliumLoadBalancerIPPool` + `CiliumL2AnnouncementPolicy` (the platform enables Cilium L2 announcements on slim). - Pool: resolved from `state.cozystack_intake.external_ips` (wizard Phase 4). The wizard already asked the operator's strategy — `internal` / `external` / `explicit` — and the reason. Here the skill **validates the strategy against live `Node.status.addresses`** and refuses to silently override: ```bash @@ -355,7 +358,7 @@ Slot legend (every slot the operator may want to edit): 10. **exposedServices** (multiSelect): `api`, `dashboard`, `vm-exportproxy`, `cdi-uploadproxy`. Default `api,dashboard`. -11. **KubeOVN `MASTER_NODES`** — branch on Phase 2 finding: +11. **KubeOVN `MASTER_NODES`** — skip on `isp-slim*` and `isp-hosted*` (no Kube-OVN). Otherwise branch on Phase 2 finding: - If CP label value matches variant expectation on at least one node → default empty (let lookup work). - Otherwise → pre-fill comma-separated INTERNAL-IPs of CP nodes from Phase 2 and explain why the lookup would fail. @@ -750,7 +753,7 @@ kubectl --context $CTX get hr --all-namespaces \ --output jsonpath='{range .items[?(@.status.conditions[?(@.type=="Ready" && @.status!="True")])]}{.metadata.namespace}/{.metadata.name} {end}' ``` -On the full `system`-bundle path you may also want the root tenant's `etcd`/`monitoring`/`seaweedfs` services (this is what cozystack's own `hack/e2e-install-cozystack.bats` patches): extend the patch to `{"spec":{"ingress":true,"host":"${HOST}","monitoring":true,"etcd":true,"seaweedfs":true}}` when those were selected in Phase 4. Leave them at their defaults otherwise. +On the full `system`-bundle path you may also want the root tenant's `etcd`/`monitoring`/`seaweedfs` services (this is what cozystack's own `hack/e2e-install-cozystack.bats` patches): extend the patch to `{"spec":{"ingress":true,"host":"${HOST}","monitoring":true,"etcd":true,"seaweedfs":true}}` when those were selected in Phase 4. Leave them at their defaults otherwise. On `isp-slim*` / `isp-hosted-slim` turn on only the switches whose application chain is in `enabledPackages` (`cozystack.etcd-application`, `cozystack.monitoring-application`, `cozystack.seaweedfs-application` and their dependencies); otherwise the Tenant's release never becomes Ready. ```text HelmRelease $NS/$NAME has been Failing for $T minutes. diff --git a/plugins/cozystack/skills/cluster-install/references/helmrelease-monitoring.md b/plugins/cozystack/skills/cluster-install/references/helmrelease-monitoring.md index 397ebca..f18a297 100644 --- a/plugins/cozystack/skills/cluster-install/references/helmrelease-monitoring.md +++ b/plugins/cozystack/skills/cluster-install/references/helmrelease-monitoring.md @@ -46,9 +46,12 @@ Typical counts for orientation (not gospel): | Variant | Approx HR count | | ----------- | ----------- | -| `isp-full` | 40–50 | -| `isp-full-generic` | 40–50 | -| `isp-hosted` | 15–20 | +| `isp-full` | ~90, plus tenant-root releases | +| `isp-full-generic` | ~90, plus tenant-root releases | +| `isp-hosted` | ~60 on v1.7.0+, plus tenant-root releases | +| `isp-slim` | ~27 before opt-ins, plus tenant-root releases | +| `isp-slim-generic` | ~27 before opt-ins, plus tenant-root releases | +| `isp-hosted-slim` | ~18 before opt-ins, plus tenant-root releases | | `default` | 1–3 | ## Reading a stuck HR diff --git a/plugins/cozystack/skills/cluster-install/references/node-checks.md b/plugins/cozystack/skills/cluster-install/references/node-checks.md index 44a0ffe..51c1532 100644 --- a/plugins/cozystack/skills/cluster-install/references/node-checks.md +++ b/plugins/cozystack/skills/cluster-install/references/node-checks.md @@ -29,7 +29,7 @@ lsmod | awk '{print $1}' | grep -Ex 'overlay|br_netfilter|nf_conntrack|ip_tables Required: `overlay`, `br_netfilter`, `nf_conntrack`, `ip_tables`, `iptable_nat`. -For Kube-OVN (non-hosted): `openvswitch`, `geneve` (autoloaded by ovs userspace — soft check). +For Kube-OVN (`isp-full*` only; not needed on hosted or slim variants): `openvswitch`, `geneve` (autoloaded by ovs userspace — soft check). For DRBD storage (non-hosted): either `drbd` already in `lsmod` or a buildable kernel headers tree (`/usr/src/linux-headers-$(uname -r)`) so piraeus-operator can compile in-cluster. Ubuntu Secure Boot needs `drbd-dkms` pre-installed and signed — `lsmod | grep drbd` must succeed before install. @@ -139,14 +139,14 @@ Skip the whole storage section on hosted variant. ### Talos cozystack-tuned image — Phase 3 gate (Talos only) -If Phase 2 detected Talos on any node, run these four checks. **All four must pass on every Talos node**, otherwise Phase 3 STOP GATE 1 fails with `cozystack:cluster-install` refusing to continue and pointing at `/cozystack:talos-bootstrap`. +If Phase 2 detected Talos on any node, run these four checks. **All four must pass on every Talos node** (on `isp-slim` the openvswitch check is dropped, there is no Kube-OVN), otherwise Phase 3 STOP GATE 1 fails with `cozystack:cluster-install` refusing to continue and pointing at `/cozystack:talos-bootstrap`. ```sh # Kernel modules (cozystack-tuned image ships these as extensions; vanilla Talos does not) lsmod | awk '{print $1}' | grep -Ex 'drbd|zfs|openvswitch' ``` -All three names must appear in the output. +All three names must appear in the output (`drbd` and `zfs` on `isp-slim`). ```sh # LVM filter — cozystack-tuned machine-config writes this verbatim @@ -181,7 +181,8 @@ KubeOVN's chart looks nodes up by a key=value pair from `MASTER_NODES_LABEL`. ** | ----------- | ----------- | | `isp-full` (Talos) | `node-role.kubernetes.io/control-plane=""` (empty value — Talos default) | | `isp-full-generic` (k3s / kubeadm / RKE2) | `node-role.kubernetes.io/control-plane=true` (literal string `true`) | -| `isp-hosted` | not used — KubeOVN is not deployed | +| `isp-hosted`, `isp-hosted-slim` | not used — KubeOVN is not deployed | +| `isp-slim`, `isp-slim-generic` | not used — networking is Cilium alone | | `default` | not used unless the operator hand-rolls KubeOVN | The lookup compares the value byte-for-byte. If it doesn't match exactly, the chart `fail`s with: @@ -214,7 +215,7 @@ If the value does **not** match what the variant expects, surface two recovery p 2. **Pin explicit IPs via `MASTER_NODES`** — bypasses the lookup entirely. Collect the CP nodes' internal IPs (`kubectl get nodes -o wide`, INTERNAL-IP column) and set `networking.kubeovn.MASTER_NODES` to the comma-separated list in the values collected in Phase 4. This is the safest choice for kubeadm. -Gate: do not apply the Platform Package until **either** the label matches the variant's expected value on at least one node, **or** `MASTER_NODES` is set explicitly in the collected values. Skip this gate on `isp-hosted`. +Gate: do not apply the Platform Package until **either** the label matches the variant's expected value on at least one node, **or** `MASTER_NODES` is set explicitly in the collected values. Skip this gate on `isp-hosted`, `isp-hosted-slim`, `isp-slim` and `isp-slim-generic`. ### CNI conflict (cluster-wide) diff --git a/plugins/cozystack/skills/cluster-install/references/requirements.md b/plugins/cozystack/skills/cluster-install/references/requirements.md index 8456e50..30a9df6 100644 --- a/plugins/cozystack/skills/cluster-install/references/requirements.md +++ b/plugins/cozystack/skills/cluster-install/references/requirements.md @@ -16,7 +16,7 @@ Source of truth: `https://cozystack.io/docs/v1.3/install/kubernetes/generic/` an - **Cluster domain**: `cozy.local`. The Package chart enforces this in `networking.clusterDomain` and components assume it. If kube-apiserver was bootstrapped with `cluster.local` (or anything else), `cozystack:cluster-install` must refuse — re-bootstrap of the cluster is the only fix. - **podCIDR / serviceCIDR**: must match what kube-apiserver and kubelet were started with. Mismatched values silently break service routing. -- **CNI**: none installed. Cozystack ships Cilium + Kube-OVN. If any other CNI pod (Calico, Flannel, Weave, AWS VPC CNI) is running in `kube-system`, refuse with explanation. Exception: managed/hosted variant — provider CNI stays. +- **CNI**: none installed. Cozystack ships Cilium + Kube-OVN (Cilium alone on slim). If any other CNI pod (Calico, Flannel, Weave, AWS VPC CNI) is running in `kube-system`, refuse with explanation. Exception: managed/hosted variant — provider CNI stays. - **kube-proxy**: disabled. Cilium replaces it. If kube-proxy DaemonSet exists, the install will conflict. - **Ingress controller**: none installed. Cozystack ships ingress-nginx (`cozy-ingress-nginx`). - **cert-manager**: none installed. Cozystack ships its own (`cozy-cert-manager`). @@ -50,7 +50,7 @@ Source of truth: `https://cozystack.io/docs/v1.3/install/kubernetes/generic/` an - All nodes in the same L2 segment (or KubeSpan with RTT < 10 ms). - LB IP range or external IPs reserved and routable. - DNS: either real FQDN under operator's control, or `.nip.io` for sandbox. -- L2 anti-spoofing disabled on the upstream switch if MetalLB L2 mode is used. +- L2 anti-spoofing disabled on the upstream switch if MetalLB L2 mode (or Cilium L2 announcements on the slim variants) is used. ## Installer variant ↔ platform variant mapping @@ -61,11 +61,14 @@ The cozy-installer Helm chart has its own variant (`cozystackOperator.variant`) | `talos` | `isp-full` (or `distro-full`) | Talos Linux nodes — full IaaS + PaaS. | | `generic` | `isp-full-generic` | kubeadm / k3s / RKE2 — full IaaS + PaaS on generic Linux. Requires `cozystack.apiServerHost` (internal IP of CP node). | | `hosted` | `isp-hosted` | Managed k8s — PaaS only (no VMs, no LINSTOR). | +| `talos` | `isp-slim` | Talos Linux nodes — base platform only, Cilium without Kube-OVN; everything else opt-in. | +| `generic` | `isp-slim-generic` | kubeadm / k3s / RKE2 — base platform only, Cilium without Kube-OVN; everything else opt-in. | +| `hosted` | `isp-hosted-slim` | Managed k8s — Cozystack control plane only; everything else opt-in. | | any | `default` | Bare minimum — controller only, no bundles. Power-user / development. | ## Where this is enforced - Chart values defaults: `~/git/github.com/cozystack/cozystack/packages/core/installer/values.yaml`. - Platform defaults: `~/git/github.com/cozystack/cozystack/packages/core/platform/values.yaml`. -- Variant overlays: `packages/core/platform/values-isp-full*.yaml`, `values-isp-hosted.yaml`. +- Variant overlays: `packages/core/platform/values-isp-full*.yaml`, `values-isp-hosted.yaml`, `values-isp-slim*.yaml`, `values-isp-hosted-slim.yaml`. - Ansible reference: `~/git/github.com/cozystack/ansible-cozystack/roles/cozystack/{defaults,tasks}/main.yml` — read this when in doubt about a value's required form. diff --git a/plugins/cozystack/skills/cluster-install/references/values-template.md b/plugins/cozystack/skills/cluster-install/references/values-template.md index 15d5a2f..34e4a46 100644 --- a/plugins/cozystack/skills/cluster-install/references/values-template.md +++ b/plugins/cozystack/skills/cluster-install/references/values-template.md @@ -82,7 +82,7 @@ kind: Package metadata: name: cozystack.cozystack-platform spec: - variant: isp-full-generic # or isp-full / isp-hosted / default + variant: isp-full-generic # or isp-full / isp-hosted / isp-slim / isp-slim-generic / isp-hosted-slim / default components: platform: values: @@ -90,11 +90,12 @@ spec: system: enabled: true iaas: - enabled: true + enabled: true # false on isp-hosted* and isp-slim*: the iaas bundle refuses to render there paas: enabled: true naas: enabled: true + enabledPackages: [] # slim variants: full Package names to opt in, with their dependsOn chains authentication: oidc: enabled: false # OIDC OFF at install (installer >=1.5.0). @@ -110,7 +111,7 @@ spec: serviceCIDR: "10.96.0.0/16" # cozystack default joinCIDR: "100.64.0.0/16" # cozystack default kubeovn: - MASTER_NODES: "" # comma-separated CP IPs; leave empty to let Helm lookup find them + MASTER_NODES: "" # comma-separated CP IPs; leave empty to let Helm lookup find them. On isp-slim* the CIDR and kubeovn values are ignored (clusterDomain still applies, stageCniPlugins only when cozystack.multus is opted in; encryption.enabled is refused) publishing: host: "example.com" apiServerEndpoint: "https://api.example.com:6443" @@ -233,5 +234,8 @@ This is what creates the `IngressClass` and brings up `ingress-nginx`. - `~/git/github.com/cozystack/cozystack/packages/core/platform/values-isp-full.yaml` - `~/git/github.com/cozystack/cozystack/packages/core/platform/values-isp-full-generic.yaml` - `~/git/github.com/cozystack/cozystack/packages/core/platform/values-isp-hosted.yaml` +- `~/git/github.com/cozystack/cozystack/packages/core/platform/values-isp-slim.yaml` +- `~/git/github.com/cozystack/cozystack/packages/core/platform/values-isp-slim-generic.yaml` +- `~/git/github.com/cozystack/cozystack/packages/core/platform/values-isp-hosted-slim.yaml` These describe **only** the bundle deltas. Use the merged values from `packages/core/platform/values.yaml` plus the variant overlay as the baseline for what the user is editing in Phase 4. diff --git a/plugins/cozystack/skills/cluster-install/references/variants.md b/plugins/cozystack/skills/cluster-install/references/variants.md index 0f671b0..e09aa7b 100644 --- a/plugins/cozystack/skills/cluster-install/references/variants.md +++ b/plugins/cozystack/skills/cluster-install/references/variants.md @@ -18,7 +18,16 @@ Same stack as `isp-full`, but for generic Linux (kubeadm / k3s / RKE2). Requires Requires the internal IP of the API server (`cozystack.apiServerHost`) — the operator needs to reach kube-apiserver to read state for things like KubeOVN's `MASTER_NODES` lookup. ### `hosted` / `isp-hosted` -PaaS-only on top of a managed Kubernetes (EKS / GKE / AKS / DOKS) or any vanilla cluster where Cozystack should not manage networking/storage/VMs. No Cilium override, no Kube-OVN, no LINSTOR, no KubeVirt. Bundles: `paas` and `naas` on by default; `system` and `iaas` off. +PaaS-only on top of a managed Kubernetes (EKS / GKE / AKS / DOKS) or any vanilla cluster where Cozystack should not manage networking/storage/VMs. No Cilium override, no Kube-OVN, no LINSTOR, no KubeVirt. Bundles: `paas` and `naas` on, `iaas` off and refused. `system` is off in the preset up to v1.6.x and on (with a noop networking Package) from v1.7.0. + +### `talos` / `isp-slim`, `generic` / `isp-slim-generic`, `hosted` / `isp-hosted-slim` +Minimal counterparts of the three variants above, for small installs such as arm64 boards and labs. Only the base platform is installed: engine, API and dashboard, tenants, ingress, gateway, and LINSTOR on `isp-slim` / `isp-slim-generic` (none on `isp-hosted-slim`). Every paas/naas application and operator, monitoring, backups, etcd, SeaweedFS, metrics-server and VPA are opt-in through `bundles.enabledPackages`; MetalLB and Multus too on `isp-slim` / `isp-slim-generic` (on `isp-hosted-slim` they are not rendered at all). An opt-in package does not pull in its `dependsOn`: the whole chain has to be listed (for example monitoring needs `cozystack.monitoring-application`, `cozystack.grafana-operator` and `cozystack.postgres-operator`, plus `cozystack.monitoring-agents`, `cozystack.metrics-server` and `cozystack.vertical-pod-autoscaler` for the agents that feed it). Entries are full Package names with the `cozystack.` prefix; a bare name matches nothing. + +- The `iaas` bundle is refused (no KubeVirt, no managed Kubernetes). +- Networking on `isp-slim` / `isp-slim-generic` is Cilium alone — no Kube-OVN, so `MASTER_NODES` and the `networking.podCIDR` / `podGateway` / `serviceCIDR` / `joinCIDR` values are ignored; pod CIDRs come from `node.spec.podCIDR`, which Talos and k3s allocate by default (kubeadm needs `--pod-network-cidr`). +- On `isp-slim` / `isp-slim-generic` MetalLB is off by default (opt-in `cozystack.metallb`): `LoadBalancer` Services get addresses from Cilium L2 announcements through an admin-created `CiliumLoadBalancerIPPool` + `CiliumL2AnnouncementPolicy`, or use `publishing.externalIPs`. On `isp-hosted-slim` the provider's load balancer serves them. +- `networking.encryption.enabled` is refused on `isp-slim` / `isp-slim-generic` (ignored on `isp-hosted-slim`, like on `isp-hosted`); `monitoring.rootEnabled` is `false` in the slim presets. +- Available only on Cozystack releases whose operator registers them. Before offering one, check that `cmd/cozystack-operator/main.go` at the target `installer_version` tag mentions `isp-slim`: in the local clone (`git -C ~/git/github.com/cozystack/cozystack grep --quiet isp-slim -- cmd/cozystack-operator/main.go` after `fetch --tags`), or at `https://raw.githubusercontent.com/cozystack/cozystack//cmd/cozystack-operator/main.go`. If the file does not mention it, or neither source can be read, offer the full variant. After the operator is installed, confirm with `kubectl --context $CTX get packagesource cozystack.cozystack-platform --output jsonpath='{.spec.variants[*].name}'` before applying the Platform Package. ### `default` Bare minimum — operator + PackageSource + reconciler. No bundles enabled. The operator can later be steered by hand-rolled Package CRs. Power-user territory; rarely the right pick for click-ops. @@ -36,6 +45,8 @@ Drive it off cluster lookups in this order: 4. Otherwise: → recommend `default`, but warn that the user is on their own. +Then pick full vs slim: when the operator does not need VMs or managed Kubernetes (small or arm64 nodes make slim the better default, but a stated need for VMs wins), recommend the slim counterpart (`isp-slim` / `isp-slim-generic` / `isp-hosted-slim`); the packages to opt in are collected by the Phase 4 `enabledPackages` slot. Otherwise recommend the full variant. + Surface the chosen recommendation as the first option in the AskUserQuestion. Always allow override. ## Bundles inside a variant @@ -44,15 +55,15 @@ Independently of variant, bundles are flags: | Bundle | Default for variant | Contains | | ----------- | ----------- | ----------- | -| `system` | on for `isp-full*`, off for `isp-hosted` | Cilium, Kube-OVN, LINSTOR, ingress-nginx, cert-manager, KubeVirt — the platform infra. | -| `iaas` | on for `isp-full*`, off for `isp-hosted` | Cluster API, Kamaji (managed k8s for tenants), VM provisioning. | -| `paas` | on everywhere | MariaDB / PostgreSQL / Redis / RabbitMQ / Kafka / Grafana / VictoriaMetrics operators. | -| `naas` | on everywhere | Network-as-a-Service for tenants. | +| `system` | on for every `isp-*` variant, except `isp-hosted` up to v1.6.x (`isp-hosted*` swap the data plane for a noop networking Package) | Cilium, Kube-OVN (not on slim), LINSTOR, cert-manager, the engine and the application catalog. | +| `iaas` | on for `isp-full*`; off and refused on `isp-hosted*` and `isp-slim*` | Cluster API, Kamaji (managed k8s for tenants), VM provisioning. | +| `paas` | on everywhere (every package opt-in on slim) | MariaDB / PostgreSQL / Redis / RabbitMQ / Kafka / Grafana / VictoriaMetrics operators. | +| `naas` | on everywhere (every package opt-in on slim) | Network-as-a-Service for tenants. | `enabledPackages` / `disabledPackages` override individual packages inside whichever bundle they belong to. ## Source of truth - `~/git/github.com/cozystack/cozystack/packages/core/installer/values.yaml` — variant key for the operator. -- `~/git/github.com/cozystack/cozystack/packages/core/platform/values-isp-full.yaml`, `values-isp-full-generic.yaml`, `values-isp-hosted.yaml` — what each platform variant turns on. +- `~/git/github.com/cozystack/cozystack/packages/core/platform/values-isp-full.yaml`, `values-isp-full-generic.yaml`, `values-isp-hosted.yaml`, `values-isp-slim.yaml`, `values-isp-slim-generic.yaml`, `values-isp-hosted-slim.yaml` — what each platform variant turns on. - `https://cozystack.io/docs/v1.3/install/kubernetes/generic/` — canonical generic flow. diff --git a/plugins/cozystack/skills/debug/references/issue-templates.md b/plugins/cozystack/skills/debug/references/issue-templates.md index 174f9da..e26b962 100644 --- a/plugins/cozystack/skills/debug/references/issue-templates.md +++ b/plugins/cozystack/skills/debug/references/issue-templates.md @@ -7,7 +7,7 @@ Phase 5 of `cozystack:debug` renders one of these templates into `/d ```markdown **Cozystack version**: **Kubernetes**: -**Platform variant**: +**Platform variant**: **Install path**: **Nodes**: × on diff --git a/plugins/cozystack/skills/ubuntu-bootstrap/references/ansible-playbook.md b/plugins/cozystack/skills/ubuntu-bootstrap/references/ansible-playbook.md index 8cf2ce1..af687a2 100644 --- a/plugins/cozystack/skills/ubuntu-bootstrap/references/ansible-playbook.md +++ b/plugins/cozystack/skills/ubuntu-bootstrap/references/ansible-playbook.md @@ -37,11 +37,11 @@ curl -sfL https://get.k3s.io | \ Flag rationale: - `--cluster-init` — initialise embedded etcd raft on this node; required for HA. Single-node sandbox still works with this flag. -- `--flannel-backend=none` — disables k3s's built-in CNI. Cozystack ships Cilium + Kube-OVN. +- `--flannel-backend=none` — disables k3s's built-in CNI. Cozystack ships Cilium + Kube-OVN (Cilium alone on slim). - `--disable=traefik` — Cozystack ships its own ingress (ingress-nginx). -- `--disable=servicelb` — Cozystack ships MetalLB inside the platform Package. +- `--disable=servicelb` — Cozystack ships MetalLB inside the platform Package (on `isp-slim-generic` Cilium L2 announcements take its place). - `--disable=local-storage` — Cozystack ships LINSTOR + piraeus-operator. -- `--disable=metrics-server` — VictoriaMetrics stack covers metrics. +- `--disable=metrics-server` — Cozystack ships its own metrics-server (opt-in `cozystack.metrics-server` on `isp-slim-generic`). - `--disable-network-policy` — Cilium implements policies. - `--disable-kube-proxy` — Cilium replaces kube-proxy. - `--cluster-domain=cozy.local` — mandatory for Cozystack. Cluster-install Phase 2 refuses if this isn't set. diff --git a/plugins/cozystack/skills/wizard/SKILL.md b/plugins/cozystack/skills/wizard/SKILL.md index c73ba5f..84a07c7 100644 --- a/plugins/cozystack/skills/wizard/SKILL.md +++ b/plugins/cozystack/skills/wizard/SKILL.md @@ -289,13 +289,13 @@ For `cluster-install` first (existing k8s target): These were historically asked by `cluster-install` Phase 4 *after the cluster existed* — that re-prompted the operator for things they already knew up front. They are policy choices, not discovery-dependent, so they belong here: -- **Bundles** (multiSelect): `system`, `paas`, `iaas`, `naas`. Default per variant overlay (`isp-full` includes all four). The variant is derived from `state.target` + `intent_hints.workload_class`. +- **Bundles** (multiSelect): `system`, `paas`, `iaas`, `naas`. Default per variant overlay (`isp-full` includes all four). The variant is derived from `state.target` + `intent_hints.workload_class`. When the operator does not need VMs or managed Kubernetes (small or arm64 nodes make slim the better default, a stated need for VMs wins) and the target release ships the slim variants (version check in the `cozystack:cluster-install` variant picker), offer the slim counterpart (`isp-slim` / `isp-slim-generic` / `isp-hosted-slim`): base platform only, `iaas` refused, everything else opt-in through `bundles.enabledPackages`. The packages to opt in are collected by `cozystack:cluster-install` Phase 4, whose variant picker has the details. - **Storage layout preference** (Talos / Ubuntu routes only): for each node that will provide storage, ask the layout when more than one is plausible — `single` (Recommended for dev), `mirror` (Recommended for prod on 2-disk nodes), `raidz` (3+ disks). Default `single`. Exact device paths are deferred to `cluster-install` Phase 5.5 because they require post-boot probing; the **preference** is recorded here and Phase 5.5 just picks the largest unmounted disk(s) that fit the layout. ZFS pool name and LINSTOR storage-pool name default `data` and are rarely changed. - **Publishing host kind**: `nip.io` (Recommended for sandbox; no DNS configuration needed) vs `custom-fqdn` (operator owns the domain). When `custom-fqdn` — collect the FQDN and run the **domain-ownership gate** here, not in `cluster-install` (the operator commits to configuring DNS for it before Phase 8 of cluster-install reaches dashboard/keycloak HRs). - **External IPs strategy**: on Talos / Ubuntu routes, when nodes have distinct `InternalIP` and `ExternalIP` (typical on OCI, GCP with external NAT, AWS with EIPs), ask: - `internal` (Recommended for NAT-fronted clouds) — Service `externalIPs` are the nodes' VCN-internal addresses. NAT on the provider fabric (OCI 1:1 NAT, GCP NAT'd public IP, AWS EIP) rewrites packets before they reach the node, so the kernel only ever sees the internal IP; Cilium's externalIPs BPF must match on that. Picking public IPs here is the single most painful misconfiguration on these providers — symptom is `Connection refused` (kernel RST) on `https://dashboard./` even though every HelmRelease is Ready. - `external` (use external/public IPs) — only when the nodes' public IPs are routed onto the interface (bare-metal, Hetzner with direct public IP, dedicated server with no NAT). - - `explicit` — operator supplies the list (MetalLB pool, external LB VIP). + - `explicit` — operator supplies the list (MetalLB pool, Cilium LB pool on slim variants, external LB VIP). - The wizard probes `kubectl get nodes -o jsonpath` on Talos / Ubuntu routes after Phase 5 bootstrap completes to confirm the chosen IPs are present on the cluster's node objects, but the *choice* is asked here. - **Cert solver**: `http01` (Recommended — works with public DNS + port 80) vs `dns01` (works on internal networks, requires DNS-provider creds wired into cert-manager values). On nip.io always `http01`. On `custom-fqdn` default `http01`, ask if operator wants `dns01`. - **Exposed services** (multiSelect): `api`, `dashboard`, `vm-exportproxy`, `cdi-uploadproxy`. Default `api,dashboard`. diff --git a/plugins/cozystack/skills/wizard/references/state-schema.md b/plugins/cozystack/skills/wizard/references/state-schema.md index f772620..d8ed996 100644 --- a/plugins/cozystack/skills/wizard/references/state-schema.md +++ b/plugins/cozystack/skills/wizard/references/state-schema.md @@ -141,13 +141,15 @@ cozystack_intake: # Both names come from upstream cozystack files; do not invent values. # Real upstream platform_variant overlays in cozystack v1.3.x: # default, isp-full, isp-full-generic, isp-hosted + # plus, on releases that ship them, the minimal isp-slim, isp-slim-generic, + # isp-hosted-slim (base platform only; iaas refused; the rest opt-in) # Typical pairing: - # installer_variant=talos ↔ platform_variant=isp-full - # installer_variant=generic ↔ platform_variant=isp-full-generic - # installer_variant=hosted ↔ platform_variant=isp-hosted + # installer_variant=talos ↔ platform_variant=isp-full / isp-slim + # installer_variant=generic ↔ platform_variant=isp-full-generic / isp-slim-generic + # installer_variant=hosted ↔ platform_variant=isp-hosted / isp-hosted-slim bundles: ["system", "paas", "iaas", "naas"] installer_variant: "talos" # generic / talos / hosted (derived from target + workload_class) - platform_variant: "isp-full" # default / isp-full / isp-full-generic / isp-hosted + platform_variant: "isp-full" # default / isp-full / isp-full-generic / isp-hosted / isp-slim / isp-slim-generic / isp-hosted-slim # Storage layout preference (Talos / Ubuntu routes; "hosted" target skips) storage_pref: @@ -202,7 +204,7 @@ Written by `cluster-install`. Mirrors what gets serialised to `/cozy ```yaml cozystack: installer_variant: "talos" # generic / talos / hosted — same key as cozystack_intake - platform_variant: "isp-full" # default / isp-full / isp-full-generic / isp-hosted — upstream overlay name + platform_variant: "isp-full" # default / isp-full / isp-full-generic / isp-hosted / isp-slim / isp-slim-generic / isp-hosted-slim — upstream overlay name bundles: ["system", "paas", "iaas", "naas"] api_server_host: "127.0.0.1" # or CP1_IP / VIP / "" (hosted) api_server_port: "7445" diff --git a/plugins/cozystack/skills/wizard/references/state.schema.json b/plugins/cozystack/skills/wizard/references/state.schema.json index e9e1e8a..0aa57b9 100644 --- a/plugins/cozystack/skills/wizard/references/state.schema.json +++ b/plugins/cozystack/skills/wizard/references/state.schema.json @@ -126,7 +126,7 @@ "items": { "enum": ["system", "paas", "iaas", "naas"] } }, "installer_variant": { "enum": ["generic", "talos", "hosted"] }, - "platform_variant": { "enum": ["default", "isp-full", "isp-full-generic", "isp-hosted"] }, + "platform_variant": { "enum": ["default", "isp-full", "isp-full-generic", "isp-hosted", "isp-slim", "isp-slim-generic", "isp-hosted-slim"] }, "storage_pref": { "type": "object", "properties": { @@ -185,7 +185,7 @@ "properties": { "installer_variant": { "enum": ["generic", "talos", "hosted"] }, "installer_version": { "type": "string", "description": "Canonical git-tag form, e.g. v1.3.3. cluster-install strips the v before passing to OCI helm --version." }, - "platform_variant": { "enum": ["default", "isp-full", "isp-full-generic", "isp-hosted"] }, + "platform_variant": { "enum": ["default", "isp-full", "isp-full-generic", "isp-hosted", "isp-slim", "isp-slim-generic", "isp-hosted-slim"] }, "bundles": { "type": "array", "items": { "type": "string" } }, "api_server_host": { "type": "string" }, "api_server_port": { "type": "string" },