diff --git a/release-management/backports/01a15d41-e5b1-7d20-b6e3-e9ebcec19b81.md b/release-management/backports/01a15d41-e5b1-7d20-b6e3-e9ebcec19b81.md new file mode 100644 index 000000000000..b867bbb53a65 --- /dev/null +++ b/release-management/backports/01a15d41-e5b1-7d20-b6e3-e9ebcec19b81.md @@ -0,0 +1,98 @@ +--- +id: 01a15d41-e5b1-7d20-b6e3-e9ebcec19b81 +title: Fix nvmeof gateway startup with aes256k daemon keys +component: other +category: fix +provenance: + type: other + other: >- + Downstream-authored fix. No upstream PR exists yet. Written against + cobaltcore-dev/ceph#28 for the squid-19.2.7 effort. Upstream tracker + https://tracker.ceph.com/issues/81778 covers the same bug on squid and + tentacle. Related: tracker #79868 and ceph/ceph-nvmeof#2203. + upstream_trackers: + - https://tracker.ceph.com/issues/81778 +justification: + type: engineering + refs: + - cobaltcore-dev/cloud-storage#623 + - https://tracker.ceph.com/issues/81778 +risk: + blast: availability + conflict: substantive + coverage: partial +--- + +## Justification (public) + +Since 19.2.6, cephadm creates daemon cephx keys with the `aes256k` key type. +The NVMe-oF gateway images for squid bundle their own librados, built before +`aes256k` existed, so a gateway cannot authenticate with such a key. It fails +to connect to the cluster and restarts. The nvmeof service never reaches its +target count. + +## Justification (internal) + +- https://tracker.ceph.com/issues/81778 +- Teuthology runs #227, #259, #303, ... Surfaced in + `orch:cephadm:smoke-roleless/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-services/nvmeof 3-final}`, + job 3763 (2026-10-10): the mon grants `client.nvmeof.foo.vm06.qoywko` a key + with `"key_type": "aes256k"`. The gateway logs `rados.IOError: [errno 5] RADOS I/O error` + on each of 5 starts, then `Unable to create OMAP, exiting!`, and no mon ever + sees a connection attempt. Ends in `MaxWhileTries` after 300s. + +## Risk notes + +- **blast: availability** — the patch changes which cephx key type cephadm + requests per daemon (`utils.key_type_for()`). nvmeof gets the previous `aes`, + every other daemon type keeps `aes256k`. A regression in that dispatch would + hand a daemon a key type its image cannot use. The same failure mode this + fixes, i.e. a daemon that will not start. No data path is touched and no + on-disk format changes, so nothing can be silently corrupted. +- **conflict: substantive** — fully custom downstream patch, not a cherry-pick + (rubric: "substantive conflicts requiring code change, OR fully custom"). It + applies cleanly to the effort branch, but it carries no upstream review + behind it; the code is new on this line. +- **coverage: partial** — new + `src/pybind/mgr/cephadm/tests/test_nvmeof_key_type.py` (5 tests) covers the + per-daemon dispatch, key creation and `key_rotate`. +- **Removal condition** — `LEGACY_CIPHER_DAEMON_TYPES` has to go back to empty + once a squid-line gateway image ships librados that understands `aes256k`. + The 1.7 and 2.0.0 images already do, but those are tentacle-related images. + Until that happens, every carry-forward of this patch re-applies the + downgrade. + +## Behavior change + +NVMe-oF gateway daemon keys are created with key type `aes` instead of +`aes256k`. All other daemon types are unchanged, including the post-upgrade key +rotation, which only covers mon/mgr/osd/mds. `ceph orch daemon rotate-key` on a +gateway also produces an `aes` key. + +This is a deliberate downgrade to a weaker cipher for one daemon type. `aes` +(`CEPH_CRYPTO_AES`) is AES-128-CBC with a fixed IV and a 16-byte key; `aes256k` +(`CEPH_CRYPTO_AES256KRB5`) is AES256-CTS-HMAC-SHA384-192 with a 32-byte key. So +the gateway's key loses both the longer key and the HMAC integrity. It is the +key type every daemon used before 19.2.6 and is still in `ALLOWED_CIPHERS`, not +a new weak mode. Exposure is limited to the `client.nvmeof.*` entity, whose +caps are `mon profile rbd` / `osd profile rbd`. + +A gateway already deployed with an `aes256k` key has its key rotated to `aes` +on its next deploy, so a failed nvmeof service heals itself on redeploy rather +than needing manual cephx surgery. That rotation keys on daemon type, not on +what the running image supports, so redeploying a gateway whose image does +understand `aes256k` downgrades its key too. + +## Upgrade notes + +Clusters without an NVMe-oF service: nothing to do. + +If an nvmeof service is already stuck from a 19.2.6 or later deploy, redeploy +it once after the upgrade: + +``` +ceph orch redeploy nvmeof. +``` + +The gateway's key is rotated to a type its image supports and the service comes +up. diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index a76c4b463e73..54eed392cf03 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -2475,7 +2475,7 @@ def key_rotate(self, daemon_spec: CephadmDaemonDeploySpec) -> None: 'prefix': 'auth rotate', 'entity': daemon_spec.entity_name(), 'format': 'json', - 'key_type': utils.ROTATION_CIPHER + 'key_type': utils.key_type_for(daemon_spec.daemon_type) }) if rc: raise OrchestratorError( diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py index 114f4d5dac7c..c41eb10a983b 100644 --- a/src/pybind/mgr/cephadm/services/cephadmservice.py +++ b/src/pybind/mgr/cephadm/services/cephadmservice.py @@ -303,14 +303,15 @@ def get_active_daemon(self, daemon_descrs: List[DaemonDescription]) -> DaemonDes # defined, return empty Daemon Desc return DaemonDescription() - def get_keyring_with_caps(self, entity: AuthEntity, caps: List[str]) -> str: + def get_keyring_with_caps(self, entity: AuthEntity, caps: List[str], + key_type: str = utils.ROTATION_CIPHER) -> str: # try with newer cipher first, it's possible this isn't supported # early in an upgrade ret, keyring, err = self.mgr.mon_command({ 'prefix': 'auth get-or-create', 'entity': entity, 'caps': caps, - 'key_type': utils.ROTATION_CIPHER + 'key_type': key_type }) if err: ret, keyring, err = self.mgr.mon_command({ diff --git a/src/pybind/mgr/cephadm/services/nvmeof.py b/src/pybind/mgr/cephadm/services/nvmeof.py index f3eb623d61a2..746a6486b6b7 100644 --- a/src/pybind/mgr/cephadm/services/nvmeof.py +++ b/src/pybind/mgr/cephadm/services/nvmeof.py @@ -30,9 +30,27 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD nvmeof_gw_id = daemon_spec.daemon_id host_ip = self.mgr.inventory.get_addr(daemon_spec.host) - keyring = self.get_keyring_with_caps(self.get_auth_entity(nvmeof_gw_id), + entity = self.get_auth_entity(nvmeof_gw_id) + key_type = utils.key_type_for(self.TYPE) + keyring = self.get_keyring_with_caps(entity, ['mon', 'profile rbd', - 'osd', 'profile rbd']) + 'osd', 'profile rbd'], + key_type=key_type) + if key_type != utils.ROTATION_CIPHER and \ + utils.keyring_key_type(keyring) == utils.ROTATION_CIPHER: + # a gateway deployed with an aes256k key (cephadm did so from + # 19.2.6 on) never started: give it a key it can use + logger.info(f'rotating {entity} to a {key_type} key: the nvmeof ' + 'gateway image cannot use an aes256k key') + ret, out, err = self.mgr.mon_command({ + 'prefix': 'auth rotate', 'entity': entity, + 'format': 'json', 'key_type': key_type}) + if ret == 0: + keyring = self.get_keyring_with_caps( + entity, ['mon', 'profile rbd', 'osd', 'profile rbd'], + key_type=key_type) + else: + logger.warning(f'could not rotate {entity}: {err}') # TODO: check if we can force jinja2 to generate dicts with double quotes instead of using json.dumps transport_tcp_options = json.dumps(spec.transport_tcp_options) if spec.transport_tcp_options else None diff --git a/src/pybind/mgr/cephadm/tests/test_nvmeof_key_type.py b/src/pybind/mgr/cephadm/tests/test_nvmeof_key_type.py new file mode 100644 index 000000000000..62f6db304254 --- /dev/null +++ b/src/pybind/mgr/cephadm/tests/test_nvmeof_key_type.py @@ -0,0 +1,87 @@ +""" +nvmeof gateways get a key of the previous type (aes): the gateway images of +this release carry a librados from before aes256k and cannot connect with +an aes256k key. +""" +import base64 +import struct +from unittest.mock import MagicMock + +from ceph.deployment.service_spec import NvmeofServiceSpec + +from cephadm import utils +from cephadm.services.cephadmservice import CephadmDaemonDeploySpec +from cephadm.services.nvmeof import NvmeofService + + +def keyring(entity: str, key_type: int) -> str: + raw = struct.pack(' Optional[str]: + """ + The type of the key in a keyring ('aes', 'aes256k'), from the key's + first two bytes (CryptoKey: le16 type, utime_t created, le16 len, + secret), or None if there is no key to read. + """ + import base64 + import struct + for line in keyring.splitlines(): + name, _, value = line.partition('=') + if name.strip() != 'key': + continue + try: + raw = base64.b64decode(value.strip()) + t = struct.unpack(' str: + """The cephx key type to create or rotate a daemon's key with.""" + if daemon_type in LEGACY_CIPHER_DAEMON_TYPES: + return LEGACY_CIPHER + return ROTATION_CIPHER + + # cipher mons should use by default for service requests # Do not set the service cipher until all core (mon/mgr/osd/mds) # daemons have been upgraded diff --git a/src/pybind/mgr/tox.ini b/src/pybind/mgr/tox.ini index 3165673cdbcd..68629e0e2d5e 100644 --- a/src/pybind/mgr/tox.ini +++ b/src/pybind/mgr/tox.ini @@ -177,11 +177,17 @@ modules = rbd_support \ rgw \ selftest +# The two greps below count how many lines contain a container registry name, +# so that a new hardcoded image was added on purpose. They count only the text +# of 24 quay.io lines, not the code. The value has to be updated from 23 to 24, +# as a new image reference has been named in a comment in utils.py. The +# additional name is for the nvmeof image, added as part of 0486502cd28 +# ("mgr/cephadm: give nvmeof gateways a key they can use"). commands = flake8 --config=tox.ini {posargs} \ {posargs:{[testenv:flake8]modules}} bash -c 'test $(git ls-files cephadm | grep ".py$" | grep -v tests | xargs grep "docker.io" | wc -l) == 3' - bash -c 'test $(git ls-files cephadm | grep ".py$" | grep -v tests | xargs grep "quay.io" | wc -l) == 23' + bash -c 'test $(git ls-files cephadm | grep ".py$" | grep -v tests | xargs grep "quay.io" | wc -l) == 24' [testenv:jinjalint] basepython = python3