From c597e0fd30da0c088b8092f36a02d6a3135016fa Mon Sep 17 00:00:00 2001 From: sp16743 Date: Mon, 19 Oct 2026 17:42:49 +0530 Subject: [PATCH] Adding storage pool resize testcase for zone scoped pool --- test/integration/plugins/ontap/README.md | 18 +- test/integration/plugins/ontap/TEST_CASES.md | 88 +- .../ontap/iscsi/pool/test_pool_lifecycle.py | 1166 +++++++++++++---- .../ontap/iscsi/pool/test_zone_scoped_pool.py | 81 +- .../ontap/nfs3/pool/test_pool_lifecycle.py | 1101 ++++++++++++---- .../ontap/nfs3/pool/test_zone_scoped_pool.py | 87 +- .../plugins/ontap/ontap_test_base.py | 747 ++++++++++- 7 files changed, 2732 insertions(+), 556 deletions(-) diff --git a/test/integration/plugins/ontap/README.md b/test/integration/plugins/ontap/README.md index ed839720c2cf..ec787d003461 100644 --- a/test/integration/plugins/ontap/README.md +++ b/test/integration/plugins/ontap/README.md @@ -43,7 +43,7 @@ for the deployment, health-gate, and artifact contracts. test/integration/plugins/ontap/ ├── ontap.cfg # Environment config (IPs, credentials, zone info) ├── ontap_test_base.py # Shared base class and ONTAP REST client -├── TEST_CASES.md # Full test case reference table (101 tests) +├── TEST_CASES.md # Full test case reference table (119 tests) ├── README.md # This file │ ├── nfs3/ @@ -333,7 +333,9 @@ another ONTAP pool is present. | `get_igroup(svm_name, name)` | iSCSI igroup existence and initiator list | iSCSI suites | | `list_luns_in_volume(svm_name, vol_name)` | LUNs present in a FlexVol | iSCSI volume/instance suites | | `list_lun_maps_for_volume(svm_name, vol_name)` | Active LUN-maps for a volume | iSCSI pool-with-volumes/instance suites | -| `list_files_in_volume(svm_name, vol_name)` | Files inside a FlexVol | NFS3 instance suite | +| `list_files_in_volume(vol_name)` | Files inside a FlexVol | NFS3 instance and pool lifecycle suites | +| `write_file_in_volume(vol_name, filename, size_bytes)` | Write incompressible data into a FlexVol file (no VM) | NFS3/iSCSI pool lifecycle suites | +| `delete_file_in_volume(vol_name, filename)` | Remove a FlexVol file | NFS3/iSCSI pool lifecycle suites | --- @@ -341,15 +343,15 @@ another ONTAP pool is present. | Suite | File | Tests | What it covers | | ------- | ------ | ------- | --------------- | -| NFS3 Pool Lifecycle | `nfs3/pool/test_pool_lifecycle.py` | 13 | Existing lifecycle plus duplicate-name, aggregate-space, and invalid-capacitybytes create rejects, and empty-pool deletion with pre-deleted FlexVol/export policy | -| NFS3 Pool with Volumes | `nfs3/pool/test_pool_with_volumes.py` | 12 | Existing lifecycle plus deletion with pre-deleted FlexVol/export policy, and host cases for an inactive libvirt pool, a read-only NFS mount, and a deleted mount point | -| NFS3 Zone-Scoped Pool | `nfs3/pool/test_zone_scoped_pool.py` | 4 | Zone-scoped pool create, disable, enable, and delete | +| NFS3 Pool Lifecycle | `nfs3/pool/test_pool_lifecycle.py` | 21 | Create rejects for aggregate space, invalid capacitybytes, and a duplicate FlexVol name; grow to 300 TiB, reject grow past 300 TiB, safe shrink, disable, resize while disabled, enable, maintenance, grow and shrink in maintenance, delete, then re-create with a volume, reject shrink below used capacity, deploy a VM and attach the volume, cleanup, plus empty-pool deletion with a pre-deleted FlexVol or export policy | +| NFS3 Pool with Volumes | `nfs3/pool/test_pool_with_volumes.py` | 12 | Lifecycle with a live volume, deletion with a pre-deleted FlexVol or export policy, and host cases for an inactive libvirt pool, a read-only NFS mount, and a deleted mount point | +| NFS3 Zone-Scoped Pool | `nfs3/pool/test_zone_scoped_pool.py` | 5 | Zone-scoped pool create, grow and shrink with CloudStack/ONTAP/KVM verification, disable, enable, and delete | | NFS3 Volume Lifecycle | `nfs3/volume/test_volume_lifecycle.py` | 5 | Volume is metadata-only; FlexVol unchanged on delete | | NFS3 VM + Volume Attach | `nfs3/instance/test_vm_volume_attach.py` | 13 | Full VM lifecycle with stopped-VM qcow2 grow; ROOT on tagged pool seeds/reuses template cache, which survives VM delete | | NFS3 Template Cache Negative | `nfs3/template/test_template_cache_negative.py` | 3 | Tag mismatch; undersized pool; out-of-band cache delete | -| iSCSI Pool Lifecycle | `iscsi/pool/test_pool_lifecycle.py` | 13 | Existing lifecycle plus duplicate-name, aggregate-space, and invalid-capacitybytes create rejects, and empty-pool deletion with pre-deleted FlexVol/igroups | -| iSCSI Pool with Volumes | `iscsi/pool/test_pool_with_volumes.py` | 13 | Existing lifecycle plus deletion with pre-deleted FlexVol/igroups, maintenance with pre-deleted LUN maps, and host cases for a logged-out iSCSI session, an existing session during volume delete, and a replaced by-path symlink | -| iSCSI Zone-Scoped Pool | `iscsi/pool/test_zone_scoped_pool.py` | 4 | Zone-scoped pool create, disable, enable, and delete | +| iSCSI Pool Lifecycle | `iscsi/pool/test_pool_lifecycle.py` | 21 | Create rejects for aggregate space, invalid capacitybytes, and a duplicate FlexVol name; grow to 300 TiB, reject grow past 300 TiB, safe shrink, disable, resize while disabled, enable, maintenance, grow and shrink in maintenance, delete + igroups, then re-create with a volume (LUN), reject shrink below used capacity, deploy a VM and attach the volume, cleanup, plus empty-pool deletion with a pre-deleted FlexVol or igroups | +| iSCSI Pool with Volumes | `iscsi/pool/test_pool_with_volumes.py` | 13 | Lifecycle with a live LUN, deletion with a pre-deleted FlexVol or igroups, maintenance with pre-deleted LUN maps, and host cases for a logged-out iSCSI session, an existing session during volume delete, and a replaced by-path symlink | +| iSCSI Zone-Scoped Pool | `iscsi/pool/test_zone_scoped_pool.py` | 5 | Zone-scoped pool create, grow and shrink with CloudStack/ONTAP verification, disable, enable, and delete | | iSCSI Volume Lifecycle | `iscsi/volume/test_volume_lifecycle.py` | 5 | LUN created per CS volume; LUN removed on delete | | iSCSI VM + Volume Attach | `iscsi/instance/test_vm_volume_attach.py` | 13 | Full VM lifecycle; stopped-VM LUN grow and LUN-map checks; ROOT on tagged pool seeds/reuses `cs_tmpl_*` LUN cache | | iSCSI Template Cache Negative | `iscsi/template/test_template_cache_negative.py` | 3 | Tag mismatch; undersized pool; out-of-band cache delete | diff --git a/test/integration/plugins/ontap/TEST_CASES.md b/test/integration/plugins/ontap/TEST_CASES.md index ed335c8d5ece..0b235925c0f0 100644 --- a/test/integration/plugins/ontap/TEST_CASES.md +++ b/test/integration/plugins/ontap/TEST_CASES.md @@ -19,7 +19,7 @@ # ONTAP Integration Test Cases -Complete reference for all 101 test cases across 12 test suites. +Complete reference for all 119 test cases across 12 test suites. Each suite is sequential — tests must run in numbered order; each step builds on state created by the previous step. --- @@ -42,7 +42,7 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `nfs3/pool/test_pool_lifecycle.py` **Class:** `TestOntapNFS3PrimaryStorageWorkflow` **Tag:** `nfs3_workflow` -**Total:** 13 tests | **Scope:** cluster-scoped NFS3 pool, no-volume workflow through test 09. tests 01 and 02 are isolated; test 04 reuses the pool from test 03; tests 12–13 are isolated +**Total:** 21 tests | **Scope:** cluster-scoped NFS3 pool. tests 01, 02, 20, and 21 are isolated; test 04 reuses the FlexVol from test 03 | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | | --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | @@ -50,15 +50,23 @@ Each suite is sequential — tests must run in numbered order; each step builds | 02 | `test_02_reject_create_with_invalid_capacitybytes_values` | Reject pool creation when `capacitybytes` has invalid values | isolated | `CloudstackAPIException` for each value containing `invalid value` for `capacitybytes`, or `must be at least` when the plugin validates the size; no CS pool created | No FlexVol created | negative | | 03 | `test_03_create_primary_storage_pool` | Create a cluster-scoped NFS3 primary storage pool | setUpClass (zone, cluster, account) | `pool.state == "Up"`, `pool.type == "NetworkFilesystem"`, `nfsmountopts` contains `vers=3` | FlexVol exists and `state == "online"`, export policy exists with each cluster host IP as a rule, at least one NFS data LIF present on SVM | positive | | 04 | `test_04_reject_create_when_flexvol_name_exists` | Reject a second pool create when the FlexVol from test 03 already exists on ONTAP | test_03 | `CloudstackAPIException`; only the test_03 pool remains, still `Up` | Existing FlexVol stays `online`; export policy still present | negative | -| 05 | `test_05_disable_storage_pool` | Disable the pool (admin operation) | test_03 | `pool.state == "Disabled"` | FlexVol still `online`; export policy still present | positive | -| 06 | `test_06_enable_storage_pool` | Re-enable the pool | test_05 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | -| 07 | `test_07_enter_maintenance_mode` | Put pool into maintenance (drains new volume allocations) | test_06 | `pool.state == "Maintenance"` | FlexVol still `online`; export policy still present (maintenance is CS-only state) | positive | -| 08 | `test_08_cancel_maintenance_mode` | Cancel maintenance, return pool to service | test_07 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | -| 09 | `test_09_delete_pool_from_maintenance` | Enter maintenance then permanently delete the original pool | test_08 | Pool no longer returned by `listStoragePools` (CS 431 error expected on ID lookup) | FlexVol and export policy deleted | positive | -| 10 | `test_10_create_volume_on_pool` | Create a fresh pool and allocate a CloudStack data volume | test_09 | New pool is `Up`; `createVolume` returns a volume | FlexVol `online`; export policy present | positive | -| 11 | `test_11_delete_volume_and_pool` | Detach and destroy the VM, delete the volume, then force-delete the pool | test_10 | VM destroyed; volume and pool no longer listed | FlexVol and export policy deleted | cleanup | -| 12 | `test_12_delete_pool_with_flexvol_predeleted` | Delete an empty pool after its FlexVol was removed directly from ONTAP | isolated | Pool removed successfully | FlexVol remains absent; export policy cleaned up | negative | -| 13 | `test_13_delete_pool_with_export_policy_predeleted` | Delete an empty pool after its export policy was removed directly from ONTAP | isolated | Pool removed successfully | FlexVol deleted; export policy remains absent | negative | +| 05 | `test_05_grow_storage_pool` | Grow the original empty pool to 300 TiB | test_03 | `capacitybytes` reaches 300 TiB; pool stays `Up` | FlexVol `space.size` reaches 300 TiB; remains `online`; export policy unchanged | positive | +| 06 | `test_06_reject_grow_above_max_size` | Reject a grow past the 300 TiB ONTAP FlexVol maximum (overshoots by 1 GiB; asserts ONTAP's autosize-maximum error, not a generic failure such as aggregate space) | test_05 | `CloudstackAPIException`; `capacitybytes` stays at 300 TiB; pool stays `Up` | FlexVol `space.size` unchanged; export policy unchanged | negative | +| 07 | `test_07_shrink_storage_pool` | Safely shrink the original empty pool to its initial capacity | test_05 | `capacitybytes` reaches the safe target; pool stays `Up` | FlexVol `space.size` reaches the safe target; remains `online`; export policy unchanged | positive | +| 08 | `test_08_disable_storage_pool` | Disable the pool (admin operation) | test_07 | `pool.state == "Disabled"` | FlexVol still `online`; export policy still present | positive | +| 09 | `test_09_grow_and_shrink_storage_pool_while_disabled` | Grow then shrink the pool back while it is Disabled | test_08 | Both `updateStoragePool` calls accepted; `capacitybytes` reaches each target; KVM hosts report each resized NFS capacity; `pool.state` stays `"Disabled"` throughout | FlexVol `space.size` reaches each target and stays `online`; export policy still present and still covers every host IP | positive | +| 10 | `test_10_enable_storage_pool` | Re-enable the pool | test_09 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | +| 11 | `test_11_enter_maintenance_mode` | Put pool into maintenance (drains new volume allocations) | test_10 | `pool.state == "Maintenance"` | FlexVol still `online`; export policy still present (maintenance is CS-only state) | positive | +| 12 | `test_12_grow_and_shrink_storage_pool_in_maintenance` | Grow then shrink the pool back while it is in Maintenance | test_11 | Both `updateStoragePool` calls accepted; `capacitybytes` reaches each target; `pool.state` stays `"Maintenance"` throughout | FlexVol `space.size` reaches each target and stays `online`; export policy still present | positive | +| 13 | `test_13_cancel_maintenance_mode` | Cancel maintenance, return pool to service | test_12 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | +| 14 | `test_14_delete_pool_from_maintenance` | Enter maintenance then permanently delete the original pool | test_13 | Pool no longer returned by `listStoragePools` (CS 431 error expected on ID lookup) | FlexVol and export policy deleted | positive | +| 15 | `test_15_create_volume_on_pool` | Create a fresh pool and allocate a CloudStack data volume | test_14 | New pool is `Up`; `createVolume` returns a volume | FlexVol `online`; export policy present | positive | +| 16 | `test_16_create_vm_and_attach_volume` | Deploy a VM and attach the ONTAP data volume to it | test_15 | VM reaches `Running`; volume reports `virtualmachineid` equal to the VM id; VM still `Running` after attach; pool capacity unchanged | FlexVol still `online`; volume data file materialised in the FlexVol (NFS3 creates it lazily at attach) | positive | +| 17 | `test_17_reject_shrink_below_used_capacity` | Reject shrink below the used space of the VM-attached volume, with the target held above the ONTAP FlexVol minimum. Writes incompressible data through the ONTAP files API when the VM-attached thin volume leaves used space under the minimum, and removes it before returning | test_16 (`pool`, `volume`, `vm`) | `CloudstackAPIException`; volume remains listed with unchanged `id`/`state`/`size`/`poolid`; pool capacity unchanged | FlexVol size and export policy unchanged; used space stays above the FlexVol minimum | negative | +| 18 | `test_18_grow_and_shrink_pool_with_vm_attached` | Grow then shrink the pool back while the volume is attached to the running VM | test_16 (`vm`, `volume`) | Both resizes reach the requested capacity; pool stays `Up`; VM stays `Running` and keeps `virtualmachineid` on the volume | FlexVol reaches each size and stays `online`; export policy still covers every host IP | positive | +| 19 | `test_19_delete_volume_and_pool` | Detach and destroy the VM, delete the volume, then force-delete the pool | test_18 | VM destroyed; volume and pool no longer listed | FlexVol and export policy deleted | cleanup | +| 20 | `test_20_delete_pool_with_flexvol_predeleted` | Delete an empty pool after its FlexVol was removed directly from ONTAP | isolated | Pool removed successfully | FlexVol remains absent; export policy cleaned up | negative | +| 21 | `test_21_delete_pool_with_export_policy_predeleted` | Delete an empty pool after its export policy was removed directly from ONTAP | isolated | Pool removed successfully | FlexVol deleted; export policy remains absent | negative | --- @@ -91,14 +99,15 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `nfs3/pool/test_zone_scoped_pool.py` **Class:** `TestOntapZoneScopedPool` **Tag:** `zone_pool` -**Total:** 4 tests | **Scope:** zone-scoped NFS3 pool (scope=ZONE, all hosts in zone connected) +**Total:** 5 tests | **Scope:** zone-scoped NFS3 pool (scope=ZONE, all hosts in zone connected) | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | | --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | | 01 | `test_01_create_zone_scoped_pool` | Create a zone-scoped NFS3 pool; CloudStack calls `attachZone()` to connect all eligible KVM hosts | setUpClass | `pool.state == "Up"` | FlexVol `online`; export policy exists and contains **every** cluster host IP; at least one NFS data LIF present | positive | -| 02 | `test_02_disable_zone_scoped_pool` | Disable the zone-scoped pool | test_01 | `pool.state == "Disabled"` | FlexVol unchanged; export policy unchanged | positive | -| 03 | `test_03_enable_zone_scoped_pool` | Re-enable the zone-scoped pool | test_02 | `pool.state == "Up"` | FlexVol unchanged; export policy unchanged | positive | -| 04 | `test_04_delete_zone_scoped_pool` | Enter maintenance and force-delete the zone-scoped pool | test_03 | Pool no longer listed | FlexVol deleted; export policy deleted | positive | +| 02 | `test_02_grow_and_shrink_zone_scoped_pool` | Grow the zone-scoped pool, then shrink it to its original size | test_01 | Both capacity updates reach the requested size; pool remains `Up`; KVM hosts report each resized NFS capacity | FlexVol reaches each requested size and remains `online`; export policy still covers every host IP | positive | +| 03 | `test_03_disable_zone_scoped_pool` | Disable the zone-scoped pool | test_02 | `pool.state == "Disabled"` | FlexVol unchanged; export policy unchanged | positive | +| 04 | `test_04_enable_zone_scoped_pool` | Re-enable the zone-scoped pool | test_03 | `pool.state == "Up"` | FlexVol unchanged; export policy unchanged | positive | +| 05 | `test_05_delete_zone_scoped_pool` | Enter maintenance and force-delete the zone-scoped pool | test_04 | Pool no longer listed | FlexVol deleted; export policy deleted | positive | --- @@ -149,23 +158,31 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `iscsi/pool/test_pool_lifecycle.py` **Class:** `TestOntapISCSIPoolLifecycle` **Tag:** `iscsi_workflow` -**Total:** 13 tests | **Scope:** cluster-scoped iSCSI pool, no-volume workflow through test 09. tests 01 and 02 are isolated; test 04 reuses the pool from test 03; tests 12–13 are isolated +**Total:** 21 tests | **Scope:** cluster-scoped iSCSI pool. tests 01, 02, 20, and 21 are isolated; test 04 reuses the FlexVol from test 03 | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | | --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | | 01 | `test_01_reject_create_when_no_aggregate_space` | Reject pool creation when requested capacity exceeds every assigned online aggregate's available space | isolated | `CloudstackAPIException` containing `No suitable aggregates`; no CS pool created | No FlexVol created | negative | | 02 | `test_02_reject_create_with_invalid_capacitybytes_values` | Reject pool creation when `capacitybytes` has invalid values | isolated | `CloudstackAPIException` for each value containing `invalid value` for `capacitybytes`, or `must be at least` when the plugin validates the size; no CS pool created | No FlexVol created | negative | -| 03 | `test_03_create_primary_storage_pool` | Create a cluster-scoped iSCSI primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "Iscsi"` | FlexVol `online`; shared `cs_{hostUuid}_{svmName}` igroups unchanged from suite-start baseline | positive | +| 03 | `test_03_create_primary_storage_pool` | Create a cluster-scoped iSCSI primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "Iscsi"` | FlexVol `online`; one igroup per cluster host (named `cs_{svmName}_{hostShortName}`) with host IQN as initiator | positive | | 04 | `test_04_reject_create_when_flexvol_name_exists` | Reject a second pool create when the FlexVol from test 03 already exists on ONTAP | test_03 | `CloudstackAPIException`; only the test_03 pool remains, still `Up` | Existing FlexVol stays `online` | negative | -| 05 | `test_05_disable_storage_pool` | Disable the pool | test_03 | `pool.state == "Disabled"` | FlexVol still `online` | positive | -| 06 | `test_06_enable_storage_pool` | Re-enable the pool | test_05 | `pool.state == "Up"` | FlexVol still `online` | positive | -| 07 | `test_07_enter_maintenance_mode` | Put pool into maintenance | test_06 | `pool.state == "Maintenance"` | FlexVol still `online`; igroups unchanged | positive | -| 08 | `test_08_cancel_maintenance_mode` | Cancel maintenance | test_07 | `pool.state == "Up"` | FlexVol still `online` | positive | -| 09 | `test_09_enter_maintenance_and_delete_pool` | Enter maintenance then delete the original pool | test_08 | Pool no longer listed | FlexVol and test-pool LUN maps deleted; shared igroup baseline restored | positive | -| 10 | `test_10_create_volume_on_pool` | Create a fresh pool and allocate a CloudStack volume | test_09 | New pool is `Up`; `createVolume` returns a volume | FlexVol `online`; at least one LUN present | positive | -| 11 | `test_11_delete_volume_and_pool` | Detach and destroy the VM, delete the volume, then force-delete the pool | test_10 | VM destroyed; volume and pool no longer listed | LUN, FlexVol, and test-pool maps deleted; shared igroup baseline restored | cleanup | -| 12 | `test_12_delete_pool_with_flexvol_predeleted` | Delete an empty pool after its FlexVol was removed directly from ONTAP | isolated | Pool removed successfully | FlexVol remains absent; igroups cleaned up | negative | -| 13 | `test_13_delete_pool_with_igroups_predeleted` | Delete an empty pool after host igroups were removed directly from ONTAP | isolated | Pool removed successfully | FlexVol deleted; igroups remain absent | negative | +| 05 | `test_05_grow_storage_pool` | Grow the original empty pool to 300 TiB | test_03 | `capacitybytes` reaches 300 TiB; pool stays `Up` | FlexVol reaches 300 TiB and remains `online`; igroups remain present | positive | +| 06 | `test_06_reject_grow_above_max_size` | Reject a grow past the 300 TiB ONTAP FlexVol maximum (overshoots by 1 GiB; asserts ONTAP's autosize-maximum error, not a generic failure such as aggregate space) | test_05 | `CloudstackAPIException`; `capacitybytes` stays at 300 TiB; pool stays `Up` | FlexVol `space.size` unchanged; igroups unchanged | negative | +| 07 | `test_07_shrink_storage_pool` | Safely shrink the original empty pool to its initial capacity | test_05 | `capacitybytes` reaches the safe target; pool stays `Up` | FlexVol reaches the safe target and remains `online`; igroups remain present | positive | +| 08 | `test_08_disable_storage_pool` | Disable the pool | test_07 | `pool.state == "Disabled"` | FlexVol still `online` | positive | +| 09 | `test_09_grow_and_shrink_storage_pool_while_disabled` | Grow then shrink the pool back while it is Disabled | test_08 | Both `updateStoragePool` calls accepted; `capacitybytes` reaches each target; `pool.state` stays `"Disabled"` throughout | FlexVol `space.size` reaches each target and stays `online`; per-host igroups still present | positive | +| 10 | `test_10_enable_storage_pool` | Re-enable the pool | test_09 | `pool.state == "Up"` | FlexVol still `online` | positive | +| 11 | `test_11_enter_maintenance_mode` | Put pool into maintenance | test_10 | `pool.state == "Maintenance"` | FlexVol still `online`; igroups unchanged | positive | +| 12 | `test_12_grow_and_shrink_storage_pool_in_maintenance` | Grow then shrink the pool back while it is in Maintenance | test_11 | Both `updateStoragePool` calls accepted; `capacitybytes` reaches each target; `pool.state` stays `"Maintenance"` throughout | FlexVol `space.size` reaches each target and stays `online`; per-host igroups still present | positive | +| 13 | `test_13_cancel_maintenance_mode` | Cancel maintenance | test_12 | `pool.state == "Up"` | FlexVol still `online` | positive | +| 14 | `test_14_enter_maintenance_and_delete_pool` | Enter maintenance then delete the original pool | test_13 | Pool no longer listed | FlexVol and all host igroups deleted | positive | +| 15 | `test_15_create_volume_on_pool` | Create a fresh pool and allocate a CloudStack volume | test_14 | New pool is `Up`; `createVolume` returns a volume | FlexVol `online`; at least one LUN present | positive | +| 16 | `test_16_create_vm_and_attach_volume` | Deploy a VM and attach the ONTAP data volume (LUN) to it | test_15 | VM reaches `Running`; volume reports `virtualmachineid` equal to the VM id; VM still `Running` after attach | FlexVol still `online`; LUN still present; per-host igroups unchanged | positive | +| 17 | `test_17_reject_shrink_below_used_capacity` | Reject shrink below the used space of the VM-attached LUN, with the target held above the ONTAP FlexVol minimum. Writes incompressible data through the ONTAP files API when the VM-attached thin volume leaves used space under the minimum, and removes it before returning | test_16 (`pool`, `volume`, `vm`) | `CloudstackAPIException`; volume remains listed with unchanged `id`/`state`/`size`/`poolid`; pool capacity unchanged | FlexVol size and LUN uuid/name list unchanged; used space stays above the FlexVol minimum | negative | +| 18 | `test_18_grow_and_shrink_pool_with_vm_attached` | Grow then shrink the pool back while the volume (LUN) is attached to the running VM | test_16 (`vm`, `volume`) | Both resizes reach the requested capacity; pool stays `Up`; VM stays `Running` and keeps `virtualmachineid` on the volume | FlexVol reaches each size and stays `online`; LUN still present | positive | +| 19 | `test_19_delete_volume_and_pool` | Detach and destroy the VM, delete the volume, then force-delete the pool | test_18 | VM destroyed; volume and pool no longer listed | LUN, FlexVol, and igroups deleted | cleanup | +| 20 | `test_20_delete_pool_with_flexvol_predeleted` | Delete an empty pool after its FlexVol was removed directly from ONTAP | isolated | Pool removed successfully | FlexVol remains absent; igroups cleaned up | negative | +| 21 | `test_21_delete_pool_with_igroups_predeleted` | Delete an empty pool after host igroups were removed directly from ONTAP | isolated | Pool removed successfully | FlexVol deleted; igroups remain absent | negative | --- @@ -199,14 +216,15 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `iscsi/pool/test_zone_scoped_pool.py` **Class:** `TestOntapISCSIZoneScopedPool` **Tag:** `iscsi_zone_pool` -**Total:** 4 tests | **Scope:** zone-scoped iSCSI pool (scope=ZONE) +**Total:** 5 tests | **Scope:** zone-scoped iSCSI pool (scope=ZONE) | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | | --- | ------------- | ------ | ------------ | ----------------------------- | ------------------------ | ------ | | 01 | `test_01_create_zone_scoped_pool` | Create a zone-scoped iSCSI pool | setUpClass | `pool.state == "Up"` | FlexVol `online`; shared host igroups unchanged from suite-start baseline | positive | -| 02 | `test_02_disable_zone_scoped_pool` | Disable pool | test_01 | `pool.state == "Disabled"` | FlexVol unchanged; igroups unchanged | positive | -| 03 | `test_03_enable_zone_scoped_pool` | Re-enable pool | test_02 | `pool.state == "Up"` | FlexVol unchanged; igroups unchanged | positive | -| 04 | `test_04_delete_zone_scoped_pool` | Enter maintenance then delete pool | test_03 | Pool no longer listed | FlexVol and test-pool maps deleted; shared igroup baseline restored | positive | +| 02 | `test_02_grow_and_shrink_zone_scoped_pool` | Grow the zone-scoped pool, then shrink it to its original size | test_01 | Both capacity updates reach the requested size; pool remains `Up` | FlexVol reaches each requested size and remains `online`; shared igroups remain unchanged | positive | +| 03 | `test_03_disable_zone_scoped_pool` | Disable pool | test_02 | `pool.state == "Disabled"` | FlexVol unchanged; igroups unchanged | positive | +| 04 | `test_04_enable_zone_scoped_pool` | Re-enable pool | test_03 | `pool.state == "Up"` | FlexVol unchanged; igroups unchanged | positive | +| 05 | `test_05_delete_zone_scoped_pool` | Enter maintenance then delete pool | test_04 | Pool no longer listed | FlexVol and test-pool maps deleted; shared igroup baseline restored | positive | --- @@ -288,16 +306,16 @@ Each suite is sequential — tests must run in numbered order; each step builds | Suite | Protocol | Scope | Tests | Status | | ------- | --------- | ------- | ------- | -------- | -| NFS3 Pool Lifecycle | NFS3 | Cluster | 13 | ✅ | +| NFS3 Pool Lifecycle | NFS3 | Cluster | 21 | ✅ | | NFS3 Pool with Volumes | NFS3 | Cluster | 12 | ✅ | -| NFS3 Zone-Scoped Pool | NFS3 | Zone | 4 | ✅ | +| NFS3 Zone-Scoped Pool | NFS3 | Zone | 5 | ✅ | | NFS3 Volume Lifecycle | NFS3 | Cluster | 5 | ✅ | | NFS3 VM + Volume Attach | NFS3 | Cluster | 13 | ⏳ resize cases pending lab validation | | NFS3 Template Cache Negative | NFS3 | Cluster | 3 | 🆕 | -| iSCSI Pool Lifecycle | iSCSI | Cluster | 13 | ✅ | +| iSCSI Pool Lifecycle | iSCSI | Cluster | 21 | ✅ | | iSCSI Pool with Volumes | iSCSI | Cluster | 13 | ✅ | -| iSCSI Zone-Scoped Pool | iSCSI | Zone | 4 | ✅ | +| iSCSI Zone-Scoped Pool | iSCSI | Zone | 5 | ✅ | | iSCSI Volume Lifecycle | iSCSI | Cluster | 5 | ✅ | | iSCSI VM + Volume Attach | iSCSI | Cluster | 13 | ⏳ resize cases pending lab validation; detach limitation | | iSCSI Template Cache Negative | iSCSI | Cluster | 3 | 🆕 | -| **Total** | | | **101** | **Resize cases pending lab validation** | +| **Total** | | | **119** | **VM resize cases pending lab validation** | diff --git a/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py index 473a171cc2e4..85663d6370d0 100644 --- a/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py +++ b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py @@ -17,39 +17,52 @@ """ Sequential workflow integration tests for NetApp ONTAP iSCSI primary storage -pool lifecycle (no volumes). +pool lifecycle. -Tests are numbered test_01 ... test_13 and must run in that order. test_01 -and test_02 are isolated create rejections. Later steps build on the shared -pool from test_03. +Tests are numbered test_01 ... test_21 and must run in that order. test_01 +and test_02 are isolated create rejections. test_04 reuses the FlexVol created +by test_03. test_20 and test_21 are isolated delete cases. Workflow: 01 Reject create when no online assigned aggregate has enough free space 02 Reject create when capacitybytes has invalid values 03 Create primary storage pool 04 Reject create when a FlexVol of that name already exists on ONTAP - 05 Disable storage pool - 06 Enable storage pool - 07 Enter maintenance mode - 08 Cancel maintenance mode - 09 Enter maintenance mode and delete the storage pool - 10 Create a new pool and allocate a CloudStack data volume (LUN created) - 11 Delete the volume (LUN removed), enter maintenance, force-delete pool - 12 Delete an empty pool whose FlexVol was deleted directly on ONTAP - 13 Delete an empty pool whose igroups were deleted on ONTAP + 05 Increase storage pool capacity to the maximum supported size + 06 Reject a grow beyond the maximum supported size + 07 Safely shrink storage pool capacity + 08 Disable storage pool + 09 Grow and shrink the pool while it is disabled + 10 Enable storage pool + 11 Enter maintenance mode + 12 Grow and shrink the pool while it is in maintenance mode + 13 Cancel maintenance mode + 14 Enter maintenance mode and delete the storage pool + 15 Create a new pool and allocate a CloudStack data volume (LUN created) + 16 Deploy a VM and attach the ONTAP volume (LUN) to it + 17 Reject shrink below the ONTAP used capacity of the attached LUN + 18 Grow and shrink the pool while the VM holds the volume (LUN) + 19 Detach/destroy the VM, delete the volume (LUN removed), enter + maintenance, force-delete pool + 20 Delete an empty pool whose FlexVol was deleted directly on ONTAP + 21 Delete an empty pool whose igroups were deleted on ONTAP Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed - KVM cluster where every host has iSCSI configured (storageUrl starts with iqn.) + - At least one ready user KVM template in the zone (test_16 skips without one) - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF - ontap.cfg populated with real values Running: nosetests --with-marvin \\ --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ - test/integration/plugins/ontap/iscsi/pool/ -v + test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py -v -Note: Tests share class-level state (sequential). Always run the full suite. +Note: Tests 03-19 share class-level state (sequential). test_04 reuses the +FlexVol created by test_03. Running a single test with -m "test_NN" will +invoke setUpClass but the guard assertion will fail immediately if earlier +steps have not yet run. Always run the full suite. """ import base64 @@ -61,8 +74,10 @@ from nose.plugins.attrib import attr from marvin.cloudstackAPI import ( + attachVolume as attachVolumeAPI, createStoragePool as createStoragePoolAPI, deleteVolume as deleteVolumeAPI, + deployVirtualMachine as deployVirtualMachineAPI, updateStoragePool as updateStoragePoolAPI, ) from marvin.cloudstackException import CloudstackAPIException @@ -70,11 +85,8 @@ from marvin.lib.common import list_storage_pools from ontap_test_base import ( - OntapRestClient, - OntapTestBase, - get_datacenter_config, - log_progress, - run_invalid_capacitybytes_rejections, + OntapRestClient, OntapTestBase, _wait_for_vm_state, get_datacenter_config, + log_progress, run_invalid_capacitybytes_rejections, ) logger = logging.getLogger("TestOntapISCSIPoolLifecycle") @@ -99,7 +111,16 @@ class TestData: DETAIL_STORAGE_IP = "storageIP" ONTAP_MIN_VOLUME_SIZE = 1677721600 + + # Maximum FlexVol size on AFF/FAS, raised from 100 TB in ONTAP 9.12.1P2. + # ONTAP writes this as "300 TB" but means binary, hence 1024**4. The + # value has to be hardcoded: no REST field reports the platform ceiling. + # Against an ONTAP older than 9.12.1P2 the real limit is 100 TB and the + # grow step will fail. CloudStack enforces no maximum of its own (the + # plugin validates only a 20 MB floor), so it is ONTAP that refuses + # anything above this. ONTAP_MAX_VOLUME_SIZE = 300 * 1024 ** 4 + ONTAP_GROW_TARGET_SIZE = ONTAP_MAX_VOLUME_SIZE def __init__(self, storage_ip, svm_name, username, password, scope="CLUSTER", provider="NetApp ONTAP", @@ -158,6 +179,10 @@ class TestOntapISCSIPoolLifecycle(OntapTestBase): # ---- iSCSI-specific state (set/cleared by individual tests) -------- _vol_name_prefix = "OntapISCSIVol" + resize_original_size = None + + # VM state (vm/template_id/network_id) lives in OntapTestBase. + _vm_network_name_prefix = "ontap-iscsi-lifecycle-net" @classmethod def setUpClass(cls): @@ -197,6 +222,8 @@ def setUpClass(cls): cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + cls._discover_vm_deploy_resources() + # No per-test tearDown — state intentionally persists between steps. # ------------------------------------------------------------------ @@ -204,7 +231,7 @@ def setUpClass(cls): # ------------------------------------------------------------------ def _create_pool(self, pool_name=None, capacitybytes=None): - """Create a pool; name and capacity default to the suite's values.""" + """Create a pool; name and capacity default to the suite values.""" ps = self.testdata[TestData.primaryStorage] storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] if pool_name is None: @@ -284,8 +311,146 @@ def _assert_pool_capacity(self, pool, label): % (label, ontap_size, configured) ) + def _throwaway_pool_name(self, suffix): + return "OntapISCSI%s_%d" % (suffix, random.randint(0, 99999)) + + def _cleanup_throwaway_pool(self, pool, flexvol_name=None): + """Best-effort teardown for an isolated test: CloudStack, then ONTAP. + + Never raises, so a failed assertion in the test body is the error + that surfaces. + """ + if pool is not None: + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + listed = None + if listed: + try: + if listed[0].state != "Maintenance": + self._enter_maintenance(pool.id) + except Exception as exc: + logger.warning( + "cleanup: could not put pool '%s' into Maintenance: %s", + pool.name, exc, + ) + try: + self._delete_pool(pool.id, forced=True) + except Exception as exc: + logger.warning( + "cleanup: could not delete pool '%s': %s", + pool.name, exc, + ) + if flexvol_name is None: + flexvol_name = pool.name + if flexvol_name: + try: + if self.ontap.get_volume(flexvol_name) is not None: + self.ontap.offline_and_delete_volume(flexvol_name) + except Exception as exc: + logger.warning( + "cleanup: could not delete ONTAP FlexVol '%s': %s", + flexvol_name, exc, + ) + if pool is None: + return + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + if not remaining: + self.__class__.pool2 = None + + def _assert_only_original_pool(self, pool): + """Assert the rejected create did not add a second pool of this name.""" + try: + listed = list_storage_pools(self.apiClient, name=pool.name) or [] + except CloudstackAPIException: + listed = [] + same_name = [ + item for item in listed if getattr(item, "name", None) == pool.name + ] + self.assertEqual( + [item.id for item in same_name], + [pool.id], + "CloudStack should still list only pool '%s' after the rejected " + "create, found: %s" % (pool.name, same_name), + ) + current = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue( + current, + "Original pool '%s' disappeared after the rejected create" + % pool.name, + ) + self.assertEqual( + current[0].state, "Up", + "Original pool '%s' should still be Up, got '%s'" + % (pool.name, current[0].state), + ) + + def _warn_if_duplicate_pool(self, duplicate, pool): + """Leave a stray duplicate in place. + + Deleting it would remove the FlexVol that the rest of the suite + still uses. + """ + if duplicate is None or getattr(duplicate, "id", None) == pool.id: + return + logger.warning( + "Duplicate pool '%s' (id=%s) was created against the shared " + "FlexVol; leaving it in place so cleanup does not delete the " + "pool the rest of the suite uses", + pool.name, duplicate.id, + ) + + def _assert_no_pool_named(self, pool_name): + """Assert CloudStack holds no storage pool with this name.""" + try: + listed = list_storage_pools(self.apiClient, name=pool_name) + except CloudstackAPIException: + listed = None + self.assertFalse( + listed, + "CloudStack should hold no pool named '%s' after a rejected " + "create, found: %s" % (pool_name, listed), + ) + + def _assert_pool_gone_from_cs(self, pool_id, pool_name): + try: + remaining = list_storage_pools(self.apiClient, id=pool_id) + except CloudstackAPIException: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' still listed in CloudStack after deletion" % pool_name, + ) + + def _sweep_tracked_pool2(self): + """Clean a prior leftover before reusing the shared pool2 slot.""" + if self.__class__.pool2 is None: + return + self._cleanup_throwaway_pool(self.__class__.pool2) + if self.__class__.pool2 is not None: + self.skipTest("A previously tracked pool could not be cleaned up") + + def _create_throwaway_pool(self, suffix): + """Create an isolated pool, park it in pool2, and return it.""" + self._sweep_tracked_pool2() + pool = self._create_pool(pool_name=self._throwaway_pool_name(suffix)) + self.__class__.pool2 = pool + self.assertEqual( + pool.state, "Up", + "Throwaway pool '%s' should be 'Up', got '%s'" + % (pool.name, pool.state), + ) + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol missing for throwaway pool '%s'" % pool.name, + ) + return pool + # ------------------------------------------------------------------ - # Step 01 - Reject create when no aggregate has enough free space + # Step 01 - reject create when no aggregate space # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) @@ -354,7 +519,7 @@ def test_01_reject_create_when_no_aggregate_space(self): ) # ------------------------------------------------------------------ - # Step 02 - Reject create when capacitybytes has invalid values + # Step 02 - reject create with invalid capacitybytes values # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) @@ -384,7 +549,7 @@ def cleanup(pool, pool_name): ) # ------------------------------------------------------------------ - # Step 03 - Create primary storage pool + # Step 03 - create primary storage pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) @@ -441,8 +606,9 @@ def test_03_create_primary_storage_pool(self): # Capacity reporting self._assert_pool_capacity(pool, "pool-created") + # ------------------------------------------------------------------ - # Step 04 - Reject create when that FlexVol name already exists + # Step 04 - reject create when flexvol name exists # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) @@ -494,107 +660,436 @@ def test_04_reject_create_when_flexvol_name_exists(self): self._warn_if_duplicate_pool(duplicate, pool) # ------------------------------------------------------------------ - # Step 05 - Disable storage pool + # Step 05 - grow storage pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_05_disable_storage_pool(self): - """ - Disable the pool and verify: - - CloudStack reports Disabled - - ONTAP: FlexVol is still online (disable is a CS-only state change) - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + def test_05_grow_storage_pool(self): + """Grow the original pool to 300 TiB and verify CloudStack and ONTAP converge.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_03 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before capacity increase") + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol missing before increase") + + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + int(self.testdata[TestData.primaryStorage]["capacitybytes"]), + ) + requested_size = TestData.ONTAP_GROW_TARGET_SIZE + self.assertGreater( + requested_size, original_size, + "Grow target %d B must exceed the current pool size %d B" + % (requested_size, original_size) + ) + self.__class__.resize_original_size = original_size cmd = updateStoragePoolAPI.updateStoragePoolCmd() - cmd.id = self.__class__.pool.id - cmd.enabled = False + cmd.id = pool.id + cmd.capacitybytes = requested_size self.apiClient.updateStoragePool(cmd) - result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) - self.assertEqual(result.state, "Disabled") - - # ONTAP: disable must not touch the FlexVol - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should still be 'online' after disable, got '%s'" - % ontap_vol.get("state") + resized_pool = self._poll_pool_capacity( + pool.id, requested_size, timeout=120 ) + self.assertEqual(resized_pool.state, "Up") + resized_ontap_vol = self._poll_ontap_volume_size( + pool.name, requested_size, timeout=120 + ) + self.assertEqual(resized_ontap_vol.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after increase" % igroup_name, + ) + # ------------------------------------------------------------------ - # Step 06 - Enable storage pool + # Step 06 - reject grow above max size # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_06_enable_storage_pool(self): + def test_06_reject_grow_above_max_size(self): """ - Re-enable the pool and verify: - - CloudStack reports Up - - ONTAP: FlexVol is still online (enable is a CS-only state change) + Request a capacity above the 300 TiB ONTAP FlexVol maximum and verify + it is refused, leaving the pool at the size test_05 grew it to. + ONTAP reports this as an autosize-maximum violation rather than an + oversized-volume error. + Verifies: + - updateStoragePool raises CloudstackAPIException + - the error text is about the size limit, not about some other + failure such as the aggregate running out of space + - CloudStack capacity and ONTAP FlexVol size are both unchanged + - Pool stays 'Up' + - ONTAP: per-host igroups are untouched """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_05 must pass first" + ) + pool = self.__class__.pool - cmd = updateStoragePoolAPI.updateStoragePoolCmd() - cmd.id = self.__class__.pool.id - cmd.enabled = True - self.apiClient.updateStoragePool(cmd) + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before over-max resize") + self.assertEqual( + int(getattr(listed[0], "capacitybytes", 0) or 0), + TestData.ONTAP_MAX_VOLUME_SIZE, + "Pool should sit at the 300 TiB maximum - test_05 must pass first" + ) - result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) - self.assertEqual(result.state, "Up") + # Overshoot by only 1 GiB: at 0.0003% of the limit this probes the + # 300 TiB boundary itself rather than asking for something absurd. + above_max = self._align_flexvol_bytes( + TestData.ONTAP_MAX_VOLUME_SIZE + 1024 ** 3 + ) + log_progress( + logger, "info", + "Requesting %d B for pool '%s' (ONTAP maximum is %d B, " + "expect reject)", + above_max, pool.name, TestData.ONTAP_MAX_VOLUME_SIZE, + ) - # ONTAP: enable must not touch the FlexVol - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + # ONTAP enforces the platform cap through autosize rather than + # rejecting the size outright, reporting "Volume maximum autosize + # must be greater than or equal to the current volume size 300.0TB". + # Matching on that keeps the test from passing when the resize fails + # for an unrelated reason, such as the aggregate running out of space. + self._assert_capacity_update_rejected( + pool.id, above_max, "grow-above-max-flexvol", + pool.id, pool.name, + expected_error=( + "maximum autosize", + "maximum size", + "exceeds", + ), + ) + + after = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(after, "Pool disappeared after rejected over-max grow") self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should be 'online' after enable, got '%s'" - % ontap_vol.get("state") + after[0].state, "Up", + "Pool should stay 'Up' after a rejected resize, got '%s'" + % after[0].state ) + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after a rejected " + "over-max grow" % igroup_name, + ) + # ------------------------------------------------------------------ - # Step 07 - Enter maintenance mode + # Step 07 - shrink storage pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_07_enter_maintenance_mode(self): - """ - Put the pool into maintenance mode and verify: - - CloudStack reports Maintenance - - ONTAP: FlexVol is still online (maintenance is a CS-only state change) - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + def test_07_shrink_storage_pool(self): + """Shrink the original pool back to its initial safe capacity.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_05 must pass first" + ) + target_size = self.__class__.resize_original_size + self.assertIsNotNone( + target_size, "Original size absent - test_05 must pass first" + ) + pool = self.__class__.pool - result = self._enter_maintenance(self.__class__.pool.id) - self.assertEqual(result.state, "Maintenance") + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target_size + self.apiClient.updateStoragePool(cmd) - # ONTAP: maintenance must not touch the FlexVol - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" - % ontap_vol.get("state") + shrunk_pool = self._poll_pool_capacity( + pool.id, target_size, timeout=120 ) + self.assertEqual(shrunk_pool.state, "Up") + shrunk_ontap_vol = self._poll_ontap_volume_size( + pool.name, target_size, timeout=120 + ) + self.assertEqual(shrunk_ontap_vol.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after safe shrink" + % igroup_name, + ) + # ------------------------------------------------------------------ - # Step 08 - Cancel maintenance mode + # Step 08 - disable storage pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_08_cancel_maintenance_mode(self): + def test_08_disable_storage_pool(self): """ - Cancel maintenance and verify: - - CloudStack reports Up - - ONTAP: FlexVol is still online + Disable the pool and verify: + - CloudStack reports Disabled + - ONTAP: FlexVol is still online (disable is a CS-only state change) """ self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") - result = self._exit_maintenance(self.__class__.pool.id) - self.assertEqual(result.state, "Up") + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + # ONTAP: disable must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable, got '%s'" + % ontap_vol.get("state") + ) + + + # ------------------------------------------------------------------ + # Step 09 - grow and shrink storage pool while disabled + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_09_grow_and_shrink_storage_pool_while_disabled(self): + """ + Grow and shrink the disabled pool and verify: + - Both capacity updates are accepted + - CloudStack and ONTAP reach each requested size + - The pool remains Disabled throughout + - The FlexVol remains online and per-host igroups are unchanged + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_03 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before disabled resize") + self.assertEqual( + listed[0].state, "Disabled", + "Pool must be Disabled - test_08 must pass first, got '%s'" + % listed[0].state, + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before disabled resize" + ) + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + for target, operation in ( + (grow_target, "grow"), + (original_size, "shrink")): + log_progress( + logger, "info", + "%s disabled iSCSI pool '%s': target=%d B", + operation.capitalize(), pool.name, target, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target + self.apiClient.updateStoragePool(cmd) + + resized = self._poll_pool_capacity(pool.id, target, timeout=120) + self.assertEqual( + resized.state, "Disabled", + "Pool should remain Disabled after %s, got '%s'" + % (operation, resized.state), + ) + resized_ontap = self._poll_ontap_volume_size( + pool.name, target, timeout=120 + ) + self.assertEqual(resized_ontap.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared during disabled resize" + % igroup_name, + ) + + + # ------------------------------------------------------------------ + # Step 10 - enable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_10_enable_storage_pool(self): + """ + Re-enable the pool and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online (enable is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + # ONTAP: enable must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable, got '%s'" + % ontap_vol.get("state") + ) + + + # ------------------------------------------------------------------ + # Step 11 - enter maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_11_enter_maintenance_mode(self): + """ + Put the pool into maintenance mode and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol is still online (maintenance is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + + result = self._enter_maintenance(self.__class__.pool.id) + self.assertEqual(result.state, "Maintenance") + + # ONTAP: maintenance must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" + % ontap_vol.get("state") + ) + + + # ------------------------------------------------------------------ + # Step 12 - grow and shrink storage pool in maintenance + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_12_grow_and_shrink_storage_pool_in_maintenance(self): + """ + Grow and shrink the pool while it is in maintenance mode and verify: + - Both capacity updates are accepted + - CloudStack and ONTAP reach each requested size + - The pool remains in Maintenance throughout + - The FlexVol remains online and per-host igroups are unchanged + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_03 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before maintenance resize") + self.assertEqual( + listed[0].state, "Maintenance", + "Pool must be in Maintenance - test_11 must pass first, got '%s'" + % listed[0].state, + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before maintenance resize" + ) + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + for target, operation in ( + (grow_target, "grow"), + (original_size, "shrink")): + log_progress( + logger, "info", + "%s iSCSI pool '%s' while in Maintenance: target=%d B", + operation.capitalize(), pool.name, target, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target + self.apiClient.updateStoragePool(cmd) + + resized = self._poll_pool_capacity(pool.id, target, timeout=120) + self.assertEqual( + resized.state, "Maintenance", + "Pool should remain in Maintenance after %s, got '%s'" + % (operation, resized.state), + ) + resized_ontap = self._poll_ontap_volume_size( + pool.name, target, timeout=120 + ) + self.assertEqual(resized_ontap.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared during maintenance resize" + % igroup_name, + ) + + + # ------------------------------------------------------------------ + # Step 13 - cancel maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_13_cancel_maintenance_mode(self): + """ + Cancel maintenance and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") + + result = self._exit_maintenance(self.__class__.pool.id) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance") self.assertEqual( ontap_vol.get("state"), "online", @@ -602,12 +1097,13 @@ def test_08_cancel_maintenance_mode(self): % ontap_vol.get("state") ) + # ------------------------------------------------------------------ - # Step 09 - Enter maintenance mode and delete the storage pool + # Step 14 - enter maintenance and delete pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_09_enter_maintenance_and_delete_pool(self): + def test_14_enter_maintenance_and_delete_pool(self): """ Enter maintenance mode then delete the pool. Verifies the pool is removed from CloudStack and the backing ONTAP @@ -621,6 +1117,7 @@ def test_09_enter_maintenance_and_delete_pool(self): self._delete_pool(pool.id) self.__class__.pool = None + self.__class__.resize_original_size = None # CloudStack: pool must be gone try: @@ -648,12 +1145,13 @@ def test_09_enter_maintenance_and_delete_pool(self): "ONTAP igroup '%s' still exists after pool deletion" % igroup_name ) + # ------------------------------------------------------------------ - # Step 10 - Create fresh pool and allocate a CloudStack volume (LUN) + # Step 15 - create volume on pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_10_create_volume_on_pool(self): + def test_15_create_volume_on_pool(self): """ Create a new iSCSI pool and allocate a CloudStack data volume. For iSCSI, createAsync creates a LUN inside the pool's ONTAP FlexVol. @@ -667,7 +1165,7 @@ def test_10_create_volume_on_pool(self): self.__class__.pool = pool log_progress( logger, "info", - "test_10: created storage pool name='%s' id=%s state=%s type=%s", + "test_15: created storage pool name='%s' id=%s state=%s type=%s", pool.name, pool.id, pool.state, pool.type, ) @@ -685,7 +1183,7 @@ def test_10_create_volume_on_pool(self): self.assertIsNotNone(vol, "createVolume returned None") log_progress( logger, "info", - "test_10: created CloudStack volume name='%s' id=%s state=%s " + "test_15: created CloudStack volume name='%s' id=%s state=%s " "on pool='%s' (id=%s) account='%s' domain='%s' — " "switch to this account in the UI to see the volume", getattr(vol, "name", "?"), getattr(vol, "id", "?"), @@ -714,15 +1212,325 @@ def test_10_create_volume_on_pool(self): # Capacity reporting self._assert_pool_capacity(pool, "volume-allocated") + + # ------------------------------------------------------------------ + # Step 16 - create vm and attach volume + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_16_create_vm_and_attach_volume(self): + """ + Deploy a VM and attach the ONTAP data volume from test_15 to it. + Verifies: + - VM reaches 'Running' + - attachVolume sets the volume's virtualmachineid (on ONTAP managed + storage CloudStack may leave the volume state at 'Ready', so + virtualmachineid is the reliable attach signal) + - VM is still 'Running' after the attach + - ONTAP: FlexVol stays online and the LUN is still present + - ONTAP: per-host igroups survive the attach + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_15 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_15 must pass first" + ) + if self.__class__.template_id is None: + self.skipTest( + "No ready user KVM template in the zone - cannot deploy a VM" + ) + self.assertIsNotNone( + self.__class__.service_offering_id, + "No service offering available - check the CloudStack setup" + ) + + pool = self.__class__.pool + vol = self.__class__.volume + + log_progress( + logger, "info", + "Deploying VM for iSCSI pool '%s' (template=%s, offering=%s)", + pool.name, self.__class__.template_id, + self.__class__.service_offering_id, + ) + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + self.__class__.vm = vm + + vm_obj = _wait_for_vm_state(self.apiClient, vm.id, "Running", + timeout=600) + self.assertIsNotNone( + vm_obj, "VM %s never appeared in listVirtualMachines" % vm.id + ) + self.assertEqual( + vm_obj.state, "Running", + "VM should be 'Running' before attach, got '%s'" % vm_obj.state + ) + + log_progress( + logger, "info", + "Attaching volume '%s' to VM '%s'", vol.id, vm.id, + ) + attach_cmd = attachVolumeAPI.attachVolumeCmd() + attach_cmd.id = vol.id + attach_cmd.virtualmachineid = vm.id + self.assertIsNotNone( + self.apiClient.attachVolume(attach_cmd), + "attachVolume returned None" + ) + + vol_vmid = self._poll_volume_attached(vol.id, timeout=180) + self.assertEqual( + vol_vmid, vm.id, + "Volume %s should report virtualmachineid=%s after attach, got %s" + % (vol.id, vm.id, vol_vmid) + ) + + vm_after = _wait_for_vm_state(self.apiClient, vm.id, "Running", + timeout=60) + self.assertEqual( + vm_after.state, "Running", + "VM should still be 'Running' after attach, got '%s'" + % vm_after.state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing after volume attach" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after attach, got '%s'" + % ontap_vol.get("state") + ) + + self.assertTrue( + self.ontap.list_luns_in_volume(self.svm_name, pool.name), + "No LUN present in FlexVol '%s' after attach" % pool.name + ) + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after volume attach" + % igroup_name, + ) + + + # ------------------------------------------------------------------ + # Step 17 - reject shrink below used capacity + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_17_reject_shrink_below_used_capacity(self): + """ + Attempt to shrink the pool below ONTAP used space and verify the + request is rejected while the volume and ONTAP LUNs remain unchanged. + + Runs after test_16, so the LUN is mapped to a running VM. The LUN is + thin-provisioned, though, and in practice leaves used space under the + 20 MiB FlexVol floor, so incompressible data is written through the + ONTAP files API to get above it and removed before this test returns. + The fill is skipped whenever used space is already high enough. + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_16 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_16 must pass first" + ) + pool = self.__class__.pool + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol not found for pool '%s'" % pool.name, + ) + + vol_before = self._cs_volume_snapshot(self.__class__.volume.id) + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs in FlexVol '%s' before rejected shrink" % pool.name, + ) + luns_before_used = sorted( + (lun.get("uuid") or "", lun.get("name") or "") + for lun in luns + ) + + # The attached volume is thin on both protocols, so it usually does + # not put 20 MiB on disk by itself. This writes incompressible data + # only when used space is still under the FlexVol floor, and removes + # it before returning. + filler_name, used_bytes = self._fill_flexvol_above_minimum(pool.name) + self.__class__.filler_filename = filler_name + self.__class__.filler_flexvol = pool.name if filler_name else None + try: + below_used = self._shrink_target_below_used(used_bytes) + log_progress( + logger, "info", + "Shrinking iSCSI pool '%s' below ONTAP used capacity: " + "used=%d B, requested=%d B " + "(FlexVol minimum=%d B, expect reject)", + pool.name, used_bytes, below_used, self.ONTAP_MIN_FLEXVOL_SIZE, + ) + self._assert_capacity_update_rejected( + pool.id, below_used, "shrink-below-used", + pool.id, pool.name, + # ONTAP's wording: "Selected volume size is too small to + # hold the current volume data. New volume size must be at + # least 34.3MB ...". Matching the specific phrase stops an + # unrelated resize failure from counting as a pass. + expected_error=( + "too small to hold the current volume data", + "too small to hold", + "cannot reduce", + ), + ) + self._assert_cs_volume_untouched(vol_before, "shrink-below-used") + self.assertGreater( + self._flexvol_used_bytes(pool.name), + self.ONTAP_MIN_FLEXVOL_SIZE, + "ONTAP used space dropped below the FlexVol minimum after " + "the rejected shrink", + ) + luns_after_used = sorted( + (lun.get("uuid") or "", lun.get("name") or "") + for lun in self.ontap.list_luns_in_volume( + self.svm_name, pool.name + ) + ) + self.assertEqual( + luns_after_used, luns_before_used, + "LUN list in FlexVol '%s' changed after rejected " + "used-capacity shrink" % pool.name, + ) + self.assertTrue( + len(luns_after_used) > 0, + "LUN disappeared from FlexVol '%s' after rejected shrink" + % pool.name, + ) + finally: + self._delete_filler_file(pool.name, filler_name) + + # ------------------------------------------------------------------ - # Step 11 - Delete volume (LUN) then force-delete the pool + # Step 18 - grow and shrink pool with vm attached # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_11_delete_volume_and_pool(self): + def test_18_grow_and_shrink_pool_with_vm_attached(self): """ - Delete the volume from test_10, enter maintenance, then force-delete - the pool. + Grow the pool and shrink it straight back while the volume is + attached to the running VM from test_16, so resize is exercised + against a pool that is genuinely in use rather than an idle one. + Verifies: + - grow: CloudStack and ONTAP both reach the requested size + - shrink back: both return to the starting size + - the VM stays 'Running' and keeps the volume across both resizes + - ONTAP: FlexVol stays online + - ONTAP: the LUN backing the volume is still present + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_15 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_15 must pass first" + ) + if self.__class__.vm is None: + self.skipTest( + "No VM was deployed (test_16 skipped) - nothing to resize " + "a pool underneath" + ) + + pool = self.__class__.pool + vm = self.__class__.vm + vol = self.__class__.volume + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before the in-use resize") + original_size = int(getattr(listed[0], "capacitybytes", 0) or 0) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + log_progress( + logger, "info", + "Growing in-use pool '%s' from %d B to %d B (VM %s holds the " + "volume)", pool.name, original_size, grow_target, vm.id, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = grow_target + self.apiClient.updateStoragePool(cmd) + + grown = self._poll_pool_capacity(pool.id, grow_target, timeout=120) + self.assertEqual( + grown.state, "Up", + "Pool should stay 'Up' after growing under a VM, got '%s'" + % grown.state + ) + grown_vol = self._poll_ontap_volume_size( + pool.name, grow_target, timeout=120 + ) + self.assertEqual(grown_vol.get("state"), "online") + self._assert_vm_running_with_volume(vm.id, vol.id, "grow-with-vm") + self.assertTrue( + self.ontap.list_luns_in_volume(self.svm_name, pool.name), + "LUN disappeared from FlexVol '%s' during the in-use resize" + % pool.name, + ) + + log_progress( + logger, "info", + "Shrinking in-use pool '%s' back from %d B to %d B", + pool.name, grow_target, original_size, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = original_size + self.apiClient.updateStoragePool(cmd) + + shrunk = self._poll_pool_capacity(pool.id, original_size, timeout=120) + self.assertEqual( + shrunk.state, "Up", + "Pool should stay 'Up' after shrinking under a VM, got '%s'" + % shrunk.state + ) + shrunk_vol = self._poll_ontap_volume_size( + pool.name, original_size, timeout=120 + ) + self.assertEqual(shrunk_vol.get("state"), "online") + self._assert_vm_running_with_volume(vm.id, vol.id, "shrink-with-vm") + self.assertTrue( + self.ontap.list_luns_in_volume(self.svm_name, pool.name), + "LUN disappeared from FlexVol '%s' during the in-use resize" + % pool.name, + ) + + + # ------------------------------------------------------------------ + # Step 19 - delete volume and pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_19_delete_volume_and_pool(self): + """ + Detach the volume and destroy the VM from test_16, then delete the + volume, enter maintenance, and force-delete the pool. Verifies: - deleteVolume removes the LUN from ONTAP - Pool transitions to Maintenance @@ -730,8 +1538,13 @@ def test_11_delete_volume_and_pool(self): - ONTAP: FlexVol deleted - ONTAP: igroups for all cluster hosts deleted """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_10 must pass first") - self.assertIsNotNone(self.__class__.volume, "Volume absent - test_10 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_15 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_15 must pass first") + + # A volume still attached to a VM cannot be deleted, so unwind test_16 + # first. Both steps are no-ops when test_16 skipped. + self._detach_volume_if_attached(self.__class__.volume.id) + self.__class__._destroy_vm_if_present() pool = self.__class__.pool pool_name = pool.name @@ -797,152 +1610,12 @@ def test_11_delete_volume_and_pool(self): "ONTAP igroup '%s' still exists after pool deletion" % igroup_name ) - def _throwaway_pool_name(self, suffix): - return "OntapISCSI%s_%d" % (suffix, random.randint(0, 99999)) - - - def _cleanup_throwaway_pool(self, pool, flexvol_name=None): - """Best-effort teardown for an isolated test: CloudStack, then ONTAP. - - Never raises, so a failed assertion in the test body is the error - that surfaces. - """ - if pool is not None: - try: - listed = list_storage_pools(self.apiClient, id=pool.id) - except Exception: - listed = None - if listed: - try: - if listed[0].state != "Maintenance": - self._enter_maintenance(pool.id) - except Exception as exc: - logger.warning( - "cleanup: could not put pool '%s' into Maintenance: %s", - pool.name, exc, - ) - try: - self._delete_pool(pool.id, forced=True) - except Exception as exc: - logger.warning( - "cleanup: could not delete pool '%s': %s", - pool.name, exc, - ) - if flexvol_name is None: - flexvol_name = pool.name - if flexvol_name: - try: - if self.ontap.get_volume(flexvol_name) is not None: - self.ontap.offline_and_delete_volume(flexvol_name) - except Exception as exc: - logger.warning( - "cleanup: could not delete ONTAP FlexVol '%s': %s", - flexvol_name, exc, - ) - if pool is None: - return - try: - remaining = list_storage_pools(self.apiClient, id=pool.id) - except Exception: - remaining = None - if not remaining: - self.__class__.pool2 = None - - - def _assert_only_original_pool(self, pool): - """Assert the rejected create did not add a second pool of this name.""" - try: - listed = list_storage_pools(self.apiClient, name=pool.name) or [] - except CloudstackAPIException: - listed = [] - same_name = [ - item for item in listed if getattr(item, "name", None) == pool.name - ] - self.assertEqual( - [item.id for item in same_name], - [pool.id], - "CloudStack should still list only pool '%s' after the rejected " - "create, found: %s" % (pool.name, same_name), - ) - current = list_storage_pools(self.apiClient, id=pool.id) - self.assertTrue( - current, - "Original pool '%s' disappeared after the rejected create" - % pool.name, - ) - self.assertEqual( - current[0].state, "Up", - "Original pool '%s' should still be Up, got '%s'" - % (pool.name, current[0].state), - ) - - def _warn_if_duplicate_pool(self, duplicate, pool): - """Leave a stray duplicate in place. - - Deleting it would remove the FlexVol that the rest of the suite - still uses. - """ - if duplicate is None or getattr(duplicate, "id", None) == pool.id: - return - logger.warning( - "Duplicate pool '%s' (id=%s) was created against the shared " - "FlexVol; leaving it in place so cleanup does not delete the " - "pool the rest of the suite uses", - pool.name, duplicate.id, - ) - - def _assert_no_pool_named(self, pool_name): - """Assert CloudStack holds no storage pool with this name.""" - try: - listed = list_storage_pools(self.apiClient, name=pool_name) - except CloudstackAPIException: - listed = None - self.assertFalse( - listed, - "CloudStack should hold no pool named '%s' after a rejected " - "create, found: %s" % (pool_name, listed), - ) - - - def _assert_pool_gone_from_cs(self, pool_id, pool_name): - try: - remaining = list_storage_pools(self.apiClient, id=pool_id) - except CloudstackAPIException: - remaining = None - self.assertFalse( - remaining, - "Pool '%s' still listed in CloudStack after deletion" % pool_name, - ) - - - def _sweep_tracked_pool2(self): - """Clean a prior leftover before reusing the shared pool2 slot.""" - if self.__class__.pool2 is None: - return - self._cleanup_throwaway_pool(self.__class__.pool2) - if self.__class__.pool2 is not None: - self.skipTest("A previously tracked pool could not be cleaned up") - - - def _create_throwaway_pool(self, suffix): - """Create an isolated pool, park it in pool2, and return it.""" - self._sweep_tracked_pool2() - pool = self._create_pool(pool_name=self._throwaway_pool_name(suffix)) - self.__class__.pool2 = pool - self.assertEqual( - pool.state, "Up", - "Throwaway pool '%s' should be 'Up', got '%s'" - % (pool.name, pool.state), - ) - self.assertIsNotNone( - self.ontap.get_volume(pool.name), - "ONTAP FlexVol missing for throwaway pool '%s'" % pool.name, - ) - return pool - + # ------------------------------------------------------------------ + # Step 20 - delete pool with flexvol predeleted + # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_12_delete_pool_with_flexvol_predeleted(self): + def test_20_delete_pool_with_flexvol_predeleted(self): """ Delete the backing FlexVol directly on ONTAP, then delete the empty pool through CloudStack. Deletion must tolerate the missing volume @@ -980,9 +1653,12 @@ def test_12_delete_pool_with_flexvol_predeleted(self): self.__class__.pool2, flexvol_name=pool.name ) + # ------------------------------------------------------------------ + # Step 21 - delete pool with igroups predeleted + # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_13_delete_pool_with_igroups_predeleted(self): + def test_21_delete_pool_with_igroups_predeleted(self): """ Delete the per-host igroups directly on ONTAP, then delete the empty pool through CloudStack. Deletion must tolerate the missing igroups diff --git a/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py b/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py index 1f2801cb65e4..52b4fcaabff2 100644 --- a/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py +++ b/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py @@ -25,9 +25,10 @@ Test order — sequential workflow that must run in order: 01 Create zone-scoped iSCSI pool — pool.state Up; ONTAP FlexVol online; pre-existing shared igroups unchanged - 02 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged - 03 Enable zone-scoped pool — pool.state Up; FlexVol unchanged - 04 Delete zone-scoped pool — pool gone; FlexVol deleted; baseline restored + 02 Grow and shrink zone-scoped pool — CloudStack and ONTAP agree + 03 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged + 04 Enable zone-scoped pool — pool.state Up; FlexVol unchanged + 05 Delete zone-scoped pool — pool gone; FlexVol deleted; baseline restored Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -61,6 +62,7 @@ OntapRestClient, OntapTestBase, get_datacenter_config, + log_progress, ) logger = logging.getLogger("TestOntapISCSIZoneScopedPool") @@ -244,18 +246,73 @@ def test_01_create_zone_scoped_pool(self): ) # ------------------------------------------------------------------ - # Step 02 — Disable zone-scoped pool + # Step 02 — Grow and shrink zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["iscsi_zone_pool"], required_hardware=True) - def test_02_disable_zone_scoped_pool(self): + def test_02_grow_and_shrink_zone_scoped_pool(self): + """Grow the zone-scoped pool and shrink it back to its original size.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_01 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Zone-scoped pool missing before resize") + self.assertEqual( + listed[0].state, "Up", + "Zone-scoped pool must be Up before resize, got '%s'" + % listed[0].state, + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before zone-scoped resize" + ) + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + for target, operation in ( + (grow_target, "grow"), + (original_size, "shrink")): + log_progress( + logger, "info", + "%s zone-scoped iSCSI pool '%s': target=%d B", + operation.capitalize(), pool.name, target, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target + self.apiClient.updateStoragePool(cmd) + + resized = self._poll_pool_capacity(pool.id, target, timeout=120) + self.assertEqual( + resized.state, "Up", + "Zone-scoped pool should remain Up after %s, got '%s'" + % (operation, resized.state), + ) + resized_ontap = self._poll_ontap_volume_size( + pool.name, target, timeout=120 + ) + self.assertEqual(resized_ontap.get("state"), "online") + self._assert_igroup_baseline_unchanged( + "after %s of empty zone-scoped pool" % operation + ) + + # ------------------------------------------------------------------ + # Step 03 — Disable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_03_disable_zone_scoped_pool(self): """ Disable the zone-scoped iSCSI pool. Verifies: - pool.state is Disabled - ONTAP: FlexVol still online; igroups unchanged """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_02 must pass first") cmd = updateStoragePoolAPI.updateStoragePoolCmd() cmd.id = self.__class__.pool.id @@ -278,18 +335,18 @@ def test_02_disable_zone_scoped_pool(self): ) # ------------------------------------------------------------------ - # Step 03 — Enable zone-scoped pool + # Step 04 — Enable zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["iscsi_zone_pool"], required_hardware=True) - def test_03_enable_zone_scoped_pool(self): + def test_04_enable_zone_scoped_pool(self): """ Re-enable the zone-scoped iSCSI pool. Verifies: - pool.state is Up - ONTAP: FlexVol still online; igroups unchanged """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") cmd = updateStoragePoolAPI.updateStoragePoolCmd() cmd.id = self.__class__.pool.id @@ -312,11 +369,11 @@ def test_03_enable_zone_scoped_pool(self): ) # ------------------------------------------------------------------ - # Step 04 — Delete zone-scoped pool + # Step 05 — Delete zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["iscsi_zone_pool"], required_hardware=True) - def test_04_delete_zone_scoped_pool(self): + def test_05_delete_zone_scoped_pool(self): """ Enter maintenance then delete the zone-scoped iSCSI pool. Verifies: @@ -324,7 +381,7 @@ def test_04_delete_zone_scoped_pool(self): - ONTAP: FlexVol deleted - ONTAP: this pool's maps are gone and shared igroups are restored """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_04 must pass first") pool = self.__class__.pool pool_name = pool.name diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py index 17a6c9b88103..84e8681796bb 100644 --- a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py @@ -18,24 +18,32 @@ """ Sequential workflow integration tests for NetApp ONTAP NFS3 primary storage pool. -Tests are numbered test_01 ... test_13 and must run in that order. test_01 -and test_02 are isolated create rejections. Later steps build on the shared -pool from test_03. +Tests are numbered test_01 ... test_21 and must run in that order. test_01 +and test_02 are isolated create rejections. test_04 reuses the FlexVol created +by test_03. test_20 and test_21 are isolated delete cases. Workflow: 01 Reject create when no online assigned aggregate has enough free space 02 Reject create when capacitybytes has invalid values 03 Create primary storage pool 04 Reject create when a FlexVol of that name already exists on ONTAP - 05 Disable storage pool - 06 Enable storage pool - 07 Enter maintenance mode - 08 Cancel maintenance mode - 09 Delete the storage pool - 10 Create fresh pool and allocate a CloudStack volume - 11 Delete volume then force-delete the pool - 12 Delete an empty pool whose FlexVol was deleted directly on ONTAP - 13 Delete an empty pool whose NFS export policy was deleted on ONTAP + 05 Increase storage pool capacity to the maximum supported size + 06 Reject a grow beyond the maximum supported size + 07 Safely shrink storage pool capacity + 08 Disable storage pool + 09 Grow and shrink the pool while it is disabled + 10 Enable storage pool + 11 Enter maintenance mode + 12 Grow and shrink the pool while it is in maintenance mode + 13 Cancel maintenance mode + 14 Delete the storage pool + 15 Create fresh pool and allocate a CloudStack volume + 16 Deploy a VM and attach the ONTAP volume to it + 17 Reject shrink below the ONTAP used capacity of the attached volume + 18 Grow and shrink the pool while the VM holds the volume + 19 Detach/destroy the VM, delete volume, then force-delete the pool + 20 Delete an empty pool whose FlexVol was deleted directly on ONTAP + 21 Delete an empty pool whose NFS export policy was deleted on ONTAP Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -48,7 +56,7 @@ --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py -v -Note: Tests 03-09 share class-level state (sequential). test_04 reuses the +Note: Tests 03-19 share class-level state (sequential). test_04 reuses the FlexVol created by test_03. Running a single test with -m "test_NN" will invoke setUpClass but the guard assertion will fail immediately if earlier steps have not yet run. Always run the full suite. @@ -62,8 +70,10 @@ from nose.plugins.attrib import attr from marvin.cloudstackAPI import ( + attachVolume as attachVolumeAPI, createStoragePool as createStoragePoolAPI, deleteVolume as deleteVolumeAPI, + deployVirtualMachine as deployVirtualMachineAPI, updateStoragePool as updateStoragePoolAPI, ) from marvin.cloudstackException import CloudstackAPIException @@ -72,7 +82,7 @@ from ontap_test_base import ( OntapRestClient, OntapTestBase, _parse_pool_details, get_datacenter_config, - log_progress, run_invalid_capacitybytes_rejections, + _wait_for_vm_state, log_progress, run_invalid_capacitybytes_rejections, ) logger = logging.getLogger("TestOntapNFS3Workflow") @@ -101,7 +111,16 @@ class TestData: DETAIL_NFS_MOUNT_OPTS = "nfsmountopts" ONTAP_MIN_VOLUME_SIZE = 1677721600 + + # Maximum FlexVol size on AFF/FAS, raised from 100 TB in ONTAP 9.12.1P2. + # ONTAP writes this as "300 TB" but means binary, hence 1024**4. The + # value has to be hardcoded: no REST field reports the platform ceiling. + # Against an ONTAP older than 9.12.1P2 the real limit is 100 TB and the + # grow step will fail. CloudStack enforces no maximum of its own (the + # plugin validates only a 20 MB floor), so it is ONTAP that refuses + # anything above this. ONTAP_MAX_VOLUME_SIZE = 300 * 1024 ** 4 + ONTAP_GROW_TARGET_SIZE = ONTAP_MAX_VOLUME_SIZE def __init__(self, storage_ip, svm_name, username, password, protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", @@ -149,8 +168,12 @@ class TestOntapNFS3PrimaryStorageWorkflow(OntapTestBase): # ---- NFS3-specific shared state ------------------------------------ pool_ep_name = None # NFS export policy name for pool - pool2_ep_name = None # export policy for pool stashed from test_01-04 + pool2_ep_name = None # export policy for pool stashed from tests 03-14 cluster_host_ips = None + resize_original_size = None + + # VM state (vm/template_id/network_id) lives in OntapTestBase. + _vm_network_name_prefix = "ontap-nfs3-lifecycle-net" _vol_name_prefix = "OntapNFS3Vol" @@ -199,6 +222,8 @@ def setUpClass(cls): if getattr(h, "ipaddress", None) ] + cls._discover_vm_deploy_resources() + # No per-test tearDown — state intentionally persists between steps. # ------------------------------------------------------------------ @@ -206,7 +231,7 @@ def setUpClass(cls): # ------------------------------------------------------------------ def _create_pool(self, pool_name=None, capacitybytes=None): - """Create a pool; name and capacity default to the suite's values.""" + """Create a pool; name and capacity default to the suite values.""" ps = self.testdata[TestData.primaryStorage] storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] if pool_name is None: @@ -315,6 +340,15 @@ def _assert_pool_capacity(self, pool, label): % (label, ontap_size, configured) ) + def _volume_exists_in_cs(self, vol_id): + """Return True if the volume is still listed by CloudStack.""" + from marvin.cloudstackAPI import listVolumes as listVolumesAPI + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) or [] + return len(vols) > 0 + def _assert_pool_gone_from_cs(self, pool_id, pool_name): try: remaining = list_storage_pools(self.apiClient, id=pool_id) @@ -394,8 +428,163 @@ def _delete_volume_then_force_delete_pool(self, pool, vol, ep_name): self._assert_pool_gone_from_cs(pool.id, pool.name) self._assert_ontap_pool_gone(pool.name, ep_name) + def _throwaway_pool_name(self, suffix): + return "OntapNFS3%s_%d" % (suffix, random.randint(0, 99999)) + + def _cleanup_throwaway_pool(self, pool, flexvol_name=None, ep_name=None): + """Best-effort teardown for an isolated test: CloudStack, then ONTAP. + + Never raises, so a failed assertion in the test body is the error + that surfaces. The KVM unmount happens while the export is still + reachable, i.e. before the FlexVol goes away. + """ + backend_cleanup_safe = pool is None + if pool is not None: + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + listed = None + backend_cleanup_safe = not listed + if listed: + try: + if listed[0].state != "Maintenance": + self._enter_maintenance(pool.id) + except Exception as exc: + logger.warning( + "cleanup: could not put pool '%s' into Maintenance: %s", + pool.name, exc, + ) + try: + self._cleanup_kvm_storage_pool_mounts(pool.id) + except Exception as exc: + logger.warning( + "cleanup: could not safely unmount pool '%s': %s", + pool.name, exc, + ) + return + backend_cleanup_safe = True + try: + self._delete_pool(pool.id, forced=True) + except Exception as exc: + logger.warning( + "cleanup: could not delete pool '%s': %s", + pool.name, exc, + ) + if flexvol_name is None: + flexvol_name = pool.name + if backend_cleanup_safe and flexvol_name: + try: + if self.ontap.get_volume(flexvol_name) is not None: + self.ontap.offline_and_delete_volume(flexvol_name) + except Exception as exc: + logger.warning( + "cleanup: could not delete ONTAP FlexVol '%s': %s", + flexvol_name, exc, + ) + if backend_cleanup_safe and ep_name: + try: + if self.ontap.get_export_policy(ep_name) is not None: + self.ontap.delete_export_policy(ep_name) + except Exception as exc: + logger.warning( + "cleanup: could not delete export policy '%s': %s", + ep_name, exc, + ) + if pool is None: + return + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + if not remaining: + self.__class__.pool2 = None + self.__class__.pool2_ep_name = None + + def _assert_only_original_pool(self, pool): + """Assert the rejected create did not add a second pool of this name.""" + try: + listed = list_storage_pools(self.apiClient, name=pool.name) or [] + except CloudstackAPIException: + listed = [] + same_name = [ + item for item in listed if getattr(item, "name", None) == pool.name + ] + self.assertEqual( + [item.id for item in same_name], + [pool.id], + "CloudStack should still list only pool '%s' after the rejected " + "create, found: %s" % (pool.name, same_name), + ) + current = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue( + current, + "Original pool '%s' disappeared after the rejected create" + % pool.name, + ) + self.assertEqual( + current[0].state, "Up", + "Original pool '%s' should still be Up, got '%s'" + % (pool.name, current[0].state), + ) + + def _warn_if_duplicate_pool(self, duplicate, pool): + """Leave a stray duplicate in place. + + Deleting it would remove the FlexVol and export policy that the + rest of the suite still uses. + """ + if duplicate is None or getattr(duplicate, "id", None) == pool.id: + return + logger.warning( + "Duplicate pool '%s' (id=%s) was created against the shared " + "FlexVol; leaving it in place so cleanup does not delete the " + "pool the rest of the suite uses", + pool.name, duplicate.id, + ) + + def _assert_no_pool_named(self, pool_name): + """Assert CloudStack holds no storage pool with this name.""" + try: + listed = list_storage_pools(self.apiClient, name=pool_name) + except CloudstackAPIException: + listed = None + self.assertFalse( + listed, + "CloudStack should hold no pool named '%s' after a rejected " + "create, found: %s" % (pool_name, listed), + ) + + def _sweep_tracked_pool2(self): + """Clean a prior leftover before reusing the shared pool2 slot.""" + if self.__class__.pool2 is None: + return + self._cleanup_throwaway_pool( + self.__class__.pool2, + ep_name=self.__class__.pool2_ep_name, + ) + if self.__class__.pool2 is not None: + self.skipTest("A previously tracked pool could not be cleaned up safely") + + def _create_throwaway_pool(self, suffix): + """Create an isolated pool, park it in pool2, and return it.""" + self._sweep_tracked_pool2() + pool = self._create_pool(pool_name=self._throwaway_pool_name(suffix)) + self.__class__.pool2 = pool + ep_name = self._get_export_policy_name(pool) + self.__class__.pool2_ep_name = ep_name + self.assertEqual( + pool.state, "Up", + "Throwaway pool '%s' should be 'Up', got '%s'" + % (pool.name, pool.state), + ) + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol missing for throwaway pool '%s'" % pool.name, + ) + return pool, ep_name + # ------------------------------------------------------------------ - # Step 01 — Reject create when no aggregate has enough free space + # Step 01 - reject create when no aggregate space # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) @@ -466,7 +655,7 @@ def test_01_reject_create_when_no_aggregate_space(self): ) # ------------------------------------------------------------------ - # Step 02 — Reject create when capacitybytes has invalid values + # Step 02 - reject create with invalid capacitybytes values # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) @@ -500,7 +689,7 @@ def cleanup(pool, pool_name): ) # ------------------------------------------------------------------ - # Step 03 — Create primary storage pool + # Step 03 - create primary storage pool # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) @@ -560,8 +749,9 @@ def test_03_create_primary_storage_pool(self): # Capacity reporting self._assert_pool_capacity(pool, "pool-created") + # ------------------------------------------------------------------ - # Step 04 — Reject create when that FlexVol name already exists + # Step 04 - reject create when flexvol name exists # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) @@ -624,11 +814,175 @@ def test_04_reject_create_when_flexvol_name_exists(self): self._warn_if_duplicate_pool(duplicate, pool) # ------------------------------------------------------------------ - # Step 05 — Disable storage pool + # Step 05 - grow storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_05_grow_storage_pool(self): + """Grow the original pool to 300 TiB and verify CloudStack and ONTAP converge.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_03 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before capacity increase") + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol missing before increase") + + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + int(self.testdata[TestData.primaryStorage]["capacitybytes"]), + ) + requested_size = TestData.ONTAP_GROW_TARGET_SIZE + self.assertGreater( + requested_size, original_size, + "Grow target %d B must exceed the current pool size %d B" + % (requested_size, original_size) + ) + self.__class__.resize_original_size = original_size + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = requested_size + self.apiClient.updateStoragePool(cmd) + + resized_pool = self._poll_pool_capacity( + pool.id, requested_size, timeout=120 + ) + self.assertEqual(resized_pool.state, "Up") + resized_ontap_vol = self._poll_ontap_volume_size( + pool.name, requested_size, timeout=120 + ) + self.assertEqual(resized_ontap_vol.get("state"), "online") + self._poll_kvm_pool_capacity( + pool.id, pool.name, requested_size, timeout=120 + ) + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared after capacity increase", + ) + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + + # ------------------------------------------------------------------ + # Step 06 - reject grow above max size + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_06_reject_grow_above_max_size(self): + """ + Request a capacity above the 300 TiB ONTAP FlexVol maximum and verify + it is refused, leaving the pool at the size test_05 grew it to. + ONTAP reports this as an autosize-maximum violation rather than an + oversized-volume error. + Verifies: + - updateStoragePool raises CloudstackAPIException + - the error text is about the size limit, not about some other + failure such as the aggregate running out of space + - CloudStack capacity and ONTAP FlexVol size are both unchanged + - Pool stays 'Up' + - ONTAP: export policy is untouched + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_05 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before over-max resize") + self.assertEqual( + int(getattr(listed[0], "capacitybytes", 0) or 0), + TestData.ONTAP_MAX_VOLUME_SIZE, + "Pool should sit at the 300 TiB maximum - test_05 must pass first" + ) + + # Overshoot by only 1 GiB: at 0.0003% of the limit this probes the + # 300 TiB boundary itself rather than asking for something absurd. + above_max = self._align_flexvol_bytes( + TestData.ONTAP_MAX_VOLUME_SIZE + 1024 ** 3 + ) + log_progress( + logger, "info", + "Requesting %d B for pool '%s' (ONTAP maximum is %d B, " + "expect reject)", + above_max, pool.name, TestData.ONTAP_MAX_VOLUME_SIZE, + ) + + # ONTAP enforces the platform cap through autosize rather than + # rejecting the size outright, reporting "Volume maximum autosize + # must be greater than or equal to the current volume size 300.0TB". + # Matching on that keeps the test from passing when the resize fails + # for an unrelated reason, such as the aggregate running out of space. + self._assert_capacity_update_rejected( + pool.id, above_max, "grow-above-max-flexvol", + pool.id, pool.name, + expected_error=( + "maximum autosize", + "maximum size", + "exceeds", + ), + ) + + after = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(after, "Pool disappeared after rejected over-max grow") + self.assertEqual( + after[0].state, "Up", + "Pool should stay 'Up' after a rejected resize, got '%s'" + % after[0].state + ) + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared after a rejected over-max grow", + ) + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + + # ------------------------------------------------------------------ + # Step 07 - shrink storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_07_shrink_storage_pool(self): + """Shrink the original pool back to its initial safe capacity.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_05 must pass first" + ) + target_size = self.__class__.resize_original_size + self.assertIsNotNone( + target_size, "Original size absent — test_05 must pass first" + ) + pool = self.__class__.pool + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target_size + self.apiClient.updateStoragePool(cmd) + + shrunk_pool = self._poll_pool_capacity( + pool.id, target_size, timeout=120 + ) + self.assertEqual(shrunk_pool.state, "Up") + shrunk_ontap_vol = self._poll_ontap_volume_size( + pool.name, target_size, timeout=120 + ) + self.assertEqual(shrunk_ontap_vol.get("state"), "online") + self._poll_kvm_pool_capacity( + pool.id, pool.name, target_size, timeout=120 + ) + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared after safe shrink", + ) + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + + # ------------------------------------------------------------------ + # Step 08 - disable storage pool # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_05_disable_storage_pool(self): + def test_08_disable_storage_pool(self): """ Disable the pool and verify: - CloudStack reports Disabled @@ -659,35 +1013,108 @@ def test_05_disable_storage_pool(self): % self.__class__.pool_ep_name ) + # ------------------------------------------------------------------ - # Step 06 — Enable storage pool + # Step 09 - grow and shrink storage pool while disabled # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_06_enable_storage_pool(self): + def test_09_grow_and_shrink_storage_pool_while_disabled(self): """ - Re-enable the pool and verify: - - CloudStack reports Up - - ONTAP: FlexVol is still online and export policy unchanged + Grow and shrink the disabled pool and verify: + - Both capacity updates are accepted + - CloudStack and ONTAP reach each requested size + - KVM hosts report each resized NFS capacity + - The pool remains Disabled throughout + - The FlexVol remains online and the export policy is unchanged """ - self.assertIsNotNone(self.__class__.pool, "Pool absent — test_03 must pass first") - - cmd = updateStoragePoolAPI.updateStoragePoolCmd() - cmd.id = self.__class__.pool.id - cmd.enabled = True - self.apiClient.updateStoragePool(cmd) - - result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) - self.assertEqual(result.state, "Up") - - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_03 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before disabled resize") self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should be 'online' after enable, got '%s'" - % ontap_vol.get("state") + listed[0].state, "Disabled", + "Pool must be Disabled - test_08 must pass first, got '%s'" + % listed[0].state, ) - if self.__class__.pool_ep_name: + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before disabled resize" + ) + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + for target, operation in ( + (grow_target, "grow"), + (original_size, "shrink")): + log_progress( + logger, "info", + "%s disabled NFS3 pool '%s': target=%d B", + operation.capitalize(), pool.name, target, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target + self.apiClient.updateStoragePool(cmd) + + resized = self._poll_pool_capacity(pool.id, target, timeout=120) + self.assertEqual( + resized.state, "Disabled", + "Pool should remain Disabled after %s, got '%s'" + % (operation, resized.state), + ) + resized_ontap = self._poll_ontap_volume_size( + pool.name, target, timeout=120 + ) + self.assertEqual(resized_ontap.get("state"), "online") + self._poll_kvm_pool_capacity( + pool.id, pool.name, target, timeout=120 + ) + + if self.__class__.pool_ep_name: + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared during disabled resize", + ) + self._assert_export_policy_has_host_ips( + self.__class__.pool_ep_name + ) + + + # ------------------------------------------------------------------ + # Step 10 - enable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_10_enable_storage_pool(self): + """ + Re-enable the pool and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online and export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_03 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) self.assertIsNotNone( policy, @@ -695,12 +1122,13 @@ def test_06_enable_storage_pool(self): % self.__class__.pool_ep_name ) + # ------------------------------------------------------------------ - # Step 07 — Enter maintenance mode + # Step 11 - enter maintenance mode # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_07_enter_maintenance_mode(self): + def test_11_enter_maintenance_mode(self): """ Put the pool into maintenance mode and verify: - CloudStack reports Maintenance @@ -727,12 +1155,80 @@ def test_07_enter_maintenance_mode(self): % self.__class__.pool_ep_name ) + # ------------------------------------------------------------------ - # Step 08 — Cancel maintenance mode + # Step 12 - grow and shrink storage pool in maintenance # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_08_cancel_maintenance_mode(self): + def test_12_grow_and_shrink_storage_pool_in_maintenance(self): + """ + Grow and shrink the pool while it is in maintenance mode and verify: + - Both capacity updates are accepted + - CloudStack and ONTAP reach each requested size + - The pool remains in Maintenance throughout + - The FlexVol remains online and the export policy is unchanged + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_03 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before maintenance resize") + self.assertEqual( + listed[0].state, "Maintenance", + "Pool must be in Maintenance - test_11 must pass first, got '%s'" + % listed[0].state, + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before maintenance resize" + ) + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + for target, operation in ( + (grow_target, "grow"), + (original_size, "shrink")): + log_progress( + logger, "info", + "%s NFS3 pool '%s' while in Maintenance: target=%d B", + operation.capitalize(), pool.name, target, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target + self.apiClient.updateStoragePool(cmd) + + resized = self._poll_pool_capacity(pool.id, target, timeout=120) + self.assertEqual( + resized.state, "Maintenance", + "Pool should remain in Maintenance after %s, got '%s'" + % (operation, resized.state), + ) + resized_ontap = self._poll_ontap_volume_size( + pool.name, target, timeout=120 + ) + self.assertEqual(resized_ontap.get("state"), "online") + + if self.__class__.pool_ep_name: + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared during maintenance resize", + ) + + + # ------------------------------------------------------------------ + # Step 13 - cancel maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_13_cancel_maintenance_mode(self): """ Cancel maintenance mode and verify the pool returns to Up. @@ -771,12 +1267,13 @@ def test_08_cancel_maintenance_mode(self): % self.__class__.pool_ep_name ) + # ------------------------------------------------------------------ - # Step 09 — Delete the storage pool + # Step 14 - delete pool from maintenance # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_09_delete_pool_from_maintenance(self): + def test_14_delete_pool_from_maintenance(self): """ Enter maintenance mode then delete the storage pool. @@ -790,12 +1287,13 @@ def test_09_delete_pool_from_maintenance(self): pool_name = pool.name ep_name = self.__class__.pool_ep_name - # Pool is Up after test_06 succeeded; must enter Maintenance before deletion. + # Pool is Up after test_13 succeeded; must enter Maintenance before deletion. self._enter_maintenance(pool.id) self._delete_pool(pool.id) self.__class__.pool = None self.__class__.pool_ep_name = None + self.__class__.resize_original_size = None # CloudStack: pool must be gone try: @@ -819,12 +1317,13 @@ def test_09_delete_pool_from_maintenance(self): "Export policy '%s' still exists after pool deletion" % ep_name ) + # ------------------------------------------------------------------ - # Step 10 - Create fresh pool and allocate a CloudStack volume + # Step 15 - create volume on pool # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_10_create_volume_on_pool(self): + def test_15_create_volume_on_pool(self): """ Create a new NFS3 pool and allocate a CloudStack data volume. For NFS3, createAsync is a no-op on ONTAP (volume is a CloudStack record @@ -839,7 +1338,7 @@ def test_10_create_volume_on_pool(self): self.__class__.pool = pool log_progress( logger, "info", - "test_10: created storage pool name='%s' id=%s state=%s", + "test_15: created storage pool name='%s' id=%s state=%s", pool.name, pool.id, pool.state, ) @@ -856,7 +1355,7 @@ def test_10_create_volume_on_pool(self): self.assertIsNotNone(vol, "createVolume returned None") log_progress( logger, "info", - "test_10: created CloudStack volume name='%s' id=%s state=%s " + "test_15: created CloudStack volume name='%s' id=%s state=%s " "on pool='%s' (id=%s) account='%s' domain='%s' — " "switch to this account in the UI to see the volume", getattr(vol, "name", "?"), getattr(vol, "id", "?"), @@ -885,216 +1384,371 @@ def test_10_create_volume_on_pool(self): # Capacity reporting: FlexVol size and reported capacity unchanged after volume allocation self._assert_pool_capacity(pool, "volume-allocated") + # ------------------------------------------------------------------ - # Step 11 - Delete volume then force-delete the pool + # Step 16 - create vm and attach volume # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_11_delete_volume_and_pool(self): + def test_16_create_vm_and_attach_volume(self): """ - Delete the volume from test_10, enter maintenance, then force-delete - the pool. + Deploy a VM and attach the ONTAP data volume from test_15 to it. Verifies: - - deleteVolume completes (or expected NFS3 libvirt pool-not-found) - - Pool transitions to Maintenance - - Pool is removed from CloudStack after force deletion - - ONTAP: FlexVol deleted - - ONTAP: export policy deleted + - VM reaches 'Running' + - attachVolume sets the volume's virtualmachineid (on NFS3 shared + storage CloudStack leaves the volume state at 'Ready', so + virtualmachineid is the reliable attach signal) + - VM is still 'Running' after the attach + - ONTAP: FlexVol stays online and the pool capacity is unchanged + - ONTAP: the volume's data file is materialised in the FlexVol + (NFS3 creates it lazily, during attach rather than createVolume) """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_10 must pass first") - self.assertIsNotNone(self.__class__.volume, "Volume absent - test_10 must pass first") + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_15 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_15 must pass first" + ) + if self.__class__.template_id is None: + self.skipTest( + "No ready user KVM template in the zone - cannot deploy a VM" + ) + self.assertIsNotNone( + self.__class__.service_offering_id, + "No service offering available - check the CloudStack setup" + ) pool = self.__class__.pool - pool_name = pool.name - ep_name = self.__class__.pool_ep_name vol = self.__class__.volume - ontap_vol = self.ontap.get_volume(pool_name) + log_progress( + logger, "info", + "Deploying VM for NFS3 pool '%s' (template=%s, offering=%s)", + pool.name, self.__class__.template_id, + self.__class__.service_offering_id, + ) + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + self.__class__.vm = vm + + vm_obj = _wait_for_vm_state(self.apiClient, vm.id, "Running", + timeout=600) self.assertIsNotNone( - ontap_vol, - "ONTAP FlexVol '%s' should still exist before cleanup" % pool_name + vm_obj, "VM %s never appeared in listVirtualMachines" % vm.id ) self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should still be 'online' before cleanup" + vm_obj.state, "Running", + "VM should be 'Running' before attach, got '%s'" % vm_obj.state ) - self._delete_volume_then_force_delete_pool(pool, vol, ep_name) - self.__class__.pool = None - self.__class__.volume = None - self.__class__.pool_ep_name = None + log_progress( + logger, "info", + "Attaching volume '%s' to VM '%s'", vol.id, vm.id, + ) + attach_cmd = attachVolumeAPI.attachVolumeCmd() + attach_cmd.id = vol.id + attach_cmd.virtualmachineid = vm.id + self.assertIsNotNone( + self.apiClient.attachVolume(attach_cmd), + "attachVolume returned None" + ) - # Clean up pool from test_01-04 (left in Maintenance when test_05/06 skipped). - pool2 = self.__class__.pool2 - if pool2 is not None: - self._force_delete_pool_in_maintenance( - pool2, self.__class__.pool2_ep_name - ) - self.__class__.pool2 = None - self.__class__.pool2_ep_name = None + vol_vmid = self._poll_volume_attached(vol.id, timeout=180) + self.assertEqual( + vol_vmid, vm.id, + "Volume %s should report virtualmachineid=%s after attach, got %s" + % (vol.id, vm.id, vol_vmid) + ) - def _throwaway_pool_name(self, suffix): - return "OntapNFS3%s_%d" % (suffix, random.randint(0, 99999)) + vm_after = _wait_for_vm_state(self.apiClient, vm.id, "Running", + timeout=60) + self.assertEqual( + vm_after.state, "Running", + "VM should still be 'Running' after attach, got '%s'" + % vm_after.state + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing after volume attach" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after attach, got '%s'" + % ontap_vol.get("state") + ) - def _cleanup_throwaway_pool(self, pool, flexvol_name=None, ep_name=None): - """Best-effort teardown for an isolated test: CloudStack, then ONTAP. + # NFS3 materialises the volume's backing file during attach, not at + # createVolume time, so it must be present now. + files = self.ontap.list_files_in_volume(pool.name) + self.assertIsNotNone( + next((f for f in files if vol.id in f), None), + "No data file matching volume UUID '%s' in FlexVol '%s' after " + "attach; files present: %s" % (vol.id, pool.name, files) + ) - Never raises, so a failed assertion in the test body is the error - that surfaces. The KVM unmount happens while the export is still - reachable, i.e. before the FlexVol goes away. - """ - backend_cleanup_safe = pool is None - if pool is not None: - try: - listed = list_storage_pools(self.apiClient, id=pool.id) - except Exception: - listed = None - backend_cleanup_safe = not listed - if listed: - try: - if listed[0].state != "Maintenance": - self._enter_maintenance(pool.id) - except Exception as exc: - logger.warning( - "cleanup: could not put pool '%s' into Maintenance: %s", - pool.name, exc, - ) - try: - self._cleanup_kvm_storage_pool_mounts(pool.id) - except Exception as exc: - logger.warning( - "cleanup: could not safely unmount pool '%s': %s", - pool.name, exc, - ) - return - backend_cleanup_safe = True - try: - self._delete_pool(pool.id, forced=True) - except Exception as exc: - logger.warning( - "cleanup: could not delete pool '%s': %s", - pool.name, exc, - ) - if flexvol_name is None: - flexvol_name = pool.name - if backend_cleanup_safe and flexvol_name: - try: - if self.ontap.get_volume(flexvol_name) is not None: - self.ontap.offline_and_delete_volume(flexvol_name) - except Exception as exc: - logger.warning( - "cleanup: could not delete ONTAP FlexVol '%s': %s", - flexvol_name, exc, - ) - if backend_cleanup_safe and ep_name: - try: - if self.ontap.get_export_policy(ep_name) is not None: - self.ontap.delete_export_policy(ep_name) - except Exception as exc: - logger.warning( - "cleanup: could not delete export policy '%s': %s", - ep_name, exc, - ) - if pool is None: - return - try: - remaining = list_storage_pools(self.apiClient, id=pool.id) - except Exception: - remaining = None - if not remaining: - self.__class__.pool2 = None - self.__class__.pool2_ep_name = None + self._assert_pool_capacity(pool, "vm-attached") - def _assert_only_original_pool(self, pool): - """Assert the rejected create did not add a second pool of this name.""" - try: - listed = list_storage_pools(self.apiClient, name=pool.name) or [] - except CloudstackAPIException: - listed = [] - same_name = [ - item for item in listed if getattr(item, "name", None) == pool.name - ] - self.assertEqual( - [item.id for item in same_name], - [pool.id], - "CloudStack should still list only pool '%s' after the rejected " - "create, found: %s" % (pool.name, same_name), + # ------------------------------------------------------------------ + # Step 17 - reject shrink below used capacity + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_17_reject_shrink_below_used_capacity(self): + """ + Attempt to shrink the pool below ONTAP used space and verify the + request is rejected while the volume, pool capacity, and NFS export + policy remain unchanged. + + Runs after test_16, so the FlexVol holds the data file of a volume + attached to a running VM. That file is thin, though, and in practice + leaves used space under the 20 MiB FlexVol floor, so incompressible + data is written through the ONTAP files API to get above it and + removed before this test returns. The fill is skipped whenever used + space is already high enough on its own. + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_16 must pass first" ) - current = list_storage_pools(self.apiClient, id=pool.id) - self.assertTrue( - current, - "Original pool '%s' disappeared after the rejected create" - % pool.name, + self.assertIsNotNone( + self.__class__.volume, "Volume absent — test_16 must pass first" ) - self.assertEqual( - current[0].state, "Up", - "Original pool '%s' should still be Up, got '%s'" - % (pool.name, current[0].state), + pool = self.__class__.pool + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol not found for pool '%s'" % pool.name, ) - def _warn_if_duplicate_pool(self, duplicate, pool): - """Leave a stray duplicate in place. - - Deleting it would remove the FlexVol and export policy that the - rest of the suite still uses. - """ - if duplicate is None or getattr(duplicate, "id", None) == pool.id: - return - logger.warning( - "Duplicate pool '%s' (id=%s) was created against the shared " - "FlexVol; leaving it in place so cleanup does not delete the " - "pool the rest of the suite uses", - pool.name, duplicate.id, + export_policy_name = self.__class__.pool_ep_name + self.assertIsNotNone( + export_policy_name, + "Export policy name was not recorded by test_15", + ) + export_policy_before = self.ontap.get_export_policy( + export_policy_name + ) + self.assertIsNotNone( + export_policy_before, + "Export policy '%s' missing before rejected used-capacity shrink" + % export_policy_name, ) - def _assert_no_pool_named(self, pool_name): - """Assert CloudStack holds no storage pool with this name.""" + vol_before = self._cs_volume_snapshot(self.__class__.volume.id) + # The attached volume is thin on both protocols, so it usually does + # not put 20 MiB on disk by itself. This writes incompressible data + # only when used space is still under the FlexVol floor, and removes + # it before returning. + filler_name, used_bytes = self._fill_flexvol_above_minimum(pool.name) + self.__class__.filler_filename = filler_name + self.__class__.filler_flexvol = pool.name if filler_name else None try: - listed = list_storage_pools(self.apiClient, name=pool_name) - except CloudstackAPIException: - listed = None - self.assertFalse( - listed, - "CloudStack should hold no pool named '%s' after a rejected " - "create, found: %s" % (pool_name, listed), + below_used = self._shrink_target_below_used(used_bytes) + log_progress( + logger, "info", + "Shrinking NFS3 pool '%s' below ONTAP used capacity: " + "used=%d B, requested=%d B " + "(FlexVol minimum=%d B, expect reject)", + pool.name, used_bytes, below_used, self.ONTAP_MIN_FLEXVOL_SIZE, + ) + self._assert_capacity_update_rejected( + pool.id, below_used, "shrink-below-used", + pool.id, pool.name, + # ONTAP's wording: "Selected volume size is too small to + # hold the current volume data. New volume size must be at + # least 34.3MB ...". Matching the specific phrase stops an + # unrelated resize failure from counting as a pass. + expected_error=( + "too small to hold the current volume data", + "too small to hold", + "cannot reduce", + ), + ) + self._assert_cs_volume_untouched(vol_before, "shrink-below-used") + self.assertGreater( + self._flexvol_used_bytes(pool.name), + self.ONTAP_MIN_FLEXVOL_SIZE, + "ONTAP used space dropped below the FlexVol minimum after " + "the rejected shrink", + ) + export_policy_after_used = self.ontap.get_export_policy( + export_policy_name + ) + self.assertIsNotNone( + export_policy_after_used, + "Export policy '%s' disappeared after rejected " + "used-capacity shrink" % export_policy_name, + ) + self.assertEqual( + export_policy_after_used.get("name"), + export_policy_before.get("name"), + "Export policy changed during rejected used-capacity shrink", + ) + self._assert_export_policy_has_host_ips(export_policy_name) + finally: + self._delete_filler_file(pool.name, filler_name) + + + # ------------------------------------------------------------------ + # Step 18 - grow and shrink pool with vm attached + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_18_grow_and_shrink_pool_with_vm_attached(self): + """ + Grow the pool and shrink it straight back while the volume is + attached to the running VM from test_16, so resize is exercised + against a pool that is genuinely in use rather than an idle one. + Verifies: + - grow: CloudStack and ONTAP both reach the requested size + - shrink back: both return to the starting size + - the VM stays 'Running' and keeps the volume across both resizes + - ONTAP: FlexVol stays online + - ONTAP: export policy still covers every cluster host IP + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_15 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_15 must pass first" ) + if self.__class__.vm is None: + self.skipTest( + "No VM was deployed (test_16 skipped) - nothing to resize " + "a pool underneath" + ) + pool = self.__class__.pool + vm = self.__class__.vm + vol = self.__class__.volume - def _sweep_tracked_pool2(self): - """Clean a prior leftover before reusing the shared pool2 slot.""" - if self.__class__.pool2 is None: - return - self._cleanup_throwaway_pool( - self.__class__.pool2, - ep_name=self.__class__.pool2_ep_name, + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before the in-use resize") + original_size = int(getattr(listed[0], "capacitybytes", 0) or 0) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + log_progress( + logger, "info", + "Growing in-use pool '%s' from %d B to %d B (VM %s holds the " + "volume)", pool.name, original_size, grow_target, vm.id, ) - if self.__class__.pool2 is not None: - self.skipTest("A previously tracked pool could not be cleaned up safely") + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = grow_target + self.apiClient.updateStoragePool(cmd) + grown = self._poll_pool_capacity(pool.id, grow_target, timeout=120) + self.assertEqual( + grown.state, "Up", + "Pool should stay 'Up' after growing under a VM, got '%s'" + % grown.state + ) + grown_vol = self._poll_ontap_volume_size( + pool.name, grow_target, timeout=120 + ) + self.assertEqual(grown_vol.get("state"), "online") + self._poll_kvm_pool_capacity( + pool.id, pool.name, grow_target, timeout=120 + ) + self._assert_vm_running_with_volume(vm.id, vol.id, "grow-with-vm") + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) - def _create_throwaway_pool(self, suffix): - """Create an isolated pool, park it in pool2, and return it.""" - self._sweep_tracked_pool2() - pool = self._create_pool(pool_name=self._throwaway_pool_name(suffix)) - self.__class__.pool2 = pool - ep_name = self._get_export_policy_name(pool) - self.__class__.pool2_ep_name = ep_name + log_progress( + logger, "info", + "Shrinking in-use pool '%s' back from %d B to %d B", + pool.name, grow_target, original_size, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = original_size + self.apiClient.updateStoragePool(cmd) + + shrunk = self._poll_pool_capacity(pool.id, original_size, timeout=120) self.assertEqual( - pool.state, "Up", - "Throwaway pool '%s' should be 'Up', got '%s'" - % (pool.name, pool.state), + shrunk.state, "Up", + "Pool should stay 'Up' after shrinking under a VM, got '%s'" + % shrunk.state ) + shrunk_vol = self._poll_ontap_volume_size( + pool.name, original_size, timeout=120 + ) + self.assertEqual(shrunk_vol.get("state"), "online") + self._poll_kvm_pool_capacity( + pool.id, pool.name, original_size, timeout=120 + ) + self._assert_vm_running_with_volume(vm.id, vol.id, "shrink-with-vm") + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + + # ------------------------------------------------------------------ + # Step 19 - delete volume and pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_19_delete_volume_and_pool(self): + """ + Detach the volume and destroy the VM from test_16, then delete the + volume, enter maintenance, and force-delete the pool. + Verifies: + - deleteVolume completes (or expected NFS3 libvirt pool-not-found) + - Pool transitions to Maintenance + - Pool is removed from CloudStack after force deletion + - ONTAP: FlexVol deleted + - ONTAP: export policy deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_15 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_15 must pass first") + + # A volume still attached to a VM cannot be deleted, so unwind test_16 + # first. Both steps are no-ops when test_16 skipped. + self._detach_volume_if_attached(self.__class__.volume.id) + self.__class__._destroy_vm_if_present() + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + vol = self.__class__.volume + + ontap_vol = self.ontap.get_volume(pool_name) self.assertIsNotNone( - self.ontap.get_volume(pool.name), - "ONTAP FlexVol missing for throwaway pool '%s'" % pool.name, + ontap_vol, + "ONTAP FlexVol '%s' should still exist before cleanup" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' before cleanup" ) - return pool, ep_name + self._delete_volume_then_force_delete_pool(pool, vol, ep_name) + self.__class__.pool = None + self.__class__.volume = None + self.__class__.pool_ep_name = None + + # Clean up pool from test_03-04 (left in Maintenance when test_08/06 skipped). + pool2 = self.__class__.pool2 + if pool2 is not None: + self._force_delete_pool_in_maintenance( + pool2, self.__class__.pool2_ep_name + ) + self.__class__.pool2 = None + self.__class__.pool2_ep_name = None + + # ------------------------------------------------------------------ + # Step 20 - delete pool with flexvol predeleted + # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_12_delete_pool_with_flexvol_predeleted(self): + def test_20_delete_pool_with_flexvol_predeleted(self): """ Delete the backing FlexVol directly on ONTAP, then delete the empty pool through CloudStack. Deletion must tolerate the missing volume @@ -1144,9 +1798,12 @@ def test_12_delete_pool_with_flexvol_predeleted(self): ep_name=ep_name, ) + # ------------------------------------------------------------------ + # Step 21 - delete pool with export policy predeleted + # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_13_delete_pool_with_export_policy_predeleted(self): + def test_21_delete_pool_with_export_policy_predeleted(self): """ Delete the NFS export policy directly on ONTAP, then delete the empty pool through CloudStack. Deletion must tolerate the missing diff --git a/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py b/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py index 21260151da9f..cc75106d9406 100644 --- a/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py +++ b/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py @@ -26,9 +26,10 @@ Test order — sequential workflow that must run in order: 01 Create zone-scoped NFS3 pool — pool.state Up; ONTAP FlexVol online; export policy has all cluster host IPs - 02 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged - 03 Enable zone-scoped pool — pool.state Up; FlexVol unchanged - 04 Delete zone-scoped pool — pool gone; FlexVol deleted; export policy deleted + 02 Grow and shrink zone-scoped pool — CloudStack, ONTAP, and KVM agree + 03 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged + 04 Enable zone-scoped pool — pool.state Up; FlexVol unchanged + 05 Delete zone-scoped pool — pool gone; FlexVol deleted; export policy deleted Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -65,6 +66,7 @@ OntapTestBase, _parse_pool_details, get_datacenter_config, + log_progress, ) logger = logging.getLogger("TestOntapZoneScopedPool") @@ -292,18 +294,77 @@ def test_01_create_zone_scoped_pool(self): ) # ------------------------------------------------------------------ - # Step 02 — Disable zone-scoped pool + # Step 02 — Grow and shrink zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["zone_pool"], required_hardware=True) - def test_02_disable_zone_scoped_pool(self): + def test_02_grow_and_shrink_zone_scoped_pool(self): + """Grow the zone-scoped pool and shrink it back to its original size.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_01 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Zone-scoped pool missing before resize") + self.assertEqual( + listed[0].state, "Up", + "Zone-scoped pool must be Up before resize, got '%s'" + % listed[0].state, + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before zone-scoped resize" + ) + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + for target, operation in ( + (grow_target, "grow"), + (original_size, "shrink")): + log_progress( + logger, "info", + "%s zone-scoped NFS3 pool '%s': target=%d B", + operation.capitalize(), pool.name, target, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target + self.apiClient.updateStoragePool(cmd) + + resized = self._poll_pool_capacity(pool.id, target, timeout=120) + self.assertEqual( + resized.state, "Up", + "Zone-scoped pool should remain Up after %s, got '%s'" + % (operation, resized.state), + ) + resized_ontap = self._poll_ontap_volume_size( + pool.name, target, timeout=120 + ) + self.assertEqual(resized_ontap.get("state"), "online") + self._poll_kvm_pool_capacity( + pool.id, pool.name, target, timeout=120 + ) + + self._assert_export_policy_has_host_ips( + self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 — Disable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_03_disable_zone_scoped_pool(self): """ Disable the zone-scoped pool. Verifies: - pool.state is Disabled - ONTAP: FlexVol still online; export policy unchanged """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_02 must pass first") cmd = updateStoragePoolAPI.updateStoragePoolCmd() cmd.id = self.__class__.pool.id @@ -329,18 +390,18 @@ def test_02_disable_zone_scoped_pool(self): ) # ------------------------------------------------------------------ - # Step 03 — Enable zone-scoped pool + # Step 04 — Enable zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["zone_pool"], required_hardware=True) - def test_03_enable_zone_scoped_pool(self): + def test_04_enable_zone_scoped_pool(self): """ Re-enable the zone-scoped pool. Verifies: - pool.state is Up - ONTAP: FlexVol still online; export policy unchanged """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") cmd = updateStoragePoolAPI.updateStoragePoolCmd() cmd.id = self.__class__.pool.id @@ -366,11 +427,11 @@ def test_03_enable_zone_scoped_pool(self): ) # ------------------------------------------------------------------ - # Step 04 — Delete zone-scoped pool + # Step 05 — Delete zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["zone_pool"], required_hardware=True) - def test_04_delete_zone_scoped_pool(self): + def test_05_delete_zone_scoped_pool(self): """ Enter maintenance then delete the zone-scoped pool. Verifies: @@ -378,7 +439,7 @@ def test_04_delete_zone_scoped_pool(self): - ONTAP: FlexVol deleted - ONTAP: export policy deleted """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_04 must pass first") pool = self.__class__.pool pool_name = pool.name @@ -427,7 +488,7 @@ def tearDownClass(cls): """ Clean up any lingering zone-scoped pool NFS mounts on KVM hosts before the base-class teardown deletes the ONTAP FlexVol. Without - this, a failed test_04 leaves a stale NFS mount that will cause + this, a failed test_05 leaves a stale NFS mount that will cause KVMHAMonitor to reboot the host. """ for pool in [p for p in (cls.pool2, cls.pool) if p is not None]: diff --git a/test/integration/plugins/ontap/ontap_test_base.py b/test/integration/plugins/ontap/ontap_test_base.py index 479a98a54cc4..37431dfa565e 100644 --- a/test/integration/plugins/ontap/ontap_test_base.py +++ b/test/integration/plugins/ontap/ontap_test_base.py @@ -29,6 +29,7 @@ """ import logging +import os import random import re import requests @@ -36,18 +37,27 @@ import time import urllib3 from unittest import SkipTest -from urllib.parse import urlparse +from urllib.parse import quote, urlparse from marvin.cloudstackAPI import ( cancelStorageMaintenance, + createNetwork as createNetworkAPI, createVolume as createVolumeAPI, + deleteNetwork as deleteNetworkAPI, deleteStoragePool as deleteStoragePoolAPI, deleteVolume as deleteVolumeAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, destroyVolume as destroyVolumeAPI, + detachVolume as detachVolumeAPI, enableStorageMaintenance, listDiskOfferings as listDiskOfferingsAPI, + listNetworkOfferings as listNetworkOfferingsAPI, + listNetworks as listNetworksAPI, + listServiceOfferings as listServiceOfferingsAPI, + listTemplates as listTemplatesAPI, listVirtualMachines as listVirtualMachinesAPI, listVolumes as listVolumesAPI, + stopVirtualMachine as stopVirtualMachineAPI, resizeVolume as resizeVolumeAPI, updateStoragePool as updateStoragePoolAPI, ) @@ -56,7 +66,10 @@ from marvin.cloudstackTestCase import cloudstackTestCase from marvin.jsonHelper import jsonDump from marvin.lib.base import Account, DiskOffering -from marvin.lib.common import get_domain, get_zone, list_clusters, list_storage_pools +from marvin.sshClient import SshClient +from marvin.lib.common import ( + get_domain, get_zone, list_clusters, list_storage_pools, +) from marvin.lib.utils import cleanup_resources from helpers import kvm_host @@ -202,10 +215,11 @@ def _raise_http(self, resp): response=resp, ) - def _patch(self, path, params=None, json_body=None, timeout=60): + def _patch(self, path, payload=None, params=None, json_body=None, timeout=60): url = self._base + path + body = json_body if json_body is not None else payload resp = requests.patch( - url, auth=self._auth, params=params, json=json_body, + url, auth=self._auth, params=params, json=body, verify=False, timeout=timeout, ) self._raise_http(resp) @@ -221,6 +235,14 @@ def _post(self, path, params=None, data=None, json_body=None, timeout=60, self._raise_http(resp) return self._response_data(resp) + def _volume_uuid(self, vol_name): + vol = self.get_volume(vol_name) + return (vol or {}).get("uuid") + + def _files_path(self, vol_uuid, filename): + encoded = quote("/" + filename.lstrip("/"), safe="") + return "/storage/volumes/%s/files/%s" % (vol_uuid, encoded) + def _wait_for_job(self, response, timeout=120): """Wait for an asynchronous ONTAP response, if it contains a job.""" job = (response or {}).get("job") or {} @@ -413,7 +435,6 @@ def delete_lun_map(self, lun_map): self._delete( "/protocols/san/lun-maps/%s/%s" % (lun_uuid, igroup_uuid) ) - def delete_volume(self, name): """Delete the ONTAP FlexVol with the given name. No-op if not found.""" data = self._get("/storage/volumes", params={"name": name}) @@ -443,7 +464,13 @@ def get_volume(self, name): uuid = records[0].get("uuid") if uuid: return self._get("/storage/volumes/%s" % uuid, - params={"fields": "name,uuid,state,space,nas.path,nas.export_policy"}) + params={ + "fields": ( + "name,uuid,state,space.size,space.used," + "space.snapshot.reserve_percent," + "nas.path,nas.export_policy" + ) + }) return records[0] # -- NFS helpers --------------------------------------------------------- @@ -554,7 +581,6 @@ def list_files_in_volume(self, vol_name, path="/"): if not vol_uuid: return [] # URL-encode the path component (/ → %2F) and embed it in the URL. - from urllib.parse import quote encoded_path = quote(path, safe="") try: resp = self._get( @@ -667,23 +693,52 @@ def delete_lun(self, svm_name, lun_path): self._delete("/storage/luns/%s" % uuid) return True - def delete_file_in_volume(self, vol_name, file_path): + def write_file_in_volume(self, vol_name, filename, size_bytes, + chunk_bytes=512 * 1024): + """Write *size_bytes* of incompressible data into a FlexVol file. + + Zeros compress to almost nothing on ONTAP, so the payload is random. + The files API requires ``multipart/form-data`` and rejects writes + larger than 1 MiB, so data is sent in chunks. No VM, NFS mount, or + CloudStack volume is required. """ - Delete a file inside a FlexVol via ONTAP files API. + vol_uuid = self._volume_uuid(vol_name) + if not vol_uuid: + raise RuntimeError("ONTAP FlexVol '%s' not found" % vol_name) + url_path = self._files_path(vol_uuid, filename) + written = 0 + size_bytes = int(size_bytes) + while written < size_bytes: + chunk = min(int(chunk_bytes), size_bytes - written) + files = { + "file": (filename, os.urandom(chunk), + "application/octet-stream"), + } + self._post( + url_path, + params={"byte_offset": written, "overwrite": "true"}, + files=files, + timeout=120, + ) + written += chunk + + def delete_file_in_volume(self, vol_name, file_path): + """Delete a file inside a FlexVol. - ``file_path`` may be absolute (``/foo/bar``) or relative to volume root. - Returns True if a delete was attempted on an existing volume. + ``file_path`` may be absolute (``/foo/bar``) or relative to the + volume root. Returns True when the volume exists and the delete was + issued. A missing file (404/409) is ignored so cleanup can retry. + Returns False when the volume is gone. """ - vol = self.get_volume(vol_name) - if not vol: - return False - vol_uuid = vol.get("uuid", "") + vol_uuid = self._volume_uuid(vol_name) if not vol_uuid: return False - from urllib.parse import quote - path = file_path if file_path.startswith("/") else "/" + file_path - encoded_path = quote(path, safe="") - self._delete("/storage/volumes/%s/files/%s" % (vol_uuid, encoded_path)) + try: + self._delete(self._files_path(vol_uuid, file_path)) + except requests.HTTPError as exc: + status = getattr(exc.response, "status_code", None) + if status not in (404, 409): + raise return True @@ -771,11 +826,31 @@ def run_invalid_capacitybytes_rejections(test, create_pool, cleanup_pool, log): class OntapTestBase(cloudstackTestCase): + # ONTAP refuses to shrink a FlexVol below this; distinct from the + # plugin's 1.56 GiB create-time floor (ONTAP_MIN_VOLUME_SIZE). + ONTAP_MIN_FLEXVOL_SIZE = 20 * 1024 * 1024 + FILLER_FILENAME = "ontap-filler.bin" + FILLER_SIZE = 32 * 1024 * 1024 + # ---- shared state (set/cleared by individual tests) ---------------- pool = None volume = None + # Volumes a single test allocates on top of ``volume`` and deletes before + # it returns; tracked here only so a failed run still cleans them up. + extra_volumes = [] + # File written into the FlexVol by tests that must raise ONTAP used space. + filler_filename = None + filler_flexvol = None pool2 = None volume2 = None + # ---- VM state, for suites that deploy an instance ------------------ + vm = None + template_id = None + service_offering_id = None + network_id = None + _created_network_id = None + # Prefix for a guest network this suite creates on Advanced zones. + _vm_network_name_prefix = "ontap-vm-net" disk_offering_id = None svm_name = None cluster_hosts = None @@ -908,9 +983,214 @@ def _cleanup_kvm_storage_pool_mounts(cls, pool_uuid): # ---- shared teardown ----------------------------------------------- + @classmethod + def _all_tracked_volumes(cls): + """Every volume the suite created, most recent first, without repeats.""" + seen = set() + ordered = [] + for vol in list(cls.extra_volumes or []) + [cls.volume2, cls.volume]: + vol_id = getattr(vol, "id", None) + if vol is None or vol_id in seen: + continue + seen.add(vol_id) + ordered.append(vol) + return ordered + + # ---- VM deploy/attach helpers -------------------------------------- + + @classmethod + def _discover_vm_deploy_resources(cls): + """Resolve the template, service offering, and network for VM deploys. + + A missing template is not fatal: ``template_id`` is left as None so + callers can skip the VM step while the rest of the suite still runs. + On Advanced zones an existing account network is reused when present, + otherwise an Isolated one is created and torn down in tearDownClass. + """ + tpl_cmd = listTemplatesAPI.listTemplatesCmd() + tpl_cmd.templatefilter = "all" + tpl_cmd.listall = True + tpl_cmd.zoneid = cls.zone.id + templates = cls.apiClient.listTemplates(tpl_cmd) or [] + kvm_ready = [ + t for t in templates + if getattr(t, "hypervisor", "").lower() == "kvm" + and getattr(t, "isready", False) + and getattr(t, "templatetype", "").upper() != "SYSTEM" + ] + cls.template_id = kvm_ready[0].id if kvm_ready else None + if cls.template_id is None: + logger.warning( + "No ready user KVM template in zone '%s' — VM steps will skip." + % cls.zone.name + ) + + so_cmd = listServiceOfferingsAPI.listServiceOfferingsCmd() + offerings = cls.apiClient.listServiceOfferings(so_cmd) or [] + if offerings: + offerings.sort(key=lambda s: getattr(s, "memory", 9999)) + cls.service_offering_id = offerings[0].id + + cls.network_id = None + if getattr(cls.zone, "networktype", "Basic").lower() != "advanced": + return + + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.account = cls.account.name + net_cmd.domainid = cls.domain.id + nets = cls.apiClient.listNetworks(net_cmd) or [] + if nets: + cls.network_id = nets[0].id + return + + no_cmd = listNetworkOfferingsAPI.listNetworkOfferingsCmd() + no_cmd.state = "Enabled" + no_cmd.guestiptype = "Isolated" + no_cmd.specifyvlan = "false" + no_offerings = cls.apiClient.listNetworkOfferings(no_cmd) or [] + snat_offering = next( + (o for o in no_offerings + if "SourceNat" in o.name and "Vpc" not in o.name + and "NSX" not in o.name and "Netris" not in o.name), + no_offerings[0] if no_offerings else None + ) + if snat_offering is None: + return + cn_cmd = createNetworkAPI.createNetworkCmd() + cn_cmd.zoneid = cls.zone.id + cn_cmd.networkofferingid = snat_offering.id + cn_cmd.name = "%s-%d" % (cls._vm_network_name_prefix, + random.randint(0, 9999)) + cn_cmd.displaytext = "ONTAP test VM network" + cn_cmd.account = cls.account.name + cn_cmd.domainid = cls.domain.id + net = cls.apiClient.createNetwork(cn_cmd) + cls.network_id = net.id + cls._created_network_id = net.id + + @classmethod + def _destroy_vm_if_present(cls): + """Stop and expunge the suite's VM. Safe to call when none exists.""" + if cls.vm is None: + return + vm_id = cls.vm.id + try: + vms = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + state = vms[0].state.lower() if vms else "unknown" + if state not in ("stopped", "destroyed", "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = vm_id + stop_cmd.forced = True + cls.apiClient.stopVirtualMachine(stop_cmd) + _wait_for_vm_state(cls.apiClient, vm_id, "Stopped", timeout=180) + except Exception as e: + logger.warning("Could not stop VM %s: %s" % (vm_id, e)) + try: + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm_id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + except Exception as e: + logger.warning("Could not destroy VM %s: %s" % (vm_id, e)) + cls.vm = None + + @classmethod + def _delete_created_network(cls): + """Delete the guest network this suite created, if any.""" + if cls._created_network_id is None: + return + try: + dn_cmd = deleteNetworkAPI.deleteNetworkCmd() + dn_cmd.id = cls._created_network_id + cls.apiClient.deleteNetwork(dn_cmd) + except Exception as e: + logger.warning( + "Could not delete network %s: %s" % (cls._created_network_id, e) + ) + cls._created_network_id = None + + def _poll_volume_attached(self, vol_id, timeout=180, interval=5): + """Poll listVolumes until virtualmachineid is set; return it or None. + + ONTAP-backed volumes stay in state 'Ready' when attached, so the + virtualmachineid field is the reliable signal. + """ + deadline = time.time() + timeout + vol_vmid = None + while time.time() < deadline: + vols = self.apiClient.listVolumes(_list_vols_cmd(vol_id)) or [] + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + if vol_vmid: + return vol_vmid + time.sleep(interval) + return vol_vmid + + def _assert_vm_running_with_volume(self, vm_id, vol_id, label): + """Assert the VM is still up and still owns the volume.""" + vm_obj = _wait_for_vm_state(self.apiClient, vm_id, "Running", + timeout=60) + self.assertIsNotNone( + vm_obj, "[%s] VM %s vanished from listVirtualMachines" + % (label, vm_id), + ) + self.assertEqual( + vm_obj.state, "Running", + "[%s] VM should still be 'Running', got '%s'" + % (label, vm_obj.state), + ) + vol = self._get_cs_volume(vol_id) + self.assertIsNotNone( + vol, "[%s] volume %s is no longer listed" % (label, vol_id), + ) + self.assertEqual( + getattr(vol, "virtualmachineid", None), vm_id, + "[%s] volume %s should still be attached to VM %s, " + "virtualmachineid is %s" + % (label, vol_id, vm_id, getattr(vol, "virtualmachineid", None)), + ) + + def _detach_volume_if_attached(self, vol_id): + """Detach the volume when a VM holds it. No-op otherwise.""" + vols = self.apiClient.listVolumes(_list_vols_cmd(vol_id)) or [] + if not vols or not getattr(vols[0], "virtualmachineid", None): + return + try: + cmd = detachVolumeAPI.detachVolumeCmd() + cmd.id = vol_id + self.apiClient.detachVolume(cmd) + except Exception as e: + logger.warning("Could not detach volume %s: %s" % (vol_id, e)) + return + deadline = time.time() + 120 + while time.time() < deadline: + vols = self.apiClient.listVolumes(_list_vols_cmd(vol_id)) or [] + if not vols or not getattr(vols[0], "virtualmachineid", None): + return + time.sleep(5) + logger.warning( + "Volume %s still reports a virtualmachineid after detach" % vol_id + ) + @classmethod def tearDownClass(cls): """Best-effort cleanup of any resources left behind by a failed run.""" + cls._destroy_vm_if_present() + cls._delete_created_network() + if (getattr(cls, "ontap", None) is not None + and getattr(cls, "filler_filename", None) + and getattr(cls, "filler_flexvol", None)): + try: + cls.ontap.delete_file_in_volume( + cls.filler_flexvol, cls.filler_filename + ) + except Exception as fe: + logger.warning( + "tearDownClass: could not delete filler file %s in %s: %s" + % (cls.filler_filename, cls.filler_flexvol, fe) + ) + cls.filler_filename = None + cls.filler_flexvol = None for pool in [p for p in (cls.pool2, cls.pool) if p is not None]: try: # Step 1: Check current pool state @@ -944,7 +1224,7 @@ def tearDownClass(cls): # state. For iSCSI this works even in Maintenance; for NFS3/KVM # it may fail with NPE ("storagePoolInformation is null") when # pool is in Maintenance — that exception is caught below. - for vol in [v for v in (cls.volume2, cls.volume) if v is not None]: + for vol in cls._all_tracked_volumes(): try: cmd = deleteVolumeAPI.deleteVolumeCmd() cmd.id = vol.id @@ -1006,7 +1286,7 @@ def tearDownClass(cls): pass # Clean up volumes that may not have been handled with pool teardown - for vol in [v for v in (cls.volume2, cls.volume) if v is not None]: + for vol in cls._all_tracked_volumes(): try: cmd = deleteVolumeAPI.deleteVolumeCmd() cmd.id = vol.id @@ -1432,6 +1712,394 @@ def _wait_for_non_cache_lun_count(self, pool, expected, timeout=180, % (expected, pool.name, current), ) + def _poll_pool_capacity(self, pool_id, expected_bytes, timeout=120, + interval=5): + """Poll listStoragePools until capacitybytes equals expected_bytes.""" + start = time.time() + deadline = start + timeout + attempt = 0 + current = 0 + log_progress( + logger, "info", + "Waiting for pool %s to report capacitybytes=%d " + "(timeout=%ds, poll every %ds).", + pool_id, expected_bytes, timeout, interval, + ) + while time.time() < deadline: + attempt += 1 + elapsed = int(time.time() - start) + remaining = max(0, int(deadline - time.time())) + pools = list_storage_pools(self.apiClient, id=pool_id) + if pools: + current = int(getattr(pools[0], "capacitybytes", 0) or 0) + if current == expected_bytes: + log_progress( + logger, "info", + "Pool %s reported capacitybytes=%d after %ds (%d polls).", + pool_id, expected_bytes, elapsed, attempt, + ) + return pools[0] + log_progress( + logger, "info", + "Capacity poll #%d: pool %s capacitybytes=%d (want %d) " + "[elapsed %ds, ~%ds left]", + attempt, pool_id, current, expected_bytes, + elapsed, remaining, + ) + time.sleep(interval) + log_progress( + logger, "error", + "Pool %s did not report capacitybytes=%d within %ds (last: %d).", + pool_id, expected_bytes, timeout, current, + ) + self.fail( + "Pool %s did not report capacitybytes=%d within %ds (last: %d)" + % (pool_id, expected_bytes, timeout, current) + ) + + def _poll_kvm_pool_capacity(self, pool_id, volume_name, expected_bytes, + timeout=120, interval=5): + """Poll the KVM NFS mount until it reports the resized capacity.""" + self.assertTrue( + self.kvm_hosts_ssh_creds, + "No KVM host SSH credentials are configured for host-side " + "capacity verification", + ) + volume = self.ontap.get_volume(volume_name) + self.assertIsNotNone( + volume, + "ONTAP FlexVol '%s' was not found for host-side capacity " + "verification" % volume_name, + ) + reserve_percent = ( + volume.get("space", {}).get("snapshot", {}).get("reserve_percent") + ) + self.assertIsNotNone( + reserve_percent, + "ONTAP FlexVol '%s' did not report " + "space.snapshot.reserve_percent" % volume_name, + ) + reserve_percent = int(reserve_percent) + self.assertGreaterEqual(reserve_percent, 0) + self.assertLessEqual(reserve_percent, 100) + # CloudStack and ONTAP report provisioned size, while statvfs reports + # capacity after the FlexVol's configurable snapshot reserve. + expected_host_bytes = ( + expected_bytes * (100 - reserve_percent) // 100 + ) + tolerance = max(1024 * 1024 * 1024, expected_host_bytes // 10000) + deadline = time.time() + timeout + observed = {} + errors = {} + + while time.time() < deadline: + observed = {} + errors = {} + for creds in self.kvm_hosts_ssh_creds: + host_ip = creds["host"] + try: + ssh = SshClient( + host_ip, 22, + creds["user"], creds["password"], + retries=3, delay=3, timeout=15.0, + ) + output = ssh.execute( + "df -B1 --output=size '/mnt/%s' 2>/dev/null | " + "awk 'NR == 2 {print $1}'" % pool_id + ) + values = [ + line.strip() for line in output if line.strip().isdigit() + ] + if values: + observed[host_ip] = int(values[-1]) + else: + errors[host_ip] = "NFS pool is not mounted or has no capacity" + except Exception as ex: + errors[host_ip] = str(ex) + + if observed and all( + abs(size - expected_host_bytes) <= tolerance + for size in observed.values()): + logger.info( + "KVM hosts report resized pool %s usable capacity near " + "%d B (%d%% snapshot reserve on %d B provisioned): %s", + pool_id, expected_host_bytes, reserve_percent, + expected_bytes, observed, + ) + return observed + time.sleep(interval) + + self.fail( + "KVM hosts did not report pool %s usable capacity near %d B " + "(%d%% snapshot reserve on %d B provisioned) within %ds " + "(tolerance=%d B, observed=%s, errors=%s)" + % (pool_id, expected_host_bytes, reserve_percent, expected_bytes, + timeout, tolerance, observed, errors) + ) + + def _poll_ontap_volume_size(self, volume_name, expected_bytes, + timeout=120, interval=5): + """Poll ONTAP until the FlexVol space.size equals expected_bytes.""" + start = time.time() + deadline = start + timeout + attempt = 0 + current = 0 + log_progress( + logger, "info", + "Waiting for ONTAP FlexVol '%s' to report space.size=%d " + "(timeout=%ds, poll every %ds).", + volume_name, expected_bytes, timeout, interval, + ) + while time.time() < deadline: + attempt += 1 + elapsed = int(time.time() - start) + remaining = max(0, int(deadline - time.time())) + volume = self.ontap.get_volume(volume_name) + if volume: + current = int(volume.get("space", {}).get("size", 0) or 0) + if current == expected_bytes: + log_progress( + logger, "info", + "ONTAP FlexVol '%s' reported space.size=%d after " + "%ds (%d polls).", + volume_name, expected_bytes, elapsed, attempt, + ) + return volume + log_progress( + logger, "info", + "ONTAP capacity poll #%d: FlexVol '%s' space.size=%d " + "(want %d) [elapsed %ds, ~%ds left]", + attempt, volume_name, current, expected_bytes, + elapsed, remaining, + ) + time.sleep(interval) + log_progress( + logger, "error", + "ONTAP FlexVol '%s' did not report space.size=%d within %ds " + "(last: %d).", + volume_name, expected_bytes, timeout, current, + ) + self.fail( + "ONTAP FlexVol '%s' did not report space.size=%d within %ds " + "(last: %d)" + % (volume_name, expected_bytes, timeout, current) + ) + + def _cs_volume_snapshot(self, vol_id): + """Capture id, state, size, and pool so a later check can detect mutation.""" + vol = self._get_cs_volume(vol_id) + self.assertIsNotNone( + vol, "CloudStack volume %s is not listed in listVolumes" % vol_id + ) + pool_id = ( + getattr(vol, "storageid", None) + or getattr(vol, "poolid", None) + ) + return { + "id": getattr(vol, "id", None), + "state": getattr(vol, "state", None), + "size": int(getattr(vol, "size", 0) or 0), + "poolid": pool_id, + } + + def _assert_cs_volume_untouched(self, before, label): + """Assert listVolumes still returns the same id, state, size, and pool.""" + after = self._cs_volume_snapshot(before["id"]) + self.assertEqual( + after["id"], before["id"], + "[%s] CloudStack volume id changed (%s -> %s)" + % (label, before["id"], after["id"]), + ) + self.assertEqual( + after["state"], before["state"], + "[%s] CloudStack volume state changed (%s -> %s)" + % (label, before["state"], after["state"]), + ) + self.assertEqual( + after["size"], before["size"], + "[%s] CloudStack volume size changed (%s -> %s)" + % (label, before["size"], after["size"]), + ) + self.assertEqual( + after["poolid"], before["poolid"], + "[%s] CloudStack volume poolid changed (%s -> %s)" + % (label, before["poolid"], after["poolid"]), + ) + + def _align_flexvol_bytes(self, value): + """Round *value* down to the 4 KiB boundary ONTAP uses for FlexVol size.""" + return (int(value) // 4096) * 4096 + + def _flexvol_used_bytes(self, vol_name): + """ONTAP physical used bytes on the FlexVol (used + reserved).""" + ontap_vol = self.ontap.get_volume(vol_name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol '%s' not found" % vol_name + ) + space = ontap_vol.get("space") or {} + return int(space.get("used") or 0) + + def _fill_flexvol_above_minimum(self, vol_name, timeout=90): + """Write a FlexVol file until ONTAP used space exceeds the minimum. + + Returns ``(filler_filename_or_None, used_bytes)``. The caller must + delete any returned filename. No CloudStack volume or VM is created. + """ + used = self._flexvol_used_bytes(vol_name) + if used > self.ONTAP_MIN_FLEXVOL_SIZE: + log_progress( + logger, "info", + "FlexVol '%s' already has %d B used (ONTAP FlexVol minimum " + "%d B); no filler file needed", + vol_name, used, self.ONTAP_MIN_FLEXVOL_SIZE, + ) + return None, used + log_progress( + logger, "info", + "FlexVol '%s' has %d B used; writing %d B filler file '%s'", + vol_name, used, self.FILLER_SIZE, self.FILLER_FILENAME, + ) + self.ontap.write_file_in_volume( + vol_name, self.FILLER_FILENAME, self.FILLER_SIZE + ) + deadline = time.time() + timeout + while time.time() < deadline: + used = self._flexvol_used_bytes(vol_name) + if used > self.ONTAP_MIN_FLEXVOL_SIZE: + break + time.sleep(2) + self.assertGreater( + used, self.ONTAP_MIN_FLEXVOL_SIZE, + "FlexVol '%s' still has only %d B used after writing a %d B " + "filler file; cannot exceed the ONTAP FlexVol minimum of %d B" + % (vol_name, used, self.FILLER_SIZE, self.ONTAP_MIN_FLEXVOL_SIZE), + ) + log_progress( + logger, "info", + "FlexVol '%s' has %d B used after filler file '%s'", + vol_name, used, self.FILLER_FILENAME, + ) + return self.FILLER_FILENAME, used + + def _delete_filler_file(self, vol_name, filename): + if not filename or not vol_name: + return + try: + self.ontap.delete_file_in_volume(vol_name, filename) + except Exception as exc: + logger.warning( + "could not delete filler file '%s' in FlexVol '%s': %s", + filename, vol_name, exc, + ) + if self.__class__.filler_filename == filename: + self.__class__.filler_filename = None + self.__class__.filler_flexvol = None + + def _shrink_target_below_used(self, used_bytes): + """4 KiB-aligned size below used_bytes but above the FlexVol min.""" + used_bytes = int(used_bytes) + self.assertGreater( + used_bytes, self.ONTAP_MIN_FLEXVOL_SIZE, + "Used capacity %d B is not above the ONTAP FlexVol " + "minimum %d B; cannot distinguish a used-capacity reject " + "from a minimum-size reject" + % (used_bytes, self.ONTAP_MIN_FLEXVOL_SIZE), + ) + target = self._align_flexvol_bytes(used_bytes - 4096) + if target <= self.ONTAP_MIN_FLEXVOL_SIZE: + target = self._align_flexvol_bytes( + (used_bytes + self.ONTAP_MIN_FLEXVOL_SIZE) // 2 + ) + self.assertGreater( + target, self.ONTAP_MIN_FLEXVOL_SIZE, + "Shrink target %d B is not above the ONTAP FlexVol minimum %d B" + % (target, self.ONTAP_MIN_FLEXVOL_SIZE), + ) + self.assertLess( + target, used_bytes, + "Shrink target %d B must be below used capacity %d B" + % (target, used_bytes), + ) + return target + + def _assert_capacity_update_rejected( + self, cmd_pool_id, capacitybytes, label, verify_pool_id, + volume_name, expected_error=None): + """Assert updateStoragePool(capacitybytes) fails and sizes stay put. + + CloudStack and ONTAP are compared against their own pre-request values + because the two do not have to agree: a FlexVol created with a snapshot + reserve reports a larger space.size than the usable capacity + CloudStack records. + """ + listed = list_storage_pools(self.apiClient, id=verify_pool_id) + self.assertTrue( + listed, + "[%s] listStoragePools returned no result for pool %s" + % (label, verify_pool_id), + ) + ontap_vol = self.ontap.get_volume(volume_name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol '%s' not found" % (label, volume_name), + ) + before_cs = int(getattr(listed[0], "capacitybytes", 0) or 0) + before_ontap = int(ontap_vol.get("space", {}).get("size", 0) or 0) + log_progress( + logger, "info", + "Negative resize %s: pool_id=%s capacitybytes=%s " + "(expect reject; CS=%d B ONTAP=%d B)", + label, cmd_pool_id, capacitybytes, before_cs, before_ontap, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = cmd_pool_id + cmd.capacitybytes = capacitybytes + with self.assertRaises(CloudstackAPIException) as caught: + self.apiClient.updateStoragePool(cmd) + error_text = str(caught.exception) + log_progress( + logger, "info", "Rejected resize %s: %s", label, error_text, + ) + if expected_error: + needles = ( + expected_error + if isinstance(expected_error, (list, tuple)) + else (expected_error,) + ) + self.assertTrue( + any(needle in error_text for needle in needles), + "[%s] expected the rejection to report one of %r, got: %s" + % (label, needles, error_text), + ) + + listed = list_storage_pools(self.apiClient, id=verify_pool_id) + self.assertTrue( + listed, + "[%s] pool disappeared after rejected resize" % label, + ) + after_cs = int(getattr(listed[0], "capacitybytes", 0) or 0) + self.assertEqual( + after_cs, before_cs, + "[%s] CloudStack capacity changed after rejected resize " + "(got %d, want %d)" % (label, after_cs, before_cs), + ) + self.assertEqual( + listed[0].state, "Up", + "[%s] pool should remain Up after rejected resize, got '%s'" + % (label, listed[0].state), + ) + ontap_vol = self.ontap.get_volume(volume_name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol disappeared after rejected resize" % label, + ) + after_ontap = int(ontap_vol.get("space", {}).get("size", 0) or 0) + self.assertEqual( + after_ontap, before_ontap, + "[%s] ONTAP FlexVol size changed after rejected resize " + "(got %d, want %d)" % (label, after_ontap, before_ontap), + ) + def _create_volume(self, pool_id): """Create a data volume on the given pool; uses _vol_name_prefix.""" cmd = createVolumeAPI.createVolumeCmd() @@ -1515,3 +2183,40 @@ def _delete_pool(self, pool_id, forced=False): if forced: cmd.forced = True self.apiClient.deleteStoragePool(cmd) + + +# --------------------------------------------------------------------------- +# Module-level VM helpers +# --------------------------------------------------------------------------- + +def _list_vms_cmd(vm_id): + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.id = vm_id + cmd.listall = True + return cmd + + +def _list_vols_cmd(vol_id): + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + return cmd + + +def _wait_for_vm_state(api_client, vm_id, target_state, timeout=120, + interval=5): + """Poll listVirtualMachines until the VM reaches target_state. + + Returns the last VM object seen, which may not be in target_state if the + timeout expires — callers assert on the state themselves. + """ + deadline = time.time() + timeout + vm_obj = None + while time.time() < deadline: + vms = api_client.listVirtualMachines(_list_vms_cmd(vm_id)) or [] + if vms: + vm_obj = vms[0] + if vm_obj.state == target_state: + return vm_obj + time.sleep(interval) + return vm_obj