From e45ea9592ee2c26ae72f26f0c93857569447da28 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 14 Sep 2026 15:29:12 -0400 Subject: [PATCH 01/20] engine-schema: upgrade path for 24.0.0 Adds a 4.23.0 to 24.0.0 upgrade path (squashed from sb/upgradepath-424: engine-schema: upgrade path for 24.0.0, fix CS version, fix upgrade unit tests for cutover, fix imports). --- .../cloud/upgrade/DatabaseUpgradeChecker.java | 15 +++-- .../cloud/upgrade/dao/Upgrade42300to2400.java | 30 +++++++++ .../db/schema-42300to2400-cleanup.sql | 20 ++++++ .../META-INF/db/schema-42300to2400.sql | 20 ++++++ .../upgrade/DatabaseUpgradeCheckerTest.java | 59 +++++++++--------- .../cloudstack/veeam/api/dto/Version.java | 8 ++- .../cloudstack/utils/CloudStackVersion.java | 62 +++++++++++++++---- .../utils/CloudStackVersionTest.java | 43 +++++++++++-- 8 files changed, 205 insertions(+), 52 deletions(-) create mode 100644 engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java create mode 100644 engine/schema/src/main/resources/META-INF/db/schema-42300to2400-cleanup.sql create mode 100644 engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index c3a982aa70e5..f3ec76a27c7a 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -91,11 +91,12 @@ import com.cloud.upgrade.dao.Upgrade42020to42030; import com.cloud.upgrade.dao.Upgrade42030to42040; import com.cloud.upgrade.dao.Upgrade42040to42100; -import com.cloud.upgrade.dao.Upgrade42100to42200; -import com.cloud.upgrade.dao.Upgrade42200to42210; import com.cloud.upgrade.dao.Upgrade420to421; +import com.cloud.upgrade.dao.Upgrade42100to42200; import com.cloud.upgrade.dao.Upgrade421to430; +import com.cloud.upgrade.dao.Upgrade42200to42210; import com.cloud.upgrade.dao.Upgrade42210to42300; +import com.cloud.upgrade.dao.Upgrade42300to2400; import com.cloud.upgrade.dao.Upgrade430to440; import com.cloud.upgrade.dao.Upgrade431to440; import com.cloud.upgrade.dao.Upgrade432to440; @@ -248,6 +249,7 @@ public DatabaseUpgradeChecker() { .next("4.21.0.0", new Upgrade42100to42200()) .next("4.22.0.0", new Upgrade42200to42210()) .next("4.22.1.0", new Upgrade42210to42300()) + .next("4.23.0.0", new Upgrade42300to2400()) .build(); } @@ -513,8 +515,13 @@ protected void doUpgrades(GlobalLock lock) { String csVersion = parseSystemVmMetadata(); final CloudStackVersion sysVmVersion = CloudStackVersion.parse(csVersion); final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue); - SystemVmTemplateRegistration.CS_MAJOR_VERSION = sysVmVersion.getMajorRelease() + "." + sysVmVersion.getMinorRelease(); - SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease()); + if (sysVmVersion.usesNewVersioning()) { + SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.valueOf(sysVmVersion.getMajorRelease()); + SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getMajorRelease()); + } else { + SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.format("%d.%d", sysVmVersion.getMajorRelease(), sysVmVersion.getMinorRelease()); + SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease()); + } LOGGER.info("DB version = {} Code Version = {}", dbVersion, currentVersion); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java new file mode 100644 index 000000000000..ce217cef9e75 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade.dao; + +public class Upgrade42300to2400 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { + + @Override + public String[] getUpgradableVersionRange() { + return new String[]{"4.23.0.0", "24.0.0"}; + } + + @Override + public String getUpgradedVersion() { + return "24.0.0"; + } +} diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400-cleanup.sql new file mode 100644 index 000000000000..861a038fe7a7 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400-cleanup.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade cleanup from 4.23.0.0 to 24.0.0 +--; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql new file mode 100644 index 000000000000..7c11013a17d2 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade from 4.23.0.0 to 24.0.0 +--; diff --git a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java index 884398cf410d..3810d03161d9 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java @@ -16,20 +16,25 @@ // under the License. package com.cloud.upgrade; -import java.sql.SQLException; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + import java.lang.reflect.Field; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; import javax.sql.DataSource; import org.apache.cloudstack.utils.CloudStackVersion; -import org.junit.Test; -import org.junit.Before; import org.junit.After; +import org.junit.Before; +import org.junit.Test; import org.junit.runner.RunWith; - import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Mockito; @@ -55,15 +60,8 @@ import com.cloud.upgrade.dao.Upgrade471to480; import com.cloud.upgrade.dao.Upgrade480to481; import com.cloud.upgrade.dao.Upgrade490to4910; - import com.cloud.utils.db.TransactionLegacy; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertArrayEquals; - @RunWith(MockitoJUnitRunner.class) public class DatabaseUpgradeCheckerTest { @@ -214,10 +212,10 @@ public void testFindUpgradePath452to490() { @Test public void testCalculateUpgradePathUnknownDbVersion() { - final CloudStackVersion dbVersion = CloudStackVersion.parse("4.99.0.0"); + final CloudStackVersion dbVersion = CloudStackVersion.parse("99.0.0"); assertNotNull(dbVersion); - final CloudStackVersion currentVersion = CloudStackVersion.parse("4.99.1.0"); + final CloudStackVersion currentVersion = CloudStackVersion.parse("99.1.0"); assertNotNull(currentVersion); final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); @@ -234,7 +232,7 @@ public void testCalculateUpgradePathFromKnownDbVersion() { final CloudStackVersion dbVersion = CloudStackVersion.parse("4.17.0.0"); assertNotNull(dbVersion); - final CloudStackVersion currentVersion = CloudStackVersion.parse("4.99.1.0"); + final CloudStackVersion currentVersion = CloudStackVersion.parse("99.1.0"); assertNotNull(currentVersion); final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); @@ -268,10 +266,7 @@ public void testCalculateUpgradePathFromLatestDbVersion() { final CloudStackVersion dbVersion = checker.getLatestVersion(); assertNotNull(dbVersion); - final CloudStackVersion currentVersion = CloudStackVersion.parse(dbVersion.getMajorRelease() + "." - + dbVersion.getMinorRelease() + "." - + dbVersion.getPatchRelease() + "." - + (dbVersion.getSecurityRelease() + 1)); + final CloudStackVersion currentVersion = getNextSecurityRelease(dbVersion); assertNotNull(currentVersion); final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); @@ -293,10 +288,7 @@ public void testCalculateUpgradePathFrom41800toNextSecurityRelease() { final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); assertNotNull(upgrades); - final CloudStackVersion nextSecurityRelease = CloudStackVersion.parse(currentVersion.getMajorRelease() + "." - + currentVersion.getMinorRelease() + "." - + currentVersion.getPatchRelease() + "." - + (currentVersion.getSecurityRelease() + 1)); + final CloudStackVersion nextSecurityRelease = getNextSecurityRelease(currentVersion); assertNotNull(nextSecurityRelease); final DbUpgrade[] upgradesToNext = checker.calculateUpgradePath(dbVersion, nextSecurityRelease); @@ -306,16 +298,26 @@ public void testCalculateUpgradePathFrom41800toNextSecurityRelease() { assertTrue(upgradesToNext[upgradesToNext.length - 1] instanceof NoopDbUpgrade); } + private static CloudStackVersion getNextSecurityRelease(CloudStackVersion version, int increment) { + String nextSecurityReleaseVersionStr = version.getMajorRelease() + "." + + version.getMinorRelease() + "." + + (version.usesNewVersioning() ? "" : version.getPatchRelease() + ".") + + (version.getSecurityRelease() + increment); + + return CloudStackVersion.parse(nextSecurityReleaseVersionStr); + } + + private static CloudStackVersion getNextSecurityRelease(CloudStackVersion version) { + return getNextSecurityRelease(version, 1); + } + @Test public void testCalculateUpgradePathFromSecurityReleaseToLatest() { final CloudStackVersion dbVersion = CloudStackVersion.parse("4.17.2.0"); // a EOL version assertNotNull(dbVersion); - final CloudStackVersion oldSecurityRelease = CloudStackVersion.parse(dbVersion.getMajorRelease() + "." - + dbVersion.getMinorRelease() + "." - + dbVersion.getPatchRelease() + "." - + (dbVersion.getSecurityRelease() + 100)); + final CloudStackVersion oldSecurityRelease = getNextSecurityRelease(dbVersion, 100); assertNotNull(oldSecurityRelease); // fake security release 4.17.2.100 final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); @@ -347,10 +349,7 @@ public void testCalculateUpgradePathFromSecurityReleaseToNextSecurityRelease() { final CloudStackVersion currentVersion = checker.getLatestVersion(); assertNotNull(currentVersion); - final CloudStackVersion nextSecurityRelease = CloudStackVersion.parse(currentVersion.getMajorRelease() + "." - + currentVersion.getMinorRelease() + "." - + currentVersion.getPatchRelease() + "." - + (currentVersion.getSecurityRelease() + 1)); + final CloudStackVersion nextSecurityRelease = getNextSecurityRelease(currentVersion); assertNotNull(nextSecurityRelease); // fake security release final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java index 7b7d80a0f16c..2d14443e2d58 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java @@ -87,8 +87,12 @@ public static Version fromPackageAndCSVersion(boolean complete) { } version.setMajor(String.valueOf(csVersion.getMajorRelease())); version.setMinor(String.valueOf(csVersion.getMinorRelease())); - version.setBuild(String.valueOf(csVersion.getPatchRelease())); - version.setRevision(String.valueOf(csVersion.getSecurityRelease())); + if (csVersion.usesNewVersioning()) { + version.setBuild(String.valueOf(csVersion.getSecurityRelease())); + } else { + version.setBuild(String.valueOf(csVersion.getPatchRelease())); + version.setRevision(String.valueOf(csVersion.getSecurityRelease())); + } return version; } } diff --git a/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java b/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java index e29bd9c4e17b..8eb4c6ab9289 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java @@ -39,22 +39,23 @@ */ public final class CloudStackVersion implements Comparable { - private final static Pattern NUMBER_VERSION_FORMAT = Pattern.compile("(\\d+\\.){2}(\\d+\\.)?\\d+"); - private final static Pattern FULL_VERSION_FORMAT = Pattern.compile("(\\d+\\.){2}(\\d+\\.)?\\d+(-[a-zA-Z]+)?(-\\d+)?(-SNAPSHOT)?"); + private final static Pattern NUMBER_VERSION_FORMAT = Pattern.compile("\\d+\\.\\d+\\.\\d+(?:\\.\\d+)?"); + private final static Pattern FULL_VERSION_FORMAT = Pattern.compile("\\d+\\.\\d+\\.\\d+(?:\\.\\d+)?(?:-[a-zA-Z]+)?(?:-\\d+)?(?:-SNAPSHOT)?"); + private final static int NEW_VERSIONING_CUTOVER_MAJOR_VERSION = 24; private final int majorRelease; private final int minorRelease; - private final int patchRelease; + private final Integer patchRelease; private final Integer securityRelease; - private CloudStackVersion(final int majorRelease, final int minorRelease, final int patchRelease, final Integer securityRelease) { + private CloudStackVersion(final int majorRelease, final int minorRelease, final Integer patchRelease, final Integer securityRelease) { super(); checkArgument(majorRelease >= 0, CloudStackVersion.class.getName() + "(int, int, int, Integer) requires a majorRelease greater than 0."); checkArgument(minorRelease >= 0, CloudStackVersion.class.getName() + "(int, int, int, Integer) requires a minorRelease greater than 0."); - checkArgument(patchRelease >= 0, CloudStackVersion.class.getName() + "(int, int, int, Integer) requires a patchRelease greater than 0."); - checkArgument((securityRelease != null && securityRelease >= 0) || (securityRelease == null), + checkArgument(patchRelease == null || patchRelease >= 0, CloudStackVersion.class.getName() + "(int, int, int, Integer) requires a patchRelease greater than 0."); + checkArgument(securityRelease == null || securityRelease >= 0, CloudStackVersion.class.getName() + "(int, int, int, Integer) requires a null securityRelease or a non-null value greater than 0."); this.majorRelease = majorRelease; @@ -69,11 +70,13 @@ private CloudStackVersion(final int majorRelease, final int minorRelease, final * Parses a String representation of a version that conforms one of the following * formats into a CloudStackVersion instance: * * + * Legacy patch-based formats remain supported for backward compatibility. + * * If the string contains a suffix that begins with a "-" character, then the "-" and all characters following it * will be dropped. * @@ -91,7 +94,7 @@ public static CloudStackVersion parse(final String value) { checkArgument(StringUtils.isNotBlank(trimmedValue), CloudStackVersion.class.getName() + ".parse(String) requires a non-blank value"); checkArgument(NUMBER_VERSION_FORMAT.matcher(trimmedValue).matches(), CloudStackVersion.class.getName() + ".parse(String) passed " + - value + ", but requires a value in the format of int.int.int(.int)(-)"); + value + ", but requires a value in the format of int.int.int(.int)(-)"); final String[] components = trimmedValue.split("\\."); @@ -100,8 +103,26 @@ public static CloudStackVersion parse(final String value) { final int majorRelease = Integer.valueOf(components[0]); final int minorRelease = Integer.valueOf(components[1]); - final int patchRelease = Integer.valueOf(components[2]); - final Integer securityRelease = components.length == 3 ? null : Integer.valueOf(components[3]); + final int thirdComponent = Integer.valueOf(components[2]); + + final int patchRelease; + final Integer securityRelease; + + if (components.length == 4) { + checkArgument(isLegacyVersioning(majorRelease), CloudStackVersion.class.getName() + ".parse(String) passed " + value + + ", but major versions at or above 24 do not support legacy int.int.int.int format"); + // Deprecated legacy format: major.minor.patch.security + patchRelease = thirdComponent; + securityRelease = Integer.valueOf(components[3]); + } else if (isNewVersioning(majorRelease)) { + // New format: major.minor.securityRelease (patch dropped) + patchRelease = 0; + securityRelease = thirdComponent; + } else { + // Deprecated legacy format: major.minor.patch + patchRelease = thirdComponent; + securityRelease = null; + } return new CloudStackVersion(majorRelease, minorRelease, patchRelease, securityRelease); @@ -207,6 +228,14 @@ private static ImmutableList normalizeVersionValues(final ImmutableList } + private static boolean isLegacyVersioning(final int majorRelease) { + return majorRelease < NEW_VERSIONING_CUTOVER_MAJOR_VERSION; + } + + private static boolean isNewVersioning(final int majorRelease) { + return majorRelease >= NEW_VERSIONING_CUTOVER_MAJOR_VERSION; + } + /** * * @return The components of this version as an {@link ImmutableList} in order of major release, minor release, @@ -244,6 +273,10 @@ public Integer getSecurityRelease() { return securityRelease; } + public boolean usesNewVersioning() { + return isNewVersioning(majorRelease); + } + @Override public boolean equals(final Object thatObject) { @@ -270,6 +303,11 @@ public int hashCode() { @Override public String toString() { + // Canonicalize cutover-and-later versions to major.minor.securityRelease. + if (securityRelease != null && patchRelease == 0 && isNewVersioning(majorRelease)) { + return Joiner.on(".").join(ImmutableList.of(majorRelease, minorRelease, securityRelease)); + } + return Joiner.on(".").join(asList()); } diff --git a/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java b/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java index dabaf9bc97d3..4d0b4cb0439b 100644 --- a/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java +++ b/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java @@ -36,7 +36,11 @@ public final class CloudStackVersionTest { "1.2.3, 1.2.3", "1.2.3.4, 1.2.3.4", "1.2.3-12, 1.2.3", - "1.2.3.4-14, 1.2.3.4" + "1.2.3.4-14, 1.2.3.4", + "23.9.5, 23.9.5", + "24.0.0, 24.0.0", + "24.0.1, 24.0.1", + "25.1.1, 25.1.1" }) public void testValidParse(final String inputValue, final String expectedVersion) { final CloudStackVersion version = CloudStackVersion.parse(inputValue); @@ -44,6 +48,28 @@ public void testValidParse(final String inputValue, final String expectedVersion assertEquals(expectedVersion, version.toString()); } + @Test + public void testParseComponentMappingForLegacyAndNewVersioning() { + final CloudStackVersion legacyVersion = CloudStackVersion.parse("23.9.5"); + assertEquals(23, legacyVersion.getMajorRelease()); + assertEquals(9, legacyVersion.getMinorRelease()); + assertEquals(5, legacyVersion.getPatchRelease()); + Assert.assertNull(legacyVersion.getSecurityRelease()); + + final CloudStackVersion newVersion = CloudStackVersion.parse("24.0.1"); + assertEquals(24, newVersion.getMajorRelease()); + assertEquals(0, newVersion.getMinorRelease()); + // Patch is retained as 0 to represent "no patch" in the new major.minor.security scheme. + assertEquals(0, newVersion.getPatchRelease()); + assertEquals(Integer.valueOf(1), newVersion.getSecurityRelease()); + + final CloudStackVersion futureNewVersion = CloudStackVersion.parse("25.1.1"); + assertEquals(25, futureNewVersion.getMajorRelease()); + assertEquals(1, futureNewVersion.getMinorRelease()); + assertEquals(0, futureNewVersion.getPatchRelease()); + assertEquals(Integer.valueOf(1), futureNewVersion.getSecurityRelease()); + } + @Test(expected = IllegalArgumentException.class) @DataProvider({ "1.2", @@ -52,7 +78,10 @@ public void testValidParse(final String inputValue, final String expectedVersion "aaaa", "", " ", - "1.2.3.4.5" + "1.2.3.4.5", + "24.0.0.1", + "25.0.0.1", + "26.2.3.4" }) public void testInvalidParse(final String invalidValue) { CloudStackVersion.parse(invalidValue); @@ -147,7 +176,9 @@ public void testEqualCompareDirect(final String value, final String thatValue) { "1.2.3.4-10, 1.0.0.0-5", "1.2.3-10, 1.0.0-5", "1.2.3.4, 1.0.0.0-5", - "1.2.3.4-10, 1.0.0" + "1.2.3.4-10, 1.0.0", + "24.0.2, 24.0.1", + "24.1.0, 24.0.9" }) public void testGreaterThanAndLessThanCompareTo(final String value, final String thatValue) { @@ -178,7 +209,9 @@ public void testGreaterThanAndLessThanCompareTo(final String value, final String "1.2.3.4-10, 1.0.0.0-5", "1.2.3-10, 1.0.0-5", "1.2.3.4, 1.0.0.0-5", - "1.2.3.4-10, 1.0.0" + "1.2.3.4-10, 1.0.0", + "24.0.2, 24.0.1", + "24.1.0, 24.0.9" }) public void testGreaterThanAndLessThanCompareDirect(final String value, final String thatValue) { @@ -213,6 +246,7 @@ private void verifyGetVMwareParentVersion(String hypervisorVersion, String expec Assert.assertEquals(CloudStackVersion.getVMwareParentVersion(hypervisorVersion), expectedParentVersion); } } + @Test public void testGetParentVersion() { verifyGetVMwareParentVersion(null, null); @@ -223,5 +257,6 @@ public void testGetParentVersion() { verifyGetVMwareParentVersion("8.0.0", "8.0"); verifyGetVMwareParentVersion("8.0.0.2", "8.0"); verifyGetVMwareParentVersion("8.0.1.0", "8.0.1"); + verifyGetVMwareParentVersion("24.1.1", "24.1"); } } From c2ebb3a6ec5be616ff9e2c1bbecd5d99b657ff99 Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Mon, 14 Sep 2026 15:33:15 -0400 Subject: [PATCH 02/20] Support multi-VLAN trunk NICs --- api/src/main/java/com/cloud/host/Host.java | 2 + api/src/main/java/com/cloud/vm/Nic.java | 2 + .../apache/cloudstack/api/ApiConstants.java | 2 + .../cloudstack/api/response/HostResponse.java | 24 +++ .../cloud/agent/manager/AgentManagerImpl.java | 12 +- .../src/main/java/com/cloud/vm/NicVO.java | 14 +- .../com/cloud/vm/dao/NicNetworkMapDao.java | 32 ++++ .../cloud/vm/dao/NicNetworkMapDaoImpl.java | 77 ++++++++++ .../com/cloud/vm/dao/NicNetworkMapVO.java | 140 ++++++++++++++++++ ...spring-engine-schema-core-daos-context.xml | 1 + .../META-INF/db/schema-42300to2400.sql | 22 +++ .../resource/LibvirtComputingResource.java | 41 +++++ .../wrapper/LibvirtReadyCommandWrapper.java | 3 + .../cloud/api/query/dao/HostJoinDaoImpl.java | 2 + 14 files changed, 372 insertions(+), 2 deletions(-) create mode 100644 engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDao.java create mode 100644 engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDaoImpl.java create mode 100644 engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapVO.java diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index c110e4ca94e1..3cf36db8ad53 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -64,6 +64,8 @@ public static String[] toStrings(Host.Type... types) { String HOST_VIRTV2V_VERSION = "host.virtv2v.version"; String HOST_SSH_PORT = "host.ssh.port"; String HOST_CDROM_MAX_COUNT = "host.cdrom.max.count"; + String HOST_VLAN_FILTERING_ENABLED = "vlan.filtering.enabled"; + String HOST_VLAN_TRUNK_XML_SUPPORTED = "vlan.trunk.xml.supported"; String GUEST_OS_CATEGORY_ID = "guest.os.category.id"; String GUEST_OS_RULE = "guest.os.rule"; diff --git a/api/src/main/java/com/cloud/vm/Nic.java b/api/src/main/java/com/cloud/vm/Nic.java index 3722e5769c92..71612cd355f1 100644 --- a/api/src/main/java/com/cloud/vm/Nic.java +++ b/api/src/main/java/com/cloud/vm/Nic.java @@ -146,6 +146,8 @@ public enum ReservationStrategy { boolean getSecondaryIp(); + boolean getMultiNetwork(); + // // IPv4 // diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index f74c46161180..de6aeed50a01 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -318,6 +318,8 @@ public class ApiConstants { public static final String MOVE_ACL_CONSISTENCY_HASH = "aclconsistencyhash"; public static final String IMAGE_PATH = "imagepath"; public static final String INSTANCE_CONVERSION_SUPPORTED = "instanceconversionsupported"; + public static final String VLAN_FILTERING_ENABLED = "vlanfilteringenabled"; + public static final String VLAN_TRUNK_XML_SUPPORTED = "vlantrunkxmlsupported"; public static final String INTERNAL_DNS1 = "internaldns1"; public static final String INTERNAL_DNS2 = "internaldns2"; public static final String INTERNET_PROTOCOL = "internetprotocol"; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java index 10bd62804fb2..52804cc980cd 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java @@ -315,6 +315,14 @@ public class HostResponse extends BaseResponseWithAnnotations { @Param(description = "true if the host supports instance conversion (using virt-v2v)", since = "4.19.1") private Boolean instanceConversionSupported; + @SerializedName(ApiConstants.VLAN_FILTERING_ENABLED) + @Param(description = "true if vlan_filtering is enabled on this host's guest bridge, so it can accept multi-VLAN trunk nics", since = "24.0.0") + private Boolean vlanFilteringEnabled; + + @SerializedName(ApiConstants.VLAN_TRUNK_XML_SUPPORTED) + @Param(description = "true if this host's libvirt version supports trunk vlan tap membership natively", since = "24.0.0") + private Boolean vlanTrunkXmlSupported; + @SerializedName(ApiConstants.ARCH) @Param(description = "CPU Arch of the host", since = "4.20") private String arch; @@ -904,6 +912,14 @@ public void setInstanceConversionSupported(Boolean instanceConversionSupported) this.instanceConversionSupported = instanceConversionSupported; } + public void setVlanFilteringEnabled(Boolean vlanFilteringEnabled) { + this.vlanFilteringEnabled = vlanFilteringEnabled; + } + + public void setVlanTrunkXmlSupported(Boolean vlanTrunkXmlSupported) { + this.vlanTrunkXmlSupported = vlanTrunkXmlSupported; + } + public Boolean getIsTagARule() { return isTagARule; } @@ -1000,6 +1016,14 @@ public Boolean getInstanceConversionSupported() { return instanceConversionSupported; } + public Boolean getVlanFilteringEnabled() { + return vlanFilteringEnabled; + } + + public Boolean getVlanTrunkXmlSupported() { + return vlanTrunkXmlSupported; + } + public void setExtensionId(String extensionId) { this.extensionId = extensionId; } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 1215829d92f8..eede907b1875 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -808,8 +808,10 @@ protected AgentAttache notifyMonitorsOfConnection(final AgentAttache attache, fi String vddkSupport = detailsMap.get(Host.HOST_VDDK_SUPPORT); String vddkLibDir = detailsMap.get(Host.HOST_VDDK_LIB_DIR); String vddkVersion = detailsMap.get(Host.HOST_VDDK_VERSION); + String vlanFilteringEnabled = detailsMap.get(Host.HOST_VLAN_FILTERING_ENABLED); + String vlanTrunkXmlSupported = detailsMap.get(Host.HOST_VLAN_TRUNK_XML_SUPPORTED); logger.debug("Got HOST_UEFI_ENABLE [{}] for host [{}]:", uefiEnabled, host); - if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion, vddkSupport, vddkLibDir, vddkVersion)) { + if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion, vddkSupport, vddkLibDir, vddkVersion, vlanFilteringEnabled, vlanTrunkXmlSupported)) { _hostDao.loadDetails(host); boolean updateNeeded = false; if (StringUtils.isNotBlank(uefiEnabled) && !uefiEnabled.equals(host.getDetails().get(Host.HOST_UEFI_ENABLE))) { @@ -828,6 +830,14 @@ protected AgentAttache notifyMonitorsOfConnection(final AgentAttache attache, fi host.getDetails().put(Host.HOST_VDDK_SUPPORT, vddkSupport); updateNeeded = true; } + if (StringUtils.isNotBlank(vlanFilteringEnabled) && !vlanFilteringEnabled.equals(host.getDetails().get(Host.HOST_VLAN_FILTERING_ENABLED))) { + host.getDetails().put(Host.HOST_VLAN_FILTERING_ENABLED, vlanFilteringEnabled); + updateNeeded = true; + } + if (StringUtils.isNotBlank(vlanTrunkXmlSupported) && !vlanTrunkXmlSupported.equals(host.getDetails().get(Host.HOST_VLAN_TRUNK_XML_SUPPORTED))) { + host.getDetails().put(Host.HOST_VLAN_TRUNK_XML_SUPPORTED, vlanTrunkXmlSupported); + updateNeeded = true; + } if (!StringUtils.defaultString(vddkLibDir).equals(StringUtils.defaultString(host.getDetails().get(Host.HOST_VDDK_LIB_DIR)))) { if (StringUtils.isBlank(vddkLibDir)) { host.getDetails().remove(Host.HOST_VDDK_LIB_DIR); diff --git a/engine/schema/src/main/java/com/cloud/vm/NicVO.java b/engine/schema/src/main/java/com/cloud/vm/NicVO.java index 65946b8d8210..bcda4f787755 100644 --- a/engine/schema/src/main/java/com/cloud/vm/NicVO.java +++ b/engine/schema/src/main/java/com/cloud/vm/NicVO.java @@ -128,6 +128,9 @@ protected NicVO() { @Column(name = "secondary_ip") boolean secondaryIp; + @Column(name = "multi_network") + boolean multiNetwork; + @Column(name = "mtu") Integer mtu; @@ -337,7 +340,7 @@ public String toString() { return String.format("Nic %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields( this, "id", "uuid", "instanceId", - "deviceId", "broadcastUri", "reservationId", "iPv4Address")); + "deviceId", "broadcastUri", "reservationId", "iPv4Address", "multiNetwork")); } @Override @@ -381,6 +384,15 @@ public void setSecondaryIp(boolean secondaryIp) { this.secondaryIp = secondaryIp; } + @Override + public boolean getMultiNetwork() { + return multiNetwork; + } + + public void setMultiNetwork(boolean multiNetwork) { + this.multiNetwork = multiNetwork; + } + public void setVmType(VirtualMachine.Type vmType) { this.vmType = vmType; } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDao.java new file mode 100644 index 000000000000..b5a1ed3137af --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDao.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm.dao; + +import java.util.List; + +import com.cloud.utils.db.GenericDao; + +public interface NicNetworkMapDao extends GenericDao { + + List listByNicId(long nicId); + + List listByNetworkId(long networkId); + + NicNetworkMapVO findByNicIdAndNetworkId(long nicId, long networkId); + + List listNicIdsByNetworkId(long networkId); +} diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDaoImpl.java new file mode 100644 index 000000000000..f905809350e5 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDaoImpl.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm.dao; + +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Func; +import com.cloud.utils.db.SearchCriteria.Op; + +@Component +public class NicNetworkMapDaoImpl extends GenericDaoBase implements NicNetworkMapDao { + + private final SearchBuilder AllFieldsSearch; + private final GenericSearchBuilder NicIdsByNetworkSearch; + + public NicNetworkMapDaoImpl() { + super(); + AllFieldsSearch = createSearchBuilder(); + AllFieldsSearch.and("nicId", AllFieldsSearch.entity().getNicId(), Op.EQ); + AllFieldsSearch.and("networkId", AllFieldsSearch.entity().getNetworkId(), Op.EQ); + AllFieldsSearch.done(); + + NicIdsByNetworkSearch = createSearchBuilder(Long.class); + NicIdsByNetworkSearch.select(null, Func.DISTINCT, NicIdsByNetworkSearch.entity().getNicId()); + NicIdsByNetworkSearch.and("networkId", NicIdsByNetworkSearch.entity().getNetworkId(), Op.EQ); + NicIdsByNetworkSearch.done(); + } + + @Override + public List listByNicId(long nicId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("nicId", nicId); + return listBy(sc); + } + + @Override + public List listByNetworkId(long networkId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("networkId", networkId); + return listBy(sc); + } + + @Override + public NicNetworkMapVO findByNicIdAndNetworkId(long nicId, long networkId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("nicId", nicId); + sc.setParameters("networkId", networkId); + return findOneBy(sc); + } + + @Override + public List listNicIdsByNetworkId(long networkId) { + SearchCriteria sc = NicIdsByNetworkSearch.create(); + sc.setParameters("networkId", networkId); + return customSearch(sc, null); + } +} diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapVO.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapVO.java new file mode 100644 index 000000000000..793445354606 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapVO.java @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm.dao; + +import java.util.Date; +import java.util.UUID; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name = "nic_network_map") +public class NicNetworkMapVO implements Identity, InternalIdentity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid = UUID.randomUUID().toString(); + + @Column(name = "nic_id") + private long nicId; + + @Column(name = "network_id") + private long networkId; + + @Column(name = "ip4_address") + private String ip4Address; + + @Column(name = "ip6_address") + private String ip6Address; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + protected NicNetworkMapVO() { + } + + public NicNetworkMapVO(long nicId, long networkId) { + this.nicId = nicId; + this.networkId = networkId; + } + + public NicNetworkMapVO(long nicId, long networkId, String ip4Address, String ip6Address) { + this.nicId = nicId; + this.networkId = networkId; + this.ip4Address = ip4Address; + this.ip6Address = ip6Address; + } + + @Override + public long getId() { + return id; + } + + @Override + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public long getNicId() { + return nicId; + } + + public void setNicId(long nicId) { + this.nicId = nicId; + } + + public long getNetworkId() { + return networkId; + } + + public void setNetworkId(long networkId) { + this.networkId = networkId; + } + + public String getIp4Address() { + return ip4Address; + } + + public void setIp4Address(String ip4Address) { + this.ip4Address = ip4Address; + } + + public String getIp6Address() { + return ip6Address; + } + + public void setIp6Address(String ip6Address) { + this.ip6Address = ip6Address; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + @Override + public String toString() { + return String.format("NicNetworkMap %s", + ReflectionToStringBuilderUtils.reflectOnlySelectedFields( + this, "id", "uuid", "nicId", "networkId", "ip4Address", "ip6Address")); + } +} diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 932db538f30b..e89d87242b22 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -135,6 +135,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql index 7c11013a17d2..aaedd564b774 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql @@ -18,3 +18,25 @@ --; -- Schema upgrade from 4.23.0.0 to 24.0.0 --; + +-- Multi-VLAN trunk nics: a nic may be associated with additional networks beyond its primary nics.network_id. +CREATE TABLE IF NOT EXISTS `cloud`.`nic_network_map` ( + `id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT, + `uuid` varchar(40), + `nic_id` bigint unsigned NOT NULL COMMENT 'nic this association belongs to', + `network_id` bigint unsigned NOT NULL COMMENT 'additional network this nic is associated with', + `ip4_address` char(40) COMMENT 'ip4 address assigned to this nic from this network', + `ip6_address` char(40) COMMENT 'ip6 address assigned to this nic from this network', + `created` datetime NOT NULL COMMENT 'date created', + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY (`id`), + CONSTRAINT `fk_nic_network_map__nic_id` FOREIGN KEY (`nic_id`) REFERENCES `nics`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_nic_network_map__network_id` FOREIGN KEY (`network_id`) REFERENCES `networks`(`id`), + CONSTRAINT `uc_nic_network_map__uuid` UNIQUE (`uuid`), + UNIQUE KEY `uk_nic_network_map__nic_id_network_id` (`nic_id`, `network_id`), + INDEX `i_nic_network_map__nic_id` (`nic_id`), + INDEX `i_nic_network_map__network_id` (`network_id`), + INDEX `i_nic_network_map__removed` (`removed`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.nics', 'multi_network', 'tinyint(1) NOT NULL DEFAULT 0 COMMENT "true if this nic has additional network associations in nic_network_map"'); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 8e8ae3143127..71c862ccfc21 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -23,6 +23,8 @@ import static com.cloud.host.Host.HOST_VDDK_SUPPORT; import static com.cloud.host.Host.HOST_VDDK_VERSION; import static com.cloud.host.Host.HOST_VIRTV2V_VERSION; +import static com.cloud.host.Host.HOST_VLAN_FILTERING_ENABLED; +import static com.cloud.host.Host.HOST_VLAN_TRUNK_XML_SUPPORTED; import static com.cloud.host.Host.HOST_VOLUME_ENCRYPTION; import static org.apache.cloudstack.utils.linux.KVMHostInfo.isHostS390x; @@ -352,6 +354,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv * It is supported since Libvirt 0.9.0 */ private static final int MIN_LIBVIRT_VERSION_FOR_GUEST_CPU_TUNE = 9000; + /** + * Libvirt supports multi-tag trunk mode (<vlan trunk='yes'>) on a standard Linux bridge since 11.0.0. + */ + private static final long MIN_LIBVIRT_VERSION_FOR_VLAN_TRUNK = 11000000; /** * Constant that defines ARM64 (aarch64) guest architectures. */ @@ -4415,6 +4421,8 @@ public StartupCommand[] initialize() { cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(instanceConversionSupported)); cmd.getHostDetails().put(HOST_VDDK_SUPPORT, String.valueOf(hostSupportsVddk())); cmd.getHostDetails().put(HOST_CDROM_MAX_COUNT, String.valueOf(LibvirtVMDef.MAX_CDROMS_PER_VM)); + cmd.getHostDetails().put(HOST_VLAN_FILTERING_ENABLED, String.valueOf(hostSupportsVlanFiltering())); + cmd.getHostDetails().put(HOST_VLAN_TRUNK_XML_SUPPORTED, String.valueOf(hostSupportsVlanTrunkXml())); if (StringUtils.isNotBlank(vddkLibDir)) { cmd.getHostDetails().put(HOST_VDDK_LIB_DIR, vddkLibDir); } @@ -6236,6 +6244,39 @@ public boolean hostSupportsInstanceConversion() { return exitValue == 0; } + /** + * Static capability: whether this host's libvirt understands multi-tag trunk vlan XML + * (<vlan trunk='yes'>) on a standard Linux bridge interface. + */ + public boolean hostSupportsVlanTrunkXml() { + return hypervisorLibvirtVersion >= MIN_LIBVIRT_VERSION_FOR_VLAN_TRUNK; + } + + /** + * Live state: whether vlan_filtering is currently enabled on this host's guest bridge. + * A vlan_filtering=0 bridge floods every tagged frame to every port regardless of tag, + * so this must be true before any multi-VLAN trunk nic can be placed on this host. + */ + public boolean hostSupportsVlanFiltering() { + return isBridgeVlanFilteringEnabled(guestBridgeName); + } + + protected boolean isBridgeVlanFilteringEnabled(String bridgeName) { + if (StringUtils.isBlank(bridgeName)) { + return false; + } + File vlanFilteringFile = new File("/sys/class/net/" + bridgeName + "/bridge/vlan_filtering"); + if (!vlanFilteringFile.exists()) { + return false; + } + try { + return "1".equals(FileUtils.readFileToString(vlanFilteringFile).trim()); + } catch (IOException e) { + LOGGER.warn("Failed to read vlan_filtering state for bridge " + bridgeName, e); + return false; + } + } + public boolean hostSupportsVddk() { return hostSupportsVddk(null); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java index 5a7d6d2c203a..8af4e5f95d00 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java @@ -59,6 +59,9 @@ public Answer execute(final ReadyCommand command, final LibvirtComputingResource hostDetails.put(Host.HOST_OVFTOOL_VERSION, libvirtComputingResource.getHostOvfToolVersion()); } + hostDetails.put(Host.HOST_VLAN_FILTERING_ENABLED, Boolean.toString(libvirtComputingResource.hostSupportsVlanFiltering())); + hostDetails.put(Host.HOST_VLAN_TRUNK_XML_SUPPORTED, Boolean.toString(libvirtComputingResource.hostSupportsVlanTrunkXml())); + return new ReadyAnswer(command, hostDetails); } diff --git a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java index 2e4025ca0013..f942a38746c6 100644 --- a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java @@ -283,6 +283,8 @@ private void setNewHostResponseBase(HostJoinVO host, EnumSet detail } else { hostResponse.setUefiCapability(new Boolean(false)); } + hostResponse.setVlanFilteringEnabled(Boolean.parseBoolean((String) hostDetails.get(Host.HOST_VLAN_FILTERING_ENABLED))); + hostResponse.setVlanTrunkXmlSupported(Boolean.parseBoolean((String) hostDetails.get(Host.HOST_VLAN_TRUNK_XML_SUPPORTED))); } if (details.contains(HostDetails.all) && Arrays.asList(Hypervisor.HypervisorType.KVM, From eb97acf2fbd775f5d4269220ff4710d8a06c0be5 Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Wed, 16 Sep 2026 10:33:07 -0400 Subject: [PATCH 03/20] deliver multi-VLAN trunk nics via libvirt vlan XML and bridge fallback --- .../java/com/cloud/agent/api/to/NicTO.java | 20 ++ .../kvm/resource/BridgeVifDriver.java | 108 +++++++++- .../hypervisor/kvm/resource/LibvirtVMDef.java | 24 ++- .../hypervisor/kvm/resource/VifDriver.java | 4 + .../wrapper/LibvirtPlugNicCommandWrapper.java | 10 + .../wrapper/LibvirtStartCommandWrapper.java | 17 ++ .../kvm/resource/BridgeVifDriverTest.java | 203 ++++++++++++++++++ .../kvm/resource/LibvirtVMDefTest.java | 51 +++++ .../cloud/hypervisor/HypervisorGuruBase.java | 33 +++ 9 files changed, 468 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/api/to/NicTO.java b/api/src/main/java/com/cloud/agent/api/to/NicTO.java index 2ed7d9f9a201..663a039ab100 100644 --- a/api/src/main/java/com/cloud/agent/api/to/NicTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/NicTO.java @@ -37,6 +37,9 @@ public class NicTO extends NetworkTO { String networkSegmentName; + boolean trunkVlan; + List associatedNetworks; + public NicTO() { super(); } @@ -163,4 +166,21 @@ public boolean isEnabled() { public void setEnabled(boolean enabled) { this.enabled = enabled; } + + // trunkVlan/associatedNetworks carry the additional networks for a multi-VLAN trunk nic; unset for ordinary nics + public boolean isTrunkVlan() { + return trunkVlan; + } + + public void setTrunkVlan(boolean trunkVlan) { + this.trunkVlan = trunkVlan; + } + + public List getAssociatedNetworks() { + return associatedNetworks; + } + + public void setAssociatedNetworks(List associatedNetworks) { + this.associatedNetworks = associatedNetworks; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index eefe491e8b27..612d0cdb8a16 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -20,9 +20,13 @@ package com.cloud.hypervisor.kvm.resource; import java.io.File; +import java.net.URI; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -33,6 +37,7 @@ import org.apache.commons.lang3.StringUtils; import org.libvirt.LibvirtException; +import com.cloud.agent.api.to.NetworkTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; @@ -43,6 +48,8 @@ public class BridgeVifDriver extends VifDriverBase { + private static final String GUEST_UPLINK_TRUNK_VLAN_RANGE = "2-4094"; + private int _timeout; private final Object _vnetBridgeMonitor = new Object(); @@ -51,6 +58,7 @@ public class BridgeVifDriver extends VifDriverBase { private String _macIpScriptPath; private String _controlCidr = NetUtils.getLinkLocalCIDR(); private Long libvirtVersion; + private final Set uplinkVlanTrunkEnsuredBridges = ConcurrentHashMap.newKeySet(); private static boolean isVxlanOrNetris(String protocol) { return protocol.equals(Networks.BroadcastDomainType.Vxlan.scheme()) || protocol.equals(Networks.BroadcastDomainType.Netris.scheme()); @@ -199,6 +207,102 @@ protected boolean isValidProtocolAndVnetId(final String vNetId, final String pro return vNetId != null && protocol != null && !vNetId.equalsIgnoreCase("untagged"); } + protected void plugTrunkVlanNic(LibvirtVMDef.InterfaceDef intf, NicTO nic, String trafficLabel, String guestOsType, String nicAdapter, + Integer networkRateKBps) throws InternalErrorException { + if (nic.getBroadcastType() != Networks.BroadcastDomainType.Vlan) { + throw new InternalErrorException("Multi-VLAN trunk nics are only supported on VLAN-isolated guest networks"); + } + if (!_libvirtComputingResource.hostSupportsVlanFiltering()) { + throw new InternalErrorException("vlan_filtering is not enabled on this host's guest bridge; " + + "this host cannot accept a multi-VLAN trunk nic"); + } + + String brName = trafficLabel != null && !trafficLabel.isEmpty() ? trafficLabel : _bridges.get("guest"); + + ensureUplinkAllowsAllVlans(brName); + + List vlanTags = collectTrunkVlanTags(nic); + + logger.debug("plugging trunk nic " + nic.getMac() + " onto guest bridge " + brName + " with vlan tags " + vlanTags); + intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps); + + if (_libvirtComputingResource.hostSupportsVlanTrunkXml()) { + intf.setTrunkVlanTags(vlanTags); + } + // else: older libvirt can't express trunk membership; ensureVlanTrunkMembership() applies it manually once the tap exists + } + + private List collectTrunkVlanTags(NicTO nic) throws InternalErrorException { + Set vlanTags = new LinkedHashSet<>(); + vlanTags.add(parseVlanTag(nic.getBroadcastUri(), "primary network of nic " + nic.getMac())); + if (nic.getAssociatedNetworks() != null) { + for (NetworkTO associatedNetwork : nic.getAssociatedNetworks()) { + if (associatedNetwork.getBroadcastType() != Networks.BroadcastDomainType.Vlan) { + throw new InternalErrorException("Multi-VLAN trunk nics only support VLAN-isolated associated networks"); + } + vlanTags.add(parseVlanTag(associatedNetwork.getBroadcastUri(), "associated network " + associatedNetwork.getUuid())); + } + } + return new ArrayList<>(vlanTags); + } + + private Integer parseVlanTag(URI broadcastUri, String description) throws InternalErrorException { + String vlanValue = broadcastUri == null ? null : Networks.BroadcastDomainType.getValue(broadcastUri); + if (StringUtils.isBlank(vlanValue)) { + throw new InternalErrorException("Cannot determine VLAN for " + description + + ": no VLAN has been assigned yet (is the network implemented?). Refusing to plug this multi-VLAN trunk nic."); + } + try { + return Integer.valueOf(vlanValue); + } catch (NumberFormatException e) { + throw new InternalErrorException("Invalid VLAN value '" + vlanValue + "' for " + description); + } + } + + private void ensureUplinkAllowsAllVlans(String brName) throws InternalErrorException { + if (uplinkVlanTrunkEnsuredBridges.contains(brName)) { + return; + } + synchronized (_vnetBridgeMonitor) { + if (uplinkVlanTrunkEnsuredBridges.contains(brName)) { + return; + } + String uplinkPif = _pifs.get(brName); + if (StringUtils.isBlank(uplinkPif)) { + throw new InternalErrorException("Cannot determine the uplink interface for guest bridge " + brName + + "; refusing to plug a multi-VLAN trunk nic"); + } + runBridgeVlanCommand("add", uplinkPif, GUEST_UPLINK_TRUNK_VLAN_RANGE); + uplinkVlanTrunkEnsuredBridges.add(brName); + } + } + + @Override + public void ensureVlanTrunkMembership(LibvirtVMDef.InterfaceDef iface, NicTO nic) throws InternalErrorException { + if (!nic.isTrunkVlan() || _libvirtComputingResource.hostSupportsVlanTrunkXml()) { + return; + } + String tapName = iface.getDevName(); + if (StringUtils.isBlank(tapName)) { + throw new InternalErrorException("Cannot apply manual VLAN trunk membership: tap device name unknown for nic " + nic.getMac()); + } + for (Integer vlanTag : collectTrunkVlanTags(nic)) { + runBridgeVlanCommand("add", tapName, String.valueOf(vlanTag)); + } + } + + protected void runBridgeVlanCommand(String operation, String dev, String vid) throws InternalErrorException { + final Script command = new Script("bridge", _timeout, logger); + command.add("vlan"); + command.add(operation); + command.add("dev", dev); + command.add("vid", vid); + final String result = command.execute(); + if (result != null) { + throw new InternalErrorException("Failed to " + operation + " VLAN " + vid + " membership on " + dev + ": " + result); + } + } + protected String createStorageVnetBridgeIfNeeded(NicTO nic, String trafficLabel, String storageBrName) throws InternalErrorException { if (nic.getBroadcastUri() == null) { @@ -248,7 +352,9 @@ public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicA } if (nic.getType() == Networks.TrafficType.Guest) { - if (isBroadcastTypeVlanOrVxlan(nic) && isValidProtocolAndVnetId(vNetId, protocol)) { + if (nic.isTrunkVlan()) { + plugTrunkVlanNic(intf, nic, trafficLabel, guestOsType, nicAdapter, networkRateKBps); + } else if (isBroadcastTypeVlanOrVxlan(nic) && isValidProtocolAndVnetId(vNetId, protocol)) { if (trafficLabel != null && !trafficLabel.isEmpty()) { logger.debug("creating a vNet dev and bridge for guest traffic per traffic label " + trafficLabel); String brName = createVnetBr(vNetId, trafficLabel, protocol); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 74529d9d5fa2..112c4e06fea3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -30,6 +30,7 @@ import com.cloud.cpu.CPU; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; import org.apache.cloudstack.utils.qemu.QemuObject; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; @@ -1602,6 +1603,8 @@ enum HostNicType { private String _virtualPortType; private String _virtualPortInterfaceId; private int _vlanTag = -1; + private boolean _vlanTrunk = false; + private List _vlanTrunkTags; private boolean _pxeDisable = false; private boolean _linkStateUp = true; private Integer _slot; @@ -1762,6 +1765,19 @@ public int getVlanTag() { return _vlanTag; } + public void setTrunkVlanTags(List vlanTags) { + _vlanTrunk = true; + _vlanTrunkTags = vlanTags; + } + + public List getTrunkVlanTags() { + return _vlanTrunkTags; + } + + public boolean isVlanTrunk() { + return _vlanTrunk; + } + public void setSlot(Integer slot) { _slot = slot; } @@ -1865,7 +1881,13 @@ public String getContent() { } netBuilder.append("\n"); } - if (_vlanTag > 0 && _vlanTag < 4095) { + if (_vlanTrunk && CollectionUtils.isNotEmpty(_vlanTrunkTags)) { + netBuilder.append("\n"); + for (Integer tag : _vlanTrunkTags) { + netBuilder.append("\n"); + } + netBuilder.append(""); + } else if (_vlanTag > 0 && _vlanTag < 4095) { netBuilder.append("\n\n"); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriver.java index 72fb82967814..08ab8b6de5ba 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriver.java @@ -46,4 +46,8 @@ public interface VifDriver { void deleteBr(NicTO nic); + // applies manual VLAN trunk membership to a trunk nic's live tap on hosts whose libvirt can't do it via XML; no-op otherwise + default void ensureVlanTrunkMembership(LibvirtVMDef.InterfaceDef iface, NicTO nic) throws InternalErrorException { + } + } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java index b0950376a93d..99c7293c9806 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java @@ -29,6 +29,7 @@ import com.cloud.hypervisor.kvm.resource.VifDriver; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import org.libvirt.Connect; import org.libvirt.Domain; @@ -67,6 +68,15 @@ public Answer execute(final PlugNicCommand command, final LibvirtComputingResour } vm.attachDevice(interfaceDef.toString()); + if (nic.isTrunkVlan()) { + try { + final InterfaceDef liveInterfaceDef = libvirtComputingResource.getInterface(conn, vmName, nic.getMac()); + vifDriver.ensureVlanTrunkMembership(liveInterfaceDef, nic); + } catch (CloudRuntimeException e) { + throw new InternalErrorException("Failed to locate live tap for trunk nic " + nic.getMac() + ": " + e.getMessage()); + } + } + // apply default network rules on new nic if (vmType == VirtualMachine.Type.User && nic.isSecurityGroupEnabled()) { final Long vmId = Long.valueOf(vmName.split("-")[2]); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java index 486989661909..2199981d4a4c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java @@ -49,6 +49,7 @@ import com.cloud.network.Networks.TrafficType; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.UserVmManager; import com.cloud.vm.VirtualMachine; @@ -96,6 +97,7 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource String vmFinalSpecification = performXmlTransformHook(vmInitialSpecification, libvirtComputingResource); libvirtComputingResource.startVM(conn, vmName, vmFinalSpecification); performAgentStartHook(vmName, libvirtComputingResource); + applyManualVlanTrunkMembership(conn, vmName, nics, libvirtComputingResource); libvirtComputingResource.applyDefaultNetworkRules(conn, vmSpec, false); @@ -179,6 +181,21 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource } } + private void applyManualVlanTrunkMembership(Connect conn, String vmName, NicTO[] nics, LibvirtComputingResource libvirtComputingResource) + throws InternalErrorException { + for (NicTO nic : nics) { + if (!nic.isTrunkVlan()) { + continue; + } + try { + LibvirtVMDef.InterfaceDef liveInterface = libvirtComputingResource.getInterface(conn, vmName, nic.getMac()); + libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).ensureVlanTrunkMembership(liveInterface, nic); + } catch (CloudRuntimeException e) { + throw new InternalErrorException("Failed to locate live tap for trunk nic " + nic.getMac() + ": " + e.getMessage()); + } + } + } + private void mountSecondaryStoragesIfNeeded(StartCommand command, LibvirtComputingResource libvirtComputingResource, List secondaryStorages) { if (CollectionUtils.isNotEmpty(command.getSecondaryStorages())) { for (String secondaryStorageUrl : command.getSecondaryStorages()) { diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriverTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriverTest.java index 00364948f828..f9a786cf1ac8 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriverTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriverTest.java @@ -18,15 +18,22 @@ import java.net.URI; import java.net.URISyntaxException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; +import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.agent.api.to.NetworkTO; import com.cloud.agent.api.to.NicTO; import com.cloud.exception.InternalErrorException; import com.cloud.network.Networks; @@ -36,10 +43,27 @@ public class BridgeVifDriverTest { private static final String BRIDGE_NAME = "cloudbr1"; + @Mock + private LibvirtComputingResource libvirtComputingResource; + @Spy @InjectMocks private BridgeVifDriver driver = new BridgeVifDriver(); + @Before + public void setup() throws InternalErrorException { + driver._libvirtComputingResource = libvirtComputingResource; + Map bridges = new HashMap<>(); + bridges.put("guest", BRIDGE_NAME); + driver._bridges = bridges; + Map pifs = new HashMap<>(); + pifs.put("private", "eth1"); + pifs.put(BRIDGE_NAME, "eth1"); + pifs.put("customLabel", "eth2"); + driver._pifs = pifs; + Mockito.lenient().doNothing().when(driver).runBridgeVlanCommand(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + } + @Test public void isBroadcastTypeVlanOrVxlan() { final NicTO nic = new NicTO(); @@ -100,4 +124,183 @@ public void createStorageVnetBridgeIfNeededCreatesVnetBridgeWhenUntaggedVlan() t String result = driver.createStorageVnetBridgeIfNeeded(nic, "trafficLabel", BRIDGE_NAME); Assert.assertEquals(BRIDGE_NAME, result); } + + private NicTO buildTrunkNic(int primaryVlan, List associatedVlans) { + NicTO nic = new NicTO(); + nic.setBroadcastType(Networks.BroadcastDomainType.Vlan); + nic.setBroadcastUri(Networks.BroadcastDomainType.Vlan.toUri(primaryVlan)); + nic.setMac("00:11:22:aa:bb:dd"); + nic.setTrunkVlan(true); + if (associatedVlans != null) { + List associated = new java.util.ArrayList<>(); + for (Integer vlan : associatedVlans) { + NetworkTO associatedTo = new NetworkTO(); + associatedTo.setBroadcastType(Networks.BroadcastDomainType.Vlan); + associatedTo.setBroadcastUri(Networks.BroadcastDomainType.Vlan.toUri(vlan)); + associated.add(associatedTo); + } + nic.setAssociatedNetworks(associated); + } + return nic; + } + + @Test + public void plugTrunkVlanNicOnOldLibvirtSkipsXmlAndLeavesManualMembershipToPostAttachHook() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(false); + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + NicTO nic = buildTrunkNic(100, Collections.singletonList(200)); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + + driver.plugTrunkVlanNic(intf, nic, null, null, null, 0); + + Assert.assertFalse(intf.isVlanTrunk()); + Assert.assertEquals(BRIDGE_NAME, intf.getBrName()); + } + + @Test(expected = InternalErrorException.class) + public void plugTrunkVlanNicFailsWhenVlanFilteringNotEnabled() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(false); + NicTO nic = buildTrunkNic(100, Collections.singletonList(200)); + driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), nic, null, null, null, 0); + } + + @Test + public void ensureVlanTrunkMembershipNoOpsWhenHostSupportsTrunkXml() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(true); + NicTO nic = buildTrunkNic(100, Collections.singletonList(200)); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + intf.setDevName("vnet5"); + + driver.ensureVlanTrunkMembership(intf, nic); + + Mockito.verify(driver, Mockito.never()).runBridgeVlanCommand(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + } + + @Test + public void ensureVlanTrunkMembershipNoOpsForNonTrunkNic() throws InternalErrorException { + NicTO nic = new NicTO(); + nic.setTrunkVlan(false); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + intf.setDevName("vnet5"); + + driver.ensureVlanTrunkMembership(intf, nic); + + Mockito.verify(driver, Mockito.never()).runBridgeVlanCommand(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + } + + @Test + public void ensureVlanTrunkMembershipAppliesManualBridgeVlanAddOnOldLibvirt() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(false); + NicTO nic = buildTrunkNic(100, Collections.singletonList(200)); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + intf.setDevName("vnet5"); + + driver.ensureVlanTrunkMembership(intf, nic); + + Mockito.verify(driver).runBridgeVlanCommand("add", "vnet5", "100"); + Mockito.verify(driver).runBridgeVlanCommand("add", "vnet5", "200"); + } + + @Test(expected = InternalErrorException.class) + public void ensureVlanTrunkMembershipFailsWhenTapNameUnknown() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(false); + NicTO nic = buildTrunkNic(100, null); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + + driver.ensureVlanTrunkMembership(intf, nic); + } + + @Test(expected = InternalErrorException.class) + public void plugTrunkVlanNicFailsForNonVlanPrimaryBroadcastType() throws InternalErrorException { + NicTO nic = new NicTO(); + nic.setBroadcastType(Networks.BroadcastDomainType.Vxlan); + nic.setTrunkVlan(true); + driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), nic, null, null, null, 0); + } + + @Test(expected = InternalErrorException.class) + public void plugTrunkVlanNicFailsForNonVlanAssociatedNetwork() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + NicTO nic = buildTrunkNic(100, null); + NetworkTO badAssociation = new NetworkTO(); + badAssociation.setBroadcastType(Networks.BroadcastDomainType.Vxlan); + nic.setAssociatedNetworks(Collections.singletonList(badAssociation)); + driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), nic, null, null, null, 0); + } + + @Test + public void plugTrunkVlanNicBuildsInterfaceWithAllVlanTagsOnGuestBridge() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(true); + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + NicTO nic = buildTrunkNic(100, java.util.Arrays.asList(200, 300)); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + + driver.plugTrunkVlanNic(intf, nic, null, null, null, 0); + + Assert.assertTrue(intf.isVlanTrunk()); + Assert.assertEquals(java.util.Arrays.asList(100, 200, 300), intf.getTrunkVlanTags()); + Assert.assertEquals(BRIDGE_NAME, intf.getBrName()); + } + + @Test + public void plugTrunkVlanNicFailsClearlyWhenAssociatedNetworkHasNoVlanYet() { + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + NicTO nic = buildTrunkNic(100, null); + NetworkTO notYetImplemented = new NetworkTO(); + notYetImplemented.setBroadcastType(Networks.BroadcastDomainType.Vlan); + notYetImplemented.setUuid("unimplemented-network-uuid"); + // broadcastUri intentionally left null, matching a network that has never been implemented + nic.setAssociatedNetworks(Collections.singletonList(notYetImplemented)); + + try { + driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), nic, null, null, null, 0); + Assert.fail("expected InternalErrorException"); + } catch (InternalErrorException e) { + Assert.assertTrue(e.getMessage().contains("unimplemented-network-uuid")); + } + } + + @Test + public void plugTrunkVlanNicUsesTrafficLabelBridgeWhenPresent() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(true); + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + NicTO nic = buildTrunkNic(100, null); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + + driver.plugTrunkVlanNic(intf, nic, "customLabel", null, null, 0); + + Assert.assertEquals("customLabel", intf.getBrName()); + Assert.assertEquals(Collections.singletonList(100), intf.getTrunkVlanTags()); + } + + @Test + public void plugTrunkVlanNicProgramsUplinkForWhicheverBridgeTheNicActuallyUses() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(true); + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + + driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), buildTrunkNic(100, null), null, null, null, 0); + driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), buildTrunkNic(300, null), "customLabel", null, null, 0); + + Mockito.verify(driver).runBridgeVlanCommand("add", "eth1", "2-4094"); + Mockito.verify(driver).runBridgeVlanCommand("add", "eth2", "2-4094"); + } + + @Test + public void plugTrunkVlanNicOnlyProgramsUplinkOnceForRepeatedNicsOnTheSameBridge() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(true); + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + + driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), buildTrunkNic(100, null), null, null, null, 0); + driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), buildTrunkNic(200, null), null, null, null, 0); + + Mockito.verify(driver, Mockito.times(1)).runBridgeVlanCommand("add", "eth1", "2-4094"); + } + + @Test(expected = InternalErrorException.class) + public void plugTrunkVlanNicFailsWhenUplinkPifUnknownForBridge() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + NicTO nic = buildTrunkNic(100, null); + + driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), nic, "unknownLabel", null, null, 0); + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java index 56ad267eac7e..d4078d958132 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java @@ -140,6 +140,57 @@ public void testInterfaceBridgeSlot() { assertEquals(expected, ifDef.toString()); } + @Test + public void testInterfaceLegacySingleVlanTag() { + LibvirtVMDef.InterfaceDef ifDef = new LibvirtVMDef.InterfaceDef(); + ifDef.defBridgeNet("targetDeviceName", null, "00:11:22:aa:bb:dd", LibvirtVMDef.InterfaceDef.NicModel.VIRTIO); + ifDef.setVlanTag(123); + + String expected = + "\n" + + "\n" + + "\n" + + "\n" + + "\n\n" + + "\n" + + "\n"; + + assertEquals(expected, ifDef.toString()); + assertFalse(ifDef.isVlanTrunk()); + } + + @Test + public void testInterfaceTrunkVlanTags() { + LibvirtVMDef.InterfaceDef ifDef = new LibvirtVMDef.InterfaceDef(); + ifDef.defBridgeNet("cloudbr1", null, "00:11:22:aa:bb:dd", LibvirtVMDef.InterfaceDef.NicModel.VIRTIO); + ifDef.setTrunkVlanTags(Arrays.asList(100, 200, 300)); + + String expected = + "\n" + + "\n" + + "\n" + + "\n" + + "\n\n\n\n" + + "\n" + + "\n"; + + assertEquals(expected, ifDef.toString()); + assertTrue(ifDef.isVlanTrunk()); + assertEquals(Arrays.asList(100, 200, 300), ifDef.getTrunkVlanTags()); + } + + @Test + public void testInterfaceTrunkVlanTagsTakesPrecedenceOverLegacyVlanTag() { + LibvirtVMDef.InterfaceDef ifDef = new LibvirtVMDef.InterfaceDef(); + ifDef.defBridgeNet("cloudbr1", null, "00:11:22:aa:bb:dd", LibvirtVMDef.InterfaceDef.NicModel.VIRTIO); + ifDef.setVlanTag(50); + ifDef.setTrunkVlanTags(Arrays.asList(50, 60)); + + String content = ifDef.toString(); + assertTrue(content.contains("")); + assertFalse(content.contains("trunk='no'")); + } + @Test public void testInterfaceWithMultiQueueAndPacked() { LibvirtVMDef.InterfaceDef ifDef = new LibvirtVMDef.InterfaceDef(); diff --git a/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java b/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java index 943d1e73c4b5..adc3b0e4ad36 100644 --- a/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java +++ b/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java @@ -67,6 +67,7 @@ import com.cloud.agent.api.Command; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DiskTO; +import com.cloud.agent.api.to.NetworkTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.configuration.ConfigurationManager; @@ -97,6 +98,8 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import com.cloud.vm.dao.NicSecondaryIpDao; import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.dao.VMInstanceDao; @@ -127,6 +130,8 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis @Inject private NicSecondaryIpDao _nicSecIpDao; @Inject + private NicNetworkMapDao nicNetworkMapDao; + @Inject private ResourceManager _resourceMgr; @Inject protected ServiceOfferingDetailsDao _serviceOfferingDetailsDao; @@ -241,6 +246,7 @@ public NicTO toNicTO(NicProfile profile) { secIps = _nicSecIpDao.getSecondaryIpAddressesForNic(nicVO.getId()); } to.setNicSecIps(secIps); + setTrunkAssociationsOnNicTO(to, nicVO); } else { logger.warn("Unable to load NicVO for NicProfile {}", profile); //Workaround for dynamically created nics @@ -255,6 +261,33 @@ public NicTO toNicTO(NicProfile profile) { return to; } + private void setTrunkAssociationsOnNicTO(final NicTO to, final NicVO nic) { + if (!nic.getMultiNetwork()) { + return; + } + final List associations = nicNetworkMapDao.listByNicId(nic.getId()); + if (associations.isEmpty()) { + return; + } + final List associatedNetworks = new ArrayList<>(); + for (final NicNetworkMapVO association : associations) { + final NetworkVO associatedNetwork = networkDao.findById(association.getNetworkId()); + if (associatedNetwork == null) { + continue; + } + final NetworkTO associatedTo = new NetworkTO(); + associatedTo.setUuid(associatedNetwork.getUuid()); + associatedTo.setBroadcastType(associatedNetwork.getBroadcastDomainType()); + associatedTo.setBroadcastUri(associatedNetwork.getBroadcastUri()); + associatedTo.setType(associatedNetwork.getTrafficType()); + associatedNetworks.add(associatedTo); + } + if (!associatedNetworks.isEmpty()) { + to.setTrunkVlan(true); + to.setAssociatedNetworks(associatedNetworks); + } + } + private String getNetworkName(long zoneId, long domainId, long accountId, VpcVO vpc, long networkId) { String prefix = String.format("D%s-A%s-Z%s", domainId, accountId, zoneId); if (Objects.isNull(vpc)) { From d1438623d3b1c398b063d4af42f1dd3941999aaa Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Mon, 28 Sep 2026 14:34:03 -0400 Subject: [PATCH 04/20] Fix nic_network_map unique constraint blocking re-association The (nic_id, network_id) unique key rejected re-associating a network after a prior disassociate, since the old row's uniqueness survives as a soft-deleted (removed) row. Switch to a plain index so re-association after disassociation works, relying on application-level checks (not the DB) to reject a true duplicate live association. --- .../src/main/resources/META-INF/db/schema-42300to2400.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql index aaedd564b774..875605b23972 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS `cloud`.`nic_network_map` ( CONSTRAINT `fk_nic_network_map__nic_id` FOREIGN KEY (`nic_id`) REFERENCES `nics`(`id`) ON DELETE CASCADE, CONSTRAINT `fk_nic_network_map__network_id` FOREIGN KEY (`network_id`) REFERENCES `networks`(`id`), CONSTRAINT `uc_nic_network_map__uuid` UNIQUE (`uuid`), - UNIQUE KEY `uk_nic_network_map__nic_id_network_id` (`nic_id`, `network_id`), + INDEX `i_nic_network_map__nic_id_network_id` (`nic_id`, `network_id`), INDEX `i_nic_network_map__nic_id` (`nic_id`), INDEX `i_nic_network_map__network_id` (`network_id`), INDEX `i_nic_network_map__removed` (`removed`) From a7b6a8ec5e73f090a6cfc8bb7d2060605fcbb5d6 Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Mon, 28 Sep 2026 14:34:35 -0400 Subject: [PATCH 05/20] Deliver multi-VLAN trunk NICs on KVM via libvirt vlan XML and bridge fallback Trunk NIC VLAN membership is delivered by reusing the existing guest bridge with vlan_filtering=1: the primary VLAN gets native/untagged membership, associated VLANs get tagged-only membership, via libvirt's element where supported, or manual `bridge vlan add` on older libvirt. Adds a live membership update path (UpdateNicVlanMembershipCommand/Answer + its KVM wrapper) so an association change on a running VM is pushed to the host without a restart, and a non-blocking diagnostic warning when the physical uplink is missing a VLAN a trunk NIC needs (CloudStack no longer manages the uplink's own VLAN membership - that is the operator's responsibility). --- .../api/UpdateNicVlanMembershipAnswer.java | 30 ++++ .../api/UpdateNicVlanMembershipCommand.java | 54 ++++++ .../kvm/resource/BridgeVifDriver.java | 160 +++++++++++++++--- .../hypervisor/kvm/resource/LibvirtVMDef.java | 14 +- .../hypervisor/kvm/resource/VifDriver.java | 6 + ...UpdateNicVlanMembershipCommandWrapper.java | 71 ++++++++ .../kvm/resource/BridgeVifDriverTest.java | 142 ++++++++++++++-- .../kvm/resource/LibvirtVMDefTest.java | 17 +- 8 files changed, 451 insertions(+), 43 deletions(-) create mode 100644 core/src/main/java/com/cloud/agent/api/UpdateNicVlanMembershipAnswer.java create mode 100644 core/src/main/java/com/cloud/agent/api/UpdateNicVlanMembershipCommand.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateNicVlanMembershipCommandWrapper.java diff --git a/core/src/main/java/com/cloud/agent/api/UpdateNicVlanMembershipAnswer.java b/core/src/main/java/com/cloud/agent/api/UpdateNicVlanMembershipAnswer.java new file mode 100644 index 000000000000..467298725a92 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/UpdateNicVlanMembershipAnswer.java @@ -0,0 +1,30 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +public class UpdateNicVlanMembershipAnswer extends Answer { + + protected UpdateNicVlanMembershipAnswer() { + } + + public UpdateNicVlanMembershipAnswer(UpdateNicVlanMembershipCommand command, boolean success, String details) { + super(command, success, details); + } +} diff --git a/core/src/main/java/com/cloud/agent/api/UpdateNicVlanMembershipCommand.java b/core/src/main/java/com/cloud/agent/api/UpdateNicVlanMembershipCommand.java new file mode 100644 index 000000000000..95c699802b75 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/UpdateNicVlanMembershipCommand.java @@ -0,0 +1,54 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import com.cloud.agent.api.to.NicTO; + +/** + * Applies the full desired VLAN membership (primary + associated networks) to an already-plugged + * multi-VLAN trunk nic on a running Instance. Carries desired state, not a delta, so it is + * idempotent and safe to retry. + */ +public class UpdateNicVlanMembershipCommand extends Command { + + private NicTO nic; + private String instanceName; + + protected UpdateNicVlanMembershipCommand() { + } + + public UpdateNicVlanMembershipCommand(NicTO nic, String instanceName) { + this.nic = nic; + this.instanceName = instanceName; + } + + public NicTO getNic() { + return nic; + } + + public String getVmName() { + return instanceName; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index 612d0cdb8a16..7f235bf6a525 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -22,11 +22,11 @@ import java.io.File; import java.net.URI; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -34,7 +34,11 @@ import com.cloud.utils.net.NetUtils; import com.cloud.utils.script.OutputInterpreter; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.apache.commons.lang3.StringUtils; +import org.libvirt.Domain; import org.libvirt.LibvirtException; import com.cloud.agent.api.to.NetworkTO; @@ -48,8 +52,6 @@ public class BridgeVifDriver extends VifDriverBase { - private static final String GUEST_UPLINK_TRUNK_VLAN_RANGE = "2-4094"; - private int _timeout; private final Object _vnetBridgeMonitor = new Object(); @@ -58,7 +60,6 @@ public class BridgeVifDriver extends VifDriverBase { private String _macIpScriptPath; private String _controlCidr = NetUtils.getLinkLocalCIDR(); private Long libvirtVersion; - private final Set uplinkVlanTrunkEnsuredBridges = ConcurrentHashMap.newKeySet(); private static boolean isVxlanOrNetris(String protocol) { return protocol.equals(Networks.BroadcastDomainType.Vxlan.scheme()) || protocol.equals(Networks.BroadcastDomainType.Netris.scheme()); @@ -207,6 +208,10 @@ protected boolean isValidProtocolAndVnetId(final String vNetId, final String pro return vNetId != null && protocol != null && !vNetId.equalsIgnoreCase("untagged"); } + protected boolean usesSharedVlanAwareBridge(final NicTO nic) { + return nic.getBroadcastType() == Networks.BroadcastDomainType.Vlan && _libvirtComputingResource.hostSupportsVlanFiltering(); + } + protected void plugTrunkVlanNic(LibvirtVMDef.InterfaceDef intf, NicTO nic, String trafficLabel, String guestOsType, String nicAdapter, Integer networkRateKBps) throws InternalErrorException { if (nic.getBroadcastType() != Networks.BroadcastDomainType.Vlan) { @@ -219,15 +224,16 @@ protected void plugTrunkVlanNic(LibvirtVMDef.InterfaceDef intf, NicTO nic, Strin String brName = trafficLabel != null && !trafficLabel.isEmpty() ? trafficLabel : _bridges.get("guest"); - ensureUplinkAllowsAllVlans(brName); - + Integer primaryVlanTag = parseVlanTag(nic.getBroadcastUri(), "primary network of nic " + nic.getMac()); List vlanTags = collectTrunkVlanTags(nic); + warnIfUplinkMissingVlanMembership(brName, vlanTags); - logger.debug("plugging trunk nic " + nic.getMac() + " onto guest bridge " + brName + " with vlan tags " + vlanTags); + logger.debug("plugging trunk nic " + nic.getMac() + " onto guest bridge " + brName + " with vlan tags " + vlanTags + + ", native vlan " + primaryVlanTag); intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps); if (_libvirtComputingResource.hostSupportsVlanTrunkXml()) { - intf.setTrunkVlanTags(vlanTags); + intf.setTrunkVlanTags(vlanTags, primaryVlanTag); } // else: older libvirt can't express trunk membership; ensureVlanTrunkMembership() applies it manually once the tap exists } @@ -259,50 +265,153 @@ private Integer parseVlanTag(URI broadcastUri, String description) throws Intern } } - private void ensureUplinkAllowsAllVlans(String brName) throws InternalErrorException { - if (uplinkVlanTrunkEnsuredBridges.contains(brName)) { + /** + * The uplink's own tagged VLAN membership is the operator's responsibility, not CloudStack's - the uplink + * is never modified here. This only gives the operator a diagnostic signal: a VLAN missing from the uplink + * still lets same-host traffic on that VLAN work (pure bridge-local forwarding never touches the uplink), + * while cross-host traffic on it silently fails - a confusing signature this warning is meant to shortcut. + * Never blocks the plug and never throws; a failure to even read the uplink's membership is itself just logged. + */ + private void warnIfUplinkMissingVlanMembership(String brName, List vlanTags) { + String uplinkPif = _pifs.get(brName); + if (StringUtils.isBlank(uplinkPif)) { + logger.warn("Cannot determine the uplink interface for guest bridge {} to check VLAN membership for tags {}; " + + "cross-host traffic for this nic will fail unless the uplink has already been configured for these VLANs", brName, vlanTags); return; } - synchronized (_vnetBridgeMonitor) { - if (uplinkVlanTrunkEnsuredBridges.contains(brName)) { - return; + try { + Map currentMembership = readCurrentVlanMembership(uplinkPif); + List missingTags = new ArrayList<>(); + for (Integer vlanTag : vlanTags) { + if (!currentMembership.containsKey(vlanTag)) { + missingTags.add(vlanTag); + } } - String uplinkPif = _pifs.get(brName); - if (StringUtils.isBlank(uplinkPif)) { - throw new InternalErrorException("Cannot determine the uplink interface for guest bridge " + brName - + "; refusing to plug a multi-VLAN trunk nic"); + if (!missingTags.isEmpty()) { + logger.warn("Uplink {} is not a tagged member of VLAN(s) {}; same-host traffic on these VLANs will work, but cross-host " + + "traffic will not, until an operator adds them (e.g. 'bridge vlan add dev {} vid ')", uplinkPif, missingTags, uplinkPif); } - runBridgeVlanCommand("add", uplinkPif, GUEST_UPLINK_TRUNK_VLAN_RANGE); - uplinkVlanTrunkEnsuredBridges.add(brName); + } catch (InternalErrorException e) { + logger.warn("Unable to check VLAN membership on uplink {}: {}", uplinkPif, e.getMessage()); } } @Override public void ensureVlanTrunkMembership(LibvirtVMDef.InterfaceDef iface, NicTO nic) throws InternalErrorException { - if (!nic.isTrunkVlan() || _libvirtComputingResource.hostSupportsVlanTrunkXml()) { + boolean usesSharedBridge = nic.isTrunkVlan() || usesSharedVlanAwareBridge(nic); + if (!usesSharedBridge || _libvirtComputingResource.hostSupportsVlanTrunkXml()) { return; } String tapName = iface.getDevName(); if (StringUtils.isBlank(tapName)) { throw new InternalErrorException("Cannot apply manual VLAN trunk membership: tap device name unknown for nic " + nic.getMac()); } + Integer primaryVlanTag = parseVlanTag(nic.getBroadcastUri(), "primary network of nic " + nic.getMac()); for (Integer vlanTag : collectTrunkVlanTags(nic)) { - runBridgeVlanCommand("add", tapName, String.valueOf(vlanTag)); + runBridgeVlanCommand("add", tapName, String.valueOf(vlanTag), vlanTag.equals(primaryVlanTag)); } } - protected void runBridgeVlanCommand(String operation, String dev, String vid) throws InternalErrorException { + protected void runBridgeVlanCommand(String operation, String dev, String vid, boolean pvidUntagged) throws InternalErrorException { final Script command = new Script("bridge", _timeout, logger); command.add("vlan"); command.add(operation); command.add("dev", dev); command.add("vid", vid); + if (pvidUntagged) { + command.add("pvid"); + command.add("untagged"); + } final String result = command.execute(); if (result != null) { throw new InternalErrorException("Failed to " + operation + " VLAN " + vid + " membership on " + dev + ": " + result); } } + @Override + public void updateVlanTrunkMembership(Domain vm, LibvirtVMDef.InterfaceDef iface, NicTO nic) throws InternalErrorException, LibvirtException { + boolean usesSharedBridge = nic.isTrunkVlan() || usesSharedVlanAwareBridge(nic); + if (!usesSharedBridge) { + throw new InternalErrorException("Nic " + nic.getMac() + " does not use the shared VLAN-aware bridge; cannot update its VLAN membership live"); + } + if (!_libvirtComputingResource.hostSupportsVlanFiltering()) { + throw new InternalErrorException("vlan_filtering is not enabled on this host's guest bridge; cannot update multi-VLAN trunk membership live"); + } + + Integer primaryVlanTag = parseVlanTag(nic.getBroadcastUri(), "primary network of nic " + nic.getMac()); + List vlanTags = collectTrunkVlanTags(nic); + + if (_libvirtComputingResource.hostSupportsVlanTrunkXml()) { + iface.setTrunkVlanTags(vlanTags, primaryVlanTag); + // CloudStack's KVM domains are transient (no persistent libvirt config to update) - matches the + // existing LIVE-only precedent in LibvirtReplugNicCommandWrapper/LibvirtUpdateVmNicCommandWrapper. + // Persistence across a restart is already guaranteed by nic_network_map/nics.network_id, not libvirt. + vm.updateDeviceFlags(iface.toString(), Domain.DeviceModifyFlags.LIVE); + } else { + applyVlanTrunkMembershipDiff(iface.getDevName(), vlanTags, primaryVlanTag); + } + } + + private void applyVlanTrunkMembershipDiff(String tapName, List desiredVlanTags, Integer primaryVlanTag) throws InternalErrorException { + if (StringUtils.isBlank(tapName)) { + throw new InternalErrorException("Cannot update VLAN trunk membership: tap device name unknown"); + } + Map currentMembership = readCurrentVlanMembership(tapName); + Set desiredSet = new LinkedHashSet<>(desiredVlanTags); + + for (Integer currentVlan : currentMembership.keySet()) { + if (!desiredSet.contains(currentVlan)) { + runBridgeVlanCommand("del", tapName, String.valueOf(currentVlan), false); + } + } + for (Integer desiredVlan : desiredSet) { + boolean shouldBePvid = desiredVlan.equals(primaryVlanTag); + Boolean currentlyPvid = currentMembership.get(desiredVlan); + if (currentlyPvid == null || currentlyPvid != shouldBePvid) { + runBridgeVlanCommand("add", tapName, String.valueOf(desiredVlan), shouldBePvid); + } + } + } + + protected Map readCurrentVlanMembership(String tapName) throws InternalErrorException { + final Script command = new Script("bridge", _timeout, logger); + command.add("-j"); + command.add("vlan"); + command.add("show"); + command.add("dev", tapName); + final OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + final String errors = command.execute(parser); + if (errors != null) { + throw new InternalErrorException("Failed to read current VLAN membership for " + tapName + ": " + errors); + } + + final Map membership = new LinkedHashMap<>(); + final String json = parser.getLines(); + if (StringUtils.isBlank(json)) { + return membership; + } + for (JsonElement deviceEl : JsonParser.parseString(json).getAsJsonArray()) { + JsonObject device = deviceEl.getAsJsonObject(); + if (!device.has("vlans")) { + continue; + } + for (JsonElement vlanEl : device.getAsJsonArray("vlans")) { + JsonObject vlanObj = vlanEl.getAsJsonObject(); + int vlan = vlanObj.get("vlan").getAsInt(); + boolean pvid = false; + if (vlanObj.has("flags")) { + for (JsonElement flagEl : vlanObj.getAsJsonArray("flags")) { + if ("PVID".equalsIgnoreCase(flagEl.getAsString())) { + pvid = true; + } + } + } + membership.put(vlan, pvid); + } + } + return membership; + } + protected String createStorageVnetBridgeIfNeeded(NicTO nic, String trafficLabel, String storageBrName) throws InternalErrorException { if (nic.getBroadcastUri() == null) { @@ -355,7 +464,12 @@ public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicA if (nic.isTrunkVlan()) { plugTrunkVlanNic(intf, nic, trafficLabel, guestOsType, nicAdapter, networkRateKBps); } else if (isBroadcastTypeVlanOrVxlan(nic) && isValidProtocolAndVnetId(vNetId, protocol)) { - if (trafficLabel != null && !trafficLabel.isEmpty()) { + if (usesSharedVlanAwareBridge(nic)) { + // host is VLAN-filtering-ready: single-VLAN nics share the same VLAN-aware bridge + // trunk nics use, rather than getting their own dedicated per-VLAN bridge, so the + // two mechanisms never end up on disconnected bridges for the same VLAN on one host + plugTrunkVlanNic(intf, nic, trafficLabel, guestOsType, nicAdapter, networkRateKBps); + } else if (trafficLabel != null && !trafficLabel.isEmpty()) { logger.debug("creating a vNet dev and bridge for guest traffic per traffic label " + trafficLabel); String brName = createVnetBr(vNetId, trafficLabel, protocol); intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 112c4e06fea3..4754b03d27b3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -1605,6 +1605,7 @@ enum HostNicType { private int _vlanTag = -1; private boolean _vlanTrunk = false; private List _vlanTrunkTags; + private Integer _nativeVlanTag; private boolean _pxeDisable = false; private boolean _linkStateUp = true; private Integer _slot; @@ -1765,15 +1766,20 @@ public int getVlanTag() { return _vlanTag; } - public void setTrunkVlanTags(List vlanTags) { + public void setTrunkVlanTags(List vlanTags, Integer nativeVlanTag) { _vlanTrunk = true; _vlanTrunkTags = vlanTags; + _nativeVlanTag = nativeVlanTag; } public List getTrunkVlanTags() { return _vlanTrunkTags; } + public Integer getNativeVlanTag() { + return _nativeVlanTag; + } + public boolean isVlanTrunk() { return _vlanTrunk; } @@ -1884,7 +1890,11 @@ public String getContent() { if (_vlanTrunk && CollectionUtils.isNotEmpty(_vlanTrunkTags)) { netBuilder.append("\n"); for (Integer tag : _vlanTrunkTags) { - netBuilder.append("\n"); + if (tag.equals(_nativeVlanTag)) { + netBuilder.append("\n"); + } else { + netBuilder.append("\n"); + } } netBuilder.append(""); } else if (_vlanTag > 0 && _vlanTag < 4095) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriver.java index 08ab8b6de5ba..a09a30916220 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriver.java @@ -23,6 +23,7 @@ import javax.naming.ConfigurationException; +import org.libvirt.Domain; import org.libvirt.LibvirtException; import com.cloud.agent.api.to.NicTO; @@ -50,4 +51,9 @@ public interface VifDriver { default void ensureVlanTrunkMembership(LibvirtVMDef.InterfaceDef iface, NicTO nic) throws InternalErrorException { } + // applies nic's full desired VLAN membership (primary + associated networks) to an already-plugged trunk nic on a running Instance + default void updateVlanTrunkMembership(Domain vm, LibvirtVMDef.InterfaceDef iface, NicTO nic) throws InternalErrorException, LibvirtException { + throw new InternalErrorException("This vif driver does not support live multi-VLAN trunk membership updates"); + } + } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateNicVlanMembershipCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateNicVlanMembershipCommandWrapper.java new file mode 100644 index 000000000000..83c81deb005e --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateNicVlanMembershipCommandWrapper.java @@ -0,0 +1,71 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.UpdateNicVlanMembershipAnswer; +import com.cloud.agent.api.UpdateNicVlanMembershipCommand; +import com.cloud.agent.api.to.NicTO; +import com.cloud.exception.InternalErrorException; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; +import com.cloud.hypervisor.kvm.resource.VifDriver; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.exception.CloudRuntimeException; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; + +@ResourceWrapper(handles = UpdateNicVlanMembershipCommand.class) +public final class LibvirtUpdateNicVlanMembershipCommandWrapper + extends CommandWrapper { + + @Override + public Answer execute(final UpdateNicVlanMembershipCommand command, final LibvirtComputingResource libvirtComputingResource) { + final NicTO nic = command.getNic(); + final String vmName = command.getVmName(); + Domain vm = null; + try { + final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); + final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName); + vm = libvirtComputingResource.getDomain(conn, vmName); + + final InterfaceDef iface = libvirtComputingResource.getInterface(conn, vmName, nic.getMac()); + + final VifDriver vifDriver = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()); + vifDriver.updateVlanTrunkMembership(vm, iface, nic); + + return new UpdateNicVlanMembershipAnswer(command, true, "success"); + } catch (final LibvirtException | InternalErrorException | CloudRuntimeException e) { + final String msg = "Update NIC VLAN membership failed due to " + e; + logger.warn(msg, e); + return new UpdateNicVlanMembershipAnswer(command, false, msg); + } finally { + if (vm != null) { + try { + vm.free(); + } catch (final LibvirtException l) { + logger.trace("Ignoring libvirt error.", l); + } + } + } + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriverTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriverTest.java index f9a786cf1ac8..e5e092d3aa09 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriverTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriverTest.java @@ -61,7 +61,7 @@ public void setup() throws InternalErrorException { pifs.put(BRIDGE_NAME, "eth1"); pifs.put("customLabel", "eth2"); driver._pifs = pifs; - Mockito.lenient().doNothing().when(driver).runBridgeVlanCommand(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + Mockito.lenient().doNothing().when(driver).runBridgeVlanCommand(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean()); } @Test @@ -173,7 +173,7 @@ public void ensureVlanTrunkMembershipNoOpsWhenHostSupportsTrunkXml() throws Inte driver.ensureVlanTrunkMembership(intf, nic); - Mockito.verify(driver, Mockito.never()).runBridgeVlanCommand(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + Mockito.verify(driver, Mockito.never()).runBridgeVlanCommand(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean()); } @Test @@ -185,7 +185,49 @@ public void ensureVlanTrunkMembershipNoOpsForNonTrunkNic() throws InternalErrorE driver.ensureVlanTrunkMembership(intf, nic); - Mockito.verify(driver, Mockito.never()).runBridgeVlanCommand(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + Mockito.verify(driver, Mockito.never()).runBridgeVlanCommand(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean()); + } + + @Test + public void usesSharedVlanAwareBridgeTrueOnlyForVlanNicOnFilteringReadyHost() { + NicTO nic = new NicTO(); + nic.setBroadcastType(Networks.BroadcastDomainType.Vlan); + + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + Assert.assertTrue(driver.usesSharedVlanAwareBridge(nic)); + + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(false); + Assert.assertFalse(driver.usesSharedVlanAwareBridge(nic)); + + nic.setBroadcastType(Networks.BroadcastDomainType.Vxlan); + Assert.assertFalse(driver.usesSharedVlanAwareBridge(nic)); + } + + @Test + public void ensureVlanTrunkMembershipAppliesNativeVlanForOrdinaryNicOnFilteringReadyHost() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(false); + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + NicTO nic = buildTrunkNic(100, null); + nic.setTrunkVlan(false); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + intf.setDevName("vnet5"); + + driver.ensureVlanTrunkMembership(intf, nic); + + Mockito.verify(driver).runBridgeVlanCommand("add", "vnet5", "100", true); + } + + @Test + public void ensureVlanTrunkMembershipNoOpsForOrdinaryNicWhenHostNotFilteringReady() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(false); + NicTO nic = buildTrunkNic(100, null); + nic.setTrunkVlan(false); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + intf.setDevName("vnet5"); + + driver.ensureVlanTrunkMembership(intf, nic); + + Mockito.verify(driver, Mockito.never()).runBridgeVlanCommand(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean()); } @Test @@ -197,8 +239,8 @@ public void ensureVlanTrunkMembershipAppliesManualBridgeVlanAddOnOldLibvirt() th driver.ensureVlanTrunkMembership(intf, nic); - Mockito.verify(driver).runBridgeVlanCommand("add", "vnet5", "100"); - Mockito.verify(driver).runBridgeVlanCommand("add", "vnet5", "200"); + Mockito.verify(driver).runBridgeVlanCommand("add", "vnet5", "100", true); + Mockito.verify(driver).runBridgeVlanCommand("add", "vnet5", "200", false); } @Test(expected = InternalErrorException.class) @@ -210,6 +252,53 @@ public void ensureVlanTrunkMembershipFailsWhenTapNameUnknown() throws InternalEr driver.ensureVlanTrunkMembership(intf, nic); } + @Test + public void updateVlanTrunkMembershipUpdatesXmlOnModernLibvirt() throws Exception { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(true); + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + NicTO nic = buildTrunkNic(100, Collections.singletonList(200)); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + intf.setDevName("vnet5"); + org.libvirt.Domain vm = Mockito.mock(org.libvirt.Domain.class); + + driver.updateVlanTrunkMembership(vm, intf, nic); + + Mockito.verify(vm).updateDeviceFlags(Mockito.contains("trunk"), + Mockito.eq(org.libvirt.Domain.DeviceModifyFlags.LIVE)); + } + + @Test + public void updateVlanTrunkMembershipAppliesDiffOnOldLibvirt() throws Exception { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(false); + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + NicTO nic = buildTrunkNic(100, Collections.singletonList(200)); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + intf.setDevName("vnet5"); + org.libvirt.Domain vm = Mockito.mock(org.libvirt.Domain.class); + Map currentMembership = new HashMap<>(); + currentMembership.put(100, true); + currentMembership.put(999, false); + Mockito.doReturn(currentMembership).when(driver).readCurrentVlanMembership("vnet5"); + + driver.updateVlanTrunkMembership(vm, intf, nic); + + Mockito.verify(driver).runBridgeVlanCommand("del", "vnet5", "999", false); + Mockito.verify(driver).runBridgeVlanCommand("add", "vnet5", "200", false); + Mockito.verify(driver, Mockito.never()).runBridgeVlanCommand("add", "vnet5", "100", true); + Mockito.verify(vm, Mockito.never()).updateDeviceFlags(Mockito.anyString(), Mockito.anyInt()); + } + + @Test(expected = InternalErrorException.class) + public void updateVlanTrunkMembershipFailsForNonTrunkNic() throws Exception { + NicTO nic = new NicTO(); + nic.setBroadcastType(Networks.BroadcastDomainType.Vlan); + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(false); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + org.libvirt.Domain vm = Mockito.mock(org.libvirt.Domain.class); + + driver.updateVlanTrunkMembership(vm, intf, nic); + } + @Test(expected = InternalErrorException.class) public void plugTrunkVlanNicFailsForNonVlanPrimaryBroadcastType() throws InternalErrorException { NicTO nic = new NicTO(); @@ -239,6 +328,7 @@ public void plugTrunkVlanNicBuildsInterfaceWithAllVlanTagsOnGuestBridge() throws Assert.assertTrue(intf.isVlanTrunk()); Assert.assertEquals(java.util.Arrays.asList(100, 200, 300), intf.getTrunkVlanTags()); + Assert.assertEquals(Integer.valueOf(100), intf.getNativeVlanTag()); Assert.assertEquals(BRIDGE_NAME, intf.getBrName()); } @@ -271,36 +361,58 @@ public void plugTrunkVlanNicUsesTrafficLabelBridgeWhenPresent() throws InternalE Assert.assertEquals("customLabel", intf.getBrName()); Assert.assertEquals(Collections.singletonList(100), intf.getTrunkVlanTags()); + Assert.assertEquals(Integer.valueOf(100), intf.getNativeVlanTag()); } @Test - public void plugTrunkVlanNicProgramsUplinkForWhicheverBridgeTheNicActuallyUses() throws InternalErrorException { + public void plugTrunkVlanNicNeverModifiesTheUplinkItself() throws InternalErrorException { Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(true); Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + Mockito.doReturn(Collections.emptyMap()).when(driver).readCurrentVlanMembership(Mockito.anyString()); driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), buildTrunkNic(100, null), null, null, null, 0); driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), buildTrunkNic(300, null), "customLabel", null, null, 0); - Mockito.verify(driver).runBridgeVlanCommand("add", "eth1", "2-4094"); - Mockito.verify(driver).runBridgeVlanCommand("add", "eth2", "2-4094"); + // the uplink's own VLAN membership is the operator's responsibility, not CloudStack's - plugging a nic + // never adds anything to eth1/eth2, regardless of whether their current membership already covers the VLAN + Mockito.verify(driver, Mockito.never()).runBridgeVlanCommand(Mockito.eq("add"), Mockito.eq("eth1"), Mockito.anyString(), Mockito.anyBoolean()); + Mockito.verify(driver, Mockito.never()).runBridgeVlanCommand(Mockito.eq("add"), Mockito.eq("eth2"), Mockito.anyString(), Mockito.anyBoolean()); } @Test - public void plugTrunkVlanNicOnlyProgramsUplinkOnceForRepeatedNicsOnTheSameBridge() throws InternalErrorException { + public void plugTrunkVlanNicChecksUplinkMembershipForTheResolvedPif() throws InternalErrorException { Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(true); Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + Mockito.doReturn(Collections.emptyMap()).when(driver).readCurrentVlanMembership(Mockito.anyString()); - driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), buildTrunkNic(100, null), null, null, null, 0); - driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), buildTrunkNic(200, null), null, null, null, 0); + driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), buildTrunkNic(100, Collections.singletonList(200)), null, null, null, 0); - Mockito.verify(driver, Mockito.times(1)).runBridgeVlanCommand("add", "eth1", "2-4094"); + Mockito.verify(driver).readCurrentVlanMembership("eth1"); } - @Test(expected = InternalErrorException.class) - public void plugTrunkVlanNicFailsWhenUplinkPifUnknownForBridge() throws InternalErrorException { + @Test + public void plugTrunkVlanNicSucceedsWhenUplinkMembershipCannotBeRead() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(true); + Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); + Mockito.doThrow(new InternalErrorException("bridge command failed")).when(driver).readCurrentVlanMembership(Mockito.anyString()); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + + // a failure to even read the uplink's current membership is only a missed diagnostic, never a reason to refuse the plug + driver.plugTrunkVlanNic(intf, buildTrunkNic(100, null), null, null, null, 0); + + Assert.assertEquals(BRIDGE_NAME, intf.getBrName()); + } + + @Test + public void plugTrunkVlanNicSucceedsWhenUplinkPifUnknownForBridge() throws InternalErrorException { + Mockito.when(libvirtComputingResource.hostSupportsVlanTrunkXml()).thenReturn(true); Mockito.when(libvirtComputingResource.hostSupportsVlanFiltering()).thenReturn(true); NicTO nic = buildTrunkNic(100, null); + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + + // not being able to even determine the uplink is only a missed diagnostic, never a reason to refuse the plug + driver.plugTrunkVlanNic(intf, nic, "unknownLabel", null, null, 0); - driver.plugTrunkVlanNic(new LibvirtVMDef.InterfaceDef(), nic, "unknownLabel", null, null, 0); + Assert.assertEquals("unknownLabel", intf.getBrName()); } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java index d4078d958132..b94ba691155f 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java @@ -163,20 +163,31 @@ public void testInterfaceLegacySingleVlanTag() { public void testInterfaceTrunkVlanTags() { LibvirtVMDef.InterfaceDef ifDef = new LibvirtVMDef.InterfaceDef(); ifDef.defBridgeNet("cloudbr1", null, "00:11:22:aa:bb:dd", LibvirtVMDef.InterfaceDef.NicModel.VIRTIO); - ifDef.setTrunkVlanTags(Arrays.asList(100, 200, 300)); + ifDef.setTrunkVlanTags(Arrays.asList(100, 200, 300), 100); String expected = "\n" + "\n" + "\n" + "\n" - + "\n\n\n\n" + + "\n\n\n\n" + "\n" + "\n"; assertEquals(expected, ifDef.toString()); assertTrue(ifDef.isVlanTrunk()); assertEquals(Arrays.asList(100, 200, 300), ifDef.getTrunkVlanTags()); + assertEquals(Integer.valueOf(100), ifDef.getNativeVlanTag()); + } + + @Test + public void testInterfaceTrunkVlanTagsWithoutNativeVlan() { + LibvirtVMDef.InterfaceDef ifDef = new LibvirtVMDef.InterfaceDef(); + ifDef.defBridgeNet("cloudbr1", null, "00:11:22:aa:bb:dd", LibvirtVMDef.InterfaceDef.NicModel.VIRTIO); + ifDef.setTrunkVlanTags(Arrays.asList(100, 200, 300), null); + + String content = ifDef.toString(); + assertFalse(content.contains("nativeMode")); } @Test @@ -184,7 +195,7 @@ public void testInterfaceTrunkVlanTagsTakesPrecedenceOverLegacyVlanTag() { LibvirtVMDef.InterfaceDef ifDef = new LibvirtVMDef.InterfaceDef(); ifDef.defBridgeNet("cloudbr1", null, "00:11:22:aa:bb:dd", LibvirtVMDef.InterfaceDef.NicModel.VIRTIO); ifDef.setVlanTag(50); - ifDef.setTrunkVlanTags(Arrays.asList(50, 60)); + ifDef.setTrunkVlanTags(Arrays.asList(50, 60), 50); String content = ifDef.toString(); assertTrue(content.contains("")); From bcb89ab26f0b190082990cfa4973720bbf6c04ae Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Mon, 28 Sep 2026 14:41:01 -0400 Subject: [PATCH 06/20] Bill usage for a trunk nic's associated networks - New UsageEventUtils.publishNicNetworkOfferingUsageEvents() emits one usage event per network a nic bills against (primary + any trunk associations), replacing the old single-event publishUsageEvent at every NIC-billing call site - Ordinary non-trunk nics still produce exactly one event, unchanged - New nullable usage_network_offering.network_id column disambiguates rows when a trunk nic's networks share one network offering - Backfills network_id on a nic's own still-open usage row(s) the moment it first converts to a trunk nic; historic rows otherwise left alone --- .../main/java/com/cloud/event/EventTypes.java | 2 + .../java/com/cloud/event/UsageEventUtils.java | 57 +++++ .../com/cloud/event/UsageEventUtilsTest.java | 212 ++++++++++++++++++ .../cloud/vm/VirtualMachineManagerImpl.java | 9 +- .../java/com/cloud/event/UsageEventVO.java | 2 +- .../cloud/usage/UsageNetworkOfferingVO.java | 17 ++ .../usage/dao/UsageNetworkOfferingDao.java | 6 + .../dao/UsageNetworkOfferingDaoImpl.java | 37 ++- .../META-INF/db/schema-42300to2400.sql | 5 + .../java/com/cloud/vm/UserVmManagerImpl.java | 35 +-- .../com/cloud/vm/UserVmStateListener.java | 6 +- .../vm/UnmanagedVMsManagerImpl.java | 5 +- .../com/cloud/usage/UsageManagerImpl.java | 79 ++++++- .../com/cloud/usage/UsageManagerImplTest.java | 150 +++++++++++++ 14 files changed, 574 insertions(+), 48 deletions(-) create mode 100644 engine/components-api/src/test/java/com/cloud/event/UsageEventUtilsTest.java diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index f7d13343d469..f7c985cc2c6c 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -228,6 +228,8 @@ public class EventTypes { public static final String EVENT_NIC_DETAIL_ADD = "NIC.DETAIL.ADD"; public static final String EVENT_NIC_DETAIL_UPDATE = "NIC.DETAIL.UPDATE"; public static final String EVENT_NIC_DETAIL_REMOVE = "NIC.DETAIL.REMOVE"; + // internal-only signal consumed by the usage job - never user-facing, not an audit event + public static final String EVENT_NIC_NETWORK_ID_BACKFILL = "NIC.NETWORK.ID.BACKFILL"; // Load Balancers public static final String EVENT_ASSIGN_TO_LOAD_BALANCER_RULE = "LB.ASSIGN.TO.RULE"; diff --git a/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java b/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java index 1c88c7df124c..753774f9360f 100644 --- a/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java +++ b/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java @@ -18,6 +18,7 @@ package com.cloud.event; import java.text.SimpleDateFormat; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -26,6 +27,11 @@ import javax.inject.Inject; import com.cloud.network.Network; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.vm.Nic; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import org.apache.commons.collections.MapUtils; import org.springframework.beans.factory.NoSuchBeanDefinitionException; @@ -48,6 +54,8 @@ public class UsageEventUtils { private static UsageEventDao s_usageEventDao; private static AccountDao s_accountDao; private static DataCenterDao s_dcDao; + private static NetworkDao s_networkDao; + private static NicNetworkMapDao s_nicNetworkMapDao; protected static Logger LOGGER = LogManager.getLogger(UsageEventUtils.class); protected static EventBus s_eventBus = null; protected static ConfigurationDao s_configDao; @@ -61,6 +69,10 @@ public class UsageEventUtils { DataCenterDao dcDao; @Inject ConfigurationDao configDao; + @Inject + NetworkDao networkDao; + @Inject + NicNetworkMapDao nicNetworkMapDao; public UsageEventUtils() { } @@ -71,6 +83,8 @@ void init() { s_accountDao = accountDao; s_dcDao = dcDao; s_configDao = configDao; + s_networkDao = networkDao; + s_nicNetworkMapDao = nicNetworkMapDao; } public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId, @@ -94,6 +108,49 @@ public static void publishUsageEvent(String usageType, long accountId, long zone } + // named distinctly, rather than as another publishUsageEvent(...) overload, because its parameter shape + // (Map, boolean) is only distinguishable at compile time from the existing (Long, boolean) + // overload above when neither argument is a literal null - several existing call sites pass null literals + // here, which would make an overload ambiguous + public static void publishUsageEventWithDetails(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId, + Long size, String entityType, String entityUUID, Map details, boolean displayResource) { + if (displayResource) { + saveUsageEvent(usageType, accountId, zoneId, resourceId, resourceName, offeringId, templateId, size, details); + } + publishUsageEvent(usageType, accountId, zoneId, entityType, entityUUID); + } + + /** + * Emits one {@code usageType} usage event per network a NIC is actually billed against - its primary + * network, plus any additional networks it is associated with as a multi-VLAN trunk nic. An ordinary, + * non-trunk nic has exactly one such network, so this is identical to today's single-event behaviour for + * every legacy nic. Each event carries the network id as a detail, so the usage-processing side can + * disambiguate rows when a trunk nic's networks share the same network offering. + */ + public static void publishNicNetworkOfferingUsageEvents(String usageType, long accountId, long zoneId, long vmId, String entityType, String entityUUID, + Nic nic, long isDefault, boolean displayResource) { + NetworkVO primaryNetwork = s_networkDao.findById(nic.getNetworkId()); + if (primaryNetwork != null) { + publishNicNetworkOfferingUsageEvent(usageType, accountId, zoneId, vmId, entityType, entityUUID, nic.getId(), primaryNetwork, isDefault, displayResource); + } + if (nic.getMultiNetwork()) { + for (NicNetworkMapVO association : s_nicNetworkMapDao.listByNicId(nic.getId())) { + NetworkVO associatedNetwork = s_networkDao.findById(association.getNetworkId()); + if (associatedNetwork != null) { + publishNicNetworkOfferingUsageEvent(usageType, accountId, zoneId, vmId, entityType, entityUUID, nic.getId(), associatedNetwork, isDefault, + displayResource); + } + } + } + } + + private static void publishNicNetworkOfferingUsageEvent(String usageType, long accountId, long zoneId, long vmId, String entityType, String entityUUID, + long nicId, NetworkVO network, long isDefault, boolean displayResource) { + Map details = Collections.singletonMap(UsageEventVO.DynamicParameters.networkId.name(), String.valueOf(network.getId())); + publishUsageEventWithDetails(usageType, accountId, zoneId, vmId, Long.toString(nicId), network.getNetworkOfferingId(), null, isDefault, entityType, entityUUID, + details, displayResource); + } + public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId, Long size, String entityType, String entityUUID, Long vmId, boolean displayResource) { if (displayResource) { diff --git a/engine/components-api/src/test/java/com/cloud/event/UsageEventUtilsTest.java b/engine/components-api/src/test/java/com/cloud/event/UsageEventUtilsTest.java new file mode 100644 index 000000000000..4b1c7b1bfd20 --- /dev/null +++ b/engine/components-api/src/test/java/com/cloud/event/UsageEventUtilsTest.java @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.event; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.event.dao.UsageEventDao; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.vm.Nic; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; + +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; + +/** + * Verifies {@link UsageEventUtils#publishNicNetworkOfferingUsageEvents} - the mechanism that emits one usage + * event per (nic, network) a nic actually bills against, rather than one per nic. The key non-regression + * assertion is that an ordinary, non-trunk nic still produces exactly the single event it always has. + */ +@RunWith(MockitoJUnitRunner.class) +public class UsageEventUtilsTest { + + private static final String USAGE_TYPE = EventTypes.EVENT_NETWORK_OFFERING_ASSIGN; + private static final long ACCOUNT_ID = 1L; + private static final long ZONE_ID = 2L; + private static final long VM_ID = 3L; + private static final long NIC_ID = 5L; + private static final String ENTITY_TYPE = "VirtualMachine"; + private static final String ENTITY_UUID = "vm-uuid"; + + protected Map staticFieldValues = new HashMap<>(); + + @Mock + protected UsageEventDao usageEventDao; + @Mock + protected AccountDao accountDao; + @Mock + protected DataCenterDao dcDao; + @Mock + protected ConfigurationDao configDao; + @Mock + protected NetworkDao networkDao; + @Mock + protected NicNetworkMapDao nicNetworkMapDao; + + @Before + public void setup() throws Exception { + staticFieldValues = new HashMap<>(); + UsageEventUtils utils = new UsageEventUtils(); + + for (Field field : UsageEventUtils.class.getDeclaredFields()) { + if (field.getAnnotation(Inject.class) != null) { + field.setAccessible(true); + try { + Field mockField = this.getClass().getDeclaredField(field.getName()); + mockField.setAccessible(true); + field.set(utils, mockField.get(this)); + Field staticField = UsageEventUtils.class.getDeclaredField("s_" + field.getName()); + staticField.setAccessible(true); + staticFieldValues.put(field.getName(), staticField.get(null)); + } catch (Exception e) { + // ignore fields with no matching mock/static counterpart + } + } + } + utils.init(); + + // configDao.getValue(...) defaults to null -> event-bus publish short-circuits without needing an + // EventDistributor bean; only the usage_event persistence path (what billing actually reads) matters here + Mockito.when(usageEventDao.persist(Mockito.any(UsageEventVO.class))).thenAnswer(invocation -> invocation.getArgument(0)); + } + + @After + public void teardown() throws Exception { + UsageEventUtils utils = new UsageEventUtils(); + for (String fieldName : staticFieldValues.keySet()) { + Field field = UsageEventUtils.class.getDeclaredField("s_" + fieldName); + field.setAccessible(true); + field.set(utils, staticFieldValues.get(fieldName)); + } + } + + private NetworkVO mockNetwork(long id, long offeringId) { + NetworkVO network = Mockito.mock(NetworkVO.class); + Mockito.when(network.getId()).thenReturn(id); + Mockito.when(network.getNetworkOfferingId()).thenReturn(offeringId); + return network; + } + + @Test + public void ordinaryNicEmitsExactlyOneEventForItsPrimaryNetworkOnly() { + Nic nic = Mockito.mock(Nic.class); + Mockito.when(nic.getId()).thenReturn(NIC_ID); + Mockito.when(nic.getNetworkId()).thenReturn(100L); + Mockito.when(nic.getMultiNetwork()).thenReturn(false); + NetworkVO primaryNetwork = mockNetwork(100L, 7L); + Mockito.when(networkDao.findById(100L)).thenReturn(primaryNetwork); + + UsageEventUtils.publishNicNetworkOfferingUsageEvents(USAGE_TYPE, ACCOUNT_ID, ZONE_ID, VM_ID, ENTITY_TYPE, ENTITY_UUID, nic, 1L, true); + + Mockito.verifyNoInteractions(nicNetworkMapDao); + + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(UsageEventVO.class); + Mockito.verify(usageEventDao, Mockito.times(1)).persist(eventCaptor.capture()); + UsageEventVO persisted = eventCaptor.getValue(); + Assert.assertEquals(Long.toString(NIC_ID), persisted.getResourceName()); + Assert.assertEquals(Long.valueOf(7L), persisted.getOfferingId()); + Assert.assertEquals(Long.valueOf(1L), persisted.getSize()); + + ArgumentCaptor> detailsCaptor = ArgumentCaptor.forClass(Map.class); + Mockito.verify(usageEventDao, Mockito.times(1)).saveDetails(Mockito.anyLong(), detailsCaptor.capture()); + Assert.assertEquals("100", detailsCaptor.getValue().get(UsageEventVO.DynamicParameters.networkId.name())); + } + + @Test + public void trunkNicEmitsOneEventForItsPrimaryNetworkAndOneForEachAssociatedNetwork() { + Nic nic = Mockito.mock(Nic.class); + Mockito.when(nic.getId()).thenReturn(NIC_ID); + Mockito.when(nic.getNetworkId()).thenReturn(100L); + Mockito.when(nic.getMultiNetwork()).thenReturn(true); + NetworkVO primaryNetwork = mockNetwork(100L, 7L); + NetworkVO associatedNetwork1 = mockNetwork(200L, 8L); + NetworkVO associatedNetwork2 = mockNetwork(300L, 9L); + Mockito.when(networkDao.findById(100L)).thenReturn(primaryNetwork); + Mockito.when(networkDao.findById(200L)).thenReturn(associatedNetwork1); + Mockito.when(networkDao.findById(300L)).thenReturn(associatedNetwork2); + List associations = Arrays.asList(new NicNetworkMapVO(NIC_ID, 200L), new NicNetworkMapVO(NIC_ID, 300L)); + Mockito.when(nicNetworkMapDao.listByNicId(NIC_ID)).thenReturn(associations); + + UsageEventUtils.publishNicNetworkOfferingUsageEvents(USAGE_TYPE, ACCOUNT_ID, ZONE_ID, VM_ID, ENTITY_TYPE, ENTITY_UUID, nic, 0L, true); + + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(UsageEventVO.class); + Mockito.verify(usageEventDao, Mockito.times(3)).persist(eventCaptor.capture()); + List offeringIds = eventCaptor.getAllValues().stream().map(UsageEventVO::getOfferingId).collect(java.util.stream.Collectors.toList()); + Assert.assertTrue(offeringIds.containsAll(Arrays.asList(7L, 8L, 9L))); + // every one of the three rows still belongs to the same nic + for (UsageEventVO event : eventCaptor.getAllValues()) { + Assert.assertEquals(Long.toString(NIC_ID), event.getResourceName()); + } + + ArgumentCaptor> detailsCaptor = ArgumentCaptor.forClass(Map.class); + Mockito.verify(usageEventDao, Mockito.times(3)).saveDetails(Mockito.anyLong(), detailsCaptor.capture()); + List networkIds = detailsCaptor.getAllValues().stream() + .map(details -> details.get(UsageEventVO.DynamicParameters.networkId.name())).collect(java.util.stream.Collectors.toList()); + Assert.assertTrue(networkIds.containsAll(Arrays.asList("100", "200", "300"))); + } + + @Test + public void displayResourceFalseNeverPersistsAnyUsageRow() { + Nic nic = Mockito.mock(Nic.class); + Mockito.when(nic.getId()).thenReturn(NIC_ID); + Mockito.when(nic.getNetworkId()).thenReturn(100L); + Mockito.when(nic.getMultiNetwork()).thenReturn(false); + NetworkVO primaryNetwork = mockNetwork(100L, 7L); + Mockito.when(networkDao.findById(100L)).thenReturn(primaryNetwork); + + UsageEventUtils.publishNicNetworkOfferingUsageEvents(USAGE_TYPE, ACCOUNT_ID, ZONE_ID, VM_ID, ENTITY_TYPE, ENTITY_UUID, nic, 1L, false); + + Mockito.verify(usageEventDao, Mockito.never()).persist(Mockito.any()); + Mockito.verify(usageEventDao, Mockito.never()).saveDetails(Mockito.anyLong(), Mockito.anyMap()); + } + + @Test + public void missingAssociatedNetworkIsSkippedRatherThanFailing() { + Nic nic = Mockito.mock(Nic.class); + Mockito.when(nic.getId()).thenReturn(NIC_ID); + Mockito.when(nic.getNetworkId()).thenReturn(100L); + Mockito.when(nic.getMultiNetwork()).thenReturn(true); + NetworkVO primaryNetwork = mockNetwork(100L, 7L); + Mockito.when(networkDao.findById(100L)).thenReturn(primaryNetwork); + Mockito.when(networkDao.findById(200L)).thenReturn(null); + Mockito.when(nicNetworkMapDao.listByNicId(NIC_ID)).thenReturn(Collections.singletonList(new NicNetworkMapVO(NIC_ID, 200L))); + + UsageEventUtils.publishNicNetworkOfferingUsageEvents(USAGE_TYPE, ACCOUNT_ID, ZONE_ID, VM_ID, ENTITY_TYPE, ENTITY_UUID, nic, 1L, true); + + Mockito.verify(usageEventDao, Mockito.times(1)).persist(Mockito.any()); + } +} diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 8c9a1653a260..a1b4e9e732f8 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -4689,8 +4689,9 @@ private NicProfile orchestrateAddVmToNetwork(final VirtualMachine vm, final Netw final long isDefault = nic.isDefaultNic() ? 1 : 0; if(VirtualMachine.Type.User.equals(vmVO.getType())) { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmVO.getAccountId(), vmVO.getDataCenterId(), vmVO.getId(), - Long.toString(nic.getId()), network.getNetworkOfferingId(), null, isDefault, VirtualMachine.class.getName(), vmVO.getUuid(), vm.isDisplay()); + NicVO nicVO = _nicsDao.findById(nic.getId()); + UsageEventUtils.publishNicNetworkOfferingUsageEvents(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmVO.getAccountId(), vmVO.getDataCenterId(), + vmVO.getId(), VirtualMachine.class.getName(), vmVO.getUuid(), nicVO, isDefault, vm.isDisplay()); } return nic; } else { @@ -4776,8 +4777,8 @@ private boolean orchestrateRemoveNicFromVm(final VirtualMachine vm, final Nic ni _userVmMgr.setupVmForPvlan(false, vm.getHostId(), nicProfile); logger.debug("NIC is unplugged successfully for Instance {} in Network {}.", vm, network); final long isDefault = nic.isDefaultNic() ? 1 : 0; - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), - Long.toString(nic.getId()), network.getNetworkOfferingId(), null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay()); + UsageEventUtils.publishNicNetworkOfferingUsageEvents(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), + VirtualMachine.class.getName(), vm.getUuid(), nic, isDefault, vm.isDisplay()); } else { logger.warn("Failed to unplug NIC for the Instance {} from Network {}.", vm, network); return false; diff --git a/engine/schema/src/main/java/com/cloud/event/UsageEventVO.java b/engine/schema/src/main/java/com/cloud/event/UsageEventVO.java index 41ecec0c7fb8..f80f00fdfa90 100644 --- a/engine/schema/src/main/java/com/cloud/event/UsageEventVO.java +++ b/engine/schema/src/main/java/com/cloud/event/UsageEventVO.java @@ -31,7 +31,7 @@ @Table(name = "usage_event") public class UsageEventVO implements UsageEvent { public enum DynamicParameters { - cpuSpeed, cpuNumber, memory, vmSnapshotId + cpuSpeed, cpuNumber, memory, vmSnapshotId, networkId }; @Id diff --git a/engine/schema/src/main/java/com/cloud/usage/UsageNetworkOfferingVO.java b/engine/schema/src/main/java/com/cloud/usage/UsageNetworkOfferingVO.java index f16f1ea5ac1c..473f169d4d65 100644 --- a/engine/schema/src/main/java/com/cloud/usage/UsageNetworkOfferingVO.java +++ b/engine/schema/src/main/java/com/cloud/usage/UsageNetworkOfferingVO.java @@ -67,11 +67,19 @@ public class UsageNetworkOfferingVO implements InternalIdentity { @Column(name = "nic_id") private Long nicId; + @Column(name = "network_id") + private Long networkId; + protected UsageNetworkOfferingVO() { } public UsageNetworkOfferingVO(long zoneId, long accountId, long domainId, long vmInstanceId, long networkOfferingId, long nicId, boolean isDefault, Date created, Date deleted) { + this(zoneId, accountId, domainId, vmInstanceId, networkOfferingId, nicId, null, isDefault, created, deleted); + } + + public UsageNetworkOfferingVO(long zoneId, long accountId, long domainId, long vmInstanceId, long networkOfferingId, long nicId, Long networkId, boolean isDefault, + Date created, Date deleted) { this.zoneId = zoneId; this.accountId = accountId; this.domainId = domainId; @@ -81,6 +89,7 @@ public UsageNetworkOfferingVO(long zoneId, long accountId, long domainId, long v this.created = created; this.deleted = deleted; this.nicId = nicId; + this.networkId = networkId; } public long getZoneId() { @@ -127,6 +136,14 @@ public void setNicId(Long nicId) { this.nicId = nicId; } + public Long getNetworkId() { + return networkId; + } + + public void setNetworkId(Long networkId) { + this.networkId = networkId; + } + @Override public long getId() { return this.id; diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworkOfferingDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworkOfferingDao.java index 9f987b8897ac..a930abe9cb05 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworkOfferingDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworkOfferingDao.java @@ -26,4 +26,10 @@ public interface UsageNetworkOfferingDao extends GenericDao getUsageRecords(Long accountId, Long domainId, Date startDate, Date endDate, boolean limit, int page); + + /** + * Backfills network_id on a nic's own currently-open, not-yet-populated usage row(s), the moment that nic + * is first converted to a multi-VLAN trunk nic. Historic rows are otherwise never backfilled in bulk. + */ + public void backfillNetworkIdForNic(long nicId, long networkId); } diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworkOfferingDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworkOfferingDaoImpl.java index b3bc06e8af40..b44b74e7e1e7 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworkOfferingDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworkOfferingDaoImpl.java @@ -39,14 +39,18 @@ public class UsageNetworkOfferingDaoImpl extends GenericDaoBase= ?)))"; protected static final String GET_USAGE_RECORDS_BY_DOMAIN = - "SELECT zone_id, account_id, domain_id, vm_instance_id, network_offering_id, nic_id, is_default, created, deleted " + "FROM usage_network_offering " + "SELECT zone_id, account_id, domain_id, vm_instance_id, network_offering_id, nic_id, is_default, created, deleted, network_id " + "FROM usage_network_offering " + "WHERE domain_id = ? AND ((deleted IS NULL) OR (created BETWEEN ? AND ?) OR " + " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?)))"; protected static final String GET_ALL_USAGE_RECORDS = - "SELECT zone_id, account_id, domain_id, vm_instance_id, network_offering_id, nic_id, is_default, created, deleted " + "FROM usage_network_offering " + "SELECT zone_id, account_id, domain_id, vm_instance_id, network_offering_id, nic_id, is_default, created, deleted, network_id " + "FROM usage_network_offering " + "WHERE (deleted IS NULL) OR (created BETWEEN ? AND ?) OR " + " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?))"; + // only ever touches rows created before network_id existed (never backfilled in bulk); scoped to a single + // nic so it can safely run the moment that nic is first converted to a multi-VLAN trunk nic + protected static final String BACKFILL_NETWORK_ID_FOR_NIC = + "UPDATE usage_network_offering SET network_id = ? WHERE nic_id = ? AND network_id IS NULL AND deleted IS NULL"; public UsageNetworkOfferingDaoImpl() { } @@ -120,7 +124,7 @@ public List getUsageRecords(Long accountId, Long domainI ResultSet rs = pstmt.executeQuery(); while (rs.next()) { - //zoneId, account_id, domain_id, vm_instance_id, network_offering_id, nic_id, is_default, created, deleted + //zoneId, account_id, domain_id, vm_instance_id, network_offering_id, nic_id, is_default, created, deleted, network_id Long zoneId = Long.valueOf(rs.getLong(1)); Long acctId = Long.valueOf(rs.getLong(2)); Long dId = Long.valueOf(rs.getLong(3)); @@ -132,6 +136,8 @@ public List getUsageRecords(Long accountId, Long domainI Date deletedDate = null; String createdTS = rs.getString(8); String deletedTS = rs.getString(9); + long networkIdValue = rs.getLong(10); + Long networkId = rs.wasNull() ? null : Long.valueOf(networkIdValue); if (createdTS != null) { createdDate = DateUtil.parseDateString(s_gmtTimeZone, createdTS); @@ -140,7 +146,7 @@ public List getUsageRecords(Long accountId, Long domainI deletedDate = DateUtil.parseDateString(s_gmtTimeZone, deletedTS); } - usageRecords.add(new UsageNetworkOfferingVO(zoneId, acctId, dId, vmId, noId, nicId, isDefault, createdDate, deletedDate)); + usageRecords.add(new UsageNetworkOfferingVO(zoneId, acctId, dId, vmId, noId, nicId, networkId, isDefault, createdDate, deletedDate)); } } catch (Exception e) { txn.rollback(); @@ -151,4 +157,25 @@ public List getUsageRecords(Long accountId, Long domainI return usageRecords; } + + @Override + public void backfillNetworkIdForNic(long nicId, long networkId) { + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + try { + txn.start(); + try (PreparedStatement pstmt = txn.prepareStatement(BACKFILL_NETWORK_ID_FOR_NIC)) { + pstmt.setLong(1, networkId); + pstmt.setLong(2, nicId); + pstmt.executeUpdate(); + } catch (SQLException e) { + throw new CloudException(String.format("Error backfilling network_id for UsageNetworkOfferingVO: %s", e.getMessage()), e); + } + txn.commit(); + } catch (Exception e) { + txn.rollback(); + logger.warn("Error backfilling network_id for UsageNetworkOfferingVO: {}", e.getMessage(), e); + } finally { + txn.close(); + } + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql index 875605b23972..e3457e508f8b 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql @@ -40,3 +40,8 @@ CREATE TABLE IF NOT EXISTS `cloud`.`nic_network_map` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8; CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.nics', 'multi_network', 'tinyint(1) NOT NULL DEFAULT 0 COMMENT "true if this nic has additional network associations in nic_network_map"'); + +-- Disambiguates usage_network_offering rows when a multi-VLAN trunk nic has more than one associated network +-- sharing the same network offering. NULL for rows created before this column existed - those are never +-- backfilled in bulk, only targeted, single-nic backfill on first conversion to a trunk nic. +CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_network_offering', 'network_id', 'bigint(20) unsigned DEFAULT NULL COMMENT "associated network this usage row belongs to"'); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 60fdee8feaa7..51c8ffcb4dee 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -1836,14 +1836,6 @@ public UserVm updateDefaultNicForVirtualMachine(UpdateDefaultNicForVMCmd cmd) th throw new CloudRuntimeException("Failed to find a NIC profile for the existing default Network. This is bad and probably means some sort of configuration corruption"); } - Network oldDefaultNetwork = null; - oldDefaultNetwork = _networkModel.getDefaultNetworkForVm(vmId); - String oldNicIdString = Long.toString(_networkModel.getDefaultNic(vmId).getId()); - long oldNetworkOfferingId = -1L; - - if (oldDefaultNetwork != null) { - oldNetworkOfferingId = oldDefaultNetwork.getNetworkOfferingId(); - } NicVO existingVO = _nicDao.findById(existing.id); int chosenID = nic.getDeviceId(); Integer existingID = existing.getDeviceId(); @@ -1869,16 +1861,14 @@ public UserVm updateDefaultNicForVirtualMachine(UpdateDefaultNicForVMCmd cmd) th throw new CloudRuntimeException("Failed to change default nic to " + nic + " and now we have no default"); } else if (newdefault.getId() == nic.getNetworkId()) { logger.debug("successfully set default network to " + network + " for " + vmInstance); - String nicIdString = Long.toString(nic.getId()); - long newNetworkOfferingId = network.getNetworkOfferingId(); - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vmInstance.getAccountId(), vmInstance.getDataCenterId(), vmInstance.getId(), - oldNicIdString, oldNetworkOfferingId, null, 1L, VirtualMachine.class.getName(), vmInstance.getUuid(), vmInstance.isDisplay()); - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmInstance.getAccountId(), vmInstance.getDataCenterId(), vmInstance.getId(), nicIdString, - newNetworkOfferingId, null, 1L, VirtualMachine.class.getName(), vmInstance.getUuid(), vmInstance.isDisplay()); - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vmInstance.getAccountId(), vmInstance.getDataCenterId(), vmInstance.getId(), nicIdString, - newNetworkOfferingId, null, 0L, VirtualMachine.class.getName(), vmInstance.getUuid(), vmInstance.isDisplay()); - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmInstance.getAccountId(), vmInstance.getDataCenterId(), vmInstance.getId(), - oldNicIdString, oldNetworkOfferingId, null, 0L, VirtualMachine.class.getName(), vmInstance.getUuid(), vmInstance.isDisplay()); + UsageEventUtils.publishNicNetworkOfferingUsageEvents(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vmInstance.getAccountId(), vmInstance.getDataCenterId(), + vmInstance.getId(), VirtualMachine.class.getName(), vmInstance.getUuid(), existingVO, 1L, vmInstance.isDisplay()); + UsageEventUtils.publishNicNetworkOfferingUsageEvents(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmInstance.getAccountId(), vmInstance.getDataCenterId(), + vmInstance.getId(), VirtualMachine.class.getName(), vmInstance.getUuid(), nic, 1L, vmInstance.isDisplay()); + UsageEventUtils.publishNicNetworkOfferingUsageEvents(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vmInstance.getAccountId(), vmInstance.getDataCenterId(), + vmInstance.getId(), VirtualMachine.class.getName(), vmInstance.getUuid(), nic, 0L, vmInstance.isDisplay()); + UsageEventUtils.publishNicNetworkOfferingUsageEvents(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmInstance.getAccountId(), vmInstance.getDataCenterId(), + vmInstance.getId(), VirtualMachine.class.getName(), vmInstance.getUuid(), existingVO, 0L, vmInstance.isDisplay()); if (vmInstance.getState() == State.Running) { try { @@ -3290,10 +3280,9 @@ private void saveUsageEvent(UserVmVO vm) { private void generateNetworkUsageForVm(VirtualMachine vm, boolean isDisplay, String eventType) { List nics = _nicDao.listByVmId(vm.getId()); for (NicVO nic : nics) { - NetworkVO network = _networkDao.findById(nic.getNetworkId()); long isDefault = (nic.isDefaultNic()) ? 1 : 0; - UsageEventUtils.publishUsageEvent(eventType, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), - Long.toString(nic.getId()), network.getNetworkOfferingId(), null, isDefault, vm.getClass().getName(), vm.getUuid(), isDisplay); + UsageEventUtils.publishNicNetworkOfferingUsageEvents(eventType, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), + vm.getClass().getName(), vm.getUuid(), nic, isDefault, isDisplay); } } @@ -5809,8 +5798,8 @@ public boolean finalizeStart(VirtualMachineProfile profile, long hostId, Command for (NicVO nic : nics) { NetworkVO network = _networkDao.findById(nic.getNetworkId()); long isDefault = (nic.isDefaultNic()) ? 1 : 0; - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), Long.toString(nic.getId()), - network.getNetworkOfferingId(), null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay()); + UsageEventUtils.publishNicNetworkOfferingUsageEvents(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), + VirtualMachine.class.getName(), vm.getUuid(), nic, isDefault, vm.isDisplay()); if (network.getTrafficType() == TrafficType.Guest) { originalIp = nic.getIPv4Address(); guestNic = nic; diff --git a/server/src/main/java/com/cloud/vm/UserVmStateListener.java b/server/src/main/java/com/cloud/vm/UserVmStateListener.java index aa1805d3366e..971ae69803f0 100644 --- a/server/src/main/java/com/cloud/vm/UserVmStateListener.java +++ b/server/src/main/java/com/cloud/vm/UserVmStateListener.java @@ -36,7 +36,6 @@ import com.cloud.event.UsageEventUtils; import com.cloud.event.dao.UsageEventDao; import com.cloud.network.dao.NetworkDao; -import com.cloud.network.dao.NetworkVO; import com.cloud.server.ManagementService; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.utils.component.ComponentContext; @@ -99,9 +98,8 @@ public boolean postStateTransitionEvent(StateMachine2.Transition t generateUsageEvent(vo.getServiceOfferingId(), vo, EventTypes.EVENT_VM_STOP); List nics = _nicDao.listByVmId(vo.getId()); for (NicVO nic : nics) { - NetworkVO network = _networkDao.findById(nic.getNetworkId()); - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vo.getAccountId(), vo.getDataCenterId(), vo.getId(), - Long.toString(nic.getId()), network.getNetworkOfferingId(), null, 0L, vo.getClass().getName(), vo.getUuid(), vo.isDisplay()); + UsageEventUtils.publishNicNetworkOfferingUsageEvents(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vo.getAccountId(), vo.getDataCenterId(), vo.getId(), + vo.getClass().getName(), vo.getUuid(), nic, 0L, vo.isDisplay()); } } else if (newState == State.Destroyed || newState == State.Error || newState == State.Expunging) { generateUsageEvent(vo.getServiceOfferingId(), vo, EventTypes.EVENT_VM_DESTROY); diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 840a1de45962..39aeb406091a 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -1044,9 +1044,8 @@ private void publishVMUsageUpdateResourceCount(final UserVm userVm, ServiceOffer List nics = nicDao.listByVmId(userVm.getId()); for (NicVO nic : nics) { try { - NetworkVO network = networkDao.findById(nic.getNetworkId()); - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, userVm.getAccountId(), userVm.getDataCenterId(), userVm.getId(), - Long.toString(nic.getId()), network.getNetworkOfferingId(), null, 1L, VirtualMachine.class.getName(), userVm.getUuid(), userVm.isDisplay()); + UsageEventUtils.publishNicNetworkOfferingUsageEvents(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, userVm.getAccountId(), userVm.getDataCenterId(), + userVm.getId(), VirtualMachine.class.getName(), userVm.getUuid(), nic, 1L, userVm.isDisplay()); } catch (Exception e) { logger.error(String.format("Failed to publish network usage records during VM import. %s", StringUtils.defaultString(e.getMessage()))); } diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index eab371ab353f..b63c480cf3a5 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -969,6 +969,8 @@ private void createHelperRecord(UsageEventVO event) { createPortForwardingHelperEvent(event); } else if (isNetworkOfferingEvent(eventType)) { createNetworkOfferingEvent(event); + } else if (isNicTrunkConversionEvent(eventType)) { + backfillNicNetworkIdOnTrunkConversion(event); } else if (isVPNUserEvent(eventType)) { handleVpnUserEvent(event); } else if (isSecurityGroupEvent(eventType)) { @@ -1048,6 +1050,10 @@ private boolean isNetworkOfferingEvent(String eventType) { eventType.equals(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN) || eventType.equals(EventTypes.EVENT_NETWORK_OFFERING_REMOVE)); } + private boolean isNicTrunkConversionEvent(String eventType) { + return EventTypes.EVENT_NIC_NETWORK_ID_BACKFILL.equals(eventType); + } + private boolean isVPNUserEvent(String eventType) { if (eventType == null) return false; @@ -1736,18 +1742,16 @@ private void createPortForwardingHelperEvent(UsageEventVO event) { } } - private void createNetworkOfferingEvent(UsageEventVO event) { + protected void createNetworkOfferingEvent(UsageEventVO event) { long zoneId = -1L; long vmId = event.getResourceId(); long networkOfferingId = event.getOfferingId(); - long nicId = 0; - try { - nicId = Long.parseLong(event.getResourceName()); - } catch (Exception e) { - logger.warn("Failed to get NIC id from resource name, resource name is: " + event.getResourceName()); - } + Long parsedNicId = parseLongOrWarn("NIC id from resource name", event.getResourceName()); + long nicId = parsedNicId != null ? parsedNicId : 0; + + Long networkId = findNetworkIdEventDetail(event); if (EventTypes.EVENT_NETWORK_OFFERING_CREATE.equals(event.getType()) || EventTypes.EVENT_NETWORK_OFFERING_ASSIGN.equals(event.getType())) { if (logger.isDebugEnabled()) { @@ -1757,7 +1761,8 @@ private void createNetworkOfferingEvent(UsageEventVO event) { Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId()); boolean isDefault = (event.getSize() == 1) ? true : false; UsageNetworkOfferingVO networkOffering = - new UsageNetworkOfferingVO(zoneId, event.getAccountId(), acct.getDomainId(), vmId, networkOfferingId, nicId, isDefault, event.getCreateDate(), null); + new UsageNetworkOfferingVO(zoneId, event.getAccountId(), acct.getDomainId(), vmId, networkOfferingId, nicId, networkId, isDefault, event.getCreateDate(), + null); _usageNetworkOfferingDao.persist(networkOffering); } else if (EventTypes.EVENT_NETWORK_OFFERING_DELETE.equals(event.getType()) || EventTypes.EVENT_NETWORK_OFFERING_REMOVE.equals(event.getType())) { SearchCriteria sc = _usageNetworkOfferingDao.createSearchCriteria(); @@ -1767,6 +1772,22 @@ private void createNetworkOfferingEvent(UsageEventVO event) { sc.addAnd("networkOfferingId", SearchCriteria.Op.EQ, networkOfferingId); sc.addAnd("deleted", SearchCriteria.Op.NULL); List noVOs = _usageNetworkOfferingDao.search(sc, null); + // A multi-VLAN trunk nic can have more than one association sharing the same offering; the plain + // (account, vm, nic, offering) key alone can't tell those rows apart. Narrow using the network id + // carried on this event, when there is one to narrow with. An ordinary nic only ever produces one + // candidate row here, and a historic (network_id = NULL) row never collides with anything, so this + // is a no-op for every legacy nic. + if (noVOs.size() > 1 && networkId != null) { + List networkMatches = new ArrayList<>(); + for (UsageNetworkOfferingVO candidate : noVOs) { + if (networkId.equals(candidate.getNetworkId())) { + networkMatches.add(candidate); + } + } + if (networkMatches.size() == 1) { + noVOs = networkMatches; + } + } if (noVOs.size() > 1) { logger.warn("More that one usage entry for networking offering: " + networkOfferingId + " for Vm: " + vmId + " assigned to account: " + event.getAccountId() + "; marking them all as deleted..."); @@ -1781,6 +1802,48 @@ private void createNetworkOfferingEvent(UsageEventVO event) { } } + /** + * Backfills network_id on a nic's own still-open usage row(s) the moment that nic is first converted to a + * multi-VLAN trunk nic - only ever needed for a row that predates the network_id column existing at all. + * Runs here, on the usage job's own thread, rather than synchronously where the conversion happens + * (NetworkServiceImpl.persistAssociations): that request thread already has an ambient `cloud` DB + * transaction open, and this table lives in the separate cloud_usage database. + */ + protected void backfillNicNetworkIdOnTrunkConversion(UsageEventVO event) { + Long nicId = parseLongOrWarn("NIC id from resource name", event.getResourceName()); + if (nicId == null) { + return; + } + Long networkId = findNetworkIdEventDetail(event); + if (networkId == null) { + logger.warn("Trunk-conversion backfill event for nic {} is missing its network id detail, skipping", nicId); + return; + } + _usageNetworkOfferingDao.backfillNetworkIdForNic(nicId, networkId); + } + + /** + * Looks up and parses this event's networkId detail, shared by every usage event that carries one + * (network-offering ASSIGN/REMOVE, and the trunk-conversion backfill signal). Returns null if the detail + * is absent or unparseable - the caller decides what "absent" means for its own event type. + */ + private Long findNetworkIdEventDetail(UsageEventVO event) { + UsageEventDetailsVO networkIdDetail = _usageEventDetailsDao.findDetail(event.getId(), UsageEventVO.DynamicParameters.networkId.name()); + return networkIdDetail == null ? null : parseLongOrWarn("network id usage event detail", networkIdDetail.getValue()); + } + + private Long parseLongOrWarn(String description, String value) { + if (value == null) { + return null; + } + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + logger.warn("Failed to parse {}, value is: {}", description, value); + return null; + } + } + /** * Handles VPN user create and remove events: *
    diff --git a/usage/src/test/java/com/cloud/usage/UsageManagerImplTest.java b/usage/src/test/java/com/cloud/usage/UsageManagerImplTest.java index 03939e430d25..5592881434de 100644 --- a/usage/src/test/java/com/cloud/usage/UsageManagerImplTest.java +++ b/usage/src/test/java/com/cloud/usage/UsageManagerImplTest.java @@ -17,13 +17,20 @@ package com.cloud.usage; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; import java.util.List; +import com.cloud.event.UsageEventDetailsVO; import com.cloud.event.dao.UsageEventDetailsDao; +import com.cloud.usage.dao.UsageNetworkOfferingDao; import com.cloud.usage.dao.UsageVMSnapshotDao; +import com.cloud.utils.db.SearchCriteria; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; @@ -58,6 +65,9 @@ public class UsageManagerImplTest { @Mock private AccountDao accountDaoMock; + @Mock + private UsageNetworkOfferingDao usageNetworkOfferingDaoMock; + @Mock private UsageVPNUserVO vpnUserMock; @@ -243,4 +253,144 @@ public void handleVMSnapshotEventTestEventIsNeitherAddNorRemove() { Mockito.verify(usageManagerImpl, Mockito.never()).createUsageVpnUser(usageEventVOMock,accountMock); Mockito.verify(usageManagerImpl, Mockito.never()).deleteUsageVpnUser(usageEventVOMock, accountMock); } + + private void mockNetworkOfferingEvent(String type, long resourceId, long offeringId, String resourceName, long size, String networkIdDetailValue) { + Mockito.when(usageEventVOMock.getType()).thenReturn(type); + Mockito.when(usageEventVOMock.getResourceId()).thenReturn(resourceId); + Mockito.when(usageEventVOMock.getOfferingId()).thenReturn(offeringId); + Mockito.when(usageEventVOMock.getResourceName()).thenReturn(resourceName); + Mockito.when(usageEventVOMock.getZoneId()).thenReturn(9L); + Mockito.when(usageEventVOMock.getAccountId()).thenReturn(accountMockId); + Mockito.when(usageEventVOMock.getSize()).thenReturn(size); + Mockito.when(usageEventVOMock.getCreateDate()).thenReturn(new Date()); + Mockito.when(usageEventVOMock.getId()).thenReturn(42L); + if (networkIdDetailValue != null) { + Mockito.when(usageEventDetailsDao.findDetail(42L, UsageEventVO.DynamicParameters.networkId.name())) + .thenReturn(new UsageEventDetailsVO(42L, UsageEventVO.DynamicParameters.networkId.name(), networkIdDetailValue)); + } + } + + @Test + public void createNetworkOfferingEventAssignPersistsRowWithNetworkIdFromEventDetail() { + // an ordinary, non-trunk nic's ASSIGN event carries a network id too now (every emit site resolves it), + // so every new row - trunk or not - gets network_id populated going forward + mockNetworkOfferingEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, 3L, 7L, "11", 1L, "205"); + + usageManagerImpl.createNetworkOfferingEvent(usageEventVOMock); + + ArgumentCaptor captor = ArgumentCaptor.forClass(UsageNetworkOfferingVO.class); + Mockito.verify(usageNetworkOfferingDaoMock, Mockito.times(1)).persist(captor.capture()); + UsageNetworkOfferingVO persisted = captor.getValue(); + Assert.assertEquals(Long.valueOf(205L), persisted.getNetworkId()); + Assert.assertEquals(Long.valueOf(7L), persisted.getNetworkOfferingId()); + Assert.assertEquals(11L, persisted.getNicId().longValue()); + } + + @Test + public void createNetworkOfferingEventAssignPersistsRowWithNullNetworkIdWhenDetailMissing() { + // an old agent/emit path with no networkId detail at all must still work exactly as it always has + mockNetworkOfferingEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, 3L, 7L, "11", 1L, null); + + usageManagerImpl.createNetworkOfferingEvent(usageEventVOMock); + + ArgumentCaptor captor = ArgumentCaptor.forClass(UsageNetworkOfferingVO.class); + Mockito.verify(usageNetworkOfferingDaoMock, Mockito.times(1)).persist(captor.capture()); + Assert.assertNull(captor.getValue().getNetworkId()); + } + + @Test + public void createNetworkOfferingEventRemoveUpdatesTheOnlyCandidateRegardlessOfNetworkId() { + // the common, non-ambiguous case (an ordinary nic, or a historic row): exactly one open row matches the + // (account, vm, nic, offering) key, and it must always be the one closed, network_id or not + mockNetworkOfferingEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, 3L, 7L, "11", 0L, "205"); + UsageNetworkOfferingVO onlyCandidate = new UsageNetworkOfferingVO(9L, accountMockId, 2L, 3L, 7L, 11L, 205L, true, new Date(), null); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(usageNetworkOfferingDaoMock.createSearchCriteria()).thenReturn(sc); + Mockito.when(usageNetworkOfferingDaoMock.search(sc, null)).thenReturn(Arrays.asList(onlyCandidate)); + + usageManagerImpl.createNetworkOfferingEvent(usageEventVOMock); + + Mockito.verify(usageNetworkOfferingDaoMock, Mockito.times(1)).update(onlyCandidate); + Assert.assertNotNull(onlyCandidate.getDeleted()); + } + + @Test + public void createNetworkOfferingEventRemoveNarrowsToTheMatchingNetworkWhenMultipleCandidatesCollide() { + // a trunk nic with two associated networks sharing the same offering: the (account, vm, nic, offering) + // key alone matches both rows, so only the one whose network_id matches this event's network may close + mockNetworkOfferingEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, 3L, 7L, "11", 0L, "206"); + UsageNetworkOfferingVO stillActiveAssociation = new UsageNetworkOfferingVO(9L, accountMockId, 2L, 3L, 7L, 11L, 205L, false, new Date(), null); + UsageNetworkOfferingVO removedAssociation = new UsageNetworkOfferingVO(9L, accountMockId, 2L, 3L, 7L, 11L, 206L, false, new Date(), null); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(usageNetworkOfferingDaoMock.createSearchCriteria()).thenReturn(sc); + Mockito.when(usageNetworkOfferingDaoMock.search(sc, null)).thenReturn(Arrays.asList(stillActiveAssociation, removedAssociation)); + + usageManagerImpl.createNetworkOfferingEvent(usageEventVOMock); + + Mockito.verify(usageNetworkOfferingDaoMock, Mockito.times(1)).update(removedAssociation); + Mockito.verify(usageNetworkOfferingDaoMock, Mockito.never()).update(stillActiveAssociation); + Assert.assertNull("the still-active association on the same nic/offering must not be closed", stillActiveAssociation.getDeleted()); + } + + @Test + public void createNetworkOfferingEventRemoveFallsBackToClosingAllCandidatesWhenNetworkIdCannotDisambiguate() { + // defence in depth: if more than one candidate remains even after narrowing by network id (should not + // happen given nic_network_map's uniqueness constraint), preserve today's existing fallback behaviour + // of closing every candidate, rather than silently picking one + mockNetworkOfferingEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, 3L, 7L, "11", 0L, null); + UsageNetworkOfferingVO candidate1 = new UsageNetworkOfferingVO(9L, accountMockId, 2L, 3L, 7L, 11L, 205L, false, new Date(), null); + UsageNetworkOfferingVO candidate2 = new UsageNetworkOfferingVO(9L, accountMockId, 2L, 3L, 7L, 11L, 206L, false, new Date(), null); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(usageNetworkOfferingDaoMock.createSearchCriteria()).thenReturn(sc); + Mockito.when(usageNetworkOfferingDaoMock.search(sc, null)).thenReturn(Arrays.asList(candidate1, candidate2)); + + usageManagerImpl.createNetworkOfferingEvent(usageEventVOMock); + + Mockito.verify(usageNetworkOfferingDaoMock, Mockito.times(1)).update(candidate1); + Mockito.verify(usageNetworkOfferingDaoMock, Mockito.times(1)).update(candidate2); + } + + @Test + public void backfillNicNetworkIdOnTrunkConversionBackfillsWhenDetailPresent() { + Mockito.when(usageEventVOMock.getResourceName()).thenReturn("11"); + Mockito.when(usageEventVOMock.getId()).thenReturn(42L); + Mockito.when(usageEventDetailsDao.findDetail(42L, UsageEventVO.DynamicParameters.networkId.name())) + .thenReturn(new UsageEventDetailsVO(42L, UsageEventVO.DynamicParameters.networkId.name(), "205")); + + usageManagerImpl.backfillNicNetworkIdOnTrunkConversion(usageEventVOMock); + + Mockito.verify(usageNetworkOfferingDaoMock, Mockito.times(1)).backfillNetworkIdForNic(11L, 205L); + } + + @Test + public void backfillNicNetworkIdOnTrunkConversionSkipsWhenDetailMissing() { + Mockito.when(usageEventVOMock.getResourceName()).thenReturn("11"); + Mockito.when(usageEventVOMock.getId()).thenReturn(42L); + Mockito.when(usageEventDetailsDao.findDetail(42L, UsageEventVO.DynamicParameters.networkId.name())).thenReturn(null); + + usageManagerImpl.backfillNicNetworkIdOnTrunkConversion(usageEventVOMock); + + Mockito.verify(usageNetworkOfferingDaoMock, Mockito.never()).backfillNetworkIdForNic(Mockito.anyLong(), Mockito.anyLong()); + } + + @Test + public void backfillNicNetworkIdOnTrunkConversionSkipsWhenResourceNameNotParseable() { + Mockito.when(usageEventVOMock.getResourceName()).thenReturn("not-a-number"); + + usageManagerImpl.backfillNicNetworkIdOnTrunkConversion(usageEventVOMock); + + Mockito.verify(usageNetworkOfferingDaoMock, Mockito.never()).backfillNetworkIdForNic(Mockito.anyLong(), Mockito.anyLong()); + } + + @Test + public void backfillNicNetworkIdOnTrunkConversionSkipsWhenNetworkIdDetailNotParseable() { + Mockito.when(usageEventVOMock.getResourceName()).thenReturn("11"); + Mockito.when(usageEventVOMock.getId()).thenReturn(42L); + Mockito.when(usageEventDetailsDao.findDetail(42L, UsageEventVO.DynamicParameters.networkId.name())) + .thenReturn(new UsageEventDetailsVO(42L, UsageEventVO.DynamicParameters.networkId.name(), "not-a-number")); + + usageManagerImpl.backfillNicNetworkIdOnTrunkConversion(usageEventVOMock); + + Mockito.verify(usageNetworkOfferingDaoMock, Mockito.never()).backfillNetworkIdForNic(Mockito.anyLong(), Mockito.anyLong()); + } } From 3d27d8970874699ce291406bb3a5cef6ca29a89e Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Mon, 28 Sep 2026 14:41:10 -0400 Subject: [PATCH 07/20] Expose trunk nic and associated-network info in NicResponse - New NicNetworkMapResponse + trunked/associatednetworks fields on NicResponse - Populated in listNics, listVirtualMachines, and the associate/disassociate/change-primary command responses - Trunk status derived from nic_network_map rows rather than a new column on the wide UserVmJoinVO view --- .../api/response/NicNetworkMapResponse.java | 137 ++++++++++++++++++ .../cloudstack/api/response/NicResponse.java | 24 +++ .../main/java/com/cloud/api/ApiDBUtils.java | 49 +++++++ .../java/com/cloud/api/ApiResponseHelper.java | 5 + .../api/query/dao/UserVmJoinDaoImpl.java | 22 +++ 5 files changed, 237 insertions(+) create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/NicNetworkMapResponse.java diff --git a/api/src/main/java/org/apache/cloudstack/api/response/NicNetworkMapResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/NicNetworkMapResponse.java new file mode 100644 index 000000000000..45d5098c3814 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/NicNetworkMapResponse.java @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +@EntityReference(value = NicNetworkMapResponse.class) +@SuppressWarnings("unused") +public class NicNetworkMapResponse extends BaseResponse { + + @SerializedName(ApiConstants.NETWORK_ID) + @Param(description = "The ID of the associated network") + private String networkId; + + @SerializedName(ApiConstants.NETWORK_NAME) + @Param(description = "The name of the associated network") + private String networkName; + + @SerializedName(ApiConstants.BROADCAST_URI) + @Param(description = "The broadcast URI of the associated network") + private String broadcastUri; + + @SerializedName(ApiConstants.GATEWAY) + @Param(description = "The gateway of the associated network") + private String gateway; + + @SerializedName(ApiConstants.NETMASK) + @Param(description = "The netmask of the associated network") + private String netmask; + + @SerializedName(ApiConstants.IP6_GATEWAY) + @Param(description = "The IPv6 gateway of the associated network") + private String ip6Gateway; + + @SerializedName(ApiConstants.IP6_CIDR) + @Param(description = "The IPv6 CIDR of the associated network") + private String ip6Cidr; + + @SerializedName(ApiConstants.IP_ADDRESS) + @Param(description = "The IPv4 address on the associated network") + private String ipAddress; + + @SerializedName(ApiConstants.IP6_ADDRESS) + @Param(description = "The IPv6 address on the associated network") + private String ip6Address; + + public String getNetworkId() { + return networkId; + } + + public void setNetworkId(String networkId) { + this.networkId = networkId; + } + + public String getNetworkName() { + return networkName; + } + + public void setNetworkName(String networkName) { + this.networkName = networkName; + } + + public String getBroadcastUri() { + return broadcastUri; + } + + public void setBroadcastUri(String broadcastUri) { + this.broadcastUri = broadcastUri; + } + + public String getGateway() { + return gateway; + } + + public void setGateway(String gateway) { + this.gateway = gateway; + } + + public String getNetmask() { + return netmask; + } + + public void setNetmask(String netmask) { + this.netmask = netmask; + } + + public String getIp6Gateway() { + return ip6Gateway; + } + + public void setIp6Gateway(String ip6Gateway) { + this.ip6Gateway = ip6Gateway; + } + + public String getIp6Cidr() { + return ip6Cidr; + } + + public void setIp6Cidr(String ip6Cidr) { + this.ip6Cidr = ip6Cidr; + } + + public String getIpAddress() { + return ipAddress; + } + + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + + public String getIp6Address() { + return ip6Address; + } + + public void setIp6Address(String ip6Address) { + this.ip6Address = ip6Address; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/NicResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/NicResponse.java index 5ad41ad62244..96d20b1b25b9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/NicResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/NicResponse.java @@ -94,6 +94,14 @@ public class NicResponse extends BaseResponse { @Param(description = "The Secondary IPv4 addr of NIC") private List secondaryIps; + @SerializedName(ApiConstants.TRUNKED) + @Param(description = "True if this nic is a multi-VLAN trunk nic (associated with additional networks beyond its primary)", since = "24.0.0") + private Boolean trunked; + + @SerializedName(ApiConstants.ASSOCIATED_NETWORKS) + @Param(description = "The additional networks this nic is associated with as a multi-VLAN trunk", since = "24.0.0") + private List associatedNetworks; + @SerializedName(ApiConstants.EXTRA_DHCP_OPTION) @Param(description = "The extra DHCP options on the NIC", since = "4.11.0") private List extraDhcpOptions; @@ -269,6 +277,14 @@ public void setSecondaryIps(List ipList) { this.secondaryIps = ipList; } + public void setTrunked(Boolean trunked) { + this.trunked = trunked; + } + + public void setAssociatedNetworks(List associatedNetworks) { + this.associatedNetworks = associatedNetworks; + } + public void setNsxLogicalSwitch(String nsxLogicalSwitch) { this.nsxLogicalSwitch = nsxLogicalSwitch; } @@ -337,6 +353,14 @@ public List getSecondaryIps() { return secondaryIps; } + public Boolean getTrunked() { + return trunked; + } + + public List getAssociatedNetworks() { + return associatedNetworks; + } + public String getDeviceId() { return deviceId; } diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 934600eb2b61..bd7e6b8c483c 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -48,6 +48,7 @@ import org.apache.cloudstack.api.response.BackupOfferingResponse; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.NicNetworkMapResponse; import org.apache.cloudstack.api.response.DomainRouterResponse; import org.apache.cloudstack.api.response.EventResponse; import org.apache.cloudstack.api.response.SharedFSResponse; @@ -339,6 +340,8 @@ import com.cloud.uservm.UserVm; import com.cloud.utils.EnumUtils; import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; +import com.cloud.utils.net.NetUtils; import com.cloud.vm.ConsoleProxyVO; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.InstanceGroup; @@ -356,6 +359,8 @@ import com.cloud.vm.dao.ConsoleProxyDao; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import com.cloud.vm.dao.NicSecondaryIpDao; import com.cloud.vm.dao.NicSecondaryIpVO; import com.cloud.vm.dao.UserVmDao; @@ -487,6 +492,7 @@ public class ApiDBUtils { static VMSnapshotDao s_vmSnapshotDao; static ClusterDetailsDao s_clusterDetailsDao; static NicSecondaryIpDao s_nicSecondaryIpDao; + static NicNetworkMapDao s_nicNetworkMapDao; static VpcProvisioningService s_vpcProvSvc; static AffinityGroupDao s_affinityGroupDao; static AffinityGroupJoinDao s_affinityGroupJoinDao; @@ -736,6 +742,8 @@ public class ApiDBUtils { @Inject private NicSecondaryIpDao nicSecondaryIpDao; @Inject + private NicNetworkMapDao nicNetworkMapDao; + @Inject private VpcProvisioningService vpcProvSvc; @Inject private AffinityGroupDao affinityGroupDao; @@ -903,6 +911,7 @@ void init() { s_vmSnapshotDao = vmSnapshotDao; s_nicDao = nicDao; s_nicSecondaryIpDao = nicSecondaryIpDao; + s_nicNetworkMapDao = nicNetworkMapDao; s_vpcProvSvc = vpcProvSvc; s_affinityGroupDao = affinityGroupDao; s_affinityGroupJoinDao = affinityGroupJoinDao; @@ -2246,6 +2255,46 @@ public static List findNicSecondaryIps(long nicId) { return s_nicSecondaryIpDao.listByNicId(nicId); } + /** + * Builds the response for every additional network a multi-VLAN trunk nic is associated with (beyond its + * primary network). Shared by every response builder that surfaces a nic's associations - {@code listNics}, + * {@code listVirtualMachines}, and the associate/disassociate/change-primary command responses all need the + * identical shape, so it's built once here rather than duplicated per builder. + */ + public static List findAssociatedNetworkResponses(long nicId) { + List associations = s_nicNetworkMapDao.listByNicId(nicId); + List associatedNetworkResponses = new ArrayList<>(); + for (NicNetworkMapVO association : associations) { + NetworkVO associatedNetwork = findNetworkById(association.getNetworkId()); + if (associatedNetwork == null) { + continue; + } + NicNetworkMapResponse associationResponse = new NicNetworkMapResponse(); + associationResponse.setNetworkId(associatedNetwork.getUuid()); + associationResponse.setNetworkName(associatedNetwork.getName()); + if (associatedNetwork.getBroadcastUri() != null) { + associationResponse.setBroadcastUri(associatedNetwork.getBroadcastUri().toString()); + } + // gateway/netmask/IPv6 gateway/IPv6 CIDR describe the associated network itself, not this nic's + // specific address on it - sourced the same way ApiResponseHelper#createNetworkResponse does for + // NetworkResponse. A Shared network with more than one IP range has these as a comma-separated + // list (one per range); only the first is surfaced here, matching that same existing convention. + associationResponse.setGateway(StringUtils.getFirstValueFromCommaSeparatedString(associatedNetwork.getGateway())); + String cidr = StringUtils.getFirstValueFromCommaSeparatedString(associatedNetwork.getCidr()); + if (associatedNetwork.getNetworkCidr() != null) { + associationResponse.setNetmask(NetUtils.cidr2Netmask(associatedNetwork.getNetworkCidr())); + } else if (cidr != null) { + associationResponse.setNetmask(NetUtils.cidr2Netmask(cidr)); + } + associationResponse.setIp6Gateway(StringUtils.getFirstValueFromCommaSeparatedString(associatedNetwork.getIp6Gateway())); + associationResponse.setIp6Cidr(StringUtils.getFirstValueFromCommaSeparatedString(associatedNetwork.getIp6Cidr())); + associationResponse.setIpAddress(association.getIp4Address()); + associationResponse.setIp6Address(association.getIp6Address()); + associatedNetworkResponses.add(associationResponse); + } + return associatedNetworkResponses; + } + public static NicVO findNicById(long nicId) { return s_nicDao.findById(nicId); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index f56cda6e557a..150d04ddc0d8 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -4880,6 +4880,11 @@ public NicResponse createNicResponse(Nic result) { response.setSecondaryIps(ipList); } } + /*trunked / associatednetworks - multi-VLAN trunk nic, if any*/ + response.setTrunked(result.getMultiNetwork()); + if (result.getMultiNetwork()) { + response.setAssociatedNetworks(ApiDBUtils.findAssociatedNetworkResponses(result.getId())); + } /*18: extra dhcp options */ List nicExtraDhcpOptionResponses = nicExtraDhcpOptionVOs .stream() diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java index ebc2326a72d5..1836ed41b090 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java @@ -104,6 +104,8 @@ import com.cloud.vm.VmIsoMapVO; import com.cloud.vm.VmStats; import com.cloud.vm.dao.NicExtraDhcpOptionDao; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import com.cloud.vm.dao.NicSecondaryIpVO; import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.dao.VmIsoMapDao; @@ -122,6 +124,8 @@ public class UserVmJoinDaoImpl extends GenericDaoBaseWithTagInformation nicExtraDhcpOptionResponses = _nicExtraDhcpOptionDao.listByNicId(nic_id).stream() @@ -630,6 +636,20 @@ private void addVnfInfoToserVmResponse(UserVmJoinVO userVm, UserVmResponse userV } } + /** + * The join view backing UserVmJoinVO doesn't select nics.multi_network, so trunk status is derived directly + * from whether nic_network_map has any rows for this nic, rather than adding a column to that wide, + * heavily-used view. + */ + private void populateTrunkNicResponse(NicResponse nicResponse, long nicId) { + List associations = _nicNetworkMapDao.listByNicId(nicId); + nicResponse.setTrunked(!associations.isEmpty()); + if (associations.isEmpty()) { + return; + } + nicResponse.setAssociatedNetworks(ApiDBUtils.findAssociatedNetworkResponses(nicId)); + } + private void addVmRxTxDataToResponse(UserVmJoinVO userVm, UserVmResponse userVmResponse) { Long bytesReceived = 0L; Long bytesSent = 0L; @@ -737,6 +757,8 @@ public UserVmResponse setUserVmResponse(ResponseView view, UserVmResponse userVm nicResponse.setPublicIp(publicIp.getAddress().toString()); } + populateTrunkNicResponse(nicResponse, nic_id); + /* 18: extra dhcp options */ nicResponse.setObjectName("nic"); List nicExtraDhcpOptionResponses = _nicExtraDhcpOptionDao.listByNicId(nic_id) From dd4ee2a665251b9c39ea7cc7e86b3e7515e2ae6f Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Mon, 28 Sep 2026 14:41:46 -0400 Subject: [PATCH 08/20] Add ApiConstants for NicResponse trunk fields - ASSOCIATED_NETWORKS/TRUNKED, needed by the previous commit's NicResponse fields --- api/src/main/java/org/apache/cloudstack/api/ApiConstants.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index de6aeed50a01..265431813f70 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -867,6 +867,8 @@ public class ApiConstants { public static final String ASSOCIATED_NETWORK = "associatednetwork"; public static final String ASSOCIATED_NETWORK_ID = "associatednetworkid"; public static final String ASSOCIATED_NETWORK_NAME = "associatednetworkname"; + public static final String ASSOCIATED_NETWORKS = "associatednetworks"; + public static final String TRUNKED = "trunked"; public static final String ASSOCIATED_VPC_ID = "associatedvpcid"; public static final String ASSOCIATED_VPC_NAME = "associatedvpcname"; public static final String SOURCE_NAT_SUPPORTED = "sourcenatsupported"; From c16a58d4241a69b6c5a76b5fc7bb713cbb2a63f8 Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Mon, 28 Sep 2026 14:48:44 -0400 Subject: [PATCH 09/20] Add multi-VLAN trunk NIC association service Core capability: - New associateNetworkToNic/disassociateNetworkFromNic/changeNicPrimaryNetwork admin APIs and associateNetworksToNic (programmatic, used by deploy) - A nic keeps one primary network plus any number of additional associated networks (nic_network_map), delivered as a VLAN trunk - changeNicPrimaryNetwork only allowed while the Instance is stopped - Rejects associating a network already reachable via another of the VM's nics, or with an overlapping CIDR to an existing association - Disassociating a network is blocked while an active PF/Static NAT/LB rule targets its allocated IP PF/Static NAT/LB support for associated networks: - NetworkModel.getNicAndIpInNetwork resolves a nic/guest-ip pair via the nic's primary match or a trunk association, reused by RulesManagerImpl and LoadBalancingRulesManagerImpl so these rules can target a trunk nic's associated networks, not just its primary Deployment planning: - Hosts without VLAN filtering enabled are excluded up front for any VM with a multi-VLAN trunk nic, with a reactive fallback exclusion during start as a backstop - Removing a nic cleans up any trunk associations it held as primary; deleting a network is blocked while still associated as a secondary Metadata: - A trunk nic's associated-network VLAN tags are exposed to the guest via the metadata service (nic-vlan-mapping file), gated by a new account-scoped config key, off by default --- .../java/com/cloud/network/NetworkModel.java | 8 + .../com/cloud/network/NetworkService.java | 29 + .../network/AssociateNetworkToNicCmd.java | 160 ++++ .../network/ChangeNicPrimaryNetworkCmd.java | 124 +++ .../DisassociateNetworkFromNicCmd.java | 123 +++ .../com/cloud/vm/VirtualMachineManager.java | 4 + .../service/NetworkOrchestrationService.java | 3 + .../cloud/vm/VirtualMachineManagerImpl.java | 20 +- .../orchestration/NetworkOrchestrator.java | 36 +- .../vm/VirtualMachineManagerImplTest.java | 113 +++ .../NetworkOrchestratorTest.java | 95 +++ .../deploy/DeploymentPlanningManagerImpl.java | 35 + .../com/cloud/network/NetworkModelImpl.java | 27 + .../com/cloud/network/NetworkServiceImpl.java | 524 ++++++++++++- .../lb/LoadBalancingRulesManagerImpl.java | 64 +- .../network/router/CommandSetupHelper.java | 39 + .../cloud/network/rules/RulesManagerImpl.java | 52 +- .../cloud/server/ManagementServerImpl.java | 6 + .../DeploymentPlanningManagerImplTest.java | 68 ++ .../cloud/network/MockNetworkModelImpl.java | 6 + .../cloud/network/NetworkModelImplTest.java | 55 ++ .../cloud/network/NetworkServiceImplTest.java | 731 ++++++++++++++++++ .../lb/LoadBalancingRulesManagerImplTest.java | 18 +- .../router/CommandSetupHelperTest.java | 130 ++++ .../com/cloud/vpc/MockNetworkManagerImpl.java | 22 + .../com/cloud/vpc/MockNetworkModelImpl.java | 6 + 26 files changed, 2430 insertions(+), 68 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/network/AssociateNetworkToNicCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/network/ChangeNicPrimaryNetworkCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/network/DisassociateNetworkFromNicCmd.java diff --git a/api/src/main/java/com/cloud/network/NetworkModel.java b/api/src/main/java/com/cloud/network/NetworkModel.java index 7e1a07ebeb69..410d5da34ead 100644 --- a/api/src/main/java/com/cloud/network/NetworkModel.java +++ b/api/src/main/java/com/cloud/network/NetworkModel.java @@ -74,6 +74,7 @@ public interface NetworkModel { String CLOUD_DOMAIN_FILE = "cloud-domain"; String CLOUD_DOMAIN_ID_FILE = "cloud-domain-id"; String CLOUD_NAME_FILE = "cloud-name"; + String NIC_VLAN_MAPPING_FILE = "nic-vlan-mapping"; int CONFIGDATA_DIR = 0; int CONFIGDATA_FILE = 1; int CONFIGDATA_CONTENT = 2; @@ -137,6 +138,13 @@ public interface NetworkModel { Nic getNicInNetwork(long vmId, long networkId); + /** + * Resolves the nic and guest ip(s) a VM answers on for a given network - the nic's own primary network match + * when one exists, or a trunk nic's association (from nic_network_map) otherwise. Returns null if the VM has + * neither a primary nor an associated nic in that network. + */ + Pair getNicAndIpInNetwork(long vmId, long networkId); + List getNicsForTraffic(long vmId, TrafficType type); Network getDefaultNetworkForVm(long vmId); diff --git a/api/src/main/java/com/cloud/network/NetworkService.java b/api/src/main/java/com/cloud/network/NetworkService.java index c32bb711c0f2..64342c6104fb 100644 --- a/api/src/main/java/com/cloud/network/NetworkService.java +++ b/api/src/main/java/com/cloud/network/NetworkService.java @@ -35,6 +35,9 @@ import org.apache.cloudstack.api.command.user.network.ResetNetworkPermissionsCmd; import org.apache.cloudstack.api.command.user.network.RestartNetworkCmd; import org.apache.cloudstack.api.command.user.network.UpdateNetworkCmd; +import org.apache.cloudstack.api.command.admin.network.AssociateNetworkToNicCmd; +import org.apache.cloudstack.api.command.admin.network.ChangeNicPrimaryNetworkCmd; +import org.apache.cloudstack.api.command.admin.network.DisassociateNetworkFromNicCmd; import org.apache.cloudstack.api.command.user.vm.ListNicsCmd; import org.apache.cloudstack.api.response.AcquirePodIpCmdResponse; import org.apache.cloudstack.framework.config.ConfigKey; @@ -249,6 +252,32 @@ Network createPrivateNetwork(String networkName, String displayText, long physic List listVmNicSecondaryIps(ListNicsCmd listNicsCmd); + /** + * Associates one or more additional networks with an existing nic, delivering them as a multi-VLAN trunk + */ + Nic associateNetworkToNic(AssociateNetworkToNicCmd cmd) + throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException, InsufficientAddressCapacityException; + + /** + * Disassociates a network from an existing nic, shrinking its multi-VLAN trunk. The primary network cannot be disassociated. + */ + Nic disassociateNetworkFromNic(DisassociateNetworkFromNicCmd cmd) throws ResourceUnavailableException; + + /** + * Changes a multi-VLAN trunk nic's primary network to one of its already-associated networks. Only allowed while + * the owning Instance is stopped, since the nic is rebuilt fresh on its next start. + */ + Nic changeNicPrimaryNetwork(ChangeNicPrimaryNetworkCmd cmd); + + /** + * Associates additional networks with a nic that was just created (e.g. during VM deploy), before the owning + * Instance has ever started. No live agent update is issued - the nic's first plug already reads these + * associations fresh. requestedIps (nullable) optionally requests a specific ip4/ip6 address per associated + * network id - a network with no entry (or a null map) auto-allocates, same as today. + */ + void associateNetworksToNic(Nic nic, List networkIds, Map requestedIps) + throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException, InsufficientAddressCapacityException; + AcquirePodIpCmdResponse allocatePodIp(Account account, String zoneId, String podId) throws ResourceAllocationException, ConcurrentOperationException; boolean releasePodIp(ReleasePodIpCmdByAdmin ip) throws CloudRuntimeException; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/AssociateNetworkToNicCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/AssociateNetworkToNicCmd.java new file mode 100644 index 000000000000..4b47a5e9462d --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/AssociateNetworkToNicCmd.java @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.network; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.NetworkResponse; +import org.apache.cloudstack.api.response.NicResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientAddressCapacityException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.network.Network; +import com.cloud.user.Account; +import com.cloud.vm.Nic; +import com.cloud.vm.VirtualMachine; + +@APICommand(name = "associateNetworkToNic", description = "Associates one or more additional networks with an existing NIC, delivering them as a multi-VLAN trunk", + responseObject = NicResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "24.0.0") +public class AssociateNetworkToNicCmd extends BaseAsyncCmd { + private static final String s_name = "associatenetworktonicresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NIC_ID, type = CommandType.UUID, entityType = NicResponse.class, required = true, + description = "the ID of the NIC to associate additional networks with") + private Long nicId; + + @Parameter(name = ApiConstants.NETWORK_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = NetworkResponse.class, required = true, + description = "the IDs of the additional networks to associate with the NIC") + private List networkIds; + + @Parameter(name = ApiConstants.IP_ADDRESSES, type = CommandType.MAP, required = false, + description = "optional per-network IP addresses, keyed by network. Example: " + + "ipaddresses[0].networkid=UUID&ipaddresses[0].ipaddress=10.0.0.5&ipaddresses[0].ip6address=fd00::5") + private Map ipAddresses; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getNicId() { + return nicId; + } + + public List getNetworkIds() { + return networkIds; + } + + public Map getIpAddressesMap() { + Map result = new HashMap<>(); + if (ipAddresses == null || ipAddresses.isEmpty()) { + return result; + } + Collection> entries = ipAddresses.values(); + for (Map entry : entries) { + String networkUuid = entry.get(ApiConstants.NETWORK_ID); + if (networkUuid == null) { + throw new InvalidParameterValueException(String.format("Each ipaddresses entry must specify %s", ApiConstants.NETWORK_ID)); + } + Network network = _entityMgr.findByUuid(Network.class, networkUuid); + if (network == null) { + throw new InvalidParameterValueException(String.format("Unable to find network with id: %s", networkUuid)); + } + String ip4Address = entry.get(ApiConstants.IP_ADDRESS); + String ip6Address = entry.get(ApiConstants.IP6_ADDRESS); + result.put(network.getId(), new Network.IpAddresses(ip4Address, ip6Address)); + } + return result; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation /////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_NIC_NETWORK_ASSOCIATE; + } + + @Override + public String getEventDescription() { + return "Associating networks " + StringUtils.join(getNetworkIds(), ",") + " with nic " + getResourceUuid(ApiConstants.NIC_ID); + } + + private Nic getNic() { + Nic nic = _entityMgr.findById(Nic.class, nicId); + if (nic == null) { + throw new InvalidParameterValueException("Can't find NIC for id specified"); + } + return nic; + } + + @Override + public long getEntityOwnerId() { + VirtualMachine vm = _entityMgr.findById(VirtualMachine.class, getNic().getInstanceId()); + if (vm == null) { + return Account.ACCOUNT_ID_SYSTEM; + } + return vm.getAccountId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } + + @Override + public Long getApiResourceId() { + return getNic().getInstanceId(); + } + + @Override + public void execute() throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException, InsufficientAddressCapacityException { + CallContext.current().setEventDetails("Nic ID: " + getResourceUuid(ApiConstants.NIC_ID)); + Nic result = _networkService.associateNetworkToNic(this); + if (result == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to associate network to NIC. Refer to server logs for details."); + } + NicResponse response = _responseGenerator.createNicResponse(result); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/ChangeNicPrimaryNetworkCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/ChangeNicPrimaryNetworkCmd.java new file mode 100644 index 000000000000..ac2e2ce2d55c --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/ChangeNicPrimaryNetworkCmd.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.network; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.NetworkResponse; +import org.apache.cloudstack.api.response.NicResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.user.Account; +import com.cloud.vm.Nic; +import com.cloud.vm.VirtualMachine; + +@APICommand(name = "changeNicPrimaryNetwork", description = "Changes a multi-VLAN trunk NIC's primary (native/untagged) network to one of its already-associated networks. " + + "Only allowed while the owning Instance is stopped. The operator is responsible for confirming the guest will " + + "reacquire its address on the new primary network (e.g. via DHCP) after the next start - CloudStack cannot verify this from the host side.", + responseObject = NicResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "24.0.0") +public class ChangeNicPrimaryNetworkCmd extends BaseAsyncCmd { + private static final String s_name = "changenicprimarynetworkresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NIC_ID, type = CommandType.UUID, entityType = NicResponse.class, required = true, + description = "the ID of the NIC whose primary network should be changed") + private Long nicId; + + @Parameter(name = ApiConstants.NETWORK_ID, type = CommandType.UUID, entityType = NetworkResponse.class, required = true, + description = "the ID of the network to make primary; it must already be associated with this NIC") + private Long networkId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getNicId() { + return nicId; + } + + public Long getNetworkId() { + return networkId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation /////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_NIC_PRIMARY_NETWORK_CHANGE; + } + + @Override + public String getEventDescription() { + return "Changing nic " + getResourceUuid(ApiConstants.NIC_ID) + "'s primary network to " + getResourceUuid(ApiConstants.NETWORK_ID); + } + + private Nic getNic() { + Nic nic = _entityMgr.findById(Nic.class, nicId); + if (nic == null) { + throw new InvalidParameterValueException("Can't find NIC for id specified"); + } + return nic; + } + + @Override + public long getEntityOwnerId() { + VirtualMachine vm = _entityMgr.findById(VirtualMachine.class, getNic().getInstanceId()); + if (vm == null) { + return Account.ACCOUNT_ID_SYSTEM; + } + return vm.getAccountId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } + + @Override + public Long getApiResourceId() { + return getNic().getInstanceId(); + } + + @Override + public void execute() { + CallContext.current().setEventDetails("Nic ID: " + getResourceUuid(ApiConstants.NIC_ID)); + Nic result = _networkService.changeNicPrimaryNetwork(this); + if (result == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to change nic's primary network. Refer to server logs for details."); + } + NicResponse response = _responseGenerator.createNicResponse(result); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/DisassociateNetworkFromNicCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/DisassociateNetworkFromNicCmd.java new file mode 100644 index 000000000000..ad860a4efe26 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/DisassociateNetworkFromNicCmd.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.network; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.NetworkResponse; +import org.apache.cloudstack.api.response.NicResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.user.Account; +import com.cloud.vm.Nic; +import com.cloud.vm.VirtualMachine; + +@APICommand(name = "disassociateNetworkFromNic", description = "Disassociates a network from an existing NIC, shrinking its multi-VLAN trunk", + responseObject = NicResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "24.0.0") +public class DisassociateNetworkFromNicCmd extends BaseAsyncCmd { + private static final String s_name = "disassociatenetworkfromnicresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NIC_ID, type = CommandType.UUID, entityType = NicResponse.class, required = true, + description = "the ID of the NIC to disassociate the network from") + private Long nicId; + + @Parameter(name = ApiConstants.NETWORK_ID, type = CommandType.UUID, entityType = NetworkResponse.class, required = true, + description = "the ID of the associated network to remove from the NIC") + private Long networkId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getNicId() { + return nicId; + } + + public Long getNetworkId() { + return networkId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation /////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_NIC_NETWORK_DISASSOCIATE; + } + + @Override + public String getEventDescription() { + return "Disassociating network " + getResourceUuid(ApiConstants.NETWORK_ID) + " from nic " + getResourceUuid(ApiConstants.NIC_ID); + } + + private Nic getNic() { + Nic nic = _entityMgr.findById(Nic.class, nicId); + if (nic == null) { + throw new InvalidParameterValueException("Can't find NIC for id specified"); + } + return nic; + } + + @Override + public long getEntityOwnerId() { + VirtualMachine vm = _entityMgr.findById(VirtualMachine.class, getNic().getInstanceId()); + if (vm == null) { + return Account.ACCOUNT_ID_SYSTEM; + } + return vm.getAccountId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } + + @Override + public Long getApiResourceId() { + return getNic().getInstanceId(); + } + + @Override + public void execute() throws ResourceUnavailableException { + CallContext.current().setEventDetails("Nic ID: " + getResourceUuid(ApiConstants.NIC_ID)); + Nic result = _networkService.disassociateNetworkFromNic(this); + if (result == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to disassociate network from NIC. Refer to server logs for details."); + } + NicResponse response = _responseGenerator.createNicResponse(result); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index c7238a513693..77f3379ee708 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -88,6 +88,10 @@ public interface VirtualMachineManager extends Manager { ConfigKey AllowExposeDomainInMetadata = new ConfigKey<>("Advanced", Boolean.class, "metadata.allow.expose.domain", "false", "If set to true, it allows the VM's domain to be seen in metadata.", true, ConfigKey.Scope.Domain); + ConfigKey AllowExposeNicVlanMapping = new ConfigKey<>("Advanced", Boolean.class, "account.allow.expose.nic.vlan.mapping", + "false", "If set to true, exposes a VM's trunk NIC associated-network VLAN tag mappings via the metadata service, " + + "for guest/VNF boot scripts to configure VLAN sub-interfaces. Has no effect on VMs without trunk NICs.", true, ConfigKey.Scope.Account); + ConfigKey MetadataCustomCloudName = new ConfigKey<>("Advanced", String.class, "metadata.custom.cloud.name", "", "If provided, a custom cloud-name in cloud-init metadata", true, ConfigKey.Scope.Zone); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index 109a44488ec4..5c141197ce12 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -98,6 +98,9 @@ public interface NetworkOrchestrationService { ConfigKey PromiscuousMode = new ConfigKey<>("Advanced", Boolean.class, "network.promiscuous.mode", "false", "Whether to allow or deny promiscuous mode on NICs for applicable network elements such as for vswitch/dvswitch portgroups.", true); + ConfigKey MultiNetworkNicEnabled = new ConfigKey<>("Advanced", Boolean.class, "multi.network.nic.enabled", "false", + "Allow a NIC to be associated with more than one network (multi-VLAN trunk NICs)", true, ConfigKey.Scope.Zone); + ConfigKey MacAddressChanges = new ConfigKey<>("Advanced", Boolean.class, "network.mac.address.changes", "true", "Whether to allow or deny mac address changes on NICs for applicable network elements such as for vswitch/dvswitch porgroups.", true); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index a1b4e9e732f8..3702787fe9f4 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -217,6 +217,7 @@ import com.cloud.exception.StorageUnavailableException; import com.cloud.ha.HighAvailabilityManager; import com.cloud.ha.HighAvailabilityManager.WorkType; +import com.cloud.host.DetailVO; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -1494,6 +1495,14 @@ public void orchestrateStart(final String vmUuid, final Map[] getConfigKeys() { VmOpLockStateRetry, VmOpWaitInterval, ExecuteInSequence, VmJobCheckInterval, VmJobTimeout, VmJobStateReportInterval, VmConfigDriveLabel, VmConfigDriveOnPrimaryPool, VmConfigDriveForceHostCacheUse, VmConfigDriveUseHostCacheOnUnsupportedPool, HaVmRestartHostUp, ResourceCountRunningVMsonly, AllowExposeHypervisorHostname, AllowExposeHypervisorHostnameAccountLevel, SystemVmRootDiskSize, - AllowExposeDomainInMetadata, MetadataCustomCloudName, VmMetadataManufacturer, VmMetadataProductName, + AllowExposeDomainInMetadata, AllowExposeNicVlanMapping, MetadataCustomCloudName, VmMetadataManufacturer, VmMetadataProductName, VmSyncPowerStateTransitioning, SystemVmEnableUserData }; } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 8af75562b31c..f676882211a7 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -269,6 +269,8 @@ import com.cloud.vm.dao.NicExtraDhcpOptionDao; import com.cloud.vm.dao.NicIpAliasDao; import com.cloud.vm.dao.NicIpAliasVO; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import com.cloud.vm.dao.NicSecondaryIpDao; import com.cloud.vm.dao.NicSecondaryIpVO; import com.cloud.vm.dao.UserVmDao; @@ -307,6 +309,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Inject NicDao _nicDao; @Inject + NicNetworkMapDao _nicNetworkMapDao; + @Inject RulesManager _rulesMgr; @Inject LoadBalancingRulesManager _lbMgr; @@ -2740,6 +2744,12 @@ && isDhcpAccrossMultipleSubnetsSupported(dhcpServiceProvider)) { if (!removeVmSecondaryIpsOfNic(nic.getId())) { logger.debug("Removing NIC {} secondary IP addresses failed", nic); } + + // remove any multi-VLAN trunk associations this nic held as a primary - otherwise they're left + // permanently orphaned, referencing a nic that no longer exists + for (final NicNetworkMapVO association : _nicNetworkMapDao.listByNicId(nic.getId())) { + _nicNetworkMapDao.remove(association.getId()); + } } public boolean isDhcpAccrossMultipleSubnetsSupported(final DhcpServiceProvider dhcpServiceProvider) { @@ -2791,6 +2801,9 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { public void removeNics(final VirtualMachineProfile vm) { final List nics = _nicDao.listByVmId(vm.getId()); for (final NicVO nic : nics) { + for (final NicNetworkMapVO association : _nicNetworkMapDao.listByNicId(nic.getId())) { + _nicNetworkMapDao.remove(association.getId()); + } _nicDao.remove(nic.getId()); } } @@ -3477,6 +3490,13 @@ public boolean destroyNetwork(final long networkId, final ReservationContext con } } + // A network with no primary nics can still be actively used as a secondary (trunk) network by a nic + // whose primary is a different network entirely - nics.network_id alone would miss this + if (MultiNetworkNicEnabled.valueIn(network.getDataCenterId()) && !_nicNetworkMapDao.listByNetworkId(networkId).isEmpty()) { + logger.warn("Can't delete the Network {}, it is still associated with at least one NIC as a secondary network", network); + return false; + } + // Don't allow to delete network via api call when it has vms assigned to it final int nicCount = getActiveNicsInNetwork(networkId); if (nicCount > 0) { @@ -3715,6 +3735,15 @@ public void reallyRun() { continue; } + final NetworkVO network = _networksDao.findById(networkId); + if (network != null && MultiNetworkNicEnabled.valueIn(network.getDataCenterId()) + && !_nicNetworkMapDao.listByNetworkId(networkId).isEmpty()) { + // only zones with the feature enabled can ever have nic_network_map rows at all, + // so this check is skipped entirely everywhere else instead of querying needlessly + logger.debug("Network {} is still associated with at least one NIC as a secondary network, skipping", networkId); + continue; + } + final Long time = _lastNetworkIdsToFree.remove(networkId); if (time == null) { logger.debug("We found network {} to be free for the first time. Adding it to the list: {}", () -> _networksDao.findById(networkId), () -> currentTime); @@ -5032,10 +5061,9 @@ public void unmanageNics(VirtualMachineProfile vm) { final List nics = _nicDao.listByVmId(vm.getId()); for (final NicVO nic : nics) { removeNic(vm, nic); - NetworkVO network = _networksDao.findById(nic.getNetworkId()); if (virtualMachine.getState() != VirtualMachine.State.Stopped) { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, virtualMachine.getAccountId(), virtualMachine.getDataCenterId(), virtualMachine.getId(), - Long.toString(nic.getId()), network.getNetworkOfferingId(), null, 0L, virtualMachine.getClass().getName(), virtualMachine.getUuid(), virtualMachine.isDisplay()); + UsageEventUtils.publishNicNetworkOfferingUsageEvents(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, virtualMachine.getAccountId(), virtualMachine.getDataCenterId(), + virtualMachine.getId(), virtualMachine.getClass().getName(), virtualMachine.getUuid(), nic, 0L, virtualMachine.isDisplay()); } } } @@ -5069,6 +5097,6 @@ public ConfigKey[] getConfigKeys() { GuestDomainSuffix, NetworkThrottlingRate, VmNetworkThrottlingRate, MinVRVersion, DhcpLeaseTimeout, PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RollingRestartEnabled, TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED, NETWORK_LB_HAPROXY_MAX_CONN, - NETWORK_LB_HAPROXY_IDLE_TIMEOUT}; + NETWORK_LB_HAPROXY_IDLE_TIMEOUT, MultiNetworkNicEnabled}; } } diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index 3b680f0bc74a..40f76b74b808 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -126,9 +126,11 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.DetailVO; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.network.NetworkService; @@ -284,6 +286,8 @@ public class VirtualMachineManagerImplTest { @Mock NicDao _nicsDao; @Mock + HostDetailsDao hostDetailsDao; + @Mock NetworkService networkService; @Mock NetworkModel networkModel; @@ -1298,6 +1302,115 @@ public void testOrchestrateStartNullPodId() throws Exception { assertNull(vmInstance.getPodIdToDeployIn()); } + @Test + public void testOrchestrateStartExcludesHostWithoutVlanFilteringForMultiNetworkNic() throws Exception { + VMInstanceVO vmInstance = new VMInstanceVO(); + ReflectionTestUtils.setField(vmInstance, "id", 1L); + ReflectionTestUtils.setField(vmInstance, "accountId", 1L); + ReflectionTestUtils.setField(vmInstance, "uuid", "vm-uuid"); + ReflectionTestUtils.setField(vmInstance, "serviceOfferingId", 2L); + ReflectionTestUtils.setField(vmInstance, "instanceName", "myVm"); + ReflectionTestUtils.setField(vmInstance, "hostId", 2L); + ReflectionTestUtils.setField(vmInstance, "type", VirtualMachine.Type.User); + ReflectionTestUtils.setField(vmInstance, "dataCenterId", 1L); + ReflectionTestUtils.setField(vmInstance, "hypervisorType", HypervisorType.KVM); + + VirtualMachineGuru vmGuru = mock(VirtualMachineGuru.class); + User user = mock(User.class); + Account account = mock(Account.class); + Account owner = mock(Account.class); + ReservationContext ctx = mock(ReservationContext.class); + ItWorkVO work = mock(ItWorkVO.class); + ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); + VirtualMachineTemplate template = mock(VirtualMachineTemplate.class); + when(template.isDeployAsIs()).thenReturn(false); + + DataCenterDeployment plan = mock(DataCenterDeployment.class); + when(plan.getDataCenterId()).thenReturn(1L); + when(plan.getPodId()).thenReturn(1L); + + Map params = new HashMap<>(); + DeploymentPlanner planner = mock(DeploymentPlanner.class); + + when(vmInstanceDaoMock.findByUuid("vm-uuid")).thenReturn(vmInstance); + doReturn(vmGuru).when(virtualMachineManagerImpl).getVmGuru(vmInstance); + + Ternary start = new Ternary<>(vmInstance, ctx, work); + Mockito.doReturn(start).when(virtualMachineManagerImpl).changeToStartState(vmGuru, vmInstance, user, account, owner, serviceOffering, template); + + when(ctx.getJournal()).thenReturn(Mockito.mock(Journal.class)); + when(serviceOfferingDaoMock.findById(vmInstance.getId(), vmInstance.getServiceOfferingId())).thenReturn(serviceOffering); + when(_entityMgr.findById(Account.class, vmInstance.getAccountId())).thenReturn(owner); + when(_entityMgr.findByIdIncludingRemoved(VirtualMachineTemplate.class, vmInstance.getTemplateId())).thenReturn(template); + + NicVO trunkNic = mock(NicVO.class); + when(trunkNic.getMultiNetwork()).thenReturn(true); + when(_nicsDao.listByVmId(vmInstance.getId())).thenReturn(java.util.Collections.singletonList(trunkNic)); + + Host notReadyHost = mock(Host.class); + when(notReadyHost.getId()).thenReturn(1L); + when(hostDetailsDao.findDetail(1L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(null); + + Host readyHost = mock(Host.class); + when(readyHost.getId()).thenReturn(2L); + DetailVO readyDetail = mock(DetailVO.class); + when(readyDetail.getValue()).thenReturn("true"); + when(hostDetailsDao.findDetail(2L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(readyDetail); + + Pod destPod = mock(Pod.class); + when(destPod.getId()).thenReturn(2L); + Cluster cluster = mock(Cluster.class); + when(cluster.getId()).thenReturn(1L); + + DeployDestination notReadyDest = mock(DeployDestination.class); + when(notReadyDest.getHost()).thenReturn(notReadyHost); + + DeployDestination readyDest = mock(DeployDestination.class); + when(readyDest.getHost()).thenReturn(readyHost); + when(readyDest.getPod()).thenReturn(destPod); + when(readyDest.getCluster()).thenReturn(cluster); + + when(_dpMgr.planDeployment(any(VirtualMachineProfileImpl.class), any(DataCenterDeployment.class), any(ExcludeList.class), any(DeploymentPlanner.class))) + .thenReturn(notReadyDest, readyDest); + + doNothing().when(virtualMachineManagerImpl).checkIfTemplateNeededForCreatingVmVolumes(vmInstance); + + when(_workDao.updateStep(any(), any())).thenReturn(true); + when(_stateMachine.transitTo(vmInstance, VirtualMachine.Event.OperationRetry, new Pair(vmInstance.getHostId(), 2L), vmInstanceDaoMock)).thenThrow(new CloudRuntimeException("Error while transitioning")); + when(_stateMachine.transitTo(vmInstance, VirtualMachine.Event.OperationFailed, new Pair(vmInstance.getHostId(), null), vmInstanceDaoMock)).thenReturn(true); + + ClusterDetailsVO cluster_detail_cpu = mock(ClusterDetailsVO.class); + ClusterDetailsVO cluster_detail_ram = mock(ClusterDetailsVO.class); + when(_clusterDetailsDao.findDetail(1L, VmDetailConstants.CPU_OVER_COMMIT_RATIO)).thenReturn(cluster_detail_cpu); + when(_clusterDetailsDao.findDetail(1L, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO)).thenReturn(cluster_detail_ram); + when(vmInstanceDetailsDao.findDetail(anyLong(), Mockito.anyString())).thenReturn(null); + when(cluster_detail_cpu.getValue()).thenReturn("1.0"); + when(cluster_detail_ram.getValue()).thenReturn("1.0"); + doReturn(false).when(virtualMachineManagerImpl).areAllVolumesAllocated(Mockito.anyLong()); + + CallContext callContext = mock(CallContext.class); + when(callContext.getCallingAccount()).thenReturn(account); + when(callContext.getCallingUser()).thenReturn(user); + try (MockedStatic ignored = Mockito.mockStatic(CallContext.class)) { + when(CallContext.current()).thenReturn(callContext); + + try { + virtualMachineManagerImpl.orchestrateStart("vm-uuid", params, plan, planner); + } catch (CloudRuntimeException e) { + assertEquals(e.getMessage(), "Error while transitioning"); + } + } + + // first attempt landed on the not-ready host and was excluded without ever touching state transition; + // second attempt landed on the ready host and proceeded (reaching the same known stopping point as the + // other orchestrateStart tests confirms execution got past the readiness check this time) + verify(_dpMgr, Mockito.times(2)).planDeployment(any(VirtualMachineProfileImpl.class), any(DataCenterDeployment.class), any(ExcludeList.class), any(DeploymentPlanner.class)); + verify(_stateMachine, Mockito.never()).transitTo(vmInstance, VirtualMachine.Event.OperationRetry, new Pair(vmInstance.getHostId(), 1L), vmInstanceDaoMock); + verify(hostDetailsDao, Mockito.atLeastOnce()).findDetail(2L, Host.HOST_VLAN_FILTERING_ENABLED); + verify(_stateMachine, Mockito.atLeastOnce()).transitTo(vmInstance, VirtualMachine.Event.OperationRetry, new Pair(vmInstance.getHostId(), 2L), vmInstanceDaoMock); + assertEquals(vmInstance.getPodIdToDeployIn(), (Long) destPod.getId()); + } + @Test public void testIsDiskOfferingSuitableForVmSuccess() { Mockito.doReturn(Mockito.mock(DiskOfferingVO.class)).when(diskOfferingDaoMock).findById(anyLong()); diff --git a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java index 66f5b699cc46..5f4aa54019de 100644 --- a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java +++ b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.lang.reflect.Field; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; @@ -89,6 +90,7 @@ import com.cloud.vm.NicProfile; import com.cloud.vm.NicVO; import com.cloud.vm.ReservationContext; +import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.Type; import com.cloud.vm.VirtualMachineProfile; @@ -96,7 +98,12 @@ import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.NicExtraDhcpOptionDao; import com.cloud.vm.dao.NicIpAliasDao; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import com.cloud.vm.dao.NicSecondaryIpDao; +import com.cloud.vm.dao.UserVmDao; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.config.ConfigKey; import junit.framework.TestCase; @@ -143,6 +150,8 @@ public void setUp() { testOrchestrator._ipAddrMgr = mock(IpAddressManager.class); testOrchestrator._entityMgr = mock(EntityManager.class); testOrchestrator.extensionHelper = mock(ExtensionHelper.class); + testOrchestrator._userVmDao = mock(UserVmDao.class); + testOrchestrator._nicNetworkMapDao = mock(NicNetworkMapDao.class); networkExtensionElement = mock(NetworkExtensionElement.class); ReflectionTestUtils.setField(testOrchestrator, "networkExtensionElement", networkExtensionElement); DhcpServiceProvider provider = mock(DhcpServiceProvider.class); @@ -210,6 +219,42 @@ public void testDontRemoveDhcpServiceFromDomainRouter() { verify(testOrchestrator._ntwkSrvcDao, never()).getProviderForServiceInNetwork(network.getId(), Service.Dhcp); verify(testOrchestrator._networksDao, times(1)).findById(nic.getNetworkId()); } + + @Test + public void removeNicCleansUpOrphanedNicNetworkMapAssociations() { + VirtualMachineProfile vm = mock(VirtualMachineProfile.class); + NicVO nic = mock(NicVO.class); + NetworkVO network = mock(NetworkVO.class); + + when(vm.getType()).thenReturn(Type.DomainRouter); + when(nic.getId()).thenReturn(25L); + when(network.getGuruName()).thenReturn(guruName); + when(testOrchestrator._networksDao.findById(nic.getNetworkId())).thenReturn(network); + NicNetworkMapVO association = new NicNetworkMapVO(25L, 210L, "10.1.1.50", null); + when(testOrchestrator._nicNetworkMapDao.listByNicId(25L)).thenReturn(Collections.singletonList(association)); + + testOrchestrator.removeNic(vm, nic); + + verify(testOrchestrator._nicNetworkMapDao).remove(association.getId()); + } + + @Test + public void removeNicsCleansUpOrphanedNicNetworkMapAssociations() { + VirtualMachineProfile vm = mock(VirtualMachineProfile.class); + NicVO nic = mock(NicVO.class); + + when(vm.getId()).thenReturn(13L); + when(nic.getId()).thenReturn(25L); + when(testOrchestrator._nicDao.listByVmId(13L)).thenReturn(Collections.singletonList(nic)); + NicNetworkMapVO association = new NicNetworkMapVO(25L, 210L, "10.1.1.50", null); + when(testOrchestrator._nicNetworkMapDao.listByNicId(25L)).thenReturn(Collections.singletonList(association)); + + testOrchestrator.removeNics(vm); + + verify(testOrchestrator._nicNetworkMapDao).remove(association.getId()); + verify(testOrchestrator._nicDao).remove(25L); + } + @Test public void testDontRemoveDhcpServiceWhenNotProvided() { // make local mocks @@ -1073,4 +1118,54 @@ public void getNetworkElementsIncludingExtensionsReturnsBaseListWhenExtensionHel assertNotNull(result); assertEquals(1, result.size()); } + + private void overrideDefaultConfigValue(final ConfigKey configKey, final String name, final Object o) throws IllegalAccessException, NoSuchFieldException { + Field f = ConfigKey.class.getDeclaredField(name); + f.setAccessible(true); + f.set(configKey, o); + } + + @Test + public void destroyNetworkFailsWhenActivelyAssociatedAsSecondaryNetwork() throws Exception { + overrideDefaultConfigValue(NetworkOrchestrationService.MultiNetworkNicEnabled, "_defaultValue", "true"); + try { + NetworkVO network = mock(NetworkVO.class); + when(network.getId()).thenReturn(1L); + when(network.getDataCenterId()).thenReturn(1L); + when(testOrchestrator._networksDao.findById(1L)).thenReturn(network); + when(testOrchestrator._userVmDao.listByNetworkIdAndStates(1L)).thenReturn(new ArrayList()); + when(testOrchestrator._nicNetworkMapDao.listByNetworkId(1L)).thenReturn(Collections.singletonList(mock(NicNetworkMapVO.class))); + + boolean result = testOrchestrator.destroyNetwork(1L, mock(ReservationContext.class), false); + + assertFalse(result); + verify(testOrchestrator._networksDao, never()).getActiveNicsIn(ArgumentMatchers.anyLong()); + } finally { + overrideDefaultConfigValue(NetworkOrchestrationService.MultiNetworkNicEnabled, "_defaultValue", "false"); + } + } + + @Test + public void destroyNetworkSkipsAssociationCheckWhenFeatureDisabledInZone() throws Exception { + overrideDefaultConfigValue(NetworkOrchestrationService.MultiNetworkNicEnabled, "_defaultValue", "false"); + + NetworkVO network = mock(NetworkVO.class); + when(network.getId()).thenReturn(1L); + when(network.getDataCenterId()).thenReturn(1L); + when(testOrchestrator._networksDao.findById(1L)).thenReturn(network); + when(testOrchestrator._userVmDao.listByNetworkIdAndStates(1L)).thenReturn(new ArrayList()); + DataCenter zone = mock(DataCenter.class); + when(zone.getNetworkType()).thenReturn(DataCenter.NetworkType.Advanced); + when(testOrchestrator._entityMgr.findById(DataCenter.class, 1L)).thenReturn(zone); + + try { + testOrchestrator.destroyNetwork(1L, mock(ReservationContext.class), false); + } catch (Exception ignored) { + // destroyNetwork continues well beyond what this test cares about (network element shutdown, etc., + // none of it mocked here) - only the association-check skip behavior matters for this test + } + + verify(testOrchestrator._nicNetworkMapDao, never()).listByNetworkId(ArgumentMatchers.anyLong()); + verify(testOrchestrator._networksDao).getActiveNicsIn(1L); + } } diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index 32e904c8de38..a842954de3ca 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -149,6 +149,8 @@ import com.cloud.vm.VirtualMachine.Event; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.NicVO; +import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; @@ -186,6 +188,8 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy HostDetailsDao _hostDetailsDao; @Inject private VMTemplateDao templateDao; + @Inject + protected NicDao _nicDao; private static final long ADMIN_ACCOUNT_ROLE_ID = 1l; private static final long INITIAL_RESERVATION_RELEASE_CHECKER_DELAY = 30L * 1000L; // thirty seconds expressed in milliseconds @@ -334,6 +338,7 @@ public DeployDestination planDeployment(VirtualMachineProfile vmProfile, Deploym avoidDisabledResources(vmProfile, dc, avoids); avoidDifferentArchResources(vmProfile, dc, avoids); + avoidHostsNotReadyForMultiNetworkNics(vmProfile, dc, avoids); String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); String uefiFlag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.UefiFlag); @@ -468,6 +473,36 @@ private void avoidDifferentArchResources(VirtualMachineProfile vmProfile, DataCe } } + /** + * A multi-VLAN trunk nic can only be plugged on a host with VLAN filtering enabled on its guest bridge - this is + * a hard capability requirement, not a preference. Excluding every not-ready host up front, in one pass, avoids + * burning retries in the caller's exclude-and-retry loop (VirtualMachineManagerImpl#orchestrateStart) on hosts + * that were never going to work - which matters once a zone has more not-ready hosts than the start.retry budget. + * No-op, and the candidate set is untouched, for a VM with no multi-VLAN trunk nic - the common case. + */ + protected void avoidHostsNotReadyForMultiNetworkNics(VirtualMachineProfile vmProfile, DataCenter dc, ExcludeList avoids) { + boolean hasMultiNetworkNic = _nicDao.listByVmId(vmProfile.getId()).stream().anyMatch(NicVO::getMultiNetwork); + if (!hasMultiNetworkNic) { + return; + } + List candidateHostIds = _hostDao.listEnabledIdsByDataCenterId(dc.getId()); + if (CollectionUtils.isEmpty(candidateHostIds)) { + return; + } + Set readyHostIds = _hostDetailsDao.findByName(Host.HOST_VLAN_FILTERING_ENABLED).stream() + .filter(detail -> Boolean.parseBoolean(detail.getValue())) + .map(DetailVO::getHostId) + .collect(Collectors.toSet()); + List notReadyHostIds = candidateHostIds.stream() + .filter(hostId -> !readyHostIds.contains(hostId)) + .collect(Collectors.toList()); + if (!notReadyHostIds.isEmpty()) { + logger.debug("Excluding {} host(s) without VLAN filtering enabled, since VM [{}] has a multi-VLAN trunk nic", + notReadyHostIds.size(), vmProfile); + avoids.addHostList(notReadyHostIds); + } + } + private DeployDestination deployInVmLastHost(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoids, DeploymentPlanner planner, VirtualMachine vm, DataCenter dc, ServiceOffering offering, int cpuRequested, long ramRequested, boolean volumesRequireEncryption) throws InsufficientServerCapacityException { diff --git a/server/src/main/java/com/cloud/network/NetworkModelImpl.java b/server/src/main/java/com/cloud/network/NetworkModelImpl.java index f47046cdc434..ea5d394cae1b 100644 --- a/server/src/main/java/com/cloud/network/NetworkModelImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkModelImpl.java @@ -149,6 +149,8 @@ import com.cloud.vm.VirtualMachine.Type; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import com.cloud.vm.dao.NicSecondaryIpDao; import com.cloud.vm.dao.VMInstanceDao; @@ -232,6 +234,8 @@ public void setNetworkElements(List networkElements) { @Inject NicSecondaryIpDao _nicSecondaryIpDao; @Inject + NicNetworkMapDao _nicNetworkMapDao; + @Inject ApplicationLoadBalancerRuleDao _appLbRuleDao; @Inject private ProjectAccountDao _projectAccountDao; @@ -1027,6 +1031,23 @@ public Nic getNicInNetwork(long vmId, long networkId) { return _nicDao.findByNtwkIdAndInstanceId(networkId, vmId); } + @Override + public Pair getNicAndIpInNetwork(long vmId, long networkId) { + NicVO nic = _nicDao.findByNtwkIdAndInstanceId(networkId, vmId); + if (nic != null) { + return new Pair<>(nic, new Network.IpAddresses(nic.getIPv4Address(), nic.getIPv6Address())); + } + // networkId isn't any nic's primary - check whether it's one of a trunk nic's associated networks instead, + // where the guest ip lives in nic_network_map rather than on the nic itself + for (NicVO candidate : _nicDao.listByVmId(vmId)) { + NicNetworkMapVO association = _nicNetworkMapDao.findByNicIdAndNetworkId(candidate.getId(), networkId); + if (association != null) { + return new Pair<>(candidate, new Network.IpAddresses(association.getIp4Address(), association.getIp6Address())); + } + } + return null; + } + @Override public String getIpInNetwork(long vmId, long networkId) { Nic guestNic = getNicInNetwork(vmId, networkId); @@ -2349,6 +2370,12 @@ public List getUsedIpsInNetwork(Network network) { //Get ips used by tungsten List tfIps = tungstenGuestNetworkIpAddressDao.listGuestIpAddressByNetworkId(network.getId()); ips.addAll(tfIps); + //Get ips used by nics that are associated with this network as a secondary (trunk) network, not their primary + for (NicNetworkMapVO association : _nicNetworkMapDao.listByNetworkId(network.getId())) { + if (StringUtils.isNotBlank(association.getIp4Address())) { + ips.add(association.getIp4Address()); + } + } return ips; } diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 2853fa96330d..a0b8c12d2096 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -41,12 +41,17 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.host.DetailVO; +import com.cloud.host.HostVO; import com.cloud.resourcelimit.CheckedReservation; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.alert.AlertService; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.address.ReleasePodIpCmdByAdmin; +import org.apache.cloudstack.api.command.admin.network.AssociateNetworkToNicCmd; +import org.apache.cloudstack.api.command.admin.network.ChangeNicPrimaryNetworkCmd; +import org.apache.cloudstack.api.command.admin.network.DisassociateNetworkFromNicCmd; import org.apache.cloudstack.api.command.admin.network.CreateNetworkCmdByAdmin; import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd; import org.apache.cloudstack.api.command.admin.network.ListDedicatedGuestVlanRangesCmd; @@ -90,8 +95,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; +import com.cloud.agent.api.UpdateNicVlanMembershipCommand; import com.cloud.agent.api.to.IpAddressTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.manager.Commands; @@ -126,6 +133,7 @@ import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; +import com.cloud.event.UsageEventVO; import com.cloud.exception.AccountLimitException; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientAddressCapacityException; @@ -137,7 +145,10 @@ import com.cloud.exception.UnsupportedServiceException; import com.cloud.host.Host; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.HypervisorGuru; +import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.network.IpAddress.State; import com.cloud.network.Network.Capability; import com.cloud.network.Network.GuestType; @@ -266,6 +277,8 @@ import com.cloud.vm.VirtualMachineProfileImpl; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import com.cloud.vm.dao.NicSecondaryIpDao; import com.cloud.vm.dao.NicSecondaryIpVO; import com.cloud.vm.dao.UserVmDao; @@ -329,6 +342,12 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C @Inject NicDao _nicDao = null; @Inject + NicNetworkMapDao _nicNetworkMapDao = null; + @Inject + AgentManager _agentMgr; + @Inject + HypervisorGuruManager _hvGuruMgr; + @Inject RulesManager _rulesMgr; List _networkGurus; @Inject @@ -374,6 +393,8 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C @Inject HostDao _hostDao; @Inject + HostDetailsDao _hostDetailsDao; + @Inject DataCenterVnetDao _dcVnetDao; @Inject AccountGuestVlanMapDao _accountGuestVlanMapDao; @@ -886,6 +907,489 @@ public boolean configureNicSecondaryIp(NicSecondaryIp secIp, boolean isZoneSgEna return success; } + @Override + @ActionEvent(eventType = EventTypes.EVENT_NIC_NETWORK_ASSOCIATE, eventDescription = "associating network to nic", create = true) + public Nic associateNetworkToNic(AssociateNetworkToNicCmd cmd) + throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException, InsufficientAddressCapacityException { + NicVO nic = getTrunkEligibleNic(cmd.getNicId()); + VirtualMachine vm = getOwningVm(nic); + if (!NetworkOrchestrationService.MultiNetworkNicEnabled.valueIn(vm.getDataCenterId())) { + throw new InvalidParameterValueException(String.format("Multi-VLAN trunk NICs are not enabled in zone %s", vm.getDataCenterId())); + } + Account caller = CallContext.current().getCallingAccount(); + _accountMgr.checkAccess(caller, null, true, vm); + + boolean vmRunning = vm.getState() == VirtualMachine.State.Running; + if (vmRunning) { + checkHostReadyForVlanTrunk(vm); + } + + associateNetworksInternal(nic, vm, caller, cmd.getNetworkIds(), cmd.getIpAddressesMap()); + + if (vmRunning) { + updateLiveVlanTrunkMembership(vm, nic); + refreshNicVlanMappingMetadata(nic); + } + + return _nicDao.findById(nic.getId()); + } + + /** + * Rejects disassociating a trunk nic's network while an active PF/Static NAT/LB rule still targets the + * association's allocated ip - mirrors the equivalent guard in releaseSecondaryIpFromNic for secondary ips. + */ + private void checkNoActiveRulesOnAssociation(long vmId, long networkId, String ip4Address) { + if (ip4Address == null) { + return; + } + List fwRulesList = _firewallDao.listByNetworkAndPurpose(networkId, Purpose.PortForwarding); + for (FirewallRuleVO rule : fwRulesList) { + if (_portForwardingDao.findByIdAndIp(rule.getId(), ip4Address) != null) { + throw new InvalidParameterValueException(String.format( + "Can't disassociate network %s, its allocated ip %s is associated with a port forwarding rule", networkId, ip4Address)); + } + } + IPAddressVO publicIpVO = _ipAddressDao.findByIpAndNetworkId(networkId, ip4Address); + if (publicIpVO != null) { + throw new InvalidParameterValueException(String.format( + "Can't disassociate network %s, its allocated ip %s is associated with static NAT rule public IP address ID: %s", networkId, ip4Address, publicIpVO)); + } + if (_loadBalancerDao.isLoadBalancerRulesMappedToVmGuestIp(vmId, ip4Address, networkId)) { + throw new InvalidParameterValueException(String.format( + "Can't disassociate network %s, its allocated ip %s is mapped to a load balancing rule", networkId, ip4Address)); + } + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_NIC_NETWORK_ASSOCIATE, eventDescription = "associating network to nic", create = true) + public void associateNetworksToNic(Nic nic, List networkIds, Map requestedIps) + throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException, InsufficientAddressCapacityException { + NicVO nicVO = getTrunkEligibleNic(nic.getId()); + VirtualMachine vm = getOwningVm(nicVO); + if (!NetworkOrchestrationService.MultiNetworkNicEnabled.valueIn(vm.getDataCenterId())) { + throw new InvalidParameterValueException(String.format("Multi-VLAN trunk NICs are not enabled in zone %s", vm.getDataCenterId())); + } + if (vm.getState() != VirtualMachine.State.Stopped) { + throw new CloudRuntimeException(String.format( + "Nic %s's additional networks can only be associated while Instance %s is stopped or being created, not while %s", + nicVO.getUuid(), vm.getUuid(), vm.getState())); + } + Account caller = CallContext.current().getCallingAccount(); + _accountMgr.checkAccess(caller, null, true, vm); + associateNetworksInternal(nicVO, vm, caller, networkIds, requestedIps != null ? requestedIps : Collections.emptyMap()); + } + + /** + * Core of associateNetworkToNic, shared with the deploy-time trunk NIC path (associateNetworksToNic). No live + * update is issued here - callers decide whether/how to push the change to a running Instance. + */ + private void associateNetworksInternal(NicVO nic, VirtualMachine vm, Account caller, List requestedNetworkIds, Map ipAddressesMap) + throws ResourceUnavailableException, InsufficientCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException { + if (requestedNetworkIds == null || requestedNetworkIds.isEmpty()) { + throw new InvalidParameterValueException("networkids must not be empty"); + } + + NetworkVO primaryNetwork = _networksDao.findById(nic.getNetworkId()); + if (primaryNetwork == null) { + throw new CloudRuntimeException(String.format("Nic %s has no primary network", nic.getUuid())); + } + + Pair, List> networkSets = buildResultingNetworkSet(vm, nic, primaryNetwork, requestedNetworkIds); + List resultingSet = networkSets.first(); + List requestedNetworks = networkSets.second(); + + validateTrunkSet(primaryNetwork, resultingSet); + validateNoOverlappingCidrs(resultingSet); + implementRequestedNetworks(requestedNetworks, vm, caller); + validateNoDuplicateVlanAfterImplement(resultingSet); + persistAssociations(nic, vm, requestedNetworks, ipAddressesMap); + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_NIC_NETWORK_DISASSOCIATE, eventDescription = "disassociating network from nic", create = true) + public Nic disassociateNetworkFromNic(DisassociateNetworkFromNicCmd cmd) throws ResourceUnavailableException { + NicVO nic = getTrunkEligibleNic(cmd.getNicId()); + VirtualMachine vm = getOwningVm(nic); + if (!NetworkOrchestrationService.MultiNetworkNicEnabled.valueIn(vm.getDataCenterId())) { + throw new InvalidParameterValueException(String.format("Multi-VLAN trunk NICs are not enabled in zone %s", vm.getDataCenterId())); + } + Account caller = CallContext.current().getCallingAccount(); + _accountMgr.checkAccess(caller, null, true, vm); + + long targetNetworkId = cmd.getNetworkId(); + if (targetNetworkId == nic.getNetworkId()) { + throw new InvalidParameterValueException(String.format( + "Network %s is nic %s's primary network and cannot be disassociated; add a new nic with the desired primary network instead", + targetNetworkId, nic.getUuid())); + } + + NicNetworkMapVO association = _nicNetworkMapDao.findByNicIdAndNetworkId(nic.getId(), targetNetworkId); + if (association == null) { + throw new InvalidParameterValueException(String.format("Nic %s is not associated with network %s", nic.getUuid(), targetNetworkId)); + } + checkNoActiveRulesOnAssociation(vm.getId(), targetNetworkId, association.getIp4Address()); + + boolean vmRunning = vm.getState() == VirtualMachine.State.Running; + if (vmRunning) { + checkHostReadyForVlanTrunk(vm); + } + + // guest IPs allocated for an association are never persisted anywhere but nic_network_map itself + // (allocateGuestIP/allocateGuestIpv6 compute availability fresh from live DB state each time), + // so removing this row is itself the release + _nicNetworkMapDao.remove(association.getId()); + + if (vmRunning) { + updateLiveVlanTrunkMembership(vm, nic); + refreshNicVlanMappingMetadata(nic); + } + + return _nicDao.findById(nic.getId()); + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_NIC_PRIMARY_NETWORK_CHANGE, eventDescription = "changing nic's primary network", create = true) + public Nic changeNicPrimaryNetwork(ChangeNicPrimaryNetworkCmd cmd) { + NicVO nic = getTrunkEligibleNic(cmd.getNicId()); + VirtualMachine vm = getOwningVm(nic); + if (!NetworkOrchestrationService.MultiNetworkNicEnabled.valueIn(vm.getDataCenterId())) { + throw new InvalidParameterValueException(String.format("Multi-VLAN trunk NICs are not enabled in zone %s", vm.getDataCenterId())); + } + Account caller = CallContext.current().getCallingAccount(); + _accountMgr.checkAccess(caller, null, true, vm); + + if (vm.getState() != VirtualMachine.State.Stopped) { + throw new InvalidParameterValueException(String.format( + "Nic %s's primary network can only be changed while Instance %s is stopped, since the nic is rebuilt fresh on its next start", + nic.getUuid(), vm.getUuid())); + } + + long newPrimaryNetworkId = cmd.getNetworkId(); + if (newPrimaryNetworkId == nic.getNetworkId()) { + throw new InvalidParameterValueException(String.format("Network %s is already nic %s's primary network", newPrimaryNetworkId, nic.getUuid())); + } + + NicNetworkMapVO association = _nicNetworkMapDao.findByNicIdAndNetworkId(nic.getId(), newPrimaryNetworkId); + if (association == null) { + throw new InvalidParameterValueException(String.format( + "Network %s must already be associated with nic %s before it can become the primary; associate it first", + newPrimaryNetworkId, nic.getUuid())); + } + + long oldPrimaryNetworkId = nic.getNetworkId(); + String oldPrimaryIp4 = nic.getIPv4Address(); + String oldPrimaryIp6 = nic.getIPv6Address(); + + // reuses the IP already reserved for this network as an association - no new allocation needed, + // it's simply moving which table (nics vs nic_network_map) tracks the same reservation + _nicNetworkMapDao.remove(association.getId()); + _nicNetworkMapDao.persist(new NicNetworkMapVO(nic.getId(), oldPrimaryNetworkId, oldPrimaryIp4, oldPrimaryIp6)); + + nic.setNetworkId(newPrimaryNetworkId); + nic.setIPv4Address(association.getIp4Address()); + nic.setIPv6Address(association.getIp6Address()); + _nicDao.update(nic.getId(), nic); + + return _nicDao.findById(nic.getId()); + } + + private void updateLiveVlanTrunkMembership(VirtualMachine vm, NicVO nic) throws ResourceUnavailableException { + if (vm.getHostId() == null) { + throw new CloudRuntimeException(String.format("Instance %s is Running but has no host assigned", vm.getUuid())); + } + DataCenter dc = _entityMgr.findById(DataCenter.class, vm.getDataCenterId()); + NicProfile nicProfile = _networkModel.getNicProfile(vm, nic, dc); + HypervisorGuru guru = _hvGuruMgr.getGuru(vm.getHypervisorType()); + NicTO nicTO = guru.toNicTO(nicProfile); + UpdateNicVlanMembershipCommand updateCmd = new UpdateNicVlanMembershipCommand(nicTO, vm.getInstanceName()); + Answer answer = _agentMgr.easySend(vm.getHostId(), updateCmd); + if (answer == null || !answer.getResult()) { + throw new CloudRuntimeException(String.format("Failed to update live VLAN trunk membership for nic %s: %s", + nic.getUuid(), answer != null ? answer.getDetails() : "no response from host")); + } + } + + /** + * Resends the nic's associated-network VLAN mapping metadata to its primary network's router(s), so the + * guest sees an up-to-date list immediately rather than only after that router's next restart. Best-effort: + * a failure here leaves guest-facing metadata briefly stale but must not fail the association change itself, + * since the actual nic_network_map/live VLAN membership updates (the correctness-critical parts) already + * succeeded by the time this runs. + */ + private void refreshNicVlanMappingMetadata(NicVO nic) { + UserVmVO vm = _userVmDao.findById(nic.getInstanceId()); + if (vm == null) { + return; + } + _userVmDao.loadDetails(vm); + List routers = routerDao.findByNetwork(nic.getNetworkId()); + for (DomainRouterVO router : routers) { + if (router.getState() != VirtualMachine.State.Running) { + continue; + } + try { + Commands cmds = new Commands(Command.OnError.Stop); + commandSetupHelper.createVmDataCommand(router, vm, nic, vm.getDetail("SSH.PublicKey"), cmds); + networkHelper.sendCommandsToRouter(router, cmds); + } catch (Exception e) { + logger.warn("Failed to refresh nic VLAN mapping metadata for nic {} on router {}: {}", nic.getUuid(), router.getUuid(), e.getMessage(), e); + } + } + } + + private NicVO getTrunkEligibleNic(long nicId) { + NicVO nic = _nicDao.findById(nicId); + if (nic == null) { + throw new InvalidParameterValueException(String.format("Unable to find nic with id %d", nicId)); + } + if (nic.getVmType() != VirtualMachine.Type.User) { + throw new InvalidParameterValueException("Multi-VLAN trunk nics are only supported on User Instances"); + } + return nic; + } + + private VirtualMachine getOwningVm(NicVO nic) { + VirtualMachine vm = _userVmDao.findById(nic.getInstanceId()); + if (vm == null) { + throw new InvalidParameterValueException(String.format("Unable to find the Instance owning nic %s", nic.getUuid())); + } + return vm; + } + + private Pair, List> buildResultingNetworkSet(VirtualMachine vm, NicVO nic, NetworkVO primaryNetwork, List requestedNetworkIds) { + List existingAssociations = _nicNetworkMapDao.listByNicId(nic.getId()); + Set associatedNetworkIds = new HashSet<>(); + associatedNetworkIds.add(primaryNetwork.getId()); + List resultingSet = new ArrayList<>(); + resultingSet.add(primaryNetwork); + for (NicNetworkMapVO association : existingAssociations) { + associatedNetworkIds.add(association.getNetworkId()); + NetworkVO existingNetwork = _networksDao.findById(association.getNetworkId()); + if (existingNetwork != null) { + resultingSet.add(existingNetwork); + } + } + + Set networksOnOtherNics = getNetworksTouchedByOtherNics(vm, nic.getId()); + + List requestedNetworks = new ArrayList<>(); + for (Long requestedNetworkId : requestedNetworkIds) { + if (associatedNetworkIds.contains(requestedNetworkId)) { + NetworkVO alreadyAssociated = _networksDao.findById(requestedNetworkId); + throw new InvalidParameterValueException(String.format("Nic %s is already associated with network %s", nic.getUuid(), + alreadyAssociated != null ? alreadyAssociated.getUuid() : requestedNetworkId)); + } + if (networksOnOtherNics.contains(requestedNetworkId)) { + NetworkVO conflictingNetwork = _networksDao.findById(requestedNetworkId); + throw new InvalidParameterValueException(String.format( + "Instance %s already has network %s on a different nic; a network can only be reachable through one of an Instance's nics at a time", + vm.getUuid(), conflictingNetwork != null ? conflictingNetwork.getUuid() : requestedNetworkId)); + } + NetworkVO requestedNetwork = _networksDao.findById(requestedNetworkId); + if (requestedNetwork == null) { + throw new InvalidParameterValueException(String.format("Unable to find network with id %d", requestedNetworkId)); + } + requestedNetworks.add(requestedNetwork); + resultingSet.add(requestedNetwork); + } + return new Pair<>(resultingSet, requestedNetworks); + } + + /** + * Networks already reachable through any of the vm's other nics - either as that nic's primary, or one of its + * own trunk associations. Mirrors VirtualMachineManagerImpl.checkIfNetworkExistsForUserVM's existing "at most + * one nic per network" rule for User VMs (trunk nics are already restricted to User Instances - see + * getTrunkEligibleNic), extended to also cover nic_network_map, which that check has no visibility into. + */ + private Set getNetworksTouchedByOtherNics(VirtualMachine vm, long excludingNicId) { + Set networkIds = new HashSet<>(); + for (NicVO otherNic : _nicDao.listByVmId(vm.getId())) { + if (otherNic.getId() == excludingNicId) { + continue; + } + networkIds.add(otherNic.getNetworkId()); + for (NicNetworkMapVO association : _nicNetworkMapDao.listByNicId(otherNic.getId())) { + networkIds.add(association.getNetworkId()); + } + } + return networkIds; + } + + private void validateTrunkSet(NetworkVO primaryNetwork, List resultingSet) { + for (NetworkVO network : resultingSet) { + validateNetworkFitsTrunk(primaryNetwork, network); + } + } + + /** + * A CIDR is a network's own fixed property (unlike its VLAN, which may still be auto-allocated at this + * point), so this can run before implementation, on the same resultingSet buildResultingNetworkSet already + * built. Two associated networks sharing address space is not a VLAN-isolation gap, but it still leaves a + * guest unable to unambiguously route between them - the same failure any multi-homed Linux host hits when + * two of its interfaces sit on identical subnets. + */ + private void validateNoOverlappingCidrs(List resultingSet) { + List> networkCidrs = new ArrayList<>(); + for (NetworkVO network : resultingSet) { + String cidr = network.getCidr(); + if (cidr == null) { + continue; + } + for (String range : cidr.split(",")) { + networkCidrs.add(new Pair<>(network, range.trim())); + } + } + for (int i = 0; i < networkCidrs.size(); i++) { + for (int j = i + 1; j < networkCidrs.size(); j++) { + NetworkVO networkA = networkCidrs.get(i).first(); + NetworkVO networkB = networkCidrs.get(j).first(); + if (networkA.getId() == networkB.getId()) { + continue; + } + String cidrA = networkCidrs.get(i).second(); + String cidrB = networkCidrs.get(j).second(); + if (NetUtils.isNetworksOverlap(cidrA, cidrB)) { + throw new InvalidParameterValueException(String.format( + "Networks %s and %s have overlapping subnets (%s and %s) and cannot be associated with the same nic; " + + "a guest cannot unambiguously route between associated networks that share address space", + networkA.getUuid(), networkB.getUuid(), cidrA, cidrB)); + } + } + } + } + + private void implementRequestedNetworks(List requestedNetworks, VirtualMachine vm, Account caller) + throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { + for (NetworkVO requestedNetwork : requestedNetworks) { + implementNetworkIfNeeded(requestedNetwork, vm, caller); + } + } + + private void validateNoDuplicateVlanAfterImplement(List resultingSet) { + // re-fetch: implementNetworkIfNeeded may have just assigned a broadcastUri to one of the requested + // networks, and existing associations are re-read too rather than trusted stale + List implementedNetworks = new ArrayList<>(); + for (NetworkVO network : resultingSet) { + NetworkVO refreshed = _networksDao.findById(network.getId()); + if (refreshed == null) { + throw new CloudRuntimeException(String.format("Network %s disappeared during association", network.getUuid())); + } + implementedNetworks.add(refreshed); + } + validateNoDuplicateVlan(implementedNetworks); + } + + private void persistAssociations(NicVO nic, VirtualMachine vm, List requestedNetworks, Map requestedIps) + throws InsufficientAddressCapacityException { + Map safeRequestedIps = requestedIps != null ? requestedIps : Collections.emptyMap(); + for (NetworkVO network : requestedNetworks) { + NetworkVO refreshed = _networksDao.findById(network.getId()); + Pair allocatedIps = allocateIpsForAssociation(refreshed, safeRequestedIps.get(network.getId())); + _nicNetworkMapDao.persist(new NicNetworkMapVO(nic.getId(), network.getId(), allocatedIps.first(), allocatedIps.second())); + } + if (!nic.getMultiNetwork()) { + nic.setMultiNetwork(true); + _nicDao.update(nic.getId(), nic); + // this nic's own still-open usage row(s) predate network_id existing on this table and were never + // backfilled in bulk; backfill them now, unconditionally, since a later association on this same + // nic could otherwise introduce an offering collision this row can no longer be told apart from. + // Published as a usage event rather than written directly: this request thread already has an + // ambient `cloud` DB transaction open, and cloud_usage is a separate database - the usage job + // consumes this event and performs the actual write on its own thread, same as every other + // usage_network_offering row in this feature. + Map details = Collections.singletonMap(UsageEventVO.DynamicParameters.networkId.name(), String.valueOf(nic.getNetworkId())); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NIC_NETWORK_ID_BACKFILL, vm.getAccountId(), vm.getDataCenterId(), nic.getId(), + Long.toString(nic.getId()), null, null, null, VirtualMachine.class.getName(), vm.getUuid(), details); + } + } + + private Pair allocateIpsForAssociation(NetworkVO network, Network.IpAddresses requested) throws InsufficientAddressCapacityException { + if (network.getGuestType() == Network.GuestType.L2) { + return new Pair<>(null, null); + } + String requestedIp4 = requested != null ? requested.getIp4Address() : null; + String requestedIp6 = requested != null ? requested.getIp6Address() : null; + String ip4Address = _ipAddrMgr.allocateGuestIP(network, requestedIp4); + String ip6Address = StringUtils.isNotBlank(network.getIp6Cidr()) ? ipv6AddrMgr.allocateGuestIpv6(network, requestedIp6) : null; + return new Pair<>(ip4Address, ip6Address); + } + + private void checkHostReadyForVlanTrunk(VirtualMachine vm) { + if (vm.getHostId() == null) { + throw new CloudRuntimeException(String.format("Instance %s is Running but has no host assigned", vm.getUuid())); + } + DetailVO vlanFilteringDetail = _hostDetailsDao.findDetail(vm.getHostId(), Host.HOST_VLAN_FILTERING_ENABLED); + if (vlanFilteringDetail == null || !Boolean.parseBoolean(vlanFilteringDetail.getValue())) { + HostVO host = _hostDao.findById(vm.getHostId()); + throw new InvalidParameterValueException(String.format( + "Host %s does not have VLAN filtering enabled on its guest bridge, so a multi-VLAN trunk nic cannot be associated on this Instance while it is running", + host != null ? host.getUuid() : vm.getHostId())); + } + } + + private void implementNetworkIfNeeded(NetworkVO network, VirtualMachine vm, Account caller) + throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { + if (network.getState() == Network.State.Implemented) { + return; + } + DataCenter dc = _entityMgr.findById(DataCenter.class, network.getDataCenterId()); + Host host = vm.getHostId() != null ? _hostDao.findById(vm.getHostId()) : null; + DeployDestination dest = new DeployDestination(dc, null, null, host); + User callerUser = _accountMgr.getActiveUser(CallContext.current().getCallingUserId()); + ReservationContext context = new ReservationContextImpl(null, null, callerUser, caller); + Pair implemented = _networkMgr.implementNetwork(network.getId(), dest, context); + if (implemented == null || implemented.first() == null) { + throw new CloudRuntimeException(String.format("Failed to implement network %s as part of associating it to a nic", network.getUuid())); + } + } + + private void validateNoDuplicateVlan(List resultingSet) { + Map seenBroadcastUris = new HashMap<>(); + for (NetworkVO network : resultingSet) { + URI broadcastUri = network.getBroadcastUri(); + if (broadcastUri == null) { + throw new CloudRuntimeException(String.format("Network %s has no VLAN assigned after implementation; this should not happen", network.getUuid())); + } + NetworkVO conflicting = seenBroadcastUris.put(broadcastUri, network); + if (conflicting != null) { + throw new InvalidParameterValueException(String.format("Networks %s and %s share the same VLAN (%s) and cannot be associated with the same nic", + conflicting.getUuid(), network.getUuid(), broadcastUri)); + } + } + } + + private void validateNetworkFitsTrunk(NetworkVO primaryNetwork, NetworkVO network) { + if (network.getGuestType() != primaryNetwork.getGuestType()) { + throw new InvalidParameterValueException(String.format( + "All networks associated with one nic must be the same type; network %s is %s but the nic's primary network %s is %s", + network.getUuid(), network.getGuestType(), primaryNetwork.getUuid(), primaryNetwork.getGuestType())); + } + if (!Objects.equals(network.getPhysicalNetworkId(), primaryNetwork.getPhysicalNetworkId())) { + throw new InvalidParameterValueException(String.format( + "All networks associated with one nic must be on the same physical network; network %s is not on the same physical network as %s", + network.getUuid(), primaryNetwork.getUuid())); + } + if (!Objects.equals(network.getDataCenterId(), primaryNetwork.getDataCenterId())) { + throw new InvalidParameterValueException(String.format( + "All networks associated with one nic must be in the same zone; network %s is not in the same zone as %s", + network.getUuid(), primaryNetwork.getUuid())); + } + if (primaryNetwork.getGuestType() == Network.GuestType.Isolated && !Objects.equals(network.getVpcId(), primaryNetwork.getVpcId())) { + throw new InvalidParameterValueException(String.format( + "A nic may only be associated with tiers of the same VPC (or all non-VPC networks); network %s does not match the VPC of %s", + network.getUuid(), primaryNetwork.getUuid())); + } + if (network.getBroadcastDomainType() != BroadcastDomainType.Vlan) { + throw new InvalidParameterValueException(String.format("Multi-VLAN trunk nics only support VLAN-isolated networks; network %s is %s", + network.getUuid(), network.getBroadcastDomainType())); + } + if (_networkModel.isSecurityGroupSupportedInNetwork(network)) { + throw new InvalidParameterValueException(String.format( + "Network %s has security groups enabled; a nic associated with multiple networks may not use a security-group-enabled network", + network.getUuid())); + } + } + /** * It allocates a secondary IP alias on the NIC. It can be either an Ipv4 or an Ipv6 or even both, according to the given IpAddresses object. */ @@ -3583,12 +4087,20 @@ public void doInTransactionWithoutResult(TransactionStatus status) { updateNetworkIpv6(network, networkOfferingId); network.setNetworkOfferingId(networkOfferingId); _networksDao.update(networkId, network, newSvcProviders); - // get all nics using this network + // get all nics using this network, either as their primary network or as an + // associated network of a multi-VLAN trunk nic // log remove usage events for old offering // log assign usage events for new offering + Set nicIds = new HashSet<>(); List nics = _nicDao.listByNetworkId(networkId); for (NicVO nic : nics) { - if (Nic.ReservationStrategy.PlaceHolder.equals(nic.getReservationStrategy())) { + nicIds.add(nic.getId()); + } + nicIds.addAll(_nicNetworkMapDao.listNicIdsByNetworkId(networkId)); + Map usageDetails = Collections.singletonMap(UsageEventVO.DynamicParameters.networkId.name(), String.valueOf(networkId)); + for (Long nicId : nicIds) { + NicVO nic = _nicDao.findById(nicId); + if (nic == null || Nic.ReservationStrategy.PlaceHolder.equals(nic.getReservationStrategy())) { continue; } long vmId = nic.getInstanceId(); @@ -3599,10 +4111,10 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } long isDefault = (nic.isDefaultNic()) ? 1 : 0; String nicIdString = Long.toString(nic.getId()); - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), nicIdString, oldNetworkOfferingId, - null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay()); - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), nicIdString, networkOfferingId, - null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay()); + UsageEventUtils.publishUsageEventWithDetails(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), nicIdString, oldNetworkOfferingId, + null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), usageDetails, vm.isDisplay()); + UsageEventUtils.publishUsageEventWithDetails(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), nicIdString, networkOfferingId, + null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), usageDetails, vm.isDisplay()); } } }); diff --git a/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java b/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java index 38f405fe1e4d..b0be0df4e0b0 100644 --- a/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java +++ b/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java @@ -977,8 +977,7 @@ public void updateLBHealthChecks(Scheme scheme) throws ResourceUnavailableExcept List lbVmMaps = _lb2VmMapDao.listByLoadBalancerId(ulb.getId()); for (LoadBalancerVMMapVO lbVmMap : lbVmMaps) { UserVm vm = _vmDao.findById(lbVmMap.getInstanceId()); - Nic nic = _nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(ulb.getNetworkId(), vm.getId()); - String dstIp = lbVmMap.getInstanceIp() == null ? nic.getIPv4Address(): lbVmMap.getInstanceIp(); + String dstIp = lbVmMap.getInstanceIp() == null ? getLbDestinationIp(vm.getId(), ulb.getNetworkId()) : lbVmMap.getInstanceIp(); for (int i = 0; i < lbto.getDestinations().length; i++) { LoadBalancerTO.DestinationTO des = lbto.getDestinations()[i]; @@ -1097,9 +1096,10 @@ public boolean assignToLoadBalancer(long loadBalancerId, List instanceIds, Network loadBalancerNetwork = _networkDao.findById(loadBalancer.getNetworkId()); _accountMgr.checkAccess(vmOwner, SecurityChecker.AccessType.UseEntry, false, loadBalancerNetwork); - Nic vmNicInLb = getVmNicInLoadBalancer(vm, loadBalancer, loadBalancerNetwork, vmIdNetworkMap, vmOwner); + Pair vmNicAndIpInLb = getVmNicInLoadBalancer(vm, loadBalancer, loadBalancerNetwork, vmIdNetworkMap, vmOwner); + Nic vmNicInLb = vmNicAndIpInLb.first(); - String priIp = vmNicInLb.getIPv4Address(); + String priIp = vmNicAndIpInLb.second().getIp4Address(); if (existingVmIdIps.containsKey(instanceId)) { // now check for ip address @@ -1123,13 +1123,14 @@ public boolean assignToLoadBalancer(long loadBalancerId, List instanceIds, if (vmIpsList != null) { - //check if the ips belongs to nic secondary ip + //check if the ips belongs to nic secondary ip - scoped by network, since the same nic can carry + //secondary ips across more than one of its associated networks for (String ip: vmIpsList) { // skip the primary ip from vm secondary ip comparisions if (ip.equals(priIp)) { continue; } - if(_nicSecondaryIpDao.findByIp4AddressAndNicId(ip,vmNicInLb.getId()) == null) { + if(_nicSecondaryIpDao.findByIp4AddressAndNetworkIdAndInstanceId(loadBalancer.getNetworkId(), instanceId, ip) == null) { throw new InvalidParameterValueException("Instance IP "+ ip + " specified does not belong to " + "NIC in Network " + vmNicInLb.getNetworkId()); } @@ -1221,11 +1222,11 @@ public void doInTransactionWithoutResult(TransactionStatus status) { return success; } - protected Nic getVmNicInLoadBalancer(UserVm vm, LoadBalancerVO loadBalancer, Network loadBalancerNetwork, Map vmIdNetworkMap, Account vmOwner) { + protected Pair getVmNicInLoadBalancer(UserVm vm, LoadBalancerVO loadBalancer, Network loadBalancerNetwork, Map vmIdNetworkMap, Account vmOwner) { boolean isVpcConserveModeEnabled = _vpcMgr.isNetworkOnVpcEnabledConserveMode(loadBalancerNetwork); boolean isNetworkPassedVpcConserveMode = isVpcConserveModeEnabled && MapUtils.isNotEmpty(vmIdNetworkMap) && vmIdNetworkMap.containsKey(vm.getId()); - Nic vmNicInLb = isNetworkPassedVpcConserveMode ? + Pair vmNicInLb = isNetworkPassedVpcConserveMode ? getNicForVmInVpcConserveModeTierNetwork(vm, vmIdNetworkMap, vmOwner, loadBalancerNetwork) : getNicForVmLbNetwork(vm, loadBalancer); @@ -1241,29 +1242,40 @@ protected Nic getVmNicInLoadBalancer(UserVm vm, LoadBalancerVO loadBalancer, Net } /** - * For Isolated Networks or Network tiers of VPCs not using Conserve mode, use the same network as the load balancer - * @return the nic of the VM in the load balancer network + * For Isolated Networks or Network tiers of VPCs not using Conserve mode, use the same network as the load balancer - + * either as the vm's primary nic, or (for a trunk nic) one of its associated networks. + * @return the nic and guest ip of the VM in the load balancer network */ - protected Nic getNicForVmLbNetwork(UserVm vm, LoadBalancerVO loadBalancer) { - List nics = _networkModel.getNics(vm.getId()); - for (Nic nic : nics) { - if (nic.getNetworkId() == loadBalancer.getNetworkId()) { - return nic; - } - } - return null; + protected Pair getNicForVmLbNetwork(UserVm vm, LoadBalancerVO loadBalancer) { + return _networkModel.getNicAndIpInNetwork(vm.getId(), loadBalancer.getNetworkId()); } /** * On VPC Conserve Mode, VMs from multiple VPC networks tiers can be assigned to the same load balancer. - * @return the nic of the VM in the specified tier network in `vmIdNetworkMap` + * @return the nic and guest ip of the VM in the specified tier network in `vmIdNetworkMap` */ - protected Nic getNicForVmInVpcConserveModeTierNetwork(UserVm vm, Map vmIdNetworkMap, Account vmOwner, Network loadBalancerNetwork) { + protected Pair getNicForVmInVpcConserveModeTierNetwork(UserVm vm, Map vmIdNetworkMap, Account vmOwner, Network loadBalancerNetwork) { Long vmNetworkId = vmIdNetworkMap.get(vm.getId()); Network vmNetwork = _networkDao.findById(vmNetworkId); _accountMgr.checkAccess(vmOwner, SecurityChecker.AccessType.UseEntry, false, vmNetwork); checkNetworkBelongsToLoadBalancerVpc(vmNetwork, loadBalancerNetwork); - return _networkModel.getNicInNetwork(vm.getId(), vmNetworkId); + return _networkModel.getNicAndIpInNetwork(vm.getId(), vmNetworkId); + } + + /** + * Resolves the guest ip a VM answers on for a LB rule's network - the nic's own primary network match when one + * exists, or a trunk nic's association (from nic_network_map) otherwise. Falls back further to a since-removed + * nic's last-known address (matching this method's replaced call sites' original use of + * NicDao.findByInstanceIdAndNetworkIdIncludingRemoved) so a VM whose nic was unplugged after being added to + * this LB rule still resolves to its last-known ip, same as before. Returns null only if none of these match. + */ + private String getLbDestinationIp(long vmId, long networkId) { + Pair nicAndIp = _networkModel.getNicAndIpInNetwork(vmId, networkId); + if (nicAndIp != null) { + return nicAndIp.second().getIp4Address(); + } + Nic removedNic = _nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(networkId, vmId); + return removedNic != null ? removedNic.getIPv4Address() : null; } protected void checkNetworkBelongsToLoadBalancerVpc(Network vmNetwork, Network loadBalancerNetwork) { @@ -2277,8 +2289,7 @@ public List getExistingDestinations(long lbId) { String dstIp = null; for (LoadBalancerVMMapVO lbVmMap : lbVmMaps) { UserVm vm = _vmDao.findById(lbVmMap.getInstanceId()); - Nic nic = _nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(lb.getNetworkId(), vm.getId()); - dstIp = lbVmMap.getInstanceIp() == null ? nic.getIPv4Address(): lbVmMap.getInstanceIp(); + dstIp = lbVmMap.getInstanceIp() == null ? getLbDestinationIp(vm.getId(), lb.getNetworkId()) : lbVmMap.getInstanceIp(); LbDestination lbDst = new LbDestination(lb.getDefaultPortStart(), lb.getDefaultPortEnd(), dstIp, lbVmMap.isRevoke()); dstList.add(lbDst); } @@ -2754,9 +2765,10 @@ public Map getLbInstances(long lbId) { for (LoadBalancerVMMapVO lbVmMap : lbVmMaps) { UserVm vm = _vmDao.findById(lbVmMap.getInstanceId()); - Nic nic = _nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(lb.getNetworkId(), vm.getId()); - Ip ip = new Ip(nic.getIPv4Address()); - dstList.put(ip, vm); + String ip4Address = getLbDestinationIp(vm.getId(), lb.getNetworkId()); + if (ip4Address != null) { + dstList.put(new Ip(ip4Address), vm); + } } return dstList; } diff --git a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java index 58b2892e09c3..a09e2f48381b 100644 --- a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java +++ b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java @@ -157,7 +157,10 @@ import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.NicIpAliasDao; import com.cloud.vm.dao.NicIpAliasVO; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import com.cloud.vm.dao.UserVmDao; +import com.google.gson.Gson; public class CommandSetupHelper { @@ -171,6 +174,8 @@ public class CommandSetupHelper { @Inject private NicDao _nicDao; @Inject + private NicNetworkMapDao _nicNetworkMapDao; + @Inject private NetworkDao _networkDao; @Inject private DomainRouterDao _routerDao; @@ -251,6 +256,8 @@ public void createVmDataCommand(final VirtualRouter router, final UserVm vm, fin vmDataCommand.addVmData(NetworkModel.METATDATA_DIR, NetworkModel.CLOUD_NAME_FILE, customCloudName); } + addNicVlanMappingToVmData(vmDataCommand, vm, nic); + cmds.addCommand("vmdata", vmDataCommand); } } @@ -1362,6 +1369,38 @@ protected void addUserDataDetailsToCommand(VmDataCommand cmd, String userDataDet } } + protected void addNicVlanMappingToVmData(VmDataCommand cmd, UserVm vm, NicVO nic) { + if (!nic.getMultiNetwork() || !VirtualMachineManager.AllowExposeNicVlanMapping.valueIn(vm.getAccountId())) { + return; + } + List associations = _nicNetworkMapDao.listByNicId(nic.getId()); + List mappings = new ArrayList<>(); + for (NicNetworkMapVO association : associations) { + NetworkVO associatedNetwork = _networkDao.findById(association.getNetworkId()); + if (associatedNetwork == null || associatedNetwork.getBroadcastDomainType() != BroadcastDomainType.Vlan || associatedNetwork.getBroadcastUri() == null) { + continue; + } + String vlanTag = BroadcastDomainType.Vlan.getValueFrom(associatedNetwork.getBroadcastUri()); + mappings.add(new NicVlanMappingEntry(associatedNetwork.getUuid(), associatedNetwork.getName(), vlanTag)); + } + // empty-string data is vmdata.py's signal to delete a previously-written file: once a nic has been + // trunked, its last association being removed must actively clear any stale file left from before, + // not just skip writing - the guest must never see associations that no longer exist + cmd.addVmData(NetworkModel.METATDATA_DIR, NetworkModel.NIC_VLAN_MAPPING_FILE, mappings.isEmpty() ? "" : new Gson().toJson(mappings)); + } + + private static final class NicVlanMappingEntry { + private final String networkId; + private final String networkName; + private final String vlan; + + private NicVlanMappingEntry(String networkId, String networkName, String vlan) { + this.networkId = networkId; + this.networkName = networkName; + this.vlan = vlan; + } + } + private NicVO findGatewayIp(final long userVmId) { final NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); return defaultNic; diff --git a/server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java b/server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java index bb3ee8d76248..b79d357c7fe6 100644 --- a/server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java +++ b/server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java @@ -276,21 +276,23 @@ public PortForwardingRule createPortForwardingRule(final PortForwardingRule rule throw new InvalidParameterValueException(String.format("Invalid user vm: %s", vm)); } - // Verify that vm has nic in the network + // Verify that vm has nic in the network - either as its primary, or (for a trunk nic) as one of + // its associated networks, whose guest ip lives in nic_network_map rather than on the nic itself Ip dstIp = rule.getDestinationIpAddress(); - guestNic = _networkModel.getNicInNetwork(vmId, networkId); - if (guestNic == null || guestNic.getIPv4Address() == null) { + Pair nicAndIp = _networkModel.getNicAndIpInNetwork(vmId, networkId); + if (nicAndIp == null || nicAndIp.second().getIp4Address() == null) { throw new InvalidParameterValueException("Vm doesn't belong to network associated with ipAddress"); - } else { - dstIp = new Ip(guestNic.getIPv4Address()); } + guestNic = nicAndIp.first(); + dstIp = new Ip(nicAndIp.second().getIp4Address()); if (vmIp != null) { //vm ip is passed so it can be primary or secondary ip addreess. if (!dstIp.equals(vmIp)) { //the vm ip is secondary ip to the nic. - // is vmIp is secondary ip or not - NicSecondaryIp secondaryIp = _nicSecondaryDao.findByIp4AddressAndNicId(vmIp.toString(), guestNic.getId()); + // is vmIp is secondary ip or not - scoped by network, since the same nic can carry + // secondary ips across more than one of its associated networks + NicSecondaryIp secondaryIp = _nicSecondaryDao.findByIp4AddressAndNetworkIdAndInstanceId(networkId, vmId, vmIp.toString()); if (secondaryIp == null) { throw new InvalidParameterValueException("IP Address is not in the VM nic's network "); } @@ -476,12 +478,14 @@ private boolean enableStaticNat(long ipId, long vmId, long networkId, boolean is throw new InvalidParameterValueException("Unable to find network by id"); } - // Check that vm has a nic in the network - guestNic = _networkModel.getNicInNetwork(vmId, networkId); - if (guestNic == null) { + // Check that vm has a nic in the network - either as its primary, or (for a trunk nic) as one of its + // associated networks, whose guest ip lives in nic_network_map rather than on the nic itself + Pair nicAndIp = _networkModel.getNicAndIpInNetwork(vmId, networkId); + if (nicAndIp == null) { throw new InvalidParameterValueException("Vm doesn't belong to the network with specified id"); } - dstIp = guestNic.getIPv4Address(); + guestNic = nicAndIp.first(); + dstIp = nicAndIp.second().getIp4Address(); if (!_networkModel.areServicesSupportedInNetwork(network.getId(), Service.StaticNat)) { throw new InvalidParameterValueException("Unable to create static nat rule; StaticNat service is not " + "supported in network with specified id"); @@ -573,8 +577,9 @@ private boolean enableStaticNat(long ipId, long vmId, long networkId, boolean is if (!secondaryIpSet) { throw new InvalidParameterValueException("VM ip " + vmGuestIp + " address not belongs to the vm"); } - //check the ip belongs to the vm or not - nicSecIp = _nicSecondaryDao.findByIp4AddressAndNicId(vmGuestIp, guestNic.getId()); + //check the ip belongs to the vm or not - scoped by network, since the same nic can carry + //secondary ips across more than one of its associated networks + nicSecIp = _nicSecondaryDao.findByIp4AddressAndNetworkIdAndInstanceId(networkId, vmId, vmGuestIp); if (nicSecIp == null) { throw new InvalidParameterValueException("VM ip " + vmGuestIp + " address not belongs to the vm"); } @@ -1479,9 +1484,10 @@ protected List createStaticNatForIp(IpAddress sourceIp, Account calle } // create new static nat rule - // Get nic IP4 address + // Get nic IP4 address - a null primary-nic match doesn't necessarily mean the vm has no nic in this + // network; it may be a trunk nic's associated (non-primary) network instead Nic guestNic = _networkModel.getNicInNetworkIncludingRemoved(vm.getId(), networkId); - if (guestNic == null) { + if (guestNic == null && _networkModel.getNicAndIpInNetwork(vm.getId(), networkId) == null) { throw new InvalidParameterValueException("Vm doesn't belong to the network with specified id"); } @@ -1659,20 +1665,22 @@ public PortForwardingRule updatePortForwardingRule(UpdatePortForwardingRuleCmd c } Ip dstIp = rule.getDestinationIpAddress(); if (virtualMachineId != null) { - // Verify that vm has nic in the network - Nic guestNic = _networkModel.getNicInNetwork(virtualMachineId, rule.getNetworkId()); - if (guestNic == null || guestNic.getIPv4Address() == null) { + // Verify that vm has nic in the network - either as its primary, or (for a trunk nic) as one of its + // associated networks, whose guest ip lives in nic_network_map rather than on the nic itself + Pair nicAndIp = _networkModel.getNicAndIpInNetwork(virtualMachineId, rule.getNetworkId()); + if (nicAndIp == null || nicAndIp.second().getIp4Address() == null) { throw new InvalidParameterValueException("Vm doesn't belong to network associated with ipAddress"); - } else { - dstIp = new Ip(guestNic.getIPv4Address()); } + Nic guestNic = nicAndIp.first(); + dstIp = new Ip(nicAndIp.second().getIp4Address()); if (vmGuestIp != null) { //vm ip is passed so it can be primary or secondary ip addreess. if (!dstIp.equals(vmGuestIp)) { //the vm ip is secondary ip to the nic. - // is vmIp is secondary ip or not - NicSecondaryIp secondaryIp = _nicSecondaryDao.findByIp4AddressAndNicId(vmGuestIp.toString(), guestNic.getId()); + // is vmIp is secondary ip or not - scoped by network, since the same nic can carry + // secondary ips across more than one of its associated networks + NicSecondaryIp secondaryIp = _nicSecondaryDao.findByIp4AddressAndNetworkIdAndInstanceId(rule.getNetworkId(), virtualMachineId, vmGuestIp.toString()); if (secondaryIp == null) { throw new InvalidParameterValueException("IP Address is not in the VM nic's network "); } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index f32857d7cf04..bb1fb289a86e 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -133,7 +133,10 @@ import org.apache.cloudstack.api.command.admin.management.RemoveManagementServerCmd; import org.apache.cloudstack.api.command.admin.network.AddNetworkDeviceCmd; import org.apache.cloudstack.api.command.admin.network.AddNetworkServiceProviderCmd; +import org.apache.cloudstack.api.command.admin.network.AssociateNetworkToNicCmd; +import org.apache.cloudstack.api.command.admin.network.ChangeNicPrimaryNetworkCmd; import org.apache.cloudstack.api.command.admin.network.CloneNetworkOfferingCmd; +import org.apache.cloudstack.api.command.admin.network.DisassociateNetworkFromNicCmd; import org.apache.cloudstack.api.command.admin.network.CreateManagementNetworkIpRangeCmd; import org.apache.cloudstack.api.command.admin.network.CreateNetworkCmdByAdmin; import org.apache.cloudstack.api.command.admin.network.CreateNetworkOfferingCmd; @@ -4355,6 +4358,9 @@ public List> getCommands() { cmdList.add(ListPublicIpAddressesCmdByAdmin.class); cmdList.add(CreateNetworkCmdByAdmin.class); cmdList.add(UpdateNetworkCmdByAdmin.class); + cmdList.add(AssociateNetworkToNicCmd.class); + cmdList.add(DisassociateNetworkFromNicCmd.class); + cmdList.add(ChangeNicPrimaryNetworkCmd.class); cmdList.add(ListNetworksCmdByAdmin.class); cmdList.add(CreateVPCCmdByAdmin.class); cmdList.add(ListVPCsCmdByAdmin.class); diff --git a/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java b/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java index ace043bcc526..fc5049980118 100644 --- a/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java +++ b/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java @@ -40,6 +40,7 @@ import com.cloud.gpu.GPU; import com.cloud.gpu.dao.HostGpuGroupsDao; import com.cloud.gpu.dao.VgpuProfileDao; +import com.cloud.host.DetailVO; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -79,6 +80,8 @@ import com.cloud.vm.VirtualMachine.Type; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.VirtualMachineProfileImpl; +import com.cloud.vm.NicVO; +import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.dao.VMInstanceDao; @@ -224,6 +227,12 @@ public class DeploymentPlanningManagerImplTest { @Mock DataStoreManager _dataStoreManager; + @Mock + NicDao nicDao; + + @Mock + HostDetailsDao hostDetailsDao; + @Inject HostPodDao _podDao; @@ -263,6 +272,7 @@ public void testSetUp() { Mockito.when(vmDetailsDao.listDetailsKeyPairs(ArgumentMatchers.anyLong())).thenReturn(null); Mockito.when(volDao.findByInstance(ArgumentMatchers.anyLong())).thenReturn(new ArrayList<>()); + Mockito.when(nicDao.listByVmId(ArgumentMatchers.anyLong())).thenReturn(new ArrayList<>()); Mockito.when(_dcDao.findById(ArgumentMatchers.anyLong())).thenReturn(dc); Mockito.when(dc.getId()).thenReturn(dataCenterId); @@ -472,6 +482,59 @@ public void avoidDisabledDataCentersTest() { } } + @Test + public void avoidHostsNotReadyForMultiNetworkNicsNoTrunkNic() { + VirtualMachineProfile vmProfile = Mockito.mock(VirtualMachineProfile.class); + Mockito.when(vmProfile.getId()).thenReturn(instanceId); + DataCenter dc = Mockito.mock(DataCenter.class); + Mockito.when(dc.getId()).thenReturn(dataCenterId); + ExcludeList avoids = new ExcludeList(); + + NicVO ordinaryNic = Mockito.mock(NicVO.class); + Mockito.when(ordinaryNic.getMultiNetwork()).thenReturn(false); + Mockito.when(nicDao.listByVmId(instanceId)).thenReturn(Arrays.asList(ordinaryNic)); + + _dpm.avoidHostsNotReadyForMultiNetworkNics(vmProfile, dc, avoids); + + assertAvoidIsEmpty(avoids, true, true, true, true); + Mockito.verify(hostDao, Mockito.never()).listEnabledIdsByDataCenterId(Mockito.anyLong()); + } + + @Test + public void avoidHostsNotReadyForMultiNetworkNicsExcludesNotReadyHosts() { + VirtualMachineProfile vmProfile = Mockito.mock(VirtualMachineProfile.class); + Mockito.when(vmProfile.getId()).thenReturn(instanceId); + DataCenter dc = Mockito.mock(DataCenter.class); + Mockito.when(dc.getId()).thenReturn(dataCenterId); + ExcludeList avoids = new ExcludeList(); + + NicVO trunkNic = Mockito.mock(NicVO.class); + Mockito.when(trunkNic.getMultiNetwork()).thenReturn(true); + Mockito.when(nicDao.listByVmId(instanceId)).thenReturn(Arrays.asList(trunkNic)); + + long readyHostId = 10L; + long notReadyHostId = 11L; + long noDetailHostId = 12L; + Mockito.when(hostDao.listEnabledIdsByDataCenterId(dataCenterId)) + .thenReturn(Arrays.asList(readyHostId, notReadyHostId, noDetailHostId)); + + DetailVO readyDetail = Mockito.mock(DetailVO.class); + Mockito.when(readyDetail.getHostId()).thenReturn(readyHostId); + Mockito.when(readyDetail.getValue()).thenReturn("true"); + DetailVO notReadyDetail = Mockito.mock(DetailVO.class); + Mockito.when(notReadyDetail.getHostId()).thenReturn(notReadyHostId); + Mockito.when(notReadyDetail.getValue()).thenReturn("false"); + Mockito.when(hostDetailsDao.findByName(Host.HOST_VLAN_FILTERING_ENABLED)) + .thenReturn(Arrays.asList(readyDetail, notReadyDetail)); + + _dpm.avoidHostsNotReadyForMultiNetworkNics(vmProfile, dc, avoids); + + Assert.assertEquals(2, avoids.getHostsToAvoid().size()); + Assert.assertTrue(avoids.getHostsToAvoid().contains(notReadyHostId)); + Assert.assertTrue(avoids.getHostsToAvoid().contains(noDetailHostId)); + Assert.assertFalse(avoids.getHostsToAvoid().contains(readyHostId)); + } + @Test public void avoidDisabledPodsTestNoDisabledPod() { DataCenter dc = Mockito.mock(DataCenter.class); @@ -965,6 +1028,11 @@ public HostDetailsDao hostDetailsDao() { return Mockito.mock(HostDetailsDao.class); } + @Bean + public NicDao nicDao() { + return Mockito.mock(NicDao.class); + } + @Bean public ClusterDetailsDao clusterDetailsDao() { diff --git a/server/src/test/java/com/cloud/network/MockNetworkModelImpl.java b/server/src/test/java/com/cloud/network/MockNetworkModelImpl.java index 3f2f3b6e955e..821935e390af 100644 --- a/server/src/test/java/com/cloud/network/MockNetworkModelImpl.java +++ b/server/src/test/java/com/cloud/network/MockNetworkModelImpl.java @@ -183,6 +183,12 @@ public Nic getNicInNetwork(long vmId, long networkId) { return null; } + @Override + public Pair getNicAndIpInNetwork(long vmId, long networkId) { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) * @see com.cloud.network.NetworkModel#getNicsForTraffic(long, com.cloud.network.Networks.TrafficType) */ diff --git a/server/src/test/java/com/cloud/network/NetworkModelImplTest.java b/server/src/test/java/com/cloud/network/NetworkModelImplTest.java index 7810662c617b..181f0965a3c3 100644 --- a/server/src/test/java/com/cloud/network/NetworkModelImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkModelImplTest.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.when; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -71,7 +72,11 @@ import com.cloud.utils.net.Ip; import com.cloud.vm.Nic; import com.cloud.vm.NicProfile; +import com.cloud.vm.NicVO; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import org.apache.cloudstack.extension.Extension; import org.apache.cloudstack.extension.ExtensionHelper; import org.apache.cloudstack.framework.extensions.network.NetworkExtensionElement; @@ -452,4 +457,54 @@ public void listSupportedNetworkServiceProvidersExcludesExtensionBackedProviders Mockito.verify(physicalNetworkServiceProviderDao, Mockito.times(1)).listAll(); Mockito.verify(physicalNetworkServiceProviderDao, Mockito.never()).listBy(Mockito.anyLong()); } + + @Test + public void getNicAndIpInNetworkReturnsPrimaryNicWhenItMatches() { + NicDao nicDao = mock(NicDao.class); + networkModel._nicDao = nicDao; + NicVO nic = mock(NicVO.class); + when(nic.getIPv4Address()).thenReturn("10.1.1.5"); + when(nic.getIPv6Address()).thenReturn(null); + when(nicDao.findByNtwkIdAndInstanceId(205L, 5L)).thenReturn(nic); + + Pair result = networkModel.getNicAndIpInNetwork(5L, 205L); + + assertEquals(nic, result.first()); + assertEquals("10.1.1.5", result.second().getIp4Address()); + } + + @Test + public void getNicAndIpInNetworkFallsBackToTrunkAssociationWhenNoPrimaryMatch() { + NicDao nicDao = mock(NicDao.class); + NicNetworkMapDao nicNetworkMapDao = mock(NicNetworkMapDao.class); + networkModel._nicDao = nicDao; + networkModel._nicNetworkMapDao = nicNetworkMapDao; + + when(nicDao.findByNtwkIdAndInstanceId(206L, 5L)).thenReturn(null); + NicVO trunkNic = mock(NicVO.class); + when(trunkNic.getId()).thenReturn(11L); + when(nicDao.listByVmId(5L)).thenReturn(Collections.singletonList(trunkNic)); + NicNetworkMapVO association = mock(NicNetworkMapVO.class); + when(association.getIp4Address()).thenReturn("10.1.1.60"); + when(association.getIp6Address()).thenReturn(null); + when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(association); + + Pair result = networkModel.getNicAndIpInNetwork(5L, 206L); + + assertEquals(trunkNic, result.first()); + assertEquals("10.1.1.60", result.second().getIp4Address()); + } + + @Test + public void getNicAndIpInNetworkReturnsNullWhenVmHasNoNicInNetworkAtAll() { + NicDao nicDao = mock(NicDao.class); + NicNetworkMapDao nicNetworkMapDao = mock(NicNetworkMapDao.class); + networkModel._nicDao = nicDao; + networkModel._nicNetworkMapDao = nicNetworkMapDao; + + when(nicDao.findByNtwkIdAndInstanceId(207L, 5L)).thenReturn(null); + when(nicDao.listByVmId(5L)).thenReturn(new ArrayList<>()); + + assertNull(networkModel.getNicAndIpInNetwork(5L, 207L)); + } } diff --git a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java index cd7d40d68951..334a73cef00c 100644 --- a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -41,6 +42,9 @@ import java.util.UUID; import org.apache.cloudstack.alert.AlertService; +import org.apache.cloudstack.api.command.admin.network.AssociateNetworkToNicCmd; +import org.apache.cloudstack.api.command.admin.network.ChangeNicPrimaryNetworkCmd; +import org.apache.cloudstack.api.command.admin.network.DisassociateNetworkFromNicCmd; import org.apache.cloudstack.api.command.admin.network.CreateNetworkCmdByAdmin; import org.apache.cloudstack.api.command.user.address.UpdateQuarantinedIpCmd; import org.apache.cloudstack.api.command.user.network.CreateNetworkCmd; @@ -65,8 +69,13 @@ import org.springframework.beans.factory.NoSuchBeanDefinitionException; import org.springframework.test.util.ReflectionTestUtils; +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.UpdateNicVlanMembershipAnswer; import com.cloud.agent.api.to.IpAddressTO; import com.cloud.agent.api.to.NicTO; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.HypervisorGuru; +import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.alert.AlertManager; import com.cloud.bgp.BGPService; import com.cloud.configuration.ConfigurationManager; @@ -75,12 +84,22 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; +import com.cloud.event.EventTypes; +import com.cloud.event.UsageEventUtils; +import com.cloud.event.UsageEventVO; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InsufficientAddressCapacityException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; +import com.cloud.host.DetailVO; +import com.cloud.host.Host; +import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; +import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.LoadBalancerDao; +import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; @@ -92,6 +111,10 @@ import com.cloud.network.router.CommandSetupHelper; import com.cloud.network.router.NetworkHelper; import com.cloud.network.vo.PublicIpQuarantineVO; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; +import com.cloud.vm.dao.UserVmDao; import com.cloud.network.vpc.VpcManager; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; @@ -118,6 +141,7 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Ip; import com.cloud.vm.DomainRouterVO; +import com.cloud.vm.Nic; import com.cloud.vm.NicVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.DomainRouterDao; @@ -145,6 +169,12 @@ public class NetworkServiceImplTest { @Mock NetworkModel networkModel; @Mock + AgentManager agentMgr; + @Mock + HypervisorGuruManager hvGuruMgr; + @Mock + HypervisorGuru hypervisorGuru; + @Mock NetworkOfferingServiceMapDao networkOfferingServiceMapDao; @Mock EntityManager entityMgr; @@ -165,8 +195,24 @@ public class NetworkServiceImplTest { @Mock NicDao nicDao; @Mock + UserVmDao userVmDao; + @Mock + HostDao hostDao; + @Mock + HostDetailsDao hostDetailsDao; + @Mock + NicNetworkMapDao nicNetworkMapDao; + @Mock + Ipv6AddressManager ipv6AddressManager; + @Mock IPAddressDao ipAddressDao; @Mock + FirewallRulesDao firewallRulesDao; + @Mock + PortForwardingRulesDao portForwardingRulesDao; + @Mock + LoadBalancerDao loadBalancerDao; + @Mock ConfigurationManager configMgr; @Mock ConfigKey publicMtuKey; @@ -267,6 +313,7 @@ public class NetworkServiceImplTest { private VpcVO vpc; private MockedStatic callContextMocked; + private MockedStatic usageEventUtilsMocked; private AutoCloseable closeable; @@ -302,6 +349,7 @@ private void registerCallContext() { public void setup() throws Exception { closeable = MockitoAnnotations.openMocks(this); overrideDefaultConfigValue(NetworkService.AllowUsersToSpecifyVRMtu, "_defaultValue", "true"); + overrideDefaultConfigValue(NetworkOrchestrationService.MultiNetworkNicEnabled, "_defaultValue", "true"); offering = Mockito.mock(NetworkOfferingVO.class); network = Mockito.mock(Network.class); dc = Mockito.mock(DataCenterVO.class); @@ -322,7 +370,18 @@ public void setup() throws Exception { service.networkHelper = networkHelper; service._ipAddrMgr = ipAddressManagerMock; service.nsxProviderDao = nsxProviderDao; + service._userVmDao = userVmDao; + service._hostDao = hostDao; + service._hostDetailsDao = hostDetailsDao; + service._nicNetworkMapDao = nicNetworkMapDao; + service.ipv6AddrMgr = ipv6AddressManager; + service._networkModel = networkModel; + service._entityMgr = entityMgr; + service._networkMgr = _networkMgr; callContextMocked = Mockito.mockStatic(CallContext.class); + // stubbed as a no-op by default: persistAssociations publishes a real UsageEventUtils call on every + // first-time trunk conversion, which this class' tests don't otherwise need to know about + usageEventUtilsMocked = Mockito.mockStatic(UsageEventUtils.class); CallContext callContextMock = Mockito.mock(CallContext.class); callContextMocked.when(CallContext::current).thenReturn(callContextMock); accountMock = Mockito.mock(Account.class); @@ -339,6 +398,7 @@ public void setup() throws Exception { @After public void tearDown() throws Exception { callContextMocked.close(); + usageEventUtilsMocked.close(); closeable.close(); } @@ -1378,4 +1438,675 @@ public void getAndValidateSupportForKeepMacAddressOnPublicNicParameterTestReturn Assert.assertFalse(service.getAndValidateSupportForKeepMacAddressOnPublicNicParameter(false, networkOfferingVO)); } + + private NicVO mockNic(long nicId, long instanceId, long networkId, boolean multiNetwork) { + NicVO nic = Mockito.mock(NicVO.class); + Mockito.when(nic.getId()).thenReturn(nicId); + Mockito.when(nic.getInstanceId()).thenReturn(instanceId); + Mockito.when(nic.getNetworkId()).thenReturn(networkId); + Mockito.when(nic.getVmType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(nic.getUuid()).thenReturn("nic-uuid"); + Mockito.when(nic.getMultiNetwork()).thenReturn(multiNetwork); + return nic; + } + + private NetworkVO mockNetwork(long networkId, Network.GuestType guestType, long physicalNetworkId, long dataCenterId, Long vpcId, + Networks.BroadcastDomainType broadcastType, URI broadcastUri) { + NetworkVO network = Mockito.mock(NetworkVO.class); + Mockito.when(network.getId()).thenReturn(networkId); + Mockito.when(network.getUuid()).thenReturn("network-uuid-" + networkId); + Mockito.when(network.getGuestType()).thenReturn(guestType); + Mockito.when(network.getPhysicalNetworkId()).thenReturn(physicalNetworkId); + Mockito.when(network.getDataCenterId()).thenReturn(dataCenterId); + Mockito.when(network.getVpcId()).thenReturn(vpcId); + Mockito.when(network.getBroadcastDomainType()).thenReturn(broadcastType); + Mockito.when(network.getBroadcastUri()).thenReturn(broadcastUri); + Mockito.when(network.getState()).thenReturn(Network.State.Implemented); + return network; + } + + private UserVmVO mockVm(VirtualMachine.State state, Long hostId) { + UserVmVO vm = Mockito.mock(UserVmVO.class); + Mockito.when(vm.getState()).thenReturn(state); + Mockito.when(vm.getHostId()).thenReturn(hostId); + return vm; + } + + private AssociateNetworkToNicCmd mockAssociateCmd(long nicId, List networkIds) { + AssociateNetworkToNicCmd cmd = Mockito.mock(AssociateNetworkToNicCmd.class); + Mockito.when(cmd.getNicId()).thenReturn(nicId); + Mockito.when(cmd.getNetworkIds()).thenReturn(networkIds); + Mockito.when(cmd.getIpAddressesMap()).thenReturn(new HashMap<>()); + return cmd; + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworkToNicFailsWhenNicNotFound() throws Exception { + Mockito.when(nicDao.findById(11L)).thenReturn(null); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworkToNicFailsWhenNicIsNotOnAUserVm() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nic.getVmType()).thenReturn(VirtualMachine.Type.DomainRouter); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + } + + @Test + public void associateNetworkToNicFailsWhenFeatureDisabledInZone() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + + overrideDefaultConfigValue(NetworkOrchestrationService.MultiNetworkNicEnabled, "_defaultValue", "false"); + try { + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + Assert.fail("expected InvalidParameterValueException"); + } catch (InvalidParameterValueException e) { + Assert.assertTrue(e.getMessage().contains("not enabled")); + } finally { + overrideDefaultConfigValue(NetworkOrchestrationService.MultiNetworkNicEnabled, "_defaultValue", "true"); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworkToNicFailsWhenOwningVmNotFound() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + Mockito.when(userVmDao.findById(5L)).thenReturn(null); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworkToNicFailsWhenRunningVmHostIsNotVlanFilteringReady() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Running, 1L); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + Mockito.when(hostDetailsDao.findDetail(1L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(null); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + } + + @Test + public void associateNetworkToNicUpdatesLiveVlanMembershipOnARunningVmWhenHostIsReady() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Running, 1L); + Mockito.when(vm.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + Mockito.when(vm.getInstanceName()).thenReturn("i-2-11-VM"); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + DetailVO readyDetail = Mockito.mock(DetailVO.class); + Mockito.when(readyDetail.getValue()).thenReturn("true"); + Mockito.when(hostDetailsDao.findDetail(1L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(readyDetail); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO requestedNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(requestedNetwork); + Mockito.when(ipAddressManagerMock.allocateGuestIP(requestedNetwork, null)).thenReturn("10.1.1.50"); + Mockito.when(networkModel.getNicProfile(Mockito.eq(vm), Mockito.eq(nic), Mockito.any())).thenReturn(null); + Mockito.when(hvGuruMgr.getGuru(Hypervisor.HypervisorType.KVM)).thenReturn(hypervisorGuru); + Mockito.when(hypervisorGuru.toNicTO(null)).thenReturn(Mockito.mock(NicTO.class)); + Mockito.when(agentMgr.easySend(Mockito.eq(1L), Mockito.any())).thenReturn(new UpdateNicVlanMembershipAnswer(null, true, "success")); + DomainRouterVO router = Mockito.mock(DomainRouterVO.class); + Mockito.when(router.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(routerDao.findByNetwork(205L)).thenReturn(Arrays.asList(router)); + + Nic result = service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + + Assert.assertNotNull(result); + Mockito.verify(agentMgr).easySend(Mockito.eq(1L), Mockito.any()); + Mockito.verify(commandSetupHelper).createVmDataCommand(Mockito.eq(router), Mockito.eq(vm), Mockito.eq(nic), Mockito.any(), Mockito.any()); + Mockito.verify(networkHelper).sendCommandsToRouter(Mockito.eq(router), Mockito.any()); + } + + @Test + public void associateNetworkToNicDoesNotRefreshMetadataOnRoutersThatAreNotRunning() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Running, 1L); + Mockito.when(vm.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + Mockito.when(vm.getInstanceName()).thenReturn("i-2-11-VM"); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + DetailVO readyDetail = Mockito.mock(DetailVO.class); + Mockito.when(readyDetail.getValue()).thenReturn("true"); + Mockito.when(hostDetailsDao.findDetail(1L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(readyDetail); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO requestedNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(requestedNetwork); + Mockito.when(ipAddressManagerMock.allocateGuestIP(requestedNetwork, null)).thenReturn("10.1.1.50"); + Mockito.when(networkModel.getNicProfile(Mockito.eq(vm), Mockito.eq(nic), Mockito.any())).thenReturn(null); + Mockito.when(hvGuruMgr.getGuru(Hypervisor.HypervisorType.KVM)).thenReturn(hypervisorGuru); + Mockito.when(hypervisorGuru.toNicTO(null)).thenReturn(Mockito.mock(NicTO.class)); + Mockito.when(agentMgr.easySend(Mockito.eq(1L), Mockito.any())).thenReturn(new UpdateNicVlanMembershipAnswer(null, true, "success")); + DomainRouterVO stoppedRouter = Mockito.mock(DomainRouterVO.class); + Mockito.when(stoppedRouter.getState()).thenReturn(VirtualMachine.State.Stopped); + Mockito.when(routerDao.findByNetwork(205L)).thenReturn(Arrays.asList(stoppedRouter)); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + + Mockito.verify(commandSetupHelper, Mockito.never()).createVmDataCommand(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test(expected = CloudRuntimeException.class) + public void associateNetworkToNicFailsWhenLiveUpdateCommandFails() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Running, 1L); + Mockito.when(vm.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + Mockito.when(vm.getInstanceName()).thenReturn("i-2-11-VM"); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + DetailVO readyDetail = Mockito.mock(DetailVO.class); + Mockito.when(readyDetail.getValue()).thenReturn("true"); + Mockito.when(hostDetailsDao.findDetail(1L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(readyDetail); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO requestedNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(requestedNetwork); + Mockito.when(ipAddressManagerMock.allocateGuestIP(requestedNetwork, null)).thenReturn("10.1.1.50"); + Mockito.when(networkModel.getNicProfile(Mockito.eq(vm), Mockito.eq(nic), Mockito.any())).thenReturn(null); + Mockito.when(hvGuruMgr.getGuru(Hypervisor.HypervisorType.KVM)).thenReturn(hypervisorGuru); + Mockito.when(hypervisorGuru.toNicTO(null)).thenReturn(Mockito.mock(NicTO.class)); + Mockito.when(agentMgr.easySend(Mockito.eq(1L), Mockito.any())).thenReturn(new UpdateNicVlanMembershipAnswer(null, false, "boom")); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworkToNicFailsWhenAlreadyAssociatedWithPrimaryNetwork() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(205L))); + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworkToNicFailsWhenRequestedNetworkIsADifferentGuestType() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO sharedNetwork = mockNetwork(206L, Network.GuestType.Shared, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(sharedNetwork); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworkToNicFailsWhenRequestedNetworkCidrOverlapsPrimaryNetworkCidr() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(primaryNetwork.getCidr()).thenReturn("10.1.1.0/24"); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO requestedNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(requestedNetwork.getCidr()).thenReturn("10.1.1.0/24"); + Mockito.when(networkDao.findById(206L)).thenReturn(requestedNetwork); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + } + + @Test + public void associateNetworkToNicSucceedsWhenRequestedNetworkCidrDoesNotOverlapPrimaryNetworkCidr() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(primaryNetwork.getCidr()).thenReturn("10.1.1.0/24"); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO requestedNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(requestedNetwork.getCidr()).thenReturn("10.1.2.0/24"); + Mockito.when(networkDao.findById(206L)).thenReturn(requestedNetwork); + Mockito.when(ipAddressManagerMock.allocateGuestIP(requestedNetwork, null)).thenReturn("10.1.2.50"); + + Nic result = service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + + Assert.assertNotNull(result); + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworkToNicFailsWhenRequestedNetworkIsOnADifferentPhysicalNetwork() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO otherPhysicalNetwork = mockNetwork(206L, Network.GuestType.Isolated, 2L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(otherPhysicalNetwork); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworkToNicFailsWhenRequestedNetworkIsInADifferentZone() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO otherZoneNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 2L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(otherZoneNetwork); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworkToNicFailsWhenRequestedIsolatedNetworkIsInADifferentVpc() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, 900L, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO otherVpcNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, 901L, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(otherVpcNetwork); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworkToNicFailsWhenRequestedNetworkIsSecurityGroupEnabled() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO sgNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(sgNetwork); + Mockito.when(networkModel.isSecurityGroupSupportedInNetwork(sgNetwork)).thenReturn(true); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworkToNicFailsWhenRequestedNetworkSharesVlanWithAnExistingAssociation() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO sameVlanNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(206L)).thenReturn(sameVlanNetwork); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + } + + @Test + public void associateNetworkToNicPersistsAssociationAndSetsMultiNetworkOnAStoppedVm() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO requestedNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(requestedNetwork); + Mockito.when(ipAddressManagerMock.allocateGuestIP(requestedNetwork, null)).thenReturn("10.1.1.50"); + + Nic result = service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + + Assert.assertNotNull(result); + Mockito.verify(nicNetworkMapDao).persist(Mockito.argThat(map -> + map.getNicId() == 11L && map.getNetworkId() == 206L && "10.1.1.50".equals(map.getIp4Address()))); + Mockito.verify(nic).setMultiNetwork(true); + Mockito.verify(nicDao).update(Mockito.eq(11L), Mockito.eq(nic)); + } + + @Test + public void associateNetworksToNicPersistsAssociationWithNoExplicitIpAddresses() throws Exception { + // the deploy-time path (persistAdditionalNicNetworkAssociations -> associateNetworksToNic) calls this with + // no ipAddresses map at all - must not NPE when that's passed through as a literal null + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO requestedNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(requestedNetwork); + Mockito.when(ipAddressManagerMock.allocateGuestIP(requestedNetwork, null)).thenReturn("10.1.1.60"); + + service.associateNetworksToNic(nic, Arrays.asList(206L), null); + + Mockito.verify(nicNetworkMapDao).persist(Mockito.argThat(map -> + map.getNicId() == 11L && map.getNetworkId() == 206L && "10.1.1.60".equals(map.getIp4Address()))); + Mockito.verify(nic).setMultiNetwork(true); + } + + @Test + public void associateNetworksToNicUsesRequestedIpForAssociatedNetwork() throws Exception { + // deploy-time path (nicnetworkslist[N].ip4addresses) - a requested ip for an associated network must + // reach allocateGuestIP as the requested value, not be silently ignored/auto-allocated + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO requestedNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(requestedNetwork); + Mockito.when(ipAddressManagerMock.allocateGuestIP(requestedNetwork, "10.1.1.99")).thenReturn("10.1.1.99"); + + service.associateNetworksToNic(nic, Arrays.asList(206L), Collections.singletonMap(206L, new Network.IpAddresses("10.1.1.99", null))); + + Mockito.verify(ipAddressManagerMock).allocateGuestIP(requestedNetwork, "10.1.1.99"); + Mockito.verify(nicNetworkMapDao).persist(Mockito.argThat(map -> + map.getNicId() == 11L && map.getNetworkId() == 206L && "10.1.1.99".equals(map.getIp4Address()))); + } + + @Test(expected = InvalidParameterValueException.class) + public void associateNetworksToNicRejectsNetworkAlreadyOnAnotherNic() throws Exception { + // vm has a second, ordinary nic (208L) whose primary is network 206L - trying to associate 206L onto + // nic 11L too must be rejected, since a network can only be reachable through one of the vm's nics + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(vm.getId()).thenReturn(5L); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + + NicVO otherNic = mockNic(12L, 5L, 206L, false); + Mockito.when(nicDao.listByVmId(5L)).thenReturn(Arrays.asList(nic, otherNic)); + Mockito.when(nicNetworkMapDao.listByNicId(12L)).thenReturn(new ArrayList<>()); + + service.associateNetworksToNic(nic, Arrays.asList(206L), null); + } + + @Test + public void associateNetworksToNicPublishesBackfillEventOnFirstConversionToTrunk() throws Exception { + // nic is not yet a trunk nic (multiNetwork=false) - this is the first association, so the nic's own + // still-open, pre-existing usage row(s) for its primary network need a one-time network_id backfill. + // The backfill itself runs later, on the usage job's own thread (see UsageManagerImpl) - this request + // thread only ever publishes the event, since it already has an ambient `cloud` DB transaction open + // and cloud_usage is a separate database. + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO requestedNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(requestedNetwork); + Mockito.when(ipAddressManagerMock.allocateGuestIP(requestedNetwork, null)).thenReturn("10.1.1.60"); + + service.associateNetworksToNic(nic, Arrays.asList(206L), null); + + usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent( + Mockito.eq(EventTypes.EVENT_NIC_NETWORK_ID_BACKFILL), Mockito.anyLong(), Mockito.anyLong(), Mockito.eq(11L), Mockito.eq("11"), + Mockito.isNull(), Mockito.isNull(), Mockito.isNull(), Mockito.anyString(), Mockito.any(), + Mockito.argThat(details -> "205".equals(details.get(UsageEventVO.DynamicParameters.networkId.name())))), Mockito.times(1)); + } + + @Test + public void associateNetworksToNicDoesNotPublishBackfillEventWhenNicIsAlreadyATrunkNic() throws Exception { + // nic is already a trunk nic - this is a further association on top of an existing one, not the + // first-time conversion, so the one-time backfill must not fire again + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO requestedNetwork = mockNetwork(207L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1190)); + Mockito.when(networkDao.findById(207L)).thenReturn(requestedNetwork); + Mockito.when(ipAddressManagerMock.allocateGuestIP(requestedNetwork, null)).thenReturn("10.1.1.70"); + + service.associateNetworksToNic(nic, Arrays.asList(207L), null); + + usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent( + Mockito.eq(EventTypes.EVENT_NIC_NETWORK_ID_BACKFILL), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), + Mockito.isNull(), Mockito.isNull(), Mockito.isNull(), Mockito.anyString(), Mockito.any(), Mockito.anyMap()), Mockito.never()); + } + + @Test(expected = CloudRuntimeException.class) + public void associateNetworksToNicFailsWhenVmNotStopped() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Running, 1L); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + + service.associateNetworksToNic(nic, Arrays.asList(206L), null); + } + + private DisassociateNetworkFromNicCmd mockDisassociateCmd(long nicId, long networkId) { + DisassociateNetworkFromNicCmd cmd = Mockito.mock(DisassociateNetworkFromNicCmd.class); + Mockito.when(cmd.getNicId()).thenReturn(nicId); + Mockito.when(cmd.getNetworkId()).thenReturn(networkId); + return cmd; + } + + @Test + public void disassociateNetworkFromNicFailsWhenFeatureDisabledInZone() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + + overrideDefaultConfigValue(NetworkOrchestrationService.MultiNetworkNicEnabled, "_defaultValue", "false"); + try { + service.disassociateNetworkFromNic(mockDisassociateCmd(11L, 206L)); + Assert.fail("expected InvalidParameterValueException"); + } catch (InvalidParameterValueException e) { + Assert.assertTrue(e.getMessage().contains("not enabled")); + } finally { + overrideDefaultConfigValue(NetworkOrchestrationService.MultiNetworkNicEnabled, "_defaultValue", "true"); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void disassociateNetworkFromNicFailsWhenTargetIsPrimaryNetwork() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + + service.disassociateNetworkFromNic(mockDisassociateCmd(11L, 205L)); + } + + @Test(expected = InvalidParameterValueException.class) + public void disassociateNetworkFromNicFailsWhenNotAssociated() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(null); + + service.disassociateNetworkFromNic(mockDisassociateCmd(11L, 206L)); + } + + @Test + public void disassociateNetworkFromNicRemovesAssociationOnAStoppedVm() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L, "10.1.1.50", null); + Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(association); + + Nic result = service.disassociateNetworkFromNic(mockDisassociateCmd(11L, 206L)); + + Assert.assertNotNull(result); + Mockito.verify(nicNetworkMapDao).remove(association.getId()); + Mockito.verify(agentMgr, Mockito.never()).easySend(Mockito.anyLong(), Mockito.any()); + } + + @Test + public void disassociateNetworkFromNicUpdatesLiveVlanMembershipOnARunningVm() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Running, 1L); + Mockito.when(vm.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + Mockito.when(vm.getInstanceName()).thenReturn("i-2-11-VM"); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + DetailVO readyDetail = Mockito.mock(DetailVO.class); + Mockito.when(readyDetail.getValue()).thenReturn("true"); + Mockito.when(hostDetailsDao.findDetail(1L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(readyDetail); + NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L, "10.1.1.50", null); + Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(association); + Mockito.when(networkModel.getNicProfile(Mockito.eq(vm), Mockito.eq(nic), Mockito.any())).thenReturn(null); + Mockito.when(hvGuruMgr.getGuru(Hypervisor.HypervisorType.KVM)).thenReturn(hypervisorGuru); + Mockito.when(hypervisorGuru.toNicTO(null)).thenReturn(Mockito.mock(NicTO.class)); + Mockito.when(agentMgr.easySend(Mockito.eq(1L), Mockito.any())).thenReturn(new UpdateNicVlanMembershipAnswer(null, true, "success")); + DomainRouterVO router = Mockito.mock(DomainRouterVO.class); + Mockito.when(router.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(routerDao.findByNetwork(205L)).thenReturn(Arrays.asList(router)); + + Nic result = service.disassociateNetworkFromNic(mockDisassociateCmd(11L, 206L)); + + Assert.assertNotNull(result); + Mockito.verify(nicNetworkMapDao).remove(association.getId()); + Mockito.verify(agentMgr).easySend(Mockito.eq(1L), Mockito.any()); + Mockito.verify(commandSetupHelper).createVmDataCommand(Mockito.eq(router), Mockito.eq(vm), Mockito.eq(nic), Mockito.any(), Mockito.any()); + Mockito.verify(networkHelper).sendCommandsToRouter(Mockito.eq(router), Mockito.any()); + } + + @Test + public void disassociateNetworkFromNicSucceedsWhenMetadataRefreshFails() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Running, 1L); + Mockito.when(vm.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + Mockito.when(vm.getInstanceName()).thenReturn("i-2-11-VM"); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + DetailVO readyDetail = Mockito.mock(DetailVO.class); + Mockito.when(readyDetail.getValue()).thenReturn("true"); + Mockito.when(hostDetailsDao.findDetail(1L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(readyDetail); + NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L, "10.1.1.50", null); + Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(association); + Mockito.when(networkModel.getNicProfile(Mockito.eq(vm), Mockito.eq(nic), Mockito.any())).thenReturn(null); + Mockito.when(hvGuruMgr.getGuru(Hypervisor.HypervisorType.KVM)).thenReturn(hypervisorGuru); + Mockito.when(hypervisorGuru.toNicTO(null)).thenReturn(Mockito.mock(NicTO.class)); + Mockito.when(agentMgr.easySend(Mockito.eq(1L), Mockito.any())).thenReturn(new UpdateNicVlanMembershipAnswer(null, true, "success")); + DomainRouterVO router = Mockito.mock(DomainRouterVO.class); + Mockito.when(router.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(routerDao.findByNetwork(205L)).thenReturn(Arrays.asList(router)); + Mockito.when(networkHelper.sendCommandsToRouter(Mockito.eq(router), Mockito.any())).thenThrow(new CloudRuntimeException("router unreachable")); + + Nic result = service.disassociateNetworkFromNic(mockDisassociateCmd(11L, 206L)); + + Assert.assertNotNull(result); + Mockito.verify(nicNetworkMapDao).remove(association.getId()); + } + + private ChangeNicPrimaryNetworkCmd mockChangePrimaryCmd(long nicId, long networkId) { + ChangeNicPrimaryNetworkCmd cmd = Mockito.mock(ChangeNicPrimaryNetworkCmd.class); + Mockito.when(cmd.getNicId()).thenReturn(nicId); + Mockito.when(cmd.getNetworkId()).thenReturn(networkId); + return cmd; + } + + @Test + public void changeNicPrimaryNetworkFailsWhenFeatureDisabledInZone() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + + overrideDefaultConfigValue(NetworkOrchestrationService.MultiNetworkNicEnabled, "_defaultValue", "false"); + try { + service.changeNicPrimaryNetwork(mockChangePrimaryCmd(11L, 206L)); + Assert.fail("expected InvalidParameterValueException"); + } catch (InvalidParameterValueException e) { + Assert.assertTrue(e.getMessage().contains("not enabled")); + } finally { + overrideDefaultConfigValue(NetworkOrchestrationService.MultiNetworkNicEnabled, "_defaultValue", "true"); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void changeNicPrimaryNetworkFailsWhenVmNotStopped() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Running, 1L); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + + service.changeNicPrimaryNetwork(mockChangePrimaryCmd(11L, 206L)); + } + + @Test(expected = InvalidParameterValueException.class) + public void changeNicPrimaryNetworkFailsWhenTargetIsCurrentPrimary() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + + service.changeNicPrimaryNetwork(mockChangePrimaryCmd(11L, 205L)); + } + + @Test(expected = InvalidParameterValueException.class) + public void changeNicPrimaryNetworkFailsWhenTargetNotAssociated() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(null); + + service.changeNicPrimaryNetwork(mockChangePrimaryCmd(11L, 206L)); + } + + @Test + public void changeNicPrimaryNetworkSwapsPrimaryAndDemotesOldPrimaryToAssociation() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nic.getIPv4Address()).thenReturn("10.1.1.10"); + Mockito.when(nic.getIPv6Address()).thenReturn(null); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L, "10.1.1.50", null); + Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(association); + + Nic result = service.changeNicPrimaryNetwork(mockChangePrimaryCmd(11L, 206L)); + + Assert.assertNotNull(result); + Mockito.verify(nicNetworkMapDao).remove(association.getId()); + Mockito.verify(nicNetworkMapDao).persist(Mockito.argThat(map -> + map.getNicId() == 11L && map.getNetworkId() == 205L && "10.1.1.10".equals(map.getIp4Address()))); + Mockito.verify(nic).setNetworkId(206L); + Mockito.verify(nic).setIPv4Address("10.1.1.50"); + Mockito.verify(nic).setIPv6Address(null); + Mockito.verify(nicDao).update(Mockito.eq(11L), Mockito.eq(nic)); + } + + private URI vlanUri(int vlanId) { + return Networks.BroadcastDomainType.Vlan.toUri(vlanId); + } } diff --git a/server/src/test/java/com/cloud/network/lb/LoadBalancingRulesManagerImplTest.java b/server/src/test/java/com/cloud/network/lb/LoadBalancingRulesManagerImplTest.java index 282d892600b5..deca36df56f9 100644 --- a/server/src/test/java/com/cloud/network/lb/LoadBalancingRulesManagerImplTest.java +++ b/server/src/test/java/com/cloud/network/lb/LoadBalancingRulesManagerImplTest.java @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -66,6 +65,7 @@ import com.cloud.user.User; import com.cloud.user.UserVO; import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; import com.cloud.utils.db.EntityManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @@ -332,11 +332,10 @@ public void testGetVmNicInLoadBalancerDefaultCase() { when(loadBalancer.getNetworkId()).thenReturn(networkId); Nic nic = Mockito.mock(Nic.class); - when(nic.getNetworkId()).thenReturn(networkId); - List nics = Collections.singletonList(nic); - Mockito.doReturn(nics).when(_networkModel).getNics(anyLong()); - Nic nicInLb = lbr.getVmNicInLoadBalancer(userVm, loadBalancer, loadBalancerNetwork, null, owner); - Assert.assertEquals(nic, nicInLb); + Pair nicAndIp = new Pair<>(nic, new Network.IpAddresses("10.1.1.5", null)); + Mockito.doReturn(nicAndIp).when(_networkModel).getNicAndIpInNetwork(anyLong(), Mockito.eq(networkId)); + Pair nicInLb = lbr.getVmNicInLoadBalancer(userVm, loadBalancer, loadBalancerNetwork, null, owner); + Assert.assertEquals(nic, nicInLb.first()); } @Test @@ -358,10 +357,11 @@ public void testGetVmNicInLoadBalancerVPCConserveMode() { when(networkTier2.getVpcId()).thenReturn(10L); when(loadBalancerNetwork.getVpcId()).thenReturn(10L); Nic nic = Mockito.mock(Nic.class); - when(_networkModel.getNicInNetwork(Mockito.eq(vmId), Mockito.eq(networkTier2Id))).thenReturn(nic); + Pair nicAndIp = new Pair<>(nic, new Network.IpAddresses("10.1.1.5", null)); + when(_networkModel.getNicAndIpInNetwork(Mockito.eq(vmId), Mockito.eq(networkTier2Id))).thenReturn(nicAndIp); - Nic nicInLb = lbr.getVmNicInLoadBalancer(userVm, loadBalancer, loadBalancerNetwork, vmIdNetworkIdMap, owner); - Assert.assertEquals(nic, nicInLb); + Pair nicInLb = lbr.getVmNicInLoadBalancer(userVm, loadBalancer, loadBalancerNetwork, vmIdNetworkIdMap, owner); + Assert.assertEquals(nic, nicInLb.first()); } @Test(expected = InvalidParameterValueException.class) diff --git a/server/src/test/java/com/cloud/network/router/CommandSetupHelperTest.java b/server/src/test/java/com/cloud/network/router/CommandSetupHelperTest.java index 34909e7e860f..2e7b1e33df88 100644 --- a/server/src/test/java/com/cloud/network/router/CommandSetupHelperTest.java +++ b/server/src/test/java/com/cloud/network/router/CommandSetupHelperTest.java @@ -42,14 +42,21 @@ import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.offerings.dao.NetworkOfferingDetailsDao; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.uservm.UserVm; import com.cloud.utils.net.Ip; import com.cloud.vm.NicVO; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; import org.apache.cloudstack.network.BgpPeerVO; import org.apache.cloudstack.network.dao.BgpPeerDetailsDao; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -63,6 +70,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -102,10 +110,13 @@ public class CommandSetupHelperTest { ASNumberDao asNumberDao; @Mock BgpPeerDetailsDao bgpPeerDetailsDao; + @Mock + NicNetworkMapDao nicNetworkMapDao; @Before public void setUp() { ReflectionTestUtils.setField(commandSetupHelper, "_nicDao", nicDao); + ReflectionTestUtils.setField(commandSetupHelper, "_nicNetworkMapDao", nicNetworkMapDao); ReflectionTestUtils.setField(commandSetupHelper, "_networkDao", networkDao); ReflectionTestUtils.setField(commandSetupHelper, "_ipAddressDao", ipAddressDao); ReflectionTestUtils.setField(commandSetupHelper, "_vlanDao", vlanDao); @@ -118,6 +129,125 @@ public void setUp() { ReflectionTestUtils.setField(commandSetupHelper, "_dcDao", dcDao); } + @After + public void tearDown() { + ConfigKey.init(null); + } + + private NetworkVO createVlanNetwork(String uuid, String name, long vlanTag) { + NetworkVO network = Mockito.mock(NetworkVO.class); + when(network.getUuid()).thenReturn(uuid); + when(network.getName()).thenReturn(name); + when(network.getBroadcastDomainType()).thenReturn(BroadcastDomainType.Vlan); + when(network.getBroadcastUri()).thenReturn(BroadcastDomainType.Vlan.toUri(vlanTag)); + return network; + } + + private void enableNicVlanMappingExposure() { + ConfigDepotImpl configDepotMock = Mockito.mock(ConfigDepotImpl.class); + ConfigKey.init(configDepotMock); + when(configDepotMock.getConfigStringValue(Mockito.eq(VirtualMachineManager.AllowExposeNicVlanMapping.toString()), Mockito.any(), Mockito.any())) + .thenReturn("true"); + } + + @Test + public void testAddNicVlanMappingToVmDataSkipsWhenNicNotTrunked() { + VmDataCommand vmDataCommand = new VmDataCommand("testVMname"); + NicVO nic = Mockito.mock(NicVO.class); + when(nic.getMultiNetwork()).thenReturn(false); + UserVm vm = Mockito.mock(UserVm.class); + + commandSetupHelper.addNicVlanMappingToVmData(vmDataCommand, vm, nic); + + Assert.assertTrue(vmDataCommand.getVmData().isEmpty()); + Mockito.verifyNoInteractions(nicNetworkMapDao); + } + + @Test + public void testAddNicVlanMappingToVmDataSkipsWhenConfigDisabled() { + VmDataCommand vmDataCommand = new VmDataCommand("testVMname"); + NicVO nic = Mockito.mock(NicVO.class); + when(nic.getMultiNetwork()).thenReturn(true); + UserVm vm = Mockito.mock(UserVm.class); + when(vm.getAccountId()).thenReturn(2L); + + commandSetupHelper.addNicVlanMappingToVmData(vmDataCommand, vm, nic); + + Assert.assertTrue(vmDataCommand.getVmData().isEmpty()); + Mockito.verifyNoInteractions(nicNetworkMapDao); + } + + @Test + public void testAddNicVlanMappingToVmDataClearsFileWhenNoAssociations() { + enableNicVlanMappingExposure(); + VmDataCommand vmDataCommand = new VmDataCommand("testVMname"); + NicVO nic = Mockito.mock(NicVO.class); + when(nic.getId()).thenReturn(11L); + when(nic.getMultiNetwork()).thenReturn(true); + UserVm vm = Mockito.mock(UserVm.class); + when(vm.getAccountId()).thenReturn(2L); + when(nicNetworkMapDao.listByNicId(11L)).thenReturn(Collections.emptyList()); + + commandSetupHelper.addNicVlanMappingToVmData(vmDataCommand, vm, nic); + + List metadata = vmDataCommand.getVmData(); + Assert.assertEquals(1, metadata.size()); + String[] metadataFile = metadata.get(0); + Assert.assertEquals("metadata", metadataFile[0]); + Assert.assertEquals(NetworkModel.NIC_VLAN_MAPPING_FILE, metadataFile[1]); + Assert.assertEquals("", metadataFile[2]); + } + + @Test + public void testAddNicVlanMappingToVmDataAddsMappingsWhenEnabledAndTrunked() { + enableNicVlanMappingExposure(); + VmDataCommand vmDataCommand = new VmDataCommand("testVMname"); + NicVO nic = Mockito.mock(NicVO.class); + when(nic.getId()).thenReturn(11L); + when(nic.getMultiNetwork()).thenReturn(true); + UserVm vm = Mockito.mock(UserVm.class); + when(vm.getAccountId()).thenReturn(2L); + + NicNetworkMapVO association = new NicNetworkMapVO(11L, 205L); + when(nicNetworkMapDao.listByNicId(11L)).thenReturn(Arrays.asList(association)); + NetworkVO associatedNetwork = createVlanNetwork("network-uuid-205", "netNormal", 1179L); + when(networkDao.findById(205L)).thenReturn(associatedNetwork); + + commandSetupHelper.addNicVlanMappingToVmData(vmDataCommand, vm, nic); + + List metadata = vmDataCommand.getVmData(); + Assert.assertEquals(1, metadata.size()); + String[] metadataFile = metadata.get(0); + Assert.assertEquals("metadata", metadataFile[0]); + Assert.assertEquals(NetworkModel.NIC_VLAN_MAPPING_FILE, metadataFile[1]); + Assert.assertTrue(metadataFile[2].contains("network-uuid-205")); + Assert.assertTrue(metadataFile[2].contains("netNormal")); + Assert.assertTrue(metadataFile[2].contains("1179")); + } + + @Test + public void testAddNicVlanMappingToVmDataClearsFileWhenOnlyNonVlanAssociatedNetworks() { + enableNicVlanMappingExposure(); + VmDataCommand vmDataCommand = new VmDataCommand("testVMname"); + NicVO nic = Mockito.mock(NicVO.class); + when(nic.getId()).thenReturn(11L); + when(nic.getMultiNetwork()).thenReturn(true); + UserVm vm = Mockito.mock(UserVm.class); + when(vm.getAccountId()).thenReturn(2L); + + NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L); + when(nicNetworkMapDao.listByNicId(11L)).thenReturn(Arrays.asList(association)); + NetworkVO nonVlanNetwork = Mockito.mock(NetworkVO.class); + when(nonVlanNetwork.getBroadcastDomainType()).thenReturn(BroadcastDomainType.Lswitch); + when(networkDao.findById(206L)).thenReturn(nonVlanNetwork); + + commandSetupHelper.addNicVlanMappingToVmData(vmDataCommand, vm, nic); + + List metadata = vmDataCommand.getVmData(); + Assert.assertEquals(1, metadata.size()); + Assert.assertEquals("", metadata.get(0)[2]); + } + @Test public void testUserDataDetails() { VmDataCommand vmDataCommand = new VmDataCommand("testVMname"); diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java index 1239dd23e7ec..0a5a58613363 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java @@ -26,6 +26,9 @@ import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.api.command.admin.address.ReleasePodIpCmdByAdmin; +import org.apache.cloudstack.api.command.admin.network.AssociateNetworkToNicCmd; +import org.apache.cloudstack.api.command.admin.network.ChangeNicPrimaryNetworkCmd; +import org.apache.cloudstack.api.command.admin.network.DisassociateNetworkFromNicCmd; import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd; import org.apache.cloudstack.api.command.admin.network.ListDedicatedGuestVlanRangesCmd; import org.apache.cloudstack.api.command.admin.network.ListGuestVlansCmd; @@ -1065,6 +1068,25 @@ public List listVmNicSecondaryIps(ListNicsCmd listNics return null; } + @Override + public Nic associateNetworkToNic(AssociateNetworkToNicCmd cmd) { + return null; + } + + @Override + public Nic disassociateNetworkFromNic(DisassociateNetworkFromNicCmd cmd) { + return null; + } + + @Override + public Nic changeNicPrimaryNetwork(ChangeNicPrimaryNetworkCmd cmd) { + return null; + } + + @Override + public void associateNetworksToNic(Nic nic, List networkIds, Map requestedIps) { + } + @Override public boolean releasePodIp(ReleasePodIpCmdByAdmin ip) throws CloudRuntimeException { return true; diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkModelImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkModelImpl.java index 83f3fda383c8..f24836c09e00 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkModelImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkModelImpl.java @@ -195,6 +195,12 @@ public Nic getNicInNetwork(long vmId, long networkId) { return null; } + @Override + public Pair getNicAndIpInNetwork(long vmId, long networkId) { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) * @see com.cloud.network.NetworkModel#getNicsForTraffic(long, com.cloud.network.Networks.TrafficType) */ From 88585ed91c464a1178ae389387309973f0eea4d6 Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Mon, 28 Sep 2026 14:50:14 -0400 Subject: [PATCH 10/20] Add EventTypes constants for the association service commit - EVENT_NIC_NETWORK_ASSOCIATE/DISASSOCIATE/PRIMARY_NETWORK_CHANGE, needed by the previous commit's @ActionEvent annotations --- .../main/java/com/cloud/event/EventTypes.java | 6 ++++ .../java/com/cloud/vm/UserVmManagerImpl.java | 34 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index f7c985cc2c6c..b2848abafa66 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -228,6 +228,9 @@ public class EventTypes { public static final String EVENT_NIC_DETAIL_ADD = "NIC.DETAIL.ADD"; public static final String EVENT_NIC_DETAIL_UPDATE = "NIC.DETAIL.UPDATE"; public static final String EVENT_NIC_DETAIL_REMOVE = "NIC.DETAIL.REMOVE"; + public static final String EVENT_NIC_NETWORK_ASSOCIATE = "NIC.NETWORK.ASSOCIATE"; + public static final String EVENT_NIC_NETWORK_DISASSOCIATE = "NIC.NETWORK.DISASSOCIATE"; + public static final String EVENT_NIC_PRIMARY_NETWORK_CHANGE = "NIC.PRIMARY.NETWORK.CHANGE"; // internal-only signal consumed by the usage job - never user-facing, not an audit event public static final String EVENT_NIC_NETWORK_ID_BACKFILL = "NIC.NETWORK.ID.BACKFILL"; @@ -982,6 +985,9 @@ public class EventTypes { // Nic Events entityEventDetails.put(EVENT_NIC_CREATE, Nic.class); + entityEventDetails.put(EVENT_NIC_NETWORK_ASSOCIATE, Nic.class); + entityEventDetails.put(EVENT_NIC_NETWORK_DISASSOCIATE, Nic.class); + entityEventDetails.put(EVENT_NIC_PRIMARY_NETWORK_CHANGE, Nic.class); // Load Balancers entityEventDetails.put(EVENT_ASSIGN_TO_LOAD_BALANCER_RULE, FirewallRule.class); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 51c8ffcb4dee..d68d4eb806cf 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -39,6 +39,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -6620,6 +6621,7 @@ private void verifyTemplate(BaseDeployVMCmd cmd, VirtualMachineTemplate template if (TemplateType.VNF.equals(template.getTemplateType())) { if (!_itMgr.isBlankInstance(template)) { vnfTemplateManager.validateVnfApplianceNics(template, cmd.getNetworkIds(), cmd.getVmNetworkMap()); + vnfTemplateManager.validateVnfApplianceTrunkNics(template, cmd.getNicNetworksList()); } } else if (cmd instanceof DeployVnfApplianceCmd) { throw new InvalidParameterValueException("Can't deploy VNF appliance from a non-VNF template"); @@ -6847,6 +6849,8 @@ private UserVm createVirtualMachine(BaseDeployVMCmd cmd, DataCenter zone, Accoun } } + persistAdditionalNicNetworkAssociations(cmd, vm); + // check if this templateId has a child ISO List child_templates = _templateDao.listByParentTemplatetId(template.getId()); for (VMTemplateVO tmpl: child_templates) { @@ -6890,6 +6894,36 @@ private UserVm createVirtualMachine(BaseDeployVMCmd cmd, DataCenter zone, Accoun return vm; } + /** + * Persists nicnetworkslist's additional (non-primary) networks into nic_network_map for each newly-created nic, + * including any per-network requested IPs (nicnetworkslist[N].ip4addresses/ip6addresses) - a network with no + * requested IP auto-allocates, same as before this was added. The primary network of each nicnetworkslist entry + * already flows through the ordinary networkIds/ipToNetworkMap pipeline above unchanged, so this only handles + * the associations that pipeline has no concept of. No live agent update is issued - the vm hasn't started yet, + * so its first plug already reads these associations fresh. + */ + private void persistAdditionalNicNetworkAssociations(BaseDeployVMCmd cmd, UserVm vm) + throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException { + List nicNetworksList = cmd.getNicNetworksList(); + if (CollectionUtils.isEmpty(nicNetworksList)) { + return; + } + List nics = _nicDao.listByVmId(vm.getId()); + nics.sort(Comparator.comparingInt(NicVO::getDeviceId)); + if (nics.size() != nicNetworksList.size()) { + throw new CloudRuntimeException(String.format( + "Instance %s was created with %d nic(s) but %d %s entries were requested", + vm.getUuid(), nics.size(), nicNetworksList.size(), ApiConstants.NIC_NETWORKS_LIST)); + } + for (int i = 0; i < nics.size(); i++) { + BaseDeployVMCmd.NicNetworkGrouping grouping = nicNetworksList.get(i); + List additionalNetworkIds = grouping.getAssociatedNetworkIds(); + if (!additionalNetworkIds.isEmpty()) { + networkService.associateNetworksToNic(nics.get(i), additionalNetworkIds, grouping.getAssociatedNetworkIps()); + } + } + } + protected void validateLeaseProperties(Integer leaseDuration, VMLeaseManager.ExpiryAction leaseExpiryAction) { if (ObjectUtils.allNull(leaseDuration, leaseExpiryAction) // both are null || (leaseDuration != null && leaseDuration == -1)) { // special condition to disable lease for instance From df74bd94d22f74ec91d4810a8a1af491b431d322 Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Mon, 28 Sep 2026 14:51:35 -0400 Subject: [PATCH 11/20] Support multi-VLAN trunk NICs at VM deploy time - New nicnetworkslist deploy param (nicnetworkslist[N].networkids, first id primary) lets a VM be deployed with trunk nics directly, mutually exclusive with networkids/iptonetworklist/vApp nicnetworklist - Also carries per-entry ip4address/ip6address for the primary and ip4addresses/ip6addresses (comma-separated) for its associated networks - a network with no requested IP auto-allocates - VNF: rejects a management-device nic from being requested as a trunk, both at deploy time and via the live associate API --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../api/command/user/vm/BaseDeployVMCmd.java | 157 ++++++++++++++++++ .../storage/template/VnfTemplateManager.java | 8 + .../api/command/user/vm/DeployVMCmdTest.java | 93 +++++++++++ .../template/VnfTemplateManagerImpl.java | 17 ++ 5 files changed, 276 insertions(+) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 265431813f70..c822f80c54f1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -422,6 +422,7 @@ public class ApiConstants { public static final String NIC = "nic"; public static final String NICS = "nics"; public static final String NIC_NETWORK_LIST = "nicnetworklist"; + public static final String NIC_NETWORKS_LIST = "nicnetworkslist"; public static final String NIC_IP_ADDRESS_LIST = "nicipaddresslist"; public static final String NIC_MULTIQUEUE_NUMBER = "nicmultiqueuenumber"; public static final String NIC_PACKED_VIRTQUEUES_ENABLED = "nicpackedvirtqueuesenabled"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java index 8d79ac4af45f..81b1debd1d8b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java @@ -19,7 +19,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -190,6 +193,15 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme + " Example: iptonetworklist[0].ip=10.10.10.11&iptonetworklist[0].ipv6=fc00:1234:5678::abcd&iptonetworklist[0].networkid=uuid&iptonetworklist[0].mac=aa:bb:cc:dd:ee::ff - requests to use ip 10.10.10.11 in network id=uuid") private Map ipToNetworkList; + @Parameter(name = ApiConstants.NIC_NETWORKS_LIST, type = CommandType.MAP, since = "24.0.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin}, + description = "one entry per nic, each naming the set of networks that nic is associated with as a multi-VLAN trunk. The first network id in each entry is the nic's primary network." + + " Can't be specified with networkIds or ipToNetworkList parameters." + + " Optional ip4addresses/ip6addresses request specific IPs for the entry's associated (non-primary) networks - comma-separated," + + " positionally aligned to networkids after its first (primary) entry; a blank token auto-allocates that network's IP, same as omitting it." + + " Example: nicnetworkslist[0].networkids=uuid1,uuid2,uuid3&nicnetworkslist[0].ip4address=10.10.10.11&nicnetworkslist[0].ip4addresses=10.10.20.5,10.10.30.5") + private Map nicNetworksList; + @Parameter(name = ApiConstants.IP_ADDRESS, type = CommandType.STRING, description = "the ip address for default vm's network") private String ipAddress; @@ -510,6 +522,18 @@ public VMLeaseManager.ExpiryAction getLeaseExpiryAction() { } public List getNetworkIds() { + if (MapUtils.isNotEmpty(nicNetworksList)) { + if (CollectionUtils.isNotEmpty(networkIds) || ipAddress != null || getIp6Address() != null + || MapUtils.isNotEmpty(ipToNetworkList) || MapUtils.isNotEmpty(vAppNetworks)) { + throw new InvalidParameterValueException(String.format("%s can't be specified along with %s, %s, %s, %s", + ApiConstants.NIC_NETWORKS_LIST, ApiConstants.NETWORK_IDS, ApiConstants.IP_ADDRESS, ApiConstants.IP_NETWORK_LIST, ApiConstants.NIC_NETWORK_LIST)); + } + List networks = new ArrayList<>(); + for (NicNetworkGrouping grouping : getNicNetworksList()) { + networks.add(grouping.getPrimaryNetworkId()); + } + return networks; + } if (MapUtils.isNotEmpty(vAppNetworks)) { if (CollectionUtils.isNotEmpty(networkIds) || ipAddress != null || getIp6Address() != null || MapUtils.isNotEmpty(ipToNetworkList)) { throw new InvalidParameterValueException(String.format("%s can't be specified along with %s, %s, %s", ApiConstants.NIC_NETWORK_LIST, ApiConstants.NETWORK_IDS, ApiConstants.IP_ADDRESS, ApiConstants.IP_NETWORK_LIST)); @@ -624,6 +648,20 @@ public boolean getStartVm() { } public Map getIpToNetworkMap() { + if (MapUtils.isNotEmpty(nicNetworksList)) { + LinkedHashMap ipToNetworkMap = new LinkedHashMap<>(); + for (NicNetworkGrouping grouping : getNicNetworksList()) { + if (grouping.getIp4Address() == null && grouping.getIp6Address() == null && grouping.getMacAddress() == null) { + continue; + } + HashMap ips = new HashMap<>(); + ips.put("ip", grouping.getIp4Address()); + ips.put("ipv6", grouping.getIp6Address()); + ips.put("mac", grouping.getMacAddress()); + ipToNetworkMap.put(grouping.getPrimaryNetworkId(), getIpAddressesFromIpMap(ips)); + } + return ipToNetworkMap.isEmpty() ? null : ipToNetworkMap; + } if ((networkIds != null || ipAddress != null || getIp6Address() != null) && ipToNetworkList != null) { throw new InvalidParameterValueException("NetworkIds and ipAddress can't be specified along with ipToNetworkMap parameter"); } @@ -643,6 +681,125 @@ public Map getIpToNetworkMap() { return ipToNetworkMap; } + /** + * Parses nicnetworkslist into one grouping per requested nic, each naming the full ordered set of networks + * (primary first) that nic is associated with. Legacy networkIds/ipToNetworkList/vAppNetworks are untouched + * when nicnetworkslist is absent - this only returns a non-empty list when it was actually supplied. + *

    + * Entries are read in ascending order of their nicnetworkslist[N] index, not map iteration order - the index + * is what both the deploy-time nic creation order (device id) and VNF's per-device-id validation rely on, and + * the underlying request map is not guaranteed to preserve insertion order. + */ + public List getNicNetworksList() { + List groupings = new ArrayList<>(); + if (MapUtils.isEmpty(nicNetworksList)) { + return groupings; + } + // the request-binding framework hands back these keys as Integer, not String - sort on the parsed value + // directly rather than assuming a type, so this works regardless of which one it actually is + List indices = new ArrayList<>(nicNetworksList.keySet()); + try { + indices.sort(Comparator.comparingInt(index -> Integer.parseInt(index.toString()))); + } catch (NumberFormatException e) { + throw new InvalidParameterValueException(String.format("%s indices must be integers", ApiConstants.NIC_NETWORKS_LIST)); + } + for (Object index : indices) { + HashMap entry = (HashMap) nicNetworksList.get(index); + String networkIdsCsv = entry.get("networkids"); + if (StringUtils.isBlank(networkIdsCsv)) { + throw new InvalidParameterValueException(String.format("%s entries must specify networkids", ApiConstants.NIC_NETWORKS_LIST)); + } + List resolvedNetworkIds = new ArrayList<>(); + for (String token : networkIdsCsv.split(",")) { + HashMap networkIdMap = new HashMap<>(); + networkIdMap.put("networkid", token.trim()); + resolvedNetworkIds.add(getNetworkIdFomIpMap(networkIdMap)); + } + if (resolvedNetworkIds.size() != new HashSet<>(resolvedNetworkIds).size()) { + throw new InvalidParameterValueException(String.format("%s entry lists the same network more than once: %s", ApiConstants.NIC_NETWORKS_LIST, networkIdsCsv)); + } + Map associatedNetworkIps = parseAssociatedNetworkIps(resolvedNetworkIds, entry.get("ip4addresses"), entry.get("ip6addresses")); + groupings.add(new NicNetworkGrouping(resolvedNetworkIds, entry.get("ip4address"), entry.get("ip6address"), entry.get("macaddress"), associatedNetworkIps)); + } + return groupings; + } + + /** + * Parses a nicnetworkslist[N] entry's optional ip4addresses/ip6addresses - comma-separated, positionally + * aligned to that entry's networkids starting from its 2nd id (the associated networks; the primary's own + * ip4address/ip6address is handled separately). A blank token, or a shorter list than the associated-network + * count, leaves that network to auto-allocate - same as not requesting an IP for it at all. + */ + private Map parseAssociatedNetworkIps(List networkIds, String ip4AddressesCsv, String ip6AddressesCsv) { + if (StringUtils.isBlank(ip4AddressesCsv) && StringUtils.isBlank(ip6AddressesCsv)) { + return Collections.emptyMap(); + } + List associatedNetworkIds = networkIds.subList(1, networkIds.size()); + String[] ip4Tokens = ip4AddressesCsv != null ? ip4AddressesCsv.split(",", -1) : new String[0]; + String[] ip6Tokens = ip6AddressesCsv != null ? ip6AddressesCsv.split(",", -1) : new String[0]; + if (ip4Tokens.length > associatedNetworkIds.size() || ip6Tokens.length > associatedNetworkIds.size()) { + throw new InvalidParameterValueException(String.format( + "%s entry's ip4addresses/ip6addresses can't list more entries than its associated networks (%d)", + ApiConstants.NIC_NETWORKS_LIST, associatedNetworkIds.size())); + } + Map associatedNetworkIps = new HashMap<>(); + for (int i = 0; i < associatedNetworkIds.size(); i++) { + String ip4 = i < ip4Tokens.length ? StringUtils.trimToNull(ip4Tokens[i]) : null; + String ip6 = i < ip6Tokens.length ? StringUtils.trimToNull(ip6Tokens[i]) : null; + if (ip4 != null || ip6 != null) { + associatedNetworkIps.put(associatedNetworkIds.get(i), new IpAddresses(ip4, ip6)); + } + } + return associatedNetworkIps; + } + + /** + * One requested nic's full network association set, primary network first, from a nicnetworkslist entry. + */ + public static class NicNetworkGrouping { + private final List networkIds; + private final String ip4Address; + private final String ip6Address; + private final String macAddress; + private final Map associatedNetworkIps; + + public NicNetworkGrouping(List networkIds, String ip4Address, String ip6Address, String macAddress, Map associatedNetworkIps) { + this.networkIds = networkIds; + this.ip4Address = ip4Address; + this.ip6Address = ip6Address; + this.macAddress = macAddress; + this.associatedNetworkIps = associatedNetworkIps; + } + + public Long getPrimaryNetworkId() { + return networkIds.get(0); + } + + public List getAssociatedNetworkIds() { + return networkIds.subList(1, networkIds.size()); + } + + public String getIp4Address() { + return ip4Address; + } + + public String getIp6Address() { + return ip6Address; + } + + public String getMacAddress() { + return macAddress; + } + + /** + * Requested ip4/ip6 addresses for this entry's associated (non-primary) networks, keyed by network id. + * A network with no entry here auto-allocates, same as today. + */ + public Map getAssociatedNetworkIps() { + return associatedNetworkIps; + } + } + @Nonnull private IpAddresses getIpAddressesFromIpMap(HashMap ips) { String requestedIp = ips.get("ip"); diff --git a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java index 3df59811561b..2232adf51d8e 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java +++ b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java @@ -26,6 +26,7 @@ import com.cloud.uservm.UserVm; import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd; +import org.apache.cloudstack.api.command.user.vm.BaseDeployVMCmd; import org.apache.cloudstack.api.command.user.vm.DeployVnfApplianceCmd; import org.apache.cloudstack.framework.config.ConfigKey; import java.util.List; @@ -45,6 +46,13 @@ public interface VnfTemplateManager { void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds, Map vmNetworkMap); + /** + * Rejects a nicnetworkslist request that would make a VNF template's management nic a multi-VLAN trunk nic - + * management access must stay single-segment. No-op when nicNetworksList is empty (i.e. the legacy + * networkids/deploy-as-is paths, which cannot address a nic as a trunk at all). + */ + void validateVnfApplianceTrunkNics(VirtualMachineTemplate template, List nicNetworksList); + SecurityGroup createSecurityGroupForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner, DeployVnfApplianceCmd cmd); void createIsolatedNetworkRulesForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner, diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java index 09d396e40237..d0b7e4fe94c9 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java @@ -265,6 +265,99 @@ public void testGetNetworkIdsIpToNetworkListAndNetworkIds() { assertTrue(thrownException.getMessage().contains("ipToNetworkMap can't be specified along with networkIds or ipAddress")); } + @Test + public void testGetNetworkIdsNicNetworksList() { + // the request-binding framework hands back these keys as Integer, not String + Map nicNetworksList = new HashMap<>(); + Map entry0 = new HashMap<>(); + entry0.put("networkids", "1,2,3"); + Map entry1 = new HashMap<>(); + entry1.put("networkids", "4"); + nicNetworksList.put(1, entry1); + nicNetworksList.put(0, entry0); + ReflectionTestUtils.setField(cmd, "nicNetworksList", nicNetworksList); + ReflectionTestUtils.setField(cmd, "networkIds", null); + ReflectionTestUtils.setField(cmd, "ipToNetworkList", null); + ReflectionTestUtils.setField(cmd, "vAppNetworks", null); + ReflectionTestUtils.setField(cmd, "ipAddress", null); + ReflectionTestUtils.setField(cmd, "ip6Address", null); + ReflectionTestUtils.setField(cmd, "_networkService", mock(NetworkService.class)); + + List result = cmd.getNetworkIds(); + + // primaries only, in ascending index order (0 then 1) despite being inserted out of order + assertEquals(Arrays.asList(1L, 4L), result); + } + + @Test + public void testGetNicNetworksListResolvesPrimaryAndAssociatedNetworkIds() { + Map nicNetworksList = new HashMap<>(); + Map entry0 = new HashMap<>(); + entry0.put("networkids", "1,2,3"); + nicNetworksList.put(0, entry0); + ReflectionTestUtils.setField(cmd, "nicNetworksList", nicNetworksList); + ReflectionTestUtils.setField(cmd, "_networkService", mock(NetworkService.class)); + + List result = cmd.getNicNetworksList(); + + assertEquals(1, result.size()); + assertEquals(Long.valueOf(1L), result.get(0).getPrimaryNetworkId()); + assertEquals(Arrays.asList(2L, 3L), result.get(0).getAssociatedNetworkIds()); + } + + @Test + public void testGetNicNetworksListParsesAssociatedNetworkIps() { + // ip4addresses/ip6addresses are positionally aligned to networkids after its first (primary) entry - + // here network 2 gets an explicit ip4/ip6, network 3 is left blank (auto-allocates) + Map nicNetworksList = new HashMap<>(); + Map entry0 = new HashMap<>(); + entry0.put("networkids", "1,2,3"); + entry0.put("ip4addresses", "10.1.1.5,"); + entry0.put("ip6addresses", "fd00::5,"); + nicNetworksList.put(0, entry0); + ReflectionTestUtils.setField(cmd, "nicNetworksList", nicNetworksList); + ReflectionTestUtils.setField(cmd, "_networkService", mock(NetworkService.class)); + + List result = cmd.getNicNetworksList(); + + Map associatedNetworkIps = result.get(0).getAssociatedNetworkIps(); + assertEquals(1, associatedNetworkIps.size()); + assertEquals("10.1.1.5", associatedNetworkIps.get(2L).getIp4Address()); + assertEquals("fd00::5", associatedNetworkIps.get(2L).getIp6Address()); + assertFalse(associatedNetworkIps.containsKey(3L)); + } + + @Test + public void testGetNicNetworksListRejectsTooManyAssociatedNetworkIps() { + Map nicNetworksList = new HashMap<>(); + Map entry0 = new HashMap<>(); + entry0.put("networkids", "1,2"); + entry0.put("ip4addresses", "10.1.1.5,10.1.1.6,10.1.1.7"); + nicNetworksList.put(0, entry0); + ReflectionTestUtils.setField(cmd, "nicNetworksList", nicNetworksList); + ReflectionTestUtils.setField(cmd, "_networkService", mock(NetworkService.class)); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getNicNetworksList(); + }); + assertTrue(thrownException.getMessage().contains("ip4addresses")); + } + + @Test + public void testGetNetworkIdsNicNetworksListAndNetworkIds() { + Map nicNetworksList = new HashMap<>(); + Map entry0 = new HashMap<>(); + entry0.put("networkids", "1,2"); + nicNetworksList.put(0, entry0); + ReflectionTestUtils.setField(cmd, "nicNetworksList", nicNetworksList); + ReflectionTestUtils.setField(cmd, "networkIds", Arrays.asList(1L, 2L)); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getNetworkIds(); + }); + assertTrue(thrownException.getMessage().contains("nicnetworkslist")); + } + @Test public void testGetIpToNetworkMap_WithNetworkIds() { ReflectionTestUtils.setField(cmd, "networkIds", Arrays.asList(1L, 2L)); diff --git a/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java index 0ebff237a44e..7b4fd7526a96 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java @@ -76,6 +76,7 @@ import org.apache.cloudstack.api.command.user.template.ListVnfTemplatesCmd; import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd; +import org.apache.cloudstack.api.command.user.vm.BaseDeployVMCmd; import org.apache.cloudstack.api.command.user.vm.DeployVnfApplianceCmd; import org.apache.cloudstack.api.command.user.vm.ListVnfAppliancesCmd; import org.apache.cloudstack.framework.config.ConfigKey; @@ -220,6 +221,22 @@ public void validateVnfApplianceNics(VirtualMachineTemplate template, List } } + @Override + public void validateVnfApplianceTrunkNics(VirtualMachineTemplate template, List nicNetworksList) { + if (CollectionUtils.isEmpty(nicNetworksList)) { + return; + } + List vnfNics = vnfTemplateNicDao.listByTemplateId(template.getId()); + for (VnfTemplateNicVO vnfNic : vnfNics) { + if (!vnfNic.isManagement() || vnfNic.getDeviceId() >= nicNetworksList.size()) { + continue; + } + if (!nicNetworksList.get((int) vnfNic.getDeviceId()).getAssociatedNetworkIds().isEmpty()) { + throw new InvalidParameterValueException("VNF nic is the management interface and cannot be a multi-VLAN trunk nic: " + vnfNic); + } + } + } + private void validateVnfApplianceNetworksMap(VirtualMachineTemplate template, Map vmNetworkMap) { if (MapUtils.isEmpty(vmNetworkMap)) { throw new InvalidParameterValueException("VNF networks map is empty"); From 3a7519db255f3c7fbda87989dcfaf90b0e607dd9 Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Mon, 28 Sep 2026 14:53:42 -0400 Subject: [PATCH 12/20] Fix static NAT lookup in disassociate guard - checkNoActiveRulesOnAssociation used findByIpAndNetworkId, which matches on a public IP's own address - the association's IP is a guest IP, so this never matched and the guard silently let disassociation through even with an active static NAT rule - Use findByAssociatedVmIdAndVmIp instead, which matches the actual static NAT target mapping --- .../com/cloud/network/NetworkServiceImpl.java | 2 +- .../com/cloud/network/NetworkServiceImplTest.java | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index a0b8c12d2096..a70ea6dc944c 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -949,7 +949,7 @@ private void checkNoActiveRulesOnAssociation(long vmId, long networkId, String i "Can't disassociate network %s, its allocated ip %s is associated with a port forwarding rule", networkId, ip4Address)); } } - IPAddressVO publicIpVO = _ipAddressDao.findByIpAndNetworkId(networkId, ip4Address); + IPAddressVO publicIpVO = _ipAddressDao.findByAssociatedVmIdAndVmIp(vmId, ip4Address); if (publicIpVO != null) { throw new InvalidParameterValueException(String.format( "Can't disassociate network %s, its allocated ip %s is associated with static NAT rule public IP address ID: %s", networkId, ip4Address, publicIpVO)); diff --git a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java index 334a73cef00c..70c48b767af4 100644 --- a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java @@ -1953,6 +1953,21 @@ public void disassociateNetworkFromNicFailsWhenNotAssociated() throws Exception service.disassociateNetworkFromNic(mockDisassociateCmd(11L, 206L)); } + @Test(expected = InvalidParameterValueException.class) + public void disassociateNetworkFromNicFailsWhenAssociationIpHasActiveStaticNat() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(vm.getId()).thenReturn(5L); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L, "10.3.1.25", null); + Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(association); + IPAddressVO staticNatIp = Mockito.mock(IPAddressVO.class); + Mockito.when(ipAddressDao.findByAssociatedVmIdAndVmIp(5L, "10.3.1.25")).thenReturn(staticNatIp); + + service.disassociateNetworkFromNic(mockDisassociateCmd(11L, 206L)); + } + @Test public void disassociateNetworkFromNicRemovesAssociationOnAStoppedVm() throws Exception { NicVO nic = mockNic(11L, 5L, 205L, true); From 68d7771fd5ca55068664b59bc29af261130554b7 Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Mon, 28 Sep 2026 14:56:36 -0400 Subject: [PATCH 13/20] Fix VNF management nic trunkable via live associate API - validateVnfApplianceTrunkNics only ran at deploy time (nicnetworkslist) - the live associateNetworkToNic/associateNetworksToNic path had no equivalent check, so a VNF's management nic could be trunked after deploy - New single-nic validateVnfApplianceTrunkNic, called from associateNetworksInternal so both entry points are covered - No-op for any non-VNF template --- .../storage/template/VnfTemplateManager.java | 7 +++++ .../com/cloud/network/NetworkServiceImpl.java | 12 ++++++++ .../template/VnfTemplateManagerImpl.java | 10 +++++++ .../cloud/network/NetworkServiceImplTest.java | 28 +++++++++++++++++++ .../template/VnfTemplateManagerImplTest.java | 20 +++++++++++++ 5 files changed, 77 insertions(+) diff --git a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java index 2232adf51d8e..de0cebd3a3a0 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java +++ b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java @@ -53,6 +53,13 @@ public interface VnfTemplateManager { */ void validateVnfApplianceTrunkNics(VirtualMachineTemplate template, List nicNetworksList); + /** + * Single-nic counterpart of {@link #validateVnfApplianceTrunkNics}, for the live associate-network-to-nic + * path (an already-deployed VNF's nic, rather than a deploy-time nicnetworkslist). No-op when the template + * has no VNF nic metadata at all, i.e. any non-VNF template. + */ + void validateVnfApplianceTrunkNic(VirtualMachineTemplate template, long deviceId); + SecurityGroup createSecurityGroupForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner, DeployVnfApplianceCmd cmd); void createIsolatedNetworkRulesForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner, diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index a70ea6dc944c..e67207a97bd7 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -83,6 +83,7 @@ import org.apache.cloudstack.network.dao.NetworkPermissionDao; import org.apache.cloudstack.network.element.InternalLoadBalancerElementService; import org.apache.cloudstack.reservation.dao.ReservationDao; +import org.apache.cloudstack.storage.template.VnfTemplateManager; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.BooleanUtils; @@ -225,6 +226,8 @@ import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.user.Account; @@ -458,6 +461,10 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C private BGPService bgpService; @Inject private ASNumberDao asNumberDao; + @Inject + VMTemplateDao _templateDao; + @Inject + VnfTemplateManager vnfTemplateManager; List internalLoadBalancerElementServices = new ArrayList<>(); Map internalLoadBalancerElementServiceMap = new HashMap<>(); @@ -994,6 +1001,11 @@ private void associateNetworksInternal(NicVO nic, VirtualMachine vm, Account cal throw new CloudRuntimeException(String.format("Nic %s has no primary network", nic.getUuid())); } + VMTemplateVO template = _templateDao.findByIdIncludingRemoved(vm.getTemplateId()); + if (template != null) { + vnfTemplateManager.validateVnfApplianceTrunkNic(template, nic.getDeviceId()); + } + Pair, List> networkSets = buildResultingNetworkSet(vm, nic, primaryNetwork, requestedNetworkIds); List resultingSet = networkSets.first(); List requestedNetworks = networkSets.second(); diff --git a/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java index 7b4fd7526a96..a21f5a7b7277 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java @@ -237,6 +237,16 @@ public void validateVnfApplianceTrunkNics(VirtualMachineTemplate template, List< } } + @Override + public void validateVnfApplianceTrunkNic(VirtualMachineTemplate template, long deviceId) { + List vnfNics = vnfTemplateNicDao.listByTemplateId(template.getId()); + for (VnfTemplateNicVO vnfNic : vnfNics) { + if (vnfNic.isManagement() && vnfNic.getDeviceId() == deviceId) { + throw new InvalidParameterValueException("VNF nic is the management interface and cannot be a multi-VLAN trunk nic: " + vnfNic); + } + } + } + private void validateVnfApplianceNetworksMap(VirtualMachineTemplate template, Map vmNetworkMap) { if (MapUtils.isEmpty(vmNetworkMap)) { throw new InvalidParameterValueException("VNF networks map is empty"); diff --git a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java index 70c48b767af4..5ac347139148 100644 --- a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java @@ -53,6 +53,7 @@ import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.network.RoutedIpv4Manager; +import org.apache.cloudstack.storage.template.VnfTemplateManager; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -126,6 +127,8 @@ import com.cloud.org.Grouping; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; @@ -213,6 +216,10 @@ public class NetworkServiceImplTest { @Mock LoadBalancerDao loadBalancerDao; @Mock + VMTemplateDao templateDao; + @Mock + VnfTemplateManager vnfTemplateManagerMock; + @Mock ConfigurationManager configMgr; @Mock ConfigKey publicMtuKey; @@ -378,6 +385,8 @@ public void setup() throws Exception { service._networkModel = networkModel; service._entityMgr = entityMgr; service._networkMgr = _networkMgr; + service._templateDao = templateDao; + service.vnfTemplateManager = vnfTemplateManagerMock; callContextMocked = Mockito.mockStatic(CallContext.class); // stubbed as a no-op by default: persistAssociations publishes a real UsageEventUtils call on every // first-time trunk conversion, which this class' tests don't otherwise need to know about @@ -1829,6 +1838,25 @@ public void associateNetworksToNicUsesRequestedIpForAssociatedNetwork() throws E map.getNicId() == 11L && map.getNetworkId() == 206L && "10.1.1.99".equals(map.getIp4Address()))); } + @Test(expected = InvalidParameterValueException.class) + public void associateNetworksToNicRejectsVnfManagementNic() throws Exception { + // the live associate-network-to-nic path must reject trunking a VNF template's management nic, same as + // the deploy-time nicnetworkslist path already does via validateVnfApplianceTrunkNics + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(vm.getTemplateId()).thenReturn(99L); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + VMTemplateVO template = Mockito.mock(VMTemplateVO.class); + Mockito.when(templateDao.findByIdIncludingRemoved(99L)).thenReturn(template); + Mockito.doThrow(new InvalidParameterValueException("VNF nic is the management interface")) + .when(vnfTemplateManagerMock).validateVnfApplianceTrunkNic(template, 0L); + + service.associateNetworksToNic(nic, Arrays.asList(206L), null); + } + @Test(expected = InvalidParameterValueException.class) public void associateNetworksToNicRejectsNetworkAlreadyOnAnotherNic() throws Exception { // vm has a second, ordinary nic (208L) whose primary is network 206L - trying to associate 206L onto diff --git a/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java index b9565ebb2922..fe64bc4e8a4f 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java @@ -249,6 +249,26 @@ public void testValidateVnfApplianceNicsWithMissingNetworkId() { vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } + @Test(expected = InvalidParameterValueException.class) + public void testValidateVnfApplianceTrunkNicRejectsManagementDevice() { + when(template.getId()).thenReturn(templateId); + VnfTemplateNicVO vnfNic1 = new VnfTemplateNicVO(templateId, 0L, "eth0", true, true, "first"); + VnfTemplateNicVO vnfNic2 = new VnfTemplateNicVO(templateId, 1L, "eth1", true, false, "second"); + Mockito.doReturn(Arrays.asList(vnfNic1, vnfNic2)).when(vnfTemplateNicDao).listByTemplateId(templateId); + + vnfTemplateManagerImpl.validateVnfApplianceTrunkNic(template, 0L); + } + + @Test + public void testValidateVnfApplianceTrunkNicAllowsDataPlaneDevice() { + when(template.getId()).thenReturn(templateId); + VnfTemplateNicVO vnfNic1 = new VnfTemplateNicVO(templateId, 0L, "eth0", true, true, "first"); + VnfTemplateNicVO vnfNic2 = new VnfTemplateNicVO(templateId, 1L, "eth1", true, false, "second"); + Mockito.doReturn(Arrays.asList(vnfNic1, vnfNic2)).when(vnfTemplateNicDao).listByTemplateId(templateId); + + vnfTemplateManagerImpl.validateVnfApplianceTrunkNic(template, 1L); + } + @Test public void testGetManagementNetworkAndIp() { when(template.getId()).thenReturn(templateId); From 485ad2652c6277bb22b14734475fdb4d9c7437cb Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Mon, 28 Sep 2026 14:58:07 -0400 Subject: [PATCH 14/20] Fix stale nic fields after changing its primary network - changeNicPrimaryNetwork updated networkId and the IPv4/IPv6 address but left gateway, netmask, broadcast/isolation uri, and IPv6 gateway/cidr pointing at the old primary network - wrong indefinitely, since nothing else ever recomputes them - New applyNetworkAddressingToNic refreshes them from the new primary network, mirroring the field derivation used when a nic is first created --- .../com/cloud/network/NetworkServiceImpl.java | 27 +++++++++++++++++++ .../cloud/network/NetworkServiceImplTest.java | 10 +++++++ 2 files changed, 37 insertions(+) diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index e67207a97bd7..fefb65b44d7a 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -1088,6 +1088,11 @@ public Nic changeNicPrimaryNetwork(ChangeNicPrimaryNetworkCmd cmd) { newPrimaryNetworkId, nic.getUuid())); } + NetworkVO newPrimaryNetwork = _networksDao.findById(newPrimaryNetworkId); + if (newPrimaryNetwork == null) { + throw new CloudRuntimeException(String.format("Unable to find network %s", newPrimaryNetworkId)); + } + long oldPrimaryNetworkId = nic.getNetworkId(); String oldPrimaryIp4 = nic.getIPv4Address(); String oldPrimaryIp6 = nic.getIPv6Address(); @@ -1100,11 +1105,33 @@ public Nic changeNicPrimaryNetwork(ChangeNicPrimaryNetworkCmd cmd) { nic.setNetworkId(newPrimaryNetworkId); nic.setIPv4Address(association.getIp4Address()); nic.setIPv6Address(association.getIp6Address()); + applyNetworkAddressingToNic(nic, newPrimaryNetwork); _nicDao.update(nic.getId(), nic); return _nicDao.findById(nic.getId()); } + /** + * Refreshes a nic's network-derived fields (gateway, netmask, broadcast/isolation uri, IPv6 gateway/cidr) to + * match the given network. These are stored on the nic row itself rather than resolved live from networkId, so + * changeNicPrimaryNetwork must update them explicitly - GuestNetworkGuru.reserve() only refreshes + * broadcastUri/isolationUri on the VM's next start, and never touches gateway/netmask/IPv6 fields at all, + * which would otherwise stay wrong indefinitely. Mirrors the same field derivation GuestNetworkGuru.allocate() + * uses when a nic is first created against a network. + */ + private void applyNetworkAddressingToNic(NicVO nic, NetworkVO network) { + nic.setIPv4Gateway(network.getGateway()); + if (network.getCidr() != null) { + nic.setIPv4Netmask(NetUtils.cidr2Netmask(_networkModel.getValidNetworkCidr(network))); + } + nic.setBroadcastUri(network.getBroadcastUri()); + nic.setIsolationUri(network.getBroadcastUri()); + if (network.getIp6Cidr() != null && network.getIp6Gateway() != null) { + nic.setIPv6Cidr(network.getIp6Cidr()); + nic.setIPv6Gateway(network.getIp6Gateway()); + } + } + private void updateLiveVlanTrunkMembership(VirtualMachine vm, NicVO nic) throws ResourceUnavailableException { if (vm.getHostId() == null) { throw new CloudRuntimeException(String.format("Instance %s is Running but has no host assigned", vm.getUuid())); diff --git a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java index 5ac347139148..8b6b3f5f14c4 100644 --- a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java @@ -143,6 +143,7 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Ip; +import com.cloud.utils.net.NetUtils; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.Nic; import com.cloud.vm.NicVO; @@ -2136,6 +2137,11 @@ public void changeNicPrimaryNetworkSwapsPrimaryAndDemotesOldPrimaryToAssociation Mockito.when(userVmDao.findById(5L)).thenReturn(vm); NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L, "10.1.1.50", null); Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(association); + NetworkVO newPrimaryNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(newPrimaryNetwork.getGateway()).thenReturn("10.1.2.1"); + Mockito.when(newPrimaryNetwork.getCidr()).thenReturn("10.1.2.0/24"); + Mockito.when(networkDao.findById(206L)).thenReturn(newPrimaryNetwork); + Mockito.when(networkModel.getValidNetworkCidr(newPrimaryNetwork)).thenReturn("10.1.2.0/24"); Nic result = service.changeNicPrimaryNetwork(mockChangePrimaryCmd(11L, 206L)); @@ -2146,6 +2152,10 @@ public void changeNicPrimaryNetworkSwapsPrimaryAndDemotesOldPrimaryToAssociation Mockito.verify(nic).setNetworkId(206L); Mockito.verify(nic).setIPv4Address("10.1.1.50"); Mockito.verify(nic).setIPv6Address(null); + Mockito.verify(nic).setIPv4Gateway("10.1.2.1"); + Mockito.verify(nic).setIPv4Netmask(NetUtils.cidr2Netmask("10.1.2.0/24")); + Mockito.verify(nic).setBroadcastUri(vlanUri(1180)); + Mockito.verify(nic).setIsolationUri(vlanUri(1180)); Mockito.verify(nicDao).update(Mockito.eq(11L), Mockito.eq(nic)); } From 721bae1929d98ea3321b93dcc1f2776fae11856f Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Tue, 29 Sep 2026 10:10:44 -0400 Subject: [PATCH 15/20] Clear multiNetwork flag when a nic's last trunk association is removed - disassociateNetworkFromNic now clears nics.multi_network once nic_network_map has no remaining rows for that nic - Without this, listNics/listVirtualMachines kept reporting trunked=true and the deployment planner kept restricting the VM to VLAN-filtering- capable hosts even after all associations were removed --- .../com/cloud/network/NetworkServiceImpl.java | 5 +++++ .../cloud/network/NetworkServiceImplTest.java | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index fefb65b44d7a..f45a4bcebbe6 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -1051,6 +1051,11 @@ public Nic disassociateNetworkFromNic(DisassociateNetworkFromNicCmd cmd) throws // so removing this row is itself the release _nicNetworkMapDao.remove(association.getId()); + if (nic.getMultiNetwork() && _nicNetworkMapDao.listByNicId(nic.getId()).isEmpty()) { + nic.setMultiNetwork(false); + _nicDao.update(nic.getId(), nic); + } + if (vmRunning) { updateLiveVlanTrunkMembership(vm, nic); refreshNicVlanMappingMetadata(nic); diff --git a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java index 8b6b3f5f14c4..86c4fdbb7f7d 100644 --- a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java @@ -2005,12 +2005,34 @@ public void disassociateNetworkFromNicRemovesAssociationOnAStoppedVm() throws Ex Mockito.when(userVmDao.findById(5L)).thenReturn(vm); NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L, "10.1.1.50", null); Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(association); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(Collections.emptyList()); Nic result = service.disassociateNetworkFromNic(mockDisassociateCmd(11L, 206L)); Assert.assertNotNull(result); Mockito.verify(nicNetworkMapDao).remove(association.getId()); Mockito.verify(agentMgr, Mockito.never()).easySend(Mockito.anyLong(), Mockito.any()); + Mockito.verify(nic).setMultiNetwork(false); + Mockito.verify(nicDao).update(11L, nic); + } + + @Test + public void disassociateNetworkFromNicKeepsMultiNetworkFlagWhenOtherAssociationsRemain() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L, "10.1.1.50", null); + Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(association); + NicNetworkMapVO remainingAssociation = new NicNetworkMapVO(11L, 207L, "10.1.1.60", null); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(Arrays.asList(remainingAssociation)); + + Nic result = service.disassociateNetworkFromNic(mockDisassociateCmd(11L, 206L)); + + Assert.assertNotNull(result); + Mockito.verify(nicNetworkMapDao).remove(association.getId()); + Mockito.verify(nic, Mockito.never()).setMultiNetwork(Mockito.anyBoolean()); + Mockito.verify(nicDao, Mockito.never()).update(Mockito.eq(11L), Mockito.any()); } @Test From 1a3ddfbe6cdd0d0c16ea15ed7fa179f23a08d8be Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Tue, 29 Sep 2026 12:53:04 -0400 Subject: [PATCH 16/20] Push DHCP entries to a trunk nic's associated networks - New CommandSetupHelper.createDhcpEntryCommand overload takes addressing explicitly instead of reading it off a NicVO, since a nic's own DB row only ever carries its primary network's IP/gateway - NetworkServiceImpl sends the entry directly to the associated network's router(s) on live associate/disassociate, bypassing the DhcpServiceProvider pipeline (which re-resolves the nic from the DB and so can never see anything but the primary IP) - UserVmManagerImpl.finalizeStart converges a nic's associated-network DHCP entries with its current nic_network_map rows on every VM start, covering associations made while the VM was stopped and cleaning up entries for associations removed while stopped - New NicNetworkMapDao.listRemovedByNicId to support that cleanup --- .../com/cloud/vm/dao/NicNetworkMapDao.java | 2 + .../cloud/vm/dao/NicNetworkMapDaoImpl.java | 8 ++ .../com/cloud/network/NetworkServiceImpl.java | 47 +++++++-- .../network/router/CommandSetupHelper.java | 23 +++-- .../java/com/cloud/vm/UserVmManagerImpl.java | 36 +++++++ .../cloud/network/NetworkServiceImplTest.java | 99 +++++++++++++++++++ .../com/cloud/vm/UserVmManagerImplTest.java | 69 +++++++++++++ 7 files changed, 270 insertions(+), 14 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDao.java index b5a1ed3137af..99aa445a7199 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDao.java @@ -24,6 +24,8 @@ public interface NicNetworkMapDao extends GenericDao { List listByNicId(long nicId); + List listRemovedByNicId(long nicId); + List listByNetworkId(long networkId); NicNetworkMapVO findByNicIdAndNetworkId(long nicId, long networkId); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDaoImpl.java index f905809350e5..c8c523e95593 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicNetworkMapDaoImpl.java @@ -53,6 +53,14 @@ public List listByNicId(long nicId) { return listBy(sc); } + @Override + public List listRemovedByNicId(long nicId) { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("nicId", SearchCriteria.Op.EQ, nicId); + sc.addAnd("removed", SearchCriteria.Op.NNULL); + return listIncludingRemovedBy(sc); + } + @Override public List listByNetworkId(long networkId) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index f45a4bcebbe6..7c2e356fb28f 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -240,6 +240,7 @@ import com.cloud.user.UserVO; import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.UserDao; +import com.cloud.uservm.UserVm; import com.cloud.utils.Journal; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; @@ -931,11 +932,14 @@ public Nic associateNetworkToNic(AssociateNetworkToNicCmd cmd) checkHostReadyForVlanTrunk(vm); } - associateNetworksInternal(nic, vm, caller, cmd.getNetworkIds(), cmd.getIpAddressesMap()); + List newlyAssociatedNetworks = associateNetworksInternal(nic, vm, caller, cmd.getNetworkIds(), cmd.getIpAddressesMap()); if (vmRunning) { updateLiveVlanTrunkMembership(vm, nic); refreshNicVlanMappingMetadata(nic); + for (NetworkVO network : newlyAssociatedNetworks) { + pushDhcpEntryForAssociation(vm, nic, network); + } } return _nicDao.findById(nic.getId()); @@ -986,11 +990,7 @@ public void associateNetworksToNic(Nic nic, List networkIds, Map requestedNetworkIds, Map ipAddressesMap) + private List associateNetworksInternal(NicVO nic, VirtualMachine vm, Account caller, List requestedNetworkIds, Map ipAddressesMap) throws ResourceUnavailableException, InsufficientCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException { if (requestedNetworkIds == null || requestedNetworkIds.isEmpty()) { throw new InvalidParameterValueException("networkids must not be empty"); @@ -1015,6 +1015,7 @@ private void associateNetworksInternal(NicVO nic, VirtualMachine vm, Account cal implementRequestedNetworks(requestedNetworks, vm, caller); validateNoDuplicateVlanAfterImplement(resultingSet); persistAssociations(nic, vm, requestedNetworks, ipAddressesMap); + return requestedNetworks; } @Override @@ -1057,6 +1058,7 @@ public Nic disassociateNetworkFromNic(DisassociateNetworkFromNicCmd cmd) throws } if (vmRunning) { + cleanupDhcpEntryForAssociation(vm, nic, targetNetworkId, association.getIp4Address(), association.getIp6Address()); updateLiveVlanTrunkMembership(vm, nic); refreshNicVlanMappingMetadata(nic); } @@ -1137,6 +1139,39 @@ private void applyNetworkAddressingToNic(NicVO nic, NetworkVO network) { } } + // Pushes a DHCP entry for a newly-associated network on a Running Instance. + private void pushDhcpEntryForAssociation(VirtualMachine vm, NicVO nic, NetworkVO network) throws ResourceUnavailableException { + NicNetworkMapVO association = _nicNetworkMapDao.findByNicIdAndNetworkId(nic.getId(), network.getId()); + if (association == null) { + return; + } + sendDhcpEntryCommandForAssociation(vm, nic, network, association.getIp4Address(), association.getIp6Address(), false); + } + + // Counterpart to pushDhcpEntryForAssociation, for a Running Instance's disassociate. + private void cleanupDhcpEntryForAssociation(VirtualMachine vm, NicVO nic, long networkId, String ip4Address, String ip6Address) throws ResourceUnavailableException { + NetworkVO network = _networksDao.findById(networkId); + if (network == null) { + return; + } + sendDhcpEntryCommandForAssociation(vm, nic, network, ip4Address, ip6Address, true); + } + + // Bypasses the DhcpServiceProvider/NetworkElement pipeline: DhcpEntryRules re-fetches the nic's own DB row + // for its IP, which only ever holds the primary network's address, never an association's. + private void sendDhcpEntryCommandForAssociation(VirtualMachine vm, NicVO nic, NetworkVO network, String ip4Address, String ip6Address, boolean remove) + throws ResourceUnavailableException { + for (DomainRouterVO router : routerDao.findByNetwork(network.getId())) { + if (router.getState() != VirtualMachine.State.Running) { + continue; + } + Commands cmds = new Commands(Command.OnError.Stop); + commandSetupHelper.createDhcpEntryCommand(router, (UserVm) vm, nic.getMacAddress(), ip4Address, ip6Address, + network.getGateway(), network.getIp6Gateway(), network.getId(), false, remove, cmds); + networkHelper.sendCommandsToRouter(router, cmds); + } + } + private void updateLiveVlanTrunkMembership(VirtualMachine vm, NicVO nic) throws ResourceUnavailableException { if (vm.getHostId() == null) { throw new CloudRuntimeException(String.format("Instance %s is Running but has no host assigned", vm.getUuid())); diff --git a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java index a09e2f48381b..d30b3968aca4 100644 --- a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java +++ b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java @@ -284,23 +284,30 @@ public void createApplyVpnUsersCommand(final List users, fina } public void createDhcpEntryCommand(final VirtualRouter router, final UserVm vm, final NicVO nic, boolean remove, final Commands cmds) { - final DhcpEntryCommand dhcpCommand = new DhcpEntryCommand(nic.getMacAddress(), nic.getIPv4Address(), vm.getHostName(), nic.getIPv6Address(), - _networkModel.getExecuteInSeqNtwkElmtCmd()); + createDhcpEntryCommand(router, vm, nic.getMacAddress(), nic.getIPv4Address(), nic.getIPv6Address(), nic.getIPv4Gateway(), nic.getIPv6Gateway(), + nic.getNetworkId(), nic.isDefaultNic(), remove, cmds); + } - String gatewayIp = nic.getIPv4Gateway(); + // Same as createDhcpEntryCommand(router, vm, NicVO, ...), but for a nic's association with a network other + // than its primary (multi-VLAN trunk nics) - takes the addressing explicitly, since a nic's own DB row only + // ever carries its primary network's IP/gateway. + public void createDhcpEntryCommand(final VirtualRouter router, final UserVm vm, final String macAddress, final String ip4Address, final String ip6Address, + final String ip4Gateway, final String ip6Gateway, final long networkId, final boolean isDefaultNic, boolean remove, final Commands cmds) { + final DhcpEntryCommand dhcpCommand = new DhcpEntryCommand(macAddress, ip4Address, vm.getHostName(), ip6Address, + _networkModel.getExecuteInSeqNtwkElmtCmd()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); - dhcpCommand.setDefaultRouter(gatewayIp); - dhcpCommand.setIp6Gateway(nic.getIPv6Gateway()); + dhcpCommand.setDefaultRouter(ip4Gateway); + dhcpCommand.setIp6Gateway(ip6Gateway); String ipaddress = null; final NicVO domrDefaultNic = findDefaultDnsIp(vm.getId()); if (domrDefaultNic != null) { ipaddress = domrDefaultNic.getIPv4Address(); } dhcpCommand.setDefaultDns(ipaddress); - dhcpCommand.setDuid(NetUtils.getDuidLL(nic.getMacAddress())); - dhcpCommand.setDefault(nic.isDefaultNic()); + dhcpCommand.setDuid(NetUtils.getDuidLL(macAddress)); + dhcpCommand.setDefault(isDefaultNic); dhcpCommand.setRemove(remove); // Set DHCP lease timeout from zone-scoped config (0 = infinite) @@ -309,7 +316,7 @@ public void createDhcpEntryCommand(final VirtualRouter router, final UserVm vm, dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId())); dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, _routerControlHelper.getRouterIpInNetwork(nic.getNetworkId(), router.getId())); + dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, _routerControlHelper.getRouterIpInNetwork(networkId, router.getId())); dhcpCommand.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("dhcp", dhcpCommand); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index d68d4eb806cf..f62b1846b3c4 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -423,6 +423,8 @@ import com.cloud.vm.dao.InstanceGroupVMMapDao; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.NicExtraDhcpOptionDao; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.dao.VMInstanceDetailsDao; @@ -525,6 +527,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Inject private NicDao _nicDao; @Inject + private NicNetworkMapDao _nicNetworkMapDao; + @Inject private RulesManager _rulesMgr; @Inject private LoadBalancingRulesManager _lbMgr; @@ -3481,6 +3485,37 @@ protected void updateUserData(UserVm vm) throws ResourceUnavailableException, In } } + // Converges a trunk nic's associated-network DHCP entries with its current nic_network_map rows on start; + // no-op for a nic without associations. + void syncNicNetworkAssociationDhcpEntries(UserVmVO vm, NicVO nic) { + for (NicNetworkMapVO association : _nicNetworkMapDao.listByNicId(nic.getId())) { + sendAssociationDhcpEntry(vm, nic, association.getNetworkId(), association.getIp4Address(), association.getIp6Address(), false); + } + for (NicNetworkMapVO removedAssociation : _nicNetworkMapDao.listRemovedByNicId(nic.getId())) { + sendAssociationDhcpEntry(vm, nic, removedAssociation.getNetworkId(), removedAssociation.getIp4Address(), removedAssociation.getIp6Address(), true); + } + } + + private void sendAssociationDhcpEntry(UserVmVO vm, NicVO nic, long networkId, String ip4Address, String ip6Address, boolean remove) { + NetworkVO network = _networkDao.findById(networkId); + if (network == null) { + return; + } + for (DomainRouterVO router : _routerDao.findByNetwork(networkId)) { + if (router.getState() != State.Running) { + continue; + } + try { + Commands commands = new Commands(Command.OnError.Stop); + commandSetupHelper.createDhcpEntryCommand(router, vm, nic.getMacAddress(), ip4Address, ip6Address, + network.getGateway(), network.getIp6Gateway(), networkId, false, remove, commands); + nwHelper.sendCommandsToRouter(router, commands); + } catch (ResourceUnavailableException e) { + logger.warn("Failed to sync DHCP entry for nic {} on associated network {}: {}", nic.getUuid(), networkId, e.getMessage(), e); + } + } + } + private void updateDns(UserVmVO vm, String hostName) throws ResourceUnavailableException, InsufficientCapacityException { if (!StringUtils.isEmpty(hostName)) { vm.setHostName(hostName); @@ -5801,6 +5836,7 @@ public boolean finalizeStart(VirtualMachineProfile profile, long hostId, Command long isDefault = (nic.isDefaultNic()) ? 1 : 0; UsageEventUtils.publishNicNetworkOfferingUsageEvents(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), VirtualMachine.class.getName(), vm.getUuid(), nic, isDefault, vm.isDisplay()); + syncNicNetworkAssociationDhcpEntries(vm, nic); if (network.getTrafficType() == TrafficType.Guest) { originalIp = nic.getIPv4Address(); guestNic = nic; diff --git a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java index 86c4fdbb7f7d..008ce3f4dc5d 100644 --- a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java @@ -1577,6 +1577,59 @@ public void associateNetworkToNicUpdatesLiveVlanMembershipOnARunningVmWhenHostIs Mockito.verify(networkHelper).sendCommandsToRouter(Mockito.eq(router), Mockito.any()); } + @Test + public void associateNetworkToNicPushesDhcpEntryForNewlyAssociatedNetworkOnARunningVm() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Running, 1L); + Mockito.when(vm.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + Mockito.when(vm.getInstanceName()).thenReturn("i-2-11-VM"); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + DetailVO readyDetail = Mockito.mock(DetailVO.class); + Mockito.when(readyDetail.getValue()).thenReturn("true"); + Mockito.when(hostDetailsDao.findDetail(1L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(readyDetail); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO requestedNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(requestedNetwork); + Mockito.when(ipAddressManagerMock.allocateGuestIP(requestedNetwork, null)).thenReturn("10.1.1.50"); + Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(new NicNetworkMapVO(11L, 206L, "10.1.1.50", null)); + Mockito.when(networkModel.getNicProfile(Mockito.eq(vm), Mockito.eq(nic), Mockito.any())).thenReturn(null); + Mockito.when(hvGuruMgr.getGuru(Hypervisor.HypervisorType.KVM)).thenReturn(hypervisorGuru); + Mockito.when(hypervisorGuru.toNicTO(null)).thenReturn(Mockito.mock(NicTO.class)); + Mockito.when(agentMgr.easySend(Mockito.eq(1L), Mockito.any())).thenReturn(new UpdateNicVlanMembershipAnswer(null, true, "success")); + Mockito.when(routerDao.findByNetwork(205L)).thenReturn(new ArrayList<>()); + DomainRouterVO associatedNetworkRouter = Mockito.mock(DomainRouterVO.class); + Mockito.when(associatedNetworkRouter.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(routerDao.findByNetwork(206L)).thenReturn(Arrays.asList(associatedNetworkRouter)); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + + Mockito.verify(commandSetupHelper).createDhcpEntryCommand(Mockito.eq(associatedNetworkRouter), Mockito.eq(vm), Mockito.any(), Mockito.eq("10.1.1.50"), + Mockito.any(), Mockito.any(), Mockito.any(), Mockito.eq(206L), Mockito.eq(false), Mockito.eq(false), Mockito.any()); + Mockito.verify(networkHelper).sendCommandsToRouter(Mockito.eq(associatedNetworkRouter), Mockito.any()); + } + + @Test + public void associateNetworkToNicDoesNotPushDhcpEntryOnAStoppedVm() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, false); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NetworkVO primaryNetwork = mockNetwork(205L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1174)); + Mockito.when(networkDao.findById(205L)).thenReturn(primaryNetwork); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(new ArrayList<>()); + NetworkVO requestedNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(requestedNetwork); + Mockito.when(ipAddressManagerMock.allocateGuestIP(requestedNetwork, null)).thenReturn("10.1.1.50"); + + service.associateNetworkToNic(mockAssociateCmd(11L, Arrays.asList(206L))); + + Mockito.verify(commandSetupHelper, Mockito.never()).createDhcpEntryCommand(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.anyString(), + Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong(), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.any()); + } + @Test public void associateNetworkToNicDoesNotRefreshMetadataOnRoutersThatAreNotRunning() throws Exception { NicVO nic = mockNic(11L, 5L, 205L, false); @@ -2065,6 +2118,52 @@ public void disassociateNetworkFromNicUpdatesLiveVlanMembershipOnARunningVm() th Mockito.verify(networkHelper).sendCommandsToRouter(Mockito.eq(router), Mockito.any()); } + @Test + public void disassociateNetworkFromNicRemovesDhcpEntryOnARunningVm() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Running, 1L); + Mockito.when(vm.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + Mockito.when(vm.getInstanceName()).thenReturn("i-2-11-VM"); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + DetailVO readyDetail = Mockito.mock(DetailVO.class); + Mockito.when(readyDetail.getValue()).thenReturn("true"); + Mockito.when(hostDetailsDao.findDetail(1L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(readyDetail); + NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L, "10.1.1.50", null); + Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(association); + NetworkVO associatedNetwork = mockNetwork(206L, Network.GuestType.Isolated, 1L, 1L, null, Networks.BroadcastDomainType.Vlan, vlanUri(1180)); + Mockito.when(networkDao.findById(206L)).thenReturn(associatedNetwork); + Mockito.when(networkModel.getNicProfile(Mockito.eq(vm), Mockito.eq(nic), Mockito.any())).thenReturn(null); + Mockito.when(hvGuruMgr.getGuru(Hypervisor.HypervisorType.KVM)).thenReturn(hypervisorGuru); + Mockito.when(hypervisorGuru.toNicTO(null)).thenReturn(Mockito.mock(NicTO.class)); + Mockito.when(agentMgr.easySend(Mockito.eq(1L), Mockito.any())).thenReturn(new UpdateNicVlanMembershipAnswer(null, true, "success")); + Mockito.when(routerDao.findByNetwork(205L)).thenReturn(new ArrayList<>()); + DomainRouterVO associatedNetworkRouter = Mockito.mock(DomainRouterVO.class); + Mockito.when(associatedNetworkRouter.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(routerDao.findByNetwork(206L)).thenReturn(Arrays.asList(associatedNetworkRouter)); + + service.disassociateNetworkFromNic(mockDisassociateCmd(11L, 206L)); + + Mockito.verify(commandSetupHelper).createDhcpEntryCommand(Mockito.eq(associatedNetworkRouter), Mockito.eq(vm), Mockito.any(), Mockito.eq("10.1.1.50"), + Mockito.any(), Mockito.any(), Mockito.any(), Mockito.eq(206L), Mockito.eq(false), Mockito.eq(true), Mockito.any()); + Mockito.verify(networkHelper).sendCommandsToRouter(Mockito.eq(associatedNetworkRouter), Mockito.any()); + } + + @Test + public void disassociateNetworkFromNicDoesNotRemoveDhcpEntryOnAStoppedVm() throws Exception { + NicVO nic = mockNic(11L, 5L, 205L, true); + Mockito.when(nicDao.findById(11L)).thenReturn(nic); + UserVmVO vm = mockVm(VirtualMachine.State.Stopped, null); + Mockito.when(userVmDao.findById(5L)).thenReturn(vm); + NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L, "10.1.1.50", null); + Mockito.when(nicNetworkMapDao.findByNicIdAndNetworkId(11L, 206L)).thenReturn(association); + + service.disassociateNetworkFromNic(mockDisassociateCmd(11L, 206L)); + + Mockito.verify(commandSetupHelper, Mockito.never()).createDhcpEntryCommand(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.anyString(), + Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong(), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.any()); + } + @Test public void disassociateNetworkFromNicSucceedsWhenMetadataRefreshFails() throws Exception { NicVO nic = mockNic(11L, 5L, 205L, true); diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 4d631b293c17..fff8d9b024a6 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -156,6 +156,8 @@ import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.network.element.UserDataServiceProvider; import com.cloud.network.guru.NetworkGuru; +import com.cloud.network.router.CommandSetupHelper; +import com.cloud.network.router.NetworkHelper; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRule; import com.cloud.network.rules.dao.PortForwardingRulesDao; @@ -208,7 +210,10 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExceptionProxyObject; import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.NicNetworkMapDao; +import com.cloud.vm.dao.NicNetworkMapVO; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.snapshot.VMSnapshotVO; @@ -239,6 +244,18 @@ public class UserVmManagerImplTest { @Mock protected NicDao nicDao; + @Mock + private NicNetworkMapDao nicNetworkMapDao; + + @Mock + private DomainRouterDao routerDao; + + @Mock + private CommandSetupHelper commandSetupHelper; + + @Mock + private NetworkHelper nwHelper; + @Mock private NetworkDao _networkDao; @@ -559,6 +576,58 @@ public void afterTest() { } } + @Test + public void syncNicNetworkAssociationDhcpEntriesIsNoOpForANicWithoutAssociations() { + UserVmVO vm = Mockito.mock(UserVmVO.class); + NicVO nic = Mockito.mock(NicVO.class); + Mockito.when(nic.getId()).thenReturn(11L); + + userVmManagerImpl.syncNicNetworkAssociationDhcpEntries(vm, nic); + + Mockito.verify(_networkDao, Mockito.never()).findById(Mockito.anyLong()); + Mockito.verifyNoInteractions(commandSetupHelper); + } + + @Test + public void syncNicNetworkAssociationDhcpEntriesPushesDhcpEntryForAnActiveAssociation() throws Exception { + UserVmVO vm = Mockito.mock(UserVmVO.class); + NicVO nic = Mockito.mock(NicVO.class); + Mockito.when(nic.getId()).thenReturn(11L); + NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L, "10.1.1.50", null); + Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(Collections.singletonList(association)); + NetworkVO associatedNetwork = Mockito.mock(NetworkVO.class); + Mockito.when(_networkDao.findById(206L)).thenReturn(associatedNetwork); + DomainRouterVO router = Mockito.mock(DomainRouterVO.class); + Mockito.when(router.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(routerDao.findByNetwork(206L)).thenReturn(Collections.singletonList(router)); + + userVmManagerImpl.syncNicNetworkAssociationDhcpEntries(vm, nic); + + Mockito.verify(commandSetupHelper).createDhcpEntryCommand(Mockito.eq(router), Mockito.eq(vm), Mockito.any(), Mockito.eq("10.1.1.50"), + Mockito.any(), Mockito.any(), Mockito.any(), Mockito.eq(206L), Mockito.eq(false), Mockito.eq(false), Mockito.any()); + Mockito.verify(nwHelper).sendCommandsToRouter(Mockito.eq(router), Mockito.any()); + } + + @Test + public void syncNicNetworkAssociationDhcpEntriesCleansUpARemovedAssociation() throws Exception { + UserVmVO vm = Mockito.mock(UserVmVO.class); + NicVO nic = Mockito.mock(NicVO.class); + Mockito.when(nic.getId()).thenReturn(11L); + NicNetworkMapVO removedAssociation = new NicNetworkMapVO(11L, 207L, "10.1.1.60", null); + Mockito.when(nicNetworkMapDao.listRemovedByNicId(11L)).thenReturn(Collections.singletonList(removedAssociation)); + NetworkVO removedNetwork = Mockito.mock(NetworkVO.class); + Mockito.when(_networkDao.findById(207L)).thenReturn(removedNetwork); + DomainRouterVO router = Mockito.mock(DomainRouterVO.class); + Mockito.when(router.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(routerDao.findByNetwork(207L)).thenReturn(Collections.singletonList(router)); + + userVmManagerImpl.syncNicNetworkAssociationDhcpEntries(vm, nic); + + Mockito.verify(commandSetupHelper).createDhcpEntryCommand(Mockito.eq(router), Mockito.eq(vm), Mockito.any(), Mockito.eq("10.1.1.60"), + Mockito.any(), Mockito.any(), Mockito.any(), Mockito.eq(207L), Mockito.eq(false), Mockito.eq(true), Mockito.any()); + Mockito.verify(nwHelper).sendCommandsToRouter(Mockito.eq(router), Mockito.any()); + } + @Test public void validateGuestOsIdForUpdateVirtualMachineCommandTestOsTypeNull() { Mockito.when(updateVmCommand.getOsTypeId()).thenReturn(null); From 5088e0b55b8a6ec2f192478c8e84cbcef4d195a5 Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Fri, 2 Oct 2026 09:06:06 -0400 Subject: [PATCH 17/20] Add KVM live migration support for multi-VLAN trunk NICs (Phase 7) - Destination-capability-driven interface XML rewrite during migration (LibvirtMigrateCommandWrapper): rewrites bridge name and trunk-vlan XML per nic based on the destination's own reported capability and bridge mapping, failing closed if that capability is unknown. - Destination reports its own per-nic bridge name and VLAN-filtering/ trunk-XML capability back to the source via PrepareForMigrationAnswer (LibvirtPrepareForMigrationCommandWrapper), rather than having the source guess it. - New PostMigrationCommand path applies manual VLAN trunk membership on destinations whose libvirt can't express it via XML (LibvirtPostMigrationCommandWrapper). - Host-readiness gating for migration destinations: a VM with a multi-VLAN trunk nic is rejected from migrating to a host without VLAN filtering enabled, both for explicit-host migration (UserVmManagerImpl) and for host listing/planner-driven migration (ManagementServerImpl, DeploymentPlanningManagerImpl). - Shared VlanTrunkMigrationHelper so capability population and post-migration membership application work identically whether a VM migrates via the plain compute-only path (VirtualMachineManagerImpl) or via storage-motion (StorageSystemDataMotionStrategy, StorPoolDataMotionStrategy, LinstorDataMotionStrategy). - Fix VM-start/add-nic VLAN membership application (LibvirtStartCommandWrapper, LibvirtPlugNicCommandWrapper) to cover any VLAN nic on a vlan_filtering-enabled host, not just trunk nics, matching the already-correct post-migration behavior. --- .../com/cloud/agent/api/MigrateCommand.java | 29 ++++ .../agent/api/PrepareForMigrationAnswer.java | 11 ++ .../agent/api/VlanTrunkMigrationHelper.java | 87 ++++++++++ .../deploy/DeploymentPlanningManager.java | 2 + .../cloud/vm/VirtualMachineManagerImpl.java | 9 +- .../vm/VirtualMachineManagerImplTest.java | 44 ++++++ .../com/cloud/host/dao/HostDetailsDao.java | 4 + .../cloud/host/dao/HostDetailsDaoImpl.java | 11 ++ .../StorageSystemDataMotionStrategy.java | 13 ++ .../wrapper/LibvirtMigrateCommandWrapper.java | 149 ++++++++++++++++++ .../wrapper/LibvirtPlugNicCommandWrapper.java | 3 +- .../LibvirtPostMigrationCommandWrapper.java | 20 +++ ...virtPrepareForMigrationCommandWrapper.java | 16 +- .../wrapper/LibvirtStartCommandWrapper.java | 3 +- .../LibvirtMigrateCommandWrapperTest.java | 98 ++++++++++++ ...ibvirtPostMigrationCommandWrapperTest.java | 47 ++++++ ...PrepareForMigrationCommandWrapperTest.java | 17 +- .../motion/LinstorDataMotionStrategy.java | 17 +- .../motion/StorPoolDataMotionStrategy.java | 17 +- .../deploy/DeploymentPlanningManagerImpl.java | 8 +- .../cloud/server/ManagementServerImpl.java | 2 + .../java/com/cloud/vm/UserVmManagerImpl.java | 19 +++ .../DeploymentPlanningManagerImplTest.java | 12 +- .../server/ManagementServerImplTest.java | 13 ++ .../com/cloud/vm/UserVmManagerImplTest.java | 69 +++++++- 25 files changed, 690 insertions(+), 30 deletions(-) create mode 100644 core/src/main/java/com/cloud/agent/api/VlanTrunkMigrationHelper.java diff --git a/core/src/main/java/com/cloud/agent/api/MigrateCommand.java b/core/src/main/java/com/cloud/agent/api/MigrateCommand.java index 7196247ffc23..eaaee660ba85 100644 --- a/core/src/main/java/com/cloud/agent/api/MigrateCommand.java +++ b/core/src/main/java/com/cloud/agent/api/MigrateCommand.java @@ -47,6 +47,11 @@ public class MigrateCommand extends Command { Map vlanToPersistenceMap = new HashMap<>(); + // null means unknown capability, not false + private Boolean destVlanFilteringEnabled; + private Boolean destVlanTrunkXmlSupported; + private Map nicBridgeMapping = new HashMap<>(); + public Map getDpdkInterfaceMapping() { return dpdkInterfaceMapping; } @@ -159,6 +164,30 @@ public void setClvmCrossPoolMigration(boolean clvmCrossPoolMigration) { this.clvmCrossPoolMigration = clvmCrossPoolMigration; } + public Boolean getDestVlanFilteringEnabled() { + return destVlanFilteringEnabled; + } + + public void setDestVlanFilteringEnabled(Boolean destVlanFilteringEnabled) { + this.destVlanFilteringEnabled = destVlanFilteringEnabled; + } + + public Boolean getDestVlanTrunkXmlSupported() { + return destVlanTrunkXmlSupported; + } + + public void setDestVlanTrunkXmlSupported(Boolean destVlanTrunkXmlSupported) { + this.destVlanTrunkXmlSupported = destVlanTrunkXmlSupported; + } + + public Map getNicBridgeMapping() { + return nicBridgeMapping; + } + + public void setNicBridgeMapping(Map nicBridgeMapping) { + this.nicBridgeMapping = nicBridgeMapping; + } + public static class MigrateDiskInfo { public enum DiskType { FILE, BLOCK; diff --git a/core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java b/core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java index 190e844ddc5f..ae6a824de966 100644 --- a/core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java @@ -30,6 +30,9 @@ public class PrepareForMigrationAnswer extends Answer { private Integer newVmCpuShares = null; + // destination bridge name per nic MAC, computed by the destination's own local capability + private Map nicBridgeMapping = new HashMap<>(); + protected PrepareForMigrationAnswer() { } @@ -60,4 +63,12 @@ public Integer getNewVmCpuShares() { public void setNewVmCpuShares(Integer newVmCpuShares) { this.newVmCpuShares = newVmCpuShares; } + + public Map getNicBridgeMapping() { + return nicBridgeMapping; + } + + public void setNicBridgeMapping(Map nicBridgeMapping) { + this.nicBridgeMapping = nicBridgeMapping; + } } diff --git a/core/src/main/java/com/cloud/agent/api/VlanTrunkMigrationHelper.java b/core/src/main/java/com/cloud/agent/api/VlanTrunkMigrationHelper.java new file mode 100644 index 000000000000..ca1d2640fb56 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/VlanTrunkMigrationHelper.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import java.util.Map; + +import org.apache.commons.collections.MapUtils; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.DetailVO; +import com.cloud.host.Host; +import com.cloud.host.dao.HostDetailsDao; +import com.cloud.network.Networks; +import com.cloud.vm.dao.NicDao; + +public class VlanTrunkMigrationHelper { + + // matches AgentManager.send's signature so callers can pass agentManager::send without core depending on AgentManager + @FunctionalInterface + public interface CommandSender { + Answer send(Long hostId, Command cmd) throws AgentUnavailableException, OperationTimedoutException; + } + + private VlanTrunkMigrationHelper() { + } + + public static void populateVlanTrunkMigrationDetails(MigrateCommand migrateCommand, PrepareForMigrationAnswer prepareForMigrationAnswer, + HostDetailsDao hostDetailsDao, long destHostId) { + Map nicBridgeMapping = prepareForMigrationAnswer.getNicBridgeMapping(); + if (MapUtils.isNotEmpty(nicBridgeMapping)) { + migrateCommand.setNicBridgeMapping(nicBridgeMapping); + } + + DetailVO destVlanFilteringDetail = hostDetailsDao.findDetail(destHostId, Host.HOST_VLAN_FILTERING_ENABLED); + migrateCommand.setDestVlanFilteringEnabled(destVlanFilteringDetail == null ? null : Boolean.parseBoolean(destVlanFilteringDetail.getValue())); + DetailVO destVlanTrunkXmlDetail = hostDetailsDao.findDetail(destHostId, Host.HOST_VLAN_TRUNK_XML_SUPPORTED); + migrateCommand.setDestVlanTrunkXmlSupported(destVlanTrunkXmlDetail == null ? null : Boolean.parseBoolean(destVlanTrunkXmlDetail.getValue())); + } + + public static boolean vmNeedsPostMigrationVlanTrunkMembership(long vmId, long dstHostId, NicDao nicDao, HostDetailsDao hostDetailsDao) { + DetailVO trunkXmlDetail = hostDetailsDao.findDetail(dstHostId, Host.HOST_VLAN_TRUNK_XML_SUPPORTED); + if (trunkXmlDetail != null && Boolean.parseBoolean(trunkXmlDetail.getValue())) { + return false; + } + DetailVO vlanFilteringDetail = hostDetailsDao.findDetail(dstHostId, Host.HOST_VLAN_FILTERING_ENABLED); + boolean destVlanFilteringEnabled = vlanFilteringDetail != null && Boolean.parseBoolean(vlanFilteringDetail.getValue()); + return nicDao.listByVmId(vmId).stream() + .anyMatch(nic -> nic.getBroadcastUri() != null && Networks.BroadcastDomainType.getSchemeValue(nic.getBroadcastUri()) == Networks.BroadcastDomainType.Vlan + && (nic.getMultiNetwork() || destVlanFilteringEnabled)); + } + + public static void sendPostMigrationVlanTrunkMembershipIfNeeded(VirtualMachineTO vmTO, long dstHostId, + NicDao nicDao, HostDetailsDao hostDetailsDao, CommandSender commandSender, Logger logger) { + if (!vmNeedsPostMigrationVlanTrunkMembership(vmTO.getId(), dstHostId, nicDao, hostDetailsDao)) { + return; + } + final String warningMsg = "Post-migration VLAN trunk membership tasks did not complete for VM [{}] " + + "on destination host [{}]: {}. Migration completed but VLAN membership may need manual correction."; + try { + PostMigrationCommand postMigrationCommand = new PostMigrationCommand(vmTO, vmTO.getName()); + Answer postMigrationAnswer = commandSender.send(dstHostId, postMigrationCommand); + if (postMigrationAnswer == null || !postMigrationAnswer.getResult()) { + String details = postMigrationAnswer != null ? postMigrationAnswer.getDetails() : "null answer returned"; + logger.warn(warningMsg, vmTO, dstHostId, details); + } + } catch (Exception e) { + logger.warn(warningMsg, vmTO, dstHostId, e.getMessage(), e); + } + } +} diff --git a/engine/components-api/src/main/java/com/cloud/deploy/DeploymentPlanningManager.java b/engine/components-api/src/main/java/com/cloud/deploy/DeploymentPlanningManager.java index ce40287bf033..25b8830fde8b 100644 --- a/engine/components-api/src/main/java/com/cloud/deploy/DeploymentPlanningManager.java +++ b/engine/components-api/src/main/java/com/cloud/deploy/DeploymentPlanningManager.java @@ -65,5 +65,7 @@ String finalizeReservation(DeployDestination plannedDestination, void checkForNonDedicatedResources(VirtualMachineProfile vmProfile, DataCenter dc, ExcludeList avoids); + void avoidHostsNotReadyForMultiNetworkNics(VirtualMachineProfile vmProfile, DataCenter dc, ExcludeList avoids); + void reorderHostsByPriority(Map priorities, List hosts); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 3702787fe9f4..9ad348112cbd 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -3375,7 +3375,8 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy } private void executePostMigrationCommand(VMInstanceVO vm, VirtualMachineTO to, long dstHostId) { - if (!(vm.getHypervisorType() == HypervisorType.KVM && hasClvmVolumes(vm.getId()))) { + if (vm.getHypervisorType() != HypervisorType.KVM + || !(hasClvmVolumes(vm.getId()) || vmNeedsPostMigrationVlanTrunkMembership(vm, dstHostId))) { return; } final String dstHostUuid = _hostDao.findById(dstHostId).getUuid(); @@ -3433,6 +3434,8 @@ protected MigrateCommand buildMigrateCommand(VMInstanceVO vmInstance, VirtualMac migrateCommand.setNewVmCpuShares(newVmCpuShares); } + VlanTrunkMigrationHelper.populateVlanTrunkMigrationDetails(migrateCommand, prepareForMigrationAnswer, hostDetailsDao, destination.getHost().getId()); + return migrateCommand; } @@ -6570,6 +6573,10 @@ private boolean hasClvmVolumes(long vmId) { .anyMatch(pool -> pool != null && ClvmPoolManager.isClvmPoolType(pool.getPoolType())); } + boolean vmNeedsPostMigrationVlanTrunkMembership(VMInstanceVO vm, long dstHostId) { + return VlanTrunkMigrationHelper.vmNeedsPostMigrationVlanTrunkMembership(vm.getId(), dstHostId, _nicsDao, hostDetailsDao); + } + private void executePreMigrationCommand(VMInstanceVO vm, VirtualMachineTO to, long srcHostId) { if (!(vm.getHypervisorType() == HypervisorType.KVM && hasClvmVolumes(vm.getId()))) { return; diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index 40f76b74b808..fbd535c33d49 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -60,6 +60,7 @@ import com.cloud.ha.HighAvailabilityManager; import com.cloud.network.Network; import com.cloud.network.NetworkModel; +import com.cloud.network.Networks; import com.cloud.resource.ResourceManager; import com.cloud.storage.clvm.ClvmPoolManager; import org.apache.cloudstack.api.ApiConstants; @@ -1411,6 +1412,49 @@ public void testOrchestrateStartExcludesHostWithoutVlanFilteringForMultiNetworkN assertEquals(vmInstance.getPodIdToDeployIn(), (Long) destPod.getId()); } + private VMInstanceVO mockVmWithId(long id) { + VMInstanceVO vm = mock(VMInstanceVO.class); + when(vm.getId()).thenReturn(id); + return vm; + } + + private NicVO mockVlanNic(int vlan, boolean multiNetwork) { + NicVO nic = mock(NicVO.class); + when(nic.getBroadcastUri()).thenReturn(Networks.BroadcastDomainType.Vlan.toUri(vlan)); + when(nic.getMultiNetwork()).thenReturn(multiNetwork); + return nic; + } + + @Test + public void vmNeedsPostMigrationVlanTrunkMembershipFalseWhenDestSupportsTrunkXml() { + DetailVO trunkXmlSupported = mock(DetailVO.class); + when(trunkXmlSupported.getValue()).thenReturn("true"); + when(hostDetailsDao.findDetail(2L, Host.HOST_VLAN_TRUNK_XML_SUPPORTED)).thenReturn(trunkXmlSupported); + + Assert.assertFalse(virtualMachineManagerImpl.vmNeedsPostMigrationVlanTrunkMembership(mockVmWithId(1L), 2L)); + verify(_nicsDao, Mockito.never()).listByVmId(anyLong()); + } + + @Test + public void vmNeedsPostMigrationVlanTrunkMembershipTrueForTrunkNicOnNonTrunkXmlDest() { + when(hostDetailsDao.findDetail(2L, Host.HOST_VLAN_TRUNK_XML_SUPPORTED)).thenReturn(null); + when(hostDetailsDao.findDetail(2L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(null); + NicVO trunkNic = mockVlanNic(100, true); + when(_nicsDao.listByVmId(1L)).thenReturn(java.util.Collections.singletonList(trunkNic)); + + Assert.assertTrue(virtualMachineManagerImpl.vmNeedsPostMigrationVlanTrunkMembership(mockVmWithId(1L), 2L)); + } + + @Test + public void vmNeedsPostMigrationVlanTrunkMembershipFalseForOrdinaryNicWithoutVlanFilteringDest() { + when(hostDetailsDao.findDetail(2L, Host.HOST_VLAN_TRUNK_XML_SUPPORTED)).thenReturn(null); + when(hostDetailsDao.findDetail(2L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(null); + NicVO ordinaryNic = mockVlanNic(100, false); + when(_nicsDao.listByVmId(1L)).thenReturn(java.util.Collections.singletonList(ordinaryNic)); + + Assert.assertFalse(virtualMachineManagerImpl.vmNeedsPostMigrationVlanTrunkMembership(mockVmWithId(1L), 2L)); + } + @Test public void testIsDiskOfferingSuitableForVmSuccess() { Mockito.doReturn(Mockito.mock(DiskOfferingVO.class)).when(diskOfferingDaoMock).findById(anyLong()); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDetailsDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDetailsDao.java index b72ec5239ffb..e60a904d7e02 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDetailsDao.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; import com.cloud.host.DetailVO; import com.cloud.utils.db.GenericDao; @@ -37,4 +38,7 @@ public interface HostDetailsDao extends GenericDao { void replaceExternalDetails(long hostId, Map details); + // Ids of hosts with Host.HOST_VLAN_FILTERING_ENABLED=true, i.e. hosts able to run a multi-VLAN trunk nic. + Set findHostIdsWithVlanFilteringEnabled(); + } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDetailsDaoImpl.java index 0f8e8623506d..cf0b801ddd95 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDetailsDaoImpl.java @@ -22,10 +22,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import org.springframework.stereotype.Component; import com.cloud.host.DetailVO; +import com.cloud.host.Host; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; @@ -139,6 +142,14 @@ public List findByName(String name) { return listBy(sc); } + @Override + public Set findHostIdsWithVlanFilteringEnabled() { + return findByName(Host.HOST_VLAN_FILTERING_ENABLED).stream() + .filter(detail -> Boolean.parseBoolean(detail.getValue())) + .map(DetailVO::getHostId) + .collect(Collectors.toSet()); + } + @Override public void removeExternalDetails(long hostId) { TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 7674f1ce25a1..99a442d77d49 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -92,6 +92,7 @@ import com.cloud.agent.api.ModifyTargetsCommand; import com.cloud.agent.api.PreMigrationCommand; import com.cloud.agent.api.PrepareForMigrationCommand; +import com.cloud.agent.api.VlanTrunkMigrationHelper; import com.cloud.agent.api.storage.CopyVolumeAnswer; import com.cloud.agent.api.storage.CopyVolumeCommand; import com.cloud.agent.api.storage.MigrateVolumeAnswer; @@ -108,6 +109,7 @@ import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.resource.ResourceState; import com.cloud.storage.DataStoreRole; @@ -145,6 +147,7 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Preconditions; import java.util.Arrays; @@ -179,6 +182,10 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { @Inject private HostDao _hostDao; @Inject + private HostDetailsDao hostDetailsDao; + @Inject + private NicDao nicDao; + @Inject protected PrimaryDataStoreDao _storagePoolDao; @Inject private SnapshotDao _snapshotDao; @@ -2193,6 +2200,8 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach .anyMatch(info -> ClvmPoolManager.isClvmPoolType(info.getSourcePoolType())); migrateCommand.setClvmCrossPoolMigration(hasClvmCrossPoolVolume); + VlanTrunkMigrationHelper.populateVlanTrunkMigrationDetails(migrateCommand, (PrepareForMigrationAnswer) pfma, hostDetailsDao, destHost.getId()); + Integer newVmCpuShares = ((PrepareForMigrationAnswer) pfma).getNewVmCpuShares(); if (newVmCpuShares != null) { logger.debug(String.format("Setting CPU shares to [%d] as part of migrate VM with volumes command for VM [%s].", newVmCpuShares, vmTO)); @@ -2224,6 +2233,10 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach handlePostMigration(success, srcVolumeInfoToDestVolumeInfo, vmTO, srcHost, destHost); + if (success) { + VlanTrunkMigrationHelper.sendPostMigrationVlanTrunkMembershipIfNeeded(vmTO, destHost.getId(), nicDao, hostDetailsDao, agentManager::send, logger); + } + if (!success) { if (migrateAnswer == null) { throw new CloudRuntimeException("Unable to get an answer to the migrate command"); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index ed02ae6da38d..48f58e25d5dd 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.InputStream; +import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -76,6 +77,8 @@ import com.cloud.agent.api.MigrateCommand.MigrateDiskInfo; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.DpdkTO; +import com.cloud.agent.api.to.NetworkTO; +import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; @@ -86,6 +89,7 @@ import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; import com.cloud.hypervisor.kvm.resource.MigrateKVMAsync; import com.cloud.hypervisor.kvm.resource.VifDriver; +import com.cloud.network.Networks; @ResourceWrapper(handles = MigrateCommand.class) public final class LibvirtMigrateCommandWrapper extends CommandWrapper { @@ -229,6 +233,8 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0. } } + xmlDesc = replaceVlanTrunkInterfaces(xmlDesc, command, libvirtComputingResource); + xmlDesc = updateVmSharesIfNeeded(command, xmlDesc, libvirtComputingResource); xmlDesc = updateGpuDevicesIfNeeded(command, xmlDesc, libvirtComputingResource); @@ -688,6 +694,149 @@ protected String replaceDpdkInterfaces(String xmlDesc, Map dpdkP return LibvirtXMLParser.getXml(doc); } + // Rewrites Vlan-broadcast nics' bridge/trunk XML for the destination's capability; fails closed if that capability is unknown. + protected String replaceVlanTrunkInterfaces(String xmlDesc, MigrateCommand command, LibvirtComputingResource libvirtComputingResource) + throws ParserConfigurationException, IOException, SAXException, TransformerException { + NicTO[] nics = command.getVirtualMachine().getNics(); + boolean sourceVlanFilteringEnabled = libvirtComputingResource.hostSupportsVlanFiltering(); + boolean anyNicNeedsHandling = false; + for (NicTO nic : nics) { + if (nic.getBroadcastType() == Networks.BroadcastDomainType.Vlan && (sourceVlanFilteringEnabled || nic.isTrunkVlan())) { + anyNicNeedsHandling = true; + break; + } + } + if (!anyNicNeedsHandling) { + return xmlDesc; + } + + InputStream in = IOUtils.toInputStream(xmlDesc); + DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + Document doc = docBuilder.parse(in); + + Node domainNode = doc.getFirstChild(); + NodeList domainChildNodes = domainNode.getChildNodes(); + for (int i = 0; i < domainChildNodes.getLength(); i++) { + Node domainChildNode = domainChildNodes.item(i); + if (!"devices".equals(domainChildNode.getNodeName())) { + continue; + } + NodeList devicesChildNodes = domainChildNode.getChildNodes(); + for (int x = 0; x < devicesChildNodes.getLength(); x++) { + Node deviceChildNode = devicesChildNodes.item(x); + if (!"interface".equals(deviceChildNode.getNodeName()) || !(deviceChildNode instanceof Element)) { + continue; + } + Element interfaceElement = (Element) deviceChildNode; + String mac = findInterfaceMacAddress(interfaceElement); + NicTO nic = mac == null ? null : findNicByMac(nics, mac); + if (nic == null || nic.getBroadcastType() != Networks.BroadcastDomainType.Vlan + || !(sourceVlanFilteringEnabled || nic.isTrunkVlan())) { + continue; + } + replaceVlanTrunkInterface(doc, interfaceElement, nic, command); + } + } + + return LibvirtXMLParser.getXml(doc); + } + + private NicTO findNicByMac(NicTO[] nics, String mac) { + for (NicTO nic : nics) { + if (mac.equalsIgnoreCase(nic.getMac())) { + return nic; + } + } + return null; + } + + private String findInterfaceMacAddress(Element interfaceElement) { + NodeList children = interfaceElement.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + Node child = children.item(i); + if ("mac".equals(child.getNodeName()) && child.getAttributes() != null) { + Node addressAttr = child.getAttributes().getNamedItem("address"); + if (addressAttr != null) { + return addressAttr.getNodeValue(); + } + } + } + return null; + } + + private void replaceVlanTrunkInterface(Document doc, Element interfaceElement, NicTO nic, MigrateCommand command) { + String destBridge = command.getNicBridgeMapping().get(nic.getMac()); + Boolean destVlanFilteringEnabled = command.getDestVlanFilteringEnabled(); + Boolean destVlanTrunkXmlSupported = command.getDestVlanTrunkXmlSupported(); + if (StringUtils.isBlank(destBridge) || destVlanFilteringEnabled == null || destVlanTrunkXmlSupported == null) { + throw new CloudRuntimeException(String.format( + "Refusing to migrate nic %s: destination host's multi-VLAN capability is unknown to this migration command", nic.getMac())); + } + + NodeList children = interfaceElement.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + Node child = children.item(i); + if ("source".equals(child.getNodeName()) && child.getAttributes() != null) { + Node bridgeAttr = child.getAttributes().getNamedItem("bridge"); + if (bridgeAttr != null) { + bridgeAttr.setNodeValue(destBridge); + } + } + } + + Node existingVlanNode = null; + for (int i = 0; i < children.getLength(); i++) { + if ("vlan".equals(children.item(i).getNodeName())) { + existingVlanNode = children.item(i); + break; + } + } + if (existingVlanNode != null) { + interfaceElement.removeChild(existingVlanNode); + } + + if (destVlanTrunkXmlSupported) { + interfaceElement.appendChild(buildVlanTrunkElement(doc, nic)); + } + // else: membership is applied manually on the destination tap instead (ensureVlanTrunkMembership) + } + + private Element buildVlanTrunkElement(Document doc, NicTO nic) { + Element vlanElement = doc.createElement("vlan"); + vlanElement.setAttribute("trunk", "yes"); + + Integer primaryTag = parseVlanTag(nic.getBroadcastUri()); + appendVlanTagElement(doc, vlanElement, primaryTag, true); + if (nic.getAssociatedNetworks() != null) { + for (NetworkTO associatedNetwork : nic.getAssociatedNetworks()) { + appendVlanTagElement(doc, vlanElement, parseVlanTag(associatedNetwork.getBroadcastUri()), false); + } + } + return vlanElement; + } + + private void appendVlanTagElement(Document doc, Element vlanElement, Integer tag, boolean nativeUntagged) { + Element tagElement = doc.createElement("tag"); + tagElement.setAttribute("id", String.valueOf(tag)); + if (nativeUntagged) { + tagElement.setAttribute("nativeMode", "untagged"); + } + vlanElement.appendChild(tagElement); + } + + private Integer parseVlanTag(URI broadcastUri) { + String vlanValue = broadcastUri == null ? null : Networks.BroadcastDomainType.getValue(broadcastUri); + if (StringUtils.isBlank(vlanValue)) { + throw new CloudRuntimeException("Cannot determine VLAN from broadcast URI " + broadcastUri + " while preparing migration XML"); + } + try { + return Integer.valueOf(vlanValue); + } catch (NumberFormatException e) { + throw new CloudRuntimeException("Invalid VLAN value '" + vlanValue + "' in broadcast URI " + broadcastUri); + } + } + /** * In case of a local file, it deletes the file on the source host/storage pool. Otherwise (for instance iScsi) it disconnects the disk on the source storage pool.
    * This method must be executed after a successful migration to a target storage pool, cleaning up the source storage. diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java index 99c7293c9806..b1bc65ec73ce 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java @@ -27,6 +27,7 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; import com.cloud.hypervisor.kvm.resource.VifDriver; +import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.utils.exception.CloudRuntimeException; @@ -68,7 +69,7 @@ public Answer execute(final PlugNicCommand command, final LibvirtComputingResour } vm.attachDevice(interfaceDef.toString()); - if (nic.isTrunkVlan()) { + if (nic.getBroadcastType() == BroadcastDomainType.Vlan) { try { final InterfaceDef liveInterfaceDef = libvirtComputingResource.getInterface(conn, vmName, nic.getMac()); vifDriver.ensureVlanTrunkMembership(liveInterfaceDef, nic); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostMigrationCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostMigrationCommandWrapper.java index 608770974dc1..2c267179274b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostMigrationCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostMigrationCommandWrapper.java @@ -29,10 +29,14 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.PostMigrationAnswer; import com.cloud.agent.api.PostMigrationCommand; +import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.exception.InternalErrorException; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtConnection; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef; +import com.cloud.network.Networks; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; @@ -68,6 +72,8 @@ public Answer execute(final PostMigrationCommand command, final LibvirtComputing LibvirtComputingResource.ClvmVolumeState.EXCLUSIVE ); + applyManualVlanTrunkMembership(conn, vmName, vm.getNics(), libvirtComputingResource); + logger.debug("Successfully completed post-migration tasks for VM {}", vmName); return new PostMigrationAnswer(command); @@ -79,4 +85,18 @@ public Answer execute(final PostMigrationCommand command, final LibvirtComputing return new PostMigrationAnswer(command, e); } } + + private void applyManualVlanTrunkMembership(Connect conn, String vmName, NicTO[] nics, LibvirtComputingResource libvirtComputingResource) + throws InternalErrorException { + if (nics == null) { + return; + } + for (NicTO nic : nics) { + if (nic.getBroadcastType() != Networks.BroadcastDomainType.Vlan) { + continue; + } + LibvirtVMDef.InterfaceDef liveInterface = libvirtComputingResource.getInterface(conn, vmName, nic.getMac()); + libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).ensureVlanTrunkMembership(liveInterface, nic); + } + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java index f7ca79127dad..7d02c728b526 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java @@ -42,7 +42,9 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef.GuestNetType; +import com.cloud.hypervisor.kvm.resource.VifDriver; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.network.Networks; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Volume; @@ -68,6 +70,7 @@ public Answer execute(final PrepareForMigrationCommand command, final LibvirtCom final NicTO[] nics = vm.getNics(); Map dpdkInterfaceMapping = new HashMap<>(); + Map nicBridgeMapping = new HashMap<>(); boolean skipDisconnect = false; @@ -78,7 +81,8 @@ public Answer execute(final PrepareForMigrationCommand command, final LibvirtCom final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vm.getName()); for (final NicTO nic : nics) { - LibvirtVMDef.InterfaceDef interfaceDef = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).plug(nic, null, "", vm.getExtraConfig()); + VifDriver vifDriver = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()); + LibvirtVMDef.InterfaceDef interfaceDef = vifDriver.plug(nic, null, "", vm.getExtraConfig()); if (vm.getDetails() != null) { libvirtComputingResource.setInterfaceDefQueueSettings(vm.getDetails(), vm.getCpus(), interfaceDef); } @@ -87,6 +91,9 @@ public Answer execute(final PrepareForMigrationCommand command, final LibvirtCom dpdkInterfaceMapping.put(nic.getMac(), to); logger.debug("Configured DPDK interface for VM {}", vm.getName()); } + if (nic.getBroadcastType() == Networks.BroadcastDomainType.Vlan && interfaceDef != null) { + nicBridgeMapping.put(nic.getMac(), interfaceDef.getBrName()); + } } /* setup disks, e.g for iso */ @@ -139,7 +146,7 @@ public Answer execute(final PrepareForMigrationCommand command, final LibvirtCom } logger.info("Successfully prepared destination host for migration of VM {}", vm.getName()); - return createPrepareForMigrationAnswer(command, dpdkInterfaceMapping, libvirtComputingResource, vm); + return createPrepareForMigrationAnswer(command, dpdkInterfaceMapping, nicBridgeMapping, libvirtComputingResource, vm); } catch (final LibvirtException | CloudRuntimeException | InternalErrorException | URISyntaxException e) { if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) { for (DpdkTO to : dpdkInterfaceMapping.values()) { @@ -155,6 +162,7 @@ public Answer execute(final PrepareForMigrationCommand command, final LibvirtCom } protected PrepareForMigrationAnswer createPrepareForMigrationAnswer(PrepareForMigrationCommand command, Map dpdkInterfaceMapping, + Map nicBridgeMapping, LibvirtComputingResource libvirtComputingResource, VirtualMachineTO vm) { PrepareForMigrationAnswer answer = new PrepareForMigrationAnswer(command); @@ -163,6 +171,10 @@ protected PrepareForMigrationAnswer createPrepareForMigrationAnswer(PrepareForMi answer.setDpdkInterfaceMapping(dpdkInterfaceMapping); } + if (MapUtils.isNotEmpty(nicBridgeMapping)) { + answer.setNicBridgeMapping(nicBridgeMapping); + } + int newCpuShares = libvirtComputingResource.calculateCpuShares(vm); logger.debug(String.format("Setting CPU shares to [%s] for the migration of VM [%s].", newCpuShares, vm)); answer.setNewVmCpuShares(newCpuShares); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java index 2199981d4a4c..fd771add7bea 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java @@ -46,6 +46,7 @@ import com.cloud.hypervisor.kvm.resource.LibvirtKvmAgentHook; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.TrafficType; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; @@ -184,7 +185,7 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource private void applyManualVlanTrunkMembership(Connect conn, String vmName, NicTO[] nics, LibvirtComputingResource libvirtComputingResource) throws InternalErrorException { for (NicTO nic : nics) { - if (!nic.isTrunkVlan()) { + if (nic.getBroadcastType() != BroadcastDomainType.Vlan) { continue; } try { diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java index aa36fb89765c..9e9a5ce61c34 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java @@ -26,6 +26,7 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -66,7 +67,10 @@ import com.cloud.agent.api.VgpuTypesInfo; import com.cloud.agent.api.to.DpdkTO; import com.cloud.agent.api.to.GPUDeviceTO; +import com.cloud.agent.api.to.NetworkTO; +import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.network.Networks; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtConnection; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef; @@ -956,6 +960,100 @@ public void testReplaceDPDKPorts() throws ParserConfigurationException, IOExcept Assert.assertFalse(replaced.contains("csdpdk-1")); } + private static final String vlanNicMac = "02:00:00:00:00:01"; + private static final String vlanNicDomainXml = + "\n" + + " i-2-4-VM\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + + private NicTO buildVlanNicTO(boolean trunk, int primaryVlan, List associatedVlans) { + NicTO nic = new NicTO(); + nic.setBroadcastType(Networks.BroadcastDomainType.Vlan); + nic.setBroadcastUri(Networks.BroadcastDomainType.Vlan.toUri(primaryVlan)); + nic.setMac(vlanNicMac); + nic.setTrunkVlan(trunk); + if (associatedVlans != null) { + List associated = new ArrayList<>(); + for (Integer vlan : associatedVlans) { + NetworkTO associatedTo = new NetworkTO(); + associatedTo.setBroadcastType(Networks.BroadcastDomainType.Vlan); + associatedTo.setBroadcastUri(Networks.BroadcastDomainType.Vlan.toUri(vlan)); + associated.add(associatedTo); + } + nic.setAssociatedNetworks(associated); + } + return nic; + } + + @Test + public void replaceVlanTrunkInterfacesLeavesOrdinaryVlanNicUntouchedOnNonFilteringSource() + throws ParserConfigurationException, IOException, SAXException, TransformerException { + NicTO nic = buildVlanNicTO(false, 100, null); + Mockito.doReturn(new NicTO[]{nic}).when(virtualMachineTOMock).getNics(); + Mockito.doReturn(virtualMachineTOMock).when(migrateCommandMock).getVirtualMachine(); + Mockito.doReturn(false).when(libvirtComputingResourceMock).hostSupportsVlanFiltering(); + + String result = libvirtMigrateCmdWrapper.replaceVlanTrunkInterfaces(vlanNicDomainXml, migrateCommandMock, libvirtComputingResourceMock); + + assertEquals(vlanNicDomainXml, result); + } + + @Test + public void replaceVlanTrunkInterfacesInsertsTrunkVlanBlockWhenDestSupportsTrunkXml() + throws ParserConfigurationException, IOException, SAXException, TransformerException { + NicTO nic = buildVlanNicTO(true, 100, Collections.singletonList(200)); + Mockito.doReturn(new NicTO[]{nic}).when(virtualMachineTOMock).getNics(); + Mockito.doReturn(virtualMachineTOMock).when(migrateCommandMock).getVirtualMachine(); + Mockito.doReturn(true).when(libvirtComputingResourceMock).hostSupportsVlanFiltering(); + Mockito.doReturn(true).when(migrateCommandMock).getDestVlanFilteringEnabled(); + Mockito.doReturn(true).when(migrateCommandMock).getDestVlanTrunkXmlSupported(); + Mockito.doReturn(Map.of(vlanNicMac, "destcloudbr0")).when(migrateCommandMock).getNicBridgeMapping(); + + String result = libvirtMigrateCmdWrapper.replaceVlanTrunkInterfaces(vlanNicDomainXml, migrateCommandMock, libvirtComputingResourceMock); + + assertTrue(result.contains("bridge=\"destcloudbr0\"")); + assertTrue(result.contains("trunk=\"yes\"")); + assertTrue(result.contains("id=\"100\"")); + assertTrue(result.contains("nativeMode=\"untagged\"")); + assertTrue(result.contains("id=\"200\"")); + } + + @Test + public void replaceVlanTrunkInterfacesStripsVlanBlockWhenDestDoesNotSupportTrunkXml() + throws ParserConfigurationException, IOException, SAXException, TransformerException { + NicTO nic = buildVlanNicTO(true, 100, Collections.singletonList(200)); + Mockito.doReturn(new NicTO[]{nic}).when(virtualMachineTOMock).getNics(); + Mockito.doReturn(virtualMachineTOMock).when(migrateCommandMock).getVirtualMachine(); + Mockito.doReturn(true).when(libvirtComputingResourceMock).hostSupportsVlanFiltering(); + Mockito.doReturn(true).when(migrateCommandMock).getDestVlanFilteringEnabled(); + Mockito.doReturn(false).when(migrateCommandMock).getDestVlanTrunkXmlSupported(); + Mockito.doReturn(Map.of(vlanNicMac, "destcloudbr0")).when(migrateCommandMock).getNicBridgeMapping(); + + String result = libvirtMigrateCmdWrapper.replaceVlanTrunkInterfaces(vlanNicDomainXml, migrateCommandMock, libvirtComputingResourceMock); + + assertTrue(result.contains("bridge=\"destcloudbr0\"")); + assertFalse(result.contains(" disks = createNonClvmDisks(); + try (MockedStatic mockedConnection = Mockito.mockStatic(LibvirtConnection.class)) { + mockedConnection.when(() -> LibvirtConnection.getConnectionByVmName(VM_NAME)).thenReturn(connect); + when(libvirtComputingResource.getDisks(connect, VM_NAME)).thenReturn(disks); + + Answer answer = wrapper.execute(postMigrationCommand, libvirtComputingResource); + + Assert.assertTrue(answer.getResult()); + Mockito.verify(vifDriver).ensureVlanTrunkMembership(liveInterface, vlanNic); + } + } + + @Test + public void testExecute_NonVlanNic_SkipsManualVlanTrunkMembership() throws LibvirtException { + NicTO publicNic = new NicTO(); + publicNic.setBroadcastType(Networks.BroadcastDomainType.Native); + publicNic.setMac("02:00:00:00:00:02"); + when(virtualMachineTO.getNics()).thenReturn(new NicTO[]{publicNic}); + + List disks = createNonClvmDisks(); + try (MockedStatic mockedConnection = Mockito.mockStatic(LibvirtConnection.class)) { + mockedConnection.when(() -> LibvirtConnection.getConnectionByVmName(VM_NAME)).thenReturn(connect); + when(libvirtComputingResource.getDisks(connect, VM_NAME)).thenReturn(disks); + + Answer answer = wrapper.execute(postMigrationCommand, libvirtComputingResource); + + Assert.assertTrue(answer.getResult()); + Mockito.verify(libvirtComputingResource, Mockito.never()).getInterface(any(), any(), any()); + } + } + @Test public void testExecute_ClvmVolumes_ConvertedToExclusiveMode() throws LibvirtException { List disks = createClvmDisks(); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapperTest.java index 864cb4482886..9a98b5c99fb2 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapperTest.java @@ -57,17 +57,28 @@ public void createPrepareForMigrationAnswerTestDpdkInterfaceNotEmptyShouldSetPar Map dpdkInterfaceMapping = new HashMap<>(); dpdkInterfaceMapping.put("Interface", new DpdkTO()); - PrepareForMigrationAnswer prepareForMigrationAnswer = libvirtPrepareForMigrationCommandWrapperSpy.createPrepareForMigrationAnswer(prepareForMigrationCommandMock, dpdkInterfaceMapping, libvirtComputingResourceMock, - virtualMachineTOMock); + PrepareForMigrationAnswer prepareForMigrationAnswer = libvirtPrepareForMigrationCommandWrapperSpy.createPrepareForMigrationAnswer(prepareForMigrationCommandMock, dpdkInterfaceMapping, null, + libvirtComputingResourceMock, virtualMachineTOMock); Assert.assertEquals(prepareForMigrationAnswer.getDpdkInterfaceMapping(), dpdkInterfaceMapping); } + @Test + public void createPrepareForMigrationAnswerTestNicBridgeMappingNotEmptyShouldSetParamOnAnswer() { + Map nicBridgeMapping = new HashMap<>(); + nicBridgeMapping.put("02:00:00:00:00:01", "cloudbr0"); + + PrepareForMigrationAnswer prepareForMigrationAnswer = libvirtPrepareForMigrationCommandWrapperSpy.createPrepareForMigrationAnswer(prepareForMigrationCommandMock, null, nicBridgeMapping, + libvirtComputingResourceMock, virtualMachineTOMock); + + Assert.assertEquals(nicBridgeMapping, prepareForMigrationAnswer.getNicBridgeMapping()); + } + @Test public void createPrepareForMigrationAnswerTestVerifyThatCpuSharesIsSet() { int cpuShares = 1000; Mockito.doReturn(cpuShares).when(libvirtComputingResourceMock).calculateCpuShares(virtualMachineTOMock); - PrepareForMigrationAnswer prepareForMigrationAnswer = libvirtPrepareForMigrationCommandWrapperSpy.createPrepareForMigrationAnswer(prepareForMigrationCommandMock,null, + PrepareForMigrationAnswer prepareForMigrationAnswer = libvirtPrepareForMigrationCommandWrapperSpy.createPrepareForMigrationAnswer(prepareForMigrationCommandMock, null, null, libvirtComputingResourceMock, virtualMachineTOMock); Assert.assertEquals(cpuShares, prepareForMigrationAnswer.getNewVmCpuShares().intValue()); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java index 0dd0de70464f..113805762e2c 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java @@ -37,12 +37,15 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.MigrateAnswer; import com.cloud.agent.api.MigrateCommand; +import com.cloud.agent.api.PrepareForMigrationAnswer; import com.cloud.agent.api.PrepareForMigrationCommand; +import com.cloud.agent.api.VlanTrunkMigrationHelper; import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; @@ -54,6 +57,7 @@ import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; @@ -116,6 +120,10 @@ public class LinstorDataMotionStrategy implements DataMotionStrategy { @Inject private AgentManager _agentManager; @Inject + private HostDetailsDao hostDetailsDao; + @Inject + private NicDao nicDao; + @Inject private PrimaryDataStoreDao _storagePoolDao; @Override @@ -441,8 +449,9 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach } PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(vmTO); + Answer pfma; try { - Answer pfma = _agentManager.send(destHost.getId(), pfmc); + pfma = _agentManager.send(destHost.getId(), pfmc); if (pfma == null || !pfma.getResult()) { String details = pfma != null ? pfma.getDetails() : "null answer returned"; @@ -471,11 +480,17 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value(); migrateCommand.setAutoConvergence(kvmAutoConvergence); + VlanTrunkMigrationHelper.populateVlanTrunkMigrationDetails(migrateCommand, (PrepareForMigrationAnswer) pfma, hostDetailsDao, destHost.getId()); + MigrateAnswer migrateAnswer = (MigrateAnswer) _agentManager.send(srcHost.getId(), migrateCommand); boolean success = migrateAnswer != null && migrateAnswer.getResult(); handlePostMigration(success, srcVolumeInfoToDestVolumeInfo, vmTO, destHost); + if (success) { + VlanTrunkMigrationHelper.sendPostMigrationVlanTrunkMembershipIfNeeded(vmTO, destHost.getId(), nicDao, hostDetailsDao, _agentManager::send, logger); + } + if (migrateAnswer == null) { throw new CloudRuntimeException("Unable to get an answer to the migrate command"); } diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java index 8d3dc868276c..16b176844faf 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java @@ -27,7 +27,9 @@ import com.cloud.agent.api.MigrateCommand.MigrateDiskInfo; import com.cloud.agent.api.ModifyTargetsAnswer; import com.cloud.agent.api.ModifyTargetsCommand; +import com.cloud.agent.api.PrepareForMigrationAnswer; import com.cloud.agent.api.PrepareForMigrationCommand; +import com.cloud.agent.api.VlanTrunkMigrationHelper; import com.cloud.agent.api.storage.StorPoolBackupTemplateFromSnapshotCommand; import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.VirtualMachineTO; @@ -37,6 +39,7 @@ import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage.ImageFormat; @@ -54,6 +57,7 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; @@ -120,6 +124,10 @@ public class StorPoolDataMotionStrategy implements DataMotionStrategy { @Inject private HostDao _hostDao; @Inject + private HostDetailsDao hostDetailsDao; + @Inject + private NicDao nicDao; + @Inject private SnapshotDetailsDao _snapshotDetailsDao; @Inject private VMTemplateDetailsDao _vmTemplateDetailsDao; @@ -342,8 +350,9 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(vmTO); + Answer pfma; try { - Answer pfma = _agentManager.send(destHost.getId(), pfmc); + pfma = _agentManager.send(destHost.getId(), pfmc); if (pfma == null || !pfma.getResult()) { String details = pfma != null ? pfma.getDetails() : "null answer returned"; @@ -371,12 +380,18 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach migrateCommand.setAutoConvergence(kvmAutoConvergence); + VlanTrunkMigrationHelper.populateVlanTrunkMigrationDetails(migrateCommand, (PrepareForMigrationAnswer) pfma, hostDetailsDao, destHost.getId()); + MigrateAnswer migrateAnswer = (MigrateAnswer) _agentManager.send(srcHost.getId(), migrateCommand); boolean success = migrateAnswer != null && migrateAnswer.getResult(); handlePostMigration(success, srcVolumeInfoToDestVolumeInfo, vmTO, destHost); + if (success) { + VlanTrunkMigrationHelper.sendPostMigrationVlanTrunkMembershipIfNeeded(vmTO, destHost.getId(), nicDao, hostDetailsDao, _agentManager::send, logger); + } + if (migrateAnswer == null) { throw new CloudRuntimeException("Unable to get an answer to the migrate command"); } diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index a842954de3ca..851a531ec95d 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -480,7 +480,8 @@ private void avoidDifferentArchResources(VirtualMachineProfile vmProfile, DataCe * that were never going to work - which matters once a zone has more not-ready hosts than the start.retry budget. * No-op, and the candidate set is untouched, for a VM with no multi-VLAN trunk nic - the common case. */ - protected void avoidHostsNotReadyForMultiNetworkNics(VirtualMachineProfile vmProfile, DataCenter dc, ExcludeList avoids) { + @Override + public void avoidHostsNotReadyForMultiNetworkNics(VirtualMachineProfile vmProfile, DataCenter dc, ExcludeList avoids) { boolean hasMultiNetworkNic = _nicDao.listByVmId(vmProfile.getId()).stream().anyMatch(NicVO::getMultiNetwork); if (!hasMultiNetworkNic) { return; @@ -489,10 +490,7 @@ protected void avoidHostsNotReadyForMultiNetworkNics(VirtualMachineProfile vmPro if (CollectionUtils.isEmpty(candidateHostIds)) { return; } - Set readyHostIds = _hostDetailsDao.findByName(Host.HOST_VLAN_FILTERING_ENABLED).stream() - .filter(detail -> Boolean.parseBoolean(detail.getValue())) - .map(DetailVO::getHostId) - .collect(Collectors.toSet()); + Set readyHostIds = _hostDetailsDao.findHostIdsWithVlanFilteringEnabled(); List notReadyHostIds = candidateHostIds.stream() .filter(hostId -> !readyHostIds.contains(hostId)) .collect(Collectors.toList()); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index bb1fb289a86e..7412d2e5fa99 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -1655,6 +1655,8 @@ public ExcludeList applyAffinityConstraints(VirtualMachine vm, VirtualMachinePro } } + _dpMgr.avoidHostsNotReadyForMultiNetworkNics(vmProfile, _dcDao.findById(plan.getDataCenterId()), excludes); + // call affinitygroup chain final long vmGroupCount = _affinityGroupVMMapDao.countAffinityGroupsForVm(vm.getId()); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index f62b1846b3c4..1c13e03e5a15 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -271,10 +271,12 @@ import com.cloud.exception.VirtualMachineMigrationException; import com.cloud.gpu.GPU; import com.cloud.ha.HighAvailabilityManager; +import com.cloud.host.DetailVO; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; @@ -455,6 +457,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Inject private HostDao _hostDao; @Inject + private HostDetailsDao _hostDetailsDao; + @Inject private ServiceOfferingDao serviceOfferingDao; @Inject private DiskOfferingDao _diskOfferingDao; @@ -7718,6 +7722,11 @@ private DeployDestination checkVmMigrationDestination(VMInstanceVO vm, Host srcH throw new CloudRuntimeException("Cannot migrate VM, VM is DPDK enabled VM but destination host is not DPDK enabled"); } + if (!isHostReadyForMultiNetworkNics(vm, destinationHost)) { + throw new InvalidParameterValueException(String.format( + "Cannot migrate VM, VM has a multi-VLAN trunk nic but destination host: %s does not have VLAN filtering enabled", destinationHost)); + } + HostVO destinationHostVO = _hostDao.findById(destinationHost.getId()); _hostDao.loadHostTags(destinationHostVO); validateStrictHostTagCheck(vm, destinationHostVO); @@ -7749,6 +7758,16 @@ private DeployDestination checkVmMigrationDestination(VMInstanceVO vm, Host srcH return dest; } + // A multi-VLAN trunk nic can only be plugged on a host with VLAN filtering enabled - true (ready) for a VM with no such nic. + boolean isHostReadyForMultiNetworkNics(VMInstanceVO vm, Host host) { + boolean hasMultiNetworkNic = _nicDao.listByVmId(vm.getId()).stream().anyMatch(NicVO::getMultiNetwork); + if (!hasMultiNetworkNic) { + return true; + } + DetailVO detail = _hostDetailsDao.findDetail(host.getId(), Host.HOST_VLAN_FILTERING_ENABLED); + return detail != null && Boolean.parseBoolean(detail.getValue()); + } + private boolean isOnSupportedHypevisorForMigration(VMInstanceVO vm) { return (vm.getHypervisorType().equals(HypervisorType.XenServer) || vm.getHypervisorType().equals(HypervisorType.VMware) || diff --git a/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java b/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java index fc5049980118..d9d71993a682 100644 --- a/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java +++ b/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java @@ -40,7 +40,6 @@ import com.cloud.gpu.GPU; import com.cloud.gpu.dao.HostGpuGroupsDao; import com.cloud.gpu.dao.VgpuProfileDao; -import com.cloud.host.DetailVO; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -132,6 +131,7 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -518,14 +518,8 @@ public void avoidHostsNotReadyForMultiNetworkNicsExcludesNotReadyHosts() { Mockito.when(hostDao.listEnabledIdsByDataCenterId(dataCenterId)) .thenReturn(Arrays.asList(readyHostId, notReadyHostId, noDetailHostId)); - DetailVO readyDetail = Mockito.mock(DetailVO.class); - Mockito.when(readyDetail.getHostId()).thenReturn(readyHostId); - Mockito.when(readyDetail.getValue()).thenReturn("true"); - DetailVO notReadyDetail = Mockito.mock(DetailVO.class); - Mockito.when(notReadyDetail.getHostId()).thenReturn(notReadyHostId); - Mockito.when(notReadyDetail.getValue()).thenReturn("false"); - Mockito.when(hostDetailsDao.findByName(Host.HOST_VLAN_FILTERING_ENABLED)) - .thenReturn(Arrays.asList(readyDetail, notReadyDetail)); + Mockito.when(hostDetailsDao.findHostIdsWithVlanFilteringEnabled()) + .thenReturn(Collections.singleton(readyHostId)); _dpm.avoidHostsNotReadyForMultiNetworkNics(vmProfile, dc, avoids); diff --git a/server/src/test/java/com/cloud/server/ManagementServerImplTest.java b/server/src/test/java/com/cloud/server/ManagementServerImplTest.java index d2c12b6c99dd..74c9c5e66821 100644 --- a/server/src/test/java/com/cloud/server/ManagementServerImplTest.java +++ b/server/src/test/java/com/cloud/server/ManagementServerImplTest.java @@ -310,6 +310,19 @@ private void overrideDefaultConfigValue(final ConfigKey configKey, final String f.set(configKey, o); } + @Test + public void applyAffinityConstraintsChecksMultiNetworkNicHostReadiness() { + Mockito.when(virtualMachineMock.getId()).thenReturn(1L); + Mockito.when(virtualMachineMock.getHostId()).thenReturn(5L); + Mockito.when(dataCenterDeploymentMock.getDataCenterId()).thenReturn(2L); + DataCenterVO dc = Mockito.mock(DataCenterVO.class); + Mockito.when(dcDao.findById(2L)).thenReturn(dc); + + spy.applyAffinityConstraints(virtualMachineMock, virtualMachineProfileMock, dataCenterDeploymentMock, Collections.emptyList()); + + Mockito.verify(dpMgr).avoidHostsNotReadyForMultiNetworkNics(Mockito.eq(virtualMachineProfileMock), Mockito.eq(dc), Mockito.any()); + } + @Test(expected = InvalidParameterValueException.class) public void testDuplicateRegistrations() { String accountName = "account"; diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index fff8d9b024a6..3d253028c6ad 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -137,10 +137,12 @@ import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.host.DetailVO; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.network.Network; import com.cloud.network.NetworkModel; @@ -343,6 +345,9 @@ public class UserVmManagerImplTest { @Mock HostDao hostDao; + @Mock + private HostDetailsDao hostDetailsDao; + @Mock private VolumeVO volumeVOMock; @@ -576,11 +581,16 @@ public void afterTest() { } } + private NicVO mockNicWithId(long id) { + NicVO nic = Mockito.mock(NicVO.class); + Mockito.when(nic.getId()).thenReturn(id); + return nic; + } + @Test public void syncNicNetworkAssociationDhcpEntriesIsNoOpForANicWithoutAssociations() { UserVmVO vm = Mockito.mock(UserVmVO.class); - NicVO nic = Mockito.mock(NicVO.class); - Mockito.when(nic.getId()).thenReturn(11L); + NicVO nic = mockNicWithId(11L); userVmManagerImpl.syncNicNetworkAssociationDhcpEntries(vm, nic); @@ -591,8 +601,7 @@ public void syncNicNetworkAssociationDhcpEntriesIsNoOpForANicWithoutAssociations @Test public void syncNicNetworkAssociationDhcpEntriesPushesDhcpEntryForAnActiveAssociation() throws Exception { UserVmVO vm = Mockito.mock(UserVmVO.class); - NicVO nic = Mockito.mock(NicVO.class); - Mockito.when(nic.getId()).thenReturn(11L); + NicVO nic = mockNicWithId(11L); NicNetworkMapVO association = new NicNetworkMapVO(11L, 206L, "10.1.1.50", null); Mockito.when(nicNetworkMapDao.listByNicId(11L)).thenReturn(Collections.singletonList(association)); NetworkVO associatedNetwork = Mockito.mock(NetworkVO.class); @@ -611,8 +620,7 @@ public void syncNicNetworkAssociationDhcpEntriesPushesDhcpEntryForAnActiveAssoci @Test public void syncNicNetworkAssociationDhcpEntriesCleansUpARemovedAssociation() throws Exception { UserVmVO vm = Mockito.mock(UserVmVO.class); - NicVO nic = Mockito.mock(NicVO.class); - Mockito.when(nic.getId()).thenReturn(11L); + NicVO nic = mockNicWithId(11L); NicNetworkMapVO removedAssociation = new NicNetworkMapVO(11L, 207L, "10.1.1.60", null); Mockito.when(nicNetworkMapDao.listRemovedByNicId(11L)).thenReturn(Collections.singletonList(removedAssociation)); NetworkVO removedNetwork = Mockito.mock(NetworkVO.class); @@ -628,6 +636,55 @@ public void syncNicNetworkAssociationDhcpEntriesCleansUpARemovedAssociation() th Mockito.verify(nwHelper).sendCommandsToRouter(Mockito.eq(router), Mockito.any()); } + private VMInstanceVO mockVmWithId(long id) { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(id); + return vm; + } + + private NicVO mockNicWithMultiNetwork(boolean multiNetwork) { + NicVO nic = Mockito.mock(NicVO.class); + Mockito.when(nic.getMultiNetwork()).thenReturn(multiNetwork); + return nic; + } + + @Test + public void isHostReadyForMultiNetworkNicsIsReadyForAVmWithNoTrunkNic() { + VMInstanceVO vm = mockVmWithId(1L); + NicVO ordinaryNic = mockNicWithMultiNetwork(false); + Mockito.when(nicDao.listByVmId(1L)).thenReturn(Collections.singletonList(ordinaryNic)); + Host host = Mockito.mock(Host.class); + + Assert.assertTrue(userVmManagerImpl.isHostReadyForMultiNetworkNics(vm, host)); + Mockito.verifyNoInteractions(hostDetailsDao); + } + + @Test + public void isHostReadyForMultiNetworkNicsRejectsATrunkNicVmOnAHostWithoutVlanFiltering() { + VMInstanceVO vm = mockVmWithId(1L); + NicVO trunkNic = mockNicWithMultiNetwork(true); + Mockito.when(nicDao.listByVmId(1L)).thenReturn(Collections.singletonList(trunkNic)); + Host host = Mockito.mock(Host.class); + Mockito.when(host.getId()).thenReturn(2L); + Mockito.when(hostDetailsDao.findDetail(2L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(null); + + Assert.assertFalse(userVmManagerImpl.isHostReadyForMultiNetworkNics(vm, host)); + } + + @Test + public void isHostReadyForMultiNetworkNicsAcceptsATrunkNicVmOnAHostWithVlanFiltering() { + VMInstanceVO vm = mockVmWithId(1L); + NicVO trunkNic = mockNicWithMultiNetwork(true); + Mockito.when(nicDao.listByVmId(1L)).thenReturn(Collections.singletonList(trunkNic)); + Host host = Mockito.mock(Host.class); + Mockito.when(host.getId()).thenReturn(2L); + DetailVO detail = Mockito.mock(DetailVO.class); + Mockito.when(detail.getValue()).thenReturn("true"); + Mockito.when(hostDetailsDao.findDetail(2L, Host.HOST_VLAN_FILTERING_ENABLED)).thenReturn(detail); + + Assert.assertTrue(userVmManagerImpl.isHostReadyForMultiNetworkNics(vm, host)); + } + @Test public void validateGuestOsIdForUpdateVirtualMachineCommandTestOsTypeNull() { Mockito.when(updateVmCommand.getOsTypeId()).thenReturn(null); From c9b98eaeb2c38f4360e886469e1a676ffb1fc2c4 Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Fri, 2 Oct 2026 13:31:18 -0400 Subject: [PATCH 18/20] fix test --- .../com/cloud/deploy/DeploymentPlanningManagerImplTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java b/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java index 527409a723dc..e7f59eadeb8b 100644 --- a/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java +++ b/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java @@ -234,9 +234,6 @@ public class DeploymentPlanningManagerImplTest { @Mock NicDao nicDao; - @Mock - HostDetailsDao hostDetailsDao; - @Inject HostPodDao _podDao; From 5f9594648e7cbeae067b711fb3dae6b35863db10 Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Fri, 2 Oct 2026 13:48:23 -0400 Subject: [PATCH 19/20] Fix CI build failures from the multi-VLAN trunk NIC changes - Add missing VlanTrunkMigrationHelper import in VirtualMachineManagerImpl - Fix AssignLoadBalancerTest's NetworkModelImpl spy to stub the new getNicAndIpInNetwork method used by LoadBalancingRulesManagerImpl, matching the existing getNics() stub it replaced --- .../java/com/cloud/vm/VirtualMachineManagerImpl.java | 1 + .../com/cloud/network/lb/AssignLoadBalancerTest.java | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 9ad348112cbd..f432cf50228f 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -159,6 +159,7 @@ import com.cloud.agent.api.UnregisterVMCommand; import com.cloud.agent.api.UpdateVmNicAnswer; import com.cloud.agent.api.UpdateVmNicCommand; +import com.cloud.agent.api.VlanTrunkMigrationHelper; import com.cloud.agent.api.VmDiskStatsEntry; import com.cloud.agent.api.VmNetworkStatsEntry; import com.cloud.agent.api.VmStatsEntry; diff --git a/server/src/test/java/com/cloud/network/lb/AssignLoadBalancerTest.java b/server/src/test/java/com/cloud/network/lb/AssignLoadBalancerTest.java index 1fc0517a727e..cbbda1f2ee74 100644 --- a/server/src/test/java/com/cloud/network/lb/AssignLoadBalancerTest.java +++ b/server/src/test/java/com/cloud/network/lb/AssignLoadBalancerTest.java @@ -40,6 +40,7 @@ import com.cloud.user.UserVO; import com.cloud.user.dao.AccountDao; import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; import com.cloud.vm.Nic; import com.cloud.vm.NicVO; import com.cloud.vm.UserVmVO; @@ -102,6 +103,15 @@ public List getNics(long vmId) { nic.add(nicvo); return nic; } + + @Override + public Pair getNicAndIpInNetwork(long vmId, long networkId) { + nicvo.setNetworkId(204L); + if (networkId != 204L) { + return null; + } + return new Pair<>(nicvo, new Network.IpAddresses(nicvo.getIPv4Address(), nicvo.getIPv6Address())); + } }; From b4c18d655ec6ebca0db7c869580c47e153a12f22 Mon Sep 17 00:00:00 2001 From: Pearl1594 Date: Fri, 2 Oct 2026 14:15:17 -0400 Subject: [PATCH 20/20] Fix NullPointerException in LibvirtComputingResourceTest.testMigrateCommand replaceVlanTrunkInterfaces iterates VirtualMachineTO.getNics(), which the test's bare mock returned null for (unstubbed), unlike every real VirtualMachineTO built via toVmTO, which always sets nics. Stub it the same way getDisks() already was. --- .../hypervisor/kvm/resource/LibvirtComputingResourceTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index 008d9444fe5f..3cbd9c628f7c 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -1749,10 +1749,12 @@ public void testMigrateCommand() { final VirtualMachineTO vmTO = Mockito.mock(VirtualMachineTO.class); final boolean executeInSequence = false; DiskTO[] diskTOS = new DiskTO[]{}; + final NicTO[] nicTOs = new NicTO[]{}; final MigrateCommand command = new MigrateCommand(vmName, destIp, isWindows, vmTO, executeInSequence ); when(libvirtComputingResourceMock.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); when(vmTO.getDisks()).thenReturn(diskTOS); + when(vmTO.getNics()).thenReturn(nicTOs); try { when(libvirtUtilitiesHelper.getConnectionByVmName(vmName)).thenReturn(conn); when(libvirtUtilitiesHelper.retrieveQemuConnection("qemu+tcp://" + command.getDestinationIp() + "/system")).thenReturn(dconn);