engine-schema: upgrade path for 24.0.0 (#14033)

* engine-schema: upgrade path for 24.0.0

Adds a 4.23.0 to 24.0.0 upgrade path.

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* fix CS version

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* fix upgrade unit tests for cutover

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* fix imports

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* fix template version

* continue using 3 part version for sys template

* fix

* fix more

* use security version after cutover

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* improvements and add tests

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* fix more scripts

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* address remaining comments

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* more changes

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* Update versioning logic in export-templates.sh

Refactor versioning logic to handle four-component metadata separately.

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Fix export-templates.sh syntax

---------

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nvazquez <nicovazquez90@gmail.com>

Author: 3 people

engine/schema/pom.xml

@@ -153,6 +153,20 @@
                 <artifactId>exec-maven-plugin</artifactId>
                 <version>${cs.exec-maven-plugin.version}</version>
                 <executions>
+                    <execution>
+                        <id>test-templateConfig</id>
+                        <phase>test</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                        <configuration>
+                            <workingDirectory>${basedir}/</workingDirectory>
+                            <executable>bash</executable>
+                            <arguments>
+                                <argument>test_templateConfig.sh</argument>
+                            </arguments>
+                        </configuration>
+                    </execution>
                     <execution>
                         <id>systemvm-template-metadata</id>
                         <phase>package</phase>

engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java

@@ -91,11 +91,12 @@
 import com.cloud.upgrade.dao.Upgrade42020to42030;
 import com.cloud.upgrade.dao.Upgrade42030to42040;
 import com.cloud.upgrade.dao.Upgrade42040to42100;
-import com.cloud.upgrade.dao.Upgrade42100to42200;
-import com.cloud.upgrade.dao.Upgrade42200to42210;
 import com.cloud.upgrade.dao.Upgrade420to421;
+import com.cloud.upgrade.dao.Upgrade42100to42200;
 import com.cloud.upgrade.dao.Upgrade421to430;
+import com.cloud.upgrade.dao.Upgrade42200to42210;
 import com.cloud.upgrade.dao.Upgrade42210to42300;
+import com.cloud.upgrade.dao.Upgrade42300to2400;
 import com.cloud.upgrade.dao.Upgrade430to440;
 import com.cloud.upgrade.dao.Upgrade431to440;
 import com.cloud.upgrade.dao.Upgrade432to440;
@@ -248,6 +249,7 @@ public DatabaseUpgradeChecker() {
                 .next("4.21.0.0", new Upgrade42100to42200())
                 .next("4.22.0.0", new Upgrade42200to42210())
                 .next("4.22.1.0", new Upgrade42210to42300())
+                .next("4.23.0.0", new Upgrade42300to2400())
                 .build();
     }
 
@@ -513,8 +515,7 @@ protected void doUpgrades(GlobalLock lock) {
             String csVersion = parseSystemVmMetadata();
             final CloudStackVersion sysVmVersion = CloudStackVersion.parse(csVersion);
             final  CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue);
-            SystemVmTemplateRegistration.CS_MAJOR_VERSION  = sysVmVersion.getMajorRelease() + "." + sysVmVersion.getMinorRelease();
-            SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease());
+            updateSystemVmTemplateVersion(sysVmVersion);
 
             LOGGER.info("DB version = {} Code Version = {}", dbVersion, currentVersion);
 
@@ -540,6 +541,18 @@ protected void doUpgrades(GlobalLock lock) {
         }
     }
 
+    /**
+     * Sets the system VM template major/tiny version used to look up the matching system VM template,
+     * from the version parsed out of the system VM template metadata file. Below the versioning cutover
+     * (major < 24) the tiny version is the legacy patch release; from the cutover onwards it is the
+     * security release, since the patch position is dropped in that scheme.
+     */
+    @VisibleForTesting
+    protected static void updateSystemVmTemplateVersion(CloudStackVersion sysVmVersion) {
+        SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.format("%d.%d", sysVmVersion.getMajorRelease(), sysVmVersion.getMinorRelease());
+        SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getTinyRelease());
+    }
+
     /**
      * Hook that is called when an upgrade is required but the management server is clustered.
      * Default behavior is to exit the JVM, tests can override to throw instead.

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.upgrade.dao;
+
+public class Upgrade42300to2400 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate {
+
+    @Override
+    public String[] getUpgradableVersionRange() {
+        return new String[]{"4.23.0.0", "24.0.0"};
+    }
+
+    @Override
+    public String getUpgradedVersion() {
+        return "24.0.0";
+    }
+}

engine/schema/src/main/resources/META-INF/db/schema-42300to2400-cleanup.sql

@@ -0,0 +1,20 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade cleanup from 4.23.0.0 to 24.0.0
+--;

engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql

@@ -0,0 +1,20 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade from 4.23.0.0 to 24.0.0
+--;

engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java

@@ -16,20 +16,25 @@
 // under the License.
 package com.cloud.upgrade;
 
-import java.sql.SQLException;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
 import java.lang.reflect.Field;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
+import java.sql.SQLException;
 
 import javax.sql.DataSource;
 
 import org.apache.cloudstack.utils.CloudStackVersion;
-import org.junit.Test;
-import org.junit.Before;
 import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
 import org.junit.runner.RunWith;
-
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mock;
 import org.mockito.Mockito;
@@ -47,6 +52,7 @@
 import com.cloud.upgrade.dao.Upgrade42020to42030;
 import com.cloud.upgrade.dao.Upgrade42030to42040;
 import com.cloud.upgrade.dao.Upgrade42040to42100;
+import com.cloud.upgrade.dao.Upgrade42300to2400;
 import com.cloud.upgrade.dao.Upgrade452to453;
 import com.cloud.upgrade.dao.Upgrade453to460;
 import com.cloud.upgrade.dao.Upgrade460to461;
@@ -55,15 +61,8 @@
 import com.cloud.upgrade.dao.Upgrade471to480;
 import com.cloud.upgrade.dao.Upgrade480to481;
 import com.cloud.upgrade.dao.Upgrade490to4910;
-
 import com.cloud.utils.db.TransactionLegacy;
 
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertArrayEquals;
-
 
 @RunWith(MockitoJUnitRunner.class)
 public class DatabaseUpgradeCheckerTest {
@@ -81,6 +80,8 @@ public class DatabaseUpgradeCheckerTest {
     ResultSet resultSet;
 
     private DataSource backupDataSource;
+    private String previousCsMajorVersion;
+    private String previousCsTinyVersion;
 
     @Before
     public void setup() throws Exception {
@@ -92,13 +93,19 @@ public void setup() throws Exception {
         Mockito.when(dataSource.getConnection()).thenReturn(connection);
         Mockito.when(connection.prepareStatement(ArgumentMatchers.anyString())).thenReturn(preparedStatement);
         Mockito.when(preparedStatement.executeQuery()).thenReturn(resultSet);
+
+        previousCsMajorVersion = SystemVmTemplateRegistration.CS_MAJOR_VERSION;
+        previousCsTinyVersion = SystemVmTemplateRegistration.CS_TINY_VERSION;
     }
 
     @After
     public void cleanup() throws Exception {
         Field dsField = TransactionLegacy.class.getDeclaredField("s_ds");
         dsField.setAccessible(true);
         dsField.set(null, backupDataSource);
+
+        SystemVmTemplateRegistration.CS_MAJOR_VERSION = previousCsMajorVersion;
+        SystemVmTemplateRegistration.CS_TINY_VERSION = previousCsTinyVersion;
     }
 
     @Test
@@ -214,10 +221,10 @@ public void testFindUpgradePath452to490() {
     @Test
     public void testCalculateUpgradePathUnknownDbVersion() {
 
-        final CloudStackVersion dbVersion = CloudStackVersion.parse("4.99.0.0");
+        final CloudStackVersion dbVersion = CloudStackVersion.parse("99.0.0");
         assertNotNull(dbVersion);
 
-        final CloudStackVersion currentVersion = CloudStackVersion.parse("4.99.1.0");
+        final CloudStackVersion currentVersion = CloudStackVersion.parse("99.1.0");
         assertNotNull(currentVersion);
 
         final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker();
@@ -234,7 +241,7 @@ public void testCalculateUpgradePathFromKnownDbVersion() {
         final CloudStackVersion dbVersion = CloudStackVersion.parse("4.17.0.0");
         assertNotNull(dbVersion);
 
-        final CloudStackVersion currentVersion = CloudStackVersion.parse("4.99.1.0");
+        final CloudStackVersion currentVersion = CloudStackVersion.parse("99.1.0");
         assertNotNull(currentVersion);
 
         final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker();
@@ -268,10 +275,7 @@ public void testCalculateUpgradePathFromLatestDbVersion() {
         final CloudStackVersion dbVersion = checker.getLatestVersion();
         assertNotNull(dbVersion);
 
-        final CloudStackVersion currentVersion = CloudStackVersion.parse(dbVersion.getMajorRelease() + "."
-                + dbVersion.getMinorRelease() + "."
-                + dbVersion.getPatchRelease() + "."
-                + (dbVersion.getSecurityRelease() + 1));
+        final CloudStackVersion currentVersion = getNextSecurityRelease(dbVersion);
         assertNotNull(currentVersion);
 
         final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion);
@@ -293,10 +297,7 @@ public void testCalculateUpgradePathFrom41800toNextSecurityRelease() {
         final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion);
         assertNotNull(upgrades);
 
-        final CloudStackVersion nextSecurityRelease = CloudStackVersion.parse(currentVersion.getMajorRelease() + "."
-                + currentVersion.getMinorRelease() + "."
-                + currentVersion.getPatchRelease() + "."
-                + (currentVersion.getSecurityRelease() + 1));
+        final CloudStackVersion nextSecurityRelease = getNextSecurityRelease(currentVersion);
         assertNotNull(nextSecurityRelease);
 
         final DbUpgrade[] upgradesToNext = checker.calculateUpgradePath(dbVersion, nextSecurityRelease);
@@ -306,16 +307,26 @@ public void testCalculateUpgradePathFrom41800toNextSecurityRelease() {
         assertTrue(upgradesToNext[upgradesToNext.length - 1] instanceof NoopDbUpgrade);
     }
 
+    private static CloudStackVersion getNextSecurityRelease(CloudStackVersion version, int increment) {
+        String nextSecurityReleaseVersionStr = version.getMajorRelease() + "."
+                + version.getMinorRelease() + "."
+                + (version.usesNewVersioning() ? "" : version.getPatchRelease() + ".")
+                + (version.getSecurityRelease() + increment);
+
+        return CloudStackVersion.parse(nextSecurityReleaseVersionStr);
+    }
+
+    private static CloudStackVersion getNextSecurityRelease(CloudStackVersion version) {
+        return getNextSecurityRelease(version, 1);
+    }
+
     @Test
     public void testCalculateUpgradePathFromSecurityReleaseToLatest() {
 
         final CloudStackVersion dbVersion = CloudStackVersion.parse("4.17.2.0");    // a EOL version
         assertNotNull(dbVersion);
 
-        final CloudStackVersion oldSecurityRelease = CloudStackVersion.parse(dbVersion.getMajorRelease() + "."
-                + dbVersion.getMinorRelease() + "."
-                + dbVersion.getPatchRelease() + "."
-                + (dbVersion.getSecurityRelease() + 100));
+        final CloudStackVersion oldSecurityRelease = getNextSecurityRelease(dbVersion, 100);
         assertNotNull(oldSecurityRelease);      // fake security release 4.17.2.100
 
         final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker();
@@ -347,10 +358,7 @@ public void testCalculateUpgradePathFromSecurityReleaseToNextSecurityRelease() {
         final CloudStackVersion currentVersion = checker.getLatestVersion();
         assertNotNull(currentVersion);
 
-        final CloudStackVersion nextSecurityRelease = CloudStackVersion.parse(currentVersion.getMajorRelease() + "."
-                + currentVersion.getMinorRelease() + "."
-                + currentVersion.getPatchRelease() + "."
-                + (currentVersion.getSecurityRelease() + 1));
+        final CloudStackVersion nextSecurityRelease = getNextSecurityRelease(currentVersion);
         assertNotNull(nextSecurityRelease);     // fake security release
 
         final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion);
@@ -422,4 +430,48 @@ public void testCalculateUpgradePath42010to42100() {
         assertTrue(upgrades[2] instanceof Upgrade42040to42100);
         assertEquals(currentVersion.toString(), upgrades[2].getUpgradedVersion());
     }
+
+    @Test
+    public void testCalculateUpgradePath42300to2400() {
+
+        final CloudStackVersion dbVersion = CloudStackVersion.parse("4.23.0.0");
+        assertNotNull(dbVersion);
+
+        final CloudStackVersion currentVersion = CloudStackVersion.parse("24.0.0");
+        assertNotNull(currentVersion);
+
+        final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker();
+        final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion);
+
+        assertNotNull(upgrades);
+        assertEquals(1, upgrades.length);
+        assertTrue(upgrades[0] instanceof Upgrade42300to2400);
+
+        assertArrayEquals(new String[]{"4.23.0.0", "24.0.0"}, upgrades[0].getUpgradableVersionRange());
+        assertEquals(currentVersion.toString(), upgrades[0].getUpgradedVersion());
+    }
+
+    @Test
+    public void testUpdateSystemVmTemplateVersionBelowCutover() {
+        DatabaseUpgradeChecker.updateSystemVmTemplateVersion(CloudStackVersion.parse("4.22.1.0"));
+
+        assertEquals("4.22", SystemVmTemplateRegistration.CS_MAJOR_VERSION);
+        assertEquals("1", SystemVmTemplateRegistration.CS_TINY_VERSION);
+    }
+
+    @Test
+    public void testUpdateSystemVmTemplateVersionAtCutover() {
+        DatabaseUpgradeChecker.updateSystemVmTemplateVersion(CloudStackVersion.parse("24.0.1"));
+
+        assertEquals("24.0", SystemVmTemplateRegistration.CS_MAJOR_VERSION);
+        assertEquals("1", SystemVmTemplateRegistration.CS_TINY_VERSION);
+    }
+
+    @Test
+    public void testUpdateSystemVmTemplateVersionAfterCutover() {
+        DatabaseUpgradeChecker.updateSystemVmTemplateVersion(CloudStackVersion.parse("25.3.2"));
+
+        assertEquals("25.3", SystemVmTemplateRegistration.CS_MAJOR_VERSION);
+        assertEquals("2", SystemVmTemplateRegistration.CS_TINY_VERSION);
+    }
 }