IGNITE-28207 IgniteClassPath creation

modules/binary/api/src/main/java/org/apache/ignite/binary/BinaryRawWriter.java

@@ -128,6 +128,14 @@ public interface BinaryRawWriter {
      */
     public void writeByteArray(@Nullable byte[] val) throws BinaryObjectException;
 
+    /**
+     * @param val Value to write.
+     * @param off Offset to start write from.
+     * @param len Count of bytes to write.
+     * @throws BinaryObjectException In case of error.
+     */
+    public void writeByteArray(@Nullable byte[] val, int off, int len) throws BinaryObjectException;
+
     /**
      * @param val Value to write.
      * @throws BinaryObjectException In case of error.

modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java

@@ -904,6 +904,21 @@ void writeBooleanField(@Nullable Boolean val) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void writeByteArray(@Nullable byte[] val, int pos, int len) throws BinaryObjectException {
+        if (val == null)
+            out.writeByte(GridBinaryMarshaller.NULL);
+        else {
+            if (len > val.length)
+                throw new IllegalArgumentException("len must be less then val.length");
+
+            out.unsafeEnsure(1 + 4);
+            out.unsafeWriteByte(GridBinaryMarshaller.BYTE_ARR);
+            out.unsafeWriteInt(len);
+            out.writeByteArray(val, pos, len);
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public void writeShortArray(String fieldName, @Nullable short[] val) throws BinaryObjectException {
         writeFieldId(fieldName);

modules/codegen/src/main/java/org/apache/ignite/internal/MessageProcessor.java

@@ -33,12 +33,14 @@
 import javax.lang.model.SourceVersion;
 import javax.lang.model.element.Element;
 import javax.lang.model.element.ElementKind;
+import javax.lang.model.element.ExecutableElement;
 import javax.lang.model.element.Modifier;
 import javax.lang.model.element.TypeElement;
 import javax.lang.model.element.VariableElement;
 import javax.lang.model.type.TypeKind;
 import javax.lang.model.type.TypeMirror;
 import javax.tools.Diagnostic;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.lang.IgniteBiTuple;
 
 import static org.apache.ignite.internal.MessageSerializerGenerator.DLFT_ENUM_MAPPER_CLS;
@@ -115,6 +117,9 @@ public class MessageProcessor extends AbstractProcessor {
             if (clazz.getModifiers().contains(Modifier.ABSTRACT))
                 continue;
 
+            if (!checkConstructors(clazz))
+                continue;
+
             List<VariableElement> fields = orderedFields(clazz);
 
             if (fields.isEmpty() && emptyMsgs.stream().noneMatch(t -> isAssignable(t, clazz))) {
@@ -146,6 +151,34 @@ public class MessageProcessor extends AbstractProcessor {
         return true;
     }
 
+    /** */
+    private boolean checkConstructors(TypeElement clazz) {
+        boolean isMarshallableMsg = isAssignable(
+            clazz.asType(),
+            processingEnv.getElementUtils().getTypeElement("org.apache.ignite.internal.MarshallableMessage")
+        );
+
+        for (Element el : clazz.getEnclosedElements()) {
+            if (el.getKind() != ElementKind.CONSTRUCTOR)
+                continue;
+
+            ExecutableElement c = (ExecutableElement)el;
+
+            boolean isDfltConstructor = F.isEmpty(c.getParameters());
+
+            if (isDfltConstructor && !isMarshallableMsg)
+                return true;
+        }
+
+        processingEnv.getMessager().printMessage(
+            Diagnostic.Kind.ERROR,
+            "Class must have default constructor: " + clazz.getQualifiedName(),
+            clazz
+        );
+
+        return false;
+    }
+
     /**
      * Collects all fields annotated with {@link Order} from the given {@link TypeElement} and all its superclasses.
      * <p>

modules/commons/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java

@@ -2153,4 +2153,18 @@ public static <T> Collection<T> emptyIfNull(@Nullable Collection<T> col) {
     public static <K, V> Map<K, V> emptyIfNull(@Nullable Map<K, V> map) {
         return map == null ? Collections.emptyMap() : map;
     }
+
+    /**
+     * @param arr Array.
+     * @param el Element.
+     * @return Element index or {@code -1} if not found.
+     */
+    public static <T> int indexOf(T[] arr, T el) {
+        for (int i = 0; i < arr.length; i++) {
+            if (Objects.equals(arr[i], el))
+                return i;
+        }
+
+        return -1;
+    }
 }

modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite2.java

@@ -22,6 +22,7 @@
 import org.apache.ignite.util.CacheMetricsCommandTest;
 import org.apache.ignite.util.CdcCommandTest;
 import org.apache.ignite.util.CdcResendCommandTest;
+import org.apache.ignite.util.GridCommandHandlerClassPathTest;
 import org.apache.ignite.util.GridCommandHandlerConsistencyBinaryTest;
 import org.apache.ignite.util.GridCommandHandlerConsistencySensitiveTest;
 import org.apache.ignite.util.GridCommandHandlerConsistencyTest;
@@ -74,7 +75,8 @@
 
     SecurityCommandHandlerPermissionsTest.class,
 
-    IdleVerifyDumpTest.class
+    IdleVerifyDumpTest.class,
+    GridCommandHandlerClassPathTest.class
 })
 public class IgniteControlUtilityTestSuite2 {
 }

modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClassPathTest.java

@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree;
+import org.junit.Test;
+
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_INVALID_ARGUMENTS;
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
+import static org.apache.ignite.testframework.GridTestUtils.assertContains;
+
+/**
+ * Test for --classpath command.
+ */
+public class GridCommandHandlerClassPathTest extends GridCommandHandlerAbstractTest {
+    /** */
+    public static final int GRID_CNT = 3;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        cleanPersistenceDir();
+
+        startGrids(GRID_CNT);
+
+        super.beforeTestsStarted();
+    }
+
+    // TODO check empty file creation.
+    // TODO add in production code checks of files integriy. Perform file integrity check on startup.
+    // Support JXM handler.
+    // Support pretty print for command.
+    // Test for different failure types: node fail, file write fail, etc.
+    // Concurrent creation of CP with the same name.
+
+    /** Tests --create command. */
+    @Test
+    public void testCreate() {
+        //grid(0).cluster().state(ClusterState.ACTIVE);
+
+        Set<Path> jars = jars(
+                Path.of(getClass().getClassLoader().getResource(".").getPath() + "../"),
+                Path.of(getClass().getClassLoader().getResource(".").getPath() + "../../../core/target")
+        );
+
+        injectTestSystemOut();
+
+        final TestCommandHandler hnd = newCommandHandler(createTestLogger());
+
+        String cpName = "mysuperapp";
+
+        try {
+            String files = jars.stream().map(Path::toFile).map(File::getAbsolutePath).collect(Collectors.joining(","));
+
+            assertEquals(EXIT_CODE_OK, execute(hnd, "--class-path", "create", "--name", cpName, "--files", files));
+        }
+        finally {
+            String outStr = testOut.toString();
+
+            System.out.println(outStr);
+        }
+
+        Set<String> cpFilesNames = fileNames(jars);
+
+        for (int i = 0; i < GRID_CNT; i++) {
+            NodeFileTree ft = grid(i).context().pdsFolderResolver().fileTree();
+
+            assertEquals("Files must be deployed on each node", cpFilesNames, fileNames(jars(ft.classPathRoot(cpName).toPath())));
+        }
+    }
+
+    /** Tests --create command arguments format. */
+    @Test
+    public void testEmptyFilesArgument() {
+        injectTestSystemOut();
+
+        assertContains(
+            log,
+            executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--class-path", "create", "--name", "mysuperapp", "--files"),
+            "Please specify a value for argument: --files"
+        );
+
+        assertContains(
+            log,
+            executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--class-path", "create", "--name", "mysuperapp"),
+            "Mandatory argument(s) missing: [--files]"
+        );
+
+        assertContains(
+            log,
+            executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--class-path", "create", "--name", "--files", "some_files"),
+            "Please specify a value for argument: --name"
+        );
+
+        assertContains(
+            log,
+            executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--class-path", "create", "--files", "some_files"),
+            "Mandatory argument(s) missing: [--name]"
+        );
+
+        assertContains(
+            log,
+            executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--class-path", "create", "--name", "mysuperapp", "--files", ""),
+            cliCommandHandler() ? "File name must not be empty" : "Argument --files required"
+        );
+    }
+
+    /** */
+    private Set<String> fileNames(Set<Path> dirs) {
+        return dirs.stream().map(Path::getFileName).map(Path::toString).collect(Collectors.toSet());
+    }
+
+    /** */
+    private Set<Path> jars(Path... dirs) {
+        return Stream.of(dirs).flatMap(dir -> {
+            try {
+                return Files.list(dir).filter(p -> p.getFileName().toString().endsWith("jar"));
+            }
+            catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }).map(Path::toAbsolutePath).collect(Collectors.toSet());
+    }
+}

modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java

@@ -23,6 +23,10 @@
 import org.apache.ignite.internal.cache.query.index.IndexQueryResultMeta;
 import org.apache.ignite.internal.cache.query.index.sorted.IndexKeyDefinition;
 import org.apache.ignite.internal.cache.query.index.sorted.IndexKeyTypeSettings;
+import org.apache.ignite.internal.classpath.ClassPathDeployToAllRequest;
+import org.apache.ignite.internal.classpath.ClassPathDeployToAllResponse;
+import org.apache.ignite.internal.classpath.DownloadClassPathFailureMessage;
+import org.apache.ignite.internal.classpath.DownloadClassPathMessage;
 import org.apache.ignite.internal.management.cache.PartitionKey;
 import org.apache.ignite.internal.managers.checkpoint.GridCheckpointRequest;
 import org.apache.ignite.internal.managers.communication.CompressedMessage;
@@ -666,6 +670,11 @@ public CoreMessagesProvider(Marshaller dfltMarsh, Marshaller schemaAwareMarsh, C
         withNoSchema(PartitionHashRecord.class);
         withNoSchema(TransactionsHashRecord.class);
 
+        withNoSchema(ClassPathDeployToAllRequest.class);
+        withNoSchema(ClassPathDeployToAllResponse.class);
+        withNoSchema(DownloadClassPathMessage.class);
+        withNoSchema(DownloadClassPathFailureMessage.class);
+
         assert msgIdx <= MAX_MESSAGE_ID;
     }
 

modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java

@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.cache.query.index.IndexProcessor;
 import org.apache.ignite.internal.cache.transform.CacheObjectTransformerProcessor;
+import org.apache.ignite.internal.classpath.ClassPathProcessor;
 import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager;
 import org.apache.ignite.internal.managers.collision.GridCollisionManager;
 import org.apache.ignite.internal.managers.communication.GridIoManager;
@@ -656,6 +657,11 @@ public interface GridKernalContext extends Iterable<GridComponent> {
      */
     public RollingUpgradeProcessor rollingUpgrade();
 
+    /**
+     * @return Class path processor.
+     */
+    public ClassPathProcessor classPath();
+
     /**
      * Executor that is in charge of processing user async continuations.
      *

modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java

@@ -41,6 +41,7 @@
 import org.apache.ignite.internal.binary.GridBinaryMarshaller;
 import org.apache.ignite.internal.cache.query.index.IndexProcessor;
 import org.apache.ignite.internal.cache.transform.CacheObjectTransformerProcessor;
+import org.apache.ignite.internal.classpath.ClassPathProcessor;
 import org.apache.ignite.internal.maintenance.MaintenanceProcessor;
 import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager;
 import org.apache.ignite.internal.managers.collision.GridCollisionManager;
@@ -368,6 +369,10 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     @GridToStringExclude
     private RollingUpgradeProcessor rollUpProc;
 
+    /** Classpath processor. */
+    @GridToStringExclude
+    private ClassPathProcessor classPathProc;
+
     /** */
     private Thread.UncaughtExceptionHandler hnd;
 
@@ -601,6 +606,8 @@ else if (comp instanceof PerformanceStatisticsProcessor)
             perfStatProc = (PerformanceStatisticsProcessor)comp;
         else if (comp instanceof RollingUpgradeProcessor)
             rollUpProc = (RollingUpgradeProcessor)comp;
+        else if (comp instanceof ClassPathProcessor)
+            classPathProc = (ClassPathProcessor)comp;
         else if (comp instanceof IndexProcessor)
             indexProc = (IndexProcessor)comp;
         else if (!(comp instanceof DiscoveryNodeValidationProcessor
@@ -1111,6 +1118,11 @@ public void recoveryMode(boolean recoveryMode) {
         return rollUpProc;
     }
 
+    /** {@inheritDoc} */
+    @Override public ClassPathProcessor classPath() {
+        return classPathProc;
+    }
+
     /** {@inheritDoc} */
     @Override public Executor getAsyncContinuationExecutor() {
         return config().getAsyncContinuationExecutor() == null

modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java

@@ -89,6 +89,7 @@
 import org.apache.ignite.internal.binary.BinaryUtils;
 import org.apache.ignite.internal.cache.query.index.IndexProcessor;
 import org.apache.ignite.internal.cache.transform.CacheObjectTransformerProcessor;
+import org.apache.ignite.internal.classpath.ClassPathProcessor;
 import org.apache.ignite.internal.cluster.ClusterGroupAdapter;
 import org.apache.ignite.internal.cluster.IgniteClusterEx;
 import org.apache.ignite.internal.maintenance.MaintenanceProcessor;
@@ -1025,6 +1026,7 @@ public void start(
             // Start the encryption manager after assigning the discovery manager to context, so it will be
             // able to register custom event listener.
             startManager(new GridEncryptionManager(ctx));
+            startProcessor(new ClassPathProcessor(ctx));
 
             startProcessor(new PdsConsistentIdProcessor(ctx));