diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68348c9e5f47..8999b6019d11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -340,7 +340,7 @@ jobs: pre-script: sudo hostname localhost ratis-args: ${{ inputs.ratis_args }} script: integration - script-args: -Ptest-${{ matrix.profile }} -Drocks_tools_native + script-args: -Ptest-${{ matrix.profile }} -Phadoop-native-lib -Drocks_tools_native sha: ${{ needs.build-info.outputs.sha }} split: ${{ matrix.profile }} timeout-minutes: 90 diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java index 2c1b19ad5bfb..6207d8cbe6eb 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java @@ -21,9 +21,9 @@ import java.io.IOException; import java.net.SocketTimeoutException; import java.nio.channels.AsynchronousCloseException; -import java.util.HashMap; import java.util.List; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -69,8 +69,8 @@ public final class XceiverServerDomainSocket implements XceiverServerSpi, Runnab private DomainSocket domainSocket; private final ConfigurationSource config; private final String threadPrefix; - private final HashMap peers = new HashMap<>(); - private final HashMap peersReceiver = new HashMap<>(); + private final ConcurrentHashMap peers = new ConcurrentHashMap<>(); + private final ConcurrentHashMap peersReceiver = new ConcurrentHashMap<>(); private int readTimeoutMs; private int writeTimeoutMs; private final ThreadPoolExecutor readExecutors; @@ -162,8 +162,7 @@ public void stop() { if (server != null) { try { if (domainSocket != null) { - // TODO: once HADOOP-19261 is merged, change it to domainSocket.close(true); - domainSocket.close(); + domainSocket.close(true); LOG.info("UNIX domain socket server listening on {} is stopped", domainSocket.getPath()); } } catch (IOException e) { diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml index 6c1ccb94f586..edc60bd8dc3a 100644 --- a/hadoop-ozone/integration-test/pom.xml +++ b/hadoop-ozone/integration-test/pom.xml @@ -571,6 +571,17 @@ none + + org.apache.maven.plugins + maven-surefire-plugin + + ${maven-surefire-plugin.argLine} ${maven-surefire-plugin.argLineAccessArgs} @{argLine} -Djava.library.path=${project.basedir}/../../target/native-lib + + ${project.basedir}/../../target/native-lib + ${project.basedir}/../../target/native-lib + + + diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverServerDomainSocket.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/TestXceiverServerDomainSocket.java similarity index 97% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverServerDomainSocket.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/TestXceiverServerDomainSocket.java index 1d5e88cb2925..debba4b8306d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverServerDomainSocket.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/TestXceiverServerDomainSocket.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hadoop.hdds.scm; +package org.apache.hadoop.hdds; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type.GetBlock; @@ -59,6 +59,8 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.scm.OzoneClientConfig; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.pipeline.MockPipeline; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.storage.DomainSocketFactory; @@ -86,7 +88,6 @@ import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.hadoop.utils.FaultInjectorImpl; import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.tag.Unhealthy; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -96,6 +97,11 @@ /** * Tests the XceiverServerDomainSocket class. + * Add Environment variables + * LD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib + * DYLD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib + * to intellij run configuration to run it locally. + * Dynamically set the java.library.path in java code doesn't affect the library loading */ @Timeout(300) public class TestXceiverServerDomainSocket { @@ -128,7 +134,6 @@ public static void setup() { } @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testIllegalDomainPathConfiguration() { // empty domain path conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, ""); @@ -172,7 +177,6 @@ public void testIllegalDomainPathConfiguration() { } @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testExistingDomainPath() { // an existing domain path, the existing file is override and changed from a normal file to a socket file conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath()); @@ -190,7 +194,6 @@ public void testExistingDomainPath() { } @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testDomainPathPermission() { // write from everyone is not allowed (permission too open) assertTrue(dir.setWritable(true, false)); @@ -287,7 +290,6 @@ public void testDomainPathPermission() { * On Linux, when there is still open file handle of a deleted file, the file handle remains open and can still * be used to read and write the file. */ - @Unhealthy("Run it locally since it requires libhadoop.so.") @ParameterizedTest @CsvSource({ "true, true", @@ -371,7 +373,6 @@ public void testReadWrite(boolean deleteFileBeforeRead, boolean deleteFileDuring * Test concurrent read/write. */ @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testConcurrentReadWrite() throws IOException { conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath()); ContainerMetrics containerMetrics = ContainerMetrics.create(conf); @@ -467,7 +468,6 @@ public void testConcurrentReadWrite() throws IOException { * Test server is not listening. */ @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testServerNotListening() { conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath()); DomainSocketFactory factory = DomainSocketFactory.getInstance(conf); @@ -488,7 +488,6 @@ public void testServerNotListening() { * Although socket can be created, read will fail, write can succeed. */ @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testServerNotStart() { conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath()); DomainSocketFactory factory = DomainSocketFactory.getInstance(conf); @@ -522,7 +521,6 @@ public void testServerNotStart() { } @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testReadTimeout() throws InterruptedException { conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath()); conf.set(OzoneConfigKeys.OZONE_CLIENT_READ_TIMEOUT, "2s"); @@ -558,7 +556,6 @@ public void testReadTimeout() throws InterruptedException { * read timeout happens. */ @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testReceiverDaemonStartSlow() { conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath()); DomainSocketFactory factory = DomainSocketFactory.getInstance(conf); @@ -586,7 +583,6 @@ public void testReceiverDaemonStartSlow() { } @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testMaxXceiverCount() throws IOException, InterruptedException { conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath()); DatanodeConfiguration datanodeConfiguration = conf.getObject(DatanodeConfiguration.class); @@ -650,7 +646,6 @@ public void testMaxXceiverCount() throws IOException, InterruptedException { * will treat it as a critical error, close the connection. */ @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testSendIrrelevantMessage() { conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath()); DomainSocketFactory factory = DomainSocketFactory.getInstance(conf); @@ -677,7 +672,6 @@ public void testSendIrrelevantMessage() { } @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testSendUnsupportedRequest() throws IOException { conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath()); DomainSocketFactory factory = DomainSocketFactory.getInstance(conf); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerSC.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerSC.java index 6e60089b0061..7fb1c612bcd6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerSC.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerSC.java @@ -31,7 +31,6 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; -import org.apache.ozone.test.tag.Unhealthy; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -40,6 +39,11 @@ /** * Test for short-circuit enabled XceiverClientManager. + * Add Environment variables + * LD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib + * DYLD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib + * to intellij run configuration to run it locally. + * Dynamically set the java.library.path in java code doesn't affect the library loading */ @Timeout(300) public class TestXceiverClientManagerSC { @@ -75,7 +79,6 @@ public static void shutdown() { } @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testAllocateShortCircuitClient() throws IOException { try (XceiverClientManager clientManager = new XceiverClientManager(config, config.getObject(ScmClientConfig.class), null)) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestDomainSocketFactory.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestDomainSocketFactory.java index bd10bc617668..7fc179b2bcce 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestDomainSocketFactory.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestDomainSocketFactory.java @@ -25,12 +25,16 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.storage.DomainSocketFactory; -import org.apache.ozone.test.tag.Unhealthy; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; /** * Tests for {@link DomainSocketFactory}'s functionality. + * For local intellij run, please follow the steps below: + * Add Environment variables + * LD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib + * DYLD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib + * to intellij run configuration. */ public class TestDomainSocketFactory { @@ -56,7 +60,6 @@ private DomainSocketFactory getDomainSocketFactory() { } @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") public void testShortCircuitDisableTemporary() { DomainSocketFactory factory = getDomainSocketFactory(); try { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestShortCircuitChunkInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestLocalChunkInputStream.java similarity index 95% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestShortCircuitChunkInputStream.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestLocalChunkInputStream.java index 5d0fd16cd6a2..f72ddfcae2f5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestShortCircuitChunkInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestLocalChunkInputStream.java @@ -44,7 +44,6 @@ import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo; import org.apache.hadoop.ozone.om.TestBucket; import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.tag.Unhealthy; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.io.TempDir; @@ -52,9 +51,14 @@ /** * Tests {@link LocalChunkInputStream}. + * For local intellij run, please follow the steps below: + * Add Environment variables + * LD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib + * DYLD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib + * to intellij run configuration. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class TestShortCircuitChunkInputStream extends TestChunkInputStream { +public class TestLocalChunkInputStream extends TestChunkInputStream { @TempDir private File dir; @@ -89,7 +93,6 @@ ReplicationConfig getRepConfig() { */ @ContainerLayoutTestInfo.ContainerTest @Override - @Unhealthy("Run it locally since it requires libhadoop.so.") void testAll(ContainerLayoutVersion layout) throws Exception { try (OzoneClient client = getCluster().newClient()) { updateConfig(layout); @@ -115,7 +118,6 @@ void testAll(ContainerLayoutVersion layout) throws Exception { } @Test - @Unhealthy("Run it locally since it requires libhadoop.so.") void testFallbackToGrpc() throws Exception { try (OzoneClient client = getCluster().newClient()) { assumeTrue(DomainSocketFactory.getInstance(getCluster().getConf()).isServiceReady()); diff --git a/pom.xml b/pom.xml index 22727c7b22d1..f3821ea2aaa8 100644 --- a/pom.xml +++ b/pom.xml @@ -2607,6 +2607,78 @@ + + com.googlecode.maven-download-plugin + download-maven-plugin + ${download-maven-plugin.version} + false + + + fetch-linux-library + + wget + + generate-resources + + https://raw.githubusercontent.com/apache/ozone-thirdparty/master/hadoop-native-lib/${hadoop.version}/libhadoop_linux_x86_64.so + false + ${project.build.directory}/native-lib + false + + + + fetch-mac-library + + wget + + generate-resources + + https://raw.githubusercontent.com/apache/ozone-thirdparty/master/hadoop-native-lib/${hadoop.version}/libhadoop_osx_aarch_64.dylib + false + ${project.build.directory}/native-lib + false + + + + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven-antrun-plugin.version} + false + + + extract-linux-library + + run + + prepare-package + + + + + + + + + + + extract-mac-library + + run + + prepare-package + + + + + + + + + + + @@ -2704,6 +2776,8 @@ org.apache.hadoop.ozone.client.** + org.apache.hadoop.hdds.TestXceiverServerDomainSocket + org.apache.hadoop.hdds.scm.TestXceiverClientManagerSC ${unstable-test-groups} @@ -2797,6 +2871,8 @@ org.apache.hadoop.hdds.** + org.apache.hadoop.hdds.TestXceiverServerDomainSocket + org.apache.hadoop.hdds.scm.TestXceiverClientManagerSC org.apache.hadoop.hdds.scm.container.** **/*$* @@ -2881,16 +2957,7 @@ - native - - - linux - x86_64 - - - fetch-native-hadoop - - + hadoop-native-lib @@ -2912,6 +2979,19 @@ false + + fetch-mac-library + + wget + + generate-resources + + https://raw.githubusercontent.com/apache/ozone-thirdparty/master/hadoop-native-lib/${hadoop.version}/libhadoop_osx_aarch_64.dylib + false + ${project.build.directory}/native-lib + false + + @@ -2935,51 +3015,6 @@ - - - - - - - native-mac - - - mac - aarch64 - - - fetch-native-hadoop - - - - - - com.googlecode.maven-download-plugin - download-maven-plugin - ${download-maven-plugin.version} - false - - - fetch-mac-library - - wget - - generate-resources - - https://raw.githubusercontent.com/apache/ozone-thirdparty/master/hadoop-native-lib/${hadoop.version}/libhadoop_osx_aarch_64.dylib - false - ${project.build.directory}/native-lib - false - - - - - - org.apache.maven.plugins - maven-antrun-plugin - ${maven-antrun-plugin.version} - false - extract-mac-library