From 350cae258800cf825bcf71636749fc6df338fe69 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Wed, 27 May 2026 14:31:07 +0800 Subject: [PATCH 1/2] HDDS-15382. Close idle connection for Datanode GRPC server --- .../common/statemachine/DatanodeConfiguration.java | 4 ++-- .../common/transport/server/XceiverServerGrpc.java | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java index 1fd094b55f1c..506dd79c37dc 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java @@ -145,7 +145,7 @@ public class DatanodeConfiguration extends ReconfigurableConfig { static final int BLOCK_DELETE_THREADS_DEFAULT = 5; public static final String GRPC_SO_BACKLOG_KEY = "hdds.datanode.grpc.so.backlog"; - public static final int GRPC_SO_BACKLOG_DEFAULT = 4096; + public static final int GRPC_SO_BACKLOG_DEFAULT = 256; public static final String BLOCK_DELETE_COMMAND_WORKER_INTERVAL = "hdds.datanode.block.delete.command.worker.interval"; @@ -167,7 +167,7 @@ public class DatanodeConfiguration extends ReconfigurableConfig { */ @Config(key = "hdds.datanode.grpc.so.backlog", type = ConfigType.INT, - defaultValue = "4096", + defaultValue = "256", tags = {DATANODE}, description = "The SO_BACKLOG value for the Datanode gRPC server socket. " + "This limits the number of pending connections in the kernel's " + diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java index 58be7965a74c..cf3ceb173ec7 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.container.common.transport.server; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Scope; @@ -139,6 +140,17 @@ public XceiverServerGrpc(DatanodeDetails datanodeDetails, .channelType(channelType) .withOption(ChannelOption.SO_BACKLOG, soBacklog) .executor(readExecutors) + // If a client does not send an actual functional business RPC for 15 minutes, + // the server kicks them off with a GOAWAY frame. + .maxConnectionIdle(15, TimeUnit.MINUTES) + // If the server receives absolutely zero network traffic from a client for + // 5 minutes, the server proactively sends an HTTP/2 PING frame to verify + // if the network wire or client machine is still alive. + .keepAliveTime(5, TimeUnit.MINUTES) + // If the server fires a ping and the client fails to respond with a + // PING ACK within 30 seconds, the server assumes the socket is a dead + // "zombie connection" and immediately destroys the TCP socket. + .keepAliveTimeout(30, TimeUnit.SECONDS) .addService(ServerInterceptors.intercept( xceiverService.bindServiceWithZeroCopy(), new GrpcServerInterceptor())); From c4fd96e1f061c2d09302d6278f62ef48a5fbc219 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Wed, 27 May 2026 14:36:10 +0800 Subject: [PATCH 2/2] remove unused import --- .../container/common/transport/server/XceiverServerGrpc.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java index cf3ceb173ec7..0afbabb96a23 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java @@ -17,7 +17,6 @@ package org.apache.hadoop.ozone.container.common.transport.server; -import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Scope;