| Metrics Name | +Type | +Description | +
|---|---|---|
| blobFetchTotal | +Counter | +Total number of blob fetch outcomes handled during write, including successful fetches, failures, and missing resources written as NULL. | +
| blobFetchSuccess | +Counter | +Total number of blob fetches that completed successfully. | +
| blobFetchSuccessBytes | +Counter | +Total bytes written by successful blob fetches. | +
| blobFetchNullWritten | +Counter | +Total number of NULL values written because a blob resource was missing or failed to fetch. | +
| blobFetchMissingFileNullWritten | +Counter | +Total number of NULL values written because blob-write-null-on-missing-file handled a missing blob resource. |
+
| blobFetchFailureNullWritten | +Counter | +Total number of NULL values written because blob-write-null-on-fetch-failure handled a non-404 blob fetch failure. |
+
| blobFetchFailure | +Counter | +Total number of blob fetch failures that were not written as NULL. | +
| blobFetchHttpNotFound | +Counter | +Total number of HTTP 404 blob fetch failures. | +
| blobFetchHttpClientError | +Counter | +Total number of non-404 HTTP 4xx blob fetch failures. | +
| blobFetchHttpServerError | +Counter | +Total number of HTTP 5xx blob fetch failures. | +
| blobFetchHttpOtherError | +Counter | +Total number of blob fetch failures with HTTP status codes outside 4xx and 5xx. | +
| blobFetchInvalidUri | +Counter | +Total number of blob fetch failures caused by invalid URIs. | +
| blobFetchOtherError | +Counter | +Total number of blob fetch failures that are not classified as HTTP status or invalid URI failures. | +
| <host>.taskmanager.<tm_id>.<job_name>.<writer_operator_name>.<subtask_index> | paimon.table.<table_name>.writeBuffer | +|
| Blob Fetch Metrics | +<host>.taskmanager.<tm_id>.<job_name>.<writer_operator_name>.<subtask_index> | +paimon.table.<table_name>.blobFetch | +
| Compaction Metrics | <host>.taskmanager.<tm_id>.<job_name>.<writer_operator_name>.<subtask_index> | diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BlobFetchMetricReporter.java b/paimon-common/src/main/java/org/apache/paimon/data/BlobFetchMetricReporter.java new file mode 100644 index 000000000000..0670c7098c71 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/data/BlobFetchMetricReporter.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data; + +/** Reporter for blob fetch metrics. */ +public interface BlobFetchMetricReporter { + + BlobFetchMetricReporter NOOP = + new BlobFetchMetricReporter() { + @Override + public void recordSuccess(long bytes) {} + + @Override + public void recordMissingFileNullWritten(boolean httpNotFound) {} + + @Override + public void recordFetchFailureNullWritten(Throwable throwable) {} + + @Override + public void recordFetchFailure(Throwable throwable) {} + }; + + void recordSuccess(long bytes); + + void recordMissingFileNullWritten(boolean httpNotFound); + + void recordFetchFailureNullWritten(Throwable throwable); + + void recordFetchFailure(Throwable throwable); +} diff --git a/paimon-core/src/main/java/org/apache/paimon/append/DedicatedFormatRollingFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/append/DedicatedFormatRollingFileWriter.java index e9ef1e15eedb..86df6b10b265 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/DedicatedFormatRollingFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/DedicatedFormatRollingFileWriter.java @@ -200,7 +200,8 @@ public DedicatedFormatRollingFileWriter( context.blobConsumer(), context.blobInlineFields(), context.writeNullOnMissingFile(), - context.writeNullOnFetchFailure()); + context.writeNullOnFetchFailure(), + context.blobFetchMetricReporter()); } else { this.blobWriterFactory = null; } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/MultipleBlobFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/append/MultipleBlobFileWriter.java index b8d2d1e6c549..d952018ee1f0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/MultipleBlobFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/MultipleBlobFileWriter.java @@ -19,6 +19,7 @@ package org.apache.paimon.append; import org.apache.paimon.data.BlobConsumer; +import org.apache.paimon.data.BlobFetchMetricReporter; import org.apache.paimon.data.InternalRow; import org.apache.paimon.fileindex.FileIndexOptions; import org.apache.paimon.format.blob.BlobFileFormat; @@ -65,7 +66,8 @@ public MultipleBlobFileWriter( @Nullable BlobConsumer blobConsumer, Set