diff --git a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java index a4fdbdfb5793..4e0943e5dd1a 100644 --- a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java +++ b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java @@ -181,6 +181,13 @@ private ObjectNode getJsonObject(SnapshotDiffResponse diffResponse) { if (StringUtils.isNotEmpty(diffResponse.getReason())) { diffResponseNode.put("reason", diffResponse.getReason()); } + if (diffResponse.getSubStatus() != null) { + SnapshotDiffResponse.SubStatus sub = diffResponse.getSubStatus(); + diffResponseNode.put("subStatus", sub.name()); + if (sub.hasProgress()) { + diffResponseNode.put("progressPercent", diffResponse.getProgressPercent()); + } + } return diffResponseNode; } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java index baf304e898c0..387bf8c6b21a 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java @@ -229,8 +229,7 @@ public String toString() { } if (status.equals(JobStatus.IN_PROGRESS) && subStatus != null) { sb.append(", subStatus: ").append(subStatus); - if (subStatus.equals(SubStatus.OBJECT_ID_MAP_GEN_FSO) || - subStatus.equals(SubStatus.OBJECT_ID_MAP_GEN_OBS)) { + if (subStatus.hasProgress()) { sb.append(String.format(", keysProcessedPct: %.2f", keysProcessedPct)); } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 2826239f02ea..4a7954f65310 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1468,12 +1468,19 @@ private SnapshotDiffResponse snapshotDiffInternal(String volumeName, OzoneManagerProtocolProtos.SnapshotDiffResponse diffResponse = omResponse.getSnapshotDiffResponse(); - return new SnapshotDiffResponse(SnapshotDiffReportOzone.fromProtobuf( - diffResponse.getSnapshotDiffReport()), + SnapshotDiffResponse result = new SnapshotDiffResponse( + SnapshotDiffReportOzone.fromProtobuf(diffResponse.getSnapshotDiffReport()), JobStatus.fromProtobuf(diffResponse.getJobStatus()), diffResponse.getWaitTimeInMs(), diffResponse.getReason(), reportOnly); + if (diffResponse.hasSubStatus()) { + result.setSubStatus(SnapshotDiffResponse.SubStatus.fromProtoBuf(diffResponse.getSubStatus())); + if (diffResponse.hasProgressPercent()) { + result.setProgressPercent(diffResponse.getProgressPercent()); + } + } + return result; } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java index de16a3aec409..816bfb0ce9fd 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java @@ -86,10 +86,18 @@ public String getReason() { return reason; } + public SubStatus getSubStatus() { + return subStatus; + } + public void setSubStatus(SubStatus subStatus) { this.subStatus = subStatus; } + public double getProgressPercent() { + return progressPercent; + } + public void setProgressPercent(double progressPercent) { this.progressPercent = progressPercent; } @@ -143,11 +151,10 @@ public String toString() { str.append(".\n"); if (subStatus != null) { str.append("SubStatus : ") - .append(subStatus); - if (subStatus.equals(SubStatus.OBJECT_ID_MAP_GEN_OBS) || - subStatus.equals(SubStatus.OBJECT_ID_MAP_GEN_FSO)) { - str.append("Keys Processed Estimated Percentage : ") - .append(progressPercent); + .append(subStatus) + .append('\n'); + if (subStatus.hasProgress()) { + str.append(String.format("Keys Processed Estimated Percentage : %.1f%n", progressPercent)); } } } @@ -182,8 +189,17 @@ public enum SubStatus { SST_FILE_DELTA_DAG_WALK, SST_FILE_DELTA_FULL_DIFF, OBJECT_ID_MAP_GEN_OBS, + @Deprecated OBJECT_ID_MAP_GEN_FSO, - DIFF_REPORT_GEN; + DIFF_REPORT_GEN, + PATH_RESOLUTION_FSO, + OBJECT_ID_MAP_GEN_FSO_FILE, + OBJECT_ID_MAP_GEN_FSO_DIR; + + public boolean hasProgress() { + return this == OBJECT_ID_MAP_GEN_OBS || this == OBJECT_ID_MAP_GEN_FSO || this == OBJECT_ID_MAP_GEN_FSO_FILE + || this == OBJECT_ID_MAP_GEN_FSO_DIR; + } public static SubStatus fromProtoBuf(OzoneManagerProtocolProtos.SnapshotDiffResponse.SubStatus subStatusProto) { return SubStatus.valueOf(subStatusProto.name()); diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/snapshot/TestSnapshotDiffResponse.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/snapshot/TestSnapshotDiffResponse.java index 74b7d73d3e25..14125911c15f 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/snapshot/TestSnapshotDiffResponse.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/snapshot/TestSnapshotDiffResponse.java @@ -17,12 +17,16 @@ package org.apache.hadoop.ozone.snapshot; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Collections; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.EnumSource.Mode; class TestSnapshotDiffResponse { @@ -57,20 +61,48 @@ void testReportOnlyFailedMessageIncludesReason() { assertTrue(message.contains("resubmit the job without using the --get-report option")); } - @Test - void testReportOnlyInProgressIncludesSubStatusAndProgress() { + @ParameterizedTest + @EnumSource(value = SubStatus.class, names = {"OBJECT_ID_MAP_GEN_OBS", "OBJECT_ID_MAP_GEN_FSO", + "OBJECT_ID_MAP_GEN_FSO_FILE", "OBJECT_ID_MAP_GEN_FSO_DIR"}) + void testInProgressWithMapGenSubStatusIncludesProgress(SubStatus subStatus) { SnapshotDiffResponse response = new SnapshotDiffResponse(createReport(), JobStatus.IN_PROGRESS, 1000L, true); - response.setSubStatus(SubStatus.OBJECT_ID_MAP_GEN_OBS); + response.setSubStatus(subStatus); response.setProgressPercent(55.5); String message = response.toString(); assertTrue(message.contains("IN_PROGRESS")); - assertTrue(message.contains("OBJECT_ID_MAP_GEN_OBS")); + assertTrue(message.contains(subStatus.name())); assertTrue(message.contains("Keys Processed Estimated Percentage")); assertTrue(message.contains("55.5")); } + @ParameterizedTest + @EnumSource(value = SubStatus.class, + names = {"OBJECT_ID_MAP_GEN_OBS", "OBJECT_ID_MAP_GEN_FSO", "OBJECT_ID_MAP_GEN_FSO_FILE", + "OBJECT_ID_MAP_GEN_FSO_DIR"}, + mode = Mode.EXCLUDE) + void testInProgressWithNonMapGenSubStatusRendersSubStatusButNotProgress(SubStatus subStatus) { + SnapshotDiffResponse response = new SnapshotDiffResponse(createReport(), JobStatus.IN_PROGRESS, 1000L, true); + response.setSubStatus(subStatus); + response.setProgressPercent(55.5); + + String message = response.toString(); + assertTrue(message.contains("IN_PROGRESS")); + assertTrue(message.contains(subStatus.name())); + assertFalse(message.contains("Keys Processed Estimated Percentage")); + assertFalse(message.contains("55.5")); + } + + @Test + void testInProgressWithNullSubStatusOmitsSubStatusAndProgressLines() { + SnapshotDiffResponse response = new SnapshotDiffResponse(createReport(), JobStatus.IN_PROGRESS, 1000L, true); + String message = response.toString(); + assertTrue(message.contains("IN_PROGRESS")); + assertFalse(message.contains("SubStatus")); + assertFalse(message.contains("Keys Processed Estimated Percentage")); + } + private SnapshotDiffReportOzone createReport() { return new SnapshotDiffReportOzone("snapshotRoot", "vol", "bucket", "fromSnap", "toSnap", Collections.emptyList(), null); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index bccc6d11f11d..2a27af60754c 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -2254,8 +2254,11 @@ message SnapshotDiffResponse { SST_FILE_DELTA_DAG_WALK = 1; SST_FILE_DELTA_FULL_DIFF = 2; OBJECT_ID_MAP_GEN_OBS = 3; - OBJECT_ID_MAP_GEN_FSO = 4; + OBJECT_ID_MAP_GEN_FSO = 4 [deprecated = true]; DIFF_REPORT_GEN = 5; + PATH_RESOLUTION_FSO = 6; + OBJECT_ID_MAP_GEN_FSO_FILE = 7; + OBJECT_ID_MAP_GEN_FSO_DIR = 8; } optional SnapshotDiffReportProto snapshotDiffReport = 1; @@ -2263,6 +2266,7 @@ message SnapshotDiffResponse { optional int64 waitTimeInMs = 3; optional string reason = 4; optional SubStatus subStatus = 5; + optional double progressPercent = 6; } message SubmitSnapshotDiffResponse { diff --git a/hadoop-ozone/interface-client/src/main/resources/proto.lock b/hadoop-ozone/interface-client/src/main/resources/proto.lock index 5f6b5806361b..88107a505578 100644 --- a/hadoop-ozone/interface-client/src/main/resources/proto.lock +++ b/hadoop-ozone/interface-client/src/main/resources/proto.lock @@ -1342,6 +1342,18 @@ { "name": "DIFF_REPORT_GEN", "integer": 5 + }, + { + "name": "PATH_RESOLUTION_FSO", + "integer": 6 + }, + { + "name": "OBJECT_ID_MAP_GEN_FSO_FILE", + "integer": 7 + }, + { + "name": "OBJECT_ID_MAP_GEN_FSO_DIR", + "integer": 8 } ] }, @@ -8103,6 +8115,12 @@ "name": "subStatus", "type": "SubStatus", "optional": true + }, + { + "id": 6, + "name": "progressPercent", + "type": "double", + "optional": true } ] }, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index e885e2b4f689..5f2adc2976cd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -41,6 +41,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT; import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE; +import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotActive; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.dropColumnFamilyHandle; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.getSnapshotInfo; @@ -60,8 +61,10 @@ import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.QUEUED; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.REJECTED; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus.DIFF_REPORT_GEN; -import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus.OBJECT_ID_MAP_GEN_FSO; +import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus.OBJECT_ID_MAP_GEN_FSO_DIR; +import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus.OBJECT_ID_MAP_GEN_FSO_FILE; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus.OBJECT_ID_MAP_GEN_OBS; +import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus.PATH_RESOLUTION_FSO; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; @@ -141,6 +144,7 @@ import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus; import org.apache.hadoop.ozone.snapshot.SubmitSnapshotDiffResponse; import org.apache.hadoop.ozone.util.ClosableIterator; +import org.apache.hadoop.util.Time; import org.apache.logging.log4j.util.Strings; import org.apache.ozone.rocksdb.util.SstFileInfo; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; @@ -1112,27 +1116,27 @@ void generateSnapshotDiffReport(final String jobKey, // repetition while constantly checking if the job is cancelled. Callable[] methodCalls = new Callable[]{ () -> { - recordActivity(jobKey, OBJECT_ID_MAP_GEN_OBS); getDeltaFilesAndDiffKeysToObjectIdToKeyMap(fsKeyTable, tsKeyTable, fsInfo, tsInfo, performNonNativeDiff, tablePrefixes, objectIdToKeyNameMapForFromSnapshot, objectIdToKeyNameMapForToSnapshot, objectIdToIsDirMap, - oldParentIds, newParentIds, deltaFileComputer, jobKey); + oldParentIds, newParentIds, deltaFileComputer, jobKey, jobId); return null; }, () -> { if (bucketLayout.isFileSystemOptimized()) { - recordActivity(jobKey, OBJECT_ID_MAP_GEN_FSO); getDeltaFilesAndDiffKeysToObjectIdToKeyMap(fsDirTable, tsDirTable, fsInfo, tsInfo, performNonNativeDiff, tablePrefixes, objectIdToKeyNameMapForFromSnapshot, objectIdToKeyNameMapForToSnapshot, objectIdToIsDirMap, - oldParentIds, newParentIds, deltaFileComputer, jobKey); + oldParentIds, newParentIds, deltaFileComputer, jobKey, jobId); } return null; }, () -> { if (bucketLayout.isFileSystemOptimized()) { + recordActivity(jobKey, PATH_RESOLUTION_FSO); + long pathResolutionStart = Time.monotonicNow(); long bucketId = toSnapshot.getMetadataManager() .getBucketId(volumeName, bucketName); String tablePrefix = tablePrefixes.getTablePrefix(fromSnapshot.getMetadataManager() @@ -1145,11 +1149,19 @@ void generateSnapshotDiffReport(final String jobKey, tablePrefix, bucketId, toSnapshot.getMetadataManager().getDirectoryTable()) .getAbsolutePathForObjectIDs(newParentIds, true)); + if (LOG.isDebugEnabled()) { + LOG.debug("Completed FSO path resolution for snapshot diff, resolved {} out of {} parent IDs, " + + "elapsed: {}ms, jobId: {}", + oldParentIdPathMap.get().size() + newParentIdPathMap.get().size(), + oldParentIds.get().size() + newParentIds.get().size(), + Time.monotonicNow() - pathResolutionStart, jobId); + } } return null; }, () -> { recordActivity(jobKey, DIFF_REPORT_GEN); + long reportGenStart = Time.monotonicNow(); Pair reportEntries = generateDiffReport(jobId, fsKeyTable, tsKeyTable, @@ -1166,6 +1178,10 @@ void generateSnapshotDiffReport(final String jobKey, if (reportEntries.getKey() >= 0 && areDiffJobAndSnapshotsActive(volumeName, bucketName, fromSnapshotName, toSnapshotName)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Generated snapshot diff report, entry count: {}, elapsed: {}ms, jobId: {}", + reportEntries.getKey(), Time.monotonicNow() - reportGenStart, jobId); + } updateJobStatusToDone(jobKey, reportEntries.getKey(), reportEntries.getValue()); } return null; @@ -1221,17 +1237,24 @@ private void getDeltaFilesAndDiffKeysToObjectIdToKeyMap( final PersistentMap newObjIdToKeyMap, final PersistentMap objectIdToIsDirMap, final Optional> oldParentIds, final Optional> newParentIds, - final DeltaFileComputer deltaFileComputer, final String jobKey) throws IOException, RocksDBException { + final DeltaFileComputer deltaFileComputer, final String jobKey, + final String jobId) throws IOException, RocksDBException { + long deltaFilesStart = Time.monotonicNow(); Set tablesToLookUp = Collections.singleton(fsTable.getName()); Collection> deltaFiles = deltaFileComputer.getDeltaFiles(fsInfo, tsInfo, tablesToLookUp); if (LOG.isDebugEnabled()) { - LOG.debug("Computed Delta SST File Set, Total count = {} ", deltaFiles.size()); + LOG.debug("Computed Delta SST File Set for table '{}', file count: {}, elapsed: {}ms, jobId: {}", + fsTable.getName(), deltaFiles.size(), Time.monotonicNow() - deltaFilesStart, jobId); } + recordActivity(jobKey, + fsTable.getName().equals(FILE_TABLE) ? OBJECT_ID_MAP_GEN_FSO_FILE + : fsTable.getName().equals(DIRECTORY_TABLE) ? OBJECT_ID_MAP_GEN_FSO_DIR + : OBJECT_ID_MAP_GEN_OBS); addToObjectIdMap(fsTable, tsTable, deltaFiles.stream().map(Pair::getLeft).collect(Collectors.toList()), !skipNativeDiff, oldObjIdToKeyMap, newObjIdToKeyMap, objectIdToIsDirMap, oldParentIds, - newParentIds, tablePrefixes, jobKey); + newParentIds, tablePrefixes, jobKey, jobId); } @VisibleForTesting @@ -1244,10 +1267,19 @@ void addToObjectIdMap(Table fsTable, PersistentMap objectIdToIsDirMap, Optional> oldParentIds, Optional> newParentIds, - TablePrefixInfo tablePrefixes, String jobKey) throws IOException, RocksDBException { + TablePrefixInfo tablePrefixes, String jobKey, + String jobId) throws IOException, RocksDBException { if (deltaFiles.isEmpty()) { + updateProgress(jobKey, 1.0); + if (LOG.isDebugEnabled()) { + LOG.debug("Skipped object ID map generation for table '{}' because there are no delta files, jobId: {}", + fsTable.getName(), jobId); + } return; } + updateProgress(jobKey, 0.0); + long objectIdMapStart = Time.monotonicNow(); + AtomicLong keysProcessed = new AtomicLong(0); String tablePrefix = tablePrefixes.getTablePrefix(fsTable.getName()); boolean isDirectoryTable = fsTable.getName().equals(DIRECTORY_TABLE); SstFileSetReader sstFileReader = new SstFileSetReader(deltaFiles); @@ -1256,8 +1288,7 @@ void addToObjectIdMap(Table fsTable, String sstFileReaderLowerBound = tablePrefix; String sstFileReaderUpperBound = null; double stepIncreasePct = 0.1; - double[] checkpoint = new double[1]; - checkpoint[0] = stepIncreasePct; + double checkpoint = stepIncreasePct; if (Strings.isNotEmpty(tablePrefix)) { sstFileReaderUpperBound = getLexicographicallyHigherString(tablePrefix); } @@ -1266,15 +1297,14 @@ void addToObjectIdMap(Table fsTable, : sstFileReader.getKeyStream(sstFileReaderLowerBound, sstFileReaderUpperBound); TableMergeIterator tableMergeIterator = new TableMergeIterator<>(keysToCheck, tablePrefix, (Table) fsTable, (Table) tsTable)) { - AtomicLong keysProcessed = new AtomicLong(0); while (tableMergeIterator.hasNext()) { Table.KeyValue> kvs = tableMergeIterator.next(); String key = kvs.getKey(); if (totalEstimatedKeysToProcess > 0) { double progressPct = (double) keysProcessed.get() / totalEstimatedKeysToProcess; - if (progressPct >= checkpoint[0]) { + if (progressPct >= checkpoint) { updateProgress(jobKey, progressPct); - checkpoint[0] += stepIncreasePct; + checkpoint += stepIncreasePct; } } @@ -1312,6 +1342,11 @@ void addToObjectIdMap(Table fsTable, } } } + updateProgress(jobKey, 1.0); + if (LOG.isDebugEnabled()) { + LOG.debug("Generated object ID map for table '{}', keys scanned: {}, elapsed: {}ms, jobId: {}", + fsTable.getName(), keysProcessed.get(), Time.monotonicNow() - objectIdMapStart, jobId); + } } private void validateEstimatedKeyChangesAreInLimits( @@ -1581,7 +1616,8 @@ synchronized void updateProgress(String jobKey, snapshotDiffJob.setKeysProcessedPct(pct * 100); snapDiffJobTable.put(jobKey, snapshotDiffJob); if (LOG.isDebugEnabled()) { - LOG.debug("Completed processing {}% of keys for snapshot diff job {}", pct, jobKey); + LOG.debug("Completed processing {}% of keys for snapshot diff job {}", + snapshotDiffJob.getKeysProcessedPct(), jobKey); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/CompositeDeltaDiffComputer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/CompositeDeltaDiffComputer.java index 4ef17d841141..6c08a88db325 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/CompositeDeltaDiffComputer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/CompositeDeltaDiffComputer.java @@ -86,6 +86,9 @@ Optional>> computeDeltaFiles(SnapshotInfo from updateActivity(SnapshotDiffResponse.SubStatus.SST_FILE_DELTA_DAG_WALK); deltaFiles = differComputer.computeDeltaFiles(fromSnapshotInfo, toSnapshotInfo, tablesToLookup, tablePrefixInfo).orElse(null); + if (deltaFiles == null) { + LOG.warn("DAG diff returned no result for tables {}, falling back to full diff.", tablesToLookup); + } } } catch (Exception e) { LOG.warn("Falling back to full diff.", e); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 701ecb6b38e0..b585fa234c8b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -1482,6 +1482,12 @@ private SnapshotDiffResponse snapshotDiff( builder.setSnapshotDiffReport( response.getSnapshotDiffReport().toProtobuf()); } + if (response.getSubStatus() != null) { + builder.setSubStatus(response.getSubStatus().toProtoBuf()); + if (response.getSubStatus().hasProgress()) { + builder.setProgressPercent(response.getProgressPercent()); + } + } return builder.build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 3dc468d409eb..c34e13a1054b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -158,6 +158,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.MockedConstruction; @@ -494,12 +495,16 @@ public void testObjectIdMapWithTombstoneEntries(boolean nativeLibraryLoaded, Set oldParentIds = Sets.newHashSet(); Set newParentIds = Sets.newHashSet(); + SnapshotDiffJob dummyJob = new SnapshotDiffJob(System.currentTimeMillis(), + "", IN_PROGRESS, VOLUME_NAME, BUCKET_NAME, "from", "to", false, false, 0, null, 0.0, ""); + db.get().put(snapDiffJobTable, codecRegistry.asRawData(""), codecRegistry.asRawData(dummyJob)); + snapshotDiffManager.addToObjectIdMap(toSnapshotTable, fromSnapshotTable, Sets.newHashSet(Paths.get("dummy.sst")), nativeLibraryLoaded, oldObjectIdKeyMap, newObjectIdKeyMap, objectIdsToCheck, Optional.of(oldParentIds), Optional.of(newParentIds), - new TablePrefixInfo(ImmutableMap.of(DIRECTORY_TABLE, "0", KEY_TABLE, "0", FILE_TABLE, "0")), ""); + new TablePrefixInfo(ImmutableMap.of(DIRECTORY_TABLE, "0", KEY_TABLE, "0", FILE_TABLE, "0")), "", ""); try (ClosableIterator> oldObjectIdIter = oldObjectIdKeyMap.iterator()) { @@ -1674,17 +1679,38 @@ public void testGetSnapshotDiffReportWhenDone() throws Exception { .containsExactlyElementsOf(expectedEntries); } + @ParameterizedTest + @EnumSource(value = SnapshotDiffResponse.SubStatus.class, + names = {"OBJECT_ID_MAP_GEN_OBS", "OBJECT_ID_MAP_GEN_FSO", "OBJECT_ID_MAP_GEN_FSO_DIR", + "OBJECT_ID_MAP_GEN_FSO_FILE"}) + public void testGetSnapshotDiffReportReportOnlyInProgressIncludesProgressDetails( + SnapshotDiffResponse.SubStatus subStatus) throws IOException { + SnapDiffTestContext ctx = setupRandomSnapDiffTestContext(); + SnapshotDiffJob existing = new SnapshotDiffJob(0L, UUID.randomUUID().toString(), + IN_PROGRESS, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName, ctx.toSnapshotName, + false, false, 0L, subStatus, 55.5, null); + snapshotDiffManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing); + + SnapshotDiffResponse response = snapshotDiffManager.getSnapshotDiffReport( + ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName, ctx.toSnapshotName, "", 1000); + assertEquals(IN_PROGRESS, response.getJobStatus()); + assertEquals(subStatus, response.getSubStatus()); + assertThat(response.getProgressPercent()).isEqualTo(55.5); + } + @Test - public void testGetSnapshotDiffReportReportOnlyInProgressIncludesProgressDetails() + public void testGetSnapshotDiffReportInProgressWithPathResolutionSubStatus() throws IOException { SnapDiffTestContext ctx = setupRandomSnapDiffTestContext(); SnapshotDiffJob existing = new SnapshotDiffJob(0L, UUID.randomUUID().toString(), IN_PROGRESS, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName, ctx.toSnapshotName, - false, false, 0L, SnapshotDiffResponse.SubStatus.OBJECT_ID_MAP_GEN_OBS, 55.5, null); + false, false, 0L, SnapshotDiffResponse.SubStatus.PATH_RESOLUTION_FSO, 0.0, null); snapshotDiffManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing); SnapshotDiffResponse response = snapshotDiffManager.getSnapshotDiffReport( ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName, ctx.toSnapshotName, "", 1000); assertEquals(IN_PROGRESS, response.getJobStatus()); + assertEquals(SnapshotDiffResponse.SubStatus.PATH_RESOLUTION_FSO, response.getSubStatus()); } + } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManagerMXBean.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManagerMXBean.java index afd755513229..10ab715594da 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManagerMXBean.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManagerMXBean.java @@ -139,7 +139,7 @@ public void testMXBeanRegistration() throws Exception { org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.QUEUED, "vol", "bucket", "snap1", "snap2", false, false, 0, - org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus.OBJECT_ID_MAP_GEN_FSO, + org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus.OBJECT_ID_MAP_GEN_FSO_DIR, 0, null); snapDiffJobTable.put("job-1", job); @@ -151,6 +151,6 @@ public void testMXBeanRegistration() throws Exception { assertEquals("job-1", jobs[0].get("jobId")); assertEquals("snap1", jobs[0].get("fromSnapshot")); assertEquals("snap2", jobs[0].get("toSnapshot")); - assertEquals("OBJECT_ID_MAP_GEN_FSO", jobs[0].get("subStatus")); + assertEquals("OBJECT_ID_MAP_GEN_FSO_DIR", jobs[0].get("subStatus")); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java index 35ee959236dc..40e9e18e9bc7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java @@ -481,4 +481,69 @@ public void testSnapshotDiffRoutingUsesCorrectServerMethodBasedOnOptionalFlags() reportOnlyResponse.getSnapshotDiffResponse().getJobStatus()); Assertions.assertEquals(0L, reportOnlyResponse.getSnapshotDiffResponse().getWaitTimeInMs()); } + + @Test + public void testSnapshotDiffHandlerSerializesSubStatusAndProgressIntoProto() throws IOException { + OzoneManagerRequestHandler handler = getRequestHandler(10); + OzoneManager ozoneManager = handler.getOzoneManager(); + + OMLayoutVersionManager lvm = Mockito.mock(OMLayoutVersionManager.class); + Mockito.when(lvm.isAllowed(Mockito.anyString())).thenReturn(true); + Mockito.when(ozoneManager.getVersionManager()).thenReturn(lvm); + + SnapshotDiffResponse diffResponse = + new SnapshotDiffResponse(null, SnapshotDiffResponse.JobStatus.IN_PROGRESS, 60000L); + diffResponse.setSubStatus(SnapshotDiffResponse.SubStatus.OBJECT_ID_MAP_GEN_FSO_DIR); + diffResponse.setProgressPercent(50.0); + Mockito.when(ozoneManager.snapshotDiff(Mockito.anyString(), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt())) + .thenReturn(diffResponse); + + OzoneManagerProtocolProtos.OMRequest request = + OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.SnapshotDiff) + .setClientId("client") + .setSnapshotDiffRequest(OzoneManagerProtocolProtos.SnapshotDiffRequest.newBuilder() + .setVolumeName("vol").setBucketName("buck") + .setFromSnapshot("s1").setToSnapshot("s2") + .setToken("t").setPageSize(10)) + .build(); + + OzoneManagerProtocolProtos.SnapshotDiffResponse proto = + handler.handleReadRequest(request).getSnapshotDiffResponse(); + Assertions.assertTrue(proto.hasSubStatus()); + Assertions.assertEquals( + OzoneManagerProtocolProtos.SnapshotDiffResponse.SubStatus.OBJECT_ID_MAP_GEN_FSO_DIR, + proto.getSubStatus()); + Assertions.assertEquals(50.0, proto.getProgressPercent(), 1e-9); + } + + @Test + public void testSnapshotDiffHandlerOmitsSubStatusFromProtoWhenNull() throws IOException { + OzoneManagerRequestHandler handler = getRequestHandler(10); + OzoneManager ozoneManager = handler.getOzoneManager(); + + OMLayoutVersionManager lvm = Mockito.mock(OMLayoutVersionManager.class); + Mockito.when(lvm.isAllowed(Mockito.anyString())).thenReturn(true); + Mockito.when(ozoneManager.getVersionManager()).thenReturn(lvm); + + Mockito.when(ozoneManager.snapshotDiff(Mockito.anyString(), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt())) + .thenReturn(new SnapshotDiffResponse(null, SnapshotDiffResponse.JobStatus.IN_PROGRESS, 60000L)); + + OzoneManagerProtocolProtos.OMRequest request = + OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.SnapshotDiff) + .setClientId("client") + .setSnapshotDiffRequest(OzoneManagerProtocolProtos.SnapshotDiffRequest.newBuilder() + .setVolumeName("vol").setBucketName("buck") + .setFromSnapshot("s1").setToSnapshot("s2") + .setToken("t").setPageSize(10)) + .build(); + + OzoneManagerProtocolProtos.SnapshotDiffResponse proto = + handler.handleReadRequest(request).getSnapshotDiffResponse(); + Assertions.assertFalse(proto.hasSubStatus()); + Assertions.assertFalse(proto.hasProgressPercent()); + } }