Skip to content

[improvement](fe) Reuse boxed IDs in cloud tablet indexes #66389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
deardeng merged 2 commits into from
Aug 4, 2026
+371 −27
Merged

[improvement](fe) Reuse boxed IDs in cloud tablet indexes #66389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a326d1c
[improvement](fe) Reuse boxed IDs in cloud tablet indexes
deardeng Aug 3, 2026
dd6909a
[fix](fe) Preserve inflight tablet IDs across route rebuilds
deardeng Aug 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 41 additions & 27 deletions fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ public int hashCode() {
}

private class InfightTask {
public long pickedTabletId;
public Long pickedTabletId;
public long srcBe;
public long destBe;
public long startTimestamp;
Expand Down Expand Up @@ -375,12 +375,12 @@ public int hashCode() {
}

private static class WarmupTabletTask {
private final long pickedTabletId;
private final Long pickedTabletId;
private final long srcBe;
private final long destBe;
private final String clusterId;

WarmupTabletTask(long pickedTabletId, long srcBe, long destBe, String clusterId) {
WarmupTabletTask(Long pickedTabletId, long srcBe, long destBe, String clusterId) {
this.pickedTabletId = pickedTabletId;
this.srcBe = srcBe;
this.destBe = destBe;
Expand Down Expand Up @@ -1058,6 +1058,15 @@ public void fillBeToTablets(long be, long tableId, long partId, long indexId, lo
ConcurrentHashMap<Long, ConcurrentHashMap<Long, Set<Long>>> beToTabletsInTable,
ConcurrentHashMap<Long, ConcurrentHashMap<Long, ConcurrentHashMap<Long, Set<Long>>>>
partToTablets) {
fillBeToTablets(Long.valueOf(be), Long.valueOf(tableId), Long.valueOf(partId), Long.valueOf(indexId),
Long.valueOf(tabletId), globalBeToTablets, beToTabletsInTable, partToTablets);
}

void fillBeToTablets(Long be, Long tableId, Long partId, Long indexId, Long tabletId,
ConcurrentHashMap<Long, Set<Long>> globalBeToTablets,
ConcurrentHashMap<Long, ConcurrentHashMap<Long, Set<Long>>> beToTabletsInTable,
ConcurrentHashMap<Long, ConcurrentHashMap<Long, ConcurrentHashMap<Long, Set<Long>>>>
partToTablets) {
// global
globalBeToTablets.putIfAbsent(be, ConcurrentHashMap.newKeySet());
globalBeToTablets.get(be).add(tabletId);
Expand Down Expand Up @@ -1176,25 +1185,29 @@ public void statRouteInfo() {
Map<Long, Boolean> tmpDbInternal = new HashMap<>();

loopCloudReplica((Database db, Table table, Partition partition, MaterializedIndex index, String cluster) -> {
boolean isColocated = Env.getCurrentColocateIndex().isColocateTable(table.getId());
tmpTableToDb.put(table.getId(), db.getId());
tmpPartitionToDb.put(partition.getId(), db.getId());
tmpDbInternal.computeIfAbsent(db.getId(), k -> {
Long dbId = db.getId();
Long tableId = table.getId();
Long partitionId = partition.getId();
Long indexId = index.getId();
boolean isColocated = Env.getCurrentColocateIndex().isColocateTable(tableId);
tmpTableToDb.put(tableId, dbId);
tmpPartitionToDb.put(partitionId, dbId);
tmpDbInternal.computeIfAbsent(dbId, k -> {
String name = db.getFullName();
return name != null && INTERNAL_DB_NAMES.contains(name);
});
for (Tablet tablet : index.getTablets()) {
long tabletId = tablet.getId();
Long tabletId = tablet.getId();
// active tablet scoring (used for scheduling order)
if (activeTabletIds != null && !activeTabletIds.isEmpty() && activeTabletIds.contains(tabletId)) {
tmpTableActive.merge(table.getId(), 1L, Long::sum);
tmpPartitionActive.merge(partition.getId(), 1L, Long::sum);
tmpDbActive.merge(db.getId(), 1L, Long::sum);
tmpTableActive.merge(tableId, 1L, Long::sum);
tmpPartitionActive.merge(partitionId, 1L, Long::sum);
tmpDbActive.merge(dbId, 1L, Long::sum);
}
for (Replica r : tablet.getReplicas()) {
CloudReplica replica = (CloudReplica) r;
if (isColocated) {
long beId = -1L;
Long beId = -1L;
try {
beId = replica.getColocatedBeId(cluster);
} catch (ComputeGroupException e) {
Expand All @@ -1209,13 +1222,13 @@ public void statRouteInfo() {
}

Backend be = replica.getPrimaryBackend(cluster, false);
long beId = be == null ? -1L : be.getId();
Long beId = be == null ? Long.valueOf(-1L) : Long.valueOf(be.getId());
if (!allBes.contains(beId)) {
continue;
}

Backend secondaryBe = replica.getSecondaryBackend(cluster);
long secondaryBeId = secondaryBe == null ? -1L : secondaryBe.getId();
Long secondaryBeId = secondaryBe == null ? Long.valueOf(-1L) : Long.valueOf(secondaryBe.getId());
if (allBes.contains(secondaryBeId)) {
Set<Long> tablets = tmpBeToTabletsGlobalInSecondary
.computeIfAbsent(secondaryBeId, k -> new HashSet<>());
Expand All @@ -1224,11 +1237,12 @@ public void statRouteInfo() {

InfightTablet taskKey = new InfightTablet(tabletId, cluster);
InfightTask task = tabletToInfightTask.get(taskKey);
long futureBeId = task == null ? beId : task.destBe;
fillBeToTablets(beId, table.getId(), partition.getId(), index.getId(), tabletId,
Long futureBeId = task == null ? beId : Long.valueOf(task.destBe);
Comment thread
deardeng marked this conversation as resolved.
Long routeTabletId = task == null ? tabletId : task.pickedTabletId;
fillBeToTablets(beId, tableId, partitionId, indexId, routeTabletId,
tmpBeToTabletsGlobal, beToTabletsInTable, this.partitionToTablets);

fillBeToTablets(futureBeId, table.getId(), partition.getId(), index.getId(), tabletId,
fillBeToTablets(futureBeId, tableId, partitionId, indexId, routeTabletId,
tmpFutureBeToTabletsGlobal, futureBeToTabletsInTable, futurePartitionToTablets);
}
}
Expand Down Expand Up @@ -1618,7 +1632,7 @@ private void handleWarmupCompletion(InfightTask task, String clusterId, boolean
}
}

private void updateBeToTablets(long tabletId, long srcBe, long destBe,
private void updateBeToTablets(Long tabletId, Long srcBe, Long destBe,
ConcurrentHashMap<Long, Set<Long>> globalBeToTablets,
ConcurrentHashMap<Long, ConcurrentHashMap<Long, Set<Long>>> beToTabletsInTable,
ConcurrentHashMap<Long, ConcurrentHashMap<Long, ConcurrentHashMap<Long,
Expand All @@ -1628,9 +1642,9 @@ private void updateBeToTablets(long tabletId, long srcBe, long destBe,
LOG.warn("tablet {} meta not found in inverted index, skip updateBeToTablets", tabletId);
return;
}
long tableId = tabletMeta.getTableId();
long partId = tabletMeta.getPartitionId();
long indexId = tabletMeta.getIndexId();
Long tableId = tabletMeta.getTableId();
Long partId = tabletMeta.getPartitionId();
Long indexId = tabletMeta.getIndexId();

Set<Long> globalSrcTablets = globalBeToTablets.get(srcBe);
if (globalSrcTablets == null || !globalSrcTablets.remove(tabletId)) {
Expand Down Expand Up @@ -1658,8 +1672,8 @@ private void updateBeToTablets(long tabletId, long srcBe, long destBe,
}
}

fillBeToTablets(destBe, tableId, partId, indexId, tabletId, globalBeToTablets, beToTabletsInTable,
partToTablets);
fillBeToTablets(destBe, tableId, partId, indexId, tabletId, globalBeToTablets,
beToTabletsInTable, partToTablets);
}

private void updateClusterToBeMap(long tabletId, long destBe, String clusterId,
Expand Down Expand Up @@ -1915,8 +1929,8 @@ private void balanceImpl(List<Long> bes, String clusterId, Map<Long, Set<Long>>
break; // no need balance
}

long srcBe = pairInfo.srcBe;
long destBe = pairInfo.destBe;
Long srcBe = pairInfo.srcBe;
Long destBe = pairInfo.destBe;

Long pickedTabletId = pickTabletPreferCold(srcBe, beToTablets.get(srcBe),
this.activeTabletIds, pickedTabletIds);
Expand Down Expand Up @@ -2071,7 +2085,7 @@ private Long reservoirPick(Set<Long> tabletIds, Set<Long> pickedTabletIds,
return chosen;
}

private boolean preheatAndUpdateTablet(long pickedTabletId, long srcBe, long destBe, String clusterId,
private boolean preheatAndUpdateTablet(Long pickedTabletId, Long srcBe, Long destBe, String clusterId,
BalanceType balanceType) {
Backend srcBackend = cloudSystemInfoService.getBackend(srcBe);
Backend destBackend = cloudSystemInfoService.getBackend(destBe);
Expand All @@ -2097,7 +2111,7 @@ private boolean preheatAndUpdateTablet(long pickedTabletId, long srcBe, long des
return true;
}

private boolean transferTablet(long pickedTabletId, long srcBe, long destBe, String clusterId,
private boolean transferTablet(Long pickedTabletId, Long srcBe, Long destBe, String clusterId,
BalanceType balanceType, List<UpdateCloudReplicaInfo> infos) {
LOG.debug("transfer {} from {} to {}, cluster {}, type {}",
pickedTabletId, srcBe, destBe, clusterId, balanceType);
Expand Down
Loading
Loading