From 9efbcb92ba3218fa68054b89827f929dc3ba8511 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 3 Aug 2026 16:27:15 +0800 Subject: [PATCH 1/2] [refactor](fe) Modularize external metadata cache ### What problem does this PR solve? Issue Number: None Related PR: #65126 Problem Summary: External metadata caching had parallel implementations in FE core and connector modules, which duplicated cache policy, statistics, invalidation, and concurrency behavior. Introduce an implementation-free MetaCache SPI module and a shared Caffeine-backed runtime module, move the common lifecycle and entry machinery into them, and keep only FE-specific catalog routing, configuration, refresh orchestration, edit-log integration, and schema validation in FE core. Migrate Hive, HMS, Iceberg, Paimon, and MaxCompute consumers to the single shared implementation while preserving refresh, scoped invalidation, bulk-load generation fences, and plugin class identity. ### Release note None ### Check List (For Author) - Test: Unit Test and FE build - SPI and shared runtime unit tests - FE metadata cache and refresh unit tests - Affected connector cache unit tests and plugin package/install - ./build.sh --fe -j8 - Behavior changed: No - Does this need documentation: No --- fe/fe-connector/README.md | 11 +- .../doris/connector/cache/MetaCacheEntry.java | 345 ------------------ .../connector/cache/MetaCacheEntryStats.java | 201 ---------- fe/fe-connector/fe-connector-hive/pom.xml | 18 +- .../doris/connector/hive/HiveConnector.java | 4 +- .../connector/hive/HiveConnectorMetadata.java | 4 +- .../connector/hive/HiveConnectorProvider.java | 2 +- .../connector/hive/HiveFileListingCache.java | 4 +- ...nnectorMetadataPartitionViewCacheTest.java | 2 +- .../HiveConnectorPartitionViewCacheTest.java | 4 +- fe/fe-connector/fe-connector-hms/pom.xml | 6 +- .../doris/connector/hms/CachingHmsClient.java | 4 +- fe/fe-connector/fe-connector-iceberg/pom.xml | 11 +- .../iceberg/IcebergCatalogFactory.java | 2 +- .../iceberg/IcebergCommentCache.java | 4 +- .../connector/iceberg/IcebergConnector.java | 4 +- .../iceberg/IcebergConnectorMetadata.java | 4 +- .../iceberg/IcebergConnectorProvider.java | 2 +- .../connector/iceberg/IcebergFormatCache.java | 4 +- .../iceberg/IcebergLatestSnapshotCache.java | 4 +- .../iceberg/IcebergManifestCache.java | 4 +- .../iceberg/IcebergPartitionCache.java | 4 +- .../iceberg/IcebergScanPlanProvider.java | 2 +- .../connector/iceberg/IcebergTableCache.java | 4 +- .../iceberg/IcebergConnectorCacheTest.java | 4 +- ...nnectorMetadataPartitionViewCacheTest.java | 2 +- .../IcebergLatestSnapshotCacheTest.java | 2 +- .../iceberg/IcebergTableCacheTest.java | 2 +- .../fe-connector-maxcompute/pom.xml | 18 +- .../maxcompute/MaxComputePartitionCache.java | 6 +- .../MaxComputePartitionCacheTest.java | 2 +- .../fe-connector-metacache-spi/pom.xml | 52 +++ .../connector/metacache/spi}/CacheSpec.java | 17 +- .../metacache/spi}/MetaCacheEntryDef.java | 30 +- .../spi}/MetaCacheEntryInvalidation.java | 32 +- .../metacache/spi}/MetaCacheEntryStats.java | 4 +- .../metacache/spi/MetaCacheLifecycle.java | 60 +++ .../metacache/spi}/CacheSpecTest.java | 8 +- .../pom.xml | 35 +- .../metacache/AbstractMetaCache.java | 280 ++++++++++++++ .../connector/metacache}/CacheFactory.java | 9 +- .../metacache/CatalogEntryGroup.java | 7 +- .../metacache}/ConnectorMetadataCache.java | 4 +- .../metacache}/ConnectorTableKey.java | 2 +- .../connector}/metacache/IdNameIndex.java | 34 +- .../connector}/metacache/MetaCacheEntry.java | 152 ++++++-- .../metacache/MetaCacheRegistry.java | 98 +++++ .../connector/metacache}/package-info.java | 11 +- .../metacache/AbstractMetaCacheTest.java | 136 +++++++ ...ectorMetaCacheEntryCompatibilityTest.java} | 12 +- .../ConnectorMetadataCacheTest.java | 2 +- .../connector}/metacache/IdNameIndexTest.java | 2 +- .../metacache/MetaCacheDeadlockTest.java | 10 +- .../metacache/MetaCacheEntryTest.java | 18 +- .../metacache/MetaCacheRegistryTest.java | 81 ++++ fe/fe-connector/fe-connector-paimon/pom.xml | 17 +- .../connector/paimon/PaimonConnector.java | 5 +- .../paimon/PaimonConnectorMetadata.java | 4 +- .../paimon/PaimonConnectorProvider.java | 2 +- .../paimon/PaimonLatestSnapshotCache.java | 4 +- .../paimon/PaimonConnectorCacheTest.java | 4 +- ...nnectorMetadataPartitionViewCacheTest.java | 2 +- .../paimon/PaimonLatestSnapshotCacheTest.java | 2 +- fe/fe-connector/pom.xml | 3 +- fe/fe-core/pom.xml | 10 + .../doris/datasource/ExternalCatalog.java | 15 +- .../doris/datasource/ExternalDatabase.java | 16 +- .../datasource/ExternalMetaCacheMgr.java | 6 +- .../doris/DorisExternalMetaCache.java | 12 +- .../metacache/AbstractExternalMetaCache.java | 267 +------------- .../doris/datasource/metacache/CacheSpec.java | 287 --------------- .../metacache/ExternalMetaCache.java | 76 +--- .../metacache/ExternalMetaCacheRegistry.java | 84 +---- .../datasource/metacache/NameCacheValue.java | 5 +- .../tablefunction/MetadataGenerator.java | 2 +- .../doris/catalog/RefreshManagerTest.java | 6 +- .../doris/datasource/ExternalCatalogTest.java | 4 +- .../datasource/ExternalDatabaseTest.java | 6 +- .../doris/DorisExternalMetaCacheTest.java | 2 +- .../AbstractExternalMetaCacheTest.java | 12 +- .../datasource/metacache/CacheSpecTest.java | 169 --------- 81 files changed, 1119 insertions(+), 1689 deletions(-) delete mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java delete mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java create mode 100644 fe/fe-connector/fe-connector-metacache-spi/pom.xml rename fe/fe-connector/{fe-connector-cache/src/main/java/org/apache/doris/connector/cache => fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi}/CacheSpec.java (90%) rename fe/{fe-core/src/main/java/org/apache/doris/datasource/metacache => fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi}/MetaCacheEntryDef.java (88%) rename fe/{fe-core/src/main/java/org/apache/doris/datasource/metacache => fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi}/MetaCacheEntryInvalidation.java (73%) rename fe/{fe-core/src/main/java/org/apache/doris/datasource/metacache => fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi}/MetaCacheEntryStats.java (98%) create mode 100644 fe/fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi/MetaCacheLifecycle.java rename fe/fe-connector/{fe-connector-cache/src/test/java/org/apache/doris/connector/cache => fe-connector-metacache-spi/src/test/java/org/apache/doris/connector/metacache/spi}/CacheSpecTest.java (96%) rename fe/fe-connector/{fe-connector-cache => fe-connector-metacache}/pom.xml (59%) create mode 100644 fe/fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache/AbstractMetaCache.java rename fe/fe-connector/{fe-connector-cache/src/main/java/org/apache/doris/connector/cache => fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache}/CacheFactory.java (89%) rename fe/{fe-core/src/main/java/org/apache/doris/datasource => fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector}/metacache/CatalogEntryGroup.java (88%) rename fe/fe-connector/{fe-connector-cache/src/main/java/org/apache/doris/connector/cache => fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache}/ConnectorMetadataCache.java (98%) rename fe/fe-connector/{fe-connector-cache/src/main/java/org/apache/doris/connector/cache => fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache}/ConnectorTableKey.java (98%) rename fe/{fe-core/src/main/java/org/apache/doris/datasource => fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector}/metacache/IdNameIndex.java (86%) rename fe/{fe-core/src/main/java/org/apache/doris/datasource => fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector}/metacache/MetaCacheEntry.java (85%) create mode 100644 fe/fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache/MetaCacheRegistry.java rename fe/fe-connector/{fe-connector-cache/src/main/java/org/apache/doris/connector/cache => fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache}/package-info.java (64%) create mode 100644 fe/fe-connector/fe-connector-metacache/src/test/java/org/apache/doris/connector/metacache/AbstractMetaCacheTest.java rename fe/fe-connector/{fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java => fe-connector-metacache/src/test/java/org/apache/doris/connector/metacache/ConnectorMetaCacheEntryCompatibilityTest.java} (96%) rename fe/fe-connector/{fe-connector-cache/src/test/java/org/apache/doris/connector/cache => fe-connector-metacache/src/test/java/org/apache/doris/connector/metacache}/ConnectorMetadataCacheTest.java (99%) rename fe/{fe-core/src/test/java/org/apache/doris/datasource => fe-connector/fe-connector-metacache/src/test/java/org/apache/doris/connector}/metacache/IdNameIndexTest.java (99%) rename fe/{fe-core/src/test/java/org/apache/doris/datasource => fe-connector/fe-connector-metacache/src/test/java/org/apache/doris/connector}/metacache/MetaCacheDeadlockTest.java (91%) rename fe/{fe-core/src/test/java/org/apache/doris/datasource => fe-connector/fe-connector-metacache/src/test/java/org/apache/doris/connector}/metacache/MetaCacheEntryTest.java (99%) create mode 100644 fe/fe-connector/fe-connector-metacache/src/test/java/org/apache/doris/connector/metacache/MetaCacheRegistryTest.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java diff --git a/fe/fe-connector/README.md b/fe/fe-connector/README.md index 5a1dcef5e2d883..b3019c35955951 100644 --- a/fe/fe-connector/README.md +++ b/fe/fe-connector/README.md @@ -56,7 +56,8 @@ endpoint properties) | Module | Role | |---|---| -| `fe-connector-cache` | Self-contained caching framework used by several connectors. No fe-core dependency; it is bundled into each consuming plugin, so shared third-party libraries stay at the consumers' lowest common version (see the version notes in consumer poms). | +| `fe-connector-metacache-spi` | JDK-only cache policy, entry-definition, invalidation, statistics, and lifecycle contracts. It has no fe-core, cache-library, or data-source SDK dependency. | +| `fe-connector-metacache` | Shared Caffeine-backed cache runtime used by fe-core and connector plugins. It owns entry generations, refresh, scoped invalidation, catalog entry groups, and the engine registry; it never depends on fe-core or a data-source SDK. | | `fe-connector-hms-hive-shade` | Slim, relocated HMS metastore-client closure for connectors that speak HMS thrift. The pom comments say exactly what relocates where and why. | | `fe-connector-paimon-hive-shade` | Paimon-private relocated HMS-thrift closure; same idea, different owner. | @@ -183,10 +184,10 @@ metastore/shade/cache). For a write path, the richest example is 6. **Property ownership.** Metadata-connection properties are parsed in your connector (or the metastore layer). Storage properties belong to `fe-filesystem`. Do not add parsing to fe-core — rule 2 above. -7. **Caching.** Reuse `fe-connector-cache` (example: - `PaimonLatestSnapshotCache`). Bundle the caching library into your plugin - zip and keep shared third-party versions aligned with the other consumers - (see the version notes in `fe-connector-paimon/pom.xml`). Respect the +7. **Caching.** Describe reusable entries with `fe-connector-metacache-spi` + and run them through `fe-connector-metacache` (example: + `PaimonLatestSnapshotCache`). Keep shared third-party versions aligned with + the other consumers (see the version notes in `fe-connector-paimon/pom.xml`). Respect the authorization invariant in `AGENTS.md`: a cross-query cache must never serve metadata that would bypass per-user, load-time authorization. 8. **Shading.** If your client stack drags a conflicting closure (hive/thrift diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java deleted file mode 100644 index 425697ef9b1098..00000000000000 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java +++ /dev/null @@ -1,345 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.cache; - -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.LoadingCache; -import com.github.benmanes.caffeine.cache.stats.CacheStats; - -import java.util.Objects; -import java.util.OptionalLong; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.function.Predicate; - -/** - * Unified cache entry abstraction. - * It stores one logical cache dataset and provides optional lazy loading, - * key/predicate/full invalidation, and lightweight runtime stats. - * - *

Connector-side copy of fe-core {@code org.apache.doris.datasource.metacache.MetaCacheEntry} - * (independent-copy meta-cache migration): connector plugins cannot import fe-core, so the framework is - * duplicated under {@code org.apache.doris.connector.cache}. The public API is Caffeine-free (Caffeine is - * encapsulated), so instances are safe to hold in connector (child-first) code. Two knobs that fe-core reads - * from static {@code Config} are here supplied by the connector via the constructor - * ({@code refreshAfterWriteSeconds}, {@code manualMissLoadEnabled}); otherwise keep in sync with fe-core. - */ -public class MetaCacheEntry { - // Use striped locks to deduplicate slow external loads without managing per-key lock lifecycle. - private static final int LOAD_LOCK_STRIPES = 128; - - private final String name; - private final Function loader; - private final CacheSpec cacheSpec; - private final boolean effectiveEnabled; - private final boolean autoRefresh; - // fe-core reads these two from Config; the connector copy has no fe-core Config, so they are injected. - // refreshAfterWriteSeconds is already in seconds (fe-core computes Config.*_minutes * 60 at its call site). - private final long refreshAfterWriteSeconds; - private final boolean manualMissLoadEnabled; - // Keep the loading cache for refreshAfterWrite and the legacy sync-load path when the feature is disabled. - private final LoadingCache loadingData; - // Use the plain cache view for manual miss load so slow I/O does not happen in Caffeine's sync load path. - private final Cache data; - // Protect one key stripe at a time to deduplicate concurrent miss loads with bounded lock count. - private final Object[] loadLocks = new Object[LOAD_LOCK_STRIPES]; - private final AtomicLong invalidateCount = new AtomicLong(0); - // Bump generation before invalidation so in-flight manual loads do not repopulate stale values. - private final AtomicLong invalidateGeneration = new AtomicLong(0); - // Track load statistics outside Caffeine because manual miss loads bypass the built-in load counters. - private final AtomicLong loadSuccessCount = new AtomicLong(0); - private final AtomicLong loadFailureCount = new AtomicLong(0); - private final AtomicLong totalLoadTimeNanos = new AtomicLong(0); - private final AtomicLong lastLoadSuccessTimeMs = new AtomicLong(-1L); - private final AtomicLong lastLoadFailureTimeMs = new AtomicLong(-1L); - private final AtomicReference lastError = new AtomicReference<>(""); - - /** - * Convenience constructor for the common connector case: a loader-backed entry with no auto-refresh and no - * manual miss load (Caffeine's sync load path). Use the full constructor for contextual-only entries, - * auto-refresh, or manual miss load. - */ - public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor) { - this(name, loader, cacheSpec, refreshExecutor, false, false, 0L, false); - } - - public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, - ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, - long refreshAfterWriteSeconds, boolean manualMissLoadEnabled) { - this.name = name; - if (contextualOnly) { - if (loader != null) { - throw new IllegalArgumentException("contextual-only entry loader must be null"); - } - if (autoRefresh) { - throw new IllegalArgumentException("contextual-only entry can not enable auto refresh"); - } - } else { - Objects.requireNonNull(loader, "loader can not be null"); - } - this.loader = loader; - this.cacheSpec = Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); - this.autoRefresh = autoRefresh; - this.refreshAfterWriteSeconds = refreshAfterWriteSeconds; - this.manualMissLoadEnabled = manualMissLoadEnabled; - Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); - this.effectiveEnabled = CacheSpec.isCacheEnabled( - this.cacheSpec.isEnable(), this.cacheSpec.getTtlSecond(), this.cacheSpec.getCapacity()); - OptionalLong expireAfterAccessSec = - effectiveEnabled ? CacheSpec.toExpireAfterAccess(this.cacheSpec.getTtlSecond()) : OptionalLong.empty(); - OptionalLong refreshAfterWriteSec = - effectiveEnabled && autoRefresh - ? OptionalLong.of(refreshAfterWriteSeconds) - : OptionalLong.empty(); - long maxSize = effectiveEnabled ? this.cacheSpec.getCapacity() : 0L; - CacheFactory cacheFactory = new CacheFactory( - expireAfterAccessSec, - refreshAfterWriteSec, - maxSize, - true, - null); - this.loadingData = cacheFactory.buildCache(this::loadFromDefaultLoader, refreshExecutor); - this.data = loadingData; - // Initialize striped locks eagerly to keep the hot path allocation-free. - for (int i = 0; i < loadLocks.length; i++) { - loadLocks[i] = new Object(); - } - } - - public String name() { - return name; - } - - public V get(K key) { - if (!isManualMissLoadEnabled()) { - return loadingData.get(key); - } - return getWithManualLoad(key, this::applyDefaultLoader); - } - - public V get(K key, Function missLoader) { - Function loadFunction = Objects.requireNonNull(missLoader, "missLoader can not be null"); - if (!isManualMissLoadEnabled()) { - return loadingData.get(key, typedKey -> loadAndTrack(typedKey, loadFunction)); - } - return getWithManualLoad(key, loadFunction); - } - - public V getIfPresent(K key) { - if (!effectiveEnabled) { - return null; - } - return data.getIfPresent(key); - } - - public void put(K key, V value) { - if (!effectiveEnabled) { - return; - } - data.put(key, value); - } - - /** - * The current invalidation generation. Capture this BEFORE a slow external load, then hand it to - * {@link #putIfNotInvalidatedSince} so a {@code flush}/{@code invalidate*} that raced the load does not - * get its clear silently undone by a stale write-back. Mirrors the guard the manual-miss-load path - * ({@link #getWithManualLoad}) applies around its own put; exposed so a caller that does its OWN bulk - * external read (e.g. a decorator batching a multi-key RPC and putting each result under its own key) - * can reuse the same generation guard instead of an unguarded {@link #put}. - */ - public long invalidationGeneration() { - return invalidateGeneration.get(); - } - - /** - * Generation-guarded put: caches {@code (key, value)} only if no invalidation has happened since - * {@code generation} was captured (before the caller's external load). If a {@code flush}/{@code - * invalidate*} raced the load — bumping the generation either before the put (skip) or between the put - * and the recheck (drop only the value we wrote, via {@link #removeLoadedValue}) — the stale value is - * NOT left cached, exactly as {@link #getWithManualLoad} does for its single-key load. Additive: the - * existing {@link #put} is unchanged; a disabled entry is a no-op. - */ - public void putIfNotInvalidatedSince(long generation, K key, V value) { - if (!effectiveEnabled) { - return; - } - synchronized (loadLock(key)) { - // A racing flush already bumped the generation before we could put: skip so a stale pre-flush - // value is not re-cached (mirrors getWithManualLoad's pre-put guard). - if (generation != invalidateGeneration.get()) { - return; - } - data.put(key, value); - // A flush landing between the check and the put: drop only the value we just wrote, keeping any - // newer replacement intact (mirrors getWithManualLoad's post-put guard). - if (generation != invalidateGeneration.get()) { - removeLoadedValue(key, value); - } - } - } - - public void invalidateKey(K key) { - invalidateGeneration.incrementAndGet(); - if (data.asMap().remove(key) != null) { - invalidateCount.incrementAndGet(); - } - } - - public void invalidateIf(Predicate predicate) { - invalidateGeneration.incrementAndGet(); - data.asMap().keySet().removeIf(key -> { - if (predicate.test(key)) { - invalidateCount.incrementAndGet(); - return true; - } - return false; - }); - } - - public void invalidateAll() { - invalidateGeneration.incrementAndGet(); - long size = data.estimatedSize(); - data.invalidateAll(); - invalidateCount.addAndGet(size); - } - - public void forEach(BiConsumer consumer) { - data.asMap().forEach(consumer); - } - - public MetaCacheEntryStats stats() { - CacheStats cacheStats = loadingData.stats(); - long successCount = loadSuccessCount.get(); - long failureCount = loadFailureCount.get(); - long totalLoadTime = totalLoadTimeNanos.get(); - long totalLoadCount = successCount + failureCount; - return new MetaCacheEntryStats( - cacheSpec.isEnable(), - effectiveEnabled, - autoRefresh, - cacheSpec.getTtlSecond(), - cacheSpec.getCapacity(), - data.estimatedSize(), - cacheStats.requestCount(), - cacheStats.hitCount(), - cacheStats.missCount(), - cacheStats.hitRate(), - successCount, - failureCount, - totalLoadTime, - totalLoadCount == 0 ? 0D : (double) totalLoadTime / totalLoadCount, - cacheStats.evictionCount(), - invalidateCount.get(), - lastLoadSuccessTimeMs.get(), - lastLoadFailureTimeMs.get(), - lastError.get()); - } - - // Injected at construction (fe-core reads Config.enable_external_meta_cache_manual_miss_load dynamically). - private boolean isManualMissLoadEnabled() { - return manualMissLoadEnabled; - } - - // Execute slow miss loads outside Caffeine's sync load path and suppress stale write-back after invalidation. - private V getWithManualLoad(K key, Function loadFunction) { - if (!effectiveEnabled) { - // Bypass cache entirely when the entry is disabled so manual miss load does not relax disable semantics. - return loadAndTrack(key, loadFunction); - } - - V value = data.getIfPresent(key); - if (value != null) { - return value; - } - - synchronized (loadLock(key)) { - value = data.asMap().get(key); - if (value != null) { - return value; - } - - long generation = invalidateGeneration.get(); - V loaded = loadAndTrack(key, loadFunction); - if (generation != invalidateGeneration.get()) { - return loaded; - } - - // Keep null results uncached so manual miss load matches LoadingCache null-return behavior. - if (loaded == null) { - return null; - } - - // Leave a narrow hook for tests to pause exactly before the cache put race window. - beforeManualCachePutForTest(key, loaded); - data.put(key, loaded); - if (generation != invalidateGeneration.get()) { - removeLoadedValue(key, loaded); - } - return loaded; - } - } - - // Remove only the value loaded by the current request and keep newer replacements intact. - private void removeLoadedValue(K key, V loaded) { - data.asMap().computeIfPresent(key, (ignored, currentValue) -> currentValue == loaded ? null : currentValue); - } - - // Map keys to a fixed lock stripe set to bound memory usage while keeping same-key deduplication. - private Object loadLock(K key) { - int hash = key == null ? 0 : key.hashCode(); - return loadLocks[(hash & Integer.MAX_VALUE) % loadLocks.length]; - } - - // Let tests pause between the first generation check and data.put without affecting production behavior. - void beforeManualCachePutForTest(K key, V loaded) { - } - - private V loadFromDefaultLoader(K key) { - return loadAndTrack(key, this::applyDefaultLoader); - } - - // Resolve the default loader separately so the manual path can share tracking without double counting. - private V applyDefaultLoader(K key) { - if (loader == null) { - throw new UnsupportedOperationException( - String.format("Entry '%s' requires a contextual miss loader.", name)); - } - return loader.apply(key); - } - - // Track load outcomes locally because manual miss loads do not contribute to Caffeine load statistics. - private V loadAndTrack(K key, Function loadFunction) { - long startNanos = System.nanoTime(); - try { - V value = loadFunction.apply(key); - loadSuccessCount.incrementAndGet(); - totalLoadTimeNanos.addAndGet(System.nanoTime() - startNanos); - lastLoadSuccessTimeMs.set(System.currentTimeMillis()); - return value; - } catch (RuntimeException | Error e) { - loadFailureCount.incrementAndGet(); - totalLoadTimeNanos.addAndGet(System.nanoTime() - startNanos); - lastLoadFailureTimeMs.set(System.currentTimeMillis()); - lastError.set(e.toString()); - throw e; - } - } -} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java deleted file mode 100644 index 41c8b89192cd1b..00000000000000 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java +++ /dev/null @@ -1,201 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.cache; - -import java.util.Objects; - -/** - * Immutable stats snapshot of one {@link MetaCacheEntry}. - * - *

Connector-side copy of fe-core {@code org.apache.doris.datasource.metacache.MetaCacheEntryStats} - * (independent-copy migration): the connector plugins cannot import fe-core, so the meta-cache framework is - * duplicated under {@code org.apache.doris.connector.cache}. Keep behaviourally in sync with the fe-core - * original until the fe-core copy is retired. - * - *

Time fields use the following units: - *

- * - *

For last-load timestamps, {@code -1} means no corresponding event happened yet. - * {@code lastError} keeps the latest load failure message; empty string means no failure recorded. - */ -public final class MetaCacheEntryStats { - private final boolean configEnabled; - private final boolean effectiveEnabled; - private final boolean autoRefresh; - private final long ttlSecond; - private final long capacity; - private final long estimatedSize; - private final long requestCount; - private final long hitCount; - private final long missCount; - private final double hitRate; - private final long loadSuccessCount; - private final long loadFailureCount; - private final long totalLoadTimeNanos; - private final double averageLoadPenaltyNanos; - private final long evictionCount; - private final long invalidateCount; - private final long lastLoadSuccessTimeMs; - private final long lastLoadFailureTimeMs; - private final String lastError; - - /** - * Build an immutable stats snapshot. - */ - public MetaCacheEntryStats( - boolean configEnabled, - boolean effectiveEnabled, - boolean autoRefresh, - long ttlSecond, - long capacity, - long estimatedSize, - long requestCount, - long hitCount, - long missCount, - double hitRate, - long loadSuccessCount, - long loadFailureCount, - long totalLoadTimeNanos, - double averageLoadPenaltyNanos, - long evictionCount, - long invalidateCount, - long lastLoadSuccessTimeMs, - long lastLoadFailureTimeMs, - String lastError) { - this.configEnabled = configEnabled; - this.effectiveEnabled = effectiveEnabled; - this.autoRefresh = autoRefresh; - this.ttlSecond = ttlSecond; - this.capacity = capacity; - this.estimatedSize = estimatedSize; - this.requestCount = requestCount; - this.hitCount = hitCount; - this.missCount = missCount; - this.hitRate = hitRate; - this.loadSuccessCount = loadSuccessCount; - this.loadFailureCount = loadFailureCount; - this.totalLoadTimeNanos = totalLoadTimeNanos; - this.averageLoadPenaltyNanos = averageLoadPenaltyNanos; - this.evictionCount = evictionCount; - this.invalidateCount = invalidateCount; - this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; - this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; - this.lastError = Objects.requireNonNull(lastError, "lastError"); - } - - public boolean isConfigEnabled() { - return configEnabled; - } - - /** - * Effective cache enable state evaluated by {@link CacheSpec#isCacheEnabled(boolean, long, long)}. - */ - public boolean isEffectiveEnabled() { - return effectiveEnabled; - } - - public boolean isAutoRefresh() { - return autoRefresh; - } - - public long getTtlSecond() { - return ttlSecond; - } - - public long getCapacity() { - return capacity; - } - - public long getEstimatedSize() { - return estimatedSize; - } - - public long getRequestCount() { - return requestCount; - } - - public long getHitCount() { - return hitCount; - } - - public long getMissCount() { - return missCount; - } - - public double getHitRate() { - return hitRate; - } - - public long getLoadSuccessCount() { - return loadSuccessCount; - } - - public long getLoadFailureCount() { - return loadFailureCount; - } - - public long getTotalLoadTimeNanos() { - return totalLoadTimeNanos; - } - - /** - * Average load penalty in nanoseconds. - */ - public double getAverageLoadPenaltyNanos() { - return averageLoadPenaltyNanos; - } - - public long getEvictionCount() { - return evictionCount; - } - - public double getEvictionRate() { - if (requestCount == 0) { - return 0D; - } - return (double) evictionCount / requestCount; - } - - public long getInvalidateCount() { - return invalidateCount; - } - - /** - * Last successful load timestamp in epoch milliseconds, or {@code -1} if absent. - */ - public long getLastLoadSuccessTimeMs() { - return lastLoadSuccessTimeMs; - } - - /** - * Last failed load timestamp in epoch milliseconds, or {@code -1} if absent. - */ - public long getLastLoadFailureTimeMs() { - return lastLoadFailureTimeMs; - } - - /** - * Latest load failure message, or empty string if no failure is recorded. - */ - public String getLastError() { - return lastError; - } -} diff --git a/fe/fe-connector/fe-connector-hive/pom.xml b/fe/fe-connector/fe-connector-hive/pom.xml index 29721c3a038425..931bb0f9eb5845 100644 --- a/fe/fe-connector/fe-connector-hive/pom.xml +++ b/fe/fe-connector/fe-connector-hive/pom.xml @@ -47,24 +47,22 @@ under the License. ${project.version} - + ${project.groupId} - fe-connector-cache + fe-connector-metacache ${project.version} - + version fe-connector-metacache is compiled against (the lowest common version across consuming plugins). --> com.github.ben-manes.caffeine caffeine diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java index e4a5dcee981246..9b1aefad95668b 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java @@ -29,12 +29,12 @@ import org.apache.doris.connector.api.procedure.ConnectorProcedureOps; import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; import org.apache.doris.connector.api.write.ConnectorWritePlanProvider; -import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.hms.CachingHmsClient; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsClientConfig; import org.apache.doris.connector.hms.ThriftHmsClient; import org.apache.doris.connector.hms.event.HmsEventSource; +import org.apache.doris.connector.metacache.ConnectorMetadataCache; import org.apache.doris.connector.metastore.HmsMetaStoreProperties; import org.apache.doris.connector.metastore.spi.MetaStoreProviders; import org.apache.doris.connector.spi.ConnectorContext; @@ -100,7 +100,7 @@ public class HiveConnector implements Connector { private final HiveFileListingCache fileListingCache; // PERF-06 (S6): cross-query DERIVED partition-view cache ("cache A", the generic ConnectorMetadataCache - // from fe-connector-cache), layered ABOVE the raw per-name HMS listing served by CachingHmsClient: it + // from fe-connector-metacache), layered ABOVE the raw per-name HMS listing served by CachingHmsClient: it // memoizes the BUILT List (HiveConnectorMetadata#listPartitionsUncached's per-name // HiveWriteUtils.toPartitionValues parse + ConnectorPartitionInfo construction), keyed by // (db, table, -1, -1) — hive is snapshot-less (beginQuerySnapshot always pins -1) and its handle carries no diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java index 603c2071e23355..feb4cc047ebea9 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java @@ -56,8 +56,6 @@ import org.apache.doris.connector.api.pushdown.ConnectorLiteral; import org.apache.doris.connector.api.pushdown.FilterApplicationResult; import org.apache.doris.connector.api.scan.ConnectorPartitionValues; -import org.apache.doris.connector.cache.ConnectorMetadataCache; -import org.apache.doris.connector.cache.ConnectorTableKey; import org.apache.doris.connector.hms.HiveShowCreateTableRenderer; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsClientException; @@ -67,6 +65,8 @@ import org.apache.doris.connector.hms.HmsPartitionInfo; import org.apache.doris.connector.hms.HmsTableInfo; import org.apache.doris.connector.hms.HmsTypeMapping; +import org.apache.doris.connector.metacache.ConnectorMetadataCache; +import org.apache.doris.connector.metacache.ConnectorTableKey; import org.apache.doris.connector.spi.ConnectorConf; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorStorageContext; diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorProvider.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorProvider.java index dfe2174d4c2c54..da15823f081f5b 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorProvider.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorProvider.java @@ -18,8 +18,8 @@ package org.apache.doris.connector.hive; import org.apache.doris.connector.api.Connector; -import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.hms.HmsClientConfig; +import org.apache.doris.connector.metacache.spi.CacheSpec; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorProvider; diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java index a62967c33760f2..1842a0a640438c 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java @@ -18,8 +18,8 @@ package org.apache.doris.connector.hive; import org.apache.doris.connector.api.DorisConnectorException; -import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.metacache.MetaCacheEntry; +import org.apache.doris.connector.metacache.spi.CacheSpec; import org.apache.doris.filesystem.FileEntry; import org.apache.doris.filesystem.FileIterator; import org.apache.doris.filesystem.FileSystem; diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java index e18487005642f4..094640a0e489b4 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java @@ -23,11 +23,11 @@ import org.apache.doris.connector.api.pushdown.ConnectorComparison; import org.apache.doris.connector.api.pushdown.ConnectorExpression; import org.apache.doris.connector.api.pushdown.ConnectorLiteral; -import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsDatabaseInfo; import org.apache.doris.connector.hms.HmsPartitionInfo; import org.apache.doris.connector.hms.HmsTableInfo; +import org.apache.doris.connector.metacache.ConnectorMetadataCache; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorPartitionViewCacheTest.java index a13c072f0c26e2..90022de73991e9 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorPartitionViewCacheTest.java @@ -18,8 +18,8 @@ package org.apache.doris.connector.hive; import org.apache.doris.connector.api.ConnectorPartitionInfo; -import org.apache.doris.connector.cache.ConnectorMetadataCache; -import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.metacache.ConnectorMetadataCache; +import org.apache.doris.connector.metacache.ConnectorTableKey; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/fe/fe-connector/fe-connector-hms/pom.xml b/fe/fe-connector/fe-connector-hms/pom.xml index b0fde2ef732c68..76ce661a10e949 100644 --- a/fe/fe-connector/fe-connector-hms/pom.xml +++ b/fe/fe-connector/fe-connector-hms/pom.xml @@ -55,7 +55,7 @@ under the License. dep is what puts CacheSpec/MetaCacheEntry on -hms's own compile classpath. --> ${project.groupId} - fe-connector-cache + fe-connector-metacache ${project.version} @@ -141,10 +141,10 @@ under the License. com.github.ben-manes.caffeine diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java index 19d46822dc65ec..f8752a8ca603b0 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java @@ -17,8 +17,8 @@ package org.apache.doris.connector.hms; -import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.metacache.MetaCacheEntry; +import org.apache.doris.connector.metacache.spi.CacheSpec; import org.apache.hadoop.hive.common.FileUtils; diff --git a/fe/fe-connector/fe-connector-iceberg/pom.xml b/fe/fe-connector/fe-connector-iceberg/pom.xml index d05edc1e8e7c10..040bf88dfb283b 100644 --- a/fe/fe-connector/fe-connector-iceberg/pom.xml +++ b/fe/fe-connector/fe-connector-iceberg/pom.xml @@ -47,16 +47,15 @@ under the License. ${project.version} - ${project.groupId} - fe-connector-cache + fe-connector-metacache ${project.version} diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java index d270b9b6897fab..06a460e14990a9 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java @@ -18,7 +18,7 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.api.DorisConnectorException; -import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.metacache.spi.CacheSpec; import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; import org.apache.doris.filesystem.properties.StorageProperties; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java index 6caf556f5780f6..4d222007733255 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java @@ -17,8 +17,8 @@ package org.apache.doris.connector.iceberg; -import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.metacache.MetaCacheEntry; +import org.apache.doris.connector.metacache.spi.CacheSpec; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java index 738cf7bd1bf7bb..ff0d0965ca3143 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java @@ -30,7 +30,7 @@ import org.apache.doris.connector.api.procedure.ConnectorProcedureOps; import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; import org.apache.doris.connector.api.write.ConnectorWritePlanProvider; -import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.metacache.ConnectorMetadataCache; import org.apache.doris.connector.metastore.HmsMetaStoreProperties; import org.apache.doris.connector.metastore.spi.JdbcDriverSupport; import org.apache.doris.connector.metastore.spi.MetaStoreProviders; @@ -191,7 +191,7 @@ public class IcebergConnector implements Connector { // authorization a shared cache would bypass. null for every other flavor. private final IcebergCommentCache commentCache; // null under session=user // PERF-06: cross-query DERIVED partition-view cache ("cache A", the generic ConnectorMetadataCache from - // fe-connector-cache), layered ABOVE the raw partitionCache (PERF-02): it memoizes the BUILT derived view + // fe-connector-metacache), layered ABOVE the raw partitionCache (PERF-02): it memoizes the BUILT derived view // (transform-to-range math + overlap merge for the MTMV view; the value-map construction for listPartitions) // keyed by (db, table, snapshotId, schemaId), so a repeated query on a partitioned table skips the derived // rebuild, not just the remote scan. Two typed fields because the two SPI hooks return structurally different diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java index 62e98dc2a0a048..87f845540b6dcc 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java @@ -43,8 +43,8 @@ import org.apache.doris.connector.api.mvcc.ConnectorMvccSnapshot; import org.apache.doris.connector.api.mvcc.ConnectorTimeTravelSpec; import org.apache.doris.connector.api.pushdown.ConnectorExpression; -import org.apache.doris.connector.cache.ConnectorMetadataCache; -import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.metacache.ConnectorMetadataCache; +import org.apache.doris.connector.metacache.ConnectorTableKey; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorStorageContext; import org.apache.doris.thrift.THiveTable; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProvider.java index 2f9e90b5a4d121..d39a39aeb41bbe 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProvider.java @@ -18,7 +18,7 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.api.Connector; -import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.metacache.spi.CacheSpec; import org.apache.doris.connector.metastore.spi.MetaStoreProviders; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorProvider; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java index cb07b3e57da124..0584320484a934 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java @@ -17,8 +17,8 @@ package org.apache.doris.connector.iceberg; -import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.metacache.MetaCacheEntry; +import org.apache.doris.connector.metacache.spi.CacheSpec; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java index 35b63c6aa8c8fc..719d98b1944bf5 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java @@ -17,8 +17,8 @@ package org.apache.doris.connector.iceberg; -import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.metacache.MetaCacheEntry; +import org.apache.doris.connector.metacache.spi.CacheSpec; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java index f3c46732fb2fc1..64312f459dc3b2 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java @@ -17,8 +17,8 @@ package org.apache.doris.connector.iceberg; -import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.metacache.MetaCacheEntry; +import org.apache.doris.connector.metacache.spi.CacheSpec; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java index a85e2187043e7c..ac5352629cb8f4 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java @@ -17,9 +17,9 @@ package org.apache.doris.connector.iceberg; -import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; import org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition; +import org.apache.doris.connector.metacache.MetaCacheEntry; +import org.apache.doris.connector.metacache.spi.CacheSpec; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java index 42db23c93ac1c4..0b2851904bd85b 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java @@ -29,7 +29,7 @@ import org.apache.doris.connector.api.scan.ConnectorScanRequest; import org.apache.doris.connector.api.scan.ConnectorSplitSource; import org.apache.doris.connector.api.scan.ScanNodePropertyKeys; -import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.metacache.spi.CacheSpec; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorStorageContext; import org.apache.doris.filesystem.properties.StorageProperties; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java index 426b706cf11f90..f1ce39bbcf71ae 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java @@ -17,8 +17,8 @@ package org.apache.doris.connector.iceberg; -import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.metacache.MetaCacheEntry; +import org.apache.doris.connector.metacache.spi.CacheSpec; import org.apache.iceberg.Table; import org.apache.iceberg.catalog.Namespace; diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorCacheTest.java index 6abe09a0f52dfd..8b3d23134c7fb0 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorCacheTest.java @@ -18,8 +18,8 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.api.ConnectorPartitionInfo; -import org.apache.doris.connector.cache.ConnectorMetadataCache; -import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.metacache.ConnectorMetadataCache; +import org.apache.doris.connector.metacache.ConnectorTableKey; import org.apache.iceberg.DataFiles; import org.apache.iceberg.ManifestFile; diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java index e5cb30b0ed97d8..0df52cb7661c97 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java @@ -21,7 +21,7 @@ import org.apache.doris.connector.api.mvcc.ConnectorMvccPartition; import org.apache.doris.connector.api.mvcc.ConnectorMvccPartitionView; import org.apache.doris.connector.api.pushdown.ConnectorExpression; -import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.metacache.ConnectorMetadataCache; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileFormat; diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCacheTest.java index c592472ffffb08..8a8a5080abff25 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCacheTest.java @@ -25,7 +25,7 @@ /** * Unit tests for {@link IcebergLatestSnapshotCache} (mirrors PaimonLatestSnapshotCacheTest). The cache is now - * backed by the shared {@link org.apache.doris.connector.cache.MetaCacheEntry} framework; these tests cover the + * backed by the shared {@link org.apache.doris.connector.metacache.MetaCacheEntry} framework; these tests cover the * adapter's contract — within-TTL stability, the {@code ttl <= 0} disable, and invalidation. Timed-expiry * mechanics are the framework's responsibility (the ttl→duration mapping is unit-tested in the framework * module's {@code CacheSpecTest}; Caffeine {@code expireAfterAccess} itself is the library's behavior), so they diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java index 9dd551c35fb483..470127cdd623e5 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java @@ -31,7 +31,7 @@ /** * Unit tests for {@link IcebergTableCache} (PERF-01). The cross-query RAW-table cache mirrors - * {@link IcebergLatestSnapshotCache} exactly (same {@link org.apache.doris.connector.cache.MetaCacheEntry} + * {@link IcebergLatestSnapshotCache} exactly (same {@link org.apache.doris.connector.metacache.MetaCacheEntry} * backing) but stores the whole {@link Table} instead of the {@code (snapshotId, schemaId)} pin, restoring the * table-caching half of the legacy {@code IcebergExternalMetaCache}. These tests cover the adapter's contract — * within-TTL stability, the {@code ttl <= 0} disable, invalidation, and the exception-propagation guarantee the diff --git a/fe/fe-connector/fe-connector-maxcompute/pom.xml b/fe/fe-connector/fe-connector-maxcompute/pom.xml index 37565fbc8a1140..752cab8a9f6bd9 100644 --- a/fe/fe-connector/fe-connector-maxcompute/pom.xml +++ b/fe/fe-connector/fe-connector-maxcompute/pom.xml @@ -45,24 +45,22 @@ under the License. ${project.version} - + ${project.groupId} - fe-connector-cache + fe-connector-metacache ${project.version} - com.github.ben-manes.caffeine diff --git a/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputePartitionCache.java b/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputePartitionCache.java index 2cc4afda30d5e7..953bef93f47898 100644 --- a/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputePartitionCache.java +++ b/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputePartitionCache.java @@ -17,8 +17,8 @@ package org.apache.doris.connector.maxcompute; -import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.metacache.MetaCacheEntry; +import org.apache.doris.connector.metacache.spi.CacheSpec; import com.aliyun.odps.Partition; @@ -31,7 +31,7 @@ /** * The MaxCompute connector's own partition-listing cache — a structural copy of the hive connector's * {@code HiveFileListingCache}, backed by the shared - * {@code fe-connector-cache} framework ({@link CacheSpec} + {@link MetaCacheEntry}). It memoizes the (expensive) + * {@code fe-connector-metacache} framework ({@link CacheSpec} + {@link MetaCacheEntry}). It memoizes the (expensive) * per-table ODPS partition listing ({@code structureHelper.getPartitions}), keyed by {@code (db, table)} — the * ODPS project is constant per catalog, so it is NOT part of the key. * diff --git a/fe/fe-connector/fe-connector-maxcompute/src/test/java/org/apache/doris/connector/maxcompute/MaxComputePartitionCacheTest.java b/fe/fe-connector/fe-connector-maxcompute/src/test/java/org/apache/doris/connector/maxcompute/MaxComputePartitionCacheTest.java index b957bb7a28579a..4c79aa7cf7c0a9 100644 --- a/fe/fe-connector/fe-connector-maxcompute/src/test/java/org/apache/doris/connector/maxcompute/MaxComputePartitionCacheTest.java +++ b/fe/fe-connector/fe-connector-maxcompute/src/test/java/org/apache/doris/connector/maxcompute/MaxComputePartitionCacheTest.java @@ -39,7 +39,7 @@ /** * Tests {@link MaxComputePartitionCache}: the connector-owned partition-listing cache (a structural copy of the - * hive connector's {@code HiveFileListingCache}), backed by the shared {@code fe-connector-cache} framework. + * hive connector's {@code HiveFileListingCache}), backed by the shared {@code fe-connector-metacache} framework. * *

WHY (Rule 9): after the max_compute cutover the fe-core engine-side external meta cache stops routing to a * MaxCompute catalog, so without this connector-owned cache every {@code SHOW PARTITIONS} / partition-pruning / diff --git a/fe/fe-connector/fe-connector-metacache-spi/pom.xml b/fe/fe-connector/fe-connector-metacache-spi/pom.xml new file mode 100644 index 00000000000000..a62451df8a715a --- /dev/null +++ b/fe/fe-connector/fe-connector-metacache-spi/pom.xml @@ -0,0 +1,52 @@ + + + + 4.0.0 + + + org.apache.doris + fe-connector + ${revision} + ../pom.xml + + + fe-connector-metacache-spi + jar + Doris FE Connector MetaCache SPI + + Stable, implementation-free contracts for external metadata cache providers. + Contains only JDK-based configuration, entry definitions, invalidation contracts + and statistics snapshots. It never depends on fe-core or a data source SDK. + + + + + org.junit.jupiter + junit-jupiter + test + + + + + doris-fe-connector-metacache-spi + + diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java b/fe/fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi/CacheSpec.java similarity index 90% rename from fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java rename to fe/fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi/CacheSpec.java index 0524b31d402f48..0f572ccf7155ca 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java +++ b/fe/fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi/CacheSpec.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.connector.cache; +package org.apache.doris.connector.metacache.spi; import java.util.HashMap; import java.util.Map; @@ -25,18 +25,9 @@ /** * Common cache specification for external metadata caches. * - *

Connector-side copy of the meta-cache property model (independent-copy meta-cache migration). fe-core is - * NOT changed: it keeps its own {@code org.apache.doris.datasource.metacache.CacheSpec}; this is a separate - * class under {@code org.apache.doris.connector.*} used only by the connector plugins. Although that prefix is - * parent-first, fe-core does not depend on this module, so the class resolves parent → miss → CHILD and is - * child-loaded per plugin — fe-core and the plugins do NOT share one {@code Class} identity. It carries no - * third-party dependency (JDK only) and never crosses the fe-core↔connector boundary as an object (only its - * {@code IllegalArgumentException}, a JDK type, crosses), so it is safe on both classpaths. - * - *

The {@code check*Property} validators throw {@link IllegalArgumentException} (fe-core's - * {@code PluginDrivenExternalCatalog.checkProperties} re-wraps it into a {@code DdlException} verbatim; the - * legacy fe-core catalogs that still call these validators declare {@code throws DdlException} but no longer - * need it). The user-facing message text is identical to the legacy one ({@code "... is wrong, value is ..."}). + *

The type is part of the connector-facing MetaCache SPI and therefore depends only on JDK types. + * Property validation reports {@link IllegalArgumentException}; fe-core adapters may translate that + * exception at their own API boundary. * *

Semantics: *