diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorStatementScopes.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorStatementScopes.java new file mode 100644 index 00000000000000..3f62dcdf993f82 --- /dev/null +++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorStatementScopes.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.api; + +import java.util.function.Supplier; + +/** + * Statement-scoped resolution helper over the neutral {@link ConnectorStatementScope} (reached via + * {@link ConnectorSession#getStatementScope()}). A connector routes its "resolve {@code db.table} once per + * statement" memo through {@link #resolveInStatement}, so every read / scan / write resolver of one statement + * shares the single loaded value and the loader runs at most once — while a reused prepared-statement scope, + * reset per execution, still isolates each execution (its {@code queryId} is part of the key). + * + *

This standardizes the security-critical key convention once instead of letting each connector re-derive it: + * dropping {@code queryId} would leak a table across executions of a reused prepared statement; dropping + * {@code catalogId} would collide across a cross-catalog {@code MERGE}. + * + *

{@code keyNamespace} is connector-owned; this module stays source-neutral. {@code keyNamespace} + * namespaces the value type stored under the shared {@code (catalogId, db, table, queryId)} coordinate, + * so a heterogeneous gateway statement touching two connectors cannot collide on {@code (db, table)} and hand one + * connector another's value (a {@link ClassCastException}). Each connector declares its own namespace constant in + * its own module (e.g. {@code IcebergStatementScope}, {@code HudiStatementScope}, {@code EsStatementScope}, + * {@code JdbcConnectorMetadata}) — this API deliberately holds no source names. The convention every + * {@code keyNamespace} passed to {@link #resolveInStatement} MUST follow: it is a compile-time constant prefixed + * with the connector's connector-type name (its {@code ConnectorProvider.getType()}, e.g. {@code "iceberg"}, + * {@code "hudi"}). Because {@code getType()} is a connector's unique identity, source-prefixing makes these + * namespaces distinct across connectors by construction; each connector's {@code *StatementScope} guards + * its own prefix with a unit test. + * + *

This convention governs only the {@code keyNamespace} passed to {@link #resolveInStatement}. Other + * per-statement memos that use {@link ConnectorStatementScope}'s {@code computeIfAbsent} / + * {@code getOrCreateMetadata} directly own their own key discipline and are out of scope here — e.g. an + * engine-reserved family that stores a single value type per key, discriminated by catalog id, stays collision-safe + * without a source prefix. A connector that instead memoizes on its own per-statement metadata instance needs no + * namespace at all. + */ +public final class ConnectorStatementScopes { + + private ConnectorStatementScopes() { + } + + /** + * Resolves {@code db.table} once per statement and shares the single value across every resolver of the + * statement. The key is {@code keyNamespace + ":" + catalogId + ":" + db + ":" + table + ":" + queryId}: the + * catalog id isolates a cross-catalog {@code MERGE}, the {@code queryId} isolates each execution of a reused + * prepared statement, and {@code keyNamespace} isolates value types across a heterogeneous gateway. + * {@code keyNamespace} MUST be a connector-owned, source-prefixed constant (see the class javadoc). + * {@code loader} runs at most once per statement; under a {@code null} session or + * {@link ConnectorStatementScope#NONE} (offline / no live statement) it runs on every call — byte-identical + * to loading every time. + */ + public static T resolveInStatement(ConnectorSession session, String keyNamespace, + String db, String table, Supplier loader) { + if (session == null) { + return loader.get(); + } + String key = keyNamespace + ":" + session.getCatalogId() + ":" + db + ":" + table + + ":" + session.getQueryId(); + return session.getStatementScope().computeIfAbsent(key, loader); + } +} diff --git a/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/ConnectorStatementScopesTest.java b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/ConnectorStatementScopesTest.java new file mode 100644 index 00000000000000..0011b3582419bb --- /dev/null +++ b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/ConnectorStatementScopesTest.java @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.api; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +/** + * Tests for {@link ConnectorStatementScopes#resolveInStatement}: the shared per-statement resolver whose key is + * {@code keyNamespace + ":" + catalogId + ":" + db + ":" + table + ":" + queryId}. It proves the memo collapses + * repeat resolves within a statement, that each key axis (namespace / catalog id / db / table / queryId) isolates, + * that a null session or {@link ConnectorStatementScope#NONE} degrades to load-every-time, and that the emitted + * key is exactly the documented string (the byte contract every connector consumer depends on). + */ +public class ConnectorStatementScopesTest { + + @Test + public void sameCoordinateResolvesOncePerStatement() { + // Same (namespace, catalog, db, table, queryId) within one scope -> loader runs once, one shared instance. + // MUTATION: not memoizing -> two loads / two instances -> red. + RecordingMemoScope scope = new RecordingMemoScope(); + TestSession session = new TestSession(7L, "q1", scope); + AtomicInteger loads = new AtomicInteger(); + Object a = ConnectorStatementScopes.resolveInStatement(session, "x.table", "db1", "t", () -> { + loads.incrementAndGet(); + return new Object(); + }); + Object b = ConnectorStatementScopes.resolveInStatement(session, "x.table", "db1", "t", () -> { + loads.incrementAndGet(); + return new Object(); + }); + Assertions.assertSame(a, b, "same coordinate -> one shared instance"); + Assertions.assertEquals(1, loads.get(), "loaded once per statement"); + } + + @Test + public void differentQueryIdIsolatesTheLoad() { + // A reused prepared statement runs each execution under its own queryId; the memo never crosses executions. + RecordingMemoScope scope = new RecordingMemoScope(); + Object a = ConnectorStatementScopes.resolveInStatement( + new TestSession(7L, "q1", scope), "x.table", "db1", "t", Object::new); + Object b = ConnectorStatementScopes.resolveInStatement( + new TestSession(7L, "q2", scope), "x.table", "db1", "t", Object::new); + Assertions.assertNotSame(a, b, "different queryId -> isolated load"); + } + + @Test + public void differentCatalogIdIsolatesTheLoad() { + // A cross-catalog MERGE resolves the two catalogs' tables independently (the key carries the catalog id). + RecordingMemoScope scope = new RecordingMemoScope(); + Object a = ConnectorStatementScopes.resolveInStatement( + new TestSession(1L, "q1", scope), "x.table", "db1", "t", Object::new); + Object b = ConnectorStatementScopes.resolveInStatement( + new TestSession(2L, "q1", scope), "x.table", "db1", "t", Object::new); + Assertions.assertNotSame(a, b, "different catalog id -> isolated load"); + } + + @Test + public void differentDbOrTableIsolatesTheLoad() { + RecordingMemoScope scope = new RecordingMemoScope(); + TestSession session = new TestSession(7L, "q1", scope); + Object t1 = ConnectorStatementScopes.resolveInStatement(session, "x.table", "db1", "t", Object::new); + Object t2 = ConnectorStatementScopes.resolveInStatement(session, "x.table", "db1", "u", Object::new); + Object t3 = ConnectorStatementScopes.resolveInStatement(session, "x.table", "db2", "t", Object::new); + Assertions.assertNotSame(t1, t2, "different table -> isolated load"); + Assertions.assertNotSame(t1, t3, "different db -> isolated load"); + } + + @Test + public void differentNamespaceIsolatesValueTypes() { + // The namespace guards a heterogeneous-gateway statement: two connectors sharing (db, table, catalog, + // queryId) but storing different value types must not collide. Without namespacing, the second resolve + // would return the first's memoized value cast to the wrong type -> ClassCastException. + // MUTATION: dropping keyNamespace from the key -> ClassCastException here -> red. + RecordingMemoScope scope = new RecordingMemoScope(); + TestSession session = new TestSession(7L, "q1", scope); + String asString = ConnectorStatementScopes.resolveInStatement( + session, "a.value", "db1", "t", () -> "stringValue"); + Integer asInt = ConnectorStatementScopes.resolveInStatement( + session, "b.value", "db1", "t", () -> 42); + Assertions.assertEquals("stringValue", asString, "namespace a keeps its String value"); + Assertions.assertEquals(Integer.valueOf(42), asInt, "namespace b keeps its Integer value (no collision)"); + } + + @Test + public void nullSessionLoadsEveryTime() { + // No session (offline / direct-construction tests): each call loads, byte-identical to loading every time. + AtomicInteger loads = new AtomicInteger(); + ConnectorStatementScopes.resolveInStatement(null, "x.table", "db1", "t", () -> { + loads.incrementAndGet(); + return new Object(); + }); + ConnectorStatementScopes.resolveInStatement(null, "x.table", "db1", "t", () -> { + loads.incrementAndGet(); + return new Object(); + }); + Assertions.assertEquals(2, loads.get(), "null session -> load every time"); + } + + @Test + public void noneScopeLoadsEveryTime() { + // A live session whose scope is NONE (no per-statement context) also loads every time. + TestSession session = new TestSession(7L, "q1", ConnectorStatementScope.NONE); + AtomicInteger loads = new AtomicInteger(); + ConnectorStatementScopes.resolveInStatement(session, "x.table", "db1", "t", () -> { + loads.incrementAndGet(); + return new Object(); + }); + ConnectorStatementScopes.resolveInStatement(session, "x.table", "db1", "t", () -> { + loads.incrementAndGet(); + return new Object(); + }); + Assertions.assertEquals(2, loads.get(), "NONE scope -> load every time"); + } + + @Test + public void keyIsNamespaceCatalogDbTableQueryId() { + // The exact byte contract: keyNamespace + ":" + catalogId + ":" + db + ":" + table + ":" + queryId. + // Every connector consumer's cross-statement isolation depends on this string; assert it verbatim. + RecordingMemoScope scope = new RecordingMemoScope(); + ConnectorStatementScopes.resolveInStatement( + new TestSession(7L, "q1", scope), "test.ns", "db1", "t", Object::new); + Assertions.assertEquals("test.ns:7:db1:t:q1", scope.lastKey, "emitted key must match the documented format"); + } + + /** A statement scope that memoizes like the engine's real one and records the last key, for the assertions. */ + private static final class RecordingMemoScope implements ConnectorStatementScope { + private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); + private String lastKey; + + @Override + @SuppressWarnings("unchecked") + public T computeIfAbsent(String key, Supplier loader) { + lastKey = key; + return (T) cache.computeIfAbsent(key, k -> loader.get()); + } + } + + /** Minimal {@link ConnectorSession} carrying a catalog id, queryId and scope for the key + memo assertions. */ + private static final class TestSession implements ConnectorSession { + private final long catalogId; + private final String queryId; + private final ConnectorStatementScope scope; + + TestSession(long catalogId, String queryId, ConnectorStatementScope scope) { + this.catalogId = catalogId; + this.queryId = queryId; + this.scope = scope; + } + + @Override + public long getCatalogId() { + return catalogId; + } + + @Override + public String getQueryId() { + return queryId; + } + + @Override + public String getSessionId() { + // Deliberately != queryId so keyIsNamespaceCatalogDbTableQueryId pins the per-execution queryId + // (cross-query isolation), not the stable per-connection sessionId; a queryId->sessionId swap in the + // helper's key would share a table across queries of one connection and MUST turn that assertion red. + return "session-" + queryId; + } + + @Override + public ConnectorStatementScope getStatementScope() { + return scope; + } + + @Override + public String getUser() { + return "u"; + } + + @Override + public String getTimeZone() { + return "UTC"; + } + + @Override + public String getLocale() { + return "en_US"; + } + + @Override + public String getCatalogName() { + return "c"; + } + + @Override + public T getProperty(String name, Class type) { + return null; + } + + @Override + public Map getCatalogProperties() { + return Collections.emptyMap(); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java index 47cd0596b46096..0524b31d402f48 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java @@ -67,6 +67,23 @@ public static CacheSpec of(boolean enable, long ttlSecond, long capacity) { return new CacheSpec(enable, ttlSecond, capacity); } + /** + * Build an ENABLED spec from a connector-resolved TTL under the "{@code <= 0} disables" contract. + * + *

A connector that resolves its own single {@code ttl-second} knob (iceberg's shared + * {@code meta.cache.iceberg.table.ttl-second}, paimon's snapshot cache) treats any non-positive TTL as + * "disable caching, always read live". That is NOT the raw {@link CacheSpec} contract, which reads + * {@code ttlSecond == -1} as {@link #CACHE_NO_TTL} ("no expiration", still ENABLED) and only + * {@code ttlSecond == 0} as {@link #CACHE_TTL_DISABLE_CACHE} ("disabled"). This factory folds any + * non-positive TTL to the disable sentinel so a negative operator value disables the cache rather than + * silently becoming a never-expiring one. It is exactly the + * {@code ttlSecond > 0 ? of(true, ttlSecond, capacity) : of(true, CACHE_TTL_DISABLE_CACHE, capacity)} + * expression each per-catalog cache used to inline. + */ + public static CacheSpec ofConnectorTtl(long ttlSecond, long capacity) { + return of(true, ttlSecond > 0 ? ttlSecond : CACHE_TTL_DISABLE_CACHE, capacity); + } + public static PropertySpec.Builder propertySpecBuilder() { return new PropertySpec.Builder(); } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorPartitionViewCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java similarity index 65% rename from fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorPartitionViewCache.java rename to fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java index cd7f03be986ee7..fde68573e73c50 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorPartitionViewCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java @@ -24,18 +24,18 @@ import java.util.function.Supplier; /** - * GENERIC (engine-agnostic) cross-query cache of a connector's derived partition view — "cache A" of the - * external-partition-derived-cache design (design doc {@code 2026-07-20-external-partition-derived-cache-design.md} - * §5). It is the generic version of {@code org.apache.doris.connector.iceberg.IcebergPartitionCache}: same - * construction pattern (a contextual-only, manual-miss-load {@link MetaCacheEntry}), same {@link CacheSpec} wiring, - * same invalidation style — but keyed by the engine-agnostic {@link PartitionViewCacheKey} and holding an opaque - * value {@code V} instead of an iceberg-specific raw-partition list, so it has no engine-specific imports and can be - * shared by every connector (iceberg/paimon/hive/maxcompute, wired in later subsystems — this class has NO - * consumers yet). + * GENERIC (engine-agnostic) cross-query cache of a connector's derived metadata, keyed by a table identity plus + * an optional MVCC coordinate ({@link ConnectorTableKey}) and holding an opaque value {@code V}. It is the generic + * form of the hand-rolled iceberg caches (e.g. {@code IcebergPartitionCache}): same construction pattern (a + * contextual-only, manual-miss-load {@link MetaCacheEntry}), same {@link CacheSpec} wiring, same invalidation style + * — but keyed by the engine-agnostic {@link ConnectorTableKey} and holding an opaque {@code V} instead of an + * engine-specific value, so it has no engine-specific imports and is shared by every connector. Consumers today: + * the hive/iceberg/paimon derived partition-view caches (entry {@code "partition_view"}); a connector may hold + * several instances under distinct entry names. * - *

Config: {@code meta.cache..partition_view.(enable|ttl-second|capacity)}, default ON / 86400s / - * 1000 entries (matching {@code IcebergPartitionCache}'s {@code DEFAULT_TABLE_CACHE_CAPACITY}). {@code enable=false} - * / {@code ttl-second=0} / {@code capacity=0} each disable the cache (see {@link CacheSpec#isCacheEnabled}): {@link + *

Config: {@code meta.cache...(enable|ttl-second|capacity)}, default ON / 86400s / 1000 + * entries (matching {@code IcebergPartitionCache}'s {@code DEFAULT_TABLE_CACHE_CAPACITY}). {@code enable=false} / + * {@code ttl-second=0} / {@code capacity=0} each disable the cache (see {@link CacheSpec#isCacheEnabled}): {@link * #get} then calls the loader on every call, matching {@code IcebergPartitionCache}'s disabled-cache bypass. * *

Concurrency: mirrors {@code IcebergPartitionCache} / {@code MaxComputePartitionCache} exactly — the @@ -43,31 +43,32 @@ * load, so a slow remote enumeration runs OUTSIDE Caffeine's compute lock (deduplicated per key by a striped lock) * on the calling thread, and its exception propagates to the caller unwrapped without poisoning the cache. */ -public final class ConnectorPartitionViewCache { +public final class ConnectorMetadataCache { - /** {@code meta.cache..partition_view.*} — the property-namespace entry name for this cache. */ - static final String ENTRY_PARTITION_VIEW = "partition_view"; /** Default TTL: 24h, matching the design doc's stated default and sibling caches' 24h TTL. */ static final long DEFAULT_TTL_SECOND = 86400L; /** Default capacity, matching {@code IcebergPartitionCache}'s {@code DEFAULT_TABLE_CACHE_CAPACITY}. */ static final long DEFAULT_CAPACITY = 1000L; - private final MetaCacheEntry entry; + private final MetaCacheEntry entry; /** - * @param engine engine token for the {@code meta.cache..partition_view.*} property namespace, e.g. - * {@code "iceberg"}/{@code "paimon"}/{@code "hive"}/{@code "max_compute"}. - * @param props the catalog properties; drives the {@link CacheSpec} (enable/ttl-second/capacity). May be - * {@code null}, treated as empty (defaults apply). + * @param engine engine token for the {@code meta.cache...*} property namespace, e.g. + * {@code "iceberg"}/{@code "paimon"}/{@code "hive"}/{@code "max_compute"}. + * @param entryName the entry name within that namespace (e.g. {@code "partition_view"}); a connector may hold + * several {@code ConnectorMetadataCache}s under distinct entry names. + * @param props the catalog properties; drives the {@link CacheSpec} (enable/ttl-second/capacity). May be + * {@code null}, treated as empty (defaults apply). */ - public ConnectorPartitionViewCache(String engine, Map props) { + public ConnectorMetadataCache(String engine, String entryName, Map props) { Objects.requireNonNull(engine, "engine can not be null"); + Objects.requireNonNull(entryName, "entryName can not be null"); Map properties = props == null ? Collections.emptyMap() : props; - CacheSpec spec = CacheSpec.fromProperties(properties, engine, ENTRY_PARTITION_VIEW, + CacheSpec spec = CacheSpec.fromProperties(properties, engine, entryName, CacheSpec.of(true, DEFAULT_TTL_SECOND, DEFAULT_CAPACITY)); // contextual-only (loader == null, supplied per-call by get()) + manual-miss-load, no auto-refresh -- // identical shape to IcebergPartitionCache / MaxComputePartitionCache's entry construction. - this.entry = new MetaCacheEntry<>(engine + "." + ENTRY_PARTITION_VIEW, null, spec, + this.entry = new MetaCacheEntry<>(engine + "." + entryName, null, spec, ForkJoinPool.commonPool(), false, true, 0L, true); } @@ -81,7 +82,7 @@ public boolean isEnabled() { * Disabled cache -> {@code loader} runs on every call. The loader runs OUTSIDE Caffeine's compute lock * (single-flight per key) and its exception propagates unwrapped; a failed load is never cached. */ - public V get(PartitionViewCacheKey key, Supplier loader) { + public V get(ConnectorTableKey key, Supplier loader) { Objects.requireNonNull(loader, "loader can not be null"); return entry.get(key, ignored -> loader.get()); } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/PartitionViewCacheKey.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorTableKey.java similarity index 84% rename from fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/PartitionViewCacheKey.java rename to fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorTableKey.java index f00d39465475a6..41ac71c9930240 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/PartitionViewCacheKey.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorTableKey.java @@ -20,7 +20,7 @@ import java.util.Objects; /** - * Immutable cache key for {@link ConnectorPartitionViewCache}: {@code (db, table, snapshotId, schemaId)}. + * Immutable cache key for {@link ConnectorMetadataCache}: {@code (db, table, snapshotId, schemaId)}. * *

Engine-agnostic (external-partition-derived-cache design doc §5, "cache A"): a table's derived partition * view is a pure function of its identity plus the MVCC coordinate it was read at, so pinning that coordinate @@ -28,18 +28,18 @@ * hit. Non-MVCC engines (hive) or engines without a separate schema version pass {@code snapshotId = -1} / * {@code schemaId = -1}; the key still holds them, it just means "unversioned" for that axis. * - *

{@link #matches} / {@link #matchesDb} back {@link ConnectorPartitionViewCache#invalidateTable} / - * {@link ConnectorPartitionViewCache#invalidateDb}, which must drop every snapshot/schema of a (db, table) or + *

{@link #matches} / {@link #matchesDb} back {@link ConnectorMetadataCache#invalidateTable} / + * {@link ConnectorMetadataCache#invalidateDb}, which must drop every snapshot/schema of a (db, table) or * every table of a db — mirrors the {@code matches}/{@code matchesDb} helpers on the sibling connector caches * ({@code MaxComputePartitionCache.PartitionKey}, {@code HiveFileListingCache.FileListingKey}). */ -public final class PartitionViewCacheKey { +public final class ConnectorTableKey { private final String db; private final String table; private final long snapshotId; private final long schemaId; - public PartitionViewCacheKey(String db, String table, long snapshotId, long schemaId) { + public ConnectorTableKey(String db, String table, long snapshotId, long schemaId) { this.db = db; this.table = table; this.snapshotId = snapshotId; @@ -77,10 +77,10 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (!(o instanceof PartitionViewCacheKey)) { + if (!(o instanceof ConnectorTableKey)) { return false; } - PartitionViewCacheKey that = (PartitionViewCacheKey) o; + ConnectorTableKey that = (ConnectorTableKey) o; return snapshotId == that.snapshotId && schemaId == that.schemaId && Objects.equals(db, that.db) @@ -94,7 +94,7 @@ public int hashCode() { @Override public String toString() { - return "PartitionViewCacheKey{db=" + db + ", table=" + table + return "ConnectorTableKey{db=" + db + ", table=" + table + ", snapshotId=" + snapshotId + ", schemaId=" + schemaId + '}'; } } diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java index 3e918bd80e811a..276735b40c2f9b 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java @@ -188,6 +188,36 @@ public void ofSemantics() { Assertions.assertFalse(disabled.isEnable()); } + @Test + public void ofConnectorTtlFoldsNonPositiveToDisabled() { + // A positive ttl passes through unchanged and stays enabled. + CacheSpec positive = CacheSpec.ofConnectorTtl(60, 100); + Assertions.assertTrue(positive.isEnable()); + Assertions.assertEquals(60, positive.getTtlSecond()); + Assertions.assertEquals(100, positive.getCapacity()); + Assertions.assertTrue(CacheSpec.isCacheEnabled( + positive.isEnable(), positive.getTtlSecond(), positive.getCapacity())); + + // ttl == 0 is already the disable sentinel and stays disabled. + CacheSpec zero = CacheSpec.ofConnectorTtl(0, 100); + Assertions.assertEquals(CacheSpec.CACHE_TTL_DISABLE_CACHE, zero.getTtlSecond()); + Assertions.assertFalse(CacheSpec.isCacheEnabled( + zero.isEnable(), zero.getTtlSecond(), zero.getCapacity())); + + // The load-bearing case: a NEGATIVE ttl must FOLD to the disable sentinel (0), NOT pass through as the + // -1 "no expiration (enabled)" sentinel. Without the fold, a negative operator value would silently + // produce a never-expiring cache -- the exact bug this factory replaces the per-cache ternary to avoid. + CacheSpec minusOne = CacheSpec.ofConnectorTtl(CacheSpec.CACHE_NO_TTL, 100); + Assertions.assertEquals(CacheSpec.CACHE_TTL_DISABLE_CACHE, minusOne.getTtlSecond()); + Assertions.assertFalse(CacheSpec.isCacheEnabled( + minusOne.isEnable(), minusOne.getTtlSecond(), minusOne.getCapacity())); + + CacheSpec minusTwo = CacheSpec.ofConnectorTtl(-2, 100); + Assertions.assertEquals(CacheSpec.CACHE_TTL_DISABLE_CACHE, minusTwo.getTtlSecond()); + Assertions.assertFalse(CacheSpec.isCacheEnabled( + minusTwo.isEnable(), minusTwo.getTtlSecond(), minusTwo.getCapacity())); + } + @Test public void isCacheEnabledMatchesLegacyFormula() { Assertions.assertTrue(CacheSpec.isCacheEnabled(true, CacheSpec.CACHE_NO_TTL, 1)); diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ConnectorPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ConnectorMetadataCacheTest.java similarity index 89% rename from fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ConnectorPartitionViewCacheTest.java rename to fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ConnectorMetadataCacheTest.java index 95a80e5d6f164d..8a6c1596fccad4 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ConnectorPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ConnectorMetadataCacheTest.java @@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** - * Unit tests for {@link ConnectorPartitionViewCache} — the GENERIC (engine-agnostic) cache A of the + * Unit tests for {@link ConnectorMetadataCache} — the GENERIC (engine-agnostic) cache A of the * external-partition-derived-cache design (design doc {@code 2026-07-20-external-partition-derived-cache-design.md} * §5). Mirrors {@link org.apache.doris.connector.iceberg.IcebergPartitionCache}'s test shape (that class is the * iceberg-specific ancestor this generic framework is modeled on), but with a generic string value type ({@code V}) @@ -33,22 +33,22 @@ * per-table / per-db / whole-cache invalidation, and the disabled-cache bypass (both {@code enable=false} and * {@code ttl-second=0}). */ -public class ConnectorPartitionViewCacheTest { +public class ConnectorMetadataCacheTest { private static final String ENGINE = "testengine"; - private static PartitionViewCacheKey key(String db, String table, long snapshotId, long schemaId) { - return new PartitionViewCacheKey(db, table, snapshotId, schemaId); + private static ConnectorTableKey key(String db, String table, long snapshotId, long schemaId) { + return new ConnectorTableKey(db, table, snapshotId, schemaId); } - private static ConnectorPartitionViewCache newCache() { - return new ConnectorPartitionViewCache<>(ENGINE, new HashMap<>()); + private static ConnectorMetadataCache newCache() { + return new ConnectorMetadataCache<>(ENGINE, "partition_view", new HashMap<>()); } @Test public void missThenHitLoaderRunsOnceWithinTtl() { AtomicInteger loads = new AtomicInteger(); - ConnectorPartitionViewCache cache = newCache(); + ConnectorMetadataCache cache = newCache(); String first = cache.get(key("db", "t", 5L, 1L), () -> { loads.incrementAndGet(); @@ -71,7 +71,7 @@ public void missThenHitLoaderRunsOnceWithinTtl() { @Test public void differentSnapshotIdIsADistinctEntry() { AtomicInteger loads = new AtomicInteger(); - ConnectorPartitionViewCache cache = newCache(); + ConnectorMetadataCache cache = newCache(); cache.get(key("db", "t", 1L, 1L), () -> { loads.incrementAndGet(); @@ -91,7 +91,7 @@ public void differentSnapshotIdIsADistinctEntry() { @Test public void differentSchemaIdIsADistinctEntry() { AtomicInteger loads = new AtomicInteger(); - ConnectorPartitionViewCache cache = newCache(); + ConnectorMetadataCache cache = newCache(); cache.get(key("db", "t", 1L, 1L), () -> { loads.incrementAndGet(); @@ -111,7 +111,7 @@ public void differentSchemaIdIsADistinctEntry() { @Test public void invalidateTableEvictsAllSnapshotsOfThatTableOnly() { AtomicInteger loads = new AtomicInteger(); - ConnectorPartitionViewCache cache = newCache(); + ConnectorMetadataCache cache = newCache(); cache.get(key("db", "t", 1L, 1L), () -> "v1"); cache.get(key("db", "t", 2L, 1L), () -> "v2"); @@ -142,7 +142,7 @@ public void invalidateTableEvictsAllSnapshotsOfThatTableOnly() { @Test public void invalidateDbClearsOnlyThatDbsTables() { AtomicInteger loads = new AtomicInteger(); - ConnectorPartitionViewCache cache = newCache(); + ConnectorMetadataCache cache = newCache(); cache.get(key("db1", "t1", 1L, 1L), () -> "a"); cache.get(key("db1", "t2", 1L, 1L), () -> "b"); @@ -171,7 +171,7 @@ public void invalidateDbClearsOnlyThatDbsTables() { @Test public void invalidateAllClearsEveryEntry() { AtomicInteger loads = new AtomicInteger(); - ConnectorPartitionViewCache cache = newCache(); + ConnectorMetadataCache cache = newCache(); cache.get(key("db1", "t1", 1L, 1L), () -> "a"); cache.get(key("db2", "t2", 1L, 1L), () -> "b"); @@ -191,7 +191,7 @@ public void enableFalseDisablesCacheAlwaysLoads() { AtomicInteger loads = new AtomicInteger(); Map props = new HashMap<>(); props.put("meta.cache." + ENGINE + ".partition_view.enable", "false"); - ConnectorPartitionViewCache cache = new ConnectorPartitionViewCache<>(ENGINE, props); + ConnectorMetadataCache cache = new ConnectorMetadataCache<>(ENGINE, "partition_view", props); String first = cache.get(key("db", "t", 5L, 1L), () -> { loads.incrementAndGet(); @@ -215,7 +215,7 @@ public void ttlZeroDisablesCacheAlwaysLoads() { AtomicInteger loads = new AtomicInteger(); Map props = new HashMap<>(); props.put("meta.cache." + ENGINE + ".partition_view.ttl-second", "0"); - ConnectorPartitionViewCache cache = new ConnectorPartitionViewCache<>(ENGINE, props); + ConnectorMetadataCache cache = new ConnectorMetadataCache<>(ENGINE, "partition_view", props); cache.get(key("db", "t", 5L, 1L), () -> { loads.incrementAndGet(); @@ -238,7 +238,7 @@ public void capacityZeroDisablesCacheAlwaysLoads() { AtomicInteger loads = new AtomicInteger(); Map props = new HashMap<>(); props.put("meta.cache." + ENGINE + ".partition_view.capacity", "0"); - ConnectorPartitionViewCache cache = new ConnectorPartitionViewCache<>(ENGINE, props); + ConnectorMetadataCache cache = new ConnectorMetadataCache<>(ENGINE, "partition_view", props); cache.get(key("db", "t", 5L, 1L), () -> { loads.incrementAndGet(); @@ -259,13 +259,13 @@ public void defaultsAreEnabledWithNoProperties() { // No meta.cache..partition_view.* properties set at all -> the built-in default (TTL 86400s, // capacity 1000, ON) applies, matching IcebergPartitionCache's DEFAULT_TABLE_CACHE_CAPACITY. MUTATION: // defaulting to disabled -> isEnabled() would be false here. - ConnectorPartitionViewCache cache = new ConnectorPartitionViewCache<>(ENGINE, new HashMap<>()); + ConnectorMetadataCache cache = new ConnectorMetadataCache<>(ENGINE, "partition_view", new HashMap<>()); Assertions.assertTrue(cache.isEnabled(), "the cache must be ON by default with no override properties"); } @Test public void loaderExceptionPropagatesUnwrappedAndIsNotCached() { - ConnectorPartitionViewCache cache = newCache(); + ConnectorMetadataCache cache = newCache(); // A failed load (e.g. a remote enumeration RPC failure) must propagate to the caller verbatim and must // NOT poison the cache -- a transient failure should not make every subsequent query fail for the TTL. Assertions.assertThrows(IllegalStateException.class, () -> cache.get(key("db", "t", 5L, 1L), () -> { diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java index 7f62f14aa7bf79..2db6f8d643cda8 100644 --- a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; /** * Metadata operations for Elasticsearch connector. @@ -44,6 +45,14 @@ public class EsConnectorMetadata implements ConnectorMetadata { private final EsConnectorRestClient restClient; private final Map properties; + // ES-F3 per-statement schema memo. This metadata instance is created fresh per statement + // (funnel-memoized one-per-statement), so an index's mapping is resolved into columns once and + // reused, collapsing the repeated getColumnHandles->getTableSchema remote mapping fetches to one + // per index per statement. Read-only metadata -> no in-statement invalidation. ConcurrentHashMap + // to match the maxcompute handle-memo precedent (cheap defensiveness against any concurrent + // metadata access within a statement). + private final Map schemaMemo = new ConcurrentHashMap<>(); + public EsConnectorMetadata(EsConnectorRestClient restClient, Map properties) { this.restClient = restClient; @@ -82,15 +91,21 @@ public ConnectorTableSchema getTableSchema( ConnectorSession session, ConnectorTableHandle handle) { EsTableHandle esHandle = (EsTableHandle) handle; String indexName = esHandle.getIndexName(); - String mapping = restClient.getMapping(indexName); - boolean mappingEsId = Boolean.parseBoolean(properties.getOrDefault( - EsConnectorProperties.MAPPING_ES_ID, - EsConnectorProperties.MAPPING_ES_ID_DEFAULT)); - - List columns = EsTypeMapping.parseMapping( - indexName, mapping, mappingEsId); - return new ConnectorTableSchema(indexName, columns, "ELASTICSEARCH", - Collections.emptyMap()); + return schemaMemo.computeIfAbsent(indexName, idx -> { + // Share the raw mapping with the scan path via the per-statement scope (ES-F2): one + // getMapping per index per statement across both paths. The schema memo above still + // collapses repeat getTableSchema calls within this metadata instance. + String mapping = EsStatementScope.sharedIndexMapping( + session, idx, () -> restClient.getMapping(idx)); + boolean mappingEsId = Boolean.parseBoolean(properties.getOrDefault( + EsConnectorProperties.MAPPING_ES_ID, + EsConnectorProperties.MAPPING_ES_ID_DEFAULT)); + + List columns = EsTypeMapping.parseMapping( + idx, mapping, mappingEsId); + return new ConnectorTableSchema(idx, columns, "ELASTICSEARCH", + Collections.emptyMap()); + }); } @Override @@ -138,7 +153,8 @@ public org.apache.doris.thrift.TTableDescriptor buildTableDescriptor( * @param columnNames column names to resolve field contexts for * @return fully populated EsMetadataState */ - public EsMetadataState fetchMetadataState(String indexName, List columnNames) { + public EsMetadataState fetchMetadataState(ConnectorSession session, String indexName, + List columnNames) { String mappingType = properties.getOrDefault( EsConnectorProperties.MAPPING_TYPE, null); boolean nodesDiscovery = Boolean.parseBoolean(properties.getOrDefault( @@ -149,7 +165,7 @@ public EsMetadataState fetchMetadataState(String indexName, List columnN EsMetadataState state = new EsMetadataState( indexName, mappingType, columnNames, nodesDiscovery, seeds); - EsMetadataFetcher fetcher = new EsMetadataFetcher(restClient, state); + EsMetadataFetcher fetcher = new EsMetadataFetcher(restClient, state, session); return fetcher.fetch(); } @@ -164,6 +180,6 @@ public EsMetadataState fetchMetadataState(ConnectorSession session, columnNames.add(col.getName()); } EsTableHandle esHandle = (EsTableHandle) handle; - return fetchMetadataState(esHandle.getIndexName(), columnNames); + return fetchMetadataState(session, esHandle.getIndexName(), columnNames); } } diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsMetadataFetcher.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsMetadataFetcher.java index f215a8125bea68..6e569abfd139f8 100644 --- a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsMetadataFetcher.java +++ b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsMetadataFetcher.java @@ -17,6 +17,8 @@ package org.apache.doris.connector.es; +import org.apache.doris.connector.api.ConnectorSession; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -39,10 +41,13 @@ public class EsMetadataFetcher { private final EsConnectorRestClient restClient; private final EsMetadataState state; + private final ConnectorSession session; - public EsMetadataFetcher(EsConnectorRestClient restClient, EsMetadataState state) { + public EsMetadataFetcher(EsConnectorRestClient restClient, EsMetadataState state, + ConnectorSession session) { this.restClient = restClient; this.state = state; + this.session = session; } /** @@ -55,7 +60,12 @@ public EsMetadataState fetch() { } private void fetchMapping() { - String indexMapping = restClient.getMapping(state.getSourceIndex()); + // Share the raw index mapping with the schema path via the per-statement scope: one + // getMapping per index per statement (ES-F2). The field-context derivation below stays + // per-scan (it depends on the projected columnNames). + String indexMapping = EsStatementScope.sharedIndexMapping( + session, state.getSourceIndex(), + () -> restClient.getMapping(state.getSourceIndex())); EsFieldContext fieldContext = EsMappingUtils.resolveFieldContext( state.getColumnNames(), state.getSourceIndex(), diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java index 7b7607752ede72..0388312c17e328 100644 --- a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java @@ -76,6 +76,15 @@ public class EsScanPlanProvider implements ConnectorScanPlanProvider { private final EsConnectorRestClient restClient; private final Map properties; + // ES-F1 per-scan hoist. planScan and buildScanNodeProperties of one scan node run on the same + // per-scan-node provider instance on the synchronous FE planning thread, and each used to fetch + // the full metadata state (mapping + shard routing + node topology) independently. Memoizing the + // last resolved state lets the second call reuse the first. Guarded on (index, columns) so a + // provider reused for a different request refetches; shard routing stays per-scan (fresh) because + // the provider is discarded at scan end. Plain field: safe ONLY because ES never enters batch mode + // (no off-thread scan pool) -- make it volatile if this provider ever declares batch scan. + private EsMetadataState memoizedState; + public EsScanPlanProvider(EsConnectorRestClient restClient, Map properties) { this.restClient = restClient; @@ -96,7 +105,7 @@ public List planScan( EsTableHandle esHandle = (EsTableHandle) handle; String indexName = esHandle.getIndexName(); - EsMetadataState state = fetchMetadataState(esHandle, columns); + EsMetadataState state = fetchMetadataState(session, esHandle, columns); EsShardPartitions shardPartitions = state.getShardPartitions(); if (shardPartitions == null) { LOG.warn("No shard partitions found for index {}", indexName); @@ -149,7 +158,7 @@ public Map getScanNodeProperties( ConnectorTableHandle handle, List columns, Optional filter) { - return buildScanNodeProperties(handle, columns, filter).getProperties(); + return buildScanNodeProperties(session, handle, columns, filter).getProperties(); } @Override @@ -158,15 +167,16 @@ public ScanNodePropertiesResult getScanNodePropertiesResult( ConnectorTableHandle handle, List columns, Optional filter) { - return buildScanNodeProperties(handle, columns, filter); + return buildScanNodeProperties(session, handle, columns, filter); } private ScanNodePropertiesResult buildScanNodeProperties( + ConnectorSession session, ConnectorTableHandle handle, List columns, Optional filter) { EsTableHandle esHandle = (EsTableHandle) handle; - EsMetadataState state = fetchMetadataState(esHandle, columns); + EsMetadataState state = fetchMetadataState(session, esHandle, columns); Map nodeProps = new HashMap<>(); @@ -270,7 +280,7 @@ private EsQueryDslResult buildQueryDsl(Optional filter, likePushDown, needCompatDateFields); } - private EsMetadataState fetchMetadataState(EsTableHandle handle, + private EsMetadataState fetchMetadataState(ConnectorSession session, EsTableHandle handle, List columns) { String indexName = handle.getIndexName(); List columnNames = new ArrayList<>(); @@ -279,6 +289,12 @@ private EsMetadataState fetchMetadataState(EsTableHandle handle, columnNames.add(((NamedColumnHandle) col).getName()); } } + EsMetadataState cached = memoizedState; + if (cached != null && cached.getSourceIndex().equals(indexName) + && cached.getColumnNames().equals(columnNames)) { + return cached; + } + String mappingType = properties.getOrDefault( EsConnectorProperties.MAPPING_TYPE, null); boolean nodesDiscovery = Boolean.parseBoolean(properties.getOrDefault( @@ -289,8 +305,10 @@ private EsMetadataState fetchMetadataState(EsTableHandle handle, EsMetadataState state = new EsMetadataState( indexName, mappingType, columnNames, nodesDiscovery, seeds); - EsMetadataFetcher fetcher = new EsMetadataFetcher(restClient, state); - return fetcher.fetch(); + EsMetadataFetcher fetcher = new EsMetadataFetcher(restClient, state, session); + state = fetcher.fetch(); + memoizedState = state; + return state; } /** diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsStatementScope.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsStatementScope.java new file mode 100644 index 00000000000000..3ef6dec969873d --- /dev/null +++ b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsStatementScope.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.es; + +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorStatementScopes; + +import java.util.function.Supplier; + +/** + * Per-statement scope helper for the ES connector: shares one index's raw mapping JSON across the + * schema path ({@link EsConnectorMetadata#getTableSchema}) and the scan path + * ({@link EsMetadataFetcher}) within a single statement. Both paths independently fetched the same + * {@code getMapping} remotely and derived different products from it (columns vs field-context); + * routing both through the shared statement scope collapses those to one remote fetch per index per + * statement while each path keeps its own derivation. + * + *

Only the raw mapping — stable within a statement — is shared this way. Shard routing and node + * topology are freshness-sensitive (ES rebalances) and must stay per-scan; they are NOT shared here. + * + *

Under a {@code null} session or {@link org.apache.doris.connector.api.ConnectorStatementScope#NONE} + * (offline / no live statement) the loader runs on every call — byte-identical to fetching every time. + */ +final class EsStatementScope { + + /** + * Namespace for es's per-statement raw index-mapping memo. Source-prefixed with the connector type ("es") + * so it stays distinct across a heterogeneous gateway; see {@link ConnectorStatementScopes}. + */ + static final String INDEX_MAPPING_NAMESPACE = "es.index_mapping"; + + private EsStatementScope() { + } + + static String sharedIndexMapping(ConnectorSession session, String indexName, + Supplier loader) { + return ConnectorStatementScopes.resolveInStatement( + session, INDEX_MAPPING_NAMESPACE, + EsConnectorMetadata.DEFAULT_DB, indexName, loader); + } +} diff --git a/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsScanPlanProviderTest.java b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsScanPlanProviderTest.java index 9b204e55f213e3..3c9c24150196a5 100644 --- a/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsScanPlanProviderTest.java @@ -18,6 +18,9 @@ package org.apache.doris.connector.es; import org.apache.doris.connector.api.ConnectorContractValidator; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorStatementScope; +import org.apache.doris.connector.api.handle.NamedColumnHandle; import org.apache.doris.connector.spi.ConnectorContext; import org.junit.jupiter.api.Assertions; @@ -26,7 +29,9 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; class EsScanPlanProviderTest { @@ -38,6 +43,7 @@ static class CountingRestClient extends EsConnectorRestClient { final AtomicInteger getMappingCount = new AtomicInteger(); final AtomicInteger searchShardsCount = new AtomicInteger(); + final AtomicInteger getHttpNodesCount = new AtomicInteger(); CountingRestClient() { super(new String[]{"localhost:9200"}, null, null, false, null); @@ -46,8 +52,10 @@ static class CountingRestClient extends EsConnectorRestClient { @Override public String getMapping(String indexName) { getMappingCount.incrementAndGet(); - // Minimal valid mapping JSON for EsMappingUtils.resolveFieldContext - return "{\"" + indexName + "\":{\"mappings\":{\"properties\":{}}}}"; + // Minimal valid mapping JSON for EsMappingUtils.resolveFieldContext; two keyword fields + // so tests that project columns (a/b) resolve a field context without erroring. + return "{\"" + indexName + "\":{\"mappings\":{\"properties\":" + + "{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"}}}}}"; } @Override @@ -58,59 +66,89 @@ public EsShardPartitions searchShards(String indexName) { @Override public Map getHttpNodes() { + getHttpNodesCount.incrementAndGet(); Map nodes = new HashMap<>(); nodes.put("node1", new EsNodeInfo("node1", "localhost:9200")); return nodes; } } - private static final org.apache.doris.connector.api.ConnectorSession EMPTY_SESSION = - new org.apache.doris.connector.api.ConnectorSession() { - @Override - public String getQueryId() { - return "test-query"; - } - - @Override - public String getUser() { - return "test"; - } - - @Override - public String getTimeZone() { - return "UTC"; - } - - @Override - public String getLocale() { - return "en_US"; - } - - @Override - public long getCatalogId() { - return 0; - } - - @Override - public String getCatalogName() { - return "test"; - } - - @Override - public T getProperty(String name, Class type) { - return null; - } - - @Override - public Map getCatalogProperties() { - return Collections.emptyMap(); - } - - @Override - public Map getSessionProperties() { - return Collections.emptyMap(); - } - }; + /** + * Test session with a settable statement scope. Defaults to {@link ConnectorStatementScope#NONE} + * (the offline default); pass a real scope to exercise the per-statement cross-path mapping memo. + */ + private static final class TestSession implements ConnectorSession { + private final ConnectorStatementScope scope; + + TestSession(ConnectorStatementScope scope) { + this.scope = scope; + } + + @Override + public String getQueryId() { + return "test-query"; + } + + @Override + public String getUser() { + return "test"; + } + + @Override + public String getTimeZone() { + return "UTC"; + } + + @Override + public String getLocale() { + return "en_US"; + } + + @Override + public long getCatalogId() { + return 0; + } + + @Override + public String getCatalogName() { + return "test"; + } + + @Override + public T getProperty(String name, Class type) { + return null; + } + + @Override + public Map getCatalogProperties() { + return Collections.emptyMap(); + } + + @Override + public Map getSessionProperties() { + return Collections.emptyMap(); + } + + @Override + public ConnectorStatementScope getStatementScope() { + return scope; + } + } + + /** A live per-statement scope: a plain CHM-backed arena, like the engine's real scope. */ + private static ConnectorStatementScope liveScope() { + return new ConnectorStatementScope() { + private final Map arena = new ConcurrentHashMap<>(); + + @Override + @SuppressWarnings("unchecked") + public T computeIfAbsent(String key, Supplier loader) { + return (T) arena.computeIfAbsent(key, k -> loader.get()); + } + }; + } + + private static final ConnectorSession EMPTY_SESSION = new TestSession(ConnectorStatementScope.NONE); private static Map minimalProps() { Map props = new HashMap<>(); @@ -119,7 +157,7 @@ private static Map minimalProps() { } @Test - void testPlanScanAndScanNodePropertiesFetchIndependently() { + void testPlanScanAndScanNodePropertiesShareOneFetch() { CountingRestClient client = new CountingRestClient(); EsScanPlanProvider provider = new EsScanPlanProvider(client, minimalProps()); EsTableHandle handle = new EsTableHandle("test_index"); @@ -127,10 +165,16 @@ void testPlanScanAndScanNodePropertiesFetchIndependently() { provider.planScan(EMPTY_SESSION, handle, Collections.emptyList(), java.util.Optional.empty()); provider.getScanNodeProperties(EMPTY_SESSION, handle, Collections.emptyList(), java.util.Optional.empty()); - Assertions.assertEquals(2, client.getMappingCount.get(), - "Each provider call should fetch mapping for the current request"); - Assertions.assertEquals(2, client.searchShardsCount.get(), - "Each provider call should fetch shard routing for the current request"); + // ES-F1: planScan and getScanNodeProperties of one scan node run on the SAME per-scan-node + // provider instance, so the metadata state (mapping + shard routing + node topology) is + // fetched once and shared -- not twice. MUTATION: removing the memoizedState guard makes + // each call refetch -> these go back to 2 -> red. + Assertions.assertEquals(1, client.getMappingCount.get(), + "the two provider calls of one scan node must share a single mapping fetch"); + Assertions.assertEquals(1, client.searchShardsCount.get(), + "the two provider calls of one scan node must share a single shard-routing fetch"); + Assertions.assertEquals(1, client.getHttpNodesCount.get(), + "the two provider calls of one scan node must share a single node-topology fetch"); } @Test @@ -141,10 +185,134 @@ void testDifferentIndexesFetchSeparately() { Collections.emptyList(), java.util.Optional.empty()); provider.planScan(EMPTY_SESSION, new EsTableHandle("index_b"), Collections.emptyList(), java.util.Optional.empty()); + // The per-scan memo is guarded on the index, so a provider reused for a different index + // still refetches -- distinct indexes never share a memo entry. Assertions.assertEquals(2, client.getMappingCount.get(), "Different indexes should each fetch their own metadata"); } + @Test + void testSeparateProviderInstancesEachFetchForFreshness() { + // Each scan node gets its OWN provider instance. Shard routing must stay fresh per scan + // (ES rebalances), so the memo must live on the provider and never be reused across scans. + // Two providers for the same index each fetch once -> 2 total, proving the memo is per-scan, + // never cross-query. + CountingRestClient client = new CountingRestClient(); + EsTableHandle handle = new EsTableHandle("test_index"); + + new EsScanPlanProvider(client, minimalProps()) + .planScan(EMPTY_SESSION, handle, Collections.emptyList(), java.util.Optional.empty()); + new EsScanPlanProvider(client, minimalProps()) + .planScan(EMPTY_SESSION, handle, Collections.emptyList(), java.util.Optional.empty()); + + Assertions.assertEquals(2, client.searchShardsCount.get(), + "a separate scan node (provider) must refetch shard routing -- memo is per-scan, not cross-query"); + } + + @Test + void testMetadataSchemaMemoizedPerStatement() { + // ES-F3: EsConnectorMetadata is per-statement; an index's mapping is resolved into columns + // once and reused, so getTableSchema + the getColumnHandles that re-invokes it share one + // remote mapping fetch. MUTATION: dropping the schemaMemo makes each call refetch -> 2 -> red. + CountingRestClient client = new CountingRestClient(); + EsConnectorMetadata metadata = new EsConnectorMetadata(client, minimalProps()); + EsTableHandle handle = new EsTableHandle("test_index"); + + metadata.getTableSchema(EMPTY_SESSION, handle); + metadata.getColumnHandles(EMPTY_SESSION, handle); + Assertions.assertEquals(1, client.getMappingCount.get(), + "an index's schema must be resolved once per statement and reused"); + } + + @Test + void testMetadataSchemaFreshPerStatement() { + // A fresh EsConnectorMetadata (a new statement) must re-resolve -- the schema memo is + // per-statement, never cross-query (mappings can change between statements). + CountingRestClient client = new CountingRestClient(); + EsTableHandle handle = new EsTableHandle("test_index"); + new EsConnectorMetadata(client, minimalProps()).getTableSchema(EMPTY_SESSION, handle); + new EsConnectorMetadata(client, minimalProps()).getTableSchema(EMPTY_SESSION, handle); + Assertions.assertEquals(2, client.getMappingCount.get(), + "a new statement's metadata must re-resolve the schema -- memo is per-statement"); + } + + @Test + void testMappingSharedAcrossSchemaAndScanPathsWithinStatement() { + // ES-F2: the schema path (EsConnectorMetadata) and the scan path (EsScanPlanProvider) each + // fetched the same index mapping remotely. Routed through the shared per-statement scope, one + // index's getMapping fires ONCE across both paths within a statement. MUTATION: dropping the + // EsStatementScope wrapping (or a NONE scope) makes the paths fetch independently -> 2 -> red. + CountingRestClient client = new CountingRestClient(); + ConnectorSession session = new TestSession(liveScope()); + EsTableHandle handle = new EsTableHandle("test_index"); + + new EsConnectorMetadata(client, minimalProps()).getTableSchema(session, handle); + new EsScanPlanProvider(client, minimalProps()) + .planScan(session, handle, Collections.emptyList(), java.util.Optional.empty()); + + Assertions.assertEquals(1, client.getMappingCount.get(), + "one index's mapping must be fetched once per statement across the schema and scan paths"); + } + + @Test + void testScopedMappingNotSharedAcrossStatements() { + // Two statements (distinct live scopes) must NOT share the mapping -- the scope memo is + // per-statement, never cross-query. Shard routing is likewise never placed in the scope. + CountingRestClient client = new CountingRestClient(); + EsTableHandle handle = new EsTableHandle("test_index"); + + new EsConnectorMetadata(client, minimalProps()) + .getTableSchema(new TestSession(liveScope()), handle); + new EsConnectorMetadata(client, minimalProps()) + .getTableSchema(new TestSession(liveScope()), handle); + + Assertions.assertEquals(2, client.getMappingCount.get(), + "a separate statement (scope) must re-fetch the mapping -- the scope memo is per-statement"); + } + + @Test + void testShardRoutingNeverSharedViaScope() { + // Hard freshness constraint: shard routing + node topology must NEVER be shared via the + // per-statement scope (ES rebalances) -- only the raw mapping is. Two scan providers sharing + // the SAME live statement scope and index each refetch shards/nodes, while the mapping is + // fetched once. MUTATION: routing searchShards or getHttpNodes through EsStatementScope would + // make the second scan reuse the first -> those counts drop to 1 -> red. + CountingRestClient client = new CountingRestClient(); + ConnectorSession session = new TestSession(liveScope()); + EsTableHandle handle = new EsTableHandle("test_index"); + + new EsScanPlanProvider(client, minimalProps()) + .planScan(session, handle, Collections.emptyList(), java.util.Optional.empty()); + new EsScanPlanProvider(client, minimalProps()) + .planScan(session, handle, Collections.emptyList(), java.util.Optional.empty()); + + Assertions.assertEquals(2, client.searchShardsCount.get(), + "shard routing must be fetched per scan, never shared via the statement scope"); + Assertions.assertEquals(2, client.getHttpNodesCount.get(), + "node topology must be fetched per scan, never shared via the statement scope"); + Assertions.assertEquals(1, client.getMappingCount.get(), + "the raw mapping is the only thing shared across the two scans of one statement"); + } + + @Test + void testDifferentColumnsRefetch() { + // The per-scan memo is guarded on the projected columns, not just the index, because the + // field-context depends on them; a different projection must refetch rather than return a + // stale field-context. MUTATION: dropping the columns comparison lets the second projection + // reuse the first index's state -> searchShards stays 1 -> red. + CountingRestClient client = new CountingRestClient(); + EsScanPlanProvider provider = new EsScanPlanProvider(client, minimalProps()); + EsTableHandle handle = new EsTableHandle("test_index"); + + provider.planScan(EMPTY_SESSION, handle, + Collections.singletonList(new NamedColumnHandle("a")), java.util.Optional.empty()); + provider.planScan(EMPTY_SESSION, handle, + Collections.singletonList(new NamedColumnHandle("b")), java.util.Optional.empty()); + + Assertions.assertEquals(2, client.searchShardsCount.get(), + "a different projection must refetch -- the memo is guarded on columns, not just index"); + } + @Test void testEsConnectorDoesNotSupportWrite() { EsConnector connector = new EsConnector(minimalProps(), new ConnectorContext() { diff --git a/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsStatementScopeTest.java b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsStatementScopeTest.java new file mode 100644 index 00000000000000..af679b6750e0ef --- /dev/null +++ b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsStatementScopeTest.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.es; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + +/** + * Guards that every ES statement-scope namespace follows the source-prefix norm: it is a compile-time constant + * prefixed with this connector's {@code ConnectorProvider.getType()} ("es"). Source-prefixing is what keeps the + * namespaces distinct from every other connector's on a heterogeneous gateway (no {@code ClassCastException} on the + * shared {@code (catalogId, db, table, queryId)} coordinate); the value-type home is + * {@link org.apache.doris.connector.api.ConnectorStatementScopes}. + */ +public class EsStatementScopeTest { + + @Test + public void allNamespacesArePrefixedWithConnectorType() throws Exception { + // NORM (self-extending): reflect over every "*_NAMESPACE" constant this connector declares and assert each + // is prefixed with the connector's getType() ("es."). Reflecting means a NEW namespace is auto-covered; a + // forgotten prefix or a getType() drift turns this red with no test upkeep. + String prefix = new EsConnectorProvider().getType() + "."; + int checked = 0; + for (Field f : EsStatementScope.class.getDeclaredFields()) { + if (Modifier.isStatic(f.getModifiers()) && f.getType() == String.class + && f.getName().endsWith("_NAMESPACE")) { + f.setAccessible(true); + String ns = (String) f.get(null); + Assertions.assertTrue(ns.startsWith(prefix), + f.getName() + " (\"" + ns + "\") must be prefixed with the connector type \"" + prefix + "\""); + checked++; + } + } + Assertions.assertTrue(checked > 0, "expected at least one *_NAMESPACE constant to guard"); + } +} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java index 223717a2bcb8f6..e41f981668c5cf 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java @@ -29,7 +29,7 @@ import org.apache.doris.connector.api.procedure.ConnectorProcedureOps; import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; import org.apache.doris.connector.api.write.ConnectorWritePlanProvider; -import org.apache.doris.connector.cache.ConnectorPartitionViewCache; +import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.hms.CachingHmsClient; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsClientConfig; @@ -98,7 +98,7 @@ public class HiveConnector implements Connector { // metastore-metadata sibling is the CachingHmsClient wrapping the HmsClient. private final HiveFileListingCache fileListingCache; - // PERF-06 (S6): cross-query DERIVED partition-view cache ("cache A", the generic ConnectorPartitionViewCache + // PERF-06 (S6): cross-query DERIVED partition-view cache ("cache A", the generic ConnectorMetadataCache // from fe-connector-cache), layered ABOVE the raw per-name HMS listing served by CachingHmsClient: it // memoizes the BUILT List (HiveConnectorMetadata#listPartitionsUncached's per-name // HiveWriteUtils.toPartitionValues parse + ConnectorPartitionInfo construction), keyed by @@ -109,7 +109,7 @@ public class HiveConnector implements Connector { // session=user / per-user credential-isolation cache-disabling convention (its per-catalog caches — // CachingHmsClient, HiveFileListingCache — are already built unconditionally below), so this is constructed // unconditionally too. - private final ConnectorPartitionViewCache> partitionViewCache; + private final ConnectorMetadataCache> partitionViewCache; // Embedded iceberg SIBLING connector: a flipped hms gateway delegates its iceberg-on-HMS tables to it. Built // once per gateway connector (lazily) in the iceberg plugin's OWN child-first classloader via @@ -131,7 +131,7 @@ public HiveConnector(Map properties, ConnectorContext context) { this.fileListingCache = new HiveFileListingCache(this.properties); // Reads its own meta.cache.hive.partition_view.(enable|ttl-second|capacity) from the catalog properties // via the framework's CacheSpec (default ON / 24h / 1000). - this.partitionViewCache = new ConnectorPartitionViewCache<>("hive", this.properties); + this.partitionViewCache = new ConnectorMetadataCache<>("hive", "partition_view", this.properties); } @Override @@ -495,7 +495,7 @@ HiveFileListingCache fileListingCacheForTest() { } /** Test-only: the derived listPartitions view cache (PERF-06). Never null (hive has no session=user gate). */ - ConnectorPartitionViewCache> partitionViewCacheForTest() { + ConnectorMetadataCache> partitionViewCacheForTest() { return partitionViewCache; } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java index 0db7c4445d4949..ca93cd870309e6 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java @@ -55,8 +55,8 @@ import org.apache.doris.connector.api.pushdown.ConnectorLiteral; import org.apache.doris.connector.api.pushdown.FilterApplicationResult; import org.apache.doris.connector.api.scan.ConnectorPartitionValues; -import org.apache.doris.connector.cache.ConnectorPartitionViewCache; -import org.apache.doris.connector.cache.PartitionViewCacheKey; +import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ConnectorTableKey; import org.apache.doris.connector.hms.HiveShowCreateTableRenderer; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsClientException; @@ -229,7 +229,7 @@ public class HiveConnectorMetadata implements ConnectorMetadata { // default (harmless for the direct-construction tests, which inject their file sizes and never list). private final HiveFileListingCache fileListingCache; - // PERF-06 (S6): cross-query DERIVED partition-view cache A (generic ConnectorPartitionViewCache), injected by + // PERF-06 (S6): cross-query DERIVED partition-view cache A (generic ConnectorMetadataCache), injected by // the owning HiveConnector; null = no cross-query derived layer (the convenience/test ctors below pass null, // matching every existing direct-construction test). Layered ABOVE the raw per-name HMS listing served by // CachingHmsClient: a hit skips both the derived-view BUILD (the per-name HiveWriteUtils.toPartitionValues @@ -238,7 +238,7 @@ public class HiveConnectorMetadata implements ConnectorMetadata { // handle carries no schema version, so both axes are pinned "unversioned". Consumed only by listPartitions: // getMvccPartitionView returns Optional.empty() for a real hive handle (the SPI default), so fe-core's generic // MTMV model already falls back to listPartitions for hive — there is no second enumeration hook to wrap. - private final ConnectorPartitionViewCache> partitionViewCache; + private final ConnectorMetadataCache> partitionViewCache; public HiveConnectorMetadata(HmsClient hmsClient, Map properties, ConnectorContext context) { this(hmsClient, properties, context, NO_ICEBERG_SIBLING, NO_HUDI_SIBLING, NO_SIBLING_OWNER); @@ -273,7 +273,7 @@ public HiveConnectorMetadata(HmsClient hmsClient, Map properties Supplier hudiSiblingSupplier, Function siblingOwnerResolver, HiveFileListingCache fileListingCache, - ConnectorPartitionViewCache> partitionViewCache) { + ConnectorMetadataCache> partitionViewCache) { this.hmsClient = hmsClient; this.properties = properties; this.context = context; @@ -1167,7 +1167,7 @@ public List listPartitions(ConnectorSession session, if (partitionViewCache == null || filter.isPresent()) { return listPartitionsUncached(hiveHandle); } - PartitionViewCacheKey key = new PartitionViewCacheKey( + ConnectorTableKey key = new ConnectorTableKey( hiveHandle.getDbName(), hiveHandle.getTableName(), -1L, -1L); return partitionViewCache.get(key, () -> listPartitionsUncached(hiveHandle)); } diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java index 3b375114572190..e18487005642f4 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java @@ -23,7 +23,7 @@ import org.apache.doris.connector.api.pushdown.ConnectorComparison; import org.apache.doris.connector.api.pushdown.ConnectorExpression; import org.apache.doris.connector.api.pushdown.ConnectorLiteral; -import org.apache.doris.connector.cache.ConnectorPartitionViewCache; +import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsDatabaseInfo; import org.apache.doris.connector.hms.HmsPartitionInfo; @@ -41,7 +41,7 @@ /** * PERF-06 (S6) tests for the cross-query DERIVED partition-view cache ("cache A", the generic - * {@link ConnectorPartitionViewCache}) wired into {@link HiveConnectorMetadata#listPartitions}. Hive does NOT + * {@link ConnectorMetadataCache}) wired into {@link HiveConnectorMetadata#listPartitions}. Hive does NOT * override {@code getMvccPartitionView} for a real hive handle (it returns the SPI default * {@code Optional.empty()} — fe-core's generic MTMV model already falls back to {@code listPartitions}), so — * like paimon's single typed field — there is exactly ONE enumeration hook to wrap. @@ -66,7 +66,7 @@ public class HiveConnectorMetadataPartitionViewCacheTest { "year=2024/month=01"); private static HiveConnectorMetadata metadataWithCache(CountingHmsClient client, - ConnectorPartitionViewCache> cache) { + ConnectorMetadataCache> cache) { return new HiveConnectorMetadata(client, Collections.emptyMap(), new FakeConnectorContext(), () -> { throw new UnsupportedOperationException(); @@ -80,8 +80,8 @@ private static HiveConnectorMetadata metadataWithCache(CountingHmsClient client, new HiveFileListingCache(Collections.emptyMap()), cache); } - private static ConnectorPartitionViewCache> partitionViewCache() { - return new ConnectorPartitionViewCache<>("hive", Collections.emptyMap()); + private static ConnectorMetadataCache> partitionViewCache() { + return new ConnectorMetadataCache<>("hive", "partition_view", Collections.emptyMap()); } private static HiveTableHandle handle() { @@ -101,7 +101,7 @@ public void listPartitionsCachesDerivedListAcrossQueries() { // round-trip. MUTATION: not consulting the cache (compute directly every call) -> the seam runs twice // -> red. CountingHmsClient client = new CountingHmsClient(PARTITIONS); - ConnectorPartitionViewCache> cache = partitionViewCache(); + ConnectorMetadataCache> cache = partitionViewCache(); HiveConnectorMetadata md = metadataWithCache(client, cache); HiveTableHandle h = handle(); @@ -120,7 +120,7 @@ public void listPartitionsInvalidateTableForcesReEnumeration() { // so the next query re-enumerates live. MUTATION: invalidateTable not wired -> the second call hits the // stale entry -> listPartitionNamesCalls stays 1 -> red. CountingHmsClient client = new CountingHmsClient(PARTITIONS); - ConnectorPartitionViewCache> cache = partitionViewCache(); + ConnectorMetadataCache> cache = partitionViewCache(); HiveConnectorMetadata md = metadataWithCache(client, cache); HiveTableHandle h = handle(); @@ -135,7 +135,7 @@ public void listPartitionsInvalidateAllForcesReEnumeration() { // WHY: REFRESH CATALOG (HiveConnector.invalidateAll -> cache.invalidateAll) must drop the cached list. // MUTATION: invalidateAll not wired -> the second call hits -> listPartitionNamesCalls stays 1 -> red. CountingHmsClient client = new CountingHmsClient(PARTITIONS); - ConnectorPartitionViewCache> cache = partitionViewCache(); + ConnectorMetadataCache> cache = partitionViewCache(); HiveConnectorMetadata md = metadataWithCache(client, cache); HiveTableHandle h = handle(); @@ -155,7 +155,7 @@ public void listPartitionsWithFilterBypassesCache() { // the second filtered call hits (count stays 1) -> red; or a bypassed call populating the cache -> the // following empty-filter call hits instead of missing -> red. CountingHmsClient client = new CountingHmsClient(PARTITIONS); - ConnectorPartitionViewCache> cache = partitionViewCache(); + ConnectorMetadataCache> cache = partitionViewCache(); HiveConnectorMetadata md = metadataWithCache(client, cache); HiveTableHandle h = handle(); diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorPartitionViewCacheTest.java index 57c765f9482e90..a13c072f0c26e2 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorPartitionViewCacheTest.java @@ -18,8 +18,8 @@ package org.apache.doris.connector.hive; import org.apache.doris.connector.api.ConnectorPartitionInfo; -import org.apache.doris.connector.cache.ConnectorPartitionViewCache; -import org.apache.doris.connector.cache.PartitionViewCacheKey; +import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ConnectorTableKey; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -63,16 +63,16 @@ public void refreshHooksInvalidatePartitionViewCache() { // loader must run again. MUTATION: an invalidate* hook not routed to the view cache -> the entry // survives -> loader not re-run -> a loads assert below red. HiveConnector connector = new HiveConnector(props(), new FakeConnectorContext()); - ConnectorPartitionViewCache> cache = connector.partitionViewCacheForTest(); + ConnectorMetadataCache> cache = connector.partitionViewCacheForTest(); Assertions.assertNotNull(cache); int[] loads = {0}; Supplier> loader = () -> { loads[0]++; return Collections.emptyList(); }; - PartitionViewCacheKey db1t1 = new PartitionViewCacheKey("db1", "t1", -1L, -1L); - PartitionViewCacheKey db1t2 = new PartitionViewCacheKey("db1", "t2", -1L, -1L); - PartitionViewCacheKey db2t1 = new PartitionViewCacheKey("db2", "t1", -1L, -1L); + ConnectorTableKey db1t1 = new ConnectorTableKey("db1", "t1", -1L, -1L); + ConnectorTableKey db1t2 = new ConnectorTableKey("db1", "t2", -1L, -1L); + ConnectorTableKey db2t1 = new ConnectorTableKey("db2", "t1", -1L, -1L); // REFRESH TABLE db1.t1 -> only db1.t1 re-loads. Uses the public no-client hook (mirrors a never-scanned // catalog's REFRESH — hmsClient is null here). @@ -108,14 +108,14 @@ public void invalidatePartitionDropsTheWholeTablesCachedView() { // it must invalidate that whole entry. MUTATION: invalidatePartition not routed to the view cache -> the // stale entry survives -> the reload assert below red. HiveConnector connector = new HiveConnector(props(), new FakeConnectorContext()); - ConnectorPartitionViewCache> cache = connector.partitionViewCacheForTest(); + ConnectorMetadataCache> cache = connector.partitionViewCacheForTest(); int[] loads = {0}; Supplier> loader = () -> { loads[0]++; return Collections.emptyList(); }; - PartitionViewCacheKey db1t1 = new PartitionViewCacheKey("db1", "t1", -1L, -1L); - PartitionViewCacheKey db1t2 = new PartitionViewCacheKey("db1", "t2", -1L, -1L); + ConnectorTableKey db1t1 = new ConnectorTableKey("db1", "t1", -1L, -1L); + ConnectorTableKey db1t2 = new ConnectorTableKey("db1", "t2", -1L, -1L); cache.get(db1t1, loader); cache.get(db1t2, loader); diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java index f00bcbc1fe56c0..185acd5437b716 100644 --- a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java @@ -23,6 +23,7 @@ import org.apache.doris.connector.api.DorisConnectorException; import org.apache.doris.connector.api.handle.ConnectorTableHandle; import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.hms.CachingHmsClient; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsClientConfig; import org.apache.doris.connector.hms.ThriftHmsClient; @@ -137,6 +138,60 @@ public ConnectorScanPlanProvider getScanPlanProvider() { return new HudiScanPlanProvider(properties, context); } + /** + * REFRESH TABLE hook: flush this table's cached HMS metadata ({@link CachingHmsClient#flush}: table info + + * partition names) so the next query re-reads it live. Reads the client field WITHOUT building it + * (getOrCreateClient would force a real client just to flush an empty cache; a never-queried catalog has no + * cache to flush). hudi is a leaf sibling (no siblings of its own) holding no file/partition-view caches, so + * the metastore flush is the only layer. The hive gateway forwards REFRESH to this sibling via + * {@code forEachBuiltSibling}, so this override is what makes REFRESH reach the sibling's own client + * (fe-core routes REFRESH TABLE to {@code connector.invalidateTable} for a plugin-driven catalog). + */ + @Override + public void invalidateTable(String dbName, String tableName) { + invalidateTable(hmsClient, dbName, tableName); + } + + // Package-private seam: a unit test can pass an observable CachingHmsClient (the hmsClient field is + // otherwise only set by getOrCreateClient building a real pooled client). + void invalidateTable(HmsClient client, String dbName, String tableName) { + if (client instanceof CachingHmsClient) { + ((CachingHmsClient) client).flush(dbName, tableName); + } + } + + /** + * REFRESH DATABASE hook: flush every cached table in this database ({@link CachingHmsClient#flushDb}). Same + * no-force-build read of the client as {@link #invalidateTable(String, String)}. + */ + @Override + public void invalidateDb(String dbName) { + invalidateDb(hmsClient, dbName); + } + + // Package-private seam (see invalidateTable above). + void invalidateDb(HmsClient client, String dbName) { + if (client instanceof CachingHmsClient) { + ((CachingHmsClient) client).flushDb(dbName); + } + } + + /** + * REFRESH CATALOG hook: flush this catalog's entire HMS metadata cache ({@link CachingHmsClient#flushAll}). + * Same no-force-build read of the client as {@link #invalidateTable(String, String)}. + */ + @Override + public void invalidateAll() { + invalidateAll(hmsClient); + } + + // Package-private seam (see invalidateTable above). + void invalidateAll(HmsClient client) { + if (client instanceof CachingHmsClient) { + ((CachingHmsClient) client).flushAll(); + } + } + private HmsClient getOrCreateClient() { if (hmsClient == null) { synchronized (this) { @@ -183,7 +238,21 @@ public T execute(Callable callable) throws Exception { } else { authAction = context::executeAuthenticated; } - return new ThriftHmsClient(config, authAction); + return wrapWithCache(new ThriftHmsClient(config, authAction)); + } + + /** + * Wraps the raw pooled client in the shared {@link CachingHmsClient} (mirrors {@code HiveConnector}): + * {@code getTable} and {@code listPartitionNames} become {@code (db,table)}-keyed and TTL-bounded + * ({@code meta.cache.hive.*}, default 24h), so repeated queries against the same hudi table stop re-hitting + * HMS; {@code tableExists}/{@code listTables} stay pass-through. Freshness is preserved two ways: the + * SHOW-PARTITIONS / {@code partition_values} path lists FRESH (bypasses the cache — see + * {@link HudiConnectorMetadata}{@code .collectPartitions}), and REFRESH flushes it (see + * {@link #invalidateTable(String, String)}). Package-private so a unit test can wrap an observable fake and + * assert the cache decoration. + */ + HmsClient wrapWithCache(HmsClient raw) { + return new CachingHmsClient(raw, properties); } /** diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorMetadata.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorMetadata.java index b4af4f8f2e0249..d443fab976c140 100644 --- a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorMetadata.java @@ -317,7 +317,7 @@ public ConnectorTableSchema getTableSchema( ConnectorSession session, ConnectorTableHandle handle) { // Latest schema (no time-travel pin). Shares the single build path with the at-instant overload below // (null instant = latest) so the two can never drift. - return buildTableSchema((HudiTableHandle) handle, null); + return buildTableSchema(session, (HudiTableHandle) handle, null); } /** @@ -336,30 +336,51 @@ public ConnectorTableSchema getTableSchema( public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle, ConnectorMvccSnapshot snapshot) { HudiTableHandle hudiHandle = (HudiTableHandle) handle; - return buildTableSchema(hudiHandle, hudiHandle.getQueryInstant()); + return buildTableSchema(session, hudiHandle, hudiHandle.getQueryInstant()); } /** - * Single build path for {@link #getTableSchema}: reads the columns from the Hudi metaClient AS OF - * {@code queryInstant} ({@code null} = latest) and wraps them in a {@link ConnectorTableSchema}. Shared by - * the latest (2-arg) and at-instant (3-arg) entry points so they cannot diverge. + * Single build path for {@link #getTableSchema}: reads the columns AS OF {@code queryInstant} + * ({@code null} = latest) and wraps them in a {@link ConnectorTableSchema}. Shared by the latest (2-arg) and + * at-instant (3-arg) entry points so they cannot diverge. + * + *

The latest branch ({@code queryInstant == null}) resolves the columns through the per-statement latest-schema + * memo ({@link HudiStatementScope#sharedLatestColumns}), so repeated latest reads in one statement share a single + * {@link #getSchemaFromMetaClient} call; value and failure semantics are byte-identical to the un-memoized read + * (that method already swallows a failure to an empty list / BE BY_NAME). The at-instant branch + * ({@code FOR TIME AS OF}) stays on {@link #getSchemaFromMetaClient}, UNMEMOIZED: the memo key does not carry the + * instant, so a latest read and an at-instant read of the same table in one statement must not share it.

*/ - private ConnectorTableSchema buildTableSchema(HudiTableHandle hudiHandle, String queryInstant) { + private ConnectorTableSchema buildTableSchema(ConnectorSession session, HudiTableHandle hudiHandle, + String queryInstant) { String basePath = hudiHandle.getBasePath(); List columns; if (basePath != null && !basePath.isEmpty()) { - columns = getSchemaFromMetaClient(basePath, queryInstant); + if (queryInstant == null) { + columns = HudiStatementScope.sharedLatestColumns(session, hudiHandle.getDbName(), + hudiHandle.getTableName(), () -> getSchemaFromMetaClient(basePath, null)); + } else { + columns = getSchemaFromMetaClient(basePath, queryInstant); + } } else { columns = getSchemaFromHms(hudiHandle.getDbName(), hudiHandle.getTableName()); } + return assembleTableSchema(hudiHandle, columns); + } + + /** + * Wraps the resolved columns in a {@link ConnectorTableSchema}, stamping the hudi table-type and the + * partition-column marker fe-core needs to model the table as partitioned (parity with the OLD + * HMSExternalTable/HudiDlaTable path, which read the HMS partition keys). The keys already live on the handle + * (getTableHandle -> tableInfo.getPartitionKeys()); emit them as the RAW-name CSV, matching the schema column + * names (the partition columns are appended to {@code columns} by both schema sources). Shared by the memoized + * latest path and the at-instant {@link #buildTableSchema} path. + */ + private ConnectorTableSchema assembleTableSchema(HudiTableHandle hudiHandle, List columns) { Map tableProperties = new HashMap<>(); tableProperties.put("hudi.table.type", hudiHandle.getHudiTableType()); - // Stamp the partition-column marker fe-core needs to model the table as partitioned (parity with the OLD - // HMSExternalTable/HudiDlaTable path, which read the HMS partition keys). The keys already live on the - // handle (getTableHandle -> tableInfo.getPartitionKeys()); emit them as the RAW-name CSV, matching the - // schema column names (the partition columns are appended to `columns` by both schema sources). List partitionKeyNames = hudiHandle.getPartitionKeyNames(); if (partitionKeyNames != null && !partitionKeyNames.isEmpty()) { tableProperties.put(PARTITION_COLUMNS_PROPERTY, String.join(",", partitionKeyNames)); @@ -389,8 +410,8 @@ public Map getProperties() { * default delegates here). It is deliberately NOT placed on {@code getWritePlanProvider}: the admission gate * calls {@code getWritePlanProvider} to DECIDE, so throwing there would make the gate throw a {@code * DorisConnectorException} the engine misclassifies as an internal error instead of the clean "does not - * support INSERT" — keep the provider {@code null} and reject explicitly only here (dormant until hms enters - * {@code SPI_READY_TYPES}). + * support INSERT" — keep the provider {@code null} and reject explicitly only here (with hms live, a user + * write against a hudi table reaches this reject on the gateway path). */ @Override public ConnectorTransaction beginTransaction(ConnectorSession session) { @@ -415,7 +436,11 @@ public ConnectorTransaction beginTransaction(ConnectorSession session) { @Override public Optional beginQuerySnapshot( ConnectorSession session, ConnectorTableHandle handle) { - return buildBeginQuerySnapshot(latestInstant((HudiTableHandle) handle)); + HudiTableHandle hudiHandle = (HudiTableHandle) handle; + // Memoized per statement so repeated snapshot pins share one instant read; the loader IS latestInstant, so + // the value (and its fail-loud-on-error semantics) is byte-identical to the un-memoized read. + return buildBeginQuerySnapshot(HudiStatementScope.sharedLatestInstant( + session, hudiHandle.getDbName(), hudiHandle.getTableName(), () -> latestInstant(hudiHandle))); } /** Builds the query-begin snapshot from a pinned instant. Static for offline unit testing. */ @@ -645,12 +670,15 @@ public List getSyntheticScanPredicates(ConnectorSession ses @Override public List listPartitions(ConnectorSession session, ConnectorTableHandle handle, Optional filter) { - return collectPartitions((HudiTableHandle) handle); + // Query-pruning / MTMV enumeration: read the cache (bypassCache=false). + return collectPartitions((HudiTableHandle) handle, false); } @Override public List listPartitionNames(ConnectorSession session, ConnectorTableHandle handle) { - List partitions = collectPartitions((HudiTableHandle) handle); + // SHOW PARTITIONS / partitions() TVF: list FRESH (bypassCache=true) so a newly hive-synced partition is + // visible immediately, not up to a TTL later. + List partitions = collectPartitions((HudiTableHandle) handle, true); List names = new ArrayList<>(partitions.size()); for (ConnectorPartitionInfo partition : partitions) { names.add(partition.getPartitionName()); @@ -661,7 +689,8 @@ public List listPartitionNames(ConnectorSession session, ConnectorTableH @Override public List> listPartitionValues(ConnectorSession session, ConnectorTableHandle handle, List partitionColumns) { - List partitions = collectPartitions((HudiTableHandle) handle); + // partition_values() TVF (user-facing enumeration): list FRESH (bypassCache=true), like listPartitionNames. + List partitions = collectPartitions((HudiTableHandle) handle, true); List> result = new ArrayList<>(partitions.size()); for (ConnectorPartitionInfo partition : partitions) { Map rawValues = partition.getPartitionValues(); @@ -684,7 +713,7 @@ public List> listPartitionValues(ConnectorSession session, * partition. Unpartitioned → {@code emptyList()} (legacy never lists partitions for an unpartitioned * table). Explicit time-travel (non-latest) partition listing is a later step. */ - private List collectPartitions(HudiTableHandle handle) { + private List collectPartitions(HudiTableHandle handle, boolean bypassCache) { List partKeyNames = handle.getPartitionKeyNames(); if (partKeyNames == null || partKeyNames.isEmpty()) { return Collections.emptyList(); @@ -693,8 +722,13 @@ private List collectPartitions(HudiTableHandle handle) { // hive-sync tables register their partitions in HMS: list the names from there (already authed via // hmsClient, no metaClient), like legacy. The instant still comes from the timeline. If HMS has none // (a hive-sync table not yet synced), fall back to the hudi metadata listing (legacy parity). - List hmsNames = hmsClient.listPartitionNames( - handle.getDbName(), handle.getTableName(), -1); + // bypassCache selects the freshness contract (mirrors HiveConnectorMetadata.collectPartitionNames): + // the SHOW-PARTITIONS / partition_values-TVF path (listPartitionNames / listPartitionValues) lists + // FRESH — legacy read the raw pooled client, so an externally hive-synced partition must not stay + // invisible until TTL/REFRESH — while the query-pruning / MTMV path (listPartitions) reads the cache. + List hmsNames = bypassCache + ? hmsClient.listPartitionNamesFresh(handle.getDbName(), handle.getTableName(), -1) + : hmsClient.listPartitionNames(handle.getDbName(), handle.getTableName(), -1); if (hmsNames != null && !hmsNames.isEmpty()) { return buildPartitionInfos(hmsNames, partKeyNames, latestInstant(handle)); } @@ -744,8 +778,11 @@ static List buildPartitionInfos( return result; } - /** Pins the latest completed instant, building the metaClient under the plugin auth + TCCL pin. */ - private long latestInstant(HudiTableHandle handle) { + /** + * Pins the latest completed instant, building the metaClient under the plugin auth + TCCL pin. Package-private so + * a same-loader unit test can override it to count instant reads without a live metaClient. + */ + long latestInstant(HudiTableHandle handle) { return metaClientExecutor.execute(() -> HudiScanPlanProvider.latestCompletedInstant( HudiScanPlanProvider.buildMetaClient(buildHadoopConf(), handle.getBasePath()))); diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiScanPlanProvider.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiScanPlanProvider.java index 9d65299509ea91..d115722773e79a 100644 --- a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiScanPlanProvider.java @@ -247,7 +247,7 @@ public List planScan( // window onto the handle). Select the files the window touches via the ported IncrementalRelation family // instead of the latest-snapshot partition scan below. NOTE: this selects FILES only — row-level // filtering to (begin, end] is a LATER step (an FE-side synthetic _hoodie_commit_time predicate), so a - // bare @incr read would over-read until that lands (harmless while the connector is dormant). The COW-vs- + // bare @incr read would over-read until that lands (a tracked gap on the now-live path). The COW-vs- // MOR relation choice is driven by the SAME isCow the snapshot path uses (metaClient.getTableType(), // hoodie.properties-authoritative): this SUBSUMES the legacy RO-as-RT flip (LogicalHudiScan:251-260 / // HudiScanNode:187-199), which existed only because legacy classifies from the hive inputFormat (a MOR diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiStatementScope.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiStatementScope.java new file mode 100644 index 00000000000000..708400ae2394e6 --- /dev/null +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiStatementScope.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hudi; + +import org.apache.doris.connector.api.ConnectorColumn; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorStatementScope; +import org.apache.doris.connector.api.ConnectorStatementScopes; + +import java.util.List; +import java.util.function.Supplier; + +/** + * Connector-private helpers over the neutral {@link ConnectorStatementScope} (reached via + * {@link ConnectorSession#getStatementScope()}), giving hudi one place to key its per-statement metaClient memos. + * + *

Two INDEPENDENT memos, so a statement builds each latest fact at most once and the two never couple: the + * latest-schema read ({@code getTableSchema} with no time-travel pin) resolves through {@link #sharedLatestColumns}, + * and the query-snapshot pin ({@code beginQuerySnapshot}) through {@link #sharedLatestInstant}. Each loader is the + * connector's existing single-fact method, so the memo only collapses REPEATED same-fact resolves within a statement + * and leaves each fact's value / failure semantics byte-identical. Distinct namespaces + * ({@link #LATEST_SCHEMA_NAMESPACE} vs {@link #LATEST_INSTANT_NAMESPACE}) keep the two value types (columns vs + * instant) from colliding on the shared {@code (catalog, db, table, queryId)} coordinate.

+ * + *

Under {@link ConnectorStatementScope#NONE} (offline planning / no live statement) or a {@code null} session each + * loader runs every time — byte-identical to resolving the fact per call, as before the memo.

+ */ +final class HudiStatementScope { + + /** + * Namespace for hudi's per-statement latest-columns memo. Source-prefixed with the connector type ("hudi") + * so it stays distinct across a heterogeneous gateway; see {@link ConnectorStatementScopes}. + */ + static final String LATEST_SCHEMA_NAMESPACE = "hudi.latest_schema"; + + /** Namespace for hudi's per-statement latest-completed-instant memo. Source-prefixed with the type ("hudi"). */ + static final String LATEST_INSTANT_NAMESPACE = "hudi.latest_instant"; + + private HudiStatementScope() {} + + /** + * Resolves the latest columns for {@code db.table} once per statement, sharing the single result across every + * latest-schema resolver of the statement. {@code loader} (the connector's existing latest-schema read) runs at + * most once per statement; the key carries the catalog id and queryId (cross-catalog / prepared-execution + * isolation). + */ + static List sharedLatestColumns(ConnectorSession session, String dbName, String tableName, + Supplier> loader) { + return ConnectorStatementScopes.resolveInStatement( + session, LATEST_SCHEMA_NAMESPACE, dbName, tableName, loader); + } + + /** + * Resolves the latest completed instant for {@code db.table} once per statement, sharing the single value across + * every snapshot-pin resolver of the statement. {@code loader} (the connector's existing instant read) runs at + * most once per statement. + */ + static long sharedLatestInstant(ConnectorSession session, String dbName, String tableName, + Supplier loader) { + return ConnectorStatementScopes.resolveInStatement( + session, LATEST_INSTANT_NAMESPACE, dbName, tableName, loader); + } +} diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java new file mode 100644 index 00000000000000..4efa41ce7c4755 --- /dev/null +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hudi; + +import org.apache.doris.connector.hms.CachingHmsClient; +import org.apache.doris.connector.hms.HmsClient; +import org.apache.doris.connector.hms.HmsDatabaseInfo; +import org.apache.doris.connector.hms.HmsPartitionInfo; +import org.apache.doris.connector.hms.HmsTableInfo; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; + +/** + * Locks the HMS-caching layer added to the hudi connector (the least-cached SPI connector), which mirrors + * {@code HiveConnector} so repeated queries against the same hudi-on-HMS table stop re-hitting the metastore. + * Two behaviors carry correctness weight and are pinned here (Rule 9): + *
    + *
  • Fresh vs cached split. The user-facing enumeration paths ({@code SHOW PARTITIONS} = + * {@code listPartitionNames}, {@code partition_values()} TVF = {@code listPartitionValues}) MUST list + * FRESH (bypass the cache), or an externally hive-synced partition stays invisible until the 24h TTL — + * a freshness regression the raw pre-cache client never had. The query-pruning / MTMV path + * ({@code listPartitions}) MUST use the cache (that is the whole point of wrapping).
    • + *
  • REFRESH flush. {@code invalidateTable}/{@code invalidateDb}/{@code invalidateAll} MUST flush the + * {@link CachingHmsClient}, or REFRESH cannot clear the sibling's own cache (the gateway forwards REFRESH + * to the sibling, so the flush must land on THIS connector's client).
    • + *
+ */ +public class HudiConnectorHmsCacheTest { + + private static final List YEAR_MONTH = Arrays.asList("year", "month"); + private static final List ONE_PARTITION = Collections.singletonList("year=2024/month=01"); + + // ── wrap ───────────────────────────────────────────────────────────────────────────────────────────── + + @Test + public void wrapWithCacheReturnsCachingHmsClient() { + // MUTATION: returning the raw client (dropping the wrap) -> not a CachingHmsClient -> red. This is the + // guard for HudiConnector.createClient wrapping the pooled ThriftHmsClient before handing it out. + HmsClient wrapped = connector().wrapWithCache(new FakeHmsClient(ONE_PARTITION)); + Assertions.assertTrue(wrapped instanceof CachingHmsClient, + "the hudi HMS client must be decorated with CachingHmsClient"); + } + + // ── fresh vs cached split (hive-sync partition source) ───────────────────────────────────────────────── + + @Test + public void showPartitionsPathListsFresh() { + // listPartitionNames backs SHOW PARTITIONS + the partitions() TVF -> must bypass the cache. + // MUTATION: routing it through the cached listPartitionNames -> freshCalls==0 -> red. + FakeHmsClient hms = new FakeHmsClient(ONE_PARTITION); + HudiConnectorMetadata md = hiveSyncMetadata(hms); + md.listPartitionNames(null, partitioned()); + Assertions.assertEquals(1, hms.freshCalls, "SHOW PARTITIONS must list FRESH (bypass cache)"); + Assertions.assertEquals(0, hms.cachedCalls, "SHOW PARTITIONS must NOT read the cached listing"); + } + + @Test + public void partitionValuesTvfListsFresh() { + // partition_values() TVF is user-facing enumeration too -> fresh, like listPartitionNames. + FakeHmsClient hms = new FakeHmsClient(ONE_PARTITION); + HudiConnectorMetadata md = hiveSyncMetadata(hms); + md.listPartitionValues(null, partitioned(), YEAR_MONTH); + Assertions.assertEquals(1, hms.freshCalls, "partition_values() must list FRESH (bypass cache)"); + Assertions.assertEquals(0, hms.cachedCalls, "partition_values() must NOT read the cached listing"); + } + + @Test + public void queryPruningPathUsesCache() { + // listPartitions backs query pruning / MTMV -> must use the cache (the wrap's intended win). + // MUTATION: routing it through listPartitionNamesFresh -> cachedCalls==0 -> red. + FakeHmsClient hms = new FakeHmsClient(ONE_PARTITION); + HudiConnectorMetadata md = hiveSyncMetadata(hms); + md.listPartitions(null, partitioned(), java.util.Optional.empty()); + Assertions.assertEquals(1, hms.cachedCalls, "query pruning must read the CACHED listing"); + Assertions.assertEquals(0, hms.freshCalls, "query pruning must NOT force a fresh listing"); + } + + // ── REFRESH flush wiring ─────────────────────────────────────────────────────────────────────────────── + + @Test + public void invalidateTableFlushesCache() { + FakeHmsClient delegate = new FakeHmsClient(ONE_PARTITION); + CachingHmsClient cache = new CachingHmsClient(delegate, Collections.emptyMap()); + // Populate the (db,t) partition-name cache: two reads = ONE delegate hit (cached). + cache.listPartitionNames("db", "t", -1); + cache.listPartitionNames("db", "t", -1); + Assertions.assertEquals(1, delegate.cachedCalls, "second read must be served from cache"); + + // REFRESH TABLE -> the connector must flush this table from the cache (MUTATION: an empty override -> the + // next read stays cached -> cachedCalls==1 -> red). + connector().invalidateTable(cache, "db", "t"); + cache.listPartitionNames("db", "t", -1); + Assertions.assertEquals(2, delegate.cachedCalls, "after REFRESH TABLE the next read must miss the cache"); + } + + @Test + public void invalidateDbFlushesCache() { + FakeHmsClient delegate = new FakeHmsClient(ONE_PARTITION); + CachingHmsClient cache = new CachingHmsClient(delegate, Collections.emptyMap()); + cache.listPartitionNames("db", "t", -1); + cache.listPartitionNames("db", "t", -1); + Assertions.assertEquals(1, delegate.cachedCalls); + + connector().invalidateDb(cache, "db"); + cache.listPartitionNames("db", "t", -1); + Assertions.assertEquals(2, delegate.cachedCalls, "after REFRESH DATABASE the next read must miss the cache"); + } + + @Test + public void invalidateAllFlushesCache() { + FakeHmsClient delegate = new FakeHmsClient(ONE_PARTITION); + CachingHmsClient cache = new CachingHmsClient(delegate, Collections.emptyMap()); + cache.listPartitionNames("db", "t", -1); + cache.listPartitionNames("db", "t", -1); + Assertions.assertEquals(1, delegate.cachedCalls); + + connector().invalidateAll(cache); + cache.listPartitionNames("db", "t", -1); + Assertions.assertEquals(2, delegate.cachedCalls, "after REFRESH CATALOG the next read must miss the cache"); + } + + @Test + public void invalidateOnUnbuiltClientIsNoOp() { + // REFRESH on a never-queried catalog must not force-build a client (nothing to flush). The public + // overrides read the null hmsClient field; they must not throw. + Assertions.assertDoesNotThrow(() -> { + connector().invalidateTable("db", "t"); + connector().invalidateDb("db"); + connector().invalidateAll(); + }); + } + + // ── helpers ──────────────────────────────────────────────────────────────────────────────────────────── + + private static HudiConnector connector() { + return new HudiConnector(Collections.emptyMap(), new ConnectorContext() { + @Override + public String getCatalogName() { + return "test_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + }); + } + + private static HudiTableHandle partitioned() { + return new HudiTableHandle.Builder("db", "t", "s3://b/t", "COPY_ON_WRITE") + .partitionKeyNames(YEAR_MONTH).build(); + } + + private static HudiConnectorMetadata hiveSyncMetadata(HmsClient hms) { + // hive-sync so collectPartitions lists partition names from HMS (where the fresh/cached split lives); + // the stub executor returns the canned instant latestInstant would read off the timeline. + return new HudiConnectorMetadata(hms, + Collections.singletonMap("use_hive_sync_partition", "true"), stub(7L)); + } + + /** Executor that ignores the action and returns a canned value (stubs out the live metaClient). */ + private static HudiMetaClientExecutor stub(Object cannedReturn) { + return new HudiMetaClientExecutor() { + @Override + @SuppressWarnings("unchecked") + public T execute(Callable action) { + return (T) cannedReturn; + } + }; + } + + /** + * {@link HmsClient} double that counts the CACHED {@link #listPartitionNames} vs the FRESH + * {@link #listPartitionNamesFresh} separately, so a test can pin which freshness contract each entry point + * selects. Everything else fails loud. + */ + private static final class FakeHmsClient implements HmsClient { + int cachedCalls; + int freshCalls; + private final List names; + + FakeHmsClient(List names) { + this.names = names; + } + + @Override + public List listPartitionNames(String dbName, String tableName, int maxParts) { + cachedCalls++; + return names; + } + + @Override + public List listPartitionNamesFresh(String dbName, String tableName, int maxParts) { + freshCalls++; + return names; + } + + @Override + public List listDatabases() { + throw new UnsupportedOperationException(); + } + + @Override + public HmsDatabaseInfo getDatabase(String dbName) { + throw new UnsupportedOperationException(); + } + + @Override + public List listTables(String dbName) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean tableExists(String dbName, String tableName) { + throw new UnsupportedOperationException(); + } + + @Override + public HmsTableInfo getTable(String dbName, String tableName) { + throw new UnsupportedOperationException(); + } + + @Override + public Map getDefaultColumnValues(String dbName, String tableName) { + throw new UnsupportedOperationException(); + } + + @Override + public List getPartitions(String dbName, String tableName, List partNames) { + throw new UnsupportedOperationException(); + } + + @Override + public HmsPartitionInfo getPartition(String dbName, String tableName, List values) { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + } + } +} diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorOwnsHandleTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorOwnsHandleTest.java index f86b5bc0b28c74..e1f883f5a35be0 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorOwnsHandleTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorOwnsHandleTest.java @@ -31,8 +31,8 @@ * concrete handle type is invisible across the plugin classloader split. * *

ownsHandle must be TRUE for this connector's own {@link HudiTableHandle} and FALSE for any other connector's - * handle (e.g. an iceberg sibling's), so the gateway routes correctly. Dormant until hms enters - * {@code SPI_READY_TYPES}: no production path asks this connector yet, so this is a Rule-9 routing guard. + * handle (e.g. an iceberg sibling's), so the gateway routes correctly. With hms live, the flipped gateway routes + * real hudi handles here, so this is a Rule-9 routing guard. */ public class HudiConnectorOwnsHandleTest { diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiReadOnlyWriteRejectTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiReadOnlyWriteRejectTest.java index a3b6d1753befda..c206e30a70482e 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiReadOnlyWriteRejectTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiReadOnlyWriteRejectTest.java @@ -34,8 +34,8 @@ * future change cannot silently make a hudi table writable or DDL-mutable, and a hudi write is never * mis-executed as an iceberg write. * - *

Dormant until hms enters {@code SPI_READY_TYPES} and hudi handles are diverted (no production path reaches - * this connector yet); this is a Rule-9 behavior lock. The correctness at flip is asserted end-to-end (a + *

With hms live and hudi handles diverted, this connector answers those calls on the production path; + * this is a Rule-9 behavior lock. The correctness at flip is asserted end-to-end (a * heterogeneous HMS catalog where hudi INSERT/DELETE/MERGE/EXECUTE all fail loud read-only). */ public class HudiReadOnlyWriteRejectTest { diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiStatementMemoTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiStatementMemoTest.java new file mode 100644 index 00000000000000..91e9d0c45cb5b2 --- /dev/null +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiStatementMemoTest.java @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hudi; + +import org.apache.doris.connector.api.ConnectorColumn; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorStatementScope; +import org.apache.doris.connector.api.ConnectorType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +/** + * Guards the per-statement metaClient memos: hudi keeps TWO INDEPENDENT memos, one for the latest columns + * (behind {@code getTableSchema} with no time-travel pin) and one for the latest completed instant (behind + * {@code beginQuerySnapshot}). Within one statement each fact is read at most once; the two never couple. Each + * assertion pins WHY it matters: + *

    + *
  • repeated latest {@code getTableSchema} reads in one statement collapse to a single + * {@code getSchemaFromMetaClient} — a mutation dropping the schema memo makes it run twice and turns this red;
    • + *
  • repeated {@code beginQuerySnapshot} pins collapse to a single {@code latestInstant}, likewise;
    • + *
  • the two memos are INDEPENDENT — a schema read triggers NO instant read and vice versa (a mutation that + * coupled them, e.g. a shared loader, would make the cross-counter non-zero);
    • + *
  • under {@link ConnectorStatementScope#NONE} and a {@code null} session each fact loads every time — + * byte-identical to resolving it per call, as before the memo (no cross-statement caching).
    • + *
+ * + *

The two loaders are overridden with canned results (no live metaClient / plugin auth), so this unit locks the + * sharing + independence wiring; each loader's byte-parity with the real read is guaranteed by reusing the exact + * connector method ({@code getSchemaFromMetaClient} / {@code latestInstant}) as the loader.

+ */ +public class HudiStatementMemoTest { + + private static final long INSTANT = 20240101120000000L; + + /** Records how many times each latest-fact loader actually ran (a memo miss). */ + private static final class RecordingMetadata extends HudiConnectorMetadata { + int schemaLoads; + int instantLoads; + + RecordingMetadata() { + super(null, Collections.emptyMap(), null); + } + + @Override + List getSchemaFromMetaClient(String basePath, String queryInstant) { + this.schemaLoads++; + return Collections.singletonList( + new ConnectorColumn("c", ConnectorType.of("INT"), "", true, null)); + } + + @Override + long latestInstant(HudiTableHandle handle) { + this.instantLoads++; + return INSTANT; + } + } + + private static HudiTableHandle handle() { + return new HudiTableHandle.Builder("db", "t", "s3://b/t", "COPY_ON_WRITE").build(); + } + + @Test + public void repeatedLatestSchemaReadsShareOneLoadWithinStatement() { + RecordingMetadata m = new RecordingMetadata(); + MemoSession session = new MemoSession(7L, "q1", new MemoScope()); + m.getTableSchema(session, handle()); + m.getTableSchema(session, handle()); + Assertions.assertEquals(1, m.schemaLoads, "repeated latest getTableSchema must share one metaClient read"); + Assertions.assertEquals(0, m.instantLoads, "a schema read must NOT trigger an instant read (memos independent)"); + } + + @Test + public void repeatedSnapshotPinsShareOneLoadWithinStatement() { + RecordingMetadata m = new RecordingMetadata(); + MemoSession session = new MemoSession(7L, "q1", new MemoScope()); + long a = m.beginQuerySnapshot(session, handle()).get().getSnapshotId(); + long b = m.beginQuerySnapshot(session, handle()).get().getSnapshotId(); + Assertions.assertEquals(INSTANT, a); + Assertions.assertEquals(INSTANT, b); + Assertions.assertEquals(1, m.instantLoads, "repeated beginQuerySnapshot must share one instant read"); + Assertions.assertEquals(0, m.schemaLoads, "an instant read must NOT trigger a schema read (memos independent)"); + } + + @Test + public void noneScopeLoadsEachTime() { + // A live session whose scope is NONE (no per-statement context) must NOT share — load every time. + RecordingMetadata m = new RecordingMetadata(); + MemoSession session = new MemoSession(7L, "q1", ConnectorStatementScope.NONE); + m.getTableSchema(session, handle()); + m.getTableSchema(session, handle()); + m.beginQuerySnapshot(session, handle()); + m.beginQuerySnapshot(session, handle()); + Assertions.assertEquals(2, m.schemaLoads, "NONE scope -> schema loads every time"); + Assertions.assertEquals(2, m.instantLoads, "NONE scope -> instant loads every time"); + } + + @Test + public void nullSessionLoadsEachTime() { + // Offline / direct-construction (null session): each resolve loads, byte-identical to loading every time. + RecordingMetadata m = new RecordingMetadata(); + m.getTableSchema(null, handle()); + m.getTableSchema(null, handle()); + m.beginQuerySnapshot(null, handle()); + m.beginQuerySnapshot(null, handle()); + Assertions.assertEquals(2, m.schemaLoads, "null session -> schema loads every time"); + Assertions.assertEquals(2, m.instantLoads, "null session -> instant loads every time"); + } + + @Test + public void allNamespacesArePrefixedWithConnectorType() throws Exception { + // NORM (self-extending): reflect over every "*_NAMESPACE" constant this connector declares and assert each + // is prefixed with the connector's ConnectorProvider.getType() ("hudi."). Source-prefixing keeps the value + // types (columns vs instant) — and any other connector's — distinct on the shared coordinate. Reflecting + // means a NEW namespace is auto-covered; a forgotten prefix or a getType() drift turns this red. + String prefix = new HudiConnectorProvider().getType() + "."; + int checked = 0; + for (Field f : HudiStatementScope.class.getDeclaredFields()) { + if (Modifier.isStatic(f.getModifiers()) && f.getType() == String.class + && f.getName().endsWith("_NAMESPACE")) { + f.setAccessible(true); + String ns = (String) f.get(null); + Assertions.assertTrue(ns.startsWith(prefix), + f.getName() + " (\"" + ns + "\") must be prefixed with the connector type \"" + prefix + "\""); + checked++; + } + } + Assertions.assertTrue(checked > 0, "expected at least one *_NAMESPACE constant to guard"); + } + + /** A statement scope that memoizes like the engine's real one (ConcurrentHashMap computeIfAbsent). */ + private static final class MemoScope implements ConnectorStatementScope { + private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); + + @Override + @SuppressWarnings("unchecked") + public T computeIfAbsent(String key, Supplier loader) { + return (T) cache.computeIfAbsent(key, k -> loader.get()); + } + } + + /** Minimal {@link ConnectorSession} carrying a catalog id, queryId and scope for the per-statement memo. */ + private static final class MemoSession implements ConnectorSession { + private final long catalogId; + private final String queryId; + private final ConnectorStatementScope scope; + + MemoSession(long catalogId, String queryId, ConnectorStatementScope scope) { + this.catalogId = catalogId; + this.queryId = queryId; + this.scope = scope; + } + + @Override + public long getCatalogId() { + return catalogId; + } + + @Override + public String getQueryId() { + return queryId; + } + + @Override + public String getSessionId() { + return "session-" + queryId; + } + + @Override + public ConnectorStatementScope getStatementScope() { + return scope; + } + + @Override + public String getUser() { + return "u"; + } + + @Override + public String getTimeZone() { + return "UTC"; + } + + @Override + public String getLocale() { + return "en_US"; + } + + @Override + public String getCatalogName() { + return "c"; + } + + @Override + public T getProperty(String name, Class type) { + return null; + } + + @Override + public Map getCatalogProperties() { + return Collections.emptyMap(); + } + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogOps.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogOps.java index f1d47d064f36b4..a7ab0f03aab137 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogOps.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogOps.java @@ -434,7 +434,10 @@ public Optional loadNamespaceLocation(String dbName) { @Override public void addColumn(String dbName, String tableName, IcebergColumnChange column, ConnectorColumnPosition position) { - UpdateSchema updateSchema = loadTable(dbName, tableName).updateSchema(); + Table table = loadTable(dbName, tableName); + IcebergNestedColumnEvolution.validateNoCaseInsensitiveSiblingCollision( + table.schema().asStruct(), "", column.getName(), null, "add"); + UpdateSchema updateSchema = table.updateSchema(); updateSchema.addColumn(column.getName(), column.getType(), column.getComment(), column.getDefaultValue()); applyPosition(updateSchema, position, column.getName()); @@ -443,7 +446,9 @@ public void addColumn(String dbName, String tableName, IcebergColumnChange colum @Override public void addColumns(String dbName, String tableName, List columns) { - UpdateSchema updateSchema = loadTable(dbName, tableName).updateSchema(); + Table table = loadTable(dbName, tableName); + IcebergNestedColumnEvolution.validateNoCaseInsensitiveTopLevelCollisions(table.schema(), columns); + UpdateSchema updateSchema = table.updateSchema(); for (IcebergColumnChange column : columns) { updateSchema.addColumn(column.getName(), column.getType(), column.getComment(), column.getDefaultValue()); @@ -460,9 +465,7 @@ public void dropColumn(String dbName, String tableName, String columnName) { @Override public void renameColumn(String dbName, String tableName, String oldName, String newName) { - UpdateSchema updateSchema = loadTable(dbName, tableName).updateSchema(); - updateSchema.renameColumn(oldName, newName); - updateSchema.commit(); + IcebergNestedColumnEvolution.renameTopLevelColumn(loadTable(dbName, tableName), oldName, newName); } @Override diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java index aa179082d3d234..6caf556f5780f6 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java @@ -55,13 +55,10 @@ final class IcebergCommentCache { private final MetaCacheEntry entry; IcebergCommentCache(long ttlSeconds, int maxSize) { - // Mirror IcebergTableCache: translate the connector's "<= 0 disables" contract to CacheSpec's ttl == 0 - // (disabled) rather than passing a negative value through, which CacheSpec reads as ttl == -1 "no - // expiration (enabled)". This ttl<=0 -> disabled mapping is load-bearing: a vended no-cache catalog builds - // this object but must NOT cache comments (operator "no meta cache" intent). - CacheSpec spec = ttlSeconds > 0 - ? CacheSpec.of(true, ttlSeconds, maxSize) - : CacheSpec.of(true, CacheSpec.CACHE_TTL_DISABLE_CACHE, maxSize); + // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). + // Load-bearing here: a vended no-cache catalog builds this object but must NOT cache comments + // (operator "no meta cache" intent). + CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = new MetaCacheEntry<>("iceberg-comment", null, spec, ForkJoinPool.commonPool(), false, true, 0L, true); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java index 350c767b92b61a..16f09143d38522 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java @@ -30,7 +30,7 @@ import org.apache.doris.connector.api.procedure.ConnectorProcedureOps; import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; import org.apache.doris.connector.api.write.ConnectorWritePlanProvider; -import org.apache.doris.connector.cache.ConnectorPartitionViewCache; +import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.metastore.HmsMetaStoreProperties; import org.apache.doris.connector.metastore.spi.JdbcDriverSupport; import org.apache.doris.connector.metastore.spi.MetaStoreProviders; @@ -162,30 +162,33 @@ public class IcebergConnector implements Connector { // connector (PluginDrivenExternalCatalog.onClose nulls + recreates it) and thus drops both caches. The // manifest cache is path-keyed, no-TTL, capacity-bounded; it is consumed only when // meta.cache.iceberg.manifest.enable is set (default off → scan uses the SDK planFiles path). - // Each cross-query cache below carries an isolation-discipline marker enforced by - // tools/check-authz-cache-sharding.sh: under iceberg.rest.session=user a shared, un-partitioned cache would - // bypass the per-user loadTable authorization (a metadata disclosure), so every cache field must declare - // either 'authz-cache-session-user-disabled' (null under isUserSessionEnabled()) or 'authz-cache-exempt'. - private final IcebergLatestSnapshotCache latestSnapshotCache; // authz-cache-session-user-disabled + // ATTN — authorization-sensitive cache isolation (a REVIEWED invariant; no build gate enforces it). + // Under iceberg.rest.session=user the per-user authorization lives INSIDE the delegated loadTable, so a + // shared, un-partitioned cross-query cache that returns metadata WITHOUT that per-user load is a + // "list != load" disclosure. Every cross-query cache below is therefore nulled under isUserSessionEnabled() + // (see the constructor's `? null : new ...` ternaries) except manifestCache (exempt: read only after a + // per-user resolveTable). If you ADD a cross-query cache here or in IcebergConnectorMetadata, it MUST be + // null under session=user and covered by IcebergConnectorCacheTest (which asserts exactly that at runtime). + private final IcebergLatestSnapshotCache latestSnapshotCache; // null under session=user // PERF-01: cross-query cache of the RAW iceberg Table (restores the legacy IcebergExternalMetaCache table // cache that the SPI cutover dropped). null when the catalog's credentials are query-dependent // (iceberg.rest.session=user / REST vended-credentials) — see the constructor. The per-statement scope // (ConnectorStatementScope) shares one loaded table across a statement's read/scan/write regardless of this // field, and is what a credential-gated catalog (this field null) relies on within a statement. - private final IcebergTableCache tableCache; // authz-cache-session-user-disabled + private final IcebergTableCache tableCache; // null under session=user // PERF-02: cross-query partition-view cache (the raw PARTITIONS-scan result, keyed by (table, snapshotId)). // The value is pure metadata (no FileIO/credential), but under session=user it is an authorization-sensitive // projection (a shared hit would disclose one user's partitions), so it is disabled there (see constructor). - private final IcebergPartitionCache partitionCache; // authz-cache-session-user-disabled + private final IcebergPartitionCache partitionCache; // null under session=user // PERF-03: cross-query inferred-file-format cache (the whole-table planFiles() fallback result, keyed by // (table, snapshotId)). Same authorization-sensitive treatment as partitionCache: disabled under session=user. - private final IcebergFormatCache formatCache; // authz-cache-session-user-disabled + private final IcebergFormatCache formatCache; // null under session=user // PERF-05: cross-query table-comment cache (value = the 'comment' property string). Built ONLY for a REST // vended-credentials catalog that is NOT session=user -- plain catalogs already reuse tableCache (PERF-01) for // the comment path, and session=user must stay live because the loadTable itself carries per-user // authorization a shared cache would bypass. null for every other flavor. - private final IcebergCommentCache commentCache; // authz-cache-session-user-disabled - // PERF-06: cross-query DERIVED partition-view cache ("cache A", the generic ConnectorPartitionViewCache from + private final IcebergCommentCache commentCache; // null under session=user + // PERF-06: cross-query DERIVED partition-view cache ("cache A", the generic ConnectorMetadataCache from // fe-connector-cache), layered ABOVE the raw partitionCache (PERF-02): it memoizes the BUILT derived view // (transform-to-range math + overlap merge for the MTMV view; the value-map construction for listPartitions) // keyed by (db, table, snapshotId, schemaId), so a repeated query on a partitioned table skips the derived @@ -194,12 +197,12 @@ public class IcebergConnector implements Connector { // deduped by partitionCache, so two derived caches never double-scan remotely. Authorization-sensitive // projection like partitionCache/formatCache: disabled (null) under iceberg.rest.session=user so a shared // (no user dimension) hit cannot disclose one user's partition view; kept for every other flavor. - private final ConnectorPartitionViewCache // authz-cache-session-user-disabled + private final ConnectorMetadataCache // null under session=user mvccPartitionViewCache; - private final ConnectorPartitionViewCache> // authz-cache-session-user-disabled + private final ConnectorMetadataCache> // null under session=user listPartitionsViewCache; // Manifest content cache — pure metadata, default-off (meta.cache.iceberg.manifest.enable), and consumed - // ONLY after a per-user resolveTable(ForRead). authz-cache-exempt (no read path without a per-user load). + // ONLY after a per-user resolveTable(ForRead) -- exempt: no read path without a per-user load. private final IcebergManifestCache manifestCache = new IcebergManifestCache(); // Lazily-built plugin-side Kerberos authenticator (single-owner auth; see TcclPinningConnectorContext). @@ -271,17 +274,17 @@ public IcebergConnector(Map properties, ConnectorContext context ? new IcebergCommentCache( resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY) : null; - // PERF-06: derived partition-view cache A (generic ConnectorPartitionViewCache). Same + // PERF-06: derived partition-view cache A (generic ConnectorMetadataCache). Same // authorization-sensitive treatment as partitionCache -- disabled (null) under iceberg.rest.session=user so // a shared (no user dimension) hit cannot disclose one user's partition view; kept otherwise. Reads its // own meta.cache.iceberg.partition_view.(enable|ttl-second|capacity) from the catalog properties via the // framework's CacheSpec (default ON / 24h / 1000). Two typed instances (MVCC view + partition-info list). this.mvccPartitionViewCache = isUserSessionEnabled() ? null - : new ConnectorPartitionViewCache<>("iceberg", this.properties); + : new ConnectorMetadataCache<>("iceberg", "partition_view", this.properties); this.listPartitionsViewCache = isUserSessionEnabled() ? null - : new ConnectorPartitionViewCache<>("iceberg", this.properties); + : new ConnectorMetadataCache<>("iceberg", "partition_view", this.properties); } /** @@ -761,12 +764,12 @@ IcebergCommentCache commentCacheForTest() { } /** Test-only: the derived MVCC partition-view cache (PERF-06), or {@code null} for a session=user catalog. */ - ConnectorPartitionViewCache mvccPartitionViewCacheForTest() { + ConnectorMetadataCache mvccPartitionViewCacheForTest() { return mvccPartitionViewCache; } /** Test-only: the derived listPartitions view cache (PERF-06), or {@code null} for a session=user catalog. */ - ConnectorPartitionViewCache> listPartitionsViewCacheForTest() { + ConnectorMetadataCache> listPartitionsViewCacheForTest() { return listPartitionsViewCache; } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java index f771a8531d68cd..158c1fbbb188b9 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java @@ -43,8 +43,8 @@ import org.apache.doris.connector.api.mvcc.ConnectorMvccSnapshot; import org.apache.doris.connector.api.mvcc.ConnectorTimeTravelSpec; import org.apache.doris.connector.api.pushdown.ConnectorExpression; -import org.apache.doris.connector.cache.ConnectorPartitionViewCache; -import org.apache.doris.connector.cache.PartitionViewCacheKey; +import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ConnectorTableKey; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.thrift.THiveTable; import org.apache.doris.thrift.TIcebergTable; @@ -161,14 +161,14 @@ public class IcebergConnectorMetadata implements ConnectorMetadata { // connector is a REST vended-credentials, non-session catalog (see IcebergConnector); the convenience ctors // used by direct-construction tests pass null. Consumed only by getTableComment. private final IcebergCommentCache commentCache; - // PERF-06: cross-query DERIVED partition-view cache A (generic ConnectorPartitionViewCache), injected by the + // PERF-06: cross-query DERIVED partition-view cache A (generic ConnectorMetadataCache), injected by the // owning IcebergConnector; null = no cross-query derived layer (the convenience ctors used by // direct-construction tests pass null; a session=user catalog also passes null). Layered ABOVE partitionCache // (raw rows): a hit skips the derived-view BUILD, keyed by (db, table, snapshotId, schemaId). Two typed fields // because getMvccPartitionView and listPartitions return different derived types. Consumed by // getMvccPartitionView / listPartitions respectively. - private final ConnectorPartitionViewCache mvccPartitionViewCache; - private final ConnectorPartitionViewCache> listPartitionsViewCache; + private final ConnectorMetadataCache mvccPartitionViewCache; + private final ConnectorMetadataCache> listPartitionsViewCache; public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, Map properties, ConnectorContext context) { @@ -214,8 +214,8 @@ public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, Map mvccPartitionViewCache, - ConnectorPartitionViewCache> listPartitionsViewCache) { + ConnectorMetadataCache mvccPartitionViewCache, + ConnectorMetadataCache> listPartitionsViewCache) { this.catalogOps = catalogOps; this.properties = properties; this.context = context; @@ -1693,11 +1693,10 @@ private static boolean isSupportedSysTable(String sysName) { * per-manager map and {@code GlobalExternalTransactionInfoMgr}) and stamped into the data sink — * the BE→FE report path finds the txn by this id to feed it commit fragments. * - *

Gate-closed / dormant until the P6.6 cutover: nothing routes plugin-driven iceberg writes - * through this path yet. The single SDK {@code org.apache.iceberg.Transaction} that backs commit is - * opened lazily by the write plan via {@link IcebergConnectorTransaction#beginWrite}; op selection - * (T04), the commit-validation suite (T05), the sink (T06), and the {@code supportsInsert/Delete/ - * Merge} capability declarations (T06/T07) land in later tasks.

+ *

Live since the iceberg SPI cutover: plugin-driven iceberg writes route through this path. The + * single SDK {@code org.apache.iceberg.Transaction} that backs commit is opened lazily by the write + * plan via {@link IcebergConnectorTransaction#beginWrite}; op selection, the commit-validation suite, + * the sink, and the {@code supportsInsert/Delete/Merge} capability declarations are all in place.

*/ @Override public ConnectorTransaction beginTransaction(ConnectorSession session) { @@ -1824,7 +1823,7 @@ public Optional getMvccPartitionView( if (mvccPartitionViewCache == null) { return Optional.of(buildMvccPartitionViewUncached(session, iceHandle)); } - PartitionViewCacheKey key = new PartitionViewCacheKey(iceHandle.getDbName(), + ConnectorTableKey key = new ConnectorTableKey(iceHandle.getDbName(), iceHandle.getTableName(), iceHandle.getSnapshotId(), iceHandle.getSchemaId()); return Optional.of(mvccPartitionViewCache.get(key, () -> buildMvccPartitionViewUncached(session, iceHandle))); @@ -1900,7 +1899,7 @@ public List listPartitions(ConnectorSession session, if (listPartitionsViewCache == null || filter.isPresent()) { return listPartitionsUncached(session, iceHandle); } - PartitionViewCacheKey key = new PartitionViewCacheKey(iceHandle.getDbName(), + ConnectorTableKey key = new ConnectorTableKey(iceHandle.getDbName(), iceHandle.getTableName(), iceHandle.getSnapshotId(), iceHandle.getSchemaId()); return listPartitionsViewCache.get(key, () -> listPartitionsUncached(session, iceHandle)); }); diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java index 10f3629601e1f3..58416097ec7213 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java @@ -109,8 +109,8 @@ * O5-2 write constraint ({@link #applyWriteConstraint}, a neutral {@link ConnectorPredicate} converted lazily * at commit) ANDed with a commit-time identity-partition filter derived from the commit fragments.

* - *

Gate-closed / dormant. Iceberg is not in {@code SPI_READY_TYPES} until the P6.6 cutover, so - * nothing routes plugin-driven iceberg writes through this class yet ({@link #beginWrite} is wired by T06's + *

Live since the iceberg SPI cutover. {@code iceberg} is in {@code SPI_READY_TYPES}, so + * plugin-driven iceberg writes route through this class ({@link #beginWrite} is wired by * {@code planWrite}). The txn-id is the engine-allocated Doris global id, so the generic * {@code PluginDrivenTransactionManager} registers it in both the per-manager map and * {@code GlobalExternalTransactionInfoMgr} — no per-connector registration code is needed, mirroring @@ -154,7 +154,7 @@ public class IcebergConnectorTransaction implements ConnectorTransaction, Rewrit // Session zone for human-readable TIMESTAMP partition value parsing (DV-T04-f). private ZoneId zone = ZoneOffset.UTC; - // ── REWRITE (rewrite_data_files, P6.4-T06; dormant until the P6.6 cutover) ── + // ── REWRITE (rewrite_data_files; live since the iceberg SPI cutover) ── // The original data files to remove, fed by updateRewriteFiles (the rewrite execution half hands it the // planner's RewriteDataGroup.getDataFiles() — FileScanTask.file()). The new compacted files arrive on the // shared commitDataList channel (like INSERT) and are materialized into filesToAdd at commit time. @@ -346,7 +346,7 @@ public void applyWriteConstraint(ConnectorPredicate targetOnlyFilter) { * REWRITE: registers original data files to remove (the rewrite execution half feeds it one bin-packed * group at a time — {@code RewriteDataGroup.getDataFiles()}). Accumulates across calls. Ported from legacy * {@code IcebergTransaction.updateRewriteFiles}; package-visible because the rewrite coordinator lives in - * the connector (fe-core cannot traffic in iceberg {@code DataFile}). Dormant until the P6.6 cutover. + * the connector (fe-core cannot traffic in iceberg {@code DataFile}). Live since the iceberg SPI cutover. */ void updateRewriteFiles(List originalFiles) { synchronized (filesToDelete) { @@ -999,8 +999,8 @@ private static int formatVersion(Table table) { *

Unlike legacy {@code IcebergTransaction.collectRewrittenDeleteFiles} (which looked the old files up in a * scan-time map fed from the read plan), this reads them from the write-time {@link #baseSnapshotId} the * RowDelta validates against, so the removed deletes are snapshot-consistent with the commit by construction - * (no read-vs-write snapshot skew). Post-flip deviation DV-S2-rederive (dormant: iceberg is not yet a - * plugin-driven type, so this commit path runs only after the C5 flip).

+ * (no read-vs-write snapshot skew). Post-flip deviation DV-S2-rederive: iceberg is a plugin-driven type + * since the SPI flip, so this commit path is live.

*/ List collectRewrittenDeleteFiles(List deleteCommitData) { if (deleteCommitData == null || deleteCommitData.isEmpty() || baseSnapshotId == null || table == null) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java index 294b29b3738e8c..cb07b3e57da124 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java @@ -91,11 +91,8 @@ public int hashCode() { private final MetaCacheEntry entry; IcebergFormatCache(long ttlSeconds, int maxSize) { - // Mirror IcebergPartitionCache: translate the connector's "<= 0 disables" contract to CacheSpec's ttl == 0 - // (disabled) rather than passing a negative value through (which CacheSpec reads as "no expiration"). - CacheSpec spec = ttlSeconds > 0 - ? CacheSpec.of(true, ttlSeconds, maxSize) - : CacheSpec.of(true, CacheSpec.CACHE_TTL_DISABLE_CACHE, maxSize); + // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). + CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = new MetaCacheEntry<>("iceberg-format", null, spec, ForkJoinPool.commonPool(), false, true, 0L, true); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java index b953e178fbe377..35b63c6aa8c8fc 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java @@ -67,13 +67,8 @@ static final class CachedSnapshot { private final MetaCacheEntry entry; IcebergLatestSnapshotCache(long ttlSeconds, int maxSize) { - // ttl-second <= 0 disables caching (always read live); a positive ttl is access-based expiry with the - // given capacity. CacheSpec treats ttl == -1 as "no expiration (enabled)" and ttl == 0 as "disabled", - // so translate the connector's "<= 0 disables" contract to ttl == 0 rather than passing a negative - // value straight through (which would otherwise flip -1 into a never-expiring cache). - CacheSpec spec = ttlSeconds > 0 - ? CacheSpec.of(true, ttlSeconds, maxSize) - : CacheSpec.of(true, CacheSpec.CACHE_TTL_DISABLE_CACHE, maxSize); + // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). + CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = new MetaCacheEntry<>("iceberg-latest-snapshot", null, spec, ForkJoinPool.commonPool(), false, true, 0L, true); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergNestedColumnEvolution.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergNestedColumnEvolution.java index f30eab3a8f0718..14facfd6ddc12f 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergNestedColumnEvolution.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergNestedColumnEvolution.java @@ -49,6 +49,11 @@ * *

No partial commit: every {@code UpdateSchema.commit()} is the final statement of each entry point, so * any validation/resolution throw aborts the whole change before anything is committed (legacy parity).

+ * + *

The TOP-LEVEL (flat) column ops stay on {@link IcebergCatalogOps}, but they share this class's name + * resolution and validators — iceberg matches field names case-SENSITIVELY while Doris column names are + * case-insensitive, so both arms must agree. See {@link #validateNoCaseInsensitiveSiblingCollision}, + * {@link #validateNoCaseInsensitiveTopLevelCollisions} and {@link #renameTopLevelColumn}.

*/ public final class IcebergNestedColumnEvolution { @@ -277,7 +282,45 @@ private static ConnectorColumnPath childPath(ConnectorColumnPath parentPath, Str return ConnectorColumnPath.of(parts); } - private static void validateNoCaseInsensitiveSiblingCollision(Types.StructType parentType, String parentPath, + /** + * Renames the TOP-LEVEL column {@code oldName} to {@code newName}. {@code oldName} is resolved + * case-insensitively (iceberg's own {@code renameColumn} is case-sensitive and would report the column as + * missing when the user types a different case than the iceberg field), then {@code newName} is checked + * against the other top-level fields. Mirrors the legacy flat {@code IcebergMetadataOps.renameColumn}. + */ + static void renameTopLevelColumn(Table table, String oldName, String newName) { + Schema schema = table.schema(); + ResolvedColumnPath oldPath = resolveColumnPath(schema, ConnectorColumnPath.of(oldName), "rename"); + validateNoCaseInsensitiveSiblingCollision(schema.asStruct(), "", newName, oldPath.getField(), "rename"); + + UpdateSchema updateSchema = table.updateSchema(); + applyRenameColumn(schema, updateSchema, oldPath, newName); + updateSchema.commit(); + } + + /** + * Rejects a batch of TOP-LEVEL columns that collides case-insensitively with an existing field or with + * another column of the same request. Validates the whole batch before the caller stages anything, so a + * partly-valid request commits nothing. Mirrors the legacy + * {@code IcebergMetadataOps.validateNoCaseInsensitiveTopLevelCollisions}. + */ + static void validateNoCaseInsensitiveTopLevelCollisions(Schema schema, List columns) { + Set requestedNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (IcebergColumnChange column : columns) { + validateNoCaseInsensitiveSiblingCollision(schema.asStruct(), "", column.getName(), null, "add"); + if (!requestedNames.add(column.getName())) { + throw new DorisConnectorException("Cannot add column '" + column.getName() + + "': conflicts with another requested column (case-insensitive)"); + } + } + } + + /** + * Rejects {@code targetName} when it case-insensitively collides with a sibling of {@code parentType} that + * is not {@code sourceField} itself (the identity escape lets a pure re-casing rename through). An empty + * {@code parentPath} addresses the table's top-level columns. + */ + static void validateNoCaseInsensitiveSiblingCollision(Types.StructType parentType, String parentPath, String targetName, NestedField sourceField, String operation) { NestedField conflictingField = parentType.caseInsensitiveField(targetName); if (conflictingField != null diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java index 3d68ab664cfb87..a85e2187043e7c 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java @@ -88,12 +88,8 @@ public int hashCode() { private final MetaCacheEntry> entry; IcebergPartitionCache(long ttlSeconds, int maxSize) { - // Mirror IcebergLatestSnapshotCache: translate the connector's "<= 0 disables" contract to CacheSpec's - // ttl == 0 (disabled) rather than passing a negative value through (which CacheSpec reads as "no - // expiration"). - CacheSpec spec = ttlSeconds > 0 - ? CacheSpec.of(true, ttlSeconds, maxSize) - : CacheSpec.of(true, CacheSpec.CACHE_TTL_DISABLE_CACHE, maxSize); + // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). + CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = new MetaCacheEntry<>("iceberg-partition", null, spec, ForkJoinPool.commonPool(), false, true, 0L, true); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java index e3c4fb45ef6625..a1a67dd210c192 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java @@ -796,7 +796,7 @@ private IcebergScanRange buildRangeForTask(FileScanTask task, Table table, int f * pushdown does not apply (a metadata table has no snapshot-summary count). The serialized {@code * FileScanTask} bytes are consumed verbatim by BE's {@code IcebergSysTableJniScanner} * ({@code deserializeFromBase64(...).asDataTask().rows()}); FE unit tests cannot reach the BE classloader, so - * the cross-version byte compatibility is covered by the P6.8 docker e2e. Dormant until P6.6. + * the cross-version byte compatibility is covered by the P6.8 docker e2e. Live since the iceberg SPI cutover. */ private List planSystemTableScan(IcebergTableHandle handle, List columns, Optional filter, diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergStatementScope.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergStatementScope.java index 28b0e28fe41d49..b4c6ee0929dd03 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergStatementScope.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergStatementScope.java @@ -19,6 +19,7 @@ import org.apache.doris.connector.api.ConnectorSession; import org.apache.doris.connector.api.ConnectorStatementScope; +import org.apache.doris.connector.api.ConnectorStatementScopes; import org.apache.doris.thrift.TIcebergDeleteFileDesc; import org.apache.iceberg.Table; @@ -47,6 +48,19 @@ */ final class IcebergStatementScope { + /** + * Namespace for iceberg's per-statement RAW {@link Table} memo. Source-prefixed with the connector type + * ("iceberg") so it stays distinct across a heterogeneous gateway; see {@link ConnectorStatementScopes}. + */ + static final String TABLE_NAMESPACE = "iceberg.table"; + + /** + * Namespace for iceberg's per-statement rewritable-delete supply map (a per-statement singleton keyed by + * catalog id + queryId, with no db/table — it aggregates across all touched data files). Source-prefixed + * with the connector type ("iceberg"). + */ + static final String REWRITABLE_DELETE_SUPPLY_NAMESPACE = "iceberg.rewritable-delete-supply"; + private IcebergStatementScope() {} /** @@ -57,13 +71,11 @@ private IcebergStatementScope() {} * (the caller wraps this in {@code executeAuthenticated}). */ static Table sharedTable(ConnectorSession session, String dbName, String tableName, Supplier loader) { - if (session == null) { - // No session (offline / direct-construction tests): load every time, like ConnectorStatementScope.NONE. - return loader.get(); - } - String key = "iceberg.table:" + session.getCatalogId() + ":" + dbName + ":" + tableName - + ":" + session.getQueryId(); - return session.getStatementScope().computeIfAbsent(key, loader); + // Delegates to the shared per-statement resolver. The TABLE_NAMESPACE ("iceberg.table") reproduces the + // historical "iceberg.table:" key prefix byte-for-byte, so the funnel keeps identical hits / misses / NONE + // fall-through (proved by IcebergStatementScopeTest#sharedTableKeyReproducesLegacyPrefixByteForByte). + return ConnectorStatementScopes.resolveInStatement( + session, TABLE_NAMESPACE, dbName, tableName, loader); } /** @@ -78,7 +90,7 @@ static Map> rewritableDeleteSupply(Connecto // No session: a throwaway map that does NOT bridge scan->write (same as NONE). return new ConcurrentHashMap<>(); } - String key = "iceberg.rewritable-delete-supply:" + session.getCatalogId() + ":" + session.getQueryId(); + String key = REWRITABLE_DELETE_SUPPLY_NAMESPACE + ":" + session.getCatalogId() + ":" + session.getQueryId(); return session.getStatementScope().computeIfAbsent(key, ConcurrentHashMap::new); } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java index 8a6b53ef599787..426b706cf11f90 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java @@ -61,12 +61,8 @@ final class IcebergTableCache { private final MetaCacheEntry entry; IcebergTableCache(long ttlSeconds, int maxSize) { - // Mirror IcebergLatestSnapshotCache: translate the connector's "<= 0 disables" contract to CacheSpec's - // ttl == 0 (disabled) rather than passing a negative value through, which CacheSpec reads as ttl == -1 - // "no expiration (enabled)". - CacheSpec spec = ttlSeconds > 0 - ? CacheSpec.of(true, ttlSeconds, maxSize) - : CacheSpec.of(true, CacheSpec.CACHE_TTL_DISABLE_CACHE, maxSize); + // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). + CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = new MetaCacheEntry<>("iceberg-table", null, spec, ForkJoinPool.commonPool(), false, true, 0L, true); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java index 8d346be5c85f74..efa0dfdd691ca0 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java @@ -95,8 +95,8 @@ * per-statement {@link IcebergStatementScope} carried across the scan→write seam (commit-bridge S4 part 2), * replacing the legacy fe-resident rewritable-delete planner.

* - *

Gate-closed / dormant. Iceberg is not in {@code SPI_READY_TYPES} until P6.6, so nothing - * routes iceberg writes through this provider yet; {@link #planWrite} requires the executor-bound + *

Live since the iceberg SPI cutover. {@code iceberg} is in {@code SPI_READY_TYPES}, so a + * plugin-driven iceberg write routes through this provider; {@link #planWrite} requires the executor-bound * connector transaction and fails loud if absent.

*/ public class IcebergWritePlanProvider implements ConnectorWritePlanProvider { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergExecuteActionFactory.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergExecuteActionFactory.java index c7fea971f86c7e..6a901feb349a74 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergExecuteActionFactory.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/action/IcebergExecuteActionFactory.java @@ -83,8 +83,8 @@ public static BaseIcebergAction createAction(String actionType, Map + * connector {@link RewriteDataFilePlanner}. Live since the iceberg SPI cutover.

*/ public class IcebergRewriteDataFilesAction extends BaseIcebergAction { diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java index 1e7eeffae30bf6..0b2bef024a9c42 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java @@ -131,6 +131,118 @@ public void testRenameColumn() { Assertions.assertNotNull(reload().findField("full_name")); } + // ---------- case-insensitive name collision on the flat ops (#65329 flat arm) ---------- + + /** + * Creates db1.mixed with Spark-style mixed-case names ({@code Id}, {@code Label}, + * {@code Info STRUCT}) — the fixture of the regression suite + * {@code test_iceberg_nested_schema_evolution_spark_doris_interop}. Iceberg itself matches names + * case-SENSITIVELY, so without the connector guards a Doris user could add a second {@code id} next to + * {@code Id} and make the table unreadable through Doris (whose column names are case-insensitive). + */ + private void createMixedCaseTable() { + Schema schema = new Schema( + Types.NestedField.optional(1, "Id", Types.IntegerType.get()), + Types.NestedField.optional(2, "Label", Types.StringType.get()), + Types.NestedField.optional(3, "Info", Types.StructType.of( + Types.NestedField.optional(4, "Metric", Types.IntegerType.get())))); + ops.createTable("db1", "mixed", schema, PartitionSpec.unpartitioned(), null, + IcebergSchemaBuilder.buildTableProperties(Collections.emptyMap())); + } + + @Test + public void testAddColumnCaseInsensitiveCollisionFailsLoud() { + createMixedCaseTable(); + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> ops.addColumn("db1", "mixed", change("id", Types.StringType.get(), "", true), null)); + Assertions.assertTrue(ex.getMessage() + .contains("conflicts with existing Iceberg field 'Id' (case-insensitive)"), + ex.getMessage()); + Assertions.assertEquals(3, reload("mixed").columns().size()); + } + + @Test + public void testAddColumnsCaseInsensitiveCollisionFailsLoud() { + createMixedCaseTable(); + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> ops.addColumns("db1", "mixed", Arrays.asList( + change("ok_col", Types.StringType.get(), "", true), + change("id", Types.StringType.get(), "", true)))); + Assertions.assertTrue(ex.getMessage() + .contains("conflicts with existing Iceberg field 'Id' (case-insensitive)"), + ex.getMessage()); + // Whole batch validated before anything is staged: the valid sibling must not be committed either. + Assertions.assertEquals(3, reload("mixed").columns().size()); + } + + @Test + public void testAddColumnsDuplicateRequestedNamesFailLoud() { + createMixedCaseTable(); + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> ops.addColumns("db1", "mixed", Arrays.asList( + change("new_field", Types.StringType.get(), "", true), + change("NEW_FIELD", Types.StringType.get(), "", true)))); + Assertions.assertTrue(ex.getMessage() + .contains("conflicts with another requested column (case-insensitive)"), + ex.getMessage()); + Assertions.assertEquals(3, reload("mixed").columns().size()); + } + + @Test + public void testRenameColumnCaseInsensitiveCollisionFailsLoud() { + createMixedCaseTable(); + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> ops.renameColumn("db1", "mixed", "label", "id")); + Assertions.assertTrue(ex.getMessage() + .contains("conflicts with existing Iceberg field 'Id' (case-insensitive)"), + ex.getMessage()); + Assertions.assertNotNull(reload("mixed").findField("Label")); + } + + @Test + public void testRenameColumnResolvesSourceNameCaseInsensitively() { + createMixedCaseTable(); + // The user types the Doris-side (case-insensitive) name; it must resolve to the iceberg field 'Label' + // rather than hitting iceberg's case-sensitive "Cannot rename missing column". + ops.renameColumn("db1", "mixed", "label", "full_label"); + Assertions.assertNull(reload("mixed").findField("Label")); + Assertions.assertNotNull(reload("mixed").findField("full_label")); + } + + @Test + public void testRenameColumnCaseOnlySelfRenameSucceeds() { + createMixedCaseTable(); + // 'Label' -> 'label' collides with itself; the field-identity escape in the guard must let it through, + // otherwise a pure re-casing of a column would be impossible. + ops.renameColumn("db1", "mixed", "label", "label"); + Assertions.assertNotNull(reload("mixed").findField("label")); + Assertions.assertEquals(3, reload("mixed").columns().size()); + } + + @Test + public void testRenameColumnPreservesIdentifierFieldPath() { + // Iceberg drops an identifier-field path when the field is renamed; the flat rename must restate it + // (legacy IcebergMetadataOps.applyRenameColumn parity, already honoured by the nested arm). + Schema schema = new Schema( + Arrays.asList( + Types.NestedField.required(1, "Key", Types.IntegerType.get()), + Types.NestedField.optional(2, "v", Types.StringType.get())), + Collections.singleton(1)); + ops.createTable("db1", "ident", schema, PartitionSpec.unpartitioned(), null, + IcebergSchemaBuilder.buildTableProperties(Collections.emptyMap())); + ops.renameColumn("db1", "ident", "key", "renamed_key"); + Schema after = reload("ident"); + Assertions.assertEquals(Collections.singleton(after.findField("renamed_key").fieldId()), + after.identifierFieldIds()); + } + + @Test + public void testAddColumnKeepsWorkingWhenNoCollision() { + createMixedCaseTable(); + ops.addColumn("db1", "mixed", change("brand_new", Types.StringType.get(), "", true), null); + Assertions.assertNotNull(reload("mixed").findField("brand_new")); + } + // ---------- modifyColumn ---------- @Test diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorCacheTest.java index 5d5b5285f07d43..6abe09a0f52dfd 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorCacheTest.java @@ -18,8 +18,8 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.api.ConnectorPartitionInfo; -import org.apache.doris.connector.cache.ConnectorPartitionViewCache; -import org.apache.doris.connector.cache.PartitionViewCacheKey; +import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ConnectorTableKey; import org.apache.iceberg.DataFiles; import org.apache.iceberg.ManifestFile; @@ -499,16 +499,16 @@ public void refreshHooksInvalidatePartitionViewCache() { // re-run -> a loads assert below red. IcebergConnector connector = new IcebergConnector(Collections.emptyMap(), new RecordingConnectorContext()); - ConnectorPartitionViewCache> cache = connector.listPartitionsViewCacheForTest(); + ConnectorMetadataCache> cache = connector.listPartitionsViewCacheForTest(); Assertions.assertNotNull(cache); int[] loads = {0}; Supplier> loader = () -> { loads[0]++; return Collections.emptyList(); }; - PartitionViewCacheKey db1t1 = new PartitionViewCacheKey("db1", "t1", 1L, 1L); - PartitionViewCacheKey db1t2 = new PartitionViewCacheKey("db1", "t2", 1L, 1L); - PartitionViewCacheKey db2t1 = new PartitionViewCacheKey("db2", "t1", 1L, 1L); + ConnectorTableKey db1t1 = new ConnectorTableKey("db1", "t1", 1L, 1L); + ConnectorTableKey db1t2 = new ConnectorTableKey("db1", "t2", 1L, 1L); + ConnectorTableKey db2t1 = new ConnectorTableKey("db2", "t1", 1L, 1L); // REFRESH TABLE db1.t1 -> only db1.t1 re-loads. cache.get(db1t1, loader); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java index 0ca659a7bd24f1..e5cb30b0ed97d8 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java @@ -21,7 +21,7 @@ import org.apache.doris.connector.api.mvcc.ConnectorMvccPartition; import org.apache.doris.connector.api.mvcc.ConnectorMvccPartitionView; import org.apache.doris.connector.api.pushdown.ConnectorExpression; -import org.apache.doris.connector.cache.ConnectorPartitionViewCache; +import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileFormat; @@ -42,7 +42,7 @@ /** * PERF-06 tests for the cross-query DERIVED partition-view cache ("cache A", the generic - * {@link ConnectorPartitionViewCache}) wired into {@link IcebergConnectorMetadata#getMvccPartitionView} / + * {@link ConnectorMetadataCache}) wired into {@link IcebergConnectorMetadata#getMvccPartitionView} / * {@link IcebergConnectorMetadata#listPartitions}. Uses the real {@link InMemoryCatalog} + * {@link RecordingIcebergCatalogOps} harness (no Mockito, no docker): the cache sits ABOVE the per-query build, * whose first step is {@code resolveTableForRead -> catalogOps.loadTable} (logged as {@code loadTable:db1.t1}), so @@ -89,7 +89,7 @@ private static org.apache.iceberg.DataFile dayFile(PartitionSpec spec, String pa } private static IcebergConnectorMetadata metadataWithMvccCache(RecordingIcebergCatalogOps ops, - ConnectorPartitionViewCache cache) { + ConnectorMetadataCache cache) { // 9-arg ctor: disabled latest-snapshot cache + null table/partition/comment caches so the loadTable count // reflects cache A alone; only the mvcc view cache under test is injected. return new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext(), @@ -97,7 +97,7 @@ private static IcebergConnectorMetadata metadataWithMvccCache(RecordingIcebergCa } private static IcebergConnectorMetadata metadataWithListCache(RecordingIcebergCatalogOps ops, - ConnectorPartitionViewCache> cache) { + ConnectorMetadataCache> cache) { return new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext(), new IcebergLatestSnapshotCache(0L, 1), null, null, null, null, cache); } @@ -110,12 +110,12 @@ private static long loadCount(RecordingIcebergCatalogOps ops) { return ops.log.stream().filter(s -> s.equals("loadTable:db1.t1")).count(); } - private static ConnectorPartitionViewCache mvccCache() { - return new ConnectorPartitionViewCache<>("iceberg", Collections.emptyMap()); + private static ConnectorMetadataCache mvccCache() { + return new ConnectorMetadataCache<>("iceberg", "partition_view", Collections.emptyMap()); } - private static ConnectorPartitionViewCache> listCache() { - return new ConnectorPartitionViewCache<>("iceberg", Collections.emptyMap()); + private static ConnectorMetadataCache> listCache() { + return new ConnectorMetadataCache<>("iceberg", "partition_view", Collections.emptyMap()); } private static List mvccNames(Optional view) { @@ -135,7 +135,7 @@ public void getMvccPartitionViewCachesDerivedViewAcrossQueries() { TwoSnap f = twoSnapshotTable(); RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = f.table; - ConnectorPartitionViewCache cache = mvccCache(); + ConnectorMetadataCache cache = mvccCache(); IcebergConnectorMetadata md = metadataWithMvccCache(ops, cache); Optional first = md.getMvccPartitionView(null, handle()); @@ -174,7 +174,7 @@ public void getMvccPartitionViewInvalidateTableForcesReEnumeration() { TwoSnap f = twoSnapshotTable(); RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = f.table; - ConnectorPartitionViewCache cache = mvccCache(); + ConnectorMetadataCache cache = mvccCache(); IcebergConnectorMetadata md = metadataWithMvccCache(ops, cache); md.getMvccPartitionView(null, handle()); @@ -208,7 +208,7 @@ public void listPartitionsCachesDerivedListAcrossQueries() { TwoSnap f = twoSnapshotTable(); RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = f.table; - ConnectorPartitionViewCache> cache = listCache(); + ConnectorMetadataCache> cache = listCache(); IcebergConnectorMetadata md = metadataWithListCache(ops, cache); List first = md.listPartitions(null, handle(), Optional.empty()); @@ -229,7 +229,7 @@ public void listPartitionsWithFilterBypassesCache() { TwoSnap f = twoSnapshotTable(); RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = f.table; - ConnectorPartitionViewCache> cache = listCache(); + ConnectorMetadataCache> cache = listCache(); IcebergConnectorMetadata md = metadataWithListCache(ops, cache); ConnectorExpression filter = Collections::emptyList; // any non-empty filter @@ -249,7 +249,7 @@ public void listPartitionsInvalidateAllForcesReEnumeration() { TwoSnap f = twoSnapshotTable(); RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = f.table; - ConnectorPartitionViewCache> cache = listCache(); + ConnectorMetadataCache> cache = listCache(); IcebergConnectorMetadata md = metadataWithListCache(ops, cache); md.listPartitions(null, handle(), Optional.empty()); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergStatementScopeTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergStatementScopeTest.java index 70238ea323f82d..c7050a6a24940a 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergStatementScopeTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergStatementScopeTest.java @@ -28,10 +28,13 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; /** * Tests for {@link IcebergStatementScope}: the per-statement table memo keying (catalog id + db + table + @@ -102,6 +105,34 @@ public void underNoneScopeLoadsEveryTime() { Assertions.assertEquals(2, loads.get(), "NONE -> load every time"); } + @Test + public void underNullSessionLoadsEveryTime() { + // Offline / direct-construction (null session): sharedTable loads every time, byte-identical to the + // pre-scope behavior. The null branch now lives in the shared helper; assert it still holds at the seam. + AtomicInteger loads = new AtomicInteger(); + IcebergStatementScope.sharedTable(null, "db1", "t", () -> { + loads.incrementAndGet(); + return table("t"); + }); + IcebergStatementScope.sharedTable(null, "db1", "t", () -> { + loads.incrementAndGet(); + return table("t"); + }); + Assertions.assertEquals(2, loads.get(), "null session -> load every time"); + } + + @Test + public void sharedTableKeyReproducesLegacyPrefixByteForByte() { + // PARITY (PR-2): sharedTable now delegates to ConnectorStatementScopes.resolveInStatement; the memo key it + // hands the scope MUST stay byte-identical to the pre-delegation + // "iceberg.table:" + catalogId + ":" + db + ":" + table + ":" + queryId, or funnel hits/misses shift. + // MUTATION: a different namespace, a dropped field, or a reordered field -> key differs -> red. + KeyCapturingScope scope = new KeyCapturingScope(); + IcebergStatementScope.sharedTable(new ScopeSession(7L, "q1", scope), "db1", "t", () -> table("t")); + Assertions.assertEquals("iceberg.table:7:db1:t:q1", scope.lastKey, + "delegated key must reproduce the legacy iceberg.table prefix byte-for-byte"); + } + @Test public void rewritableDeleteSupplyIsSharedPerStatementAndIsolatedPerCatalog() { // The scan seam and the write seam of one statement (same catalog + queryId) share ONE supply map; a @@ -119,6 +150,39 @@ public void rewritableDeleteSupplyIsSharedPerStatementAndIsolatedPerCatalog() { Assertions.assertNotSame(supplyScan, supplyOtherCatalog, "a different catalog (cross-catalog MERGE) is isolated"); } + @Test + public void allNamespacesArePrefixedWithConnectorType() throws Exception { + // NORM (self-extending): reflect over every "*_NAMESPACE" constant this connector declares and assert each + // is prefixed with the connector's ConnectorProvider.getType() ("iceberg."). Source-prefixing keeps the + // namespaces distinct across connectors on a heterogeneous gateway (no ClassCastException on the shared + // coordinate). Reflecting means a NEW namespace is auto-covered; a forgotten prefix or a getType() drift + // turns this red with no test upkeep. + String prefix = new IcebergConnectorProvider().getType() + "."; + int checked = 0; + for (Field f : IcebergStatementScope.class.getDeclaredFields()) { + if (Modifier.isStatic(f.getModifiers()) && f.getType() == String.class + && f.getName().endsWith("_NAMESPACE")) { + f.setAccessible(true); + String ns = (String) f.get(null); + Assertions.assertTrue(ns.startsWith(prefix), + f.getName() + " (\"" + ns + "\") must be prefixed with the connector type \"" + prefix + "\""); + checked++; + } + } + Assertions.assertTrue(checked > 0, "expected at least one *_NAMESPACE constant to guard"); + } + + /** A scope that records the last key handed to {@link #computeIfAbsent}, for the byte-key parity assertion. */ + private static final class KeyCapturingScope implements ConnectorStatementScope { + private String lastKey; + + @Override + public T computeIfAbsent(String key, Supplier loader) { + lastKey = key; + return loader.get(); + } + } + /** Minimal {@link ConnectorSession} carrying a catalog id, queryId and scope for the key + memo assertions. */ private static final class ScopeSession implements ConnectorSession { private final long catalogId; @@ -141,6 +205,14 @@ public String getQueryId() { return queryId; } + @Override + public String getSessionId() { + // Deliberately != queryId. The memo key MUST use the per-EXECUTION queryId (cross-query isolation), not + // the stable per-connection sessionId; a queryId->sessionId swap in the key would share a table across + // queries of one connection and MUST turn sharedTableKeyReproducesLegacyPrefixByteForByte red. + return "session-" + queryId; + } + @Override public ConnectorStatementScope getStatementScope() { return scope; diff --git a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadata.java b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadata.java index c20a60318cf7a5..4897367e57fa06 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadata.java @@ -20,6 +20,7 @@ import org.apache.doris.connector.api.ConnectorColumn; import org.apache.doris.connector.api.ConnectorMetadata; import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorStatementScopes; import org.apache.doris.connector.api.ConnectorTableSchema; import org.apache.doris.connector.api.ConnectorTableStatistics; import org.apache.doris.connector.api.ConnectorType; @@ -49,6 +50,14 @@ public class JdbcConnectorMetadata implements ConnectorMetadata { private static final Logger LOG = LogManager.getLogger(JdbcConnectorMetadata.class); + /** + * Namespace for jdbc's per-statement RAW remote-columns memo (a {@code List}), shared by the + * schema path ({@link #getTableSchema}), the scan column-handle path ({@link #getColumnHandles}) and the write + * INSERT-SQL shaping ({@code JdbcWritePlanProvider#buildInsertSql}). Source-prefixed with the connector type + * ("jdbc") so it stays distinct across a heterogeneous gateway; see {@link ConnectorStatementScopes}. + */ + static final String COLUMNS_NAMESPACE = "jdbc.columns"; + private final JdbcConnectorClient client; private final Map properties; @@ -116,7 +125,9 @@ public ConnectorTableSchema getTableSchema( String dbName = jdbcHandle.getRemoteDbName(); String tableName = jdbcHandle.getRemoteTableName(); - List fields = client.getJdbcColumnsInfo(dbName, tableName); + List fields = ConnectorStatementScopes.resolveInStatement( + session, COLUMNS_NAMESPACE, dbName, tableName, + () -> client.getJdbcColumnsInfo(dbName, tableName)); List columns = new ArrayList<>(fields.size()); for (JdbcFieldInfo field : fields) { @@ -159,7 +170,9 @@ public Map getColumnHandles( String tableName = jdbcHandle.getRemoteTableName(); JdbcIdentifierMapper mapper = getIdentifierMapper(session); - List fields = client.getJdbcColumnsInfo(dbName, tableName); + List fields = ConnectorStatementScopes.resolveInStatement( + session, COLUMNS_NAMESPACE, dbName, tableName, + () -> client.getJdbcColumnsInfo(dbName, tableName)); Map handles = new LinkedHashMap<>(fields.size()); for (JdbcFieldInfo field : fields) { String remoteName = field.getColumnName(); diff --git a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcWritePlanProvider.java b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcWritePlanProvider.java index 9ed3d56df57dd3..9fc66961d2e5ed 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcWritePlanProvider.java +++ b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcWritePlanProvider.java @@ -126,6 +126,12 @@ public void appendExplainInfo(StringBuilder output, String prefix, * {@link #appendExplainInfo} so the EXPLAIN SQL is identical to the one sent to BE. Resolves * the local -> remote column name mapping via the metadata column handles (same client + * properties as the legacy {@code getWriteConfig}, which called its own getColumnHandles). + * + *

The {@code new JdbcConnectorMetadata} is a cheap stateless wrapper; its + * {@link JdbcConnectorMetadata#getColumnHandles} resolves the raw remote-column fetch through the + * per-statement scope memo ({@code JdbcConnectorMetadata.COLUMNS_NAMESPACE}), so it shares the single + * {@code getJdbcColumnsInfo} round-trip with the read path and with the sibling + * planWrite/appendExplainInfo call of the same statement — EXPLAIN INSERT no longer double-fetches. */ private String buildInsertSql(ConnectorSession session, JdbcTableHandle jdbcHandle, List columns) { diff --git a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadataTest.java b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadataTest.java index d431873e8b69e0..0546eb09ee09ba 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadataTest.java +++ b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadataTest.java @@ -17,15 +17,30 @@ package org.apache.doris.connector.jdbc; +import org.apache.doris.connector.api.ConnectorColumn; import org.apache.doris.connector.api.ConnectorPushdownOps; import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorStatementScope; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.jdbc.client.JdbcConnectorClient; +import org.apache.doris.connector.jdbc.client.JdbcFieldInfo; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; /** * Unit tests for {@link JdbcConnectorMetadata}. @@ -33,6 +48,10 @@ class JdbcConnectorMetadataTest { private ConnectorSession sessionWithProps(Map props) { + return sessionWithScope(props, ConnectorStatementScope.NONE); + } + + private ConnectorSession sessionWithScope(Map props, ConnectorStatementScope scope) { return new ConnectorSession() { @Override public String getQueryId() { @@ -78,9 +97,92 @@ public Map getCatalogProperties() { public Map getSessionProperties() { return props; } + + @Override + public ConnectorStatementScope getStatementScope() { + return scope; + } + }; + } + + /** Live per-statement scope (map-backed) so resolveInStatement memoizes within the statement. */ + private static ConnectorStatementScope liveScope() { + return new ConnectorStatementScope() { + private final Map arena = new HashMap<>(); + + @Override + @SuppressWarnings("unchecked") + public T computeIfAbsent(String key, Supplier loader) { + return (T) arena.computeIfAbsent(key, k -> loader.get()); + } }; } + /** Test double that counts the remote column fetches (getJdbcColumnsInfo round-trips). */ + private static final class CountingJdbcClient extends JdbcConnectorClient { + private final AtomicInteger columnsFetches = new AtomicInteger(); + private final List fields; + + private CountingJdbcClient(List fields) { + super("test_catalog", JdbcDbType.MYSQL, "jdbc:mysql://h:3306/test_db", + false, null, null, false, false); + this.fields = fields; + } + + @Override + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { + columnsFetches.incrementAndGet(); + return fields; + } + + @Override + public ConnectorType jdbcTypeToConnectorType(JdbcFieldInfo fieldInfo) { + return ConnectorType.of("INT"); + } + } + + private static JdbcFieldInfo field(String name) { + return new JdbcFieldInfo(name, Optional.empty(), 0, + Optional.empty(), Optional.empty(), Optional.empty()); + } + + /** + * Like {@link CountingJdbcClient} but its type conversion MUTATES the field in place (allowNull=true) for + * column "d", mirroring the real MySQL/ClickHouse overrides that run on the now-shared raw memo. + */ + private static final class MutatingJdbcClient extends JdbcConnectorClient { + private final AtomicInteger columnsFetches = new AtomicInteger(); + private final List fields; + + private MutatingJdbcClient(List fields) { + super("test_catalog", JdbcDbType.MYSQL, "jdbc:mysql://h:3306/test_db", + false, null, null, false, false); + this.fields = fields; + } + + @Override + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { + columnsFetches.incrementAndGet(); + return fields; + } + + @Override + public ConnectorType jdbcTypeToConnectorType(JdbcFieldInfo fieldInfo) { + if ("d".equals(fieldInfo.getColumnName())) { + fieldInfo.setAllowNull(true); // idempotent, mirrors the real convertDateToNull path + } + return ConnectorType.of("INT"); + } + } + + private static List columnSummary(ConnectorTableSchema schema) { + List out = new ArrayList<>(); + for (ConnectorColumn c : schema.getColumns()) { + out.add(c.getName() + ":" + c.isNullable()); + } + return out; + } + @Test void testSupportsCastPredicatePushdown_defaultTrue() { JdbcConnectorMetadata metadata = new JdbcConnectorMetadata(null, Collections.emptyMap()); @@ -112,4 +214,88 @@ void testDefaultPushdownOps_alwaysTrue() { ConnectorSession session = sessionWithProps(Collections.emptyMap()); Assertions.assertTrue(defaultOps.supportsCastPredicatePushdown(session)); } + + @Test + void columnHandlesAndSchemaShareOneRemoteFetchPerStatement() { + // WHY (HP-1): within one statement the scan-path getColumnHandles (called ~2x per scan node) and a + // schema-cache-miss getTableSchema each hit client.getJdbcColumnsInfo -- a remote + // DatabaseMetaData.getColumns round-trip. Routing the raw fetch through the per-statement scope memo + // (JdbcConnectorMetadata.COLUMNS_NAMESPACE, keyed by (catalogId, db, table, queryId)) collapses them to + // ONE remote fetch. MUTATION: fetching directly on each call (pre-fix) -> counter 3 -> red. + CountingJdbcClient client = new CountingJdbcClient(Arrays.asList(field("id"), field("name"))); + JdbcConnectorMetadata md = new JdbcConnectorMetadata(client, Collections.emptyMap()); + ConnectorSession session = sessionWithScope(Collections.emptyMap(), liveScope()); + JdbcTableHandle handle = new JdbcTableHandle("db", "t"); + + Map handles = md.getColumnHandles(session, handle); + md.getColumnHandles(session, handle); + md.getTableSchema(session, handle); + + Assertions.assertEquals(1, client.columnsFetches.get(), + "getColumnHandles x2 + getTableSchema must share ONE remote column fetch per statement"); + Assertions.assertEquals(2, handles.size()); + Assertions.assertTrue(handles.containsKey("id") && handles.containsKey("name"), + "the mapping is still applied per call on the shared raw fetch"); + } + + @Test + void noneScopeFetchesColumnsEveryCall() { + // Parity: under ConnectorStatementScope.NONE (offline / no live statement -- the default session scope), + // the memo runs the loader on every call, byte-identical to the pre-fix always-fetch behavior. + // MUTATION: memoizing under NONE -> counter 1 -> red. + CountingJdbcClient client = new CountingJdbcClient(Collections.singletonList(field("id"))); + JdbcConnectorMetadata md = new JdbcConnectorMetadata(client, Collections.emptyMap()); + ConnectorSession session = sessionWithProps(Collections.emptyMap()); // default scope = NONE + + JdbcTableHandle handle = new JdbcTableHandle("db", "t"); + md.getColumnHandles(session, handle); + md.getColumnHandles(session, handle); + + Assertions.assertEquals(2, client.columnsFetches.get(), + "under NONE scope each call must fetch (no cross-call memo) -- parity with pre-fix"); + } + + @Test + void getTableSchemaStableOverSharedMutatedMemoUnderMutatingTypeConversion() { + // WHY: getTableSchema's jdbcTypeToConnectorType mutates the field in place (allowNull->true for some + // date types); under the per-statement memo the raw list is SHARED across getTableSchema/getColumnHandles, + // so a later getTableSchema reads an already-mutated field. Pin that the mutation is idempotent and + // non-corrupting: two getTableSchema calls over the shared memo (with a getColumnHandles between, which + // reads only names) yield byte-identical columns and reflect allowNull=true for the mutated column "d". + // MUTATION: a non-idempotent field mutation, or getColumnHandles depending on the flag -> schemas diverge. + MutatingJdbcClient client = new MutatingJdbcClient(Arrays.asList(field("d"), field("id"))); + JdbcConnectorMetadata md = new JdbcConnectorMetadata(client, Collections.emptyMap()); + ConnectorSession session = sessionWithScope(Collections.emptyMap(), liveScope()); + JdbcTableHandle handle = new JdbcTableHandle("db", "t"); + + List first = columnSummary(md.getTableSchema(session, handle)); + md.getColumnHandles(session, handle); + List second = columnSummary(md.getTableSchema(session, handle)); + + Assertions.assertEquals(1, client.columnsFetches.get(), "one shared fetch across all three calls"); + Assertions.assertEquals(first, second, "getTableSchema over the shared (mutated) memo is stable"); + Assertions.assertTrue(first.contains("d:true"), + "the mutating type conversion's idempotent allowNull=true is reflected in the schema"); + } + + @Test + void allNamespacesArePrefixedWithConnectorType() throws Exception { + // NORM (self-extending): reflect over every "*_NAMESPACE" constant this connector declares and assert each + // is prefixed with the connector's ConnectorProvider.getType() ("jdbc."). Source-prefixing keeps it distinct + // from every other connector's namespaces on a heterogeneous gateway. Reflecting means a NEW namespace is + // auto-covered; a forgotten prefix or a getType() drift turns this red. + String prefix = new JdbcConnectorProvider().getType() + "."; + int checked = 0; + for (Field f : JdbcConnectorMetadata.class.getDeclaredFields()) { + if (Modifier.isStatic(f.getModifiers()) && f.getType() == String.class + && f.getName().endsWith("_NAMESPACE")) { + f.setAccessible(true); + String ns = (String) f.get(null); + Assertions.assertTrue(ns.startsWith(prefix), + f.getName() + " (\"" + ns + "\") must be prefixed with the connector type \"" + prefix + "\""); + checked++; + } + } + Assertions.assertTrue(checked > 0, "expected at least one *_NAMESPACE constant to guard"); + } } diff --git a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcWritePlanProviderTest.java b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcWritePlanProviderTest.java index 62a698c3f0839a..46f6ff2c013a92 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcWritePlanProviderTest.java +++ b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcWritePlanProviderTest.java @@ -19,6 +19,7 @@ import org.apache.doris.connector.api.ConnectorColumn; import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorStatementScope; import org.apache.doris.connector.api.ConnectorType; import org.apache.doris.connector.api.handle.ConnectorTableHandle; import org.apache.doris.connector.api.handle.ConnectorWriteHandle; @@ -40,6 +41,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; /** * Byte-parity tests for {@link JdbcWritePlanProvider} (P6.3-T02 / RFC OQ-1). @@ -59,6 +62,7 @@ class JdbcWritePlanProviderTest { */ private static final class FakeJdbcClient extends JdbcConnectorClient { private final List fields; + private final AtomicInteger columnsFetches = new AtomicInteger(); private FakeJdbcClient(JdbcDbType dbType, List fields) { super("test_catalog", dbType, "jdbc:mysql://h:3306/test_db", @@ -68,6 +72,7 @@ private FakeJdbcClient(JdbcDbType dbType, List fields) { @Override public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { + columnsFetches.incrementAndGet(); return fields; } @@ -111,6 +116,23 @@ public Map getWriteContext() { } private static ConnectorSession session(long catalogId, Map sessionProps) { + return session(catalogId, sessionProps, ConnectorStatementScope.NONE); + } + + private static ConnectorStatementScope liveScope() { + return new ConnectorStatementScope() { + private final Map arena = new HashMap<>(); + + @Override + @SuppressWarnings("unchecked") + public T computeIfAbsent(String key, Supplier loader) { + return (T) arena.computeIfAbsent(key, k -> loader.get()); + } + }; + } + + private static ConnectorSession session(long catalogId, Map sessionProps, + ConnectorStatementScope scope) { return new ConnectorSession() { @Override public String getQueryId() { @@ -156,6 +178,11 @@ public Map getCatalogProperties() { public Map getSessionProperties() { return sessionProps; } + + @Override + public ConnectorStatementScope getStatementScope() { + return scope; + } }; } @@ -268,4 +295,56 @@ void planWritePoolAndTxnFieldsFallBackToLegacyDefaultsWhenAbsent() { // enable_odbc_transcation absent -> false (note the legacy session-key spelling preserved). Assertions.assertFalse(sink.isUseTransaction()); } + + @Test + void explainInsertSharesOneColumnFetchWithinStatement() { + // WHY (HP-2): planWrite and appendExplainInfo each shape the INSERT SQL via getColumnHandles -> + // getJdbcColumnsInfo; an EXPLAIN INSERT fires both on the same session -> two remote column fetches + // before. Routing that fetch through the per-statement scope collapses them to ONE. MUTATION: newing a + // fresh stateless JdbcConnectorMetadata that re-fetches (pre-fix) -> counter 2 -> red. + Map props = new HashMap<>(); + props.put("jdbc_url", "jdbc:mysql://h:3306/test_db"); + FakeJdbcClient client = new FakeJdbcClient( + JdbcDbType.MYSQL, Arrays.asList(field("id"), field("name"))); + JdbcWritePlanProvider provider = new JdbcWritePlanProvider(client, props); + ConnectorSession session = session(7L, Collections.emptyMap(), liveScope()); + ConnectorWriteHandle handle = writeHandle( + new JdbcTableHandle("test_db", "t1"), Arrays.asList("id", "name")); + + ConnectorSinkPlan plan = provider.planWrite(session, handle); + StringBuilder sb = new StringBuilder(); + provider.appendExplainInfo(sb, " ", session, handle); + + Assertions.assertEquals(1, client.columnsFetches.get(), + "planWrite + appendExplainInfo must share ONE remote column fetch per statement"); + // byte-parity of the produced SQL is unaffected by the memo + Assertions.assertEquals("INSERT INTO `test_db`.`t1`(`id`,`name`) VALUES (?, ?)", + plan.getDataSink().getJdbcTableSink().getInsertSql()); + Assertions.assertTrue(sb.toString().contains( + "INSERT SQL: INSERT INTO `test_db`.`t1`(`id`,`name`) VALUES (?, ?)")); + } + + @Test + void scanAndWriteShareOneColumnFetchWithinStatement() { + // WHY (HP-1 + HP-2 composition): because the memo lives in the statement SCOPE (not the metadata + // instance), the scan-path getColumnHandles and the write-path buildInsertSql -- even though the write + // provider news up its own JdbcConnectorMetadata -- share the ONE fetch for the same (catalog, db, + // table, query). MUTATION: an instance-level memo instead of scope-keyed -> the write's fresh instance + // misses -> counter 2 -> red. + Map props = new HashMap<>(); + props.put("jdbc_url", "jdbc:mysql://h:3306/test_db"); + FakeJdbcClient client = new FakeJdbcClient( + JdbcDbType.MYSQL, Arrays.asList(field("id"), field("name"))); + ConnectorSession session = session(7L, Collections.emptyMap(), liveScope()); + JdbcTableHandle tableHandle = new JdbcTableHandle("test_db", "t1"); + + // scan path resolves column handles on the funnel metadata + new JdbcConnectorMetadata(client, props).getColumnHandles(session, tableHandle); + // write path news up its own metadata inside buildInsertSql, yet shares the same scope entry + JdbcWritePlanProvider provider = new JdbcWritePlanProvider(client, props); + provider.planWrite(session, writeHandle(tableHandle, Arrays.asList("id", "name"))); + + Assertions.assertEquals(1, client.columnsFetches.get(), + "scan getColumnHandles and write buildInsertSql must share ONE fetch (scope-keyed, not instance)"); + } } diff --git a/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeConnectorMetadata.java b/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeConnectorMetadata.java index f70935a73e593b..f07ed7ea2430ef 100644 --- a/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeConnectorMetadata.java @@ -54,6 +54,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; /** * ConnectorMetadata implementation for MaxCompute. @@ -78,6 +79,15 @@ public class MaxComputeConnectorMetadata implements ConnectorMetadata { private final Map properties; private final MaxComputePartitionCache partitionCache; + // Per-statement table-handle memo. This metadata instance is created fresh per statement + // (MaxComputeDorisConnector.getMetadata), so a table resolved once -- one remote exists() + // probe plus one lazy ODPS Table -- is reused by every planning site in the same statement + // instead of each site re-probing and rebuilding it (killing the redundant exists() round + // trips and the per-Table schema reload). ConcurrentHashMap because a scan reuses the same + // per-statement session (hence this instance) across off-thread pool tasks. Key is the + // (dbName, tableName) pair via List's value equality -- no separator collision to reason about. + private final Map, MaxComputeTableHandle> tableHandleMemo = new ConcurrentHashMap<>(); + public MaxComputeConnectorMetadata(Odps odps, McStructureHelper structureHelper, String defaultProject, @@ -118,15 +128,24 @@ public boolean tableExists(ConnectorSession session, String dbName, @Override public Optional getTableHandle( ConnectorSession session, String dbName, String tableName) { - if (!structureHelper.tableExist(odps, dbName, tableName)) { - return Optional.empty(); - } - Table odpsTable = structureHelper.getOdpsTable( - odps, dbName, tableName); - TableIdentifier tableId = structureHelper.getTableIdentifier( - dbName, tableName); - return Optional.of(new MaxComputeTableHandle( - dbName, tableName, odpsTable, tableId)); + // Only present tables are memoized: returning null from the mapping function records no + // mapping, so an absent table re-probes on every call -- keeping the Optional.empty() + // ("table not found") behavior byte-identical to before. computeIfAbsent runs the + // mapping function at most once per key even under a concurrent first touch, so the + // exists() probe fires once per (db, table) per statement. + MaxComputeTableHandle handle = tableHandleMemo.computeIfAbsent( + List.of(dbName, tableName), k -> { + if (!structureHelper.tableExist(odps, dbName, tableName)) { + return null; + } + Table odpsTable = structureHelper.getOdpsTable( + odps, dbName, tableName); + TableIdentifier tableId = structureHelper.getTableIdentifier( + dbName, tableName); + return new MaxComputeTableHandle( + dbName, tableName, odpsTable, tableId); + }); + return Optional.ofNullable(handle); } @Override @@ -323,10 +342,9 @@ public boolean supportsCastPredicatePushdown(ConnectorSession session) { * matches the id registered in the engine transaction registry and stamped * into the data sink (see {@link MaxComputeConnectorTransaction}). * - *

Gate-closed / dormant until the {@code max_compute} cutover: nothing - * routes plugin-driven MaxCompute writes through this path yet. The ODPS - * write session that backs commit / block allocation is created by the write - * plan (P4-T04), which binds it via + *

Live since the MaxCompute cutover: plugin-driven MaxCompute writes route + * through this path. The ODPS write session that backs commit / block + * allocation is created by the write plan, which binds it via * {@link MaxComputeConnectorTransaction#setWriteSession}.

*/ @Override diff --git a/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeConnectorTransaction.java b/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeConnectorTransaction.java index deec881c53652f..af798df4492c72 100644 --- a/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeConnectorTransaction.java @@ -52,12 +52,12 @@ * write session — the session id / target identifier / environment settings used * by {@link #commit()}.

* - *

Gate-closed / dormant. Nothing routes plugin-driven MaxCompute writes - * through this class until the {@code max_compute} cutover: the executor wiring + *

Live since the MaxCompute cutover. Plugin-driven MaxCompute writes + * route through this class: the executor wiring * ({@code beginTransaction} → {@code PluginDrivenTransactionManager.begin}) - * and {@code GlobalExternalTransactionInfoMgr} registration are deferred to that - * step. {@link #commit()} depends on the write-session state populated by P4-T04 - * (via {@link #setWriteSession}); it is intentionally not runnable before then.

+ * and {@code GlobalExternalTransactionInfoMgr} registration are in place. + * {@link #commit()} depends on the write-session state populated by the write plan + * (via {@link #setWriteSession}).

*/ public class MaxComputeConnectorTransaction implements ConnectorTransaction, WriteBlockAllocatingConnectorTransaction { diff --git a/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeWritePlanProvider.java b/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeWritePlanProvider.java index f8da97fed69ef7..438897b3046930 100644 --- a/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeWritePlanProvider.java +++ b/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeWritePlanProvider.java @@ -71,10 +71,9 @@ * engine transaction ({@link MaxComputeConnectorTransaction#allocateWriteBlockRange}) * keyed by {@code txn_id}.

* - *

Gate-closed / dormant. Nothing routes plugin-driven MaxCompute writes - * through this provider until the {@code max_compute} cutover. In particular - * {@link #planWrite} requires the session to carry the connector transaction - * (bound by the executor wiring added at cutover); it fails loud if absent.

+ *

Live since the MaxCompute cutover. A plugin-driven MaxCompute write + * routes through this provider. {@link #planWrite} requires the session to carry + * the connector transaction (bound by the executor wiring); it fails loud if absent.

*/ public class MaxComputeWritePlanProvider implements ConnectorWritePlanProvider { diff --git a/fe/fe-connector/fe-connector-maxcompute/src/test/java/org/apache/doris/connector/maxcompute/MaxComputeConnectorMetadataHandleMemoTest.java b/fe/fe-connector/fe-connector-maxcompute/src/test/java/org/apache/doris/connector/maxcompute/MaxComputeConnectorMetadataHandleMemoTest.java new file mode 100644 index 00000000000000..7d3bf31c721bae --- /dev/null +++ b/fe/fe-connector/fe-connector-maxcompute/src/test/java/org/apache/doris/connector/maxcompute/MaxComputeConnectorMetadataHandleMemoTest.java @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.maxcompute; + +import org.apache.doris.connector.api.handle.ConnectorTableHandle; + +import com.aliyun.odps.Odps; +import com.aliyun.odps.OdpsException; +import com.aliyun.odps.Partition; +import com.aliyun.odps.Table; +import com.aliyun.odps.TableSchema; +import com.aliyun.odps.Tables; +import com.aliyun.odps.table.TableIdentifier; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * Guards the per-statement table-handle memo added to {@link MaxComputeConnectorMetadata}. + * + *

WHY this matters: a single MaxCompute table is resolved by ~a dozen independent + * planning sites within one statement (scan build, partition pruning, row-count, per-column + * stats, write-capability probes, SHOW CREATE). Each {@code getTableHandle} used to fire a + * fresh remote ODPS {@code tables().exists()} probe and build a new lazy {@code Table} whose + * first schema access triggers its own reload. Because the metadata instance is created fresh + * per statement (funnel-memoized one-per-statement), memoizing the resolved handle collapses + * those repeats to one probe + one Table per (db, table) per statement, and makes read and + * write share the same already-warmed Table.

+ * + *

The maxcompute test module builds no live ODPS client, so a hand-written counting + * {@link McStructureHelper} records how many times the remote probe / lazy-Table build fire. + * {@code getTableHandle} never dereferences {@code odps} directly (it only forwards it to the + * helper), so a {@code null} odps keeps the test offline — the same pattern as + * {@link MaxComputeConnectorMetadataDropDbTest}.

+ */ +public class MaxComputeConnectorMetadataHandleMemoTest { + + private MaxComputeConnectorMetadata metadataWith(McStructureHelper helper) { + return new MaxComputeConnectorMetadata( + null /* odps */, helper, "proj", "ep", "quota", Collections.emptyMap(), + null); // null: partition cache unused by this test + } + + @Test + public void sameTableResolvedTwiceProbesAndBuildsOnce() { + CountingStructureHelper helper = new CountingStructureHelper("db1.t1"); + MaxComputeConnectorMetadata metadata = metadataWith(helper); + + Optional first = metadata.getTableHandle(null, "db1", "t1"); + Optional second = metadata.getTableHandle(null, "db1", "t1"); + + // WHY: the second resolution of the same table in one statement must cost ZERO remote + // round trips and reuse the identical handle (hence the identical lazy Table, so the + // schema reload happens once). MUTATION: dropping the memo makes the second call + // re-probe/rebuild -> both lists size 2 (red) and the handles become distinct + // (assertSame red). + Assertions.assertTrue(first.isPresent() && second.isPresent(), + "an existing table must resolve to a present handle"); + Assertions.assertEquals(Collections.singletonList("db1.t1"), helper.existProbes, + "a table resolved twice in one statement must hit the remote exists() probe only once"); + Assertions.assertEquals(Collections.singletonList("db1.t1"), helper.odpsTableBuilds, + "the lazy ODPS Table must be built once and shared, not rebuilt per resolution"); + Assertions.assertSame(first.get(), second.get(), + "both resolutions must return the identical memoized handle (shared Table => shared reload)"); + } + + @Test + public void distinctTablesAreResolvedIndependently() { + CountingStructureHelper helper = new CountingStructureHelper("db1.t1", "db1.t2"); + MaxComputeConnectorMetadata metadata = metadataWith(helper); + + ConnectorTableHandle h1 = metadata.getTableHandle(null, "db1", "t1").orElseThrow(); + ConnectorTableHandle h2 = metadata.getTableHandle(null, "db1", "t2").orElseThrow(); + + // WHY: distinct (db, table) keys must not collide in the memo — each table is resolved + // exactly once on its own. This pins the value-equality List key (a broken key that + // collapsed the two names would return one shared handle -> assertNotSame red). + Assertions.assertNotSame(h1, h2, "different tables must resolve to different handles"); + Assertions.assertEquals(Arrays.asList("db1.t1", "db1.t2"), helper.existProbes, + "each distinct table is probed once"); + Assertions.assertEquals(Arrays.asList("db1.t1", "db1.t2"), helper.odpsTableBuilds, + "each distinct table builds its Table once"); + } + + @Test + public void sameTableNameInDifferentDatabasesDoNotCollide() { + CountingStructureHelper helper = new CountingStructureHelper("db1.t", "db2.t"); + MaxComputeConnectorMetadata metadata = metadataWith(helper); + + MaxComputeTableHandle a = (MaxComputeTableHandle) metadata.getTableHandle(null, "db1", "t").orElseThrow(); + MaxComputeTableHandle b = (MaxComputeTableHandle) metadata.getTableHandle(null, "db2", "t").orElseThrow(); + + // WHY: the memo key must include the database, not just the table name. A statement that + // joins db1.t and db2.t routes both resolutions through the one per-statement metadata + // instance; a db-blind key would hand db1's handle back for db2.t -> the wrong project's + // Table resolved silently. MUTATION: keying on tableName only makes the second call hit + // db1's entry -> b == a (assertNotSame red), b.getDbName() == "db1" (red), and existProbes + // drops to size 1 (red). + Assertions.assertNotSame(a, b, "same table name in different databases must not share a memo entry"); + Assertions.assertEquals("db1", a.getDbName(), "first handle must carry its own database"); + Assertions.assertEquals("db2", b.getDbName(), "second handle must resolve db2, not reuse db1's handle"); + Assertions.assertEquals(Arrays.asList("db1.t", "db2.t"), helper.existProbes, + "each database's table is probed independently"); + } + + @Test + public void absentTableReprobesEachCallAndIsNeverMemoized() { + CountingStructureHelper helper = new CountingStructureHelper(); // nothing present + MaxComputeConnectorMetadata metadata = metadataWith(helper); + + Optional first = metadata.getTableHandle(null, "db1", "missing"); + Optional second = metadata.getTableHandle(null, "db1", "missing"); + + // WHY: negatives are intentionally NOT memoized, so a missing table keeps the exact + // pre-change behavior — a fresh exists() probe and a clean Optional.empty() on EVERY + // call, never a cached present handle. MUTATION: caching the empty result would drop + // existProbes to size 1 -> red. + Assertions.assertFalse(first.isPresent(), "a missing table must resolve to empty"); + Assertions.assertFalse(second.isPresent(), "a missing table must resolve to empty every time"); + Assertions.assertEquals(Arrays.asList("db1.missing", "db1.missing"), helper.existProbes, + "an absent table must re-probe on every call (negatives are not memoized)"); + Assertions.assertTrue(helper.odpsTableBuilds.isEmpty(), + "an absent table must never build an ODPS Table handle"); + } + + /** + * Counting fake: a table "exists" iff its {@code db.table} was passed to the constructor. + * Records every {@code tableExist} probe and every {@code getOdpsTable} build so the tests + * can assert the memo collapses repeats. All other methods return harmless defaults — they + * are never invoked by {@code getTableHandle}. + */ + private static final class CountingStructureHelper implements McStructureHelper { + private final Set presentTables; + private final List existProbes = new ArrayList<>(); + private final List odpsTableBuilds = new ArrayList<>(); + + CountingStructureHelper(String... present) { + this.presentTables = new HashSet<>(Arrays.asList(present)); + } + + @Override + public boolean tableExist(Odps mcClient, String dbName, String tableName) { + String key = dbName + "." + tableName; + existProbes.add(key); + return presentTables.contains(key); + } + + @Override + public Table getOdpsTable(Odps mcClient, String dbName, String tableName) { + odpsTableBuilds.add(dbName + "." + tableName); + // A null Table is enough: the tests assert handle identity and build counts, never + // touch the Table itself, keeping the fake offline. + return null; + } + + @Override + public TableIdentifier getTableIdentifier(String dbName, String tableName) { + return null; + } + + // ---- unused by getTableHandle: harmless defaults ---- + + @Override + public List listTableNames(Odps mcClient, String dbName) { + return Collections.emptyList(); + } + + @Override + public List listDatabaseNames(Odps mcClient, String defaultProject) { + return Collections.emptyList(); + } + + @Override + public boolean databaseExist(Odps mcClient, String dbName) { + return false; + } + + @Override + public List getPartitions(Odps mcClient, String dbName, String tableName) { + return Collections.emptyList(); + } + + @Override + public Iterator getPartitionIterator(Odps mcClient, String dbName, String tableName) { + return Collections.emptyIterator(); + } + + @Override + public Tables.TableCreator createTableCreator(Odps mcClient, String dbName, + String tableName, TableSchema schema) { + return null; + } + + @Override + public void dropTable(Odps mcClient, String dbName, String tableName, boolean ifExists) + throws OdpsException { + } + + @Override + public void createDb(Odps mcClient, String dbName, boolean ifNotExists) { + } + + @Override + public void dropDb(Odps mcClient, String dbName, boolean ifExists) { + } + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java index 12c85993a69f94..dc22d050fcba56 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java @@ -24,7 +24,7 @@ import org.apache.doris.connector.api.ConnectorSession; import org.apache.doris.connector.api.ConnectorValidationContext; import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; -import org.apache.doris.connector.cache.ConnectorPartitionViewCache; +import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.metastore.HmsMetaStoreProperties; import org.apache.doris.connector.metastore.spi.JdbcDriverSupport; import org.apache.doris.connector.metastore.spi.MetaStoreProviders; @@ -129,18 +129,19 @@ public class PaimonConnector implements Connector { // Cleared wholesale on REFRESH CATALOG (the connector is rebuilt). See PaimonSchemaAtMemo. private final PaimonSchemaAtMemo schemaAtMemo = new PaimonSchemaAtMemo(PaimonSchemaAtMemo.DEFAULT_MAX_SIZE); - // PERF-06: cross-query DERIVED partition-view cache ("cache A", the generic ConnectorPartitionViewCache from + // PERF-06: cross-query DERIVED partition-view cache ("cache A", the generic ConnectorMetadataCache from // fe-connector-cache), layered ABOVE the raw remote catalog.listPartitions call (PaimonCatalogOps#listPartitions): // it memoizes the BUILT List (display-name rendering + null-sentinel normalization, // see PaimonConnectorMetadata#collectPartitions) keyed by (db, table, snapshotId, schemaId), so a repeated // query on a partitioned table skips the derived rebuild AND the remote catalog round-trip. ONE typed field // (unlike iceberg's two): paimon does not override getMvccPartitionView, so the generic MTMV model falls - // back to its default listPartitions/LIST/timestamp path for paimon -- listPartitions is the only - // enumeration hook to wrap. Unlike iceberg, paimon has NO session=user / per-user credential-isolation + // back to its default listPartitions/LIST/timestamp path for paimon -- all three partition-enumeration + // hooks (listPartitions/Names/Values) share it via PaimonConnectorMetadata#cachedPartitions. Unlike + // iceberg, paimon has NO session=user / per-user credential-isolation // cache-disabling convention (a paimon catalog authenticates at catalog-creation time -- Kerberos UGI / // HMS principal -- not per-query session identity), so this is constructed unconditionally: never null on // a live connector (only PaimonConnectorMetadata's convenience/test constructors pass null). - private final ConnectorPartitionViewCache> partitionViewCache; + private final ConnectorMetadataCache> partitionViewCache; public PaimonConnector(Map properties, ConnectorContext context) { this.properties = properties; @@ -155,7 +156,7 @@ public PaimonConnector(Map properties, ConnectorContext context) new PaimonLatestSnapshotCache(resolveTableCacheTtlSecond(properties), DEFAULT_TABLE_CACHE_CAPACITY); // Reads its own meta.cache.paimon.partition_view.(enable|ttl-second|capacity) from the catalog // properties via the framework's CacheSpec (default ON / 24h / 1000). - this.partitionViewCache = new ConnectorPartitionViewCache<>("paimon", properties); + this.partitionViewCache = new ConnectorMetadataCache<>("paimon", "partition_view", properties); } /** @@ -341,7 +342,7 @@ public Set getCapabilities() { } /** Test-only: the derived listPartitions view cache (PERF-06). Never null (paimon has no session=user gate). */ - ConnectorPartitionViewCache> partitionViewCacheForTest() { + ConnectorMetadataCache> partitionViewCacheForTest() { return partitionViewCache; } diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java index 3888a160e96185..2df695a8f8866e 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java @@ -32,8 +32,8 @@ import org.apache.doris.connector.api.mvcc.ConnectorTimeTravelSpec; import org.apache.doris.connector.api.pushdown.ConnectorExpression; import org.apache.doris.connector.api.scan.ConnectorPartitionValues; -import org.apache.doris.connector.cache.ConnectorPartitionViewCache; -import org.apache.doris.connector.cache.PartitionViewCacheKey; +import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ConnectorTableKey; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.thrift.THiveTable; import org.apache.doris.thrift.TTableDescriptor; @@ -97,14 +97,15 @@ public class PaimonConnectorMetadata implements ConnectorMetadata { // existing direct-construction tests compile unchanged; production goes through the 5-arg ctor. private final PaimonLatestSnapshotCache latestSnapshotCache; - // PERF-06: cross-query DERIVED partition-view cache A (generic ConnectorPartitionViewCache), injected by the + // PERF-06: cross-query DERIVED partition-view cache A (generic ConnectorMetadataCache), injected by the // owning PaimonConnector; null = no cross-query derived layer (the convenience/test ctors used by ~15 // existing direct-construction tests pass null). Layered ABOVE the raw remote catalogOps.listPartitions // call: a hit skips both the derived-view BUILD (collectPartitions) and the remote round-trip, keyed by - // (db, table, snapshotId, schemaId). Consumed only by listPartitions -- paimon does not override - // getMvccPartitionView (see ConnectorMetadata's default), so the generic MTMV model already uses - // listPartitions for its LIST/timestamp partition view; there is no second hook to wrap. - private final ConnectorPartitionViewCache> partitionViewCache; + // (db, table, snapshotId, schemaId). Consumed by all three partition-enumeration hooks (listPartitions, + // listPartitionNames, listPartitionValues) via the shared cachedPartitions collector -- paimon does not + // override getMvccPartitionView (see ConnectorMetadata's default), so the generic MTMV model already uses + // listPartitions for its LIST/timestamp partition view. + private final ConnectorMetadataCache> partitionViewCache; public PaimonConnectorMetadata(PaimonCatalogOps catalogOps, Map properties, ConnectorContext context) { @@ -132,7 +133,7 @@ public PaimonConnectorMetadata(PaimonCatalogOps catalogOps, Map PaimonConnectorMetadata(PaimonCatalogOps catalogOps, Map properties, ConnectorContext context, PaimonSchemaAtMemo schemaAtMemo, PaimonLatestSnapshotCache latestSnapshotCache, - ConnectorPartitionViewCache> partitionViewCache) { + ConnectorMetadataCache> partitionViewCache) { this.catalogOps = catalogOps; this.typeMappingOptions = buildTypeMappingOptions(properties); this.context = context; @@ -1059,7 +1060,7 @@ private static Map buildColumnHandles(List listPartitionNames(ConnectorSession session, ConnectorTableHandle handle) { - List partitions = collectPartitions((PaimonTableHandle) handle); + List partitions = cachedPartitions((PaimonTableHandle) handle); List names = new ArrayList<>(partitions.size()); for (ConnectorPartitionInfo partition : partitions) { names.add(partition.getPartitionName()); @@ -1073,26 +1074,42 @@ public List listPartitionNames(ConnectorSession session, ConnectorTableH * pushing predicates into the Paimon catalog, and this preserves that behavior (mirrors * {@code MaxComputeConnectorMetadata}). * - *

PERF-06 cache A: the BUILT {@code List} is memoized across queries in - * {@link #partitionViewCache}, keyed by {@code (db, table, snapshotId, schemaId)} (see - * {@link #partitionViewCacheKey}) — a hit skips both {@link #collectPartitions} and the remote - * {@code catalogOps.listPartitions} round-trip. The cache is BYPASSED (compute directly, never populated) - * when: {@code partitionViewCache} is {@code null} (the convenience/test ctors); {@code filter} is present - * (not the pruning path, and not keyed by filter); or the handle is unpartitioned (mirrors - * {@link #collectPartitions}'s own empty-partitionKeys short-circuit, so an unpartitioned table never - * touches {@link #latestSnapshotCache} either — preserving the "no seam call" contract the unpartitioned - * path already guarantees). + *

A present {@code filter} BYPASSES the derived cache (computes directly, never populates) — it is + * not the pruning path and not keyed by filter. Every other case routes through {@link #cachedPartitions}, + * the shared cache-aware collector this hook shares with {@link #listPartitionNames} and + * {@link #listPartitionValues}. */ @Override public List listPartitions(ConnectorSession session, ConnectorTableHandle handle, Optional filter) { PaimonTableHandle paimonHandle = (PaimonTableHandle) handle; + if (filter.isPresent()) { + return collectPartitions(paimonHandle); + } + return cachedPartitions(paimonHandle); + } + + /** + * Shared cache-aware partition collector backing the no-filter path of {@link #listPartitions}, + * plus {@link #listPartitionNames} and {@link #listPartitionValues}. Returns the BUILT + * {@code List} from {@link #partitionViewCache} (PERF-06 cache A), keyed by + * {@code (db, table, snapshotId, schemaId)} (see {@link #partitionViewCacheKey}) — a hit skips both + * {@link #collectPartitions} and the remote {@code catalogOps.listPartitions} round-trip, so repeated + * SHOW PARTITIONS / {@code partition_values()} / pruning over the same {@code (db, table, snapshotId)} + * render the list once. + * + *

The cache is BYPASSED (compute directly via {@link #collectPartitions}, never populated) when + * {@code partitionViewCache} is {@code null} (the convenience/test ctors) or the handle is unpartitioned + * (mirrors {@link #collectPartitions}'s own empty-partitionKeys short-circuit, so an unpartitioned table + * never touches {@link #latestSnapshotCache} either — preserving the "no seam call" contract the + * unpartitioned path already guarantees). + */ + private List cachedPartitions(PaimonTableHandle paimonHandle) { List partitionKeys = paimonHandle.getPartitionKeys(); - if (partitionViewCache == null || filter.isPresent() - || partitionKeys == null || partitionKeys.isEmpty()) { + if (partitionViewCache == null || partitionKeys == null || partitionKeys.isEmpty()) { return collectPartitions(paimonHandle); } - PartitionViewCacheKey key = partitionViewCacheKey(paimonHandle); + ConnectorTableKey key = partitionViewCacheKey(paimonHandle); return partitionViewCache.get(key, () -> collectPartitions(paimonHandle)); } @@ -1109,7 +1126,7 @@ public List listPartitions(ConnectorSession session, * and a new snapshot (data change, once the entry expires or REFRESH invalidates it) naturally mints a new key. * *

schemaId: pinned {@code -1} ("unversioned" for that axis, matching - * {@link PartitionViewCacheKey}'s documented convention). Unlike iceberg, paimon's {@link PaimonTableHandle} + * {@link ConnectorTableKey}'s documented convention). Unlike iceberg, paimon's {@link PaimonTableHandle} * carries no schemaId — {@code applySnapshot} threads only {@code scanOptions} (an opaque properties map; * see its javadoc) onto the handle, and {@link #beginQuerySnapshot} (the common latest-pin path) never * resolves a schemaId either (its {@code ConnectorMvccSnapshot} keeps the builder default {@code -1}). This @@ -1117,18 +1134,18 @@ public List listPartitions(ConnectorSession session, * (fixed at handle-build time) and paimon's raw partition specs, and paimon partition columns are immutable * post-creation, so schema evolution (e.g. ADD COLUMN) does not change what this method computes. */ - private PartitionViewCacheKey partitionViewCacheKey(PaimonTableHandle paimonHandle) { + private ConnectorTableKey partitionViewCacheKey(PaimonTableHandle paimonHandle) { Identifier identifier = Identifier.create(paimonHandle.getDatabaseName(), paimonHandle.getTableName()); long snapshotId = latestSnapshotCache.getOrLoad(identifier, () -> catalogOps.latestSnapshotId(resolveTable(paimonHandle)).orElse(-1L)); - return new PartitionViewCacheKey( + return new ConnectorTableKey( paimonHandle.getDatabaseName(), paimonHandle.getTableName(), snapshotId, -1L); } @Override public List> listPartitionValues(ConnectorSession session, ConnectorTableHandle handle, List partitionColumns) { - List partitions = collectPartitions((PaimonTableHandle) handle); + List partitions = cachedPartitions((PaimonTableHandle) handle); List> result = new ArrayList<>(partitions.size()); for (ConnectorPartitionInfo partition : partitions) { Map rawValues = partition.getPartitionValues(); @@ -1144,8 +1161,9 @@ public List> listPartitionValues(ConnectorSession session, } /** - * Shared partition collector backing {@link #listPartitionNames}, {@link #listPartitions} and - * {@link #listPartitionValues}. Replicates the fe-core display-name logic + * Shared (uncached) partition collector behind {@link #cachedPartitions} — the underlying compute for + * {@link #listPartitionNames}, {@link #listPartitions} and {@link #listPartitionValues}, also reached + * directly on the filter / unpartitioned / null-cache bypass. Replicates the fe-core display-name logic * ({@code PaimonUtil.generatePartitionInfo} + {@code isLegacyPartitionName}) so the rendered * partition names stay byte-identical to fe-core — including #65904, which drives value order from * the partition columns and escapes path-special characters in the name via the Paimon SDK. diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java index 33b62686d07386..eb538c7d29f0e2 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java @@ -50,13 +50,8 @@ final class PaimonLatestSnapshotCache { private final MetaCacheEntry entry; PaimonLatestSnapshotCache(long ttlSeconds, int maxSize) { - // ttl-second <= 0 disables caching (always read live); a positive ttl is access-based expiry with the - // given capacity. CacheSpec treats ttl == -1 as "no expiration (enabled)" and ttl == 0 as "disabled", - // so translate the connector's "<= 0 disables" contract to ttl == 0 rather than passing a negative - // value straight through (which would otherwise flip -1 into a never-expiring cache). - CacheSpec spec = ttlSeconds > 0 - ? CacheSpec.of(true, ttlSeconds, maxSize) - : CacheSpec.of(true, CacheSpec.CACHE_TTL_DISABLE_CACHE, maxSize); + // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). + CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = new MetaCacheEntry<>("paimon-latest-snapshot", null, spec, ForkJoinPool.commonPool(), false, true, 0L, true); } diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorCacheTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorCacheTest.java index f952e6d47a7def..ce6773d67d29ea 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorCacheTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorCacheTest.java @@ -18,8 +18,8 @@ package org.apache.doris.connector.paimon; import org.apache.doris.connector.api.ConnectorPartitionInfo; -import org.apache.doris.connector.cache.ConnectorPartitionViewCache; -import org.apache.doris.connector.cache.PartitionViewCacheKey; +import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ConnectorTableKey; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -113,16 +113,16 @@ public void refreshHooksInvalidatePartitionViewCache() { // invalidation the loader must run again. MUTATION: an invalidate* hook not routed to the view cache // -> the entry survives -> loader not re-run -> a loads assert below red. PaimonConnector connector = connector(Collections.emptyMap()); - ConnectorPartitionViewCache> cache = connector.partitionViewCacheForTest(); + ConnectorMetadataCache> cache = connector.partitionViewCacheForTest(); Assertions.assertNotNull(cache); int[] loads = {0}; Supplier> loader = () -> { loads[0]++; return Collections.emptyList(); }; - PartitionViewCacheKey db1t1 = new PartitionViewCacheKey("db1", "t1", 1L, -1L); - PartitionViewCacheKey db1t2 = new PartitionViewCacheKey("db1", "t2", 1L, -1L); - PartitionViewCacheKey db2t1 = new PartitionViewCacheKey("db2", "t1", 1L, -1L); + ConnectorTableKey db1t1 = new ConnectorTableKey("db1", "t1", 1L, -1L); + ConnectorTableKey db1t2 = new ConnectorTableKey("db1", "t2", 1L, -1L); + ConnectorTableKey db2t1 = new ConnectorTableKey("db2", "t1", 1L, -1L); // REFRESH TABLE db1.t1 -> only db1.t1 re-loads. cache.get(db1t1, loader); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java index e93126549df16d..4c36d42053e4ce 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java @@ -19,7 +19,7 @@ import org.apache.doris.connector.api.ConnectorPartitionInfo; import org.apache.doris.connector.api.pushdown.ConnectorExpression; -import org.apache.doris.connector.cache.ConnectorPartitionViewCache; +import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.paimon.partition.Partition; import org.apache.paimon.types.DataTypes; @@ -38,10 +38,11 @@ /** * PERF-06 tests for the cross-query DERIVED partition-view cache ("cache A", the generic - * {@link ConnectorPartitionViewCache}) wired into {@link PaimonConnectorMetadata#listPartitions}. Paimon does - * NOT override {@code getMvccPartitionView} (the generic MTMV model falls back to its default - * listPartitions/LIST/timestamp path), so — unlike iceberg's two typed fields — there is exactly ONE - * enumeration hook to wrap. + * {@link ConnectorMetadataCache}) wired into all three partition-enumeration hooks + * ({@link PaimonConnectorMetadata#listPartitions}, {@code listPartitionNames}, {@code listPartitionValues}) + * via the shared {@code cachedPartitions} collector. Paimon does NOT override {@code getMvccPartitionView} + * (the generic MTMV model falls back to its default listPartitions/LIST/timestamp path), so — unlike + * iceberg's two typed fields — there is a single typed cache field, now shared by all three hooks. * *

Uses the real {@link RecordingPaimonCatalogOps} + {@link FakePaimonTable} harness (no Mockito, no docker): * {@link PaimonConnectorMetadata#collectPartitions}'s first (and only) remote call is @@ -57,14 +58,14 @@ public class PaimonConnectorMetadataPartitionViewCacheTest { private static PaimonConnectorMetadata metadataWithCache(RecordingPaimonCatalogOps ops, - ConnectorPartitionViewCache> cache) { + ConnectorMetadataCache> cache) { return new PaimonConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext(), new PaimonSchemaAtMemo(PaimonSchemaAtMemo.DEFAULT_MAX_SIZE), new PaimonLatestSnapshotCache(0L, 1), cache); } - private static ConnectorPartitionViewCache> partitionViewCache() { - return new ConnectorPartitionViewCache<>("paimon", Collections.emptyMap()); + private static ConnectorMetadataCache> partitionViewCache() { + return new ConnectorMetadataCache<>("paimon", "partition_view", Collections.emptyMap()); } private static RowType regionRowType() { @@ -113,7 +114,7 @@ public void listPartitionsCachesDerivedListAcrossQueries() { ops.table = table; ops.latestSnapshotId = OptionalLong.of(100L); ops.partitions = Arrays.asList(partition("cn"), partition("us")); - ConnectorPartitionViewCache> cache = partitionViewCache(); + ConnectorMetadataCache> cache = partitionViewCache(); PaimonConnectorMetadata md = metadataWithCache(ops, cache); PaimonTableHandle h = handle(table); @@ -135,7 +136,7 @@ public void listPartitionsDifferentSnapshotReEnumerates() { RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); FakePaimonTable table = regionTable(); ops.table = table; - ConnectorPartitionViewCache> cache = partitionViewCache(); + ConnectorMetadataCache> cache = partitionViewCache(); PaimonConnectorMetadata md = metadataWithCache(ops, cache); PaimonTableHandle h = handle(table); @@ -162,7 +163,7 @@ public void listPartitionsInvalidateTableForcesReEnumeration() { ops.table = table; ops.latestSnapshotId = OptionalLong.of(100L); ops.partitions = Collections.singletonList(partition("cn")); - ConnectorPartitionViewCache> cache = partitionViewCache(); + ConnectorMetadataCache> cache = partitionViewCache(); PaimonConnectorMetadata md = metadataWithCache(ops, cache); PaimonTableHandle h = handle(table); @@ -181,7 +182,7 @@ public void listPartitionsInvalidateAllForcesReEnumeration() { ops.table = table; ops.latestSnapshotId = OptionalLong.of(100L); ops.partitions = Collections.singletonList(partition("cn")); - ConnectorPartitionViewCache> cache = partitionViewCache(); + ConnectorMetadataCache> cache = partitionViewCache(); PaimonConnectorMetadata md = metadataWithCache(ops, cache); PaimonTableHandle h = handle(table); @@ -205,7 +206,7 @@ public void listPartitionsWithFilterBypassesCache() { ops.table = table; ops.latestSnapshotId = OptionalLong.of(100L); ops.partitions = Collections.singletonList(partition("cn")); - ConnectorPartitionViewCache> cache = partitionViewCache(); + ConnectorMetadataCache> cache = partitionViewCache(); PaimonConnectorMetadata md = metadataWithCache(ops, cache); PaimonTableHandle h = handle(table); @@ -251,7 +252,7 @@ public void unpartitionedHandleBypassesCacheWithoutTouchingSnapshotSeam() { PaimonTableHandle h = new PaimonTableHandle( "db1", "t1", Collections.emptyList(), Collections.emptyList()); h.setPaimonTable(table); - ConnectorPartitionViewCache> cache = partitionViewCache(); + ConnectorMetadataCache> cache = partitionViewCache(); PaimonConnectorMetadata md = metadataWithCache(ops, cache); List result = md.listPartitions(null, h, Optional.empty()); @@ -259,4 +260,99 @@ public void unpartitionedHandleBypassesCacheWithoutTouchingSnapshotSeam() { Assertions.assertTrue(result.isEmpty()); Assertions.assertTrue(ops.log.isEmpty(), "an unpartitioned handle must not touch any remote seam"); } + + @Test + public void listPartitionNamesCachesAcrossQueries() { + // WHY (PA-1): listPartitionNames now routes through the SAME partitionViewCache as listPartitions + // (SHOW PARTITIONS re-rendered the full list on every call before). Two calls on the same latest + // snapshot must share one entry: the seam runs once and both calls return identical names. + // MUTATION: listPartitionNames calling collectPartitions directly (the pre-fix code) -> loadCount 2 -> red. + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + FakePaimonTable table = regionTable(); + ops.table = table; + ops.latestSnapshotId = OptionalLong.of(100L); + ops.partitions = Arrays.asList(partition("cn"), partition("us")); + ConnectorMetadataCache> cache = partitionViewCache(); + PaimonConnectorMetadata md = metadataWithCache(ops, cache); + PaimonTableHandle h = handle(table); + + List first = md.listPartitionNames(null, h); + List second = md.listPartitionNames(null, h); + + Assertions.assertEquals(Arrays.asList("region=cn", "region=us"), first); + Assertions.assertEquals(first, second, "the cached list drives identical names"); + Assertions.assertEquals(1, loadCount(ops), "listPartitionNames must hit the shared cache (enumerate once)"); + } + + @Test + public void listPartitionValuesCachesAcrossQueries() { + // WHY (PA-1): same as above for the partition_values() TVF path -- it re-rendered on every call before. + // MUTATION: listPartitionValues calling collectPartitions directly -> loadCount 2 -> red. + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + FakePaimonTable table = regionTable(); + ops.table = table; + ops.latestSnapshotId = OptionalLong.of(100L); + ops.partitions = Arrays.asList(partition("cn"), partition("us")); + ConnectorMetadataCache> cache = partitionViewCache(); + PaimonConnectorMetadata md = metadataWithCache(ops, cache); + PaimonTableHandle h = handle(table); + + List cols = Collections.singletonList("region"); + List> first = md.listPartitionValues(null, h, cols); + List> second = md.listPartitionValues(null, h, cols); + + Assertions.assertEquals( + Arrays.asList(Collections.singletonList("cn"), Collections.singletonList("us")), first); + Assertions.assertEquals(first, second, "the cached list drives identical values"); + Assertions.assertEquals(1, loadCount(ops), "listPartitionValues must hit the shared cache (enumerate once)"); + } + + @Test + public void allThreeHooksShareOneCacheEntry() { + // WHY (PA-1): the three enumeration hooks share ONE (db,table,snapshotId) entry -- listPartitions + // populates it, listPartitionNames and listPartitionValues then derive from the SAME cached list + // without re-enumerating, and the derived outputs stay byte-consistent with listPartitions' rendered + // list. MUTATION: any hook bypassing the shared cachedPartitions -> loadCount > 1 -> red. + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + FakePaimonTable table = regionTable(); + ops.table = table; + ops.latestSnapshotId = OptionalLong.of(100L); + ops.partitions = Arrays.asList(partition("cn"), partition("us")); + ConnectorMetadataCache> cache = partitionViewCache(); + PaimonConnectorMetadata md = metadataWithCache(ops, cache); + PaimonTableHandle h = handle(table); + + List full = md.listPartitions(null, h, Optional.empty()); + List namesOut = md.listPartitionNames(null, h); + List> valuesOut = md.listPartitionValues(null, h, Collections.singletonList("region")); + + Assertions.assertEquals(1, loadCount(ops), "all three hooks must share one cache entry (enumerate once)"); + Assertions.assertEquals(names(full), namesOut, "listPartitionNames equals the names of listPartitions' list"); + Assertions.assertEquals( + full.stream().map(p -> Collections.singletonList(p.getPartitionValues().get("region"))) + .collect(Collectors.toList()), + valuesOut, "listPartitionValues equals the values derived from listPartitions' list"); + } + + @Test + public void unpartitionedNamesAndValuesBypassCacheWithoutTouchingSnapshotSeam() { + // WHY (PA-1): the "no seam call for unpartitioned" contract must hold for the new routing of + // listPartitionNames and listPartitionValues too -- cachedPartitions short-circuits before building a + // key, so neither latestSnapshotId nor listPartitions is called. MUTATION: routing through the cache + // before the emptiness check -> "latestSnapshotId" appears in ops.log -> red. + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + FakePaimonTable table = new FakePaimonTable( + "t1", RowType.builder().field("id", DataTypes.INT()).build(), + Collections.emptyList(), Collections.emptyList()); + ops.table = table; + PaimonTableHandle h = new PaimonTableHandle( + "db1", "t1", Collections.emptyList(), Collections.emptyList()); + h.setPaimonTable(table); + ConnectorMetadataCache> cache = partitionViewCache(); + PaimonConnectorMetadata md = metadataWithCache(ops, cache); + + Assertions.assertTrue(md.listPartitionNames(null, h).isEmpty()); + Assertions.assertTrue(md.listPartitionValues(null, h, Collections.singletonList("region")).isEmpty()); + Assertions.assertTrue(ops.log.isEmpty(), "unpartitioned names/values must not touch any remote seam"); + } } diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index 065e21387f2d66..254712404947ce 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -100,25 +100,6 @@ under the License. - - - check-authz-cache-sharding - validate - - exec - - - ${project.basedir}/../../tools/check-authz-cache-sharding.sh - - ${project.basedir} - - - diff --git a/fe/fe-core/src/test/java/org/apache/doris/connector/ConnectorStatementScopeTest.java b/fe/fe-core/src/test/java/org/apache/doris/connector/ConnectorStatementScopeTest.java index b7d868e4a3f43e..19a9b529b16129 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/connector/ConnectorStatementScopeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/connector/ConnectorStatementScopeTest.java @@ -24,7 +24,13 @@ import org.apache.doris.connector.api.handle.ConnectorTransaction; import org.apache.doris.datasource.plugin.CatalogStatementTransaction; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.commands.ExecuteCommand; +import org.apache.doris.nereids.trees.plans.commands.PrepareCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.PreparedStatementContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.StmtExecutor; import org.apache.doris.transaction.PluginDrivenTransactionManager; import org.junit.jupiter.api.Assertions; @@ -38,8 +44,10 @@ /** * Tests for the per-statement {@link ConnectorStatementScope}: the {@link ConnectorStatementScope#NONE} no-op, - * the memoizing {@link ConnectorStatementScopeImpl}, and the {@link StatementContext} hosting + per-execution - * reset a reused prepared statement relies on. + * the memoizing {@link ConnectorStatementScopeImpl}, the {@link StatementContext} hosting + per-execution + * reset a reused prepared statement relies on, and that {@link ExecuteCommand} actually invokes that reset on + * every execution (external/connector tables are planned through the reused prepared context, so a missing + * reset would leak one execution's loaded table into the next — see {@code executeCommandResetsConnectorScope*}). */ public class ConnectorStatementScopeTest { @@ -175,6 +183,49 @@ public void resetClosesTheDroppedScopeBeforeStartingFresh() { "reset drops the scope so the next execution/attempt starts fresh"); } + @Test + public void executeCommandResetsConnectorScopePerExecution() throws Exception { + // WIRING test: the reset above is only load-bearing if ExecuteCommand.run() actually calls it. A prepared + // statement reuses ONE StatementContext across every EXECUTE, and an EXTERNAL/connector table is planned + // through that reused context each execution (external tables never take the OLAP short-circuit fast path; + // they always fall to the normal executor.execute() planner, which re-resolves the table per execution). + // So run() must drop the connector per-statement scope at the top of every execution, or a prior execution's + // memoized (loaded) table leaks into the next. The tests above cover only the reset PRIMITIVE; this covers + // that the command invokes it. MUTATION: delete `statementContext.resetConnectorStatementScope()` from + // ExecuteCommand.run() -> the seeded value survives -> the assertNotSame below flips -> red. + StatementContext statementContext = new StatementContext(); + // Seed a value the way a first execution's connector planning would (one loaded table the statement shares). + ConnectorStatementScope firstScope = statementContext.getOrCreateConnectorStatementScope(); + Object memoizedTable = firstScope.computeIfAbsent("table:1", Object::new); + + // A prepared statement wrapping that reused StatementContext, with a plain (non-cache, non-insert) plan. + LogicalPlan plan = Mockito.mock(LogicalPlan.class); + PrepareCommand prepareCommand = Mockito.mock(PrepareCommand.class); + Mockito.when(prepareCommand.getLogicalPlan()).thenReturn(plan); + + ConnectContext ctx = Mockito.mock(ConnectContext.class); + PreparedStatementContext preparedStmtCtx = new PreparedStatementContext( + prepareCommand, ctx, statementContext, "SELECT * FROM ext_catalog.db.t WHERE id = ?"); + Mockito.when(ctx.getPreparedStementContext("s")).thenReturn(preparedStmtCtx); + // Force the plain-SELECT path: skip the OLAP group-commit fast path so run() falls to executor.execute(). + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.enableGroupCommitFullPrepare = false; + Mockito.when(ctx.getSessionVariable()).thenReturn(sessionVariable); + Mockito.when(ctx.getStatementContext()).thenReturn(statementContext); + + // A no-op executor: we pin the reset wiring, not the planner. execute() is a mock no-op. + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + Mockito.when(executor.getContext()).thenReturn(ctx); + + new ExecuteCommand("s", prepareCommand, statementContext).run(ctx, executor); + + ConnectorStatementScope secondScope = statementContext.getOrCreateConnectorStatementScope(); + Assertions.assertNotSame(firstScope, secondScope, + "ExecuteCommand drops the reused context's connector scope so the next execution starts fresh"); + Assertions.assertNotSame(memoizedTable, secondScope.computeIfAbsent("table:1", Object::new), + "a prior execution's memoized connector table must not leak into the next EXECUTE"); + } + @Test public void closeClosesScopeWhenReturningResultLocally() { // A statement that never reaches the query-finish callback (external DDL / SHOW via Command.run) is diff --git a/plan-doc/HANDOFF.md b/plan-doc/HANDOFF.md index fac08121d87a6f..648ce10c346e9d 100644 --- a/plan-doc/HANDOFF.md +++ b/plan-doc/HANDOFF.md @@ -5,7 +5,71 @@ --- -# 🆕 下一个 session = **重跑 CI 验证本轮 3 个修复** +# 🆕🆕 最新一轮（2026-07-25）：CI **1005291** 的 iceberg 大小写列名回归 —— 已修待 CI 验 + +> 任务 = TeamCity `Doris_External_Regression` **#1005291**（PR **66028** @ `7ff51a106f0`）中 +> `external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy:273`。 +> 该 build 整体 SUCCESS（603 passed），这条是 **muted** 的失败。 + +## 定性：本分支独有的**真回归**，不是 flaky、不是集群故障 + +同一用例在 pull/66011、65847、66006、65851、65126 上均 SUCCESS；另外两个 FAILURE（66007、65851） +是完全不同的原因（`can not cast from origin type STRUCT<...>`、`No backend available / not alive`），**与本问题无关**。 +本地 HEAD 与 PR 66028 head 一致，`fe/fe-core/.../datasource/iceberg/` 已整体删除 ⇒ 走的一定是连接器路径。 + +## 根因：#65329 的**平坦（top-level）臂**只移植了一半 + +上游 `IcebergMetadataOps`（`git show 70a82532325:fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java`） +有 **5 处** `validateNoCaseInsensitiveSiblingCollision` 调用：flat ADD(:885)、nested ADD(:918)、 +ADD COLUMNS 批量(:946→helper :1428)、flat RENAME(:1005，且前置 :1000 用 `resolveColumnPath` 大小写不敏感解析**源列名**)、nested RENAME(:1028)。 +移植只带进了**两处 nested**（`IcebergNestedColumnEvolution:71/99`）。 + +顶层 DDL **根本到不了**那个类：fe-core `PluginDrivenExternalCatalog:908-913/949-951` 对 `!columnPath.isNested()` 直接短路回平坦 SPI， +`IcebergConnectorMetadata.addNestedColumn/renameNestedColumn` 再短路一次；终点 +`CatalogBackedIcebergCatalogOps.addColumn/addColumns/renameColumn` 直接 `UpdateSchema` 零校验。 +Iceberg 自己也挡不住 —— `SchemaUpdate` 构造器 `caseSensitive = true`，`findField("id")` 看不见 `Id`。 +⇒ 该校验器里 `parentPath.isEmpty()` 那条顶层分支一直是**死代码**，正是漏掉调用点的信号。 + +## ✅ 已交付：commit `f1104a6880d` + +三个平坦入口接上 nested 臂已有的校验器；`validateNoCaseInsensitiveSiblingCollision` 放宽为包内可见， +新增 `validateNoCaseInsensitiveTopLevelCollisions`（批量 + 请求内去重）与 `renameTopLevelColumn` +（大小写不敏感解析源名 + 冲突校验 + 复用 `applyRenameColumn` 的 identifier-field 修复）。**未碰 fe-core**。 + +验证：先 RED（7 个新用例在修前失败，症状与 CI 完全一致：ADD 是 `nothing was thrown`，RENAME 是 +`Cannot rename missing column: label`）→ 后 GREEN；模块 **1145/1145**（5 个既有 live-connectivity skip）、 +**全 reactor `test-compile` BUILD SUCCESS**、checkstyle 0。5 路对抗复审 + 3 skeptic 表决：**0 条成立**。 +该 suite 内**唯一**的顶层列 DDL 就是 274–289 行，其余全是 dotted 嵌套路径（未受影响）； +289 行 `RENAME COLUMN label TO label` 会把列名变小写，但 `mixedCaseTable` 最后一次被引用就在 296 行，无下游影响。 + +## ⏭ 下一个 session + +1. **重跑 CI 是唯一真闸门**。预期 273/277/281/285/289 五条全过（后四条此前从未被执行到）。 +2. **同族平坦臂缺口，本轮故意未打包**（复审提出、经表决判定为既有问题且超出本次范围，非本次引入）： + - flat `dropColumn`（`IcebergCatalogOps` 内）仍按大小写敏感解析 ⇒ 对 Spark 混合大小写表 `DROP COLUMN label` 会失败（上游 :961 用 `resolveColumnPath`）； + - flat `modifyColumn` 用大小写敏感 `Schema.findField` ⇒ `MODIFY COLUMN id` 报 "Column id does not exist"； + - flat `applyPosition` 的 `AFTER ` 参照列未做大小写不敏感解析； + - `reorderColumns` 既不规范化大小写也不查重复。 + 以上**均未被本 suite 触发**（该 suite 的 DROP/MODIFY 全是 dotted 嵌套路径），故不阻塞本次 CI。 + 另：row-lineage mutation guard 的缺失是**本分支既有的、有文档的有意偏离**（见 `IcebergConnectorMetadata:1282`），不是缺口。 +3. 复审旁获（未验证、与本次无关）：有 agent 声称 hive 网关未把 5 个 ColumnPath 列操作委派给 iceberg 兄弟， + 导致 iceberg-on-HMS 的 `MODIFY COLUMN COMMENT` 恒不可用。**未经我独立核实**，要动前先自己 trace。 + +## 🧰 本轮新增构建坑（补充下面第 1 条） + +**`-Dmaven.build.cache.enabled=false` 会连带打破 shade 依赖链**：`fe-connector-hms-hive-shade` 的 shade 插件绑在 +`package` 阶段，而 `test`/`compile` 生命周期到不了 `package` ⇒ 关缓存后 `fe-connector-hms` 编译报 +`package org.apache.hadoop.hive.metastore.api does not exist`（**与被改代码无关**，该 reactor 里根本没有 iceberg 模块）。 +正确姿势：先 `-pl <目标模块> -am install -DskipTests -Dmaven.build.cache.enabled=false`（跑到 package 产出 shade jar）， +再 `-pl <目标模块> test -Dmaven.build.cache.enabled=false`（**不带 `-am`**）。 + +**另注**：`fe-connector-api` 等上游模块的**已安装 jar 常落后于工作区**，只 `-pl <模块>` 不带 `-am` 会撞 +"no suitable method found" / "cannot find symbol" 之类的**假编译错**（本轮撞了两次：`ConnectorType.structOf` 5 参、 +`isChildCommentSpecified`）—— 那不是代码坏了，是 jar 陈旧。 + +--- + +# 🕗 上一轮 = **重跑 CI 验证 3 个修复**（997422） > **本轮任务** = TeamCity `Doris_External_Regression` **#997422**（PR 65474 @ `6a450c9fa79`） > **10 failed + 2 muted**（occurrence 口径 12）。 diff --git a/plan-doc/connector-cache-unification/00-research-report.md b/plan-doc/connector-cache-unification/00-research-report.md new file mode 100644 index 00000000000000..4e7c4ea6a8a92f --- /dev/null +++ b/plan-doc/connector-cache-unification/00-research-report.md @@ -0,0 +1,168 @@ +Spot-checks confirm the spine: `SPI_READY_TYPES` declared at `CatalogFactory.java:56`, the 7-type `ImmutableSet` literal at `:57`; `PluginDrivenMetadata.get` at `:53` with the identity pin at `:63-69` and the funnel key `"metadata:"+catalogId` at `:70`; both gates exist, `check-authz-cache-sharding.sh` hard-wired to `IcebergConnector.java` (`TARGET_REL`, `:57`). Now the report. + +--- + +# 调研报告:`0b4f72582e7` 统一 iceberg 缓存 + per-statement metadata funnel 框架能否推广到其余连接器 + +> 基线:`branch-catalog-spi` @ HEAD。参考 commit `0b4f72582e7`。本报告的每条论断均挂在 HEAD 代码的 `file:line` 上;连接器逐条结论来自已经过对抗式复核(adversarial verify)的 per-connector 审计。凡输入之间有分歧或低置信处,本报告显式标注。 + +--- + +## 1. 结论先行 (Executive Summary) + +一句话回答 owner:**这个框架有三层,其中 Layer-2(per-statement `ConnectorMetadata` funnel)以及它所依托的共享底座 `fe-connector-cache` 已经是连接器无关(connector-agnostic)、对全部 7 个 `SPI_READY_TYPES` 生效的通用设施——"统一(goal-1)"这一半其实已经做完;真正剩下的、需要逐连接器补齐的,是 Layer-1 那种"连接器自己是否 memoize 重活"的具体填充,而这只在极少数连接器上是真缺口。** + +- **Goal-1(统一缓存框架)最重要的一条建议:不要造"一个连接器无关的元数据大缓存"。** Trino 的实证(INPUT B §3)证明正确的统一粒度是**工具箱(toolkit)**而非缓存本身——因为被缓存对象的类型、失效语义、授权语义天生按连接器不同。Doris 已经站在这条 Trino-correct 路径上:`fe-connector-cache`(`MetaCacheEntry`/`CacheSpec`/`CacheFactory`/`ConnectorPartitionViewCache`)就是 Doris 版的 `io.trino.cache`,并已被 hive/hms/iceberg/maxcompute/paimon 采用。统一工作 = 把**尚未采用工具箱且确有热路径缺口**的连接器接上去(唯一强需求是 **hudi**),外加把 3 个"格式中立"的 iceberg 缓存(table-handle / comment / file-format)在**出现第 2 个消费者时**上收为 `ConnectorXViewCache` 泛型封装——而不是投机式地提前抽象(遵循 Rule 2)。 + +- **Goal-2(热路径性能 + 一致性)最重要的一条建议:唯一 loop-amplified(iceberg-式)的 P1 缺口是 hudi;maxcompute、es 则是 constant-factor(2–4x)的 P1 收尾。** hudi 是所有 SPI 连接器里缓存最薄的一个:零连接器侧 cross-query 元数据缓存、零 per-statement memo,甚至没有像 hive 那样包 `CachingHmsClient`(用的是裸 `ThriftHmsClient`),导致 `HoodieTableMetaClient` 每个 planning pass 重建约 5–6 次、schema 重解析约 4 次(INPUT D `HD-P01/HD-P02/HD-P03`,P1)。这是与"翻闸前 iceberg 那批被删缓存"最像的问题。其余连接器要么已经很好地缓存(hive/paimon/maxcompute-分区),要么结构上不需要(jdbc/es/trino 是廉价透传或委托嵌入式引擎缓存),只剩若干常数倍(2–4x)的 P1/P2 收尾。 + +**净结论:引擎接缝(Layer-2 + 其 gate + `fe-connector-cache` + 泛型分区视图缓存 + 通用 `shouldBypass*` 谓词)已通用且承重;iceberg 特有的是"填充物"。推广工作是有选择地填充,不是重新造框架——重点投在 hudi,其次是 maxcompute / es 的少量常数倍收尾。** + +--- + +## 2. 框架解剖:三层 + 已有共享底座 + +参考 commit 引入的框架分三层(INPUT A 在 HEAD 上核实并锐化): + +**Layer-2 —— 连接器无关的 per-statement metadata funnel(已通用、已承重)。** +这是 Doris 版的 Trino per-transaction `CatalogMetadata`(INPUT B §1):保证 *每个 (statement, catalog) 只有一个 `ConnectorMetadata` 实例*,让一条语句里所有 read/scan/DDL/MVCC/write 解析器共享同一个 metadata。核心件: +- funnel 本体 `PluginDrivenMetadata.get(session, connector)`(`PluginDrivenMetadata.java:53`),按 `"metadata:"+catalogId` 记忆(`:70`),并有 fail-loud 身份钉(`:63-69`),把跨用户凭证复用变成硬错误。这是 fe-core 中**唯一**被允许调 `Connector#getMetadata` 的地方。 +- SPI 中立接缝 `ConnectorStatementScope`(默认 `NONE` no-op),物理归宿 `StatementContext.connectorStatementScope`,写事务共持有者 `CatalogStatementTransaction`,两遍有序 teardown `ConnectorStatementScopeImpl.closeAll`。 +- **关键事实:Layer-2 对全部 `SPI_READY_TYPES = {jdbc, es, trino-connector, max_compute, paimon, iceberg, hms}`(`CatalogFactory.java:56-57`)一体适用,scope 捕获无任何连接器类型分支;hudi 作为 hms 网关下的 sibling 也经 `memoizedSiblingMetadata` 骑同一 scope。** 由 `tools/check-fecore-metadata-funnel.sh` 全构建强制(当前 0 个 exempt marker,写臂已全部入 funnel)。fe-core 中经 funnel 的 call sites 约 **58–59 处**(计法差异,不影响结论)。 + +**Layer-1 —— 6 个 iceberg 连接器侧 cross-query 缓存(一个*可复制的模式*,不是通用代码)。** +`IcebergTableCache / IcebergPartitionCache / IcebergFormatCache / IcebergCommentCache / IcebergManifestCache / IcebergLatestSnapshotCache`,全部建在共享 `MetaCacheEntry` 之上。其中 **table-handle / comment / file-format** 在意图上格式中立(每个连接器都要 load 表对象 / 取注释 / 推断格式),只是 value 类型 iceberg 化;而 **PARTITIONS 元数据表扫描 / manifest / snapshot+schemaId 原子钉** 是真正 iceberg-格式特有。 + +**Layer-3 —— `iceberg.rest.session=user` 下的授权缓存隔离。** 名字为 key 的缓存会把"可 LIST 不可 LOAD"用户的元数据泄漏给他;iceberg 在 `isUserSessionEnabled()` 下把这些缓存置 null,fe-core 侧加通用谓词 `shouldBypass{Schema,TableName,DbName}Cache`。由 `tools/check-authz-cache-sharding.sh` 强制,但**该 gate 硬编码只盯 `IcebergConnector.java`**(核实:`TARGET_REL`,`:57`)。 + +**已有共享底座 `fe-connector-cache`(统一的天然归宿,且泛化已在进行)。** `MetaCacheEntry`(Caffeine-free 的统一 entry API)、`CacheSpec`(`meta.cache...*` 配置)、`CacheFactory`,以及一个**已经写好的 `IcebergPartitionCache` 泛型版** `ConnectorPartitionViewCache`/`PartitionViewCacheKey`。采用现状:`MetaCacheEntry` 被 hive/hms/iceberg/maxcompute/paimon 用;`ConnectorPartitionViewCache` 至少被 hive、paimon 实例化(INPUT D 确认;iceberg 使用其泛型形态见 INPUT A)。**一处 drift 需修:`ConnectorPartitionViewCache` 类 javadoc 写"NO consumers yet"是 HEAD 陈旧——已有 ≥2 个确认消费者(hive、paimon)(Rule 7/12,doc-only)。** + +**这一层的含义:"统一"不是从零造框架,而是:(a) 已通用的 Layer-2 无需再做;(b) 剩余缺口 = Layer-1 式 per-connector memoization + 把确有缺口的连接器接上已有工具箱。** + +--- + +## 3. 横向矩阵 (Cross-Connector Matrix) + +来源 INPUT D(已对抗式复核;`verifyVerdict` 见括注)。iceberg 为参考实现,已完成。 + +| 连接器 | live? | funnel 记忆已加载表? | 现有缓存(要点) | 头号热路径缺口 | authz / session=user 相关? | 优先级 | +|---|---|---|---|---|---|---| +| **iceberg**(参考) | live | **是**(唯一真正 one-load-per-statement:`IcebergStatementScope.sharedTable` + `IcebergTableCache`) | 6 缓存 + per-scan/per-file hoist | —(已修) | **是**(唯一声明 `SUPPORTS_USER_SESSION`) | 参考/已完成 | +| **hive**(=hms) | live | 部分(实例无 memo;靠 cross-query `CachingHmsClient` 折叠) | `CachingHmsClient`(表/分区名/分区/列统计 4 缓存)+ `HiveFileListingCache` + `ConnectorPartitionViewCache` + sibling memo + 写事务快照 | `HMS-H4` ACID `getAcidState` 未缓存(severity none);`HMS-H6` 罕见 TTL/eviction 二次 RPC(P2) | **否**(单一 catalog 身份;RBAC 独立把关) | **LOW(仅 doc 修)** | +| **hudi**(hms sibling,**不在** `SPI_READY_TYPES`) | live | **否**(对象每语句一个,但内部零 memo;scan provider 另建 metaClient) | **零 cross-query 缓存;裸 `ThriftHmsClient`(连 `CachingHmsClient` 都没包)** | `HD-P01` metaClient 5–6x/pass、`HD-P02` schema 4x、`HD-P03` 分区扫描(均 **P1**);`HD-P04` 裸 HMS(P2);`HD-P05` hoist(P2) | 否(无 `getCapabilities` override → 无 `SUPPORTS_USER_SESSION`) | **HIGH / P1(旗舰目标)** | +| **maxcompute** | live | 否(fat-handle 携带 ODPS `Table`,不跨 handle 共享) | `MaxComputePartitionCache`(600s,建在工具箱)+ fe-core schema 缓存;split 枚举干净 | `MC-1` 每次 handle 解析冗余 `tables().exists()` 远程探测,k×/语句(**P1**);`MC-2` lazy `Table` reload(P2) | 否(catalog 静态 AK/SK,单身份) | **MEDIUM / P1(单个小修)** | +| **es** | live | 否(schema/scan-state 均不 memo) | 仅 per-catalog REST client + fe-core schema/rowcount 缓存;连接器侧零 scan 元数据缓存 | `ES-F1` `fetchMetadataState` 2x/查询(shards+nodes,**P1**);`ES-F2` mapping 重取 2x(**P1**);`ES-F3` `getColumnHandles` mapping 2x(P2) | 否(单一 catalog user/password) | **MEDIUM / P1(per-scan hoist)** | +| **jdbc** | live | 否(metadata 构造近零成本) | HikariCP 连接池(per-catalog 单例)+ driver classloader map + fe-core schema/rowcount/name 缓存 | `HP-1` scan 期 `getColumnHandles` 冗余远程取列(P2);`HP-2` 写路径 `buildInsertSql` 新建实例绕开 funnel(P2) | 否(固定 catalog 凭证) | **LOW / P2** | +| **trino-connector** | live(bridge) | 否(委托嵌入式 Trino 连接器自有缓存) | Trino 引导单例 + 每 catalog 一个嵌入式 Trino `Connector`(自带 `CachingHiveMetastore` 等)+ fe-core 缓存 | `TRINO-H1` cold 3x `getTableHandle`(P2,`memoizedAlready=true`);`TRINO-H2` 2N `getColumnMetadata`(P2);`TRINO-H3` 2x `applyFilter`(P2) | 否(静态单身份 `Identity.ofUser("user")`) | **LOW / P2(L1 缓存被明确反指)** | +| **paimon** | live | 部分(fat-handle + paimon SDK `CachingCatalog`) | `PaimonLatestSnapshotCache` + `PaimonSchemaAtMemo` + `partitionViewCache` + SDK `CachingCatalog`(tableCache/partitionCache/manifestCache) | `PA-1` `listPartitionNames/Values` 绕过 `partitionViewCache`(P2,仅 CPU 重渲染) | 否(catalog-创建期认证;REST vended token 是表级、非用户级) | **LOW / P2(一致性小清理)** | + +--- + +## 4. Goal-1:统一缓存框架的建议 + +**总纲:统一在工具箱层,不在缓存层(Trino-correct)。** INPUT B §3 用 Trino 源码证明:Trino 没有"一个统一连接器缓存",只有共享的 `io.trino.cache` 构造/淘汰安全/统计机具,各连接器在其上各建各的缓存。Doris 已如此。因此 goal-1 拆成四件可落地的事: + +**(1) Layer-2 已通用——不动。** funnel + 其 gate + `NONE` read-through + `CatalogStatementTransaction` + `PluginDrivenScanNode` provider/metadata memo,都已连接器无关且被强制。这半个"统一"已完成。 + +**(2) 真正的"统一"缺口不是缓存,是 per-statement 表解析去重(INPUT B §3 第 2 点)。** Trino 天然免费拿到"每事务每表 load 一次",因为 metadata 实例就是事务级 memo 家;Doris funnel 只保证"每语句一个 metadata 实例",是否折叠重复 load 因连接器而异。**但复核(INPUT C/D)显示这个"统一 helper"实际只对 iceberg(已有)和 hudi(metaClient 重建)是刚需**;maxcompute 因 metadata 已是每语句一个,只需在实例内加一个 `Map<(db,table),Handle>`(比 iceberg 的 scope-keyed memo 更简单);paimon/jdbc/es/trino 不构成远程放大。**建议:提供一个共享的"经 statement scope 解析表"helper 作为可选接缝,但只在 iceberg + hudi 两个真消费者上启用**;不要为 4 个不需要的连接器强推(Rule 2)。 + +**(3) 可复用原语 vs 必须留在 iceberg 模块的格式特有物——明确区分:** + +*可上收为通用原语(有 ≥2 消费者或即将有):* +- **`ConnectorPartitionViewCache`(分区视图缓存)——已经是通用原语,已被 hive、paimon 实例化(iceberg 用其泛型形态)。** 建议 hudi 采用它解 `HD-P03`;maxcompute 现用自建 `MaxComputePartitionCache`(同样建在 `MetaCacheEntry`)可保留或迁移,非必须。 +- **`CachingHmsClient`(metastore-client 缓存包装)——已被 hive/hms 共享。** hudi 应直接采用(INPUT D `HD-P04`:一行 parity,立即修 hudi 的裸 HMS 问题)。这是最干净的复用收益。 +- **格式中立的 3 个 iceberg 缓存(table-handle / comment / file-format)**:结构上都是 `new MetaCacheEntry<>(name, null, spec, commonPool, false, true, 0L, true)`,可各自折叠成 `ConnectorXViewCache` 泛型封装。**但复核显示其余连接器几乎都不需要连接器侧 table 缓存**(paimon 有 SDK 缓存、hive 有 `CachingHmsClient`、trino 委托嵌入式、jdbc/es 廉价)。唯一新消费者是 hudi(需要 cross-query metaClient/table 缓存)。**因此建议:泛型 table/handle 缓存在 iceberg+hudi 双消费者出现时再上收;在此之前保持单模块,不投机抽象。** + +*必须留在 iceberg 模块(真正格式特有,不要泛化):* +- `IcebergManifestCache`(manifest 文件)、`IcebergPartitionCache`(PARTITIONS 元数据表 + transform 分区)、`IcebergLatestSnapshotCache`(snapshot id + schema id 原子钉)。hudi 的对应物(metaClient / timeline / Hudi 分区元数据表)语义不同,应在 hudi 模块各自实现,只共享底层 `MetaCacheEntry`/`CacheSpec`。 + +**(4) doc-only 统一债:** 修 `ConnectorPartitionViewCache` 的"no consumers yet"陈旧 javadoc;修 hive/hudi/maxcompute 中"Dormant / hms is not in SPI_READY_TYPES / dormant until hms enters SPI_READY_TYPES / today getTableHandle is never called"等一批陈旧注释(hms 已 live,#65473)。 + +--- + +## 5. Goal-2:热路径性能 + 一致性路线图 + +**重要前提:其余连接器里没有一个存在 iceberg 翻闸前那种 P0 loop-amplified 远程回归——那批 P0 就是 iceberg 本身,已修。** 下面按证据 id 与预期倍数削减排序。 + +### 5.1 查询路径 (query path) + +**P1(旗舰):hudi 全套 memo。** 目标削减: +- `HD-P01`(`HudiScanPlanProvider.java:148` 等):`HoodieTableMetaClient` 每 pass 重建 **~5–6x → 1x**(最小无条件 ~3–4x,含条件站点达 5–6x;复核 `verifyVerdict=ADJUSTED`,上界成立)。做法:per-statement 解析后的 metaClient+schema memo,由 metadata 与 scan provider 共享同一 `ConnectorStatementScope`。 +- `HD-P02`(`HudiConnectorMetadata.java:797` 等):Avro+InternalSchema 重解析 **~4x → 1x**。 +- `HD-P03`(`HudiScanPlanProvider.java:734`):分区元数据表扫描——复核修正:**过滤查询已去重到 ~1 次**(`HudiConnectorMetadata.java:288-291` 注释 + `resolvePartitions` 短路 `:635-638`),"3x/pass" 是上界,真正放大发生在 fe-core 独立多次调 `listPartitions/Names/Values` 或无过滤扫描,以及一次 MTMV refresh 的 4–6 次重枚举。做法:`(table,instant)`-keyed 分区缓存(用 `ConnectorPartitionViewCache`)。 + +**P1:maxcompute `MC-1`。** `getTableHandle` 每次解析都发冗余 ODPS `tables().exists()` 远程探测(`MaxComputeConnectorMetadata.java:118-130`);funnel 只 memo metadata 不 memo handle,故一条语句 k 个独立 handle 解析站点(复核确认 **10 个直接 `resolveConnectorTableHandle` 站点** + 写能力探测)各付一次 RPC。做法:在**已经是每语句一个**的 metadata 实例内加 `Map<(db,table),Handle>`,并对已解析读路径去掉多余存在性探测。削减:**k×/语句 → 1×/语句**,顺带收 `MC-2` 的 lazy `Table` reload。低风险高杠杆。 + +**P1:es `ES-F1`/`ES-F2`。** `fetchMetadataState`(shards+nodes+field-context)每查询被调 2 次(`EsScanPlanProvider.java:99,169`),mapping 又被重取(`EsMetadataFetcher.java:57-58`)。做法:provider 已是每 scan-node 单例,加一个按 `(index,columnNames)` 的字段 memo,把 **2x→1x**,零 staleness 风险(同语句)。**重要修正(复核 `verifyVerdict=ADJUSTED`):`ES-F2` 不是"零新增缓存直接复用 schema"——fe-core `ExternalSchemaCache` 只存解析后的列,`EsConnectorMetadata.getTableSchema` 丢弃了原始 mapping(`:90-93` 传空 properties map),而 `resolveFieldContext` 需要原始 mapping。消除它要么让 `ConnectorTableSchema` 携带 field-context,要么加一个 per-statement/cross-query mapping 缓存。** `ES-F3` getColumnHandles mapping 2x 是 P2(在每语句一个的 metadata 上 memo schema 即可)。**注意:ES shard routing 必须留在 per-statement,不能 cross-query 缓存(ES rebalance/refresh 模型)。** + +**P2 收尾:** hudi `HD-P04`(裸 HMS→包 `CachingHmsClient`)、`HD-P05`(per-scan hoist metaClient/schema/storage-config/uri-normalizer);paimon `PA-1`(`listPartitionNames/Values` 路由进 `partitionViewCache`,仅省 CPU 重渲染,底层远程已被 SDK partitionCache 挡);jdbc `HP-1`;es `ES-F3`;trino `H1/H2/H3`(见 §7,L1 缓存被反指,只做 CPU 清理)。 + +### 5.2 写路径一致性 (write path,read+write 共享一 metadata/一 txn) + +**机制已通用且被强制:** fe-core 的写路径 `getMetadata` 接缝都经同一 funnel,身份钉护住跨用户凭证复用;对**有写事务的连接器(hive / maxcompute / iceberg)**,读写共享同一 memoized `ConnectorMetadata`,写事务由 `CatalogStatementTransaction` 在同一 scope 上共持有,两遍有序 teardown(先 finalize txn 再 close metadata)。当前 gate 0 exempt marker。 +> 例外(不构成一致性风险):jdbc 的 `buildInsertSql` 在连接器内部 new 一个**非 funnel** 的 `JdbcConnectorMetadata`(INPUT D `HP-2`,P2),故 jdbc 的读与写不共享同一 memoized metadata;但 jdbc 写是 BE 逐行 auto-commit(`NoOpConnectorTransaction`),无读写快照一致性需求,因此正确性中立。 + +**哪些连接器有写路径、需要写事务共持有者:** +- **已有且已接线:hive**(`HiveConnectorTransaction`,捕获 begin-time 表快照供 reject-guard 与 sink)、**maxcompute**(`MaxComputeConnectorTransaction`,write session + block 分配按 txn_id;非-MVCC,无需读写快照共享)、以及 **iceberg**(`CatalogStatementTransaction` 参考)。 +- **不需要:** paimon(写**未**迁移,`getCapabilities` 不声明写,`PaimonConnector.java:318`)、jdbc(BE 逐行 auto-commit,`NoOpConnectorTransaction`)、es/trino/hudi(只读,`beginTransaction` throws 或无 override)。 + +**写路径一致性缺口(相对 Trino,INPUT B §2/§4):** Doris 对 hive 走"粗 REFRESH+TTL",不像 Trino 在事务内围绕写做失效——存在 TTL 有界的 read-your-write 窗口。建议:要么给 `CachingHmsClient` 加写路径失效,要么让写后读走 live(`NONE`/bypass)读。**优先级低(TTL 有界),且仅 hive 相关。** + +--- + +## 6. 一致性与授权 (session=user) 的跨连接器影响 + +**核心事实:在全部 8 个连接器中,只有 iceberg-REST 声明 `SUPPORTS_USER_SESSION` 并做 per-user 凭证 vending / delegated `loadTable`。** 其余 7 个连接器逐一核实(INPUT D)均为**单一 catalog 级身份**: + +- hive/hudi:catalog 级单一 Kerberos keytab principal 或 `context.executeAuthenticated`,无 per-user delegated 凭证、无 REST OIDC、无 per-identity metastore。hudi 的安全性(复核 `verifyVerdict=ADJUSTED`)不靠 hive 网关那个 iceberg-专用 guard(`HiveConnector.java:548-556` 文案是 iceberg-specific,`getOrCreateHudiSibling` 无等价 guard),而靠两条已核实事实:(a) hudi 无 `getCapabilities` override → 继承空默认,无 `SUPPORTS_USER_SESSION`;(b) hive 前门本身从不是 session=user。 +- maxcompute:catalog 静态 AK/SK。jdbc:固定 catalog user/password。es:单一 catalog user/password。trino:静态 `Identity.ofUser("user")`,底层用 catalog 属性里的静态凭证。paimon:catalog-创建期认证;REST vended token 是**表级、从共享 catalog session 取,非按 Doris 用户 key**(`PaimonScanPlanProvider.extractVendedToken`),故名字为 key 的缓存不会 list≠load 泄漏。 + +**推论——直接影响 goal-1 的安全性:** + +1. **今天给这 7 个连接器加名字为 key 的 cross-query 缓存都是 authz-安全的**(所有用户看同一 catalog 身份视图,per-user 访问由 fe-core RBAC 独立把关)。因此推荐给 hudi 加的缓存(`CachingHmsClient` + 分区缓存)**当前无需 Layer-3 隔离**。 +2. **但这是"当前"的安全,不是"结构性"的安全。** 一旦任何连接器未来加入 per-user 凭证 vending(例如 hudi/hive 接入 REST-OIDC,或 paimon 把 vended token 变成 per-user),名字为 key 的缓存立刻变成 list≠load 泄漏面。而 `check-authz-cache-sharding.sh` **硬编码只盯 `IcebergConnector.java`**(核实 `:57`),不会保护第 2 个连接器。 +3. Layer-3 的**策略谓词** `shouldBypass{Schema,TableName,DbName}Cache` 已是连接器通用的 fe-core 代码(capability+credential gated),只是目前只有 iceberg-REST 触发它——这一半已经为未来准备好了;缺的是**缓存置 null 的连接器侧动作**与**gate 的通用化**。 + +**建议:** 不要"统一"进一个元数据泄漏 bug。在给任何连接器加名字为 key 的 cross-query 缓存前,把"该连接器是否 vend per-user 凭证"作为准入检查项;并考虑把 `check-authz-cache-sharding.sh` 从 iceberg-only 改造成"扫描任意声明 `SUPPORTS_USER_SESSION` 的连接器"(见 §8 决策 3)。授权对比 Trino(INPUT B §2b):Trino 在 impersonation 下是**按身份分片缓存**(`LoadingCache`)而非禁用;Doris 目前是**禁用**——安全但更慢,长期方向是 Trino 式 per-identity keying,但那是更大的改动,当前"禁用"是正确的保守首版。 + +--- + +## 7. 风险与反对意见 + +**(1) 对廉价/委托型连接器套 iceberg 重框架 = 投机,违反 Rule 2。** +- **jdbc**:关系型透传,`planScan` 零远程 IO、恒一个 scan range,无 split/file/manifest/partition/snapshot 层。昂贵元数据(schema/columns、row-count、名录)已被 fe-core cross-query 缓存前置,连接池是 per-catalog 单例。加连接器侧 table 缓存无对象可缓存。 +- **es**:只读、非分区、单凭证、无 snapshot/manifest/stats/write。除 `ES-F1` 的 per-scan hoist 外,几乎每个 iceberg 件都不适用;且 **shard routing 绝不能 cross-query 缓存**(freshness/rebalance)。 +- **trino**:**主动反指(actively contraindicated)。** 它是 bridge,元数据缓存/失效/split 枚举全委托嵌入式 Trino 连接器。加 Doris 侧 table/handle 缓存会**双重缓存**并把 schema 跨外部 ALTER 冻结,重新引入 Trino 已解决的 stale-metadata bug 类;且 `TrinoTableHandle` 是事务派生、transient/不可序列化。只做 P2 CPU 清理(`H2` 去掉 2N 重取、`H3` 去掉 double `applyFilter`)。 +- **paimon**:SDK `CachingCatalog`(默认开)已提供 in-memory tableCache/partitionCache/manifestCache;连接器又已恢复 3 个 legacy 语义缓存。加连接器级 table 缓存会与 SDK 缓存重复。只有 `PA-1` 一个 P2 一致性清理值得做。 + +**(2) 强推所有连接器上同一种缓存形状 = 与 per-connector 语义作对。** 被缓存 value 类型、key、失效、授权语义天生不同(一个 iceberg `Table` 不是 `HmsTableInfo` 不是 paimon `Table` 不是 ODPS `Table`)。Trino 明确拒绝"一个缓存统治所有"(INPUT B §3),Doris 若比 Trino 抽象得更狠会打同一场必输的仗。**正确姿势:统一 toolkit,按连接器各建缓存。** + +**(3) hive 已 framework-aligned,勿冗余套用。** hive 已有 `CachingHmsClient`(4 缓存)+ `HiveFileListingCache` + `ConnectorPartitionViewCache` + sibling memo + 写事务快照;`HMS-H4`(ACID `getAcidState` 未缓存)是 snapshot/write-id 依赖、legacy-parity 正确,**应保持不缓存**;`HMS-H6` 罕见二次 RPC 的 belt-and-suspenders per-statement memo **不划算**(HMS `getTable` 远比 iceberg `loadTable` 便宜)。hive 的实际工作只有 doc 修。 + +**(4) 一致性回退风险(低但需记):** §5.2 的 hive 写后读 TTL 窗口;若强行给 hudi/es 的 freshness-敏感数据加 cross-query 缓存会引入陈旧读——故 hudi 的 metaClient/分区缓存要 TTL+REFRESH 有界,es 的 shard routing 保持 per-statement。 + +--- + +## 8. 给用户的开放问题 / 需拍板的决策 + +1. **范围(scope):** 本轮只打 hudi(唯一 P1 真缺口),还是把 3 个 P1 常数倍收尾(maxcompute `MC-1`、es `ES-F1/F2`)一并纳入?我的建议是 **hudi 单独立项 + maxcompute/es 各一个小 PR**,jdbc/paimon/trino P2 进 backlog。请拍板是否同意这个切分。 + +2. **排序(sequencing):先建共享底座还是先按连接器修?** 由于泛型 table/handle 缓存目前只有 hudi 一个新消费者,我倾向 **per-connector-first**:hudi 直接复用**已存在**的 `CachingHmsClient` + `ConnectorPartitionViewCache` + 一个 hudi 模块内的 metaClient 缓存(建在 `MetaCacheEntry`),等 iceberg 的 3 个格式中立缓存出现第 2 个消费者再上收为 `ConnectorXViewCache`。是否接受"先修后抽象、不投机泛化"? + +3. **`check-authz-cache-sharding.sh` 是否现在就通用化?** 现硬编码只盯 iceberg。选项:(a) 现在改成"扫描任意声明 `SUPPORTS_USER_SESSION` 的连接器"(前瞻,防止未来第 2 个 session=user 连接器 unify 出泄漏);(b) 保持 iceberg-only,待真出现第 2 个再改。今天所有推荐的新缓存都 authz-安全,故 (b) 无当下风险,但 (a) 更稳。请选。 + +4. **是否投入共享的"经 statement scope 解析表"helper?** 它是 INPUT B 眼中"统一连接器缓存框架"的真正含义,但复核显示只有 iceberg(已有)+ hudi 刚需。选项:(a) 提炼成 fe-core 共享接缝供两者用;(b) 让 hudi 在自己模块内做 memo,不提炼。建议 (b)(Rule 2),除非预期近期有第 3 个消费者。 + +--- + +## 9. 建议的后续任务空间 (workspaces & sequencing) + +建议按 `plan-doc/perf-hotpath-iceberg/` 的布局镜像出 per-connector workspace(每个含:热路径审计、problem-class 归类、修复设计、mutation/验证门): + +- **`plan-doc/perf-hotpath-hudi/`(P1,旗舰,先做)** —— 覆盖 `HD-P01/02/03/04/05`。核心交付:(1) per-statement resolved-metaClient+schema memo,由 `HudiConnectorMetadata` 与 `HudiScanPlanProvider` 共享 scope(杀 `HD-P01/02`);(2) 包 `CachingHmsClient`(一行 parity,修 `HD-P04`);(3) cross-query metaClient/table 缓存 + `(table,instant)`-keyed 分区缓存(用 `ConnectorPartitionViewCache`,修 `HD-P03` 的 MTMV/SHOW PARTITIONS 重枚举);(4) per-scan hoist(`HD-P05`)。无 authz、无写事务工作。 +- **`plan-doc/perf-hotpath-maxcompute/`(P1,小)** —— 单点:在每语句一个的 `MaxComputeConnectorMetadata` 内加 `Map<(db,table),Handle>` 并去掉已解析读路径的冗余 `exists()` 探测(修 `MC-1`,顺带 `MC-2`)。 +- **`plan-doc/perf-hotpath-es/`(P1,小)** —— per-scan hoist `EsMetadataState`(修 `ES-F1`)+ 决定 `ES-F2` 的 mapping/field-context 承载方式(enrich `ConnectorTableSchema` 或新增 mapping 缓存——注意这不是"零新增缓存"改动)+ 在 metadata 上 memo schema(`ES-F3`)。**约束:shard routing 保持 per-statement。** +- **`plan-doc/cleanup-stale-connector-docs/`(doc-only,可随手做)** —— 修 `ConnectorPartitionViewCache` "no consumers yet";修 hive(`CachingHmsClient:85-88`、`HiveFileListingCache:72-74`、`HiveConnector.wrapWithCache:667-668` 及 `:118,126-127` sibling 字段注释)、hudi(`HudiConnectorMetadata.java:391`、`HiveConnectorMetadata.java:410,1802,1942,1972,2007`)、maxcompute(`beginTransaction:332`、`MaxComputeWritePlanProvider:74`)的"dormant / 未在 SPI_READY_TYPES / never called"陈旧注释。 +- **`plan-doc/perf-hotpath-backlog-p2/`(P2,可延后)** —— paimon `PA-1`;jdbc `HP-1/HP-2`;trino `H1/H2/H3`(仅 CPU 清理,不加 L1 缓存);hive 写后读一致性(给 `CachingHmsClient` 加写路径失效或 bypass 读)。 + +**排序建议:** 先 hudi(唯一 P1 真缺口、收益最大)→ 并行 maxcompute + es 两个小修 → doc-only 清理随时 → P2 backlog 按热点 profile 触发。共享底座泛化(iceberg 3 缓存上收 + `check-authz-cache-sharding.sh` 通用化)留待 §8 决策拍板后,且仅在出现第 2 个消费者时启动,避免投机抽象。 \ No newline at end of file diff --git a/plan-doc/connector-cache-unification/HANDOFF.md b/plan-doc/connector-cache-unification/HANDOFF.md new file mode 100644 index 00000000000000..5029fdc58d8cb0 --- /dev/null +++ b/plan-doc/connector-cache-unification/HANDOFF.md @@ -0,0 +1,129 @@ +# 🤝 Session Handoff — 连接器缓存框架统一 (connector cache unification) + +> **滚动文档**：顶部「下一步」区每 session 结束**覆盖式更新**，只保留下一个 session 必须的上下文； +> 底部「稳定区」（推进流程 / 铁律 / build 坑 / 并发探测）**勿随意覆盖**，是跨 session 常驻规则。 +> 完成明细进 `git log` + 各兄弟空间的 `designs/` + 本空间 [`progress.md`](./progress.md)。 +> **本空间是"伞形协调空间"**：调研结论在 [`00-research-report.md`](./00-research-report.md) + [`connectors/*.md`](./connectors/) + [`data/connector-audits.json`](./data/connector-audits.json)； +> 真正的逐连接器执行**各自另开兄弟空间**（`plan-doc/perf-hotpath-/`，镜像 `plan-doc/perf-hotpath-iceberg/` 布局），不并进本空间（报告 §9）。 + +--- + +## 📖 新 session 开场读序 + +``` +1. Read 本 HANDOFF.md ← 你在这（下一步 + 稳定区规则） +2. Read tasklist.md ← workstream 追踪到哪了、决策签了没 +3. 需要某条发现细节时才 Read 00-research-report.md 对应节 / connectors/.md / connector-audits.json + —— 别默认全读；报告很长，按需取。 +4. 参考实现模板：plan-doc/perf-hotpath-iceberg/（README=流程 / HANDOFF / tasklist / designs） + 问题类：plan-doc/perf-heavy-op-hot-path-problem-class.md +``` + +--- + +# 🆕 下一步（覆盖区） + +## 🎯 上一 session 收尾（2026-07-24 (7)，HEAD `7df22cd1c71`）：owner 点名三项 = **2 做 + 1 证伪** + +owner 2026-07-24 点名的三项 P2 已收尾（详见 [`progress.md`](./progress.md) "2026-07-24 (7)" + 兄弟空间 `perf-hotpath-paimon` / `perf-hotpath-jdbc`）： + +1. **paimon 分区列举去重** ✅ commit `59b65912104`：抽 `cachedPartitions` 让 `listPartitionNames/Values` 与 `listPartitions` 共享 `partitionViewCache`。**owner 拍板"走缓存"**——SHOW PARTITIONS/`partition_values()` 新鲜度从"每次重算"→24h TTL+REFRESH（与裁剪路径/Trino 自洽；与 hive 故意实时不一致但 paimon 只读无写后读问题）。26 测试 + 3-lens 净室。 +2. **jdbc 两处取列** ✅ commit `7df22cd1c71`：读侧（`getColumnHandles`×2 + `getTableSchema`）+ 写侧（`buildInsertSql`）经新 `ConnectorStatementScopes.JDBC_COLUMNS` 每语句作用域记忆化原始列，**读写自动共享（一个改动点修两处）**。**owner 拍板"语句作用域统一去重"**。199 测试 + 3-lens 净室（修 `jdbcTypeToConnectorType` 原地改 allowNull 的共享-mutable 不变量）。 +3. **hive 写后读一致性** 🚫 **非 bug（侦察证伪，owner 关闭）**：前提"写不失效"错——每次 `hms` INSERT 提交后 `PluginDrivenInsertExecutor.doAfterCommit → RefreshManager.handleRefreshTable → connector.invalidateTable` 已**同步全表失效**（协调 FE 同 session 写后读到新数据）。残留仅跨 FE 编辑日志回放毫秒窗口（通用 Doris 非 hive 专有）+ 过度失效（fe-core 侧 perf 非正确性，碰铁律 A）。印证 `execution-blueprint-overestimates-recon-first`。 + +**下一 session 议程（无 owner 新指令时）**：本条线主体已收官。剩余候选（均可延后，等 owner 点名或热点触发）：① **各连接器 e2e 统一补**（需集群，paimon/mc/es/hudi/jdbc 均标"待集群"）；② **trino P2 纯 CPU 清理**（TRINO-H1/2/3，未排期——owner 未点名 + 受"禁加 L1 缓存"硬约束）；③ **iceberg 5 缓存全量收敛**（远期，PR-3 已只做安全 ttl 去重）。动任何一项前仍守：按 HEAD 重侦察、铁律 A（fe-core 0 行）、净室对抗复审。 + +## 当前状态（历史，2026-07-23 快照 — 以上「议程」为准；round-1 + 门禁移除 + 注释清理均已完成） + +- **调研 + 设计均完成；owner 4 决策已签字 + 设计后二次确认**（[`tasklist.md`](./tasklist.md) §0）。 +- **设计定稿** → [`designs/foundation-design-FINAL.md`](./designs/foundation-design-FINAL.md)（配 `foundation-design-draft.md` + `review-1..3.md`）。11-agent 只读设计调研 + 3 路对抗评审，全程逐符号核对 HEAD。 +- **底座 PR-0/1/2 已落地（下方「进行中」为准）；连接器消费 PR-3~7 + e2e 未跑。** 设计里 `file:line` 是 2026-07-23 HEAD 侦察快照，动码前须按 HEAD 重侦察。 +- **核心结论**：走"先建底座"，但**底座几乎现成** → 整套 A/B/C + hudi/mc/es 消费 **全程 0 行 fe-core 改动**（D4 原"提炼进 fe-core"被侦察推翻：per-statement memo 底座已在 `fe-connector-api`，owner 二次确认不动 fe-core；iceberg 本轮就改挂）。 + +## 历史：底座 PR-0/1/2 + ttl 去重 + hudi/mc/es round-1 + **owner 点名 P2 三项（paimon/jdbc 做、hive 证伪）全部收尾**（2026-07-23/24）；门禁改**删门禁+ATTN**、陈旧注释清理已完成；各连接器 e2e 待集群；动每个文件前先按 HEAD 重侦察 + +**maxcompute（WS-MC）round-1 完成**（commit `58daadd10e0`，见 `plan-doc/perf-hotpath-maxcompute/`）：per-statement `Map<(db,table),handle>`（CHM）present-only memo，收冗余 ODPS `exists()` k×→1× + Table reload 每表 1×；0 fe-core；120 测试绿 + 两处变异 + 净室复审（concurrency 经 verify REFUTED）。 + +**es（WS-ES）round-1 完成（owner 拍板"三件全做"）**（commits `7d74ba1161b` + `7466b354901`，见 `plan-doc/perf-hotpath-es/`）：①per-scan hoist（`EsScanPlanProvider` memo `EsMetadataState`，plain 字段——ES 非 batch）②per-statement schema memo（`EsConnectorMetadata` CHM，镜像 mc）③cross-path raw-mapping 经每语句 `ConnectorStatementScope`（新命名空间 `ES_INDEX_MAPPING`，schema/scan 两路共享，**分片/节点绝不入作用域**）。每语句 getMapping/shard/node 各→1；**0 fe-core**；94 测试绿 + 三处变异 + 4-lens 净室（parity/concurrency/freshness HOLD，补 `ShardRoutingNeverSharedViaScope` 钉硬约束）。 + +**至此三个 P1 连接器（hudi 旗舰 / mc / es）round-1 全部收尾。** 剩余：P2 backlog（paimon PA-1 / jdbc HP-1·2 / trino CPU / hive 写后读，热点触发）、各连接器 e2e 统一补（需集群）。iceberg 5 缓存全量收敛仍延后（PR-3 已只做安全 ttl 去重）。**WS-DOC 陈旧注释清理已完成**（2026-07-24 (6)：原点名的 HMS-flip 注释早已更新；另清 iceberg+maxcompute 写/过程"翻闸前休眠"陈旧注释 9 文件纯注释；hudi 增量真休眠留存）。 + +**⚠ 授权缓存门禁项（原 PR-7 门禁通用化）已关闭——不是通用化而是删除（owner 2026-07-24 拍板）。** 尝试把 `check-authz-cache-sharding.sh` 改成"模块内构造点扫描 + 结构化验证置空"，经**两轮对抗净室复审**证伪：shell 无法可靠判断"缓存在每用户模式下是否真置空"（须理解任意 Java 布尔/多行语法——复合 `&&`/`||`、多行 lambda loader、字符串字面量、嵌套三元式；两轮共暴露 10+11 个误报/漏报，误报会挡合法构建、漏报会放走泄漏，每修冒新边界）。而置空正确性**已由运行时 `IcebergConnectorCacheTest` 证明**（比静态检查强），门禁的机器验证是越界。owner 拍板**删掉整个门禁**（`tools/check-authz-cache-sharding.sh` + 自测 + `fe-connector/pom.xml` 执行块），改在 `IcebergConnector.java` 加显式 `ATTN` 注释（讲清 list!=load 风险、每缓存已构造函数置空、新增缓存必须置空且被单测覆盖、无构建门禁靠评审+单测）。落地严格 4 文件、`mvn validate` 过。见 [`progress.md`](./progress.md) "2026-07-24 (5)"。**通用教训**：shell 门禁只配"存在性/前缀"类不变量；要理解语言语义就不是它的活。 + +设计已 owner 确认，已开工。**PR-0 完成**（预编译执行连接器作用域 reset 回归守门 + 外表可达性查实 + FINAL 设计机制描述更正；见 [`progress.md`](./progress.md) "2026-07-23 (3)"）。**PR-1 完成**（通用缓存封装升格为 `ConnectorMetadataCache`，纯重命名 + 构造器加 `entryName`，66 分区视图缓存测试证零变化；commit `a804145faaa`，见 "2026-07-23 (4)"）。**PR-2 完成**（语句作用域通用 helper `ConnectorStatementScopes.resolveInStatement` 放 `fe-connector-api`=**0 行 fe-core**，iceberg `sharedTable` 改委派 byte-identical；8+7 单测 + iceberg 全模块 1133 测试 0 失败 + 4 路对抗净室复审 PARITY_HOLDS + Rule-9 变异验证；commit `ae8c925074d`，见 "2026-07-24"）。 + +**PR-3 重定范围（owner 2026-07-24 拍板）**：原计划"iceberg 5 手写缓存全量收敛到 `ConnectorMetadataCache`"经动码前重侦察 + 向 owner 讲清成本后**改为只做安全 DRY、全量收敛延后**——收敛零功能收益、改动面宽（~19 文件/重写 ~37 测试/6 重载构造函数波及/调用点退化成字符串四元组键），且框架已被 3 连接器分区视图缓存证明可用、hudi/mc/es 不依赖它，Trino 亦是"共享原语+各连接器自持缓存"。**已落地的安全那一块**：新增 `CacheSpec.ofConnectorTtl` 折叠"`ttl≤0` 禁用"契约，**6 处**复制三元式（`IcebergComment/Format/LatestSnapshot/Partition/Table` + `PaimonLatestSnapshotCache`）改调它，逐字节等价、既有测试全绿（见 "2026-07-24 (2)"，commit `e1adddf22e8`）。**旗舰 hudi Round 1 已完成**（`plan-doc/perf-hotpath-hudi/`：文档 + HMS 缓存 + 每语句 memo 三块落地，commit `26690775c81` 为旗舰 memo——采"两块独立 per-statement memo"，动码前重侦察推翻蓝图多处、合并版经净室复审后 owner 拍板退回零耦合方案）。**下一步 = mc / es 两个小 PR**（`WS-MC` / `WS-ES`），或 hudi Round 2 延后项（跨查询缓存 `PERF-H04` 等）。 + +**设计里行号/乘数是 2026-07-23 快照，动每个文件前按符号重 grep 确认**（memory `execution-blueprint-overestimates-recon-first`；已累计侦察更正——PR-0 机制"新 context"实为"复用+显式 reset"、PR-1"删兼容子类"实为无子类可删、PR-2 `rewritableDeleteSupply` 须留 iceberg 私有、PR-3 全量收敛经侦察判为高成本零收益已延后）。**若日后重启 iceberg 收敛**，侦察须重确认：5 缓存全独立 `final class` 建于 `MetaCacheEntry`、entry 名 hyphen（`iceberg-table` 等非 `iceberg.table`）、`formatCache` 挂 `IcebergScanPlanProvider`/`IcebergConnectorMetadata` 双持、`invalidateDb` 现走 `Namespace.of(db)+id.namespace().equals`（收敛须证单层 namespace 等价 String-equals）。 + +## 实施路线（8 个独立 PR，foundation-first；详见 FINAL 设计 §Sequencing / §Risk register / §Verification plan） + +| PR | 主题 | 关键约束 / 守门 | +|---|---|---| +| **PR-0** ✅（2026-07-23 完成） | 验预编译 `EXECUTE` 重执行时 statement scope 是否每次刷新 | 已加 `ConnectorStatementScopeTest.executeCommandResetsConnectorScopePerExecution`（驱动 `ExecuteCommand.run()`、变异验证仅该测试变红）；**机制更正**：不是"新 context"而是复用 context + `ExecuteCommand.java:95` 显式 reset；**外表可达性已查实**（走普通 `executor.execute()` 路，非 OLAP 短路） | +| **PR-1** ✅（2026-07-23 完成，commit `a804145faaa`） | 升格 `ConnectorPartitionViewCache[V]`→`ConnectorMetadataCache[V]`；`PartitionViewCacheKey`→`ConnectorTableKey`；构造器加 `entryName` 参数；iceberg/hive/paimon 改挂；修陈旧 javadoc。**更正**：无"旧类可删"（是重命名非删除）；**收窄**：ttl 去重 + 预解析 CacheSpec 构造器推迟到 PR-3（避免二次翻动 iceberg 手写缓存） | ✅ `install -pl cache,hive,iceberg,paimon -am` BUILD SUCCESS；66 分区视图缓存测试 0 失败，条目名/配置项/键逐字节不变 | +| **PR-2** ✅（2026-07-24 完成，commit `ae8c925074d`） 语句作用域 helper（`fe-connector-api`）+ iceberg 改挂 | 加 `ConnectorStatementScopes.resolveInStatement` + namespace 注册表（`ICEBERG_TABLE`）；iceberg 私有包装改委派（key 逐字节不变，`rewritableDeleteSupply` 留私有） | ✅ **fe-core 0 行**；8+7 单测 + iceberg 全模块 1133 测试 0 失败 + 4 路对抗净室 PARITY_HOLDS + Rule-9 变异（queryId→sessionId 恰红 2 byte-key 测试） | +| **PR-3** ✅（2026-07-24 **重定范围**：只做安全 ttl 去重，全量收敛延后） | 新增 `CacheSpec.ofConnectorTtl` 折叠"`ttl≤0` 禁用"契约，6 处三元式改调它（iceberg 5 + paimon 1），逐字节等价 | ✅ BUILD SUCCESS；`CacheSpecTest`+1、6 缓存既有测试原样全绿、iceberg 1134/paimon 379 0 失败。**收敛部分延后**：5 缓存类/键/测试原样保留，`invalidateDb` Namespace→String parity、pre-resolved 名/loadCount 访问器等出现真实需要再上收 | +| **PR-4** 旗舰 hudi | metaClient+schema 每语句 1 次；HMS 走 `CachingHmsClient`；`(表,已完成instant)` 跨查询分区缓存；per-scan hoist | **记忆"不可关闭投影"非原始 metaClient**（scope 末关所有 AutoCloseable）；instant **每语句重取最新已完成**、只缓存分区列表；pom 加 `fe-connector-cache` + **验 Caffeine≥2.9.3 自带**；**e2e 必需**（异构+独立 hudi-on-HMS + 并发提交分区列举） | +| **PR-5** maxcompute（小） | `getTableHandle` 每语句 memo，消冗余远程 `exists()`（14–17→1） | 连接器侧、fe-core 0 行；仅 in-statement `buildConnectorSession` 路径 | +| **PR-6** es（小） | `EsMetadataState` per-scan hoist（2→1）；raw mapping 挂语句作用域态 | **分片路由必须每语句、拆 `fetch()`**、绝不跨查询缓存 | +| **PR-7** ~~门禁通用化~~ 🚫 **撤销→删门禁**（owner 2026-07-24） | 结构化验证置空正确性 shell 做不稳（两轮对抗复审 10+11 误报/漏报）；运行时 `IcebergConnectorCacheTest` 已证明置空 → **删整个门禁 + 加 `ATTN` 注释** | 已落地；见 progress.md "2026-07-24 (5)" | + +- **PR-1 + PR-2 是底座**，解锁 PR-3/4/5/6；**PR-7 独立**；**PR-0 阻塞 PR-2**。 +- **连接器 PR（4/5/6）用兄弟空间**（`plan-doc/perf-hotpath-/`，镜像 iceberg 布局）逐项立项；**底座 PR（1/2/3/7）跨连接器**，进本伞形空间 `designs/`。 +- **本轮全程守铁律 A（fe-core 0 行）**——实施中若发现某步"不得不碰 fe-core"，**停手交 review**（大概率又是侦察能推翻的伪需求）。 + +--- + +# 🧱 稳定区（勿覆盖） + +## 从调研到执行的推进流程（每个 workstream） + +> 一次一个 workstream 串行推进；这些是**性能/结构修复 → 必须行为不变**，复核与"证明只减不改"是重点。对齐 `step-by-step-fix` skill 与参考空间 `perf-hotpath-iceberg/README.md` 的立项流程。 + +``` +1. 拿 owner 签字（D1 决定本 workstream 是否在本轮范围内）。 +2. 新开兄弟空间 plan-doc/perf-hotpath-/（镜像 perf-hotpath-iceberg 布局）， + 把该连接器的发现（HD-P0x / MC-x / ES-Fx …）拆成 PERF-NN 逐项。 +3. 【动码前必做】按 HEAD 重侦察：重新 grep 行号、重新确认 ①重操作真实成本 ②乘数 ③是否真无缓存兜住。 + 调研是估算，复核可能缩小/推翻某条（如 hudi HD-P03 对过滤查询已去重到 ~1 次）。 +4. 写设计文档 + 设计红队（clean-room 对抗，本项目偏好）。 +5. 实现：最小改动、守铁律、连接器侧、匹配风格。 +6. 验证：相关 UT/e2e 全绿（parity 禁回归）+ 补调用计数守门（如 metaClient build 从 N 降到 1）。 + e2e 需集群，本地不跑的留标注。 +7. 独立 commit：[perf](catalog) fe-connector-: (PERF-NN)。commit log 全英文。 +8. 更新该兄弟空间 tasklist/HANDOFF；回本伞形空间更新 tasklist workstream 状态 + append progress.md。 +``` + +## 🔒 铁律（违反即返工，动码前记牢） + +1. **fe-core 源只出不进 + 禁 scaffolding 搬迁**（memory `fe-core-source-isolation-iron-rules`）：新缓存/memo 一律放**连接器侧**（`fe-connector-` / handle / `fe-connector-cache` 工具箱），**不得**为省事把 table 缓存/属性解析/派生 helper 塞进 fe-core。hudi/mc/es 的所有推荐修复都在连接器侧，天然合规；**D4 若选"提炼进 fe-core"= 净增 fe-core SPI，碰铁律 A，须用户显式签字**（报告建议不提炼）。 +2. **fe-core 通用节点保持 connector-agnostic**：不在 `PluginDrivenScanNode` 等通用 SPI 层写 source-specific 分支；通用热路径若要改须证 **byte + cost 对所有连接器双不变**。本轮推荐修复**都不碰** fe-core 通用节点（纯连接器侧），若某项需要碰，停手交 review。 +3. **fe-core 不解析属性**（memory `catalog-spi-no-property-parsing-in-fecore`）：属性派生放插件侧组装 → BE thrift → 回传。 +4. **新缓存准入必查 authz**（报告 §6）：给任一连接器加"名字为 key 的 cross-query 缓存"前，先确认该连接器**是否 vend per-user 凭证**。今天 7 个非-iceberg 连接器都是**单一 catalog 身份**（hudi/mc/es/jdbc/trino/paimon/hive 逐一核实），故名字为 key 的缓存 authz-安全、**无需 Layer-3 bypass**。**但这是"当前"安全非"结构"安全**：一旦某连接器未来接 per-user 凭证 vending（REST-OIDC 等），名字为 key 的缓存立刻变泄漏面，而 `check-authz-cache-sharding.sh` 现只盯 iceberg（见 D3）。 +5. **freshness-敏感数据的 cross-query 缓存必 TTL + REFRESH 有界**：hudi 的 metaClient/分区缓存要有界；**es 的 shard routing 绝不能 cross-query 缓存**（ES rebalance/refresh 模型），保持 per-statement。 +6. **反指别硬套**（报告 §7）：jdbc / trino / paimon / hive **不要**再加 iceberg-式重缓存（廉价透传 / 委托嵌入式引擎缓存 / SDK 已缓存 / 已 framework-aligned）；详见 [`tasklist.md`](./tasklist.md)「反指/不立项」。 + +## 🧰 build / 验证坑（继承自参考空间 + 本程序特有） + +1. **maven 用绝对 `-f /fe/pom.xml`**（cwd 跨调用持久，`cd` 破相对路径）；`${revision}` 未 flatten → 必 `-am`；测试用 **`install`** 而非 `test`（`-am test` 不产 shade jar）。 +2. **测试必加 `-Dmaven.build.cache.enabled=false`**（否则 surefire 静默跳过）；**别用 `mvn -q` 跑测试**（抑制 `BUILD SUCCESS`/`Tests run`）；grep 日志 `BUILD SUCCESS` + `Tests run:`。 +3. **别 `nohup … &` 套 `run_in_background`**（maven 变孤儿假 exit 0）；后台跑直接 `mvn … >> log 2>&1`，**通知里 exit code 是 echo 的**，读日志 `BUILD SUCCESS` 行。 +4. **动 fe-core 者两段验**（若某项不得已碰 fe-core）：连接器模块 `-pl -am` 反应堆不含 fe-core；fe-core 改动须另 `mvn test -pl fe-core -am -Dtest=… -f /fe/pom.xml`。 +5. **⚠ hudi 特有**：`fe-connector-hudi` 目前**零 `fe-connector-cache` 依赖**（pom 没引工具箱，也没包 `CachingHmsClient`）——WS-HUDI 第一步要**改 pom 引入工具箱**，这是 hudi 独有的前置（hive/paimon/mc 已引）。 +6. **checkstyle 主源 ≤120 且扫 test 源**；import 序 `SAME_PACKAGE→THIRD_PARTY→STANDARD_JAVA`，组内字母序组间空行。 + +## 🔎 并发探测（动码前） + +- 本 worktree：`find fe -newermt '-120 sec'`（滤 `/target/`）+ `pgrep -a -f maven`；`/mnt/disk1/yy/git/doris` 是另一 worktree，不冲突（memory `concurrent-sessions-shared-worktree-hazard`）。 +- 本 worktree 的 `.git` 是**文件**不是目录（linked worktree）——rebase 脚本用 `git rev-parse --git-path`（memory `worktree-gitdir-is-a-file`）。 +- 提交只精确 `git add` 本任务文件：工作树有大量无关 untracked（`.claude/`、`docker/`、`regression-conf.groovy` 本就脏），**禁 `git add -A`**。 + +--- + +# 🔗 关系 + +- **参考实现（模板）**：[`plan-doc/perf-hotpath-iceberg/`](../perf-hotpath-iceberg/)（iceberg 那批被删缓存的逐条修复，`PERF-01..11`）+ [`perf-heavy-op-hot-path-problem-class.md`](../perf-heavy-op-hot-path-problem-class.md)（本调研套用的"热路径重操作放大"问题类）。 +- **主线**：`plan-doc/HANDOFF.md`（catalog-spi 主线）——**并行但不混流**。 +- **协作规范 / context 预算 / subagent 纪律**：沿用 [`../AGENT-PLAYBOOK.md`](../AGENT-PLAYBOOK.md)。 +- **交接纪律**（memory `handoff-discipline-per-phase`）：每轮完成即更新 HANDOFF/tasklist；每轮起步先读 HANDOFF 再对照 HEAD 真实代码 review（计划/行号可能过时）。context 用量过 30% 就找干净节点覆写 HANDOFF 并通知用户开新 session（memory `session-handoff-at-30pct-context`）。 diff --git a/plan-doc/connector-cache-unification/README.md b/plan-doc/connector-cache-unification/README.md new file mode 100644 index 00000000000000..12ade93bc8f93e --- /dev/null +++ b/plan-doc/connector-cache-unification/README.md @@ -0,0 +1,51 @@ +# 📦 调研任务空间 — 连接器缓存框架统一 (connector cache unification) + +> **独立调研空间**，与 `plan-doc/HANDOFF.md`(catalog-spi 主线)及 `plan-doc/perf-hotpath-iceberg/`(iceberg 热路径修复,本报告的参考实现)**并行但不混流**。 +> **一句话任务**:参考 commit `0b4f72582e7` 为 iceberg 建了一套「统一元数据缓存 + per-statement metadata funnel」框架;本调研判定**这套框架是否/如何推广到其余连接器**,目标 (1) 统一连接器缓存框架、(2) 保证热路径(查询 + 写入)的缓存性能与一致性。 +> **基线**:`branch-catalog-spi` @ HEAD(2026-07-22)。参考 commit `0b4f72582e7`。 + +--- + +## 🚩 结论(先看这一句) + +**框架三层里,承重的引擎接缝(Layer-2 per-statement `ConnectorMetadata` funnel + 其 build-gate + 共享底座 `fe-connector-cache` + 泛型 `ConnectorPartitionViewCache` + fe-core 通用 `shouldBypass*` 谓词)已经是连接器无关、对全部 7 个 `SPI_READY_TYPES` 生效的通用设施——"统一(goal-1)"这一半其实已经做完。** iceberg 特有的只是"填充物"(6 个连接器侧缓存)。因此推广 = **有选择地填充,不是重新造框架**:唯一 loop-amplified(iceberg-式)的 P1 真缺口是 **hudi**(缓存最薄:零连接器侧缓存、裸 `ThriftHmsClient`、metaClient 每 pass 重建 5–6 次);maxcompute / es 是少量 constant-factor(2–4x)P1 收尾;jdbc / es / trino / paimon 结构上**反指**再加重缓存(廉价透传 / 委托嵌入式引擎 / SDK 已缓存)。 + +> **Trino-correct 原则**(经 Trino 源码核实,见 appendix B):正确的统一粒度是**工具箱**而非"一个大缓存"——被缓存对象的类型、失效语义、授权语义天生按连接器不同。Doris 已站在这条路径上。 + +--- + +## 📄 文件索引 + +| 文件 | 内容 | +|---|---| +| [`00-research-report.md`](./00-research-report.md) | **主报告(交付物)** — 9 节:结论先行 / 框架解剖 / 横向矩阵 / goal-1 统一建议 / goal-2 热路径+一致性路线图 / session=user 授权跨连接器影响 / 风险与反对意见 / 需拍板决策 / 建议后续任务空间 | +| [`appendix-A-framework-taxonomy.md`](./appendix-A-framework-taxonomy.md) | 框架三层 + 共享底座的精确解剖(哪些已通用 / 哪些是 iceberg-only 模式),全部挂 HEAD `file:line` | +| [`appendix-B-trino-reference.md`](./appendix-B-trino-reference.md) | Trino 的 per-transaction metadata + 连接器缓存模型,逐条映射到 Doris 现状并指出 gap(遵循"连接器 SPI 决策前先读 Trino") | +| [`appendix-C-funnel-universality.md`](./appendix-C-funnel-universality.md) | 结构性论断核验:funnel 是否已覆盖每个连接器 + 每个连接器的 ConnectorMetadata 是否 memoize 已加载表(per-connector 表) | +| [`appendix-D-critic-punchlist.md`](./appendix-D-critic-punchlist.md) | 对主报告的矛盾/完整性对抗复核清单(5 条 minor 已在主报告修正;附通过项) | +| [`connectors/*.md`](./connectors/) | **7 份 per-connector 审计附录**(hive / hudi / paimon / maxcompute / jdbc / es / trino),各含:迁移状态、现有缓存表、funnel 参与、热路径重复加载审计表、授权/一致性、所需框架件、优先级,**末尾附对抗式复核结论(verdict + 更正表)** | +| [`data/connector-audits.json`](./data/connector-audits.json) | 7 连接器结构化审计数据(缓存清单 / hotPathFindings / 复核 verdict),矩阵与路线图的原始来源 | + +--- + +## 🔬 本调研如何产出(方法与可信度) + +- **19-agent 编排研究**(clean-room 对抗式):3 个框架 agent(taxonomy + Trino 参考 + funnel 通用性核验)‖ 7 个连接器 × (审计 → **独立对抗验证**)‖ 综合 → 矛盾/完整性 critic。共 ~2.4M token,0 error。 +- **全部论断挂 HEAD `file:line`**,agent 被要求"行号信 HEAD 不信文档/commit message";7 连接器结论各经一个**默认怀疑**的验证 agent 复核:**5 CONFIRMED / 2 ADJUSTED**(hudi、es;更正已折进主报告与对应附录)。 +- **⚠ 这是纯调研,未改任何产品代码,未跑 e2e。** `file:line` 为调研时(2026-07-22 HEAD)快照,落地实现时以 `grep` 为准。所有"倍数/成本"是审计估算,建议在真正立项时用调用计数守门(对齐 `perf-hotpath-iceberg` 的度量门做法)复测。 + +--- + +## 🧭 需 owner 拍板的 4 个决策(详见主报告 §8) + +1. **范围**:本轮只打 hudi(唯一 P1 真缺口),还是把 maxcompute `MC-1`、es `ES-F1/F2` 两个小修一并纳入?(建议:hudi 单独立项 + mc/es 各一小 PR,余进 backlog) +2. **排序**:先建共享底座泛化,还是 per-connector-first?(建议 per-connector-first——泛型 table/handle 缓存目前只有 hudi 一个新消费者,出现第 2 个再上收,避免投机抽象 / Rule 2) +3. **`check-authz-cache-sharding.sh` 是否现在就通用化?**(现硬编码只盯 `IcebergConnector.java`;今天所有推荐新缓存都 authz-安全,故可延后,但前瞻通用化更稳) +4. **是否投入共享的"经 statement scope 解析表"helper?**(Trino 眼中"统一"的真正含义,但复核显示只有 iceberg+hudi 刚需;建议让 hudi 在自己模块内做,不提前提炼) + +--- + +## 🔗 关系 + +- **参考实现**:`plan-doc/perf-hotpath-iceberg/`(iceberg 那批被删缓存的逐条修复)+ `plan-doc/perf-heavy-op-hot-path-problem-class.md`(本调研套用的"热路径重操作放大"问题类)。 +- **建议后续任务空间**(见主报告 §9):`plan-doc/perf-hotpath-hudi/`(P1 旗舰)、`perf-hotpath-maxcompute/`、`perf-hotpath-es/`(均小)、`cleanup-stale-connector-docs/`(doc-only)、`perf-hotpath-backlog-p2/`。各连接器建**各自兄弟空间**,不并进本空间。 diff --git a/plan-doc/connector-cache-unification/appendix-A-framework-taxonomy.md b/plan-doc/connector-cache-unification/appendix-A-framework-taxonomy.md new file mode 100644 index 00000000000000..341b7950d2e385 --- /dev/null +++ b/plan-doc/connector-cache-unification/appendix-A-framework-taxonomy.md @@ -0,0 +1,128 @@ +I have all the grounding I need. Here is the taxonomy. + +--- + +# Taxonomy of the "framework" introduced by `0b4f72582e7`, at HEAD (`branch-catalog-spi`) + +Every file:line below is verified against HEAD, not against the commit or plan-docs. + +Scope note: the reference commit has three layers. Grounding at HEAD confirms the commit's own framing but sharpens one point — **Layer 2 (the funnel) is 100% connector-agnostic and already live for all 7 `SPI_READY_TYPES`; Layer 1 (the 6 caches) and one of the two gates are iceberg-only *implementations of a replicable pattern*, and the substrate to generalize them (`fe-connector-cache`) already exists and is already partly adopted.** + +--- + +## 1. Layer 2 — the connector-agnostic per-statement `ConnectorMetadata` funnel + +**What it is.** A per-statement memoization arena (the Doris analog of Trino's per-transaction `CatalogMetadata`) that guarantees *one `ConnectorMetadata` instance per (statement, catalog)*, so every read / scan / DDL / MVCC / write resolver in a statement shares one metadata — and, for a connector that memoizes its loaded table on that metadata, one loaded remote table. + +Pieces, at HEAD: + +- **SPI seam (neutral):** `fe-connector-api/.../connector/api/ConnectorStatementScope.java`. Interface with `computeIfAbsent` (line 45), typed `getOrCreateMetadata` default (lines 55-57), best-effort idempotent `closeAll` default no-op (lines 66-67), and the `NONE` no-op scope that memoizes nothing (lines 70-75). Reached from a connector purely via `ConnectorSession.getStatementScope()`, whose SPI default is `NONE` (`ConnectorSession.java:142-144`) — so any connector that does nothing still compiles and behaves byte-identically to "load every time." +- **The funnel itself:** `fe-core/.../datasource/plugin/PluginDrivenMetadata.get(session, connector)` (lines 53-71). Keyed `"metadata:" + catalogId` (line 70); builds via `connector.getMetadata(session)` at most once per statement. It also pins the *building identity* under `"metadata-identity:" + catalogId` and throws `IllegalStateException` if a second Doris principal reuses the instance (lines 63-69) — turning a would-be cross-user credential reuse into a hard error. This is the **only** file in fe-core allowed to call `Connector#getMetadata` (enforced by gate §5). +- **The arena impl:** `fe-core/.../connector/ConnectorStatementScopeImpl.java`. A `ConcurrentHashMap` (line 44) so off-thread scan pumps that reuse the one session reach the same scope; `computeIfAbsent` (lines 49-51); **two-tier `closeAll`** (lines 64-98, guarded by `closed` for idempotency): pass 1 finalizes any `CatalogStatementTransaction` (rolls back an uncommitted txn), pass 2 closes every remaining `AutoCloseable` value (the memoized metadata) — txn finalized *before* the metadata it was minted from is closed. +- **The physical home:** `fe-core/.../nereids/StatementContext.java`. Field `connectorStatementScope` (line 209); lazily built by `getOrCreateConnectorStatementScope()` (lines 656-661); dropped/closed per prepared-statement execution by `resetConnectorStatementScope()` (lines 672-677); **fallback close** in `close()` for statements that never reach a coordinator (external DDL/SHOW/DESCRIBE/EXPLAIN/foreground ANALYZE), guarded by `isReturnResultFromLocal` (lines 995-997). +- **Write-transaction co-holder:** `fe-core/.../datasource/plugin/CatalogStatementTransaction.java`. Minted from the *shared* metadata's write facet via `begin()` (lines 80-84); `finalizeAtStatementEnd()` is the deterministic backstop (lines 100-107). Connector-agnostic — traffics only in neutral `ConnectorWriteOps`/`ConnectorSession`/`ConnectorTransaction` SPI types. + +**Lifecycle** (open → memoize → two-tier close): +1. **Open:** `ConnectorSessionBuilder.captureStatementScope()` (`ConnectorSessionBuilder.java:190-203`) reads the scope off the live `ConnectContext`'s `StatementContext` at session-build time (request thread). **This is *not* gated by connector type** — there is no `iceberg`/type branch; any session built from a live statement gets the real scope, else `NONE`. +2. **Memoize:** every resolver calls `PluginDrivenMetadata.get(...)`. Verified breadth at HEAD: **58 call sites across 15 fe-core files** (`PluginDrivenExternalCatalog/Table/Database`, `PluginDrivenMvccExternalTable`, `PluginDrivenScanNode`, `PluginDrivenInsertExecutor`, `BindSink`, `PhysicalPlanTranslator`, `PhysicalExternalRowLevelMergeSink`, `ShowPartitionsCommand`, `IcebergRowLevelDmlTransform`, `ConnectorExecuteAction`, `CallExecuteStmtFunc`, `MetadataGenerator`, `JdbcQueryTableValueFunction`). +3. **Close (two-tier):** primary is the getSplits query-finish callback — `PluginDrivenScanNode` registers `statementScope::closeAll` via `QeProcessorImpl.registerQueryFinishCallback`, skipping `NONE` (lines 1235-1237); fallback is `StatementContext.close()` (line 995). Both idempotent. + +**Cross-statement background loaders** are deliberately forced through a `NONE` scope (a read-through, not a hazard): `PluginDrivenExternalCatalog` builds their sessions `.withStatementScope(ConnectorStatementScope.NONE)` (lines 1161, 1168), so a synchronous background `fetchRowCount` cannot capture the live statement's scope. + +**Two connector-agnostic memos ride on top** (both byte-identical to re-resolving): +- `PluginDrivenScanNode.resolveScanProvider()` memoizes the `(handle, provider)` pair keyed on `currentHandle` *identity* via the immutable `ResolvedScanProvider` holder (lines 245-275), so the per-split `getFileCompressType`/`getDeleteFiles`/`classifyColumn` hot path stops re-allocating + TCCL-swapping a provider per split. This is the "generic-node provider memo (C14)." +- `PluginDrivenScanNode.metadata()` caches this node's funnel result in a volatile field (lines 189, 203-210) so per-method resolvers don't re-hit the scope map. + +**Heterogeneous-gateway extension (still Layer-2 mechanics, lives in the hive connector):** `HiveConnectorMetadata.memoizedSiblingMetadata()` (lines 302-304) memoizes the iceberg/hudi-on-HMS **sibling** metadata on *the same per-statement scope*, keyed `"metadata:" + catalogId + ":" + ownerLabel` (the three gateway connectors share one catalogId, so the owner label keeps them distinct — lines 293-295). It uses only `fe-connector-api` types (no fe-core dep). + +**Does Layer 2 already apply to ALL of `SPI_READY_TYPES`?** **Yes, universally, with no iceberg gate.** `SPI_READY_TYPES = {jdbc, es, trino-connector, max_compute, paimon, iceberg, hms}` (`CatalogFactory.java:56-57`); every one is a `PluginDrivenExternalCatalog` whose reads/scans go through `PluginDrivenExternalTable`/`PluginDrivenScanNode`, i.e. through the 58 funnel call sites; and `captureStatementScope()` has no type branch. hudi is not its own type — it is served only as a sibling under the hms gateway (`CatalogFactory.java:51-55`), and *that* path also rides the funnel via `memoizedSiblingMetadata`. + +**Caveat worth stating plainly:** the funnel guarantees *one metadata per statement*, but whether that collapses repeated remote loads is **per-connector** — it only helps if that connector's `ConnectorMetadata` (or its long-lived `Connector`) actually memoizes the loaded table/schema. Layer 2 is the seam; Layer 1 is what fills it for iceberg. jdbc/es/trino have **no** connector-side cross-query cache at all (verified: no `MetaCacheEntry`/`CacheSpec`/Caffeine anywhere in `fe-connector-{jdbc,es,trino-connector}/src/main/java`), so for them the funnel collapses only the *within-statement* repeats, not cross-query. + +--- + +## 2. Layer 1 — the six iceberg connector-side caches (a replicable *pattern*, catalogued) + +All six live in `fe-connector-iceberg/.../connector/iceberg/`, are package-private `final`, and are **all built on the shared `MetaCacheEntry` substrate** (§3) with the identical construction shape `new MetaCacheEntry<>(name, null, spec, ForkJoinPool.commonPool(), false, true, 0L, true)` — contextual-only (caller supplies the loader per call), manual-miss-load (loader runs outside Caffeine's compute lock, single-flight per key, exception propagates unwrapped and un-cached). All are per-catalog, cross-query, and live on the long-lived `IcebergConnector` (a REFRESH/ADD/MODIFY CATALOG rebuilds the connector → drops the caches). + +| # | Class (file) | Heavy op memoized | Key | Value | TTL | Invalidation | Scope | Authz gate (see §4) | Format-specific? | +|---|---|---|---|---|---|---|---|---|---| +| PERF-01 | `IcebergTableCache.java:59-117` | remote `loadTable` (metastore RPC + `metadata.json` read) | `TableIdentifier` (db.table) | **raw iceberg `Table`** | `meta.cache.iceberg.table.ttl-second`, default 24h; `≤0` disables | table / db(namespace) / all (lines 91-109) | cross-query **+** a per-statement "fat handle" (transient `resolvedTable`) | **null** under `session=user` **or** REST vended-creds (`IcebergConnector.java:243-247`) — value carries FileIO credentials | **Format-neutral in spirit** (every connector loads a table object); value type is iceberg-specific | +| PERF-02 | `IcebergPartitionCache.java:58-142` | the `PARTITIONS` metadata-table scan (SDK reads every data+delete manifest of the snapshot) | `(TableIdentifier, snapshotId)` | `List` | same 24h knob; `≤0` disables | table / db / all (lines 116-129) | cross-query; shared by MVCC view, `listPartitions`, `SHOW PARTITIONS`, and the 4–6 re-enumerations of one MTMV refresh | **null** under `session=user` (`:254`) — pure metadata but authz-sensitive | **Iceberg-format-specific** (PARTITIONS metadata table, transform partitions) | +| PERF-03 | `IcebergFormatCache.java:61-145` | #64134's unfiltered whole-table `table.newScan().planFiles()` format fallback | `(TableIdentifier, snapshotId)` | format-name `String` | same 24h knob; `≤0` disables | table / db / all (lines 118-131) | cross-query | **null** under `session=user` (`:260`) | **Format-neutral in spirit** (file-format inference); bare `String` value | +| PERF-04 | `IcebergManifestCache.java:62-235` | parsing a manifest's files (remote FileIO read) | `IcebergManifestEntryKey` (manifest path + content) | `ManifestCacheValue` (copied `DataFile`/`DeleteFile` list) | **no TTL**, capacity-bounded (`100_000`), **REFRESH CATALOG only** (lines 224-227) | `invalidateAll` only; REFRESH TABLE deliberately keeps it (immutable manifests) | cross-query + cross-table (any table referencing the manifest) | **exempt** — default-off (`meta.cache.iceberg.manifest.enable`) + read after a per-user load (`IcebergConnector.java` marker `authz-cache-exempt`) | **Genuinely iceberg-format-specific** (manifest files) | +| PERF-05 | `IcebergCommentCache.java:53-111` | per-table `loadTable` that `information_schema.tables` / `SHOW TABLE STATUS` pays N-per-query for the comment | `TableIdentifier` | comment `String` | same 24h knob; `≤0` disables | table / db / all (lines 85-98) | cross-query | **built ONLY** for REST vended-creds **and NOT** `session=user` (`IcebergConnector.java:269-273`); plain catalogs reuse PERF-01 instead | **Format-neutral in spirit** (table comment is universal metadata); bare `String` | +| (survivor) | `IcebergLatestSnapshotCache.java:54-124` | `currentSnapshot()` + latest-schema read | `TableIdentifier` | `CachedSnapshot(snapshotId, schemaId)` | same 24h knob; `≤0` disables | table / db / all (lines 98-115) | cross-query; read by `beginQuerySnapshot` | **null** under `session=user` (`IcebergConnector.java:231-234`) | **Iceberg-specific** (snapshot id **+** schema id atomic pin — the one deviation from paimon's `long`-only mirror) | + +Notes grounded at HEAD: +- The commit message says the P6 cutover "kept only the latest-snapshot pin" — so `IcebergLatestSnapshotCache` is the pre-existing survivor; PERF-01/02/03/05 restore the dropped halves and PERF-04 re-wires the opt-in manifest cache. At HEAD all six are the current family. +- Snapshot-keyed caches (PERF-02/03, latest-snapshot) are "always correct" because a snapshot is immutable and a new commit yields a new key; stability across queries comes from the latest-snapshot cache holding the snapshot id stable within the TTL. +- The **per-scan/per-file hoists** (PERF-06 storage-uri normalizer once per scan via `ConnectorContext.newStorageUriNormalizer`; PERF-11 per-file partition-JSON/values/delete-carrier memo) are part of the same *pattern* but are per-statement/per-scan hoists inside `IcebergScanPlanProvider`, not cross-query cache classes; the one fe-core piece of PERF-11 (the provider memo) is Layer-2 and already connector-agnostic (§1). + +**Verdict on Layer 1:** it is a *pattern* (contextual `MetaCacheEntry` + snapshot/name key + REFRESH invalidation + authz gate), not universal code. Two members (manifest, PARTITIONS-scan) and the snapshot+schema pin are intrinsically iceberg-format-specific; three (table handle, comment, file-format) are format-neutral in intent and differ only in value type. + +--- + +## 3. The pre-existing shared substrate `fe-connector-cache` — the natural generalization home + +Module `fe/fe-connector/fe-connector-cache/.../connector/cache/`: + +- **`CacheFactory`** — Caffeine factory; framework-internal (returns Caffeine types, which must never cross into child-first connector code). Connector-side copy of fe-core's `common.CacheFactory` (plugins can't import fe-core). +- **`CacheSpec`** — the `(enable, ttl-second, capacity)` config value; `fromProperties(props, engine, entry, default)` reads `meta.cache...*`; `CACHE_TTL_DISABLE_CACHE`(0)/`CACHE_NO_TTL`(-1) sentinels. This is exactly the knob every Layer-1 cache uses. +- **`MetaCacheEntry`** — the Caffeine-**free** unified cache-entry API: lazy/contextual loading, manual-miss-load (I/O outside Caffeine's lock, striped per-key dedup), key/predicate/full invalidation, and stats (`getLoadSuccessCount`, backing the `loadCountForTest` measurement gates). This is the shared engine under all six iceberg caches. +- **`MetaCacheEntryStats`** — the stats/`isEffectiveEnabled` view. +- **`ConnectorPartitionViewCache`** and **`PartitionViewCacheKey`** — an already-written **generic version of `IcebergPartitionCache`**: same contextual-only, manual-miss-load `MetaCacheEntry` shape, same `CacheSpec` wiring, but keyed by the engine-agnostic `PartitionViewCacheKey` with opaque value `V` and no engine imports. Its own javadoc states it "is the generic version of `IcebergPartitionCache`." + +**Who uses the substrate today** (verified imports/instantiations across `fe-connector/*/src/main/java`): +- `MetaCacheEntry`: **hive** (`HiveFileListingCache`), **hms** (`CachingHmsClient`), **iceberg** (all six above), **maxcompute** (`MaxComputePartitionCache`), **paimon** (`PaimonLatestSnapshotCache`). **Not** jdbc / es / trino-connector (none). +- `ConnectorPartitionViewCache` — actual `new …()` sites: **hive** `HiveConnector.java:136`, **iceberg** `IcebergConnector.java:281` & `:284` (mvcc view + list-partitions view), **paimon** `PaimonConnector.java:158`. + +**Is it the natural home to generalize Layer-1?** **Yes, and generalization is already underway.** The substrate exists, is Caffeine-free (safe across the loader split), reads a uniform `meta.cache...*` config namespace, and `ConnectorPartitionViewCache` is a *proof* that the exact iceberg pattern generalizes (it is `IcebergPartitionCache` with the key/value abstracted). The other five iceberg caches are structurally identical (all `new MetaCacheEntry<>(name, null, spec, commonPool, false, true, 0L, true)`), so `IcebergTableCache`/`IcebergCommentCache`/`IcebergFormatCache`/latest-snapshot could each collapse into a typed `ConnectorXViewCache` wrapper the same way. + +**One drift flag (Rule 7/12):** `ConnectorPartitionViewCache`'s class javadoc says "this class has **NO consumers yet**" — that is **stale at HEAD**: three connectors (hive, iceberg, paimon) already instantiate it. Worth a doc-only fix. + +--- + +## 4. Layer 3 — authorization cache isolation under `iceberg.rest.session=user` + +The hazard: name-only-keyed caches would let a user who can `LIST` but not `LOAD` a table receive metadata a *different* user's `loadTable` produced ("list ≠ load" disclosure), because per-user authorization lives *inside* the delegated `loadTable` round-trip. + +**iceberg-REST-specific mechanisms:** +- `IcebergConnector.isUserSessionEnabled()` (`:870-874`) = `rest.session == user` **and** flavor is REST. +- Under it, the connector **nulls** its live cross-query caches in the constructor: latest-snapshot (`:231`), table (`:243`, also nulled for REST vended-creds), partition (`:254`), format (`:260`), and both `ConnectorPartitionViewCache` instances (`:279`, `:282`). `IcebergCommentCache` is *only* built for REST-vended-and-not-session-user (`:269`). So under `session=user` every shared cache is off → `beginQuerySnapshot`/reads go live per-user. Each field carries the reviewed marker `authz-cache-session-user-disabled` or `authz-cache-exempt` (enforced by gate §5). + +**fe-core-general mechanisms** (apply to *any* `SUPPORTS_USER_SESSION` connector, not just iceberg — iceberg-REST is simply the only one that currently declares the capability): +- `ExternalCatalog.shouldBypassSchemaCache(SessionContext)` — base default `false` (`ExternalCatalog.java:367-369`); the generic name-keyed schema cache is bypassed live when true. +- `PluginDrivenExternalCatalog` overrides it as `supportsUserSession() && ctx != null && ctx.hasDelegatedCredential()` (`:1212-1214`), with sibling `shouldBypassTableNameCache` (`:1190-1192`) and `shouldBypassDbNameCache` (`:1200-1202`) closing the same disclosure at the db-name/table-name-cache level. Consumed by `ExternalTable.java:439`. + +So Layer-3 is split: the *policy predicate* (`shouldBypass*`, capability+credential gated) is connector-general fe-core; the *cache-nulling* is iceberg-connector-local because that's where the caches live. + +--- + +## 5. The two anti-drift build gates + +- **`tools/check-fecore-metadata-funnel.sh`** — **connector-GENERAL.** Fails the build on any executable `Connector#getMetadata()` in `fe/fe-core/src/main/java` outside the one legal carrier `datasource/plugin/PluginDrivenMetadata.java` (allow-list: the funnel file; a `getMetadata-funnel-exempt` marker; no-arg `getMetadata()`; comment lines). Invariant enforced: *one `ConnectorMetadata` per (statement, catalog) for every connector* — i.e. Layer 2. Its root is `fe/fe-core`, no iceberg specificity (lines 20-59, 67-105). +- **`tools/check-authz-cache-sharding.sh`** — **iceberg-ONLY.** `TARGET_REL` is hard-coded to `fe-connector-iceberg/.../IcebergConnector.java` (line 57); it requires every `final …Cache[ <]` holder field there to carry `authz-cache-session-user-disabled` or `authz-cache-exempt` (lines 65-96). Invariant enforced: *no new shared cross-query cache on IcebergConnector without a declared `session=user` isolation discipline* — i.e. Layer 3, for iceberg. It explicitly notes the fe-core schema cache is a *different* cache protected separately by `shouldBypassSchemaCache` (lines 37-39). If a second connector ever declares `SUPPORTS_USER_SESSION`, this gate would need a new target — it does not generalize as written. + +--- + +## 6. Bottom line + +| Framework piece | Already universal? | +|---|---| +| Per-statement `ConnectorMetadata` funnel (`PluginDrivenMetadata`, `ConnectorStatementScope`+`Impl`, `StatementContext` home, two-tier close) | **Yes** — connector-agnostic; live for all 7 `SPI_READY_TYPES` via 58 fe-core call sites; scope capture has no type gate | +| `NONE` no-op scope + background-loader read-through | **Yes** — SPI default; used to shield cross-statement loaders | +| `CatalogStatementTransaction` (one write txn per statement, ordered teardown) | **Yes** — fe-core-internal, neutral SPI types only | +| `PluginDrivenScanNode` provider memo + per-node metadata cache (PERF-11 C14) | **Yes** — keyed on handle identity, no connector branch | +| HMS-gateway sibling metadata memo (`memoizedSiblingMetadata`) | **Yes as mechanism (rides the universal scope)**, but **lives in the hive connector** and is keyed for the iceberg/hudi-on-HMS sibling case | +| `fe-connector-cache` substrate (`CacheFactory`/`CacheSpec`/`MetaCacheEntry`/`Stats`) | **Yes** — shared; adopted by hive, hms, iceberg, maxcompute, paimon (not jdbc/es/trino) | +| `ConnectorPartitionViewCache` / `PartitionViewCacheKey` (generic partition-view cache) | **Yes (universal by design)** — already instantiated by hive, iceberg, paimon; javadoc "no consumers yet" is stale | +| The 6 iceberg caches — **table handle, comment, file-format** | **iceberg-only pattern**, format-neutral in spirit → the natural next candidates to fold into typed `ConnectorXViewCache` wrappers on the substrate | +| The 6 iceberg caches — **PARTITIONS-scan, manifest, latest-snapshot(+schemaId)** | **iceberg-only pattern**, genuinely iceberg-format-specific | +| Per-scan/per-file hoists (PERF-06 storage-uri normalizer, PERF-11 per-file invariant memo) | **iceberg-only** (inside `IcebergScanPlanProvider`); the SPI seam `ConnectorContext.newStorageUriNormalizer` is neutral but only iceberg consumes it | +| Layer-3 policy predicate `shouldBypass{Schema,TableName,DbName}Cache` (capability+credential gated) | **Partial → connector-general in fe-core**, but only iceberg-REST currently declares `SUPPORTS_USER_SESSION`, so only iceberg exercises it | +| Layer-3 cache-nulling under `isUserSessionEnabled()` | **iceberg-only** (the caches being nulled are iceberg-connector-local) | +| Gate `check-fecore-metadata-funnel.sh` | **Yes — connector-general** (enforces Layer 2 across all of fe-core) | +| Gate `check-authz-cache-sharding.sh` | **iceberg-only** (hard-coded to `IcebergConnector.java`; would need extension for any second user-session connector) | + +**Net:** the *engine seam* (Layer 2 + its gate + the `fe-connector-cache` substrate + the generic partition-view cache + the general `shouldBypass*` predicate) is already universal and load-bearing for every plugin connector. What is iceberg-specific is the *filling*: the six concrete caches (three format-specific, three format-neutral-but-not-yet-generalized), the per-scan hoists, the cache-nulling, and the second gate. The substrate to universalize the format-neutral half already exists and is already adopted by three connectors. \ No newline at end of file diff --git a/plan-doc/connector-cache-unification/appendix-B-trino-reference.md b/plan-doc/connector-cache-unification/appendix-B-trino-reference.md new file mode 100644 index 00000000000000..3c736e0ec603e1 --- /dev/null +++ b/plan-doc/connector-cache-unification/appendix-B-trino-reference.md @@ -0,0 +1,140 @@ +I have everything grounded. Returning the analysis. + +--- + +# Trino connector-metadata caching & per-transaction lifecycle — mapped onto the Doris catalog-SPI framework + +Scope note: every Doris claim below is hung on HEAD (`branch-catalog-spi`, reference commit `0b4f72582e7`) with `file:line`. Trino claims are grounded in the actual current source I fetched (`InMemoryTransactionManager`, `CatalogMetadata`, `CatalogTransaction`, `SharedHiveMetastoreCache`); where I rely on architectural knowledge rather than a fetched line I say so. + +--- + +## 1. Trino's per-transaction metadata lifecycle, and how faithful the Doris Layer-2 funnel is + +### 1a. What Trino actually does + +The lifecycle unit in Trino is the **transaction**, not the session and not the statement. Three layers cooperate: + +- **`InMemoryTransactionManager.TransactionMetadata`** holds `activeCatalogs`, a `ConcurrentHashMap`. `getTransactionCatalogMetadata(catalogHandle)` lazily does `catalog.beginTransaction(...)` on first touch of a catalog and caches the resulting `CatalogMetadata` for the transaction; `asyncCommit()`/`asyncAbort()` iterate `activeCatalogs.values()` and `commit`/`abort` each exactly once. (fetched from `InMemoryTransactionManager.java`.) +- **`CatalogMetadata`** wraps up to three `CatalogTransaction`s (the normal catalog, `information_schema`, system tables). `getMetadata(session)` returns the normal one; `getMetadataFor(session, handle)` routes by handle. (fetched from `CatalogMetadata.java`.) +- **`CatalogTransaction`** is where the one-metadata-per-transaction guarantee physically lives: + ``` + @GuardedBy("this") private ConnectorMetadata connectorMetadata; + synchronized ConnectorMetadata getConnectorMetadata(Session session) { + if (connectorMetadata == null) { + ConnectorSession cs = session.toConnectorSession(catalogHandle); + connectorMetadata = connector.getMetadata(cs, transactionHandle); + } + return connectorMetadata; + } + ``` + and `commit()`/`abort()` are gated by an `AtomicBoolean finished` so the connector transaction finishes once. (fetched from `CatalogTransaction.java`.) + +Two structural properties fall out of this and matter for Doris: + +1. **The `ConnectorMetadata` instance lives for the whole transaction and is the memo home.** Because Trino hands the connector one long-lived instance per transaction, connectors keep per-transaction working state *on that instance* (Iceberg's `IcebergMetadata` retains loaded tables/statistics; Hive's metadata holds a per-transaction metastore view). Repeated `resolveTable`/`getTableStatistics` within the transaction therefore collapse **for free** — the engine never needs a separate "load once per statement" seam. +2. **`getMetadata(session, transactionHandle)`** — the transaction handle is woven into the metadata at creation, so read and write are the same transaction by construction. `MetadataManager` (the engine's façade) routes every metadata call to `transactionManager.getCatalogMetadata(...).getMetadata(session)`, i.e. always through this cached instance. + +### 1b. What Doris does, and whether the emulation is sound + +Doris cannot use the session as the memo home, and the framework is explicit about why: + +- `ConnectorSessionImpl` is **immutable** and is **rebuilt** at each seam (`ConnectorSessionBuilder`); the PR cites ~26 rebuilds/statement. I confirmed the load-bearing fact — the session is immutable (`ConnectorSessionImpl.java:36`, all fields `final`) and constructed on demand rather than retained — so it *cannot* carry mutable per-statement memo. I did not independently count 26. +- Instead the memo hangs on the one **`StatementContext`**: `getOrCreateConnectorStatementScope()` (`StatementContext.java:656`) lazily builds a `ConnectorStatementScopeImpl` (`StatementContext.java:209` field). The scope is a `ConcurrentHashMap` (`ConnectorStatementScopeImpl.java:44`). +- Each session **captures the scope reference at build time** (`ConnectorSessionBuilder.captureStatementScope()` :190–203) so off-thread scan pumps that reuse one session still reach the same scope even without a `ConnectContext` thread-local — the direct analog of Trino handing the same `CatalogTransaction` to every operation on the transaction. +- The funnel is **`PluginDrivenMetadata.get(session, connector)`** (`PluginDrivenMetadata.java:53`), which memoizes `connector.getMetadata(session)` under key `"metadata:"+catalogId` on the scope (:70). This is the Doris analog of `CatalogTransaction.getConnectorMetadata`. It is the *only* place fe-core may call `Connector#getMetadata` — enforced build-wide by `tools/check-fecore-metadata-funnel.sh` (present, executable). I count **59** call sites in fe-core routed through it. + +**Assessment — is hanging the memo on `StatementContext` (not the session) sound?** Yes, with two caveats worth stating plainly: + +- **Sound mechanically.** The session is the wrong home precisely because it is immutable and rebuilt; the `StatementContext` is the one object that lives exactly as long as the statement and is reachable from every seam. The `ConcurrentHashMap` + `computeIfAbsent` gives every caller of a key the same instance (required because scan and write both mutate the shared table/delete-supply). Teardown is deterministic and idempotent: `closeAll()` (`ConnectorStatementScopeImpl.java:65`) runs two ordered passes (finalize any `CatalogStatementTransaction`, then close every `AutoCloseable` metadata), fired from the `getSplits` query-finish callback (`PluginDrivenScanNode.java:1237`) with a `StatementContext.resetConnectorStatementScope()` fallback (`StmtExecutor.java:1077`, `ExecuteCommand.java:95`). This mirrors Trino's `activeCatalogs.values()` commit/abort sweep + `AtomicBoolean finished`. +- **Caveat 1 — statement ≠ transaction.** Trino's unit is the whole `BEGIN…COMMIT` block; a multi-statement Trino transaction shares one `ConnectorMetadata` (and one connector transaction) across all its statements. Doris's unit is **one statement** (the scope key includes `queryId`, e.g. `IcebergStatementScope.java:64`), and a prepared-statement `EXECUTE` explicitly *drops* the scope per execution (`resetConnectorStatementScope`). For external-catalog reads and autocommit external DML (iceberg/hive) this is equivalent, because each such statement is its own transaction. But the read-consistency-within-a-transaction guarantee Trino gives across a multi-statement block is, in Doris, only a within-*statement* guarantee. Fine today; document it so nobody assumes cross-statement metadata sharing. +- **Caveat 2 — the metadata carries no transaction handle.** Trino's `getMetadata(session, transactionHandle)` bakes the transaction in; Doris's `getMetadata(session)` does not, and recovers write-transaction coupling with a *separate* co-holder (`CatalogStatementTransaction`, see §4) plus a **fail-loud identity pin** (`PluginDrivenMetadata.java:63–69`): the memo records the building user and throws if a second identity reuses the instance. Trino never needs that check because a transaction is single-identity by construction; Doris adds it because one shared singleton connector serves all users and a `session=user` metadata bakes in a delegated credential. This is a reasonable adaptation, not a defect. + +**Net:** the emulation is faithful in shape (lazy one-metadata-per-(statement,catalog), deterministic teardown, single routing funnel, arch-gate-enforced) and the choice to hang it on `StatementContext` is the correct — indeed the only — option given Doris's immutable-rebuilt session. + +--- + +## 2. Cross-transaction connector caches: the long-lived-shared vs per-transaction-view split + +### 2a. Trino + +Trino keeps a **long-lived shared cache** separate from a **per-transaction view**, per connector: + +- **Metastore.** `CachingHiveMetastore` is the shared, TTL-bounded cache (`getTable`, `getPartition(s)`, `getTableStatistics`/`getPartitionStatistics`, `listPartitionNames`). `SharedHiveMetastoreCache.ImpersonationCachingHiveMetastoreFactory` holds a `LoadingCache` **keyed by identity user** (`cache.getUnchecked(identity.getUser())`), configured `expireAfterWrite(userCacheTtl)` + `maximumSize(userCacheMaximumSize)` — i.e. under impersonation the cache is **sharded per identity**, not shared across users (fetched from `SharedHiveMetastoreCache.java`; corroborated by PR #9482 "add impersonation to SharedHiveMetastoreCache and remove per-user metadata caching from CachingHiveMetastore"). Over that shared cache Trino layers a **per-transaction `CachingHiveMetastore`** (the `createPerTransactionCache`/memoize wrapper) so that within one transaction repeated reads are consistent and never re-hit the metastore. +- **Directory listing** is a *separate* cache layer: `CachingDirectoryLister` (shared) with `TransactionScopeCachingDirectoryLister` giving the per-transaction consistent view — deliberately not folded into the metastore cache. +- **Iceberg** caches the resolved `Table` in the `TrinoCatalog` implementations (a Caffeine `Cache` gated by the `iceberg.metadata-cache` / expiration config), and `IcebergMetadata` additionally holds per-transaction working state on the instance. +- **Invalidation** is TTL + explicit: `CachingHiveMetastore.invalidateTable(...)` is called around writes so a write *within the transaction* invalidates the relevant entries; the per-transaction wrapper prevents a stale read of the just-written table. + +### 2b. Doris equivalents at HEAD + +Doris re-homed exactly this shape inside the connectors (the SPI cutover had left plain-hive caching nothing once a catalog became plugin-driven): + +| Trino | Doris (HEAD) | +|---|---| +| `CachingHiveMetastore` (metastore RPCs) | `CachingHmsClient` — decorates `HmsClient`, caches `getTable`/`listPartitionNames`/`getPartitions`/`getTableColumnStatistics`, each on its own `MetaCacheEntry`, keyed to Trino/legacy shape; class doc explicitly cites "Trino `CachingHiveMetastore` shape" (`fe-connector-hms/.../CachingHmsClient.java`) | +| `CachingDirectoryLister` / `TransactionScopeCachingDirectoryLister` | `HiveFileListingCache` — memoizes `FileSystem.listStatus` per `(db,table,location)` on a `MetaCacheEntry`; doc cites "Trino keeps the equivalent `CachingDirectoryLister` as a layer separate from its metastore cache" (`fe-connector-hive/.../HiveFileListingCache.java`) | +| Iceberg `TrinoCatalog` table cache + metadata memo | Two-level: cross-query `IcebergTableCache` (`getOrLoad`, `IcebergTableCache.java:86`) **plus** per-statement `IcebergStatementScope.sharedTable` (`IcebergStatementScope.java:59`), reached from `IcebergScanPlanProvider.resolveTable` (:2353 → `loadRawTable` :2378) | +| Iceberg partition/format/comment/manifest/snapshot metadata caches | `IcebergPartitionCache`, `IcebergFormatCache`, `IcebergCommentCache`, `IcebergManifestCache`, `IcebergLatestSnapshotCache`, plus the generic derived `ConnectorPartitionViewCache` (`IcebergConnector.java:169–203`) | + +**How Doris splits shared vs per-statement:** the cross-query caches live as `final` fields on the long-lived per-catalog connector (rebuilt only by `REFRESH CATALOG`), and the per-statement view is the `StatementContext` scope. For iceberg this is a genuine two-level split (`sharedTable` over `IcebergTableCache`) that matches Trino's per-transaction-view-over-shared-cache. For hive, `CachingHmsClient`/`HiveFileListingCache` are shared-only; the per-statement view is only the single `ConnectorMetadata` the funnel memoizes. + +**Invalidation** is coarser than Trino by design: Doris uses TTL + `REFRESH TABLE/DATABASE/CATALOG` flush (`IcebergTableCache.invalidate/invalidateDb/invalidateAll`; `CachingHmsClient.flush/flushDb/flushAll`). The `CachingHmsClient` doc states it **does not self-invalidate around writes** ("coarse REFRESH + TTL bound staleness"). Trino *does* invalidate within the transaction around a write. **Gap:** a Doris-side write leaves a TTL-bounded stale window for a subsequent read of the same table (no read-your-write within the coarse cache). Recommendation: either add write-path invalidation to `CachingHmsClient` or force the post-write read through a live (scope-`NONE` / bypass) read, matching Trino's per-transaction invalidation. + +**Authorization.** This is the sharpest divergence. Trino, under impersonation, **still caches but shards the cache key by identity** (`LoadingCache`). Doris, under `iceberg.rest.session=user`, **disables the caches entirely** — `latestSnapshotCache`/`tableCache`/`partitionCache`/`formatCache`/`commentCache` are set `null` under `isUserSessionEnabled()` (`IcebergConnector.java:231–273`), and fe-core adds `shouldBypassSchemaCache(SessionContext)` so schema is re-read live per user. This is enforced by `tools/check-authz-cache-sharding.sh` (every cache field must carry `authz-cache-session-user-disabled` or `authz-cache-exempt`). It is *safe* (it closes the "can-LIST-cannot-LOAD" metadata-disclosure the name-only keys would leak) but it is **coarser and slower** than Trino: `session=user` catalogs get no cross-query metadata cache at all. Recommendation (future): follow Trino and **key the caches by identity** rather than disabling them, restoring caching under `session=user`. That is a larger change (every key gains a user dimension) and the current "disable" is the correct conservative first cut. + +--- + +## 3. The unification question — does Trino have one generic connector cache, or per-connector caches on a shared toolkit? + +**Trino has no single unified connector cache.** It has a shared **low-level toolkit** — `io.trino.cache` (`EvictableCacheBuilder`, `SafeCaches`, `NonEvictableCache`/`NonKeyEvictableLoadingCache`, `CacheStatsMBean`) over Guava/Caffeine — and each connector builds *its own* caches on top: `CachingHiveMetastore`, `CachingJdbcClient`, the Iceberg catalog `tableCache`, `CachingDirectoryLister`. The reused unit is the *builder + eviction-safety + stats* machinery, not a cache and not a cache key model. This is deliberate: the cached objects and their key/invalidation/authorization semantics differ irreducibly per connector (an iceberg `Table` is not an `HmsTableInfo` is not a paimon `Table`). + +**Doris is already on this Trino-correct path.** `fe-connector-cache` is exactly the shared toolkit: `MetaCacheEntry` (Caffeine encapsulated behind a Caffeine-free API, striped single-flight locks, manual-miss-load, stats — `MetaCacheEntry.java`), `CacheSpec`, `CacheFactory`, and the one genuinely generic reusable *cache* `ConnectorPartitionViewCache`/`PartitionViewCacheKey`. Adopters confirmed by import: **hive, hms, iceberg, maxcompute, paimon** all build on `MetaCacheEntry`/`CacheSpec`/`ConnectorPartitionViewCache`; the only direct-Caffeine use in a connector is `org/apache/iceberg/DeleteFileIndex.java`, which is vendored iceberg code, not a Doris cache. + +**Lesson for the "unify the connector cache framework" goal:** + +1. **Do not build a single connector-agnostic metadata cache.** Trino proves the right seam is the toolkit, because the cached value types and their invalidation/authz rules are per-connector. Doris already has that toolkit (`MetaCacheEntry`) and it is already broadly adopted. Unifying *harder* than Trino (one cache to rule them all) would fight the same semantics Trino refused to fight. +2. **The real unification opportunity Doris is missing is not the caches — it is the per-statement dedup.** Trino gets "load each table once per transaction" *for free* because the `ConnectorMetadata` instance is the transaction-lived memo home (§1a-1). Doris's funnel guarantees one `ConnectorMetadata` per statement, but whether repeated `resolveTable` within that statement collapses is **per-connector and not uniform** — see §5. Iceberg routes table loads through the statement scope (`IcebergStatementScope.sharedTable`); the others do not. The highest-value "unification" is a shared **"resolve table via the statement scope"** helper so every connector inherits one-load-per-statement, closing the Variant-B gap uniformly instead of connector-by-connector. +3. **Converge the last bespoke caches onto `MetaCacheEntry`** — notably paimon still lacks a cross-query raw-`Table` cache (§5), and jdbc/es keep their own structures. + +--- + +## 4. Write-path consistency — read + write sharing one metadata / one transaction + +**Trino.** Read and write share the identical `ConnectorMetadata` for the transaction (the single `CatalogTransaction.connectorMetadata`): `beginInsert`/`finishInsert`/`beginMerge`/`finishMerge` are invoked on the same instance the reads used, and the connector transaction is finished by `CatalogTransaction.commit()`/`abort()` on the same `transactionHandle`. The write therefore sees exactly the reads' snapshot; there is one handle, one metadata, one commit. + +**Doris (HEAD).** The framework copies this "one-metadata-one-write-transaction-per-statement" model deliberately: + +- The **8 write-path `getMetadata` seams are routed through the same `PluginDrivenMetadata` funnel**, so read and write share the one memoized `ConnectorMetadata`, guarded by the fail-loud identity pin (`PluginDrivenMetadata.java:63–69`) — a `session=user` metadata baking in one user's delegated credential can never be silently reused to execute another user's write. +- The write transaction is hoisted into **`CatalogStatementTransaction`** (`CatalogStatementTransaction.java`), co-held on the scope next to the shared metadata. `begin(writeHandle)` mints the connector transaction from the **shared metadata's own write facet** (`writeOps.beginTransaction(session, writeHandle)`, :81) and registers it in `PluginDrivenTransactionManager`, so the write inherits exactly the read arm's client/ops — the class doc explicitly names this "mirroring Trino's `CatalogTransaction`: one metadata instance and one transaction per (statement, catalog)." +- Teardown is the deterministic two-pass in `closeAll()`: pass 1 `finalizeAtStatementEnd()` rolls back a transaction the executor never committed (only a mid-flight abort leaves one active), pass 2 closes the shared metadata — transaction is always finished **before** the instance it was minted from is closed (`ConnectorStatementScopeImpl.java:70–96`, `CatalogStatementTransaction.java:100`). `finalizeAtStatementEnd` is a no-op on every normal path (the executor already finished the txn, so the manager no longer holds it), and can never undo a committed write. + +**Faithfulness / gaps:** +- Faithful in the essential: read and write provably share one metadata and one connector transaction, torn down in the right order, with a fail-loud cross-identity guard Trino gets structurally. +- **Narrower scope than Trino:** the co-holder is per-*statement* (per `queryId` scope), so a *multi-statement* Doris transaction that mixes external writes and reads does not share one metadata/txn across statements the way a Trino `BEGIN…COMMIT` does. For Doris's autocommit external DML (iceberg/hive) this is equivalent; flag it if multi-statement external transactions are ever targeted. + +--- + +## 5. Summary table — Trino mechanism | Doris equivalent today | gap / recommendation + +| Trino mechanism | Doris equivalent at HEAD | Gap / recommendation | +|---|---|---| +| **Transaction is the lifecycle unit**; `TransactionManager.activeCatalogs: Map`, lazy `beginTransaction`, commit/abort sweep | **Statement** is the unit; `PluginDrivenMetadata.get` memoizes `ConnectorMetadata` per `(statement,catalog)` on `ConnectorStatementScopeImpl` (scope keyed incl. `queryId`); `closeAll` two-pass teardown | Sound. Narrower: no metadata sharing across a multi-statement `BEGIN…COMMIT`. Fine for autocommit external DML; **document the statement-not-transaction scope.** | +| `CatalogTransaction.connectorMetadata` (`@GuardedBy(this)`, lazy `connector.getMetadata(session, txnHandle)`) — **the instance is the per-txn memo home**, so repeated table/stat resolves collapse for free | Funnel gives **one `ConnectorMetadata` per statement**, but per-statement *table-load* dedup is only guaranteed where the connector routes through the scope | **Non-uniform.** Only iceberg uses the scope for table load (`IcebergStatementScope.sharedTable`). **Paimon** re-resolves via a transient `Table` on the handle (`PaimonTableResolver.resolve` → `handle.getPaimonTable()`; branch/withScanOptions handles carry `null` → live re-load) and has **no cross-query raw-`Table` cache** (only `PaimonLatestSnapshotCache` + `schemaAtMemo`). Recommend a **shared "resolve table via statement scope" helper** so every connector gets one-load-per-statement. | +| `getMetadata(session, ConnectorTransactionHandle)` — txn handle woven into the metadata | `getMetadata(session)` — no txn handle; write txn is a **separate** `CatalogStatementTransaction` co-holder + identity pin | Acceptable adaptation; co-holder + `PluginDrivenMetadata.java:63–69` identity pin recover the guarantee Trino gets structurally. Keep. | +| `CachingHiveMetastore` (shared, long-lived; getTable/partitions/stats) | `CachingHmsClient` (getTable/listPartitionNames/getPartitions/getTableColumnStatistics on `MetaCacheEntry`) | Faithful shape. **Gap:** no write-path self-invalidation ("coarse REFRESH+TTL"); Trino invalidates within the txn. **Add write-path invalidation or a bypass read for read-your-write.** | +| **Per-transaction** `CachingHiveMetastore` wrapper over the shared cache (read consistency within a txn) | Shared `CachingHmsClient`/`HiveFileListingCache` with **no per-statement scoped view** over them (only the single memoized metadata) | Minor: within one statement a re-list/re-get could observe concurrent change. Low priority (TTL bounds it); add a statement-scoped consistent view if strict intra-statement consistency is needed. | +| `SharedHiveMetastoreCache` shards the cache **per identity** under impersonation (`LoadingCache`, `expireAfterWrite`) | Under `iceberg.rest.session=user`, caches are **nulled** (`IcebergConnector.java:231–273`) + fe-core `shouldBypassSchemaCache` | Safe but coarse: `session=user` gets **no** cross-query cache. **Recommend key-by-identity sharding** (Trino model) to restore caching under `session=user`; current disable is the correct conservative first cut. | +| `CachingDirectoryLister` (+ `TransactionScopeCachingDirectoryLister`), **separate** from the metastore cache | `HiveFileListingCache` on `MetaCacheEntry`, separate layer from `CachingHmsClient` | Faithful (doc cites the Trino split). No per-transaction consistency view (see above). | +| Iceberg `TrinoCatalog` Caffeine `tableCache` + `IcebergMetadata` per-txn memo | **Two-level**: cross-query `IcebergTableCache` + per-statement `IcebergStatementScope.sharedTable` | Faithful, arguably more explicit than Trino. Good — this is the reference the other connectors should copy. | +| `io.trino.cache` toolkit (`EvictableCacheBuilder`/`SafeCaches`/`CacheStatsMBean`) reused by all connectors; **no single unified cache** | `fe-connector-cache`: `MetaCacheEntry`/`CacheSpec`/`CacheFactory`/`ConnectorPartitionViewCache`, reused by hive/hms/iceberg/maxcompute/paimon | **Already Trino-correct.** Unify at the *toolkit*, not one cache. Remaining work: converge paimon's raw-table load and jdbc/es onto `MetaCacheEntry`; unify **per-statement dedup**, not the caches. | +| Per-txn teardown: `activeCatalogs` commit/abort iterate; `CatalogTransaction.finished` (`AtomicBoolean`) | `closeAll()` two-pass (finalize `CatalogStatementTransaction`, then close metadata), idempotent, fired from query-finish callback + prepared-stmt reset | Faithful. Good. | +| Single identity per transaction (structural) → no cross-user metadata reuse possible | Shared singleton connector serves all users → **explicit** fail-loud identity pin in the funnel + `check-authz-cache-sharding.sh` | Doris must assert what Trino gets for free; the guards are the right shape. Keep enforced. | + +### Bottom line for the redesign + +The Doris framework is a faithful, deliberately-Trino-shaped emulation: the funnel = `CatalogTransaction`, `CatalogStatementTransaction` = the one-metadata-one-transaction write model, `CachingHmsClient`/`HiveFileListingCache`/`IcebergTableCache` = `CachingHiveMetastore`/`CachingDirectoryLister`/iceberg catalog cache, and `fe-connector-cache` = the `io.trino.cache` toolkit. The three places it diverges from Trino are all worth tracking: + +1. **Per-statement table dedup is non-uniform** — Trino gets it for free from the transaction-lived metadata instance; Doris has it only for iceberg (via the scope). **This is the single highest-value generalization** and the true meaning of "unify the connector cache framework": a shared statement-scope table-resolution helper, not a shared cache. Paimon is the concrete offender (transient-on-handle only, no cross-query table cache). +2. **Write-path cache invalidation** — Doris relies on coarse REFRESH+TTL where Trino invalidates within the transaction (read-your-write gap). +3. **`session=user` authorization** — Doris *disables* caches where Trino *shards by identity*; correct-but-slow, and the natural next step is Trino-style per-identity keying. + +Sources: [InMemoryTransactionManager](https://github.com/trinodb/trino/blob/master/core/trino-main/src/main/java/io/trino/transaction/InMemoryTransactionManager.java), [CatalogMetadata](https://github.com/trinodb/trino/blob/master/core/trino-main/src/main/java/io/trino/metadata/CatalogMetadata.java), [CatalogTransaction](https://github.com/trinodb/trino/blob/master/core/trino-main/src/main/java/io/trino/metadata/CatalogTransaction.java), [SharedHiveMetastoreCache](https://github.com/trinodb/trino/blob/master/lib/trino-metastore/src/main/java/io/trino/metastore/cache/SharedHiveMetastoreCache.java), [PR #9482 (impersonation in SharedHiveMetastoreCache)](https://github.com/trinodb/trino/pull/9482). \ No newline at end of file diff --git a/plan-doc/connector-cache-unification/appendix-C-funnel-universality.md b/plan-doc/connector-cache-unification/appendix-C-funnel-universality.md new file mode 100644 index 00000000000000..3dd487052a1f00 --- /dev/null +++ b/plan-doc/connector-cache-unification/appendix-C-funnel-universality.md @@ -0,0 +1,77 @@ +All evidence is grounded. Here is the report. + +--- + +# Verdict on the CLAIM + +**CLAIM:** "The per-statement `ConnectorMetadata` funnel (Layer 2) already routes EVERY plugin connector, so all connectors already share one metadata per statement — the only per-connector gap is whether each connector's own `ConnectorMetadata` memoizes its heavy loads." + +**Verdict: CONFIRMED on the routing half; DIRECTIONALLY CORRECT but imprecise on the "memoizes its heavy loads" half.** + +- **Half 1 (routing) — TRUE.** The funnel `PluginDrivenMetadata.get(ConnectorSession, Connector)` is fully connector-agnostic: it keys the memo on `catalogId` only and has **no connector-type branch** (`PluginDrivenMetadata.java:53-70`). Every fe-core seam — read, scan, DDL, MVCC, write/insert — acquires metadata through it, for all 8 SPI types (`jdbc, es, trino-connector, max_compute, paimon, iceberg, hms`, + hudi as an HMS-gateway sibling). Enforced build-wide by `tools/check-fecore-metadata-funnel.sh` (gate passes, `exit=0`; **0** remaining exempt markers). + +- **Half 2 (the gap) — needs correction.** It is true that the open per-connector question is memoization. But the CLAIM's phrase "each connector's *own* `ConnectorMetadata` memoizes" is **misleading: no connector stores the loaded table as a `ConnectorMetadata` instance field.** The load-collapsing lives either **one level up** (iceberg → the per-statement scope; hive/hudi/paimon → connector-owned cross-query caches) or **one level down** (paimon/maxcompute/trino → fat handle). Consequently, "one metadata per statement" translates into "one remote load per statement" **only for Iceberg** — the one connector that binds the memo to `session.getStatementScope()`. For every other connector the collapse happens at a *different granularity* (per-handle, or cross-query TTL), which does **not** guarantee one-load-per-statement. + +--- + +## 1. The funnel is connector-general and covers all seams + +`PluginDrivenMetadata.get` (`fe/fe-core/.../datasource/plugin/PluginDrivenMetadata.java:53-70`) memoizes `connector.getMetadata(session)` on the statement scope keyed by `"metadata:" + catalogId`; the only conditional is a fail-loud **identity** guard (`getUser` mismatch), not a connector-type branch. + +| Seam | Callsites through the funnel (file:line) | +|---|---| +| Read path | `PluginDrivenExternalTable.java:133,159,189,449,565,583,606,716,820,881,923,954,1025,1060,1126,1277` | +| Scan planning | `PluginDrivenScanNode.java:206,222` | +| DDL (catalog/table) | `PluginDrivenExternalCatalog.java:296,312,333,423,502,551,599,671,715,808…1019,1107,1113`; `ConnectorExecuteAction.java:129`; `CallExecuteStmtFunc.java:102` | +| MVCC | `PluginDrivenMvccExternalTable.java:148,363,782` | +| Write / insert | `PluginDrivenInsertExecutor.java:219`; `BindSink.java:674,713`; `PhysicalExternalRowLevelMergeSink.java:289`; `IcebergRowLevelDmlTransform.java:124` | +| Translation / TVF / SHOW | `PhysicalPlanTranslator.java:613,661`; `MetadataGenerator.java:1266`; `JdbcQueryTableValueFunction.java:53`; `ShowPartitionsCommand.java:286` | + +**Connector-type branch inside the funnel: NONE.** The only `switch(catalogType)` in the plugin classes (`PluginDrivenExternalTable.java:1211-1258`) is purely cosmetic — user-visible engine names for `SHOW TABLE STATUS`/`information_schema` — not metadata routing. The one heterogeneous case is the **HMS gateway**, which keys its *sibling* memo by owner label inside `fe-connector-hive` (`HiveConnectorMetadata.memoizedSiblingMetadata:302`), not in fe-core. + +## 2. What the gate enforces — connector-general + +`tools/check-fecore-metadata-funnel.sh` greps **any** `.getMetadata()` under `fe/fe-core/src/main/java` and fails the build unless it is (1) inside `PluginDrivenMetadata.java` itself, (2) marked `getMetadata-funnel-exempt`, (3) a no-arg `getMetadata()`, or (4) a comment. It is entirely connector-agnostic (matches the SPI call *form*, no connector names). Currently **0** exempt markers remain — the write arm is fully funneled — and the gate exits 0. + +## 3. Per-connector funnel-memoization table + +The funnel guarantees **one `ConnectorMetadata` instance per statement per catalog** for all connectors. Whether that collapses repeated remote loads to one-per-statement depends on where each connector puts its memo: + +| Connector | Funneled? | Does its ConnectorMetadata collapse repeated table loads to **one per statement**? | Mechanism & evidence (file:line) | +|---|---|---|---| +| **Iceberg** (ref) | Yes | **YES** — the only true per-statement collapse | `resolveTableForRead` routes through `IcebergStatementScope.sharedTable(session,…)` = `session.getStatementScope().computeIfAbsent(key, loader)` (`IcebergStatementScope.java:59-66`; `IcebergConnectorMetadata.java:644-650`). Plus cross-query `IcebergTableCache` (`:646-648`). One `loadTable` per statement even across distinct `getTableHandle` calls. | +| **Paimon** | Yes | **PARTIAL** — per-*handle*, not per-statement | Fat handle: `getTableHandle` loads + `handle.setPaimonTable(table)` (`PaimonConnectorMetadata.java:211,221`); `resolveTable` prefers `handle.getPaimonTable()` (`PaimonTableResolver.java:63-67`). No statement-scope memo → each *fresh* `getTableHandle` re-issues `catalogOps.getTable`; only paimon's own cross-query `CachingCatalog` (TTL `meta.cache.paimon.table.ttl-second`) dedups across handles. | +| **MaxCompute** | Yes | **PARTIAL** — per-*handle* | Fat handle carries the ODPS `Table`: `getTableHandle`→`getOdpsTable` into handle (`MaxComputeConnectorMetadata.java:124-129`); `getTableSchema` reads `mcHandle.getOdpsTable()` (`:136`). ODPS SDK lazily loads+caches schema on that `Table` object; no statement/instance memo across separate `getTableHandle` calls. | +| **Trino** | Yes | **PARTIAL** — per-*handle*, and re-opens a txn | `getTableHandle` eagerly bakes trino handle + column-handle/metadata maps into `TrinoTableHandle` (`TrinoConnectorDorisMetadata.java:166-179`). `getTableSchema` reuses the handle's `columnHandleMap` (`:200`) but **re-opens a fresh trino transaction and re-calls `getColumnMetadata` per column** (`:194-209`). Each `getTableHandle`/`getTableSchema` = a new `beginTransaction`. | +| **Hive** | Yes | **NO instance/statement memo** — relies on connector-owned cross-query cache | `getTableHandle` (`HiveConnectorMetadata.java:399`) and `getTableSchema` (`:493`) each call `hmsClient.getTable(db,table)`. Collapse lives in the cross-query `CachingHmsClient` decorator keyed by `(db,table)` (`CachingHmsClient.java:53`), owned by the long-lived connector, TTL `meta.cache.hive.*`. Cache-disabled ⇒ live thrift RPC per call; no per-statement guarantee. | +| **Hudi** | Yes (HMS sibling) | **NO** | HMS part cached via `CachingHmsClient` (`getTableHandle`→`hmsClient.getTable`, `HudiConnectorMetadata.java:207`), but the schema read **rebuilds `HoodieTableMetaClient` and `reloadActiveTimeline()` on every** `getSchemaFromMetaClient` call (`:797-806`). No memo of the metaClient/schema. | +| **Jdbc** | Yes | **NO** | `getTableHandle` is a lightweight existence check (`JdbcConnectorMetadata.java:102`). `getTableSchema`→`client.getJdbcColumnsInfo` runs a live JDBC `DatabaseMetaData` column query with **no cache** (`JdbcClient.getJdbcColumnsInfo`, `JdbcClient.java:395`). (Normally served by fe-core's cross-statement schema cache, not the metadata instance.) | +| **Es** | Yes | **NO** | `getTableSchema`→`restClient.getMapping(index)` per call (`EsConnectorMetadata.java:85`); `getColumnHandles` re-invokes `getTableSchema` (`:99`). No memo. | + +## 4. The "9 cross-statement background loaders forced through NONE-scope" — connector-general + +They route through `PluginDrivenExternalCatalog.buildCrossStatementSession()` (`:1153-1170`), which forces `ConnectorStatementScope.NONE`. Exactly **9** callers, all in connector-agnostic fe-core base classes (no connector-type branch): + +1. `PluginDrivenExternalTable.initSchema` (`:448`) — schema cache +2. `PluginDrivenExternalTable.getColumnStatistic` (`:1024`) — column-stat cache +3. `PluginDrivenExternalTable.getChunkSizes` (`:1059`) +4. `PluginDrivenExternalTable.fetchRowCount` (`:1125`) — row-count cache (the fix for `ANALYZE`'s synchronous `fetchRowCount` capturing a live statement scope) +5. `PluginDrivenExternalCatalog.listDatabaseNames` (`:295`) — db-name cache +6. `PluginDrivenExternalCatalog.listTableNamesFromRemote` (`:311`) — table-name cache +7. `PluginDrivenExternalCatalog.fromRemoteDatabaseName` (`:1106`) +8. `PluginDrivenExternalCatalog.fromRemoteTableName` (`:1112`) +9. `MetadataGenerator.dealPluginDrivenCatalog` (`:1265`) — the BE-driven metadata TVF + +Under `NONE` the funnel memoizes nothing (`PluginDrivenMetadata.java:62`; `ConnectorSessionBuilder.captureStatementScope:196,200`), so these fill caches that outlive any statement without ever binding into (or being closed with) a live statement's scope. **Connector-general, not iceberg-specific.** + +--- + +## Surprises / corrections to the CLAIM + +1. **No connector memoizes on the `ConnectorMetadata` instance itself.** The CLAIM's mental model ("each connector's own ConnectorMetadata memoizes its heavy loads") does not match the code: the memo is always one level up (iceberg statement scope; hive/hudi/paimon connector-level cross-query caches) or one level down (paimon/maxcompute/trino fat handle). "One metadata per statement" is therefore **necessary but not sufficient** — a shared metadata instance with no memo still re-loads. + +2. **Only Iceberg realizes the funnel's promise (one load per statement).** It is the sole connector that ties the collapse to `session.getStatementScope()`. Every other connector's "collapse" is either per-handle (breaks across distinct `getTableHandle` calls from different fe-core entrypoints — e.g. `initSchema`, `listPartitions`, `getColumnStatistics`, scan planning each re-resolve a handle) or cross-query TTL (breaks when the connector's cache is disabled). So the per-connector gap is not merely "add a cache" — for parity with iceberg they would need a **statement-scoped** memo, not just any memo. + +3. **Trino is internally inconsistent:** it bakes a full `columnMetadataMap` into the fat handle at `getTableHandle` (`:178`) yet `getTableSchema` ignores it and re-derives column metadata through a fresh transaction (`:206-209`) — a redundant re-fetch even when the handle already carries the answer. + +4. **Hudi is the weakest:** even within one handle it rebuilds the `HoodieTableMetaClient` and force-reloads the timeline on every schema read (`:800-806`) — a Problem-Class-A loop-amplified heavy op with no hoist, the closest analog to the pre-cutover iceberg regressions this framework was built to fix. \ No newline at end of file diff --git a/plan-doc/connector-cache-unification/appendix-D-critic-punchlist.md b/plan-doc/connector-cache-unification/appendix-D-critic-punchlist.md new file mode 100644 index 00000000000000..30fd0c1f0078a4 --- /dev/null +++ b/plan-doc/connector-cache-unification/appendix-D-critic-punchlist.md @@ -0,0 +1,20 @@ +## Overall: no major issues. Report is well-aligned with the evidence and has correctly folded in all three ADJUSTED corrections. A few minor edits below. + +### Punch-list + +**[low-med] §5.2 -> internal contradiction:** "全部 8 个写路径 getMetadata 接缝都经同一 funnel,**读写共享同一 memoized ConnectorMetadata**" is stated universally, but §3/HP-2 says jdbc's `buildInsertSql` news up a non-funneled `JdbcConnectorMetadata` — so for jdbc, read and write do NOT share one memoized metadata. -> Scope the sentence to the write-*transaction* connectors (hive/maxcompute/iceberg); add a clause that jdbc `buildInsertSql` is a connector-internal, non-funneled exception (P2, HP-2), correctness-neutral because jdbc write is BE auto-commit / `NoOpConnectorTransaction`. + +**[low] §1 -> phrasing vs audit severities:** "唯一接近 P0/P1 的真缺口是 hudi" reads as if mc/es aren't P1, but audits mark MC-1 and ES-F1/F2 as **P1**. -> Soften to "唯一 loop-amplified(iceberg-式)P1 缺口是 hudi;maxcompute/es 为 constant-factor P1",消除与 §5.1/§9 的口径分歧. + +**[low] §2 -> unverifiable attribution:** "fe-core 中经 funnel 的 call sites 约 58–59 处(INPUT A/C 计 58,INPUT B 计 59)" — INPUT C lists the seams but states no aggregate of 58. -> Drop the per-input attribution: "约 58–59(计法差异,不影响结论)". + +**[low] §2 -> consumer count only partly grounded:** "ConnectorPartitionViewCache 被 hive/iceberg/paimon 用 … 已有 3 个消费者". INPUT D confirms only **hive + paimon**; iceberg's use of the *generic* class is from INPUT A (not in the per-connector JSON). The javadoc-stale ("no consumers yet") claim still holds (≥2 confirmed). -> Attribute iceberg to INPUT A, or state "≥2 confirmed consumers (hive, paimon)". + +**[low] §9 cleanup list -> incomplete cite:** hive stale-doc list names `HiveConnector.wrapWithCache` but omits its line (hive audit pins `wrapWithCache:667-668`). -> Add `:667-668`. + +### Checks that PASSED (state for confidence) +- **"已通用" vs INPUT C:** matches — INPUT C CONFIRMS the routing half is universal; the report correctly scopes true one-load-per-statement to iceberg only (§3 table, §4(2)) and never claims connectors memoize on the metadata instance. **No overclaim.** +- **Migration status:** none mis-stated; hudi correctly "sibling / not in SPI_READY_TYPES / live", others correctly live. +- **No recommendation built on a knocked-down finding:** the three ADJUSTED corrections (ES-F2 is not a zero-new-cache fix; HD-P03 "3x" is an upper bound; hudi authz-safety rests on empty `getCapabilities`, not the iceberg-specific guard) are all correctly incorporated. +- **Roadmap completeness / no over-engineering:** every real P1 (hudi, maxcompute MC-1, es ES-F1/F2) is in the roadmap; jdbc/es/trino/paimon L1 caching correctly contraindicated, not assigned. +- **authz/session=user (security):** iceberg correctly the sole `SUPPORTS_USER_SESSION`; all 7 others correctly single-identity — **no false negative** (incl. paimon REST vended-token correctly noted as table-level, not user-keyed; gate hard-wired to `IcebergConnector.java` correctly flagged as not protecting a future 2nd session=user connector). \ No newline at end of file diff --git a/plan-doc/connector-cache-unification/connectors/es.md b/plan-doc/connector-cache-unification/connectors/es.md new file mode 100644 index 00000000000000..9e4f4e5d12996a --- /dev/null +++ b/plan-doc/connector-cache-unification/connectors/es.md @@ -0,0 +1,103 @@ +## 连接器附录:es (Elasticsearch, fe-connector-es) + +### 结论摘要 + +`es` 属于**读路径为主、无写入、无分区、单一凭证**的轻量连接器。它已经在 `SPI_READY_TYPES` 中,`EsConnectorMetadata` 已被 Layer-2 per-statement funnel 正确路由,但**连接器自身没有任何扫描元数据缓存**。iceberg 那套重型框架(Table/Partition/Format/Comment/Manifest cache、authz session=user 隔离、write-txn co-holder、per-file memo)**几乎全部不适用**。唯一真正值得做的是一个 **per-scan 的 `EsMetadataState` hoist**(把 shard routing + node 拓扑 + field-context 在一次扫描内解析一次,而不是两次),外加让扫描路径复用已被 `ExternalSchemaCache` 缓存的 mapping。都是 P1/P2 级别的**常数倍(2–4×)**冗余,而非 iceberg 那种 per-file 循环放大的 P0。 + +### 迁移状态 + +- `SPI_READY_TYPES` 含 `"es"`:`CatalogFactory.java:56-57`,createCatalog 走 SPI 路径(`:110` → `ConnectorFactory.createConnector` → `PluginDrivenExternalCatalog`)。 +- `EsConnectorProvider.getType()=="es"`(`EsConnectorProvider.java:34`),经 `META-INF/services` 发现。 +- **liveness = live**:legacy fe-core `EsExternalCatalog/EsExternalDatabase/EsExternalTable/EsScanNode` 已在 commit `4f455da5950` 删除,`BUILD_CONNECTOR_ES` 默认由 0 翻为 1;fe-core 仅残留与之无关的内表 `EsTable.java`/`EsResource.java` 与 `EsQuery` 函数。连接器是唯一活路径。 + +### 现有缓存(caches table) + +| 缓存 | scope | key | TTL / 失效 | 位置 | +|---|---|---|---|---| +| `EsConnector.restClient`(REST 连接对象,DCL 懒建) | metastore-client | 无(每 catalog 一个) | catalog 生命周期,不失效 | `EsConnector.java:43,82-107` | +| `EsConnectorRestClient.PLAIN_CLIENT / sslClient`(共享 OkHttp) | metastore-client | 无(JVM 全局) | 进程生命周期 | `EsConnectorRestClient.java:61-63,310-319` | +| Layer-2 funnel:每 statement 记忆一个 `EsConnectorMetadata` | per-statement | `"metadata:"+catalogId`,`getUser` 身份钉死 | 一条语句;NONE 不记忆 | `PluginDrivenMetadata.java:64,70` | +| `ResolvedScanProvider`:每 scan node 一个 `EsScanPlanProvider` | per-scan | `currentHandle` identity | scan node 生命周期 | `PluginDrivenScanNode.java:254-259` | +| fe-core `ExternalSchemaCache` 包住 `initSchema()→getTableSchema()→getMapping()` | cross-query | `SchemaCacheKey`(index) | schema cache 过期 / REFRESH | `PluginDrivenExternalTable.java:430-470`;`ExternalTable.java:385-445` | +| fe-core `RowCountCache` 包住 `fetchRowCount()`(ES 无 stats → UNKNOWN,无 ES 侧重 IO) | cross-query | table id | row-count cache 过期 | `PluginDrivenExternalTable.java:907,1121-1133` | +| **连接器专用扫描元数据缓存(mapping/shard/node)** | **none** | **不存在** | — | `EsScanPlanProvider.java:99,169,273-294`(每次重取) | + +### Funnel 参与度 + +`EsConnectorMetadata` 只经 `PluginDrivenMetadata.get(session, connector)` 获取(`PluginDrivenExternalTable.java:449,…,1277`;`PluginDrivenScanNode.java:1918-1920`),**确实被 funnel 路由**(每语句一个实例,arch gate 满足)。但该实例**不记忆任何东西**: + +- `getTableSchema`(`EsConnectorMetadata.java:81-94`)每次远程 `restClient.getMapping()`;`getColumnHandles`(`:97-106`)再调 `getTableSchema` → 第二次远程 mapping。 +- 更重的扫描态**根本不在 funneled metadata 上**:它在 `EsScanPlanProvider` 这个**独立对象**里(`EsConnector.getScanPlanProvider()` 每次 new,`EsConnector.java:56-58`;仅被 `ResolvedScanProvider` 每 scan node 记忆一次)。`planScan`(`:99`)与 `buildScanNodeProperties`(`:169`)各自独立 `fetchMetadataState`(`:273-294`)→ `getMapping + searchShards + getHttpNodes`。 + +因此 funnel 的"每语句一份 metadata"**并未**折叠 ES 的重复远程加载。`metadataMemoizesLoadedTable = no`。 + +### 热点重复加载审计(hot-path findings table) + +| id | area | 问题 | 倍数 | cost | 严重度 | 已记忆? | 位置 | +|---|---|---|---|---|---|---|---| +| ES-F1 | split-enum | `fetchMetadataState` 每查询 2 次(`planScan` `:99` + `buildScanNodeProperties` `:169`),每次 `searchShards` + `getHttpNodes`(`NODES_DISCOVERY_DEFAULT=true`)+ `getMapping`;同一 shard 路由/节点拓扑在一次 planning pass 内取两遍。provider 已是每 scan node 单实例,加个 `(index,columnNames)` 字段 memo 即 2→1,零陈旧风险 | 2×/查询 | remote-io | **P1** | 否 | `EsScanPlanProvider.java:99,169,273-294`;`EsMetadataFetcher.java:51-89` | +| ES-F2 | predicate | `fetchMapping`(`EsMetadataFetcher.java:57-58` → `getMapping` `RestClient:161`)取的 index mapping 已在 schema 解析时被 `ExternalSchemaCache` 缓存;扫描路径不复用而重新远程取 | ~2×/查询 | remote-io | **P1** | 否 | `EsMetadataFetcher.java:57-58`;`EsConnectorRestClient.java:161-168` | +| ES-F3 | schema | `buildColumnHandles→getColumnHandles→getTableSchema→getMapping`(裸远程,绕过 schema cache);`buildColumnHandles` 每查询≥2 次(`PluginDrivenScanNode.java:1263` getSplits、`:1841` getOrLoadPropertiesResult)。funneled metadata 是每语句单实例,在其上记忆 schema 即可折叠到 1× | 2×/查询 | remote-io | P2 | 否 | `EsConnectorMetadata.java:81-106`;`PluginDrivenScanNode.java:1263,1841,1917-1920` | +| ES-F4 | file-list | `getTableHandle→existIndex` 每次解析 handle 发一次 `index/_mapping` GET(`EsConnectorMetadata.java:74`;`RestClient:104-114`);轻量、row-count 侧有 RowCountCache;仅列完整性,非热循环 | 1×/handle | remote-io | P2 | 否 | `EsConnectorMetadata.java:71-78`;`EsConnectorRestClient.java:104-114` | + +合计:一条普通 `SELECT ... WHERE ...`(schema cache 命中的暖态)对同一 ES 集群大约发出 **~4× mapping + 2× search_shards + 2× _nodes/http** 远程往返(每次 OkHttp 10s read timeout),其中 mapping 完全冗余(已被 schema cache 缓存),shard/node 在语句内 2× 冗余。 + +**没有 iceberg 式的 per-file/per-split 循环放大**:`planScan` 从内存中的 shard 路由 map 简单循环生成 scan range(`EsScanPlanProvider.java:113-138`),循环内无远程调用。所以是 variant B/D(单链重复 k 次 / 循环不变量未提升),不是 variant A。 + +### Authz / 一致性 + +- `perUserCredential = false`、`sessionUserRelevant = false`。ES 用**单一 catalog 级 user/password**构建共享 `EsConnectorRestClient`(`EsConnector.java:95-100`;`EsConnectorRestClient.java:76-78` 一个 Basic auth header),无 `iceberg.rest.session=user`、无 kerberos doAs、无 REST/OIDC 每身份委派——所有 Doris 用户以同一身份访问 ES。因此 **name-only 缓存不会造成 list≠load 元数据泄露**,`tools/check-authz-cache-sharding.sh` 的 session=user 隔离对本连接器 **N/A**。 +- 唯一一致性关切是**数据新鲜度**:ES shard 会 rebalance(ES 自身 refresh 模型),shard/node 拓扑必须每语句解析、**不得跨查询缓存**——这与被删的 legacy `EsScanNode`/`EsMetaStateTracker` 每次扫描重解析 shard 一致。 +- **无写路径**(`EsConnectorMetadata` 只覆写 list/exists/getTableHandle/getTableSchema/getColumnHandles/buildTableDescriptor;无 beginWrite/commit/insert/sink),故无"读写共享一份 metadata/一个 txn"的需求;ES 只读且非分区(未覆写 listPartitions),无 snapshot/version 钉需求。 + +### 框架各件需求 + +| piece | 需要 | 理由 | +|---|---|---| +| per-scan-hoist | **yes** | 唯一真正收益:把 `EsMetadataState` 每扫描解析一次(ES-F1)。provider 已单实例,字段 memo 即可,零陈旧风险。 | +| per-statement-funnel-memoization | partial | funnel 已路由且 arch gate 满足,但连接器未利用:schema/getColumnHandles 仍每次远程取(ES-F3),重扫描态在 funnel 覆盖不到的独立 provider 上。 | +| cross-query-table-cache | partial | mapping 已被 `ExternalSchemaCache` 跨查询缓存,扫描路径复用即可(ES-F2);无需新建连接器 Table cache;shard/node **禁止**跨查询缓存。 | +| shared-fe-connector-cache-adoption | no | 连接器未用 fe-connector-cache,且推荐修复是 per-statement/per-scan memo 而非托管跨查询缓存,无需引入该 toolkit。 | +| partition-cache | no | ES 单索引非分区,未覆写 listPartitions;无 PARTITIONS/SHOW PARTITIONS 重路径。 | +| format-cache | no | 格式固定常量 `es_http`(`EsScanPlanProvider.java:174`),无 iceberg PERF-03 的整表 planFiles 回退。 | +| comment-cache | no | 未覆写 getComment,无 N-per-query 远程 comment 加载。 | +| manifest-or-file-list-cache | no | 无 manifest/文件清单;shard 取数是结构类比但新鲜度敏感,归入 per-scan-hoist。 | +| per-file-split-memo | no | scan range 从内存 shard map 循环生成,循环内无远程调用。 | +| authz-session-user-isolation | no | 单凭证,无 session=user/OIDC/kerberos。 | +| write-txn-coholder | no | 只读连接器。 | + +### 优先级建议 + +1. **(P1)ES-F1 per-scan hoist**:在 `EsScanPlanProvider` 上按 `(indexName, columnNames)` 记忆一次 `EsMetadataState`,让 `planScan` 与 `getScanNodePropertiesResult` 共享同一次 shard/node 解析。这是最干净、零风险、收益最大的改动(shard 取数 2→1、node 取数 2→1)。 +2. **(P1)ES-F2 复用已缓存 mapping**:扫描路径的 field-context 从已被 `ExternalSchemaCache` 缓存的 schema 派生,消除扫描侧的冗余 mapping 远程取,避免重复 `getMapping`。 +3. **(P2)ES-F3 metadata schema memo**:在每语句单实例的 `EsConnectorMetadata` 上记忆已解析 schema,使 `getColumnHandles` 折叠到 1×。 +4. **不做**:跨查询 shard 缓存(违背 ES rebalance/refresh 模型)、authz 隔离、write-txn、partition/format/manifest/comment 缓存。 + +--- + +## 复核结论 (adversarial verify) + +**Verdict: ADJUSTED** · 确认发现 IDs: ES-F1, ES-F2, ES-F3, ES-F4 + +| 原始声明 | 问题 | 更正 | 证据 | +|---|---|---|---| +| frameworkPiecesNeeded.cross-query-table-cache + ES-F2 fix rationale: 'Serving the scan-path field-context from the already-cached schema removes the redundant cross-query mapping fetch WITHOUT a new cache' / 'reuse the cached schema'. | The fe-core ExternalSchemaCache stores only the parsed ConnectorTableSchema (column list). EsConnectorMetadata.getTableSchema builds `new ConnectorTableSchema(indexName, columns, "ELASTICSEARCH", Collections.emptyMap())` (EsConnectorMetadata.java:90-93) — an EMPTY properties map; the raw mapping JSON is discarded. But EsMetadataFetcher.fetchMapping needs the raw mapping (keyword-sniff / doc_value / date-compat) via EsMappingUtils.resolveFieldContext(columnNames, sourceIndex, indexMapping, mappingType) (EsMetadataFetcher.java:57-63). So the scan-path field context CANNOT be derived from the currently cached schema value. | The redundant remote mapping fetch (ES-F2) is REAL and the multiplier holds, but the proposed fix is understated: eliminating it requires either enriching ConnectorTableSchema to carry the raw mapping/field-context, or adding a per-statement/cross-query mapping (field-context) cache — i.e. it is NOT a zero-new-cache 'just reuse the schema' change. Files: fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java:90-93; EsMetadataFetcher.java:57-63; EsMappingUtils.resolveFieldContext. | fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java | +| currentCaches RowCountCache row: 'ES has no stats -> returns UNKNOWN, no ES-side remote heavy op'. | fetchRowCount (PluginDrivenExternalTable.java:1121-1133) first calls resolveConnectorTableHandle -> EsConnectorMetadata.getTableHandle -> restClient.existIndex (a remote `index/_mapping` GET, EsConnectorRestClient.java:104-114) BEFORE it can return UNKNOWN. EsConnectorMetadata does not override getTableStatistics (confirmed by grep), so getTableStatistics itself makes no ES RPC, but the handle-resolution step does perform one lightweight remote GET. | The phrase is defensible for 'heavy' op (there is none), but 'no ES-side remote op' would be wrong: the row-count seam still issues one lightweight existIndex `_mapping` GET during handle resolution (this is the same op as ES-F4). It sits behind RowCountCache so it is not per-query hot; precision note only. File: fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java:1121-1133. | fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java | + +> 复核备注：CORE STANDS — every material claim verified against HEAD (aaab68ef474). Corrections are limited to fix-feasibility/wording nuances; no finding, multiplier, severity, or memoized-status is refuted. + +MIGRATION STATUS — CONFIRMED. SPI_READY_TYPES includes "es" at CatalogFactory.java:56-57 (ImmutableSet.of("jdbc","es","trino-connector","max_compute","paimon","iceberg","hms")); SPI routing at ~line 110. EsConnectorProvider.getType() returns "es" (line 34). Legacy fe-core EsExternalCatalog/EsExternalDatabase/EsExternalTable/EsScanNode are ABSENT (find returned nothing); internal-catalog EsTable.java/EsResource.java remain. Connector is sole live ES path. (Did not locate BUILD_CONNECTOR_ES in pom.xml/build.sh — a peripheral historical detail, not required for any technical claim.) + +CACHES — CONFIRMED. EsConnector.restClient DCL memo (EsConnector.java:43,82-107); static PLAIN_CLIENT/sslClient with 10s readTimeout (EsConnectorRestClient.java:61-63,310-319); single Basic authHeader built once (76-78). Layer-2 funnel memo keyed "metadata:"+catalogId + identity pin "metadata-identity:"+catalogId via getUser (PluginDrivenMetadata.java:64,70). ResolvedScanProvider memo keyed on currentHandle identity (PluginDrivenScanNode.java:254-256, class at 267). RowCountCache at PluginDrivenExternalTable.java:907. No dedicated connector scan-metadata cache — confirmed absent (EsScanPlanProvider re-fetches). + +FUNNEL PARTICIPATION — CONFIRMED (funneled=true, metadataMemoizesLoadedTable=no). EsConnectorMetadata holds only restClient+properties; getTableSchema (81-94) calls restClient.getMapping every call; getColumnHandles (97-106) re-invokes getTableSchema -> 2nd remote GET; no memo fields. EsScanPlanProvider is a separate object (EsConnector.java:56-58 returns new provider each call), memoized only per scan node. + +HOT-PATH FINDINGS — all four CONFIRMED: +- ES-F1 (P1, 2x, not memoized): fetchMetadataState called from planScan (EsScanPlanProvider.java:99, reached via getSplits->planScan at PluginDrivenScanNode.java:1303) AND from buildScanNodeProperties (line 169, reached via getScanNodePropertiesResult at PluginDrivenScanNode.java:1863). getOrLoadPropertiesResult IS cached (cachedPropertiesResult, line 1838/1870), so the second path fires exactly once — 2x total, constant, verified. Each fetch = fetchMapping+searchShards+getHttpNodes (EsMetadataFetcher.java:51-89); NODES_DISCOVERY_DEFAULT="true" (EsConnectorProperties.java:38) so getHttpNodes really runs. planScan loop (113-138) is over an in-memory routing map with no remote call — correctly characterized as variant B/D, not loop-amplified. +- ES-F2 (P1, ~2x): fetchMapping->getMapping (EsMetadataFetcher.java:57-58; RestClient:161-168) duplicates the schema-resolution mapping fetch. Real; see correction on fix scope. +- ES-F3 (P2, 2x): buildColumnHandles (PluginDrivenScanNode.java:1917-1920) -> getColumnHandles -> getTableSchema -> raw getMapping, bypassing ExternalSchemaCache; buildColumnHandles runs at 1263 (getSplits) and 1841 (getOrLoadPropertiesResult) — 2x confirmed. +- ES-F4 (P2, 1x/handle): getTableHandle->existIndex `_mapping` GET (EsConnectorMetadata.java:74; RestClient:104-114). Confirmed low-cost. + +Per-SELECT remote tally (~4x getMapping + 2x searchShards + 2x _nodes/http) reproduces from the code. No missed P0: no heavy op sits in a per-split/per-file loop, so the audit does not UNDERSTATE anything — worst case is constant-factor. + +AUTHZ — CONFIRMED. Single catalog-level Basic credential (EsConnector.java:95-100; RestClient:76-78); no session=user/OIDC/kerberos. EsConnectorMetadata overrides no write/partition/comment/stats methods (grep for getTableStatistics/listPartitions/getTableComment/beginWrite/applyFilter returned nothing) — read-only, non-partitioned confirmed. authz-session-user-isolation N/A is correct. Shard/node freshness rationale (do NOT cross-query cache) is sound. diff --git a/plan-doc/connector-cache-unification/connectors/hive.md b/plan-doc/connector-cache-unification/connectors/hive.md new file mode 100644 index 00000000000000..b2f680f9763156 --- /dev/null +++ b/plan-doc/connector-cache-unification/connectors/hive.md @@ -0,0 +1,97 @@ +## 连接器附录 — hive (catalog type `hms`) + +### 结论速览 + +hive 连接器在 HEAD 已经是**框架对齐 + 缓存充分**的状态，**不需要**再套用 iceberg 的 hot-path 缓存改造。它已经具备参考 commit `0b4f72582e7` 三层框架里对 hive 有意义的全部部件：Layer-1 的连接器侧 cross-query 缓存以 hive 自己的形态存在（`CachingHmsClient` 四缓存 + `HiveFileListingCache` + `ConnectorPartitionViewCache`），Layer-2 的 per-statement funnel 完整接入（主连接器走 `PluginDrivenMetadata.get`，sibling 走 `memoizedSiblingMetadata`，写路径有 per-statement 的 `HiveConnectorTransaction`），Layer-3 的 authz 隔离对 hive **不适用**（无 session=user）。所有 planning 入口的重 IO 都命中缓存，**没有** iceberg 那种「同一 planning pass 里 loadTable 3-7 次」的回归。 + +> ⚠ 文档漂移：`CachingHmsClient.java:85-88`、`HiveFileListingCache.java:72-74`、`HiveConnector.wrapWithCache` 的 javadoc（:667-668）仍写着 "Dormant / `hms` is not in `SPI_READY_TYPES`"，这在 HEAD 是**错的**——commit `6e521aa64b2`(#65473) 同时把 `hms` 放进 `SPI_READY_TYPES`(CatalogFactory.java:57) 并在 `createClient()` 里接线了 `wrapWithCache`(:645-646)，却没同步更新这几处类注释。纯文档问题，不影响运行。 + +### 迁移状态 + +- `hms` ∈ `SPI_READY_TYPES`（`CatalogFactory.java:57`）→ **LIVE**。fe-core 用 `PluginDrivenExternalCatalog` 包 `HiveConnector`（`CatalogFactory.java:110-118`）。 +- 同时是**异构网关（gateway）**：iceberg-on-HMS / hudi-on-HMS 作为 embedded SIBLING 连接器，在各自 child-first classloader 里按需构建（`HiveConnector.java:530-592`），其 `ConnectorMetadata` 按 owner label 做 per-statement 记忆化（`HiveConnectorMetadata.memoizedSiblingMetadata`，:302-305）。hudi 不是独立 SPI 类型（`CatalogFactory.java:51-55`）。 + +### 缓存清单 + +| 缓存 | scope | key | TTL/容量 | 失效 | 位置 | +|---|---|---|---|---|---| +| `CachingHmsClient.tableCache` | metastore-client | (db,table)→HmsTableInfo | 24h / 1万；兼容 `schema.cache.ttl-second` | REFRESH flush/flushDb/flushAll | `CachingHmsClient.java:116,172` | +| `CachingHmsClient.partitionNamesCache` | metastore-client | (db,table,maxParts) | 24h / 1万；兼容 `partition.cache.ttl-second` | 同上 + `invalidatePartitions` | `CachingHmsClient.java:117,187` | +| `CachingHmsClient.partitionsCache` | metastore-client | (db,table,partitionValues)，一分区一条 | 24h / 10万；put 有 generation guard 防 REFRESH race | 同上（按分区） | `CachingHmsClient.java:118,202-243` | +| `CachingHmsClient.columnStatsCache` | metastore-client | (db,table,columns) | 24h / 1万 | 同上 | `CachingHmsClient.java:119,274` | +| `HiveFileListingCache` | cross-query | (db,table,location,partitionValues)→List\ | 24h / 1万；兼容 `file.meta.cache.ttl-second` | table/db/partitions/all | `HiveFileListingCache.java:114,160-176` | +| `HiveConnector.partitionViewCache`（`ConnectorPartitionViewCache`，#65829 的 PERF-06 cache A） | cross-query | (db,table,-1,-1)（hive 无 snapshot/无 schema version，一表一条）→List\ | 24h / 1000 | table/db/all | `HiveConnector.java:112,136`；消费 `HiveConnectorMetadata.java:1160-1172` | +| `memoizedSiblingMetadata`（iceberg/hudi sibling 元数据） | per-statement | `metadata:{catalogId}:{ownerLabel}` | statement 生命周期；NONE scope 不记忆 | 随 scope 关闭 | `HiveConnectorMetadata.java:302-305` | +| `HiveConnectorTransaction` begin-time 表快照 | per-statement | 唯一写目标表（`hmsTableInfo`/`tableActions`） | 写事务生命周期 | 事务结束 | `HiveConnectorTransaction.java:209,546-558` | +| `ThriftHmsClient` 连接池（非元数据缓存，仅传输） | metastore-client | pool size = `hms.client.pool.size` | 连接生命周期 | — | `HiveConnector.java:613-617` | + +关键点：`HiveConnector.createClient()` 用 `wrapWithCache` 把裸 `ThriftHmsClient` 包成 `CachingHmsClient`（:645-646,670-672），所以 `HiveConnectorMetadata` 里**所有** `hmsClient.getTable/listPartitionNames/getPartitions/getTableColumnStatistics` 调用都透明走 cross-query 缓存——这正是 hive 不需要 iceberg PERF-07（per-statement 表加载）的原因：iceberg 在 cutover 后**没有** cross-query 表缓存，hive 有。 + +### funnel 参与 + +- **主路径**：fe-core 所有 seam 经 `PluginDrivenMetadata.get(session, connector)`（funnel 唯一合法直呼点，`PluginDrivenMetadata.java:52-72`；调用点 `PluginDrivenExternalTable.java:133,159,189,449,...`）→ `HiveConnector.getMetadata`→`newMetadata(getOrCreateClient())`（`HiveConnector.java:140-142`）。**一 statement 一 ConnectorMetadata** 不变式成立（`tools/check-fecore-metadata-funnel.sh` 全构建守门）。 +- **读侧是否 per-metadata 记忆化已加载表**：**否**——`HiveConnectorMetadata` 无 `resolvedTable` 字段（字段仅 hmsClient/缓存/sibling seam，:203-241），`getTableHandle`(:399)/`getTableSchema`(:493)/`getColumnHandles`(:604)/`getColumnStatistics`(:854)/`applyFilter`(:1093) 各自 fresh 调 getTable，但**由 CachingHmsClient cross-query 缓存兜底**，同 statement 内重复加载 = 1 RPC + N 命中。**写侧**额外在写事务内记忆 begin-time 表快照（`HiveConnectorTransaction.java:549-551`）。 +- **sibling 路径**：iceberg/hudi 元数据按 (catalogId, ownerLabel) per-statement 记忆化，保证一 statement 内同 owner 复用一个 metadata（:302-350）。 + +### hot-path 逐项审计 + +| ID | 入口 | 重 op & 倍率 | 是否已记忆 | 严重度 | 位置 | +|---|---|---|---|---|---| +| HMS-H1 | table/schema 加载 | getTable 每 pass 3-4 次 → 1 RPC + N 命中（tableCache） | ✅ | none | `HiveConnectorMetadata.java:399,493,604,854` | +| HMS-H2 | 分区枚举（pruning/scan/stats/MTMV） | listPartitionNames+getPartitions 多次 → 全缓存（names/partitions/view 三层） | ✅ | none | `:1093-1106,1019-1025,1160-1172`；`HiveScanPlanProvider.java:492-499` | +| HMS-H3 | 分区目录 listStatus（scan 最重 IO，O(分区)） | 每分区一次 → `HiveFileListingCache` cross-query，scan 与 size-estimate 互暖 | ✅ | none | `HiveScanPlanProvider.java:537`；`HiveConnectorMetadata.java:945,1062` | +| HMS-H4 | ACID split 枚举 | `HiveAcidUtil.getAcidState` 每分区每 scan，**未缓存** | ❌ | none（按设计） | `HiveScanPlanProvider.java:314` | +| HMS-H5 | 写规划 | loadTable + beginWrite 双载 + buildExistingPartitions → 缓存 + 事务内快照复用 | ✅ | none | `HiveWritePlanProvider.java:103,105,255-258`；`HiveConnectorTransaction.java:549-551` | +| HMS-H6 | 读侧无 per-statement 表 memo | 仅靠 CachingHmsClient TTL；同 statement 两次 getTable 之间若发生淘汰/过期 → 第二次 RPC | ❌ | **P2** | `HiveConnectorMetadata.java:203-241` | + +- **HMS-H4 是「有意不缓存」**：ACID 目录状态依赖 valid write-id 快照（每事务变），cross-query 缓存会读到过期数据；`getFileListingCache` 注释明确只服务非 ACID 路径（`HiveScanPlanProvider.java:103-104`），与 legacy 一致，**不是缺陷**。 +- **SHOW PARTITIONS 走 `listPartitionNamesFresh`（绕缓存）**（`CachingHmsClient.java:192-199`；`HiveConnectorMetadata.java:1130-1136`）也是**有意 fresh**（`use_meta_cache` 契约，legacy parity），非缺陷。 + +### authz / 一致性 + +hive **不 vend 任何 per-user 凭证**、**无 session=user 类比**。metastore RPC 使用**单一 catalog 级身份**：要么 plugin 侧 Kerberos keytab 固定 principal（`HiveConnector.buildPluginAuthenticator`，:710-740），要么 `context.executeAuthenticated`（catalog 级），**从不**是查询用户的委派凭证，也没有 REST OIDC / per-identity metastore。因此 name-only 的缓存 key（db,table[,parts/cols/location]）**不会**像 `iceberg.rest.session=user` 那样泄漏跨用户元数据——所有用户看到同一 metastore 身份的视图，用户级访问由 Doris fe-core RBAC 独立管控。这正是 authz 守门脚本 `tools/check-authz-cache-sharding.sh` **只针对 `IcebergConnector.java`** 的原因，hive 无条件构建缓存（`HiveConnector.java:108-112,133-136`）。另有一处 fail-loud 不变式（`HiveConnector.java:548-555`）：若 iceberg-on-HMS sibling 竟声明 `SUPPORTS_USER_SESSION` 就直接抛错——恰因 hive 前门不是 session=user、fe-core 的 per-user schema/name 缓存 bypass 是按前门连接器 keyed 的。写读一致性：读写共享同一 cross-query `CachingHmsClient`（不可变 `HmsTableInfo`/`HmsPartitionInfo`），写事务钉住 begin-time 表快照，ACID 读每 scan 钉 write-id 快照（`getValidWriteIds`）。陈旧性由 coarse REFRESH（flush/flushDb/flushAll，`HiveConnector.java:357-419`）+ TTL 收敛。**Layer-3 对 hive 不适用。** + +### 框架部件需求判定 + +| 部件 | 判定 | 依据 | +|---|---|---| +| cross-query-table-cache | already-has | `CachingHmsClient.tableCache`（iceberg `IcebergTableCache` 的 metastore-client 类比） | +| partition-cache | already-has | partitionNamesCache + partitionsCache + `ConnectorPartitionViewCache` 三层 | +| manifest-or-file-list-cache | already-has | `HiveFileListingCache`（manifest cache 的 hive 类比） | +| shared-fe-connector-cache-adoption | already-has | 全程用 `MetaCacheEntry`/`CacheSpec`/`ConnectorPartitionViewCache`/`PartitionViewCacheKey` | +| write-txn-coholder | already-has | `HiveConnectorTransaction` per-statement 写事务，co-hold begin-time 表 | +| per-scan-hoist | already-has | format/split-size/splittable/storage-props 均在分区循环前算一次 | +| per-statement-funnel-memoization | no | cross-query 缓存已折叠重复加载；HMS getTable 远轻于 iceberg loadTable，不值当 | +| format-cache | no | 从 handle inputFormat/serde 廉价推断，无 iceberg 那种整表 planFiles fallback | +| comment-cache | no | comment 来自已缓存的 getTable params/ConnectorColumn，无独立 N-per-query 远程 getComment | +| per-file-split-memo | no | splitFile 按引用共享分区 value map，无 per-file 派生可 memo | +| authz-session-user-isolation | no | 无 per-user 凭证 / 无 session=user；RBAC 独立管控 | + +### 优先建议（均为 LOW） + +1. **文档修正**：把 `CachingHmsClient.java:85-88`、`HiveFileListingCache.java:72-74`、`HiveConnector.wrapWithCache`(:667-668) 里过期的 "Dormant / hms not in SPI_READY_TYPES" 注释改为「live」——纯文档。 +2. **可选 P2**：给读侧 `HiveConnectorMetadata` 加一个 per-statement `resolvedTable` memo（iceberg PERF-07 类比），关掉 HMS-H6 那个极罕见的「statement 中途缓存淘汰导致二次 RPC」race。成本收益看**不建议**做（HMS getTable 很轻）。 +3. **保持** ACID `getAcidState` per-scan 不缓存（HMS-H4）——快照/write-id 依赖，legacy parity 正确。 + +--- + +## 复核结论 (adversarial verify) + +**Verdict: CONFIRMED** · 确认发现 IDs: HMS-H1, HMS-H2, HMS-H3, HMS-H4, HMS-H5, HMS-H6 + +| 原始声明 | 问题 | 更正 | 证据 | +|---|---|---|---| +| HiveConnectorTransaction begin-time table snapshot ... hmsTableInfo captured in beginWrite (HiveConnectorTransaction.java:209) | Off-by-2 line cite: the hmsTableInfo field is declared at :123 and assigned inside beginWrite at :211, not :209. Immaterial — :209 falls within the beginWrite method and the reuse-in-getTable range :546-558 (specifically :549-551) is exact. | Field decl HiveConnectorTransaction.java:123 (private volatile HmsTableInfo hmsTableInfo); assignment this.hmsTableInfo = table at :211; reuse at :551. No substantive error. | | +| detailedMarkdown enumerates only 3 stale 'Dormant/hms not in SPI_READY_TYPES' javadocs (CachingHmsClient:85-88, HiveFileListingCache:72-74, HiveConnector.wrapWithCache:667-668) | Non-exhaustive, not wrong: the icebergSibling/hudiSibling field comments at HiveConnector.java:118 and :126-127 also carry stale 'Dormant until hms enters SPI_READY_TYPES' text at HEAD. | Two additional stale doc sites exist at HiveConnector.java:118,126-127; harmless, additive to the same doc-fix recommendation (LOW). | | + +> 复核备注：Every material claim holds against HEAD (aaab68ef474). MIGRATION: hms in SPI_READY_TYPES (CatalogFactory.java:56-57); hudi correctly not a standalone type; createClient wraps ThriftHmsClient in CachingHmsClient (HiveConnector.java:645-646, wrapWithCache:670-672). The three stale 'Dormant' javadocs are genuinely factually wrong at HEAD (hms is live) — doc-fix recommendation valid. + +CACHES (all 9 verified, no hallucinations, scope/key/TTL correct): tableCache field :116 / getTable :172-175 (db,table); partitionNamesCache :117 / :187-190 (db,table,maxParts); partitionsCache :118 / :202-243 per-partition entry keyed by values, generation-guarded put :235-238; columnStatsCache :119 / :274-279; DEFAULT_TTL 86400 (:109), caps 10000/10000/100000/10000 (:110-113) all match. HiveFileListingCache :114 / listDataFiles cache.get :174 keyed (db,table,location,partitionValues); TTL 86400 cap 10000 (:92-93); legacy file.meta.cache.ttl-second remap :87,141-143. partitionViewCache field HiveConnector.java:112 constructed :136 (ConnectorPartitionViewCache), consumed HiveConnectorMetadata listPartitions :1170-1172 keyed (db,table,-1,-1). memoizedSiblingMetadata :302-305 keyed 'metadata:'+catalogId+':'+ownerLabel via statementScope.getOrCreateMetadata (per-statement, NONE=no memo). Shared toolkit classes (ConnectorPartitionViewCache/PartitionViewCacheKey/MetaCacheEntry/CacheSpec) all exist in fe-connector-cache and are imported (CachingHmsClient :20-21). + +FUNNEL: PluginDrivenMetadata.get (fe-core :53, memoizes on scope.getOrCreateMetadata('metadata:'+catalogId) :70); PluginDrivenExternalTable calls it 16×; getMetadata->newMetadata(getOrCreateClient()) HiveConnector :140-142,188-192. Read-side HiveConnectorMetadata fields :203-241 hold hmsClient/caches/sibling seams only — NO resolvedTable field, so getTable at :399/:493/:604 and col-stats :854 re-call fresh but collapse via CachingHmsClient; write-side reuses hmsTableInfo begin-time snapshot :551. 'metadataMemoizesLoadedTable: partial' is accurate. + +HOT-PATH FINDINGS all 6 confirmed at exact sites: H1 getTable :399/:493/:604 + getTableColumnStatistics :854, all via CachingHmsClient (severity none, memoized) — multiplier 3-4x not overstated. H2 applyFilter listPartitionNames :1093 + getPartitions :1105; resolvePartitionRefs :1019-1025; listPartitions partitionViewCache :1160-1172 — three-layer cache, none. H3 fileListingCache.listDataFiles HiveScanPlanProvider :537 (non-ACID) + sumCachedFileSizes HiveConnectorMetadata :1062-1063, HiveFileListingCache :174 — shared scan/estimate, none. H4 HiveAcidUtil.getAcidState HiveScanPlanProvider :314 inside per-partition loop (:307), genuinely uncached, snapshot/write-id dependent (by design, none). H5 loadTable HiveWritePlanProvider :103 (hmsClient.getTable :353) + beginWrite :105 double-load accepted + buildExistingPartitions listPartitionNames/getPartitions :255-258, collapsed by cache + txn snapshot :549-551. H6 no per-statement resolvedTable memo (fields :203-241) — residual mid-statement TTL/eviction re-RPC race, P2, correctly characterized as not warranted. + +AUTHZ: getCapabilities HiveConnector :301,341-344 returns SUPPORTS_VIEW/METADATA_PRELOAD/MVCC_SNAPSHOT — NO SUPPORTS_USER_SESSION; no per-user credential (auth via fixed keytab principal buildPluginAuthenticator :710-740 or context.executeAuthenticated :638). authz-cache-sharding gate targets ONLY IcebergConnector (TARGET_REL :57); fail-loud sibling guard :548-555; shouldBypassSchemaCache/SUPPORTS_USER_SESSION exist in fe-core PluginDrivenExternalCatalog :1179,1212. Layer-3 N/A for hive is correct. + +NO MISSED SEVERE FINDING: MTMV freshness (getTableFreshness :1296-1327, getPartitionFreshnessMillis :1338-1350) reads cached collectPartitionNames + hmsClient.getPartitions — no uncached heavy op. SHOW PARTITIONS intentionally fresh (listPartitionNamesFresh :193-199, HiveConnectorMetadata :1130-1136) is interactive DDL, not a hot path. splitFile :628 shares PartitionScanInfo :707 by reference (no per-file recompute). Overall verdict 'DO NOT apply new iceberg-style caches to hive' is well-supported: every planning entrypoint sits behind a live REFRESH/TTL-invalidated cache. Confidence: high. diff --git a/plan-doc/connector-cache-unification/connectors/hudi.md b/plan-doc/connector-cache-unification/connectors/hudi.md new file mode 100644 index 00000000000000..500108a54b5920 --- /dev/null +++ b/plan-doc/connector-cache-unification/connectors/hudi.md @@ -0,0 +1,94 @@ +## 连接器附录 — hudi (fe-connector-hudi) + +### 1. 迁移状态 (Migration status) + +- **`hudi` 不在 `SPI_READY_TYPES`（设计如此，正确）**。`CatalogFactory.java:56-57` 的白名单是 `{jdbc, es, trino-connector, max_compute, paimon, iceberg, hms}`；`:50-55` 的注释明确禁止加入 `"hudi"`。没有独立的 `HudiExternalCatalog`；`HudiConnectorProvider.java:45-47` 返回的是 sibling-only 类型串 `"hudi"`。 +- **hudi 通过 hms gateway 以 sibling 形式暴露，且已 LIVE**。`HiveConnector.newMetadata` 把 `this::getOrCreateHudiSibling` 接入 `HiveConnectorMetadata`（`HiveConnector.java:188-191`），`HiveConnectorMetadata.getTableHandle` 对检测为 HUDI 的表转发到 `hudiSiblingMetadata(session).getTableHandle(...)`（`HiveConnectorMetadata.java:415-417`）。`getOrCreateHudiSibling` 通过 `context.createSiblingConnector("hudi", ...)` 构建唯一 sibling（`HiveConnector.java:576-591`）。 +- **liveness = live（非 dormant）**：`"hms"` 已由 **#65473 / commit `6e521aa64b2`（2026-07-16）** 加入 `SPI_READY_TYPES`；且 fe-core 侧的 legacy hudi 路径（`HudiScanNode` / `HudiExternalMetaCache` / `HMSExternalTable` / `HudiDlaTable`）已全部删除，连接器路径是 hudi 的唯一路径。 +- **⚠ 需暴露的冲突（Rule 7）**：源码中仍散落大量 `"dormant until hms enters SPI_READY_TYPES"` / `"today getTableHandle is never called"` 注释（`HudiConnectorMetadata.java:391`、`HiveConnectorMetadata.java:410,1802,1942,1972,2007`）。这些注释写于 #65473 之前，已 **STALE**，与 HEAD 的 `SPI_READY_TYPES` 事实矛盾，不能据其判定 liveness，应清理。 + +### 2. 现有缓存 (Current caches) + +| 缓存 | scope | key | TTL / 失效 | file:line | 说明 | +|---|---|---|---|---|---| +| per-statement `ConnectorMetadata`（funnel） | per-statement | `(catalogId, HUDI_LABEL)` on `ConnectorStatementScope` | 语句结束 | `HiveConnectorMetadata.java:302` | Layer-2 唯一 memo；只 memo **对象**，不 memo 已加载的 metaClient/schema | +| HMS `ThriftHmsClient`（**RAW，无结果缓存**） | metastore-client | 单个连接池 client memo 在 `HudiConnector` 上 | 无（连接器生命周期） | `HudiConnector.java:186` | **未包 `CachingHmsClient`**：`getTable/tableExists/listTables/listPartitionNames` 每次都是新 Thrift RPC | +| `pluginAuth`（`HadoopAuthenticator`） | cross-query | catalog 级单一 Kerberos 身份 | 连接器生命周期 | `HudiConnector.java:196` | 认证对象，非元数据缓存 | +| Hudi `InternalSchemaCache`（库内部，非 Doris 缓存） | none | `(commitTime, metaClient)` in hudi-common | 库自管 | `HudiSchemaUtils.java:278` | 随每次新建 metaClient 基本重置 | + +**结论**：hudi **没有任何连接器侧 cross-query 元数据缓存**，**没有 per-statement 的 metaClient/schema memo**，**完全没用**共享工具箱（pom 无 `fe-connector-cache` 依赖，无 `CacheFactory/CacheSpec/MetaCacheEntry/ConnectorPartitionViewCache`），甚至没有像 `HiveConnector` 那样 `wrapWithCache` 包 `CachingHmsClient`。它是所有 SPI 连接器里缓存最薄的一个。 + +### 3. Funnel 参与 (Funnel participation) + +- **已接入 Layer-2 funnel**：sibling 元数据经 `HiveConnectorMetadata.memoizedSiblingMetadata`（`:302-305`）→ `session.getStatementScope().getOrCreateMetadata("metadata:"+catalogId+":"+ownerLabel, () -> owner.getMetadata(session))`，`hudiSiblingMetadata`（`:332-334`）与 by-handle `siblingMetadata`（`:347-350`）在同一语句内共用同一实例（`HUDI_LABEL` keyed）。故一条语句内只构建 **一个** `HudiConnectorMetadata`。 +- **但该实例内部零 memo**：`HudiConnectorMetadata` 字段仅 `{hmsClient, properties, metaClientExecutor, storageHadoopConfig}`（`:153-174`），无任何 resolved metaClient/schema/timeline/partitions 缓存字段。每个 `getTableSchema` / `getColumnHandles` / `applyFilter` / `listPartitions*` / `beginQuerySnapshot` / `collectPartitions` 都经 `HudiScanPlanProvider.buildMetaClient` 重新构建 `HoodieTableMetaClient`。 +- **scan provider 与 metadata 割裂**：`HudiConnector.getScanPlanProvider` 每次返回 `new HudiScanPlanProvider`（`:136-138`），并在 `planScan`（`:148-151`）与 `getScanNodeProperties`（`:363`）各建一次自己的 metaClient，与 metadata 的完全独立。 +- **`metadataMemoizesLoadedTable = no`**：funnel 把「元数据对象」收敛为一条语句一个，但**没有**收敛重复的远程 table load。需要 iceberg 式的内部 memo（PERF-01/PERF-07 类比）才能把每 pass 5-6 次 metaClient 构建塌缩为一次。 + +### 4. 热点重复加载审计 (Hot-path repeated-load audit) + +重活 = 构建 `HoodieTableMetaClient`（远程读 `.hoodie/hoodie.properties` + table config + 列举 active timeline 目录）+ schema 解析（`getTableAvroSchema` + `InternalSchema`）+ 分区列举（`listAllPartitionPaths` = `HoodieTableMetadata` 扫描）。对一条「带 WHERE 分区谓词、非 hive-sync 的分区 hudi SELECT」，单个 planning pass 内对**同一张表**的重复远程加载： + +| ID | area | 问题 | 倍数 | cost | 严重度 | memoized? | file:line | +|---|---|---|---|---|---|---|---| +| HD-P01 | query-plan | `HoodieTableMetaClient` 在每个 metadata+scan 入口各重建一次（getColumnHandles→getTableSchema、beginQuerySnapshot→latestInstant、applyFilter、MVCC getTableSchema、planScan、getScanNodeProperties），~5-6 次/pass；每次远程读 hoodie.properties + 列举 timeline。iceberg PERF-01/07 直接类比。 | ~5-6×/pass (B/D) | remote-io | **P1** | no | `HudiScanPlanProvider.java:148` | +| HD-P02 | schema | 最新 Avro schema + InternalSchema 在独立 metaClient 上重复解析 ~4 次：`getSchemaFromMetaClient`（`:819-821`）供 getColumnHandles 与 3-arg MVCC getTableSchema；planScan（`:188,194,220`）；getScanNodeProperties（`:382-383`）。 | ~4×/pass (B) | remote-io | **P1** | no | `HudiConnectorMetadata.java:797` | +| HD-P03 | partitions | 分区列举（`listAllPartitionPaths` = `HoodieTableMetadata.create`+`getAllPartitionPaths`，元数据表扫描）+ `latestCompletedInstant` timeline 读，无 `(table,instant)` 缓存：collectPartitions 支撑 listPartitions/Names/Values（`:707-713`）、applyFilter 非 hive-sync 再列举（`:289-291`）、planScan resolvePartitions 再列举（`:281,648`）；一次 MTMV refresh 内 fe-core 重列举 4-6 次。无 IcebergPartitionCache / hive partitionViewCache 类比。 | 3×/pass + 4-6×/MTMV (A/B) | remote-io | **P1** | no | `HudiScanPlanProvider.java:734` | +| HD-P04 | schema | HMS 元数据走 **RAW** `ThriftHmsClient` — hudi **未包** `CachingHmsClient`（`HudiConnector.createClient:186`；对照 `HiveConnector.wrapWithCache`）。getTableHandle 的 tableExists+getTable（2 RPC，`:204-207`）、hive-sync 下 applyFilter（`:278`）与 collectPartitions（`:695`）各自 listPartitionNames，均每查询重打且跨查询无缓存。缓存类 `CachingHmsClient`（keyed `(db,table)`、24h TTL）已存在且被 hive 使用，hudi 热点却绕过 = Variant C。 | 2+ RPC/pass，跨查询无缓存 (C) | remote-io | P2 | no | `HudiConnector.java:186` | +| HD-P05 | split-enum | 每 scan loop 不变量重算：`storageHadoopConfig(context)`（翻译全部 StorageProperties）+ `buildHadoopConf` 在 planScan（`:912-927`）与 getScanNodeProperties（`:341,363`）各建一遍；storage-URI 归一化 `context.normalizeStorageUri` 逐 base file / 逐 slice 调用（collectCowSplits `:445`、collectMorSplits `:474-476`），未按 iceberg PERF-06 每 scan 提升一次。 | 2×/pass（config）+ O(N_files)（uri）(D) | cpu | P2 | no | `HudiScanPlanProvider.java:912` | + +倍数与 cost 说明：HD-P01/P02/P03 是真远程 IO 且真倍数、真无 memo，属问题类 Variant B（单链重复 k 次）/D（跨入口未提升的 loop-invariant）/A（MTMV 循环放大）；HD-P04 属 Variant C（缓存存在但热点未命中）；HD-P05 只是 CPU/alloc，低危。已 skeptical 排除误报：per-file schemaId resolver 是真·逐文件（各文件自带写入 schema 版本），非可提升不变量，不计入。 + +### 5. 授权 / 一致性 (Authz / consistency) + +- **不 vend 每用户凭证**：hudi 用 catalog 级单一身份 —— plugin 端 Kerberos `doAs`（单 keytab，`buildPluginAuthenticator:223-238`、`metaClientExecutor:102-122`）或 FE 注入的 `context.executeAuthenticated`（sibling 下为 NOOP/SIMPLE）。无 REST-OIDC / `session=user` 类比，无 per-identity metastore。 +- `HudiConnector` 未覆写 `getCapabilities`，继承空默认（无 `SUPPORTS_USER_SESSION`）；hive gateway 前门守卫已保证不接入 session=user sibling（`HiveConnector:548-562` 的 iceberg 守卫，同一契约覆盖 hudi）。 +- **name-only 缓存是 authz-安全的**：所有用户看到同一 catalog-身份视图，与 hive 的共享 `CachingHmsClient` 一致。**不需要** iceberg Layer-3 的 `isUserSessionEnabled` 置空 / `shouldBypassSchemaCache`。 +- **写路径一致性：无**。hudi 只读（`beginTransaction` 抛错 `:395-398`；`getWritePlanProvider` 留 SPI 默认 null），不存在读+写共享 metadata/txn 的需求，无需 write-txn co-holder。 + +### 6. 所需框架部件 (Framework pieces) + +- **per-statement-funnel-memoization = partial**：funnel 路由已在（一语句一 metadata 对象），但缺「语句内 resolved metaClient+schema memo」且 scan provider 独立建 metaClient。需 iceberg PERF-07 式：让 `HudiConnectorMetadata` 与 `HudiScanPlanProvider` 经 `ConnectorStatementScope` 共享同一 resolved metaClient/schema，把 5-6 次塌缩为一次。 +- **cross-query-table-cache = yes**：无 metaClient/table-config 跨查询缓存（IcebergTableCache 类比），按 basePath keyed（TTL + REFRESH 失效），塌缩 HD-P01。 +- **partition-cache = yes**：无 `(table,instant)` 分区缓存（IcebergPartitionCache / hive ConnectorPartitionViewCache 类比），解决 HD-P03（MTMV/SHOW PARTITIONS 重列举）。 +- **format-cache = no**：COW/MOR 从权威 table config/handle 读（`HudiScanPlanProvider:156`），`detectFileFormat` 是廉价后缀判断，无 iceberg getFileFormat whole-table planFiles 回退可 memo。 +- **comment-cache = no**：未覆写 getComment，无 information_schema/SHOW TABLE STATUS 的逐表远程 comment load。 +- **manifest-or-file-list-cache = partial**：planScan 每查询建 FileSystemView + 列 partition path（`:273-281`），无文件列举缓存（hive 有 HiveFileListingCache）；可加，但优先级低于 metaClient/schema/partition memo。 +- **per-scan-hoist = yes**：planScan 与 getScanNodeProperties 各建 metaClient/重解析 schema（HD-P01/P02）；storageHadoopConfig/buildHadoopConf 建两遍、normalizeStorageUri 逐文件（HD-P05）。每 scan 解析一次并复用。 +- **per-file-split-memo = no**：分区值已每分区解析一次并跨文件共享；per-file schemaId 是真·逐文件。仅剩 per-file uri-normalize 归入 per-scan-hoist。 +- **authz-session-user-isolation = no**：非 session=user，name-only 缓存不会泄漏。 +- **shared-fe-connector-cache-adoption = yes**：完全没用共享工具箱，也没包 `CachingHmsClient`。采用之（像 hive 一样 `wrapWithCache` 包 `CachingHmsClient`；分区用 `ConnectorPartitionViewCache`）是 HD-P03/P04 最低风险的底座，且与参考模式一致。 +- **write-txn-coholder = no**：只读。 + +### 7. 结论与建议优先级 (Verdict + recommendations) + +hudi 已 LIVE（hms sibling），且是 SPI 连接器中缓存最薄的一个：**零** cross-query 元数据缓存、**零** per-statement metaClient/schema memo，甚至未包 `CachingHmsClient`。它已正确接入 Layer-2 funnel，但 funnel 只塌缩「元数据对象」，重远程加载（metaClient 5-6×/pass、schema ~4×、分区元数据表扫描 3×/pass 且 4-6×/MTMV）在每个入口重复。iceberg 框架模式**直接适用且必要**。建议按序： + +1. **per-statement resolved-metaClient+schema memo**（metadata 与 scan provider 共享）——消灭 HD-P01/HD-P02，旗舰改动（iceberg PERF-07/PERF-01）。 +2. **把 HMS client 包进 `CachingHmsClient`**——与 hive 对齐的一行改动，立即修 HD-P04（Variant C）。 +3. **cross-query metaClient/table 缓存 + `(table,instant)` 分区缓存（采用 `ConnectorPartitionViewCache`）**——修 HD-P03（MTMV/SHOW PARTITIONS 重列举）。 +4. **per-scan 提升** metaClient/schema/storage-config/uri-normalizer（HD-P05）。 + +**不需要** authz 隔离（非 session=user）、**不需要** write-txn 工作（只读）。附带：清理散落的 stale `"dormant until hms enters SPI_READY_TYPES"` 注释。 + +--- + +## 复核结论 (adversarial verify) + +**Verdict: ADJUSTED** · 确认发现 IDs: HD-P01, HD-P02, HD-P03, HD-P04, HD-P05 + +| 原始声明 | 问题 | 更正 | 证据 | +|---|---|---|---| +| authzConsistency: 'the hive gateway front-door guard already enforces that no session=user sibling can attach (HiveConnector:548-562, mirrored contract)' covering hudi. | The SUPPORTS_USER_SESSION fail-loud guard at HiveConnector.java:548-556 lives inside getOrCreateIcebergSibling and its error text is iceberg-specific ('iceberg-on-HMS sibling ... unexpectedly declares SUPPORTS_USER_SESSION'). getOrCreateHudiSibling (:576-591) has NO equivalent guard, so the '548-562 mirrored contract' does not literally cover the hudi sibling. | hudi's authz-safety instead rests on two verified facts: (a) HudiConnector does not override getCapabilities, so it inherits the empty default with no SUPPORTS_USER_SESSION (confirmed — the only getCapabilities hits are javadoc in HudiConnectorMetadata, no override); and (b) the hive front door itself is never session=user. The audit's substantive conclusion (name-only caches are authz-safe, no Layer-3 isolation / shouldBypassSchemaCache needed) is UNCHANGED and correct. | | +| HD-P03 multiplier: '3x within one pass' partition metadata-table scans for a filtered partitioned SELECT. | For a FILTERED query the code deliberately dedups to ~1 listing: applyFilter (non-hive-sync) lists once at HudiConnectorMetadata.java:289-291, then resolvePartitions returns the prunedPaths and short-circuits WITHOUT calling listAllPartitionPaths (HudiScanPlanProvider.java:635-638). The connector's own comment at HudiConnectorMetadata.java:288 states 'a filtered query lists exactly once (here) instead of there.' So '3x within one pass' is an upper bound, not the single-pass norm. | The '3x' only materializes when fe-core independently invokes multiple of listPartitions / listPartitionNames / listPartitionValues (each re-calls collectPartitions -> a fresh HoodieTableMetadata scan, HudiConnectorMetadata.java:707-713) and/or on an UNFILTERED scan where resolvePartitions does list (HudiScanPlanProvider.java:648). The CORE finding — a metadata-table partition scan re-run at multiple entrypoints with NO (table,instant)-keyed cache — is CONFIRMED; only the per-pass count is query-shape-dependent. | | +| HD-P01 multiplier '~5-6x per planning pass' and HD-P05 citation 'planScan.buildHadoopConf (:912-927)'. | Several metaClient build sites are conditional: applyFilter builds one only when a partition predicate is present AND non-hive-sync (:289-291); getScanNodeProperties builds one only when !force_jni (:363); the 3-arg MVCC getTableSchema fires only on the time-travel/MVCC path. HD-P05's '912-927' is the buildHadoopConf METHOD definition, not the planScan call site (the actual call is planScan:147). | Minimum unconditional metaClient builds per pass are ~3-4 (getColumnHandles->getSchemaFromMetaClient :800-801, beginQuerySnapshot->latestInstant :749-750, planScan :148-151, getScanNodeProperties :363), reaching 5-6 with the conditional sites — so '5-6x' is a valid upper bound. HD-P05 substance (buildHadoopConf/storageHadoopConfig built twice per pass at :147 and :363; normalizeNativeUri per-file at :445 and :476) is correct; only the line citation is loose. Neither adjustment weakens the 'not memoized' core. | | + +> 复核备注：Verified against HEAD aaab68ef474. All MATERIAL claims hold; corrections are precision-only and do not overturn any finding. + +MIGRATION STATUS — CONFIRMED. SPI_READY_TYPES = {jdbc, es, trino-connector, max_compute, paimon, iceberg, hms} (CatalogFactory.java:56-57), with the :50-55 comment explicitly forbidding 'hudi'. HudiConnectorProvider.getType() returns sibling-only 'hudi' (:45-47). Live path wired: HiveConnector.newMetadata passes this::getOrCreateHudiSibling (:190); HiveConnectorMetadata.getTableHandle diverts HUDI to hudiSiblingMetadata(session).getTableHandle (:415-416); getOrCreateHudiSibling builds via createSiblingConnector (:576-591). Legacy fe-core HudiScanNode/HudiExternalMetaCache/HMSExternalTable/HudiDlaTable ALL removed (find returns none) — confirms live, not dormant. Stale-comment conflict is REAL and correctly surfaced: 'dormant until hms enters SPI_READY_TYPES' / 'today getTableHandle is never called' present at HiveConnectorMetadata.java:410,1802,1942,1972,2007 and HudiConnectorMetadata.java:391 — all stale since hms IS in the set. + +CURRENT CACHES — all 4 CONFIRMED. (1) per-statement funnel memo via getOrCreateMetadata key 'metadata:'+catalogId+':'+HUDI_LABEL (HiveConnectorMetadata.java:302-304, hudiSiblingMetadata :332-333, by-handle siblingMetadata :347). (2) RAW ThriftHmsClient — HudiConnector.createClient returns new ThriftHmsClient (:186), NOT wrapped in CachingHmsClient; contrast HiveConnector which imports+uses CachingHmsClient/wrapWithCache (CachingHmsClient.java exists in fe-connector-hms). (3) pluginAuth double-checked memo (:196-206). (4) hudi-lib InternalSchemaCache.searchSchemaAndCache (HudiSchemaUtils.java:278). + +FUNNEL PARTICIPATION — CONFIRMED. Exactly one HudiConnectorMetadata per statement, but it memoizes NOTHING: fields are only {hmsClient:153, properties:154, metaClientExecutor:156, storageHadoopConfig:161} — grep for any HoodieTableMetaClient/InternalSchema/Schema/HoodieTimeline/partition/schema FIELD returns only method signatures, no cache field. Scan provider is a separate object (HudiConnector.getScanPlanProvider returns new HudiScanPlanProvider, :136-138) building its own metaClient in planScan (:148-151) and getScanNodeProperties (:363). + +HOT-PATH FINDINGS — all 5 independently CONFIRMED as real heavy remote-IO ops with genuinely NO memoization (no pre-existing cache catches them): HD-P01 (HoodieTableMetaClient.build reads .hoodie/hoodie.properties + loads active timeline, per line 916-917 comment; rebuilt at every metadata+scan entrypoint), HD-P02 (getTableAvroSchema(true) + resolveTableInternalSchema re-resolved at getSchemaFromMetaClient:819-821, planScan:188/194/220, getScanNodeProperties:382-383), HD-P03 (listAllPartitionPaths = HoodieTableMetadata.create+getAllPartitionPaths at :734-742, no (table,instant) cache), HD-P04 (raw ThriftHmsClient, Variant C — CachingHmsClient exists+used by hive but hudi bypasses it), HD-P05 (buildHadoopConf/storageHadoopConfig 2x + normalizeNativeUri per-file, CPU-only, correctly rated P2). The audit correctly EXCLUDED false positives (per-file schemaIdResolver at :232-241 is genuinely per-file, not a hoistable invariant; format-cache/comment-cache not applicable — getTableType is authoritative at :156, detectFileFormat is a cheap suffix check at :899-910, no getComment override). No severe finding was understated and no finding is covered by an existing cache. Overall verdict (hudi is the least-cached SPI connector; iceberg framework pattern applies and is needed) STANDS. diff --git a/plan-doc/connector-cache-unification/connectors/jdbc.md b/plan-doc/connector-cache-unification/connectors/jdbc.md new file mode 100644 index 00000000000000..24fab9d705b80f --- /dev/null +++ b/plan-doc/connector-cache-unification/connectors/jdbc.md @@ -0,0 +1,90 @@ +## 连接器附录:jdbc(fe-connector-jdbc,catalog type `jdbc`) + +### 结论速览 +`jdbc` 是**关系型透传(relational passthrough)**连接器:FE 只负责元数据发现 + 生成远程 SQL,真正的数据扫描由 BE 的 `JdbcJniReader` 直接对远端库执行。它**没有** iceberg 那套 split / file / manifest / partition / snapshot 规划层,所以 iceberg 热路径缓存框架(cross-query Table/Partition/Format/Manifest cache)对它基本无对应目标。其昂贵远程元数据早已被 **fe-core 跨查询缓存**前置,连接池是 **per-catalog 单例**。审计结论:**不套用 iceberg 式连接器缓存**;只存在两处 P2 级冗余远程取列,适合用轻量 per-statement memo 收口。 + +Migration:`jdbc` ∈ `SPI_READY_TYPES`(`CatalogFactory.java:57`),`JdbcConnectorProvider.getType()=="jdbc"`(`JdbcConnectorProvider.java:41`),经 SPI 路由到 `PluginDrivenExternalCatalog`。legacy fe-core `JdbcExternalCatalog/JdbcExternalTable` 已删;残留 fe-core `datasource/jdbc/client/JdbcClient` 仅服务 CDC/streaming-job 校验,不在 catalog 查询热路径。**LIVE**。 + +### 缓存与记忆一览 + +| 缓存 / 记忆 | 层 | scope | key | TTL / 失效 | file:line | 前置了哪个连接器远程调用 | +|---|---|---|---|---|---|---| +| HikariDataSource 连接池 | 连接器 | cross-query(per-catalog 单例) | 每 client 一池 | connector 生命周期,`close()` 销毁 | `JdbcConnectorClient.java:89` / `JdbcDorisConnector.java:188` | 所有远程 round-trip 的实际资源复用点 | +| CLASS_LOADER_MAP 驱动 classloader | 连接器 | 进程级 cross-query | driver URL | 永不淘汰(986696 Metaspace 泄漏修复) | `JdbcConnectorClient.java:78,268` | 驱动加载 | +| OceanBase compat-mode delegate | 连接器 | cross-query(per-client volatile) | 单 delegate | client 生命周期 | `JdbcOceanBaseConnectorClient.java:45,59` | `ob_compatibility_mode()` 方言探测(一次) | +| ExternalSchemaCache | fe-core | cross-query | `SchemaCacheKey(table)` | expire-after-access + refresh/DDL | `ExternalMetaCacheMgr.java:365` / `ExternalTable.java:447` | `getTableSchema`(via `initSchema`,`PluginDrivenExternalTable.java:469`) | +| ExternalRowCountCache | fe-core | cross-query | `RowCountKey(catalog,db,table)` | expireAfterWrite ~1 天,异步 | `ExternalRowCountCache.java:45` | `getTableStatistics`(via `fetchRowCount`,`:1133`) | +| MetaCache 名录/对象 | fe-core | cross-query | db 名 / (db,table) 名 | expire-after-access + refresh | `metacache/MetaCache.java:65` | `listDatabaseNames`/`listTableNamesFromRemote`(`PluginDrivenExternalCatalog.java:293,310`) | + +**关键事实**:连接器模块(`fe-connector-jdbc`)自身**没有任何元数据缓存**(grep `Cache/memo/Caffeine/LoadingCache` 在连接器代码零命中)。上表前三行是资源/方言单例,后三行是 fe-core 缓存。`JdbcConnectorMetadata` 无状态,每次调用都原样转发 `JdbcConnectorClient` 做一次新的 JDBC 元数据 round-trip(`JdbcConnectorMetadata.java:119/162/144`)。 + +### Funnel 参与 + +- **已 funnel**:所有 fe-core 读 seam 经 `PluginDrivenMetadata.get` 取 metadata(`initSchema:449`、`fetchRowCount:1126`、`getComment:923`、`PluginDrivenScanNode` 的 `metadata()`、`JdbcQueryTableValueFunction:53`)。 +- **metadata 不 memo loaded table**:`JdbcConnectorMetadata` 只持 `client+properties` 引用,`getTableSchema/getColumnHandles/getTableStatistics` 每次都远程取,内部无 memo。JDBC 也**没有**可缓存的昂贵 `Table` 对象。 +- **funnel 对 JDBC 收益有限**:funnel 保证「每语句一个 metadata 实例」,但该实例构造近乎零成本,真正昂贵的连接池已是 per-catalog 单例;funnel 唯一有用之处是提供了一个活的 per-statement scope,可用来 memo `getColumnHandles`——但目前没用上。cross-statement 缓存加载器(`buildCrossStatementSession`,`:1153`)显式用 `ConnectorStatementScope.NONE`,把 schema/rowcount/名录的跨查询缓存契约固化,合理。 + +### 热路径重复取数审计 + +| # | 入口 | 重复取的信息 | 倍率 | 代价 | 已 memo | 严重度 | file:line | +|---|---|---|---|---|---|---|---| +| HP-1 | scan 规划 `buildColumnHandles` | `getColumnHandles`→`getJdbcColumnsInfo`(远程 `DatabaseMetaData.getColumns`;MySQL 追加 information_schema) | 每 scan node ~1-2 次(getSplits/startSplit 二选一 + getOrLoadPropertiesResult) | remote-io | 否 | **P2** | `JdbcConnectorMetadata.java:162`;`PluginDrivenScanNode.java:1917` | +| HP-2 | write 整形 `buildInsertSql` | `new JdbcConnectorMetadata(...)`(**绕开 funnel**)→`getColumnHandles`→远程取列 | 每 INSERT 1 次;EXPLAIN INSERT 2 次 | remote-io | 否 | **P2** | `JdbcWritePlanProvider.java:136` | +| HP-3 | SHOW/DESC `getComment` | `getTableComment` 无缓存直连(MySQL 远程 INFORMATION_SCHEMA) | 每 SHOW/DESC 1 次(非查询规划热路径) | remote-io | 否 | P2 | `PluginDrivenExternalTable.java:925` | +| HP-4 | split 枚举 `planScan` | (阴性)零远程 IO,恒 1 个 scan range,纯 SQL 拼装 | 1(常量) | cpu | 是 | none | `JdbcScanPlanProvider.java:66,152` | +| HP-5 | row-count | (阴性)`getTableStatistics` 被 ExternalRowCountCache 吸收;base 返回 -1 | 缓存命中后 0 | remote-io | 是 | none | `PluginDrivenExternalTable.java:1133` | + +**分析**:HP-1/HP-2 是同一根问题的两面——列句柄解析(local→remote 名映射)在扫描期和写整形期各自重新远程取列,而这些列名+remote 名早已随 `getTableSchema` 进了 `ExternalSchemaCache`(两者底层同为 `getJdbcColumnsInfo`)。这是 problem-class 的 **variant C(缓存存在但热路径绕过)**。但要克制:`getJdbcColumnsInfo` 是一次「cheap-ish」的 JDBC 元数据 round-trip,倍率是 per-scan-node / per-write(非 per-split/per-file——JDBC 恒 1 split),绝对成本与放大都远小于 iceberg 的 manifest/planFiles。故 **P2**。HP-2 额外坏在自构实例、连 funnel 记忆的实例都不复用。 + +### Authz / 一致性 + +JDBC 用 **catalog 固定 user/password**(`JdbcConnectorProperties.USER/PASSWORD`),烘焙进 HikariCP 与 BE 的 `TJdbcTable`。**无 per-user 凭证、无 SUPPORTS_USER_SESSION**(`JdbcDorisConnector.java:102-105 只有 PASSTHROUGH_QUERY + METADATA_PRELOAD`),无 kerberos doAs / REST OIDC / per-identity metastore。所有身份共享同一远程用户 → name-only 缓存**不会**泄漏 can-LIST-cannot-LOAD 式 per-user 元数据,不需要 iceberg 的 `isUserSessionEnabled()` 缓存置空或 `shouldBypassSchemaCache` 分片。写侧 BE 逐行 auto-commit、`beginTransaction` 返回 `NoOpConnectorTransaction`(`JdbcConnectorMetadata.java:286`),读写无需共享 txn/snapshot。 + +### 框架各件需求判定 + +| 框架件 | 需要? | 理由 | +|---|---|---| +| per-statement-funnel-memoization | **partial** | 唯一真适用件:HP-1/HP-2 的 `getColumnHandles` 冗余远程取列。补救 = 在活 scope 上 memo,或(更优)让 fe-core `buildColumnHandles` 直接从已缓存的 `PluginDrivenSchemaCacheValue` 派生列句柄。P2,低优先。 | +| cross-query-table-cache | no | 无昂贵 loaded Table;schema/rowcount/名录已由 fe-core 跨查询缓存前置。 | +| partition-cache | no | 无分区,恒 1 scan range。 | +| format-cache | no | 无文件格式解析(关系型)。 | +| comment-cache | no | HP-3 存在但仅 SHOW/DESC,非查询热路径,价值边际。 | +| manifest-or-file-list-cache | no | 无 manifest/文件清单;无 file-list row-count 路径。 | +| per-scan-hoist | no | planScan 已 O(1) 零远程 IO,无 loop-invariant 可外提。 | +| per-file-split-memo | no | 恒 1 split,无 per-file 不变量。 | +| authz-session-user-isolation | no | 固定凭证,name-only 缓存不泄漏。 | +| shared-fe-connector-cache-adoption | no | 连接器无跨查询元数据缓存可迁移。 | +| write-txn-coholder | no | BE auto-commit,无 FE 读写共享 txn。 | + +### 优先级建议 + +1. **(可选,P2)** 收口 HP-1/HP-2 的列句柄冗余远程取:最干净的做法是**在 fe-core 让 `buildColumnHandles`/写路径从已跨查询缓存的 `PluginDrivenSchemaCacheValue`(已含 columns + remote 名)派生列句柄**,对所有连接器都零远程 IO——这是框架级小改,而非 jdbc 专属缓存;退一步可在 `JdbcConnectorMetadata` 上按 `session.getStatementScope()` memo `getJdbcColumnsInfo` 结果。收益小、风险低,非阻塞。 +2. **不新增任何 iceberg 式连接器缓存**(Rule 2 简单优先):JDBC 无对应的重复 loadTable/manifest/planFiles 目标,新增缓存纯属投机且会引入与固定凭证无关的失效/一致性负担。 +3. HP-3(comment)可暂不处理——off hot-path。 + +Confidence:high(全部锚定 HEAD file:line;e2e 未跑,判定基于代码结构与调用图)。 + +--- + +## 复核结论 (adversarial verify) + +**Verdict: CONFIRMED** · 确认发现 IDs: HP-1, HP-2, HP-3, HP-4, HP-5 + +| 原始声明 | 问题 | 更正 | 证据 | +|---|---|---|---| +| jdbc 在 CatalogFactory.SPI_READY_TYPES 中 (CatalogFactory.java:57) | Line-number anchor slightly off. The claim itself is TRUE, but the ImmutableSet containing "jdbc" is not at line 57. | At HEAD the declaration `private static final Set SPI_READY_TYPES =` is on line 56 and the `ImmutableSet.of("jdbc", "es", "trino-connector", "max_compute", "paimon", "iceberg", "hms")` literal is on line 63-64 of fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java. Membership is confirmed either way. | | +| ExternalSchemaCache TTL = expire-after-access(external_cache_expire_time_minutes_after_access) | Config key name imprecise. | The actual Config field seen at ExternalRowCountCache.java:46 is `external_cache_expire_time_seconds_after_access` (seconds, not minutes) and `external_cache_refresh_time_minutes`. This is a cosmetic naming imprecision; the expire-after-access + refresh semantics claimed are correct. | | +| ExternalSchemaCache file: ExternalMetaCacheMgr.java:365 | Anchor points to the accessor, not a distinct cache class. | Line 365 of ExternalMetaCacheMgr.java is `public Optional getSchemaCacheValue(ExternalTable table, SchemaCacheKey key)` — the fe-core schema-cache accessor. Valid anchor for the mechanism; the schema-cache front-loading of getTableSchema is real and confirmed (initSchema at PluginDrivenExternalTable.java:449→getTableSchema :469). | | + +> 复核备注：All material claims hold at HEAD (aaab68ef474). Verified: (1) Migration — jdbc live/SPI-routed, legacy fe-core Jdbc catalog/table deleted, fe-core JdbcClient only for CDC/streaming validators. (2) All 6 caches exist at ~cited locations, no invented classes: CLASS_LOADER_MAP (JdbcConnectorClient.java:78), HikariDataSource per-catalog (:89 field/:220 new/getOrCreateClient :188 single-client guard), OceanBase volatile delegate double-checked (JdbcOceanBaseConnectorClient.java:45/59), fe-core ExternalSchemaCache/ExternalRowCountCache(:40-45 AsyncLoadingCache)/MetaCache(:65). (3) Funnel: JdbcConnectorMetadata is stateless-forwarding (getTableSchema:119, getTableStatistics:144, getColumnHandles:162, getTableComment:252 all fresh client round-trips) — metadataMemoizesLoadedTable=no CONFIRMED; write path new JdbcConnectorMetadata at JdbcWritePlanProvider.java:136 bypasses funnel CONFIRMED. + +Hot-path findings independently confirmed: +- HP-1 CONFIRMED: buildColumnHandles() (PluginDrivenScanNode.java:1916) → metadata.getColumnHandles → client.getJdbcColumnsInfo (real remote DatabaseMetaData.getColumns at JdbcConnectorClient.java:408). Callers: getSplits(:1263), startSplit(:1587,:1672), getOrLoadPropertiesResult(:1841) → 1-2 remote round-trips per scan node, NOT memoized, distinct from ExternalSchemaCache-fronted getTableSchema path. Variant C. Multiplier accurate. +- HP-2 CONFIRMED: buildInsertSql (JdbcWritePlanProvider.java:130) constructs `new JdbcConnectorMetadata(client, properties).getColumnHandles(...)` at :136 — fresh instance bypassing funnel + remote refetch. Called from planWrite(:68) and appendExplainInfo(:119). +- HP-3 CONFIRMED (minor/off-hot-path, as audit states): getComment (PluginDrivenExternalTable.java:925) → metadata.getTableComment uncached; base returns "" but MySQL override does remote INFORMATION_SCHEMA.TABLES query (JdbcMySQLConnectorClient.java:165). Only SHOW/DESC path. +- HP-4 CONFIRMED (negative control): planScan (JdbcScanPlanProvider.java:75) is pure SQL-string assembly + property reads, zero client/remote calls, returns Collections.singletonList(scanRange); estimateScanRangeCount()==1 (:153). No split/file/partition loop. +- HP-5 CONFIRMED (negative control): base getRowCount returns -1 (:467), MySQL override remote (:191); fronted cross-query by ExternalRowCountCache.loadRowCount→table.fetchRowCountWithMetaCache (ExternalRowCountCache.java:90-94). + +Authz CONFIRMED: getCapabilities = EnumSet.of(SUPPORTS_PASSTHROUGH_QUERY, SUPPORTS_METADATA_PRELOAD) (JdbcDorisConnector.java:98-105), no SUPPORTS_USER_SESSION; buildConnectorSession injects credential only withUserSessionCapability(supportsUserSession()) which is false for jdbc; shouldBypassSchemaCache/shouldBypassTableNameCache gated on same (PluginDrivenExternalCatalog.java:1177-1212). Fixed catalog user/password → no per-user metadata divergence, no leak. beginTransaction returns NoOpConnectorTransaction (:286). + +No missed severe finding: no per-split/per-file/per-partition amplification exists (structurally 1 scan range), names via MetaCache, schema/rowcount cross-query cached — audit's max severity P2 is appropriate. frameworkPiecesNeeded (partial funnel-memoization, no to all iceberg-style caches) and overallVerdict (do not apply iceberg cache framework, Rule 2) are well-supported. Corrections listed are cosmetic line-anchor/config-name nits that do not affect any conclusion. diff --git a/plan-doc/connector-cache-unification/connectors/maxcompute.md b/plan-doc/connector-cache-unification/connectors/maxcompute.md new file mode 100644 index 00000000000000..19d33b38b4c9ea --- /dev/null +++ b/plan-doc/connector-cache-unification/connectors/maxcompute.md @@ -0,0 +1,95 @@ +## 连接器附录：maxcompute（`fe-connector-maxcompute`，catalog type `max_compute`） + +### 1. 迁移状态（Migration status） + +`max_compute` **已在** `SPI_READY_TYPES`（`CatalogFactory.java:57`），因此 MaxCompute catalog 走 SPI plugin 路径，被包成 `PluginDrivenExternalCatalog` + `MaxComputeDorisConnector`（`CatalogFactory.java:110-118`）。fe-core 中已无任何遗留 MaxCompute 类（`find fe/fe-core -iname '*MaxCompute*'` 为空）；仅剩显示兼容用的 engine-name switch（`PluginDrivenExternalTable.java:1227-1230,1260-1261`）与 GSON 迁移引用。**读路径完全翻闸、live。** 写路径也已 live 接线（`PhysicalPlanTranslator.visitPhysicalConnectorTableSink` → `getWritePlanProvider(handle)` → `planWrite`）；`MaxComputeConnectorMetadata.beginTransaction:332` / `MaxComputeWritePlanProvider:74` 里"gate-closed / dormant until cutover"的 javadoc 相对 HEAD 的 `SPI_READY_TYPES` **已过时**。 + +liveness = **live**。 + +### 2. 现有缓存（Current caches） + +| 缓存 | 作用域 | Key | TTL / 容量 | 失效 | file:line | +|---|---|---|---|---|---| +| `MaxComputePartitionCache` | cross-query | `(db, table)`；project 每 catalog 恒定，不入 key（`PartitionKey:138`） | 600s / 10000，可经 `meta.cache.max_compute.partition.*` 覆盖（`CacheSpec.fromProperties:92`） | REFRESH TABLE/DB/CATALOG + 分区增删 → 整表 flush（`invalidateTable:113`/`invalidateDb:118`/`invalidateAll:123`；connector 钩子 `MaxComputeDorisConnector.java:189/198/204/213`） | `MaxComputePartitionCache.java:60-173`；`final` 挂在长寿命 connector 上（`MaxComputeDorisConnector.java:61,79-80`） | +| ODPS `Table` 对象内 lazy memo | per-scan | 单个 handle 携带的一个 `com.aliyun.odps.Table`（transient，`MaxComputeTableHandle.java:37`） | handle 生命周期；首次访问 `reload()` 后 in-object 缓存 | 随 handle GC | `getTableHandle:124`（`getOdpsTable` 惰性、无 RPC）；首次元数据访问 reload（`MaxComputeScanPlanProvider.java:187,189,194`）。**不跨 handle 共享** | +| fe-core `SchemaCacheValue` | cross-query | fe-core ExternalTable schema 缓存（每表） | fe-core 外表元缓存 TTL / REFRESH | REFRESH | `PluginDrivenExternalTable.initSchema:430-471`；`getTableSchema` 每 schema-cache 刷新一次，非每查询 | +| `PluginDrivenScanNode.cachedMetadata` + `resolvedScanProvider` | per-scan | 每 scan node；metadata 按 catalogId，provider 按 handle identity | scan node 生命周期 | — | `PluginDrivenScanNode.java:189,184,245-260`（fe-core 通用设施） | +| `EnvironmentSettings`/`SplitOptions`/providers | cross-query | 每 connector 一次性 lazy init（配置，非远端数据） | connector 生命周期 | — | `MaxComputeDorisConnector.doInit:94-128`；`MaxComputeScanPlanProvider.initFromProperties:117-161` | + +连接器**没有**：跨查询 table/schema 缓存（依赖 fe-core `SchemaCacheValue`）、per-statement handle memo、format 缓存（N/A）、comment 缓存（default 空）、manifest/file-list 缓存（N/A）。分区缓存用的是共享 `fe-connector-cache`（`CacheSpec`+`MetaCacheEntry`），但未用更高层的 `ConnectorPartitionViewCache`/`PartitionViewCacheKey`（那是分区派生视图的另一议题）。 + +### 3. Funnel 参与（Layer-2） + +**已接入 funnel：是。** `MaxComputeConnectorMetadata` 只经 `MaxComputeDorisConnector.getMetadata(session)`（`MaxComputeDorisConnector.java:177-181`）产生，而它被 `PluginDrivenMetadata.get` → `scope.getOrCreateMetadata("metadata:"+catalogId)`（`PluginDrivenMetadata.java:70`）按 (statement, catalog) memo，故一条语句只有一个 `MaxComputeConnectorMetadata`（由 `tools/check-fecore-metadata-funnel.sh` 全构建强制）。 + +**但该 metadata 不 memoize 已加载表：否。** `getTableHandle`（`MaxComputeConnectorMetadata.java:118-130`）每次都重跑 `structureHelper.tableExist`（line 121 → ODPS `tables().exists()` 远程探测，`McStructureHelper.java:129-137/218-225`）**并**新建一个惰性 `Table`（line 124）。`getMetadata` 本身极廉价（只包已 init 的 `odps`/`structureHelper`/`partitionCache`），所以对 MaxCompute 而言 funnel 的 metadata memo **省不掉任何远程 IO**。funnel 收敛的是"metadata 构造"，不是"表解析"——metadata 内无 `(db,table)→handle` 映射，于是一条语句里 ~13 个 `resolveConnectorTableHandle` 站点（`PluginDrivenExternalTable.java:160,463,607,717,821,882,955,1026,1061,1128` + `resolveWriteCapabilityHandle` 205/222/375/410）与 translator 的 `getTableHandle`（`PhysicalPlanTranslator.java:624,676`；`BindSink.java:675,714`）各自付一次 `exists()` 探测；每个访问 schema 的路径各 reload 一次新 `Table`。**这正是 iceberg PERF-07 的缺口，在 MaxCompute 上尚未处理。** + +metadataMemoizesLoadedTable = **no**。 + +### 4. 热路径重复加载审计（问题类应用） + +| # | 区域 | 问题 | 乘数 | 成本 | 严重度 | 已 memo | file:line | +|---|---|---|---|---|---|---|---| +| MC-1 | schema | `getTableHandle` 每次解析都发一次**冗余**的 ODPS `tables().exists()` 远程探测（`:121`→`McStructureHelper.tableExist`），且 connector metadata 与 funnel 均不 memo handle。一条语句在多个独立站点（scan 翻译、分区裁剪、row-count、逐列 stats、写能力探测）各解析一次 handle，各付一次 `exists()`。表本就是已解析的 catalog `ExternalTable`，此探测纯属浪费（变体 B 单链重复 k 次 + C funnel 只 memo metadata 不 memo handle）。 | k = 每语句触达的解析站点数；暖 fe-core stats 缓存下的分区 SELECT ≈2（scan create + `getNameToPartitionItems`），冷 stats 缓存/写路径更高 | remote-io | **P1** | 否 | `MaxComputeConnectorMetadata.java:118-130`；`McStructureHelper.java:129-137,218-225` | +| MC-2 | schema | handle 不按语句 memo → 每个新 handle 携带各自未加载的 `Table`，每个访问 schema 的路径（`planScan` 的 `checkOperationSupported`/`getFileNum`/`getSchema`，冷 schema-cache 时的 `initSchema.getTableSchema`）各触发一次 `Table.reload()`。放大温和（多数非 scan 路径只用 db/table 名，不碰 `Table` 对象），约每语句 1–2 次 reload，而非 iceberg 的 3–7×（变体 B）。 | ~1–2×/语句 | remote-io | P2 | 是（per-handle in-object，不跨 handle 共享） | `MaxComputeConnectorMetadata.java:124`；`MaxComputeScanPlanProvider.java:183-194` | +| MC-3 | split-enum | **非问题/已干净**：split 枚举无 per-split 远程放大。读会话只建一次（`buildBatchReadSession`，规划固有成本），循环前只序列化一次（`serializeSession:329`），per-split 循环仅构造 `MaxComputeScanRange` 并按引用共享同一序列化串（`337-372`）。`getSplits` 只调一次 `planScan`。 | 1×（无放大） | cpu | none | 是 | `MaxComputeScanPlanProvider.java:326-377` | +| MC-4 | partitions | **数据侧非问题，handle 侧并入 MC-1**：分区列举（`listPartitions`/`listPartitionNames`/`listPartitionValues` + fe-core `getNameToPartitionItems`/`getNameToPartitionValues`）由跨查询 `MaxComputePartitionCache`（TTL 600s）兜住，per-table `getPartitions()` 不会每查询重发；仅 `getNameToPartitionItems` 里的 `getTableHandle` `exists()` 探测未缓存（已计入 MC-1）。`listPartitions` 故意忽略下推 filter（与遗留 SHOW PARTITIONS 一致）——全量列举但已缓存。 | 每 TTL 窗口每 (db,table) 1 次远程 `getPartitions` | remote-io | none | 是 | `MaxComputeConnectorMetadata.java:239-296`；`MaxComputePartitionCache.java:107` | + +### 5. Authz / 一致性 + +- **perUserCredential = false，sessionUserRelevant = false。** MaxCompute 用 **catalog 级静态凭证**（AK/SK、RAM-Role-ARN、ECS-RAM-Role），CREATE CATALOG 时固化，`doInit` 里一次性建客户端（`MaxComputeDorisConnector.java:102`；`MCConnectorClientFactory.java:86-127`）。无 per-user 凭证下发、无 session=user、无 REST/OIDC 会话凭证、无 kerberos doAs、无 per-identity metastore——一个 catalog 的所有用户共享同一 ODPS 身份。 +- 因此 **Layer-3 授权缓存隔离不适用**：name-only 的 `MaxComputePartitionCache`（db+table）不会泄漏 can-LIST-cannot-LOAD 用户的元数据；MC-1 的 per-statement handle memo 同样授权安全（一语句=一身份，已由 `PluginDrivenMetadata.java:63-69` 钉住）。`tools/check-authz-cache-sharding.sh` 适用但 MaxCompute 天然合规。 +- 元数据陈旧有界且正确性安全：分区新增 ≤600s 内可见（与遗留 TTL 一致），删除至多陈旧一个 TTL，miss 时自愈。 +- **写一致性**：MaxCompute **非 MVCC**（用基类 `PluginDrivenExternalTable`，非 `MvccTable`），无需 read+write snapshot 对齐；写正确性只需把 ODPS 写会话绑到 per-statement `MaxComputeConnectorTransaction`（`setWriteSession`）、block 分配按 `txn_id`（`MaxComputeConnectorTransaction.java:61-131`）——已具备。 + +### 6. 框架各件是否需要 + +| 件 | 需要 | 理由 | +|---|---|---| +| per-statement-funnel-memoization | **yes（当前 partial）** | 最高价值。funnel 已给每语句一个 metadata，但 `getTableHandle` 每次重探存在性+新建惰性表。在 per-statement `MaxComputeConnectorMetadata` 实例里加 `Map<(db,table),MaxComputeTableHandle>`，让 `getTableHandle` 整条语句返回同一 handle（一次 `exists()`、一次 `reload()`），一举收敛 MC-1、MC-2。比 iceberg 的 scope-keyed load memo 更简单——metadata 实例本就 per-statement。并顺手去掉已解析读的冗余 `tableExist()` 探测。 | +| cross-query-table-cache | partial | fe-core `SchemaCacheValue` 已跨查询缓存 schema，跨查询主需求已覆盖。连接器侧 cached ODPS `Table`（iceberg `IcebergTableCache` 类比，24h/REFRESH）可再省每查询 `planScan` reload（MC-2 跨查询维度），但有陈旧/TTL 代价、收益温和。可选、优先级低于 per-statement memo。 | +| partition-cache | already-has | `MaxComputePartitionCache` 已是完整跨查询分区缓存（db+table，TTL 600s，容量 10000，REFRESH 失效），MC-4 证实其兜住热点分区读。 | +| format-cache | no | N/A：MaxCompute 走 ODPS Storage API（arrow），无 per-table 文件格式推断（无 `getFileFormat`/`planFiles` 兜底），iceberg PERF-03 不存在。 | +| comment-cache | no | 未 override `getTableComment`，SPI default 返回 `""`（`ConnectorTableOps.java:336`）无远程，无需缓存。 | +| manifest-or-file-list-cache | no | N/A：无 manifest/snapshot 模型；读会话/split assigner 是每查询规划固有成本；`estimateDataSizeByListingFiles`/`listFileSizes` 未 override（default -1/空）。 | +| per-scan-hoist | already-has | scan 不变配置（`EnvironmentSettings`/`SplitOptions`/超时）已提到 connector/provider 一次性 lazy init；序列化会话在 split 循环前算一次。iceberg PERF-06 per-scan storage-URI normalizer 不适用（无 per-file 下发存储）。 | +| per-file-split-memo | no | N/A：split 只带 index/row-offset + 一份共享序列化会话串（已按引用共享），无 per-file 分区 JSON/值/删除载体需 memo。 | +| authz-session-user-isolation | no | catalog 静态凭证、单一共享 ODPS 身份；name-only 缓存不会跨用户泄漏或供陈旧授权，无需 iceberg 的 `isUserSessionEnabled` 置空/`shouldBypassSchemaCache`。 | +| shared-fe-connector-cache-adoption | already-has | `MaxComputePartitionCache` 基于共享 `CacheSpec`+`MetaCacheEntry`（`:20-21,92-97`），镜像 `CachingHmsClient`/`HiveFileListingCache`。 | +| write-txn-coholder | already-has | `MaxComputeConnectorTransaction` 持 per-statement 写态（写会话/标识/settings/`TMCCommitData`/block 高水位），`planWrite.setWriteSession` 绑定、`txn_id` 提交；非 MVCC 无需 snapshot 共享；funnel 的一语句一写事务已协调。 | + +### 7. 结论与优先建议 + +**结论：live，且已相当好地缓存——不是 iceberg 那样的 P0 目标。** 具备跨查询分区缓存（共享工具箱）、干净的 split 枚举（无 per-split 远程放大）、per-statement 写事务，并复用 fe-core 跨查询 schema 缓存；非 MVCC、静态凭证，Layer-3 授权隔离不适用。 + +**唯一真正适用的框架件是 per-statement handle/table memoization：** `getTableHandle` 每次解析都发冗余的 ODPS `tables().exists()` 并返回新惰性表，而 funnel 只 memo metadata 不 memo handle——于是一条语句里 k 个独立解析站点各付一次 `exists()` RPC（MC-1，P1），访问 schema 的路径重复 reload 表（MC-2，P2）。 + +**建议（低风险、高杠杆）：** 在 per-statement `MaxComputeConnectorMetadata` 实例内按 `(db,table)` memo 解析出的 `MaxComputeTableHandle`，并对已解析读去掉冗余存在性探测；可选地后续再加跨查询 ODPS `Table` 缓存。改动远小于 iceberg 那套 buildout。 + +--- + +## 复核结论 (adversarial verify) + +**Verdict: CONFIRMED** · 确认发现 IDs: MC-1, MC-2, MC-3, MC-4 + +| 原始声明 | 问题 | 更正 | 证据 | +|---|---|---|---| +| find fe/fe-core -iname '*MaxCompute*' returns nothing | Literally false: it returns two compiled dirs fe/fe-core/target/classes/org/apache/doris/datasource/maxcompute and target/test-classes/.../maxcompute (stale build artifacts). | The SOURCE tree fe/fe-core/src is clean (no MaxCompute .java). The read/write cutover conclusion is unaffected; only the stated grep evidence is imprecise. Verify with: find fe/fe-core/src -iname '*MaxCompute*' (empty). | | +| PluginDrivenScanNode.cachedMetadata + resolvedScanProvider ... PluginDrivenScanNode.java:189/184/245-260 | File path implied by the funnel narrative (datasource/plugin/) is wrong. | PluginDrivenScanNode is at fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java. All cited line numbers are correct there: currentHandle:152, resolvedScanProvider:184, cachedMetadata:189, metadata():203, resolveScanProvider():245-260. | | +| ~13 resolveConnectorTableHandle / getTableHandle sites (PluginDrivenExternalTable.java:160,463,607,717,821,882,955,1026,1061,1128 + resolveWriteCapabilityHandle 205/222/375/410) | Direct resolveConnectorTableHandle count is 10, not 13; the extra 205/222/375/410 line refs point at resolveWriteCapabilityHandle callers which I did not individually confirm (the method definition is at 187). | Exactly 10 direct resolveConnectorTableHandle call sites confirmed at the listed lines, each in a distinct real planning entrypoint (getSyntheticScanPredicates:152, initSchema:430, getShowCreateTableDdl:599, fetchSyntheticWriteColumns:700, getNameToPartitionItems:807, getNameToPartitionValues:870, getSupportedSysTables:946, getColumnStatistic:1014, getChunkSizes:1049, fetchRowCount:1121). The write-capability probes add a few more. The 'k independent sites, none memoized' core of MC-1 holds regardless of the exact count. | | +| MC-2: ~1-2 reloads per statement | Slight understatement risk on the cold per-column stats path was checked and cleared, worth recording. | getColumnStatistic (fe-core:1014-1035) resolves a FRESH handle per column (so MC-1 exists()-probe amplifies to N-per-column on cold stats), BUT MaxComputeConnectorMetadata does NOT override getColumnStatistics (SPI default no-op, no remote call and no odpsTable access), so it triggers NO extra Table.reload(). MC-2's ~1-2x reload multiplier therefore holds; the per-column amplification is exists()-probe only, which the audit already flags qualitatively under MC-1 ('higher on cold stats caches'). | | + +> 复核备注：All material claims hold against HEAD (aaab68ef474). Core thesis verified: MaxCompute is LIVE and reasonably cached (not a P0 target like iceberg), and the one real applicable framework piece is per-statement handle/table memoization. + +CONFIRMED in code: +- Migration: max_compute in SPI_READY_TYPES (CatalogFactory.java:57); write path live (PhysicalPlanTranslator getTableHandle:676, getWritePlanProvider:687); dormant-until-cutover javadoc at beginTransaction is stale. +- Funnel: PluginDrivenMetadata.get memoizes ONE metadata per (statement,catalog) (line 70). getMetadata (MaxComputeDorisConnector:177-181) returns a fresh instance; MaxComputeConnectorMetadata carries no instance-level handle map (only odps/structureHelper/defaultProject/endpoint/quota/properties/partitionCache). getTableHandle (118-130) re-runs tableExist (line 121 -> tables().exists(), remote per ODPS SDK) + fresh lazy getOdpsTable (124). resolveConnectorTableHandle (PluginDrivenExternalTable:116-119) does NOT memoize -> each of 10 call sites re-resolves. => metadataMemoizesLoadedTable = no, confirmed. +- MC-1 (P1): CONFIRMED real, not memoized at any layer, multiplier honest (~2 warm, per-column on cold stats). +- MC-2 (P2): CONFIRMED real; per-handle in-object memo (MaxComputeTableHandle.java:37 transient), not cross-handle; reload sites MaxComputeScanPlanProvider 183/187/189/194; ~1-2x correct. +- MC-3: CONFIRMED valid non-finding. serializeSession once (329), per-split loop (337-372) shares the one serialized string by reference (.scanSerialize at 346/364); no per-split remote IO. +- MC-4: CONFIRMED valid non-finding. All three partition SPI methods use partitionCache.getPartitions (listPartitionNames:243, listPartitions:265, listPartitionValues:284); the only uncached cost is the getTableHandle exists() probe, correctly folded into MC-1. +- MaxComputePartitionCache: cross-query, keyed (db,table) PartitionKey:138, TTL 600s (74), cap 10000 (75), CacheSpec.fromProperties (92), built on shared fe-connector-cache (imports 20-21), REFRESH hooks 189/198/204/213 including whole-table flush on invalidatePartition:213. +- Authz: static catalog credentials (MCConnectorClientFactory:86-127 AK/SK / RAM-Role-ARN / ECS-RAM-Role), no per-user/session=user; Layer-3 isolation N/A; name-only cache authz-safe under one shared ODPS identity. +- frameworkPiecesNeeded all consistent: getTableComment default returns "" (ConnectorTableOps.java:336) and is NOT overridden -> comment-cache 'no' correct; write-txn-coholder present (MaxComputeConnectorTransaction:61-131, setWriteSession:97, TMCCommitData/nextBlockId). + +No hallucinated caches, no invented classes, no over-stated multipliers, and no missed severe finding. Only immaterial nits (see corrections): the 'find returns nothing' grep evidence, the PluginDrivenScanNode directory path, and the '~13 sites' count. None change any conclusion. diff --git a/plan-doc/connector-cache-unification/connectors/paimon.md b/plan-doc/connector-cache-unification/connectors/paimon.md new file mode 100644 index 00000000000000..5f0cde58af4297 --- /dev/null +++ b/plan-doc/connector-cache-unification/connectors/paimon.md @@ -0,0 +1,86 @@ +## 附录:fe-connector-paimon 热点路径与缓存审计 + +### 一、迁移状态(Migration status) + +`paimon` 已在 `SPI_READY_TYPES` 白名单内(`fe/fe-core/.../datasource/CatalogFactory.java:57`),是一个**一等公民独立 catalog 类型**,经 SPI plugin 路径 `PaimonConnectorProvider → PaimonConnector → PaimonConnectorMetadata / PaimonScanPlanProvider`,由 `PluginDrivenExternalCatalog` 承载。它是 **LIVE**,不是 sibling:读、MVCC/time-travel、DDL(建/删库+表)、system tables、stats、SHOW CREATE 均在连接器落地。**写路径故意未迁移**(`getCapabilities` 不声明写能力,`PaimonConnector.java:318`),因此本连接器无 beginWrite/commit。 + +关键结构性事实:生产 catalog 被 **paimon SDK `CachingCatalog`** 包裹(`CatalogFactory.createCatalog → CachingCatalog.tryToCreate`,`CACHE_ENABLED` 默认 true,已 javap 证实 paimon-core 1.3.1)。`CachingCatalog` 自带 `tableCache`(Identifier→Table)、`partitionCache`(Identifier→List)、`manifestCache`(Path→manifest segments)。**这是 iceberg SPI 路径所缺、而 paimon 天然具备的 metastore-client 级缓存层**,是本审计结论的决定性因素。 + +### 二、当前缓存(Caches) + +| 缓存 | 作用域 | Key | 值 | TTL / 失效 | file:line | +|---|---|---|---|---|---| +| `PaimonLatestSnapshotCache` | cross-query | `Identifier(db,table)` | latest snapshot id (long) | `meta.cache.paimon.table.ttl-second` 默认 24h;`<=0` 关闭(no-cache catalog);Caffeine expireAfterAccess + maxSize 1000;REFRESH TABLE/DB/CATALOG 失效 | `PaimonLatestSnapshotCache.java:48`(字段 `PaimonConnector.java:124`) | +| `PaimonSchemaAtMemo` | cross-query | `(db,table,sysTable,branch,schemaId)` | `PaimonSchemaSnapshot`(fields+partKeys+pkKeys) | 无 TTL,bounded 10000 clear-on-overflow;值不可变(schemaId 内容 write-once);REFRESH 失效 | `PaimonSchemaAtMemo.java:49`(元数据 time-travel `PaimonConnectorMetadata.java:299` 与 scan schema-evolution dict `PaimonScanPlanProvider.java:1567` 共享) | +| `partitionViewCache`(`ConnectorPartitionViewCache`,共享工具箱) | cross-query | `PartitionViewCacheKey(db,table,snapshotId,schemaId=-1)` | 已构建 `List` | `meta.cache.paimon.partition_view.*` 默认 ON/24h/1000;snapshotId 复用 latestSnapshotCache 追踪"current";REFRESH 失效 | `PaimonConnectorMetadata.java:1012-1052`(字段 `PaimonConnector.java:143`) | +| Transient Table 胖句柄 | per-scan / per-op | `PaimonTableHandle` 实例身份 | 已加载 Table 的 transient 引用 | 句柄对象生命周期;FE→BE 序列化后丢失并 reload;不跨不同 fe-core op(每次 `resolveConnectorTableHandle` 重建),但在单个 SPI 方法内 / 单个 scan node 内(`PluginDrivenScanNode` 缓存 currentHandle+resolveScanProvider)共享 | `PaimonTableHandle.java:89`(set `PaimonConnectorMetadata.java:221`,read `PaimonTableResolver.java:65`) | +| paimon SDK `CachingCatalog`(tableCache/partitionCache/manifestCache) | metastore-client | `Identifier` / `Path` | Table / List / manifest segments | paimon `cache.*` SDK 默认;`CACHE_ENABLED` 默认 true 故生产 catalog 被包裹 | 外部 SDK `org.apache.paimon.catalog.CachingCatalog`(包裹于 `PaimonConnector.java:454`) | +| `DRIVER_CLASS_LOADER_CACHE` / `REGISTERED_DRIVER_KEYS` | none | driver URL / url#class | ClassLoader / 已注册标记 | 进程级 JDBC 驱动去重,非元数据缓存 | `PaimonConnector.java:89-90` | + +### 三、Funnel 参与(Funnel participation) + +已完全接入 Layer-2 per-statement funnel:`PluginDrivenExternalTable.resolveConnectorTableHandle` 经 `PluginDrivenMetadata.get`(每 (statement,catalog) memoize 一个 metadata);`PluginDrivenScanNode` 另在 `cachedMetadata` 缓存并按 currentHandle 身份 memoize `resolveScanProvider()`(`:182,189,245-259`)。故一个 scan node 内 `planScan` 与 `getScanNodeProperties` 共享同一 provider 与同一 currentHandle(携带 transient Table),`resolveScanTable` 命中 transient。`schemaAtMemo` 由长生命周期 `PaimonConnector` 同时注入 `getMetadata`(`:247`)与 `getScanPlanProvider`(`:311`),即每 catalog 一份。 + +`metadataMemoizesLoadedTable = partial`:`getMetadata` 每语句返回**全新** `PaimonConnectorMetadata`(`:244-248`),每个 fe-core op 重新 `getTableHandle → catalogOps.getTable`(`:211`)构造新句柄。这与 iceberg 审计指出的 re-getTableHandle 模式相同,**但对 paimon 不构成 remote-IO 放大**:paimon SDK `CachingCatalog.tableCache` 使每次 getTable 成为内存命中,transient 胖句柄在单 op 内折叠重载。因此 funnel 的"每语句一 metadata"本身并不折叠 paimon 的表加载,折叠由 paimon SDK 缓存 + transient 句柄完成。iceberg PERF-07 式"每语句加载一次"表 memo 对 paimon 仅省下内存 tableCache 查找与 `setPaimonTable` 抖动 —— 可忽略。 + +### 四、热点路径审计(Hot-path findings) + +| id | 区域 | 问题 | 倍率 | 成本 | 严重度 | 已memo? | file:line | +|---|---|---|---|---|---|---|---| +| PA-1 | partitions | `listPartitionNames`(`:988`)/`listPartitionValues`(`:1057`)直接调 `collectPartitions()`,**绕过** partitionViewCache;仅 `listPartitions`(`:1019-1022`)走派生缓存。SHOW PARTITIONS 与 partition_values() TVF 重跑渲染(变体 C) | 每次调用 1x(非循环);原始 remote listPartitions 已被 paimon SDK partitionCache 挡下,残留仅 CPU 重渲染 | cpu | **P2** | 否 | `PaimonConnectorMetadata.java:988` | +| PA-2 | query-plan | 每 fe-core op 经 getTableHandle→getTable(re-getTableHandle 模式),未由 metadata 折叠 | ~4-7 次/语句,首次后均为 CachingCatalog 内存命中(非 remote) | remote-io | none | 是(SDK+transient) | `PaimonConnectorMetadata.java:211` | +| PA-3 | stats-rowcount | `getTableStatistics → rowCount` 用 `newReadBuilder().newScan().plan().splits()` 全 manifest 计划求行数,连接器无 memo | 每语句 1 次;fe-core `RowCountCache`(`PluginDrivenExternalTable.java:907`)跨查询缓存,manifest 读被 paimon manifestCache 挡 | remote-io | none | 是(fe-core 层) | `PaimonConnectorMetadata.java:1205` | +| PA-4 | schema | `getTableSchema → latestSchema` 为 `schemaManager().latest()` 活读(刻意不用冻结 rowType(),以捕获外部 ALTER) | 每 schema-cache miss 1 次;fe-core 通用 schema 缓存(经 `schemaCacheTtlSecondOverride`)跨查询挡下 | remote-io | none | 是(fe-core schema cache) | `PaimonConnectorMetadata.java:250` | +| PA-5 | split-enum | `planScanInternal → planSplits(scan)` 读快照/manifest 枚举 split(即查询本身的数据规划,不可缓存);周边不变量已提升一次/scan | 每 scan 1 次;跨查询 manifest 读被 paimon manifestCache 挡 | remote-io | none | 是(paimon manifestCache) | `PaimonScanPlanProvider.java:461` | + +per-scan/per-file 不变量已充分提升:vended token(`:559-560`)、FE 权重分母(`:565`)、native 目标 split size 惰性一次(`:593,624-626`)、partitionValues 每 DataSplit 一次并复用于全部字节切片子 split(`:605,634`)。即 iceberg PERF-06/PERF-11 的等价已就位。 + +### 五、授权 / 一致性(Authz / consistency) + +`perUserCredential=false`,`sessionUserRelevant=false`。paimon **无** 按 Doris 用户的会话凭证模型:catalog 在创建时一次性认证(Kerberos 插件 UGI / HMS principal / 静态或 JDBC 凭证);REST vended 凭证在 scan 时从**表级** `RESTTokenFileIO` 提取(`PaimonScanPlanProvider.extractVendedToken:916`),**不按用户键控**。故 name-only 缓存无 "list≠load" 泄漏风险。已证实:fe-connector-paimon 与 fe-connector-metastore-paimon 内无 `SUPPORTS_USER_SESSION`/`isUserSessionEnabled`;`PaimonRestMetaStoreProvider` 无任何 per-user/delegation 逻辑;`tools/check-authz-cache-sharding.sh` 仅针对 `IcebergConnector`。三个连接器缓存无条件构造(从不置空)对 paimon 是正确的(`PaimonConnector.java:138-142` 记录了理由)。写路径 read+write 共享一 txn 的一致性(iceberg `CatalogStatementTransaction`)对 paimon **N/A**(写未迁移)。读侧 MVCC 一致性(查询内稳定快照 pin)由 `PaimonLatestSnapshotCache → beginQuerySnapshot → applySnapshot → scan.snapshot-id` 提供。 + +### 六、框架部件需求(Framework pieces) + +- **already-has(6)**:per-statement funnel 路由;partition-cache(partitionViewCache+SDK partitionCache);manifest-cache(SDK manifestCache);per-scan-hoist;per-file-split-memo;shared-fe-connector-cache 采用(partitionViewCache/latestSnapshotCache 用共享工具箱,仅 schemaAtMemo 用裸 CHM,可选统一)。 +- **no(5)**:cross-query-table-cache(与 SDK tableCache 重复,且 latestSnapshotCache 已提供稳定快照语义);format-cache(格式来自 `table.options()` 内存读,无 whole-table planFiles 回退);comment-cache(comment 来自 coreOptions,无 N-per-query remote getComment);authz-session-user-isolation(无 per-user 会话模型);write-txn-coholder(写未迁移)。 + +### 七、结论与建议 + +paimon **已充分缓存,无需套用 iceberg 热点/框架移植**。根因:(1) 生产 catalog 由 paimon SDK `CachingCatalog` 包裹,提供 iceberg SPI 路径缺失的 table/partition/manifest 内存缓存;(2) 连接器已恢复三个 legacy 语义缓存 + transient 胖句柄,并完成 per-scan/per-file 不变量提升;(3) funnel 已路由其 ConnectorMetadata。 + +优先建议: +1. **不要**对 paimon 移植 iceberg 式 table/format/comment/manifest 缓存或 session=user 隔离(均结构性不适用或与 SDK 缓存重复)。 +2. (可选,P2)将 `listPartitionNames`/`listPartitionValues` 也路由经 `partitionViewCache`(修 PA-1),消除派生视图重渲染的兄弟路径旁路——价值偏低,因原始 remote 读已被 paimon partitionCache 挡下。 +3. (可选,整洁性)将 `PaimonSchemaAtMemo` 迁到共享 `MetaCacheEntry` 工具箱,统一 TTL/metrics —— 纯粹一致性,非正确性或性能缺口。 + +--- + +## 复核结论 (adversarial verify) + +**Verdict: CONFIRMED** · 确认发现 IDs: PA-1, PA-2, PA-3, PA-4, PA-5 + +| 原始声明 | 问题 | 更正 | 证据 | +|---|---|---|---| +| paimon SDK CachingCatalog ... CACHE_ENABLED default true so the live catalog IS wrapped (CatalogFactory.createCatalog -> CachingCatalog.tryToCreate); paimon cache.* options (cache.expiration-interval etc.) | Immaterial label imprecision: the actual paimon option key string is 'cache-enabled' (hyphen), and the connector calls the paimon (not Doris) org.apache.paimon.catalog.CatalogFactory at PaimonConnector.java:454. The substantive claim is fully verified in bytecode. | Verified: paimon-core-1.3.1 CatalogFactory.createCatalog(ctx,cl) invokestatic CachingCatalog.tryToCreate, which gates on CatalogOptions.CACHE_ENABLED and areturns the raw catalog when false; the CACHE_ENABLED field is built as booleanType().defaultValue(Boolean.valueOf(true)) (iconst_1 at offset 202), description 'Controls whether the catalog will cache databases, tables, manifests and partitions.' Default IS true; wrap covers db/table/manifest/partition. No change to any conclusion. | | + +> 复核备注：Every material claim holds against HEAD (aaab68ef474) code and the paimon 1.3.1 SDK bytecode. No hallucinated file:line, no invented cache class, no overstated multiplier, no false-positive finding, no missed severe finding. + +MIGRATION: paimon in SPI_READY_TYPES (CatalogFactory.java:57); LIVE via PaimonConnectorProvider->PaimonConnector->PaimonConnectorMetadata/PaimonScanPlanProvider; no write capability (getCapabilities PaimonConnector.java:321-341 declares only MVCC/PARTITION_STATS/COLUMN_AUTO_ANALYZE/SHOW_CREATE_DDL). CONFIRMED. + +CACHES (all 6 exist, scope/key correct): +- PaimonLatestSnapshotCache.java:48 MetaCacheEntry, shared CacheSpec toolkit, ttl=meta.cache.paimon.table.ttl-second, maxSize 1000, <=0 disables. Built PaimonConnector.java:154-155, field :124. Invalidations :256/278/285. CONFIRMED. +- PaimonSchemaAtMemo.java:49 raw ConcurrentHashMap (:54) clear-on-overflow (:81-82) maxSize 10000, key MemoKey(db,table,sysTableName,branchName,schemaId) (:129-142), immutable value. Injected into getMetadata (:247) AND getScanPlanProvider (:311). CONFIRMED including the 'only schemaAtMemo uses a raw CHM' framework-piece note. +- partitionViewCache: ConnectorPartitionViewCache + PartitionViewCacheKey exist in fe-connector-cache; routed in PaimonConnectorMetadata.listPartitions :1012-1022, key (db,table,snapshotId,-1) via latestSnapshotCache :1046-1052; field PaimonConnector.java:143 built :158; invalidations :263/280/287. CONFIRMED. +- transient fat-handle: PaimonTableHandle.java:89 (transient Table), set PaimonConnectorMetadata.java:221, read PaimonTableResolver.java:65. CONFIRMED. +- paimon SDK CachingCatalog: DECISIVELY verified in bytecode (see corrections) - wraps db/table/manifest/partition, default on. CONFIRMED - this is the load-bearing fact behind the whole 'do not port iceberg caches' verdict and it holds. +- DRIVER_CLASS_LOADER_CACHE/REGISTERED_DRIVER_KEYS PaimonConnector.java:89-90, process-wide, not a metadata cache. CONFIRMED. + +FUNNEL (partial memoization): PluginDrivenMetadata.get memoizes one ConnectorMetadata per (statement,catalog) via scope.getOrCreateMetadata (:70) with per-statement identity guard (:64-69). getMetadata returns a FRESH PaimonConnectorMetadata per statement (:244-248). resolveConnectorTableHandle (:116-119) re-calls getTableHandle->catalogOps.getTable (:211) on EVERY fe-core op with NO caching (~15 call sites in PluginDrivenExternalTable each do PluginDrivenMetadata.get + resolveConnectorTableHandle). So the metadata does NOT memoize the loaded Table; collapse comes from SDK tableCache + transient handle. PluginDrivenScanNode: cachedMetadata (:189/204-207) + resolveScanProvider memo keyed on currentHandle identity (:245-259). All CONFIRMED. + +HOTPATH FINDINGS: +- PA-1 CONFIRMED as real Variant-C cache bypass: listPartitionNames (PaimonConnectorMetadata.java:988) and listPartitionValues (:1057) call collectPartitions() directly; only listPartitions (:1012-1022) routes through partitionViewCache. This is the ONLY actionable item. Its P2 rating is arguably slightly generous (CPU-only re-render since raw remote read is blunted by SDK partitionCache) but the audit itself flags it 'low-value', so self-consistent. +- PA-2..PA-5 CONFIRMED as accurately characterized non-findings (severity none / memoizedAlready true): heavy ops are all REAL (getTable :211; rowCount = full newReadBuilder().newScan().plan().splits() sum PaimonCatalogOps.java:368-376; latestSchema = live schemaManager().latest() :345-355, intentionally not off frozen rowType; planSplits :461) but each is genuinely blunted by SDK tableCache/manifestCache + transient handle + fe-core RowCountCache (PluginDrivenExternalTable.java:907) / generic schema cache. Per-scan/per-file loop-invariants are all hoisted as claimed (vendedToken :559-560, weightDenominator :565, targetSplitSize lazy-once :624-626, getPartitionInfoMap once-per-DataSplit :605 reused across sub-splits :634). No hidden per-split remote loadTable in any loop. + +AUTHZ: no SUPPORTS_USER_SESSION/isUserSessionEnabled anywhere in fe-connector-paimon or fe-connector-metastore-paimon (only comments explaining paimon has no such model); tools/check-authz-cache-sharding.sh TARGET_REL is IcebergConnector.java only (:57); REST vended token extracted table-level via extractVendedToken (PaimonScanPlanProvider.java:916), not Doris-user-keyed. Name-only caches safe. CONFIRMED. + +Overall verdict (do NOT port iceberg-style caching; optionally take PA-1; optionally migrate PaimonSchemaAtMemo to MetaCacheEntry) stands unchanged. diff --git a/plan-doc/connector-cache-unification/connectors/trino.md b/plan-doc/connector-cache-unification/connectors/trino.md new file mode 100644 index 00000000000000..8466a0cf8d2b27 --- /dev/null +++ b/plan-doc/connector-cache-unification/connectors/trino.md @@ -0,0 +1,100 @@ +## 连接器附录:trino-connector(`fe/fe-connector/fe-connector-trino`) + +### 定位:唯一的 LIVE pass-through bridge + +trino-connector 在 8 个 SPI 连接器里独一无二:它不是自己实现元数据,而是**内嵌一个真正的 Trino Connector SPI**,把 Doris 的 metadata/scan 请求转译后委托给内嵌 Trino 连接器。链路: + +- `TrinoConnectorProvider.getType()` = `"trino-connector"`,`create()` → `TrinoDorisConnector`(`TrinoConnectorProvider.java:35,40`)。 +- `TrinoDorisConnector` 双检锁 `doInitialize()` 通过 `TrinoBootstrap` 加载 Trino 插件、每个 Doris catalog 造**一个** live `io.trino.spi.connector.Connector` 并 volatile 持有(`TrinoDorisConnector.java:53-57,133-188`)。 +- `getMetadata()` 每次 new 一个 `TrinoConnectorDorisMetadata`(funnel 会把它 per-statement memo 掉),`getScanPlanProvider()` new `TrinoScanPlanProvider`(`TrinoDorisConnector.java:64-75`)。 + +**迁移状态**:`SPI_READY_TYPES` 含 `"trino-connector"`(`CatalogFactory.java:57`),经 `CatalogFactory.java:110-118` 走 SPI/`PluginDrivenExternalCatalog` 路径。**live**,非 sibling-only / dormant / legacy。**只读**:`TrinoConnectorDorisMetadata` 对 create/drop/rename/beginWrite/executeStmt 的 override 数 = 0。 + +### Caches 一览 + +| Cache | 作用域 | Key | TTL / 失效 | 位置 | +|---|---|---|---|---| +| `TrinoBootstrap.instance`(typeRegistry/handleResolver/pluginManager 插件基座) | 进程级(cross-query) | pluginDir(全 FE 唯一) | 进程生命周期,不失效 | `TrinoBootstrap.java:100,141-156` | +| `TrinoDorisConnector.trinoConnector`(每 catalog 内嵌的 live Trino Connector) | cross-query | Doris catalog | catalog 生命周期;`close()`→`shutdown()` 失效 | `TrinoDorisConnector.java:53-57,133-141,95-102` | +| **内嵌 Trino 连接器自身的元数据缓存**(CachingHiveMetastore / iceberg metadata cache / CachingJdbcClient)= Trino 表真正的跨查询元数据缓存 | metastore-client | Trino 自持 key;由连接器配置启用+定大小(如 `hive.metastore-cache-ttl`) | Trino 配置驱动(多数 Trino 连接器默认关闭) | 位于 `trinoConnector` 内部,本模块无源码;每次 `trinoConnector.getMetadata(...).getTableHandle(...)` read-through | +| `TrinoServicesProvider.catalogs` | cross-query | `CatalogHandle` | catalog 生命周期 | `TrinoServicesProvider.java:63` | +| `TrinoPluginManager.connectorFactories` | cross-query | `ConnectorName` | 进程生命周期 | `TrinoPluginManager.java:64` | +| **Doris 连接器层的 table/handle/schema 缓存** | **none** | 不存在 —— `TrinoConnectorDorisMetadata` 零 memo;pom 无 `fe-connector-cache` 依赖 | n/a | `TrinoConnectorDorisMetadata.java`(整类) | +| (fe-core 通用,非 trino 专属)`ExternalSchemaCache` + `RowCountCache` | cross-query | catalog.db.table 名;全连接器共享 | `external_cache_expire_time`,REFRESH 失效 | `PluginDrivenExternalTable.java:430,1121` | + +一句话:**Doris 连接器层没有任何跨查询/每语句元数据缓存**;Trino 表的元数据缓存全部在内嵌 Trino 连接器内部,fe-core 通用的 schema/rowcount 缓存兜底跨查询重复。 + +### Funnel 参与情况 + +**已进 funnel**:`PluginDrivenScanNode`/`PluginDrivenExternalTable` 所有元数据获取均经 `PluginDrivenMetadata.get(session, connector)`(`PluginDrivenScanNode.java:206,222`;`PluginDrivenExternalTable.java:449,1126`),每语句每 catalog 只 memo 一个 `TrinoConnectorDorisMetadata`,`tools/check-fecore-metadata-funnel.sh` 全库把关。 + +**关键点:该 wrapper 内部零 memo**。`listDatabaseNames/listTableNames/getTableHandle/getTableSchema/applyFilter/applyProjection` 每个方法都 `beginTransaction(READ_UNCOMMITTED)` 起**一笔全新 Trino 事务** + `trinoConnector.getMetadata(...)` 再 commit(`TrinoConnectorDorisMetadata.java:101,121,153,195,273,338`);`TrinoScanPlanProvider.planScan` 再起一笔(`TrinoScanPlanProvider.java:111-113`)。`getTableHandle` 还会 eager 拉全部 column handle + 每列 `getColumnMetadata`(`:166-176`)。scope 上不 stash 任何已解析的 `TrinoTableHandle`/Trino `Table`。 + +所以 funnel 给出的是"每语句一个 Doris wrapper",**并不折叠 wrapper 内部的重复加载**。但残余 multiplier 低:(a) fe-core 的 `ExternalSchemaCache`/`RowCountCache` 跨查询兜底 `getTableHandle`/`getTableSchema`;(b) 每次 `getTableHandle` 又 read-through 内嵌 Trino 连接器自身的长生命周期 metastore 缓存。`metadataMemoizesLoadedTable = no`。 + +### Hot-path 审计(应用 problem class) + +Trino wrapper 只 override 了 list/handle/schema/pushdown 这一小撮方法;`listPartitions`、`getTableStatistics`、`estimateDataSizeByListingFiles`、`getMvccPartitionView`、`getTableFreshness`、`getSyntheticScanPredicates` 以及整条写路径**全走接口默认值(空 / -1 / handle 原样返回)**。因此 iceberg 的 PERF-02(PARTITIONS 重扫)、PERF-03(format 回退)、PERF-04(manifest)、PERF-07/R6(写)在这里**没有对应热点**。 + +| ID | 区域 | 问题 | multiplier | cost | 严重度 | 已 memo? | 位置 | +|---|---|---|---|---|---|---|---| +| TRINO-H1 | query-plan | `getTableHandle`(唯一近似 remote 的元数据 op)在冷语句里从 3 个 fe-core 站点被调:initSchema(`PluginDrivenExternalTable.java:463`)、fetchRowCount(`:1128`)、scan create(`PluginDrivenScanNode.java:228`);wrapper 内不 memo 已解析 handle | 暖缓存 ~1x/SELECT;冷 ~3x —— 但 schema 走 ExternalSchemaCache、rowcount 走 RowCountCache,且各次 read-through Trino 自身缓存 | mixed | P2 | 是(fe-core 层) | `PluginDrivenScanNode.java:228` | +| TRINO-H2 | schema | 一次冷 schema 加载里 `getTableHandle` 已按列建好 `columnMetadataMap`(`:170-176`),`getTableSchema` 却弃之不用、每列重新 `getColumnMetadata`(`:207-209`)= 2N 次 + 第二笔 Trino 事务 | 2N getColumnMetadata / 冷 schema miss(非每查询) | cpu | P2 | 否 | `TrinoConnectorDorisMetadata.java:207` | +| TRINO-H3 | predicate | applyFilter 每次 scan 跑两遍:fe-core `convertPredicate`(`PluginDrivenScanNode.java:824`,自带一笔 Trino 事务)+ `planScan` 内再一遍(`TrinoScanPlanProvider.java:126`,另一笔事务);projection 同理 | 2x applyFilter + 2x applyProjection / scan | cpu | P2 | 否 | `TrinoScanPlanProvider.java:126` | +| TRINO-H4 | split-enum | split 枚举全委托 Trino `ConnectorSplitManager`+`BufferingSplitSource`;per-split 循环只序列化各自 split JSON + host。scan 不变字段(tableHandle/txn/columnHandles/columnMetadata/options JSON)已在循环前**一次性**预序列化 | 共享字段 1x/scan(已 hoist) | cpu | none | 是 | `TrinoScanPlanProvider.java:221-227` | +| TRINO-H5 | partitions/stats/write | 相关方法均未 override → 默认空/‑1/原样;`getNameToPartitionItems` 见空分区,`fetchRowCount` 返回 UNKNOWN;无写路径 | 0(无此调用) | cpu | none | 是 | `TrinoConnectorDorisMetadata.java:64` | + +**无 P0/P1**。不存在"重 remote op × 高 multiplier × 无 memo"的 A/B/C/D 变体:唯一的 remote-ish op(`getTableHandle`)既被 fe-core 跨查询缓存兜住,又被内嵌 Trino 连接器自身缓存兜住;真正的 per-split 循环只做不可避免的 split 序列化。 + +### Authz / 一致性 + +`perUserCredential = false`,`sessionUserRelevant = false`。`TrinoBootstrap` 在 CREATE CATALOG 时把**单一静态身份** `Identity.ofUser("user")` 烤进每 catalog 的 Trino Session(`TrinoBootstrap.java:264-265`),对所有 Doris 用户复用。无 REST OIDC、无 kerberos doAs、无 per-identity metastore、无 vended per-user 凭证;内嵌 Trino 连接器用 catalog 属性里的**静态**凭证(metastore auth / 对象存储 key)访问底层。访问控制由 Doris 自身权限系统负责。 + +因此:**name-only 缓存不会泄漏 per-user 授权**(根本没有 per-user 维度可分片),iceberg 的 `session=user` "list ≠ load" 元数据泄漏、Layer-3 隔离与 `shouldBypassSchemaCache` 在此**N/A**。`PluginDrivenMetadata` 的 builder-identity 断言(`PluginDrivenMetadata.java:63-69`)仍跑但恒真。写一致性(读写共享一 metadata/txn)也 N/A —— 只读连接器,无写事务。 + +### 框架各件是否需要 + +- **per-statement-funnel-memoization** = partial:已进 funnel;wrapper 内部再 memo 已解析 `TrinoTableHandle` 可省掉每 SELECT 多出的 Trino 事务,但收益小(schema/rowcount 缓存 + Trino 自身缓存已兜底)。低优先。 +- **cross-query-table-cache** = **no**:会与内嵌 Trino 连接器自身缓存**双重缓存**,并把外部 ALTER 后的 schema 冻住,重新引入 Trino 已解决的 stale-metadata bug;且 `TrinoTableHandle` 是事务派生 + transient/不可序列化。**禁止加**。 +- **partition-cache / format-cache / comment-cache / manifest-or-file-list-cache** = no:分区(默认空)、file-format(在 Trino/BE 侧)、comment(默认 ""、随缓存 schema)、file-listing(默认 -1、split 交给 Trino)均无 FE 侧热点。 +- **per-scan-hoist** = already-has(`TrinoScanPlanProvider.java:221-227` 已把 scan 不变字段循环前一次序列化)。 +- **per-file-split-memo** = no:无跨 split 的 per-file 不变量可 memo。 +- **authz-session-user-isolation** = no:静态单身份,无可分片缓存、无泄漏面。 +- **shared-fe-connector-cache-adoption** = no:连接器层无可安全缓存之物,采纳只会制造双缓存正确性隐患。 +- **write-txn-coholder** = no:只读,无写事务。 + +### 结论与建议 + +**不要**把 iceberg 热点缓存框架套到 trino-connector。它是委托型 bridge,元数据缓存 / 分区&stats / split 枚举 / 缓存失效全部由内嵌 Trino 连接器负责;跨查询重复由 fe-core 通用 `ExternalSchemaCache`/`RowCountCache` + Trino 自身缓存两层兜底。Layer-1 连接器缓存与 Layer-3 授权隔离在此**反而有害**(双缓存 + stale 风险;无 per-user 维度);Layer-2 funnel 已接好且被 gate 强制,残余 multiplier 低。 + +优先级:**维持现状**(委托正确)。若某个热 catalog 的 BI profile 真的暴露出来,再排期三个 P2 纯 CPU 清理:(1)per-statement memo 已解析 `TrinoTableHandle` 以消掉多余 Trino 事务;(2)冷 schema 加载去掉 `getTableHandle`/`getTableSchema` 的 2N `getColumnMetadata` 重复;(3)消除 `convertPredicate` 与 `planScan` 之间的双重 applyFilter/applyProjection。 + +--- + +## 复核结论 (adversarial verify) + +**Verdict: CONFIRMED** · 确认发现 IDs: TRINO-H1, TRINO-H2, TRINO-H3, TRINO-H4, TRINO-H5 + +| 原始声明 | 问题 | 更正 | 证据 | +|---|---|---|---| +| Cache #4 'TrinoServicesProvider.catalogs' keyedBy: CatalogHandle | The map is actually keyed by the catalog-name String, not a CatalogHandle object; lookups derive the name from the handle. | TrinoServicesProvider.java:63 declares `ConcurrentMap catalogs`; getConnectorServices(CatalogHandle) looks up via `catalogs.get(catalogHandle.getCatalogName())` (line 126). One entry per catalog (correct), but keyed by name String. Non-material. | fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoServicesProvider.java:63,126 | +| TRINO-H3: '2x applyFilter per scan' | Precisely 2x only for a scan carrying a pushable predicate. For a filterless scan the fe-core convertPredicate arm short-circuits (guarded by empty conjuncts, and by TupleDomain.isAll() in the wrapper), so only planScan's applyFilter (with Constraint.alwaysTrue) runs = 1x. | convertPredicate returns early when conjuncts empty (PluginDrivenScanNode.java:819-820); TrinoConnectorDorisMetadata.applyFilter returns empty before opening a txn when tupleDomain.isAll() (TrinoConnectorDorisMetadata.java:266-268); planScan always calls Trino applyFilter (TrinoScanPlanProvider.java:126). So '2x' holds for the WHERE-filtered case the finding targets; overstated only for filterless scans. Finding stands. | fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java:819 | +| authzConsistency: the PluginDrivenMetadata builder-identity pin is 'vacuous for Trino since the identity is constant' | The pin keys on the Doris principal session.getUser(), not the constant Trino Identity.ofUser("user"); it does vary per Doris user across statements. It is trivially satisfied WITHIN one statement (one statement = one user), which is why it never fires for trino — not because the identity is constant. | PluginDrivenMetadata.java:63-69 uses session.getUser() (Doris principal). The static Trino identity is a separate layer (TrinoBootstrap.java:264-265). Conclusion (no per-user leak) is correct; the stated reason conflates the two identity layers. Non-material. | fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenMetadata.java:63 | + +> 复核备注：Verified against HEAD aaab68ef474 (branch branch-catalog-spi). Every material claim holds; the 3 corrections above are minor/non-material and do not undermine the audit's core. + +CONFIRMED facts with file:line (HEAD): +- Migration: CatalogFactory.java:57 (SPI_READY_TYPES includes "trino-connector"), :110-118 routing to PluginDrivenExternalCatalog. TrinoConnectorProvider.java:35-36 getType()="trino-connector", :40-41 create()->TrinoDorisConnector. TrinoDorisConnector implements Connector (:45), double-checked-lock init (:133-141), embedded live trino Connector volatile (:53-57), created via TrinoBootstrap (:143-188), shutdown on close (:95-102). getMetadata() returns a FRESH TrinoConnectorDorisMetadata every call (:65-69). LIVE bridge — CONFIRMED, not sibling/dormant/legacy. +- Wrapper memoizes nothing: TrinoConnectorDorisMetadata every method opens beginTransaction(READ_UNCOMMITTED)+getMetadata+commit-in-finally: listDatabaseNames :100-101, listTableNames :120-121, getTableHandle :152-153, getTableSchema :194-195, applyFilter :272-273, applyProjection :337-338. metadataMemoizesLoadedTable=no CONFIRMED. +- Funnel: PluginDrivenMetadata.get memoizes one ConnectorMetadata per (statement,catalog) at :70, identity pin :63-69. fe-core sites route through it: PluginDrivenScanNode.java:206 (metadata()), :222 (create), :825 (convertPredicate applyFilter); PluginDrivenExternalTable.java:449 (initSchema), :1126 (fetchRowCount). Both gate scripts present: tools/check-fecore-metadata-funnel.sh, tools/check-authz-cache-sharding.sh. Only Connector#getMetadata(session) caller in fe-core is inside PluginDrivenMetadata (the other .getMetadata hits are TestExternalCatalog's Map getter). CONFIRMED. +- Caches: TrinoBootstrap.instance singleton keyed by pluginDir with mismatch guard (:100,141-156); TrinoServicesProvider.catalogs (:63, String-keyed); TrinoPluginManager.connectorFactories (:64, ConnectorName-keyed). Module-wide grep shows NO LoadingCache/Caffeine/fe-connector-cache — only those two Trino-infra ConcurrentHashMaps. Doris-layer metadata cache = NONE CONFIRMED. fe-core ExternalSchemaCache filled by initSchema (getTableSchema :469), RowCountCache read at getCachedRowCount :907-908. + +Hot-path findings (all independently CONFIRMED): +- H1: getTableHandle is the only remote-ish op; invoked from initSchema (PluginDrivenExternalTable:463), fetchRowCount (:1128), scan create (PluginDrivenScanNode:228), all via resolveConnectorTableHandle->metadata.getTableHandle (:119). No resolved-handle memo. ~1x warm / ~3x cold. P2. Multiplier fair. +- H2: getTableHandle builds columnMetadataMap by per-column getColumnMetadata (TrinoConnectorDorisMetadata.java:170-176); getTableSchema ignores that map and re-loops getColumnMetadata per column (:207-209) => 2N + a second Trino txn. CONFIRMED, P2 cpu, not memoized. +- H3: applyFilter runs in fe-core convertPredicate (PluginDrivenScanNode:825 -> wrapper applyFilter, own txn) AND in planScan (TrinoScanPlanProvider:126, direct on Trino metadata); applyProjection in tryPushDownProjection (:1264 -> wrapper) AND planScan (:144). Predicate re-conversion via new TrinoPredicateConverter each time (:262-265 and :267-270). CONFIRMED (see correction re: filterless-case count). +- H4: scan-invariant fields pre-serialized ONCE before the split loop (TrinoScanPlanProvider:221-227, serializer built once :217-219); per-split loop (:235-256) serializes only each split JSON + hosts. severity none CONFIRMED. +- H5: no override of listPartitions/getTableStatistics/estimateDataSizeByListingFiles/getMvccPartitionView/getTableFreshness/getSyntheticScanPredicates/getTableComment or any ConnectorWriteOps method. Interface defaults verified: getTableComment->"" (ConnectorTableOps:336-339), listPartitions->emptyList (:400-405), getTableStatistics->empty (ConnectorStatisticsOps:32-36), estimateDataSizeByListingFiles->-1 (:62-64), ConnectorWriteOps all default/read-only. fetchRowCount->UNKNOWN. severity none CONFIRMED. No PERF-02/03/04/07 analog. +- Authz: TrinoBootstrap bakes static Identity.ofUser("user") into setIdentity/setOriginalIdentity (:264-265); no REST OIDC / kerberos doAs / per-user credential. No per-user cache dimension to leak. CONFIRMED. + +No missed severe (P0/P1) finding: the single remote-ish op (getTableHandle) is caught cross-query by ExternalSchemaCache/RowCountCache and read-through by the embedded Trino connector's own cache; the genuine per-split loop does only unavoidable per-split serialization. overallVerdict (do-not-apply-framework; optional 3 P2 cleanups) is sound. diff --git a/plan-doc/connector-cache-unification/data/connector-audits.json b/plan-doc/connector-cache-unification/data/connector-audits.json new file mode 100644 index 00000000000000..26ef4f1b05d66d --- /dev/null +++ b/plan-doc/connector-cache-unification/data/connector-audits.json @@ -0,0 +1,1419 @@ +[ + { + "connector": "hive", + "migrationStatus": { + "inSpiReadyTypes": true, + "liveness": "live", + "notes": "hms is in SPI_READY_TYPES at CatalogFactory.java:57 ({jdbc,es,trino-connector,max_compute,paimon,iceberg,hms}). HiveConnector + HiveConnectorMetadata are real impls; fe-core builds a PluginDrivenExternalCatalog around HiveConnector (CatalogFactory.java:110-118) and every seam routes through PluginDrivenMetadata.get (PluginDrivenExternalTable.java:133,159,189,449,565,606,716,820,881,923,954,1025,1060,1126,1277). This connector is ALSO a heterogeneous GATEWAY: it hosts iceberg-on-HMS and hudi-on-HMS as embedded SIBLING connectors (HiveConnector.getOrCreateIcebergSibling/getOrCreateHudiSibling, HiveConnector.java:530-592), whose metadata is per-statement-memoized keyed by owner label (HiveConnectorMetadata.memoizedSiblingMetadata, :302-305). hudi is NOT its own SPI type (CatalogFactory.java:51-55). Note: CachingHmsClient.java:85-88 and HiveFileListingCache.java:72-74 still carry a STALE 'Dormant / hms is not in SPI_READY_TYPES' javadoc — factually wrong at HEAD (commit 6e521aa64b2 / #65473 flipped hms live AND wired wrapWithCache in the same change but did not update these class docs); harmless to runtime, worth a doc fix." + }, + "currentCaches": [ + { + "name": "CachingHmsClient.tableCache (HmsTableInfo by (db,table))", + "scope": "metastore-client", + "keyedBy": "(dbName, tableName)", + "ttl": "24h default (86400s), cap 10000; legacy schema.cache.ttl-second remapped", + "file": "fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java:116,172-175" + }, + { + "name": "CachingHmsClient.partitionNamesCache (partition-name list)", + "scope": "metastore-client", + "keyedBy": "(dbName, tableName, maxParts)", + "ttl": "24h default, cap 10000; legacy partition.cache.ttl-second remapped", + "file": "fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java:117,187-190" + }, + { + "name": "CachingHmsClient.partitionsCache (per-partition HmsPartitionInfo, Trino/legacy shape)", + "scope": "metastore-client", + "keyedBy": "(dbName, tableName, partitionValues) — one entry PER partition object", + "ttl": "24h default, cap 100000; generation-guarded put vs REFRESH race", + "file": "fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java:118,202-243" + }, + { + "name": "CachingHmsClient.columnStatsCache (HMS column statistics)", + "scope": "metastore-client", + "keyedBy": "(dbName, tableName, requested-column-list)", + "ttl": "24h default, cap 10000", + "file": "fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java:119,274-279" + }, + { + "name": "HiveFileListingCache (partition directory listStatus -> List)", + "scope": "cross-query", + "keyedBy": "(dbName, tableName, location, partitionValues); connector-owned final field shared by scan provider + size-estimate", + "ttl": "24h default (86400s), cap 10000; legacy file.meta.cache.ttl-second remapped", + "file": "fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java:114,160-176" + }, + { + "name": "HiveConnector.partitionViewCache (shared fe-connector-cache ConnectorPartitionViewCache, PERF-06 'cache A' from #65829)", + "scope": "cross-query", + "keyedBy": "PartitionViewCacheKey(db, table, -1, -1) — hive is snapshot-less, no schema version, so one entry per table", + "ttl": "24h default, cap 1000 (meta.cache.hive.partition_view.*)", + "file": "fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java:112,136; consumed HiveConnectorMetadata.java:1160-1172" + }, + { + "name": "memoizedSiblingMetadata (per-statement iceberg/hudi sibling ConnectorMetadata memo)", + "scope": "per-statement", + "keyedBy": "'metadata:' + catalogId + ':' + ownerLabel (ICEBERG/HUDI)", + "ttl": "statement lifetime (ConnectorStatementScope); NONE scope = no memo", + "file": "fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java:302-305" + }, + { + "name": "HiveConnectorTransaction begin-time table snapshot memo (write path)", + "scope": "per-statement", + "keyedBy": "the single write-target table (hmsTableInfo captured in beginWrite; tableActions map)", + "ttl": "write-transaction lifetime", + "file": "fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java:209,546-558" + }, + { + "name": "ThriftHmsClient pooled connections (transport pool, NOT a metadata cache)", + "scope": "metastore-client", + "keyedBy": "connection pool of size hms.client.pool.size", + "ttl": "connection lifetime", + "file": "fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java:613-617" + } + ], + "funnelParticipation": { + "funneled": true, + "metadataMemoizesLoadedTable": "partial", + "evidence": "Primary path: fe-core acquires the ONE per-statement ConnectorMetadata via PluginDrivenMetadata.get(session, connector) (PluginDrivenMetadata.java:52-72; call sites PluginDrivenExternalTable.java:133 etc), which calls HiveConnector.getMetadata -> newMetadata(getOrCreateClient()) (HiveConnector.java:140-142,188-192). Sibling path: iceberg/hudi-on-HMS metadata is funneled per-statement via memoizedSiblingMetadata keyed by (catalogId, ownerLabel) (HiveConnectorMetadata.java:302-305,318-350). The read-side HiveConnectorMetadata holds NO per-instance resolvedTable field (fields at :203-241 are hmsClient/caches/sibling seams only), so getTableHandle (:399), getTableSchema (:493), getColumnHandles (:604), getColumnStatistics (:854), applyFilter (:1093), resolvePartitions all re-call hmsClient.getTable/listPartitionNames/getPartitions FRESH — but hmsClient is the CachingHmsClient wrapper (HiveConnector.createClient wraps at :645-646,670-672), a CROSS-QUERY cache, so repeated same-statement loads collapse to 1 remote RPC + N cache hits. The write side additionally memoizes the begin-time table snapshot per write-transaction (HiveConnectorTransaction.java:546-551). Net: the funnel 'one metadata per statement' invariant holds, and repeated remote loads are collapsed by the metastore-client cache rather than by a per-metadata-instance memo (this is why hive did not need iceberg's PERF-07 per-statement table load — iceberg had NO cross-query table cache after its cutover; hive does).", + "gaps": "Read-side has no per-statement resolvedTable memo: it relies wholly on CachingHmsClient's cross-query TTL cache. In the rare case where an entry is evicted or TTLs out BETWEEN two getTable calls of one statement, a 2nd remote RPC occurs (very low probability, P2). A belt-and-suspenders per-statement resolvedTable field (iceberg PERF-07 analog) would close it but is not warranted — an HMS getTable thrift RPC is far cheaper than an iceberg loadTable, and the cross-query cache already collapses the common case." + }, + "hotPathFindings": [ + { + "id": "HMS-H1", + "area": "query-plan", + "problem": "Table metadata load (getTable) is called repeatedly per planning pass across getTableHandle -> getTableSchema -> getColumnHandles -> getColumnStatistics, but every call routes through CachingHmsClient.tableCache (cross-query, 24h). First call = 1 thrift RPC, rest = cache hits. NOT the iceberg 3-7x re-load regression.", + "multiplier": "3-4x getTable per statement collapsed to 1 remote RPC + N hits", + "cost": "remote-io", + "severity": "none", + "memoizedAlready": true, + "file": "fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java:399,493,604,854 via CachingHmsClient.java:172" + }, + { + "id": "HMS-H2", + "area": "partitions", + "problem": "Partition enumeration for pruning/scan/stats/MTMV (listPartitionNames + getPartitions). All go through CachingHmsClient.partitionNamesCache + per-partition partitionsCache; the built List is additionally memoized cross-query by ConnectorPartitionViewCache (PERF-06 cache A).", + "multiplier": "scan+applyFilter+stats+MTMV each re-enumerate; all served from cache", + "cost": "remote-io", + "severity": "none", + "memoizedAlready": true, + "file": "fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java:1093-1106,1019-1025,1160-1172; HiveScanPlanProvider.java:492-499" + }, + { + "id": "HMS-H3", + "area": "file-list", + "problem": "Per-partition directory listStatus (the heaviest remote-IO in the scan hot path, O(partitions)) is served from the connector-owned HiveFileListingCache, shared by the scan path (listAndSplitFiles) and the row-count/size-estimate path (sumCachedFileSizes/estimateDataSizeByListingFiles) so a scan warms the estimate and vice-versa.", + "multiplier": "one listStatus per partition per scan, cached cross-query", + "cost": "remote-io", + "severity": "none", + "memoizedAlready": true, + "file": "fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java:537; HiveConnectorMetadata.java:945-950,1062-1063; HiveFileListingCache.java:174" + }, + { + "id": "HMS-H4", + "area": "split-enum", + "problem": "ACID (transactional) scan resolves surviving base/delta + delete-delta dirs via HiveAcidUtil.getAcidState per partition, UNCACHED (fileListingCache only serves the non-ACID path). This is a per-scan directory walk with no memo.", + "multiplier": "one getAcidState per partition per ACID scan, uncached", + "cost": "remote-io", + "severity": "none", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java:314" + }, + { + "id": "HMS-H5", + "area": "write", + "problem": "Write planning loads the table twice (planWrite.loadTable + beginWrite) and lists existing partitions (buildExistingPartitions: listPartitionNames+getPartitions). Double-load is explicitly accepted; both hit CachingHmsClient (1 RPC + 1 hit) and beginWrite reuses the begin-time snapshot within the txn.", + "multiplier": "2x getTable + partition list per write, collapsed by cache + per-txn memo", + "cost": "remote-io", + "severity": "none", + "memoizedAlready": true, + "file": "fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWritePlanProvider.java:103,105,255-258; HiveConnectorTransaction.java:549-551" + }, + { + "id": "HMS-H6", + "area": "schema", + "problem": "Read-side HiveConnectorMetadata has no per-statement resolvedTable field; a mid-statement CachingHmsClient TTL-expiry/eviction between two getTable calls would trigger a 2nd remote RPC. Common case fully collapsed by the cross-query cache; residual race only.", + "multiplier": "<=1 extra RPC per statement in a rare eviction race", + "cost": "remote-io", + "severity": "P2", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java:203-241" + } + ], + "authzConsistency": { + "perUserCredential": false, + "sessionUserRelevant": false, + "staleOrLeakRisk": "low", + "notes": "Hive vends NO per-user credential and has NO session=user analog. The metastore RPC runs under a SINGLE catalog-level identity: either the plugin-side Kerberos keytab principal (HiveConnector.buildPluginAuthenticator, :710-740, a fixed principal/keytab) or context.executeAuthenticated (catalog-level auth) — never a per-querying-user delegated credential, and there is no REST OIDC / per-identity metastore. Therefore the name-only cache keys (db,table[,parts/cols/location]) on CachingHmsClient/HiveFileListingCache/partitionViewCache CANNOT leak cross-user metadata the way iceberg.rest.session=user could: all users observe the same metastore identity's view, and per-user access is gated by Doris's own fe-core RBAC (AccessController) independent of these caches. This is exactly why the authz-cache-sharding build gate targets ONLY IcebergConnector.java (tools/check-authz-cache-sharding.sh TARGET_REL) and hive builds its caches unconditionally (HiveConnector.java:108-112,133-136). A fail-loud invariant guard (HiveConnector.java:548-555) rejects any iceberg-on-HMS sibling that ever declared SUPPORTS_USER_SESSION, precisely because the hive front door is not session=user and fe-core's per-user schema/name-cache bypass keys off the front-door connector. Write/read consistency: both share the same cross-query CachingHmsClient (immutable HmsTableInfo/HmsPartitionInfo), and the write transaction pins a begin-time table snapshot (HiveConnectorTransaction.java:549-551); ACID reads pin a write-id snapshot via getValidWriteIds per scan (HiveScanPlanProvider.java:305). Staleness is bounded by coarse REFRESH TABLE/DB/CATALOG (flush/flushDb/flushAll wired at HiveConnector.java:357-419) + TTL — Layer-3 authz isolation is N/A for hive." + }, + "frameworkPiecesNeeded": [ + { + "piece": "cross-query-table-cache", + "needed": "already-has", + "rationale": "CachingHmsClient.tableCache (db,table)->HmsTableInfo, 24h TTL, REFRESH-invalidated, wired live via HiveConnector.createClient->wrapWithCache (CachingHmsClient.java:116,172; HiveConnector.java:645-646). This is the metastore-client analog of iceberg's IcebergTableCache." + }, + { + "piece": "partition-cache", + "needed": "already-has", + "rationale": "Three layers already present: partitionNamesCache + per-partition partitionsCache (CachingHmsClient.java:117-118) AND the derived ConnectorPartitionViewCache (HiveConnector.java:112, from shared toolkit / #65829). Covers pruning, scan, stats, MTMV, listPartitions." + }, + { + "piece": "manifest-or-file-list-cache", + "needed": "already-has", + "rationale": "HiveFileListingCache (cross-query, keyed by db/table/location/partitionValues, 24h) is the hive analog of the iceberg manifest cache; serves both scan and size-estimate. ACID directory walk is intentionally uncached (snapshot/write-id dependent, legacy parity)." + }, + { + "piece": "shared-fe-connector-cache-adoption", + "needed": "already-has", + "rationale": "Uses fe-connector-cache substrate throughout: ConnectorPartitionViewCache + PartitionViewCacheKey (HiveConnector.java:112,136) and MetaCacheEntry/CacheSpec for CachingHmsClient (CachingHmsClient.java:20-21,133-140) and HiveFileListingCache (HiveFileListingCache.java:21-22,144-148)." + }, + { + "piece": "per-statement-funnel-memoization", + "needed": "no", + "rationale": "Funnel participation confirmed (PluginDrivenMetadata.get for primary; memoizedSiblingMetadata for siblings). A per-statement resolvedTable memo is NOT warranted: the cross-query CachingHmsClient already collapses repeated same-statement loads to 1 RPC, and an HMS getTable is far cheaper than an iceberg loadTable. Optional P2 belt-and-suspenders only." + }, + { + "piece": "write-txn-coholder", + "needed": "already-has", + "rationale": "HiveConnectorTransaction is the per-statement write transaction (CatalogStatementTransaction analog); it captures the begin-time table snapshot and reuses it for the reject-guard and sink build (HiveConnectorTransaction.java:209,549-551), so read+write share one metadata/txn." + }, + { + "piece": "format-cache", + "needed": "no", + "rationale": "Hive infers file format cheaply from the handle's inputFormat/serde (HiveFileFormat.detect, HiveScanPlanProvider.java:136-138; detectFormatType from cached tableInfo), hoisted once per scan. No unfiltered whole-table planFiles() fallback like iceberg PERF-03, so no format cache is needed." + }, + { + "piece": "comment-cache", + "needed": "no", + "rationale": "Hive serves table/column comments from the already-cached getTable (tableInfo params / ConnectorColumn), not via a separate N-per-query remote getComment load. Iceberg's PERF-05 comment cache existed because vended-credential catalogs null the table cache; hive always has the table cache, so no comment cache is needed." + }, + { + "piece": "per-scan-hoist", + "needed": "already-has", + "rationale": "Scan invariants (format detect, target split size, LZO/splittable flags, backend storage props) are computed once per scan before the partition loop (HiveScanPlanProvider.planScan:136-145, getScanNodeProperties:386-435). Per-file normalizeNativeUri is a cheap string op, not a remote/heavy invariant." + }, + { + "piece": "per-file-split-memo", + "needed": "no", + "rationale": "splitFile slices a file into byte ranges sharing the partition's value map by reference via PartitionScanInfo (HiveScanPlanProvider.java:628-655,707-718); no per-file JSON/partition-value recompute across a file's splits (hive has no iceberg-style per-file delete-carrier derivation to memoize)." + }, + { + "piece": "partition-cache", + "needed": "already-has", + "rationale": "(duplicate axis) covered by partitionNamesCache + partitionsCache + partitionViewCache above." + }, + { + "piece": "authz-session-user-isolation", + "needed": "no", + "rationale": "Hive has no per-user delegated credential and no session=user; metastore RPC uses a single catalog identity and access is gated by fe-core RBAC. Name-only caches cannot leak cross-user (see authzConsistency). The authz-cache-sharding gate deliberately targets only IcebergConnector." + }, + { + "piece": "cross-query-table-cache", + "needed": "already-has", + "rationale": "(duplicate axis) covered by CachingHmsClient.tableCache above." + } + ], + "overallVerdict": "DO NOT apply new iceberg-style hot-path caches to hive — the connector is already framework-aligned and well-cached, and applying iceberg's Layer-1/Layer-3 would be redundant. Hive already has (a) the connector-side cross-query cache pattern: CachingHmsClient's 4 metastore caches (table/partition-names/partition/column-stats) + HiveFileListingCache (directory listings) + ConnectorPartitionViewCache (derived partition views), all live and REFRESH/TTL-invalidated; (b) full Layer-2 funnel participation — primary via PluginDrivenMetadata.get, siblings via per-statement memoizedSiblingMetadata, and a per-statement write transaction (HiveConnectorTransaction) that co-holds the begin-time table snapshot; (c) Layer-3 is N/A because hive has no session=user. No P0/P1 repeated-remote-load gap exists on any planning entrypoint (table/schema/partition/file-list/pushdown/stats/MTMV/write) — each heavy op sits behind a cache. Prioritized recommendations, all LOW: (1) fix the STALE 'Dormant / hms is not in SPI_READY_TYPES' javadocs on CachingHmsClient.java:85-88, HiveFileListingCache.java:72-74, and HiveConnector.wrapWithCache:667-668 (hms is live at HEAD) — documentation only; (2) OPTIONAL P2 belt-and-suspenders per-statement resolvedTable memo on the read-side HiveConnectorMetadata to close the rare mid-statement TTL/eviction re-RPC race (HMS-H6) — not warranted on cost grounds; (3) leave the ACID getAcidState per-scan listing uncached (HMS-H4) — it is snapshot/write-id dependent and legacy-parity correct.", + "confidence": "high", + "verify": { + "verdict": "CONFIRMED", + "confirmed": [ + "HMS-H1", + "HMS-H2", + "HMS-H3", + "HMS-H4", + "HMS-H5", + "HMS-H6" + ], + "corrections": [ + { + "claim": "HiveConnectorTransaction begin-time table snapshot ... hmsTableInfo captured in beginWrite (HiveConnectorTransaction.java:209)", + "issue": "Off-by-2 line cite: the hmsTableInfo field is declared at :123 and assigned inside beginWrite at :211, not :209. Immaterial — :209 falls within the beginWrite method and the reuse-in-getTable range :546-558 (specifically :549-551) is exact.", + "correction": "Field decl HiveConnectorTransaction.java:123 (private volatile HmsTableInfo hmsTableInfo); assignment this.hmsTableInfo = table at :211; reuse at :551. No substantive error." + }, + { + "claim": "detailedMarkdown enumerates only 3 stale 'Dormant/hms not in SPI_READY_TYPES' javadocs (CachingHmsClient:85-88, HiveFileListingCache:72-74, HiveConnector.wrapWithCache:667-668)", + "issue": "Non-exhaustive, not wrong: the icebergSibling/hudiSibling field comments at HiveConnector.java:118 and :126-127 also carry stale 'Dormant until hms enters SPI_READY_TYPES' text at HEAD.", + "correction": "Two additional stale doc sites exist at HiveConnector.java:118,126-127; harmless, additive to the same doc-fix recommendation (LOW)." + } + ] + } + }, + { + "connector": "hudi", + "migrationStatus": { + "inSpiReadyTypes": false, + "liveness": "live", + "notes": "\"hudi\" is deliberately NOT in SPI_READY_TYPES (CatalogFactory.java:56-57 lists {jdbc, es, trino-connector, max_compute, paimon, iceberg, hms}; the :50-55 comment forbids adding it; HudiConnectorProvider.java:45-47 returns the sibling-only type string \"hudi\"). hudi is reached ONLY as an embedded sibling of the hms gateway: HiveConnector.newMetadata wires this::getOrCreateHudiSibling (HiveConnector.java:188-191), and HiveConnectorMetadata.getTableHandle diverts a HUDI-detected table to hudiSiblingMetadata(session).getTableHandle(...) (HiveConnectorMetadata.java:415-417). LIVENESS = live (not dormant): \"hms\" WAS added to SPI_READY_TYPES by #65473 / commit 6e521aa64b2 (2026-07-16), and the legacy fe-core hudi path is fully removed (no HudiScanNode/HudiExternalMetaCache/HMSExternalTable/HudiDlaTable remain in fe-core). CONFLICT TO SURFACE (Rule 7): numerous in-tree comments still say \"dormant until hms enters SPI_READY_TYPES\" / \"today getTableHandle is never called\" (HudiConnectorMetadata.java:391, HiveConnectorMetadata.java:410,1802,1942,1972,2007) — these are STALE, written before #65473 flipped hms live; they no longer reflect HEAD and should not be trusted for liveness." + }, + "currentCaches": [ + { + "name": "Per-statement ConnectorMetadata (Layer-2 funnel memo)", + "scope": "per-statement", + "keyedBy": "(catalogId, HUDI_LABEL) on ConnectorStatementScope", + "ttl": "statement lifetime", + "file": "fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java:302" + }, + { + "name": "HMS ThriftHmsClient (RAW — no result caching)", + "scope": "metastore-client", + "keyedBy": "single pooled client memoized on HudiConnector; NOT wrapped in CachingHmsClient, so getTable/tableExists/listTables/listPartitionNames are fresh Thrift RPCs on every call", + "ttl": "none (connector lifetime pool)", + "file": "fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java:186" + }, + { + "name": "pluginAuth (HadoopAuthenticator)", + "scope": "cross-query", + "keyedBy": "catalog-level single-owner Kerberos identity; double-checked memo on HudiConnector", + "ttl": "connector lifetime", + "file": "fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java:196" + }, + { + "name": "Hudi InternalSchemaCache (library-internal, NOT a Doris cache)", + "scope": "none", + "keyedBy": "(commitTime, metaClient) inside hudi-common; reset per fresh metaClient build", + "ttl": "hudi-lib managed", + "file": "fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiSchemaUtils.java:278" + } + ], + "funnelParticipation": { + "funneled": true, + "metadataMemoizesLoadedTable": "no", + "evidence": "The sibling metadata IS routed through the Layer-2 funnel: HiveConnectorMetadata.memoizedSiblingMetadata (HiveConnectorMetadata.java:302-305) calls session.getStatementScope().getOrCreateMetadata(\"metadata:\"+catalogId+\":\"+ownerLabel, () -> owner.getMetadata(session)); hudiSiblingMetadata (:332-334) and the by-handle siblingMetadata (:347-350) share that one instance per statement under HUDI_LABEL. So exactly ONE HudiConnectorMetadata object is built per statement. BUT that object memoizes NOTHING internally: HudiConnectorMetadata holds only {hmsClient, properties, metaClientExecutor, storageHadoopConfig} (fields at :153-174) — no resolved metaClient/schema/timeline/partitions field. Every getTableSchema/getColumnHandles/applyFilter/listPartitions*/beginQuerySnapshot/collectPartitions call rebuilds a fresh HoodieTableMetaClient via HudiScanPlanProvider.buildMetaClient (metadata: getSchemaFromMetaClient :800-801, latestInstant :748-750, collectPartitions :707-713, applyFilter :289-291). Worse, the scan provider is a SEPARATE object (HudiConnector.getScanPlanProvider returns new HudiScanPlanProvider, :136-138) and builds its OWN metaClient twice (planScan :148-151 and getScanNodeProperties :363), independent of the metadata.", + "gaps": "The funnel collapses the metadata OBJECT to one-per-statement, but does NOT collapse repeated remote table loads: metaClient is (re)built 5-6x per planning pass and the schema/timeline re-read each time. Iceberg-style INTERNAL memoization (per-statement resolvedTable + cross-query table cache, PERF-01/PERF-07 analog) is still required; the funnel alone does not fix it." + }, + "hotPathFindings": [ + { + "id": "HD-P01", + "area": "query-plan", + "problem": "HoodieTableMetaClient is rebuilt fresh at EVERY metadata + scan entrypoint (getColumnHandles->getTableSchema, beginQuerySnapshot->latestInstant, applyFilter, MVCC getTableSchema, planScan, getScanNodeProperties) — ~5-6 builds per planning pass for the SAME table. Each build does remote IO: reads .hoodie/hoodie.properties + table config and loads/lists the active timeline directory. Zero memoization on the funnel-memoized metadata or across the metadata/scan-provider split. Direct analog of iceberg PERF-01/PERF-07 (loadTable 3-7x/pass).", + "multiplier": "~5-6x per planning pass (constant, per-query; Variant B/D)", + "cost": "remote-io", + "severity": "P1", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiScanPlanProvider.java:148" + }, + { + "id": "HD-P02", + "area": "schema", + "problem": "Latest table Avro schema + InternalSchema re-resolved ~4x per pass on independent metaClients: getSchemaFromMetaClient (getTableAvroSchema(true)+resolveTableInternalSchema, HudiConnectorMetadata:819-821) fires for getColumnHandles AND the 3-arg MVCC getTableSchema; planScan re-resolves (getTableAvroSchema :188, resolveJniColumnSchema :194, resolveTableInternalSchema :220); getScanNodeProperties re-resolves again (getTableAvroSchema :382, dict :383). Each re-reads latest-commit schema/metadata remotely. No schema memo.", + "multiplier": "~4x per pass (Variant B)", + "cost": "remote-io", + "severity": "P1", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorMetadata.java:797" + }, + { + "id": "HD-P03", + "area": "partitions", + "problem": "Partition listing (listAllPartitionPaths = HoodieTableMetadata.create + getAllPartitionPaths, a metadata-table scan) + latestCompletedInstant timeline read are re-run with NO (table,instant)-keyed cache: collectPartitions backs listPartitions/listPartitionNames/listPartitionValues (:707-713); applyFilter non-hive-sync lists again (:289-291); planScan resolvePartitions lists once more (:281,648). Across one MTMV refresh fe-core re-enumerates listPartitions/freshness 4-6x, each a fresh metadata-table scan. No iceberg-IcebergPartitionCache / hive-partitionViewCache analog.", + "multiplier": "3x within one pass + 4-6x per MTMV refresh (Variant A/B)", + "cost": "remote-io", + "severity": "P1", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiScanPlanProvider.java:734" + }, + { + "id": "HD-P04", + "area": "schema", + "problem": "HMS metadata is served by a RAW ThriftHmsClient — hudi does NOT wrap CachingHmsClient the way HiveConnector does (HudiConnector.createClient returns new ThriftHmsClient at :186; contrast HiveConnector.wrapWithCache/CachingHmsClient). So getTableHandle's tableExists+getTable (2 RPCs, :204-207) and, for hive-sync tables, listPartitionNames in BOTH applyFilter (:278) and collectPartitions (:695) re-hit HMS every query with no cross-query cache. Variant C: the cache class (CachingHmsClient, keyed (db,table), 24h TTL) already exists and is used by hive but hudi's hot path bypasses it.", + "multiplier": "2+ metastore RPCs/pass, uncached across every query (Variant C)", + "cost": "remote-io", + "severity": "P2", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java:186" + }, + { + "id": "HD-P05", + "area": "split-enum", + "problem": "Per-scan loop-invariants recomputed: storageHadoopConfig(context) (translates ALL StorageProperties -> hadoop map) + buildHadoopConf are rebuilt in BOTH planScan.buildHadoopConf (:912-927) and getScanNodeProperties (:341,363); the storage-URI normalizer context.normalizeStorageUri is invoked per base file / per file-slice (collectCowSplits :445, collectMorSplits :474-476) rather than hoisted once per scan (iceberg PERF-06 hoisted this). CPU/alloc only, not remote IO; low impact but a clean per-scan/per-file hoist.", + "multiplier": "2x per pass (config) + O(N_files) (uri normalize) (Variant D)", + "cost": "cpu", + "severity": "P2", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiScanPlanProvider.java:912" + } + ], + "authzConsistency": { + "perUserCredential": false, + "sessionUserRelevant": false, + "staleOrLeakRisk": "low", + "notes": "hudi authenticates with a CATALOG-LEVEL single identity, never per-user: HudiConnector runs metaClient/HMS work under a plugin-side Kerberos doAs on one keytab (buildPluginAuthenticator :223-238, metaClientExecutor :102-122) or the FE-injected context.executeAuthenticated (NOOP/SIMPLE for a sibling). There is NO REST-OIDC / iceberg.rest.session=user analog and NO per-identity metastore. HudiConnector does not override getCapabilities, so it inherits the empty default (no SUPPORTS_USER_SESSION), and the hive gateway front-door guard already enforces that no session=user sibling can attach (HiveConnector:548-562, mirrored contract). CONSEQUENCE: name-only caches are authz-SAFE — every user sees the same catalog-identity view, exactly like hive's shared CachingHmsClient. The iceberg Layer-3 isolation (nulling caches under isUserSessionEnabled / shouldBypassSchemaCache) is NOT needed here. WRITE CONSISTENCY: none required — hudi is read-only (beginTransaction throws :395-398; getWritePlanProvider left at SPI-default null), so there is no read+write shared-metadata/txn co-holding requirement." + }, + "frameworkPiecesNeeded": [ + { + "piece": "per-statement-funnel-memoization", + "needed": "partial", + "rationale": "The funnel already routes hudi (one HudiConnectorMetadata per statement via memoizedSiblingMetadata), so the OBJECT-level piece is in place. But the metadata does not memoize its loaded metaClient/schema within the statement, and the scan provider builds its own metaClient independently. Needed: a per-statement resolved-metaClient+schema memo (iceberg PERF-07 analog) that BOTH HudiConnectorMetadata and HudiScanPlanProvider share via the ConnectorStatementScope, collapsing the 5-6 builds/pass to one." + }, + { + "piece": "cross-query-table-cache", + "needed": "yes", + "rationale": "No cross-query HoodieTableMetaClient/table-config cache exists (iceberg IcebergTableCache analog). A cache keyed by basePath (TTL + REFRESH-invalidated) would collapse repeated metaClient builds across successive queries of the same table, the single biggest cost (HD-P01)." + }, + { + "piece": "partition-cache", + "needed": "yes", + "rationale": "No (table,instant)-keyed partition-listing cache (iceberg IcebergPartitionCache / hive ConnectorPartitionViewCache analog). Directly addresses HD-P03: shared by listPartitions/listPartitionNames/listPartitionValues, applyFilter, resolvePartitions, and the 4-6 re-enumerations of one MTMV refresh + SHOW PARTITIONS." + }, + { + "piece": "format-cache", + "needed": "no", + "rationale": "Not applicable. COW/MOR is read from the authoritative Hudi table config / handle (metaClient.getTableType, HudiScanPlanProvider:156) and detectFileFormat is a cheap suffix check (:899-910). There is no iceberg-style getFileFormat whole-table planFiles() fallback to memoize." + }, + { + "piece": "comment-cache", + "needed": "no", + "rationale": "HudiConnectorMetadata does not override getComment, so there is no per-table remote comment load on the information_schema/SHOW TABLE STATUS path to cache (unlike iceberg PERF-05)." + }, + { + "piece": "manifest-or-file-list-cache", + "needed": "partial", + "rationale": "planScan builds a FileSystemView + lists partition paths per query (HudiScanPlanProvider:273-281); there is no file-listing cache (hive has HiveFileListingCache). A partition-path/file-listing cache could help repeated planning and MTMV, but per-query freshness matters and this is lower priority than the metaClient/schema/partition memos above." + }, + { + "piece": "per-scan-hoist", + "needed": "yes", + "rationale": "planScan and getScanNodeProperties each build their own metaClient and re-resolve schema (HD-P01/HD-P02); storageHadoopConfig/buildHadoopConf are rebuilt twice and normalizeStorageUri runs per file (HD-P05). Resolve metaClient+schema+timeline once per scan and reuse; bake the storage/hadoop config and uri-normalizer once per scan." + }, + { + "piece": "per-file-split-memo", + "needed": "no", + "rationale": "Already largely fine: partition values are parsed once per partition and shared across a partition's files (collectCowSplits/collectMorSplits); the per-file schemaId resolver is genuinely per-file (each base file has its own written schema version), not a hoistable invariant. Only the per-file uri-normalize (folded into per-scan-hoist) remains." + }, + { + "piece": "authz-session-user-isolation", + "needed": "no", + "rationale": "hudi is catalog-level single-identity (Kerberos doAs / simple), never session=user, so name-only caches cannot leak a can-LIST-cannot-LOAD user's metadata. No isUserSessionEnabled nulling / shouldBypassSchemaCache needed." + }, + { + "piece": "shared-fe-connector-cache-adoption", + "needed": "yes", + "rationale": "hudi uses NONE of the shared toolkit — no fe-connector-cache dependency in its pom, no CacheFactory/CacheSpec/MetaCacheEntry/ConnectorPartitionViewCache, and it does not even wrap CachingHmsClient. Adopting the toolkit (wrap the HMS client in CachingHmsClient like HiveConnector; use ConnectorPartitionViewCache for partitions) is the natural, lowest-risk substrate for HD-P03/HD-P04 and matches the reference pattern." + }, + { + "piece": "write-txn-coholder", + "needed": "no", + "rationale": "hudi is read-only in this catalog (beginTransaction throws; no write plan provider), so there is no write transaction to co-hold with the read metadata." + } + ], + "overallVerdict": "hudi is LIVE (reachable in production as the hudi-on-HMS sibling now that hms is in SPI_READY_TYPES and the getTableHandle HUDI divert is wired) and is the LEAST-cached of the SPI connectors: ZERO connector-side cross-query metadata caches, ZERO per-statement metaClient/schema memoization, and — unlike hive — it does not even wrap CachingHmsClient (raw ThriftHmsClient). It IS correctly routed through the Layer-2 per-statement funnel, but the funnel only collapses the metadata OBJECT; the heavy remote loads (HoodieTableMetaClient build 5-6x/pass, Avro+InternalSchema resolve ~4x, partition metadata-table scan 3x/pass and 4-6x/MTMV) are re-done at every entrypoint with no memo. The iceberg framework pattern applies directly and is NEEDED. Priority order: (1) per-statement resolved-metaClient+schema memo shared by metadata AND scan provider (kills HD-P01/HD-P02, the flagship); (2) wrap the HMS client in CachingHmsClient — a one-liner parity with hive that instantly fixes HD-P04; (3) cross-query metaClient/table cache + (table,instant)-keyed partition cache via ConnectorPartitionViewCache for HD-P03 (MTMV/SHOW PARTITIONS re-enumeration); (4) per-scan hoist of metaClient/schema/storage-config/uri-normalizer (HD-P05). NO authz-isolation work (not session=user) and NO write-txn work (read-only) are required. Also surface the stale 'dormant until hms enters SPI_READY_TYPES' comments as a cleanup.", + "confidence": "high", + "verify": { + "verdict": "ADJUSTED", + "confirmed": [ + "HD-P01", + "HD-P02", + "HD-P03", + "HD-P04", + "HD-P05" + ], + "corrections": [ + { + "claim": "authzConsistency: 'the hive gateway front-door guard already enforces that no session=user sibling can attach (HiveConnector:548-562, mirrored contract)' covering hudi.", + "issue": "The SUPPORTS_USER_SESSION fail-loud guard at HiveConnector.java:548-556 lives inside getOrCreateIcebergSibling and its error text is iceberg-specific ('iceberg-on-HMS sibling ... unexpectedly declares SUPPORTS_USER_SESSION'). getOrCreateHudiSibling (:576-591) has NO equivalent guard, so the '548-562 mirrored contract' does not literally cover the hudi sibling.", + "correction": "hudi's authz-safety instead rests on two verified facts: (a) HudiConnector does not override getCapabilities, so it inherits the empty default with no SUPPORTS_USER_SESSION (confirmed — the only getCapabilities hits are javadoc in HudiConnectorMetadata, no override); and (b) the hive front door itself is never session=user. The audit's substantive conclusion (name-only caches are authz-safe, no Layer-3 isolation / shouldBypassSchemaCache needed) is UNCHANGED and correct." + }, + { + "claim": "HD-P03 multiplier: '3x within one pass' partition metadata-table scans for a filtered partitioned SELECT.", + "issue": "For a FILTERED query the code deliberately dedups to ~1 listing: applyFilter (non-hive-sync) lists once at HudiConnectorMetadata.java:289-291, then resolvePartitions returns the prunedPaths and short-circuits WITHOUT calling listAllPartitionPaths (HudiScanPlanProvider.java:635-638). The connector's own comment at HudiConnectorMetadata.java:288 states 'a filtered query lists exactly once (here) instead of there.' So '3x within one pass' is an upper bound, not the single-pass norm.", + "correction": "The '3x' only materializes when fe-core independently invokes multiple of listPartitions / listPartitionNames / listPartitionValues (each re-calls collectPartitions -> a fresh HoodieTableMetadata scan, HudiConnectorMetadata.java:707-713) and/or on an UNFILTERED scan where resolvePartitions does list (HudiScanPlanProvider.java:648). The CORE finding — a metadata-table partition scan re-run at multiple entrypoints with NO (table,instant)-keyed cache — is CONFIRMED; only the per-pass count is query-shape-dependent." + }, + { + "claim": "HD-P01 multiplier '~5-6x per planning pass' and HD-P05 citation 'planScan.buildHadoopConf (:912-927)'.", + "issue": "Several metaClient build sites are conditional: applyFilter builds one only when a partition predicate is present AND non-hive-sync (:289-291); getScanNodeProperties builds one only when !force_jni (:363); the 3-arg MVCC getTableSchema fires only on the time-travel/MVCC path. HD-P05's '912-927' is the buildHadoopConf METHOD definition, not the planScan call site (the actual call is planScan:147).", + "correction": "Minimum unconditional metaClient builds per pass are ~3-4 (getColumnHandles->getSchemaFromMetaClient :800-801, beginQuerySnapshot->latestInstant :749-750, planScan :148-151, getScanNodeProperties :363), reaching 5-6 with the conditional sites — so '5-6x' is a valid upper bound. HD-P05 substance (buildHadoopConf/storageHadoopConfig built twice per pass at :147 and :363; normalizeNativeUri per-file at :445 and :476) is correct; only the line citation is loose. Neither adjustment weakens the 'not memoized' core." + } + ] + } + }, + { + "connector": "paimon", + "migrationStatus": { + "inSpiReadyTypes": true, + "liveness": "live", + "notes": "paimon is in SPI_READY_TYPES (fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java:57). It is a first-class standalone catalog type routed through the SPI plugin path: PaimonConnectorProvider -> PaimonConnector -> PaimonConnectorMetadata / PaimonScanPlanProvider, wrapped by PluginDrivenExternalCatalog. It is fully LIVE, not a sibling: read + MVCC/time-travel + DDL (create/drop db+table) + system tables + stats + SHOW CREATE are all implemented on the connector. Write is deliberately NOT migrated (getCapabilities declares no write capability; PaimonConnector.java:318)." + }, + "currentCaches": [ + { + "name": "PaimonLatestSnapshotCache (latest-snapshot pin)", + "scope": "cross-query", + "keyedBy": "org.apache.paimon.catalog.Identifier(db,table)", + "ttl": "meta.cache.paimon.table.ttl-second, default 86400s (24h); <=0 disables (always-live no-cache catalog). Caffeine expireAfterAccess + maxSize 1000. Invalidated by REFRESH TABLE/DB/CATALOG (PaimonConnector.java:256/278/285).", + "file": "fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java:48 (field PaimonConnector.java:124, built :154)" + }, + { + "name": "PaimonSchemaAtMemo (schema-at-schemaId memo)", + "scope": "cross-query", + "keyedBy": "(db,table,sysTableName,branchName,schemaId) -> PaimonSchemaSnapshot(fields+partKeys+pkKeys)", + "ttl": "no TTL; bounded maxSize 10000, clear-on-overflow. Immutable value (schemaId content is write-once). Invalidated by REFRESH TABLE/DB/CATALOG.", + "file": "fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonSchemaAtMemo.java:49 (shared by metadata time-travel getTableSchema PaimonConnectorMetadata.java:299-300 AND scan schema-evolution dict PaimonScanPlanProvider.java:1567)" + }, + { + "name": "partitionViewCache (ConnectorPartitionViewCache, shared fe-connector-cache toolkit)", + "scope": "cross-query", + "keyedBy": "PartitionViewCacheKey(db,table,snapshotId,schemaId=-1) -> built List", + "ttl": "meta.cache.paimon.partition_view.(enable|ttl-second|capacity), default ON / 86400s / 1000. snapshotId read from latestSnapshotCache so it tracks 'current'. Invalidated by REFRESH TABLE/DB/CATALOG (PaimonConnector.java:263/280/287).", + "file": "fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java:1012-1052 (field PaimonConnector.java:143, built :158)" + }, + { + "name": "Transient Table fat-handle (per-op resolved-table memo)", + "scope": "per-scan", + "keyedBy": "PaimonTableHandle instance identity (transient Table set at getTableHandle)", + "ttl": "life of the handle object; lost on Java serialization round-trip (FE->BE) -> reload via PaimonTableResolver. NOT shared across distinct fe-core ops (each resolveConnectorTableHandle rebuilds the handle); IS shared within one SPI method and within one scan node (PluginDrivenScanNode caches currentHandle + resolveScanProvider).", + "file": "fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTableHandle.java:89 (set PaimonConnectorMetadata.java:221, read PaimonTableResolver.java:65)" + }, + { + "name": "paimon SDK CachingCatalog (tableCache / partitionCache / manifestCache / databaseCache)", + "scope": "metastore-client", + "keyedBy": "Identifier -> Table; Identifier -> List; Path -> manifest segments (Caffeine + SegmentsCache)", + "ttl": "paimon cache.* options (cache.expiration-interval etc.), SDK defaults; CACHE_ENABLED default true so the live catalog IS wrapped (CatalogFactory.createCatalog -> CachingCatalog.tryToCreate). This is the substrate that makes repeated getTable / listPartitions / manifest reads in-memory hits.", + "file": "org.apache.paimon.catalog.CachingCatalog (paimon-core 1.3.1, external SDK; wrapped in PaimonConnector.createCatalogFromContext at PaimonConnector.java:454)" + }, + { + "name": "DRIVER_CLASS_LOADER_CACHE / REGISTERED_DRIVER_KEYS", + "scope": "none", + "keyedBy": "resolved driver URL / url#class (JDBC-flavor driver jar dedup, process-wide static)", + "ttl": "process lifetime; not a metadata cache", + "file": "fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java:89-90" + } + ], + "funnelParticipation": { + "funneled": true, + "metadataMemoizesLoadedTable": "partial", + "evidence": "PaimonConnectorMetadata is acquired only through the Layer-2 funnel: PluginDrivenExternalTable.resolveConnectorTableHandle uses PluginDrivenMetadata.get(session, connector) (PluginDrivenMetadata.java:54-72 memoizes one ConnectorMetadata per (statement,catalog)); PluginDrivenScanNode additionally caches it in cachedMetadata (PluginDrivenScanNode.java:189/201-207) and memoizes resolveScanProvider() by currentHandle identity (PluginDrivenScanNode.java:182,245-259). So within a scan node planScan + getScanNodeProperties share one provider AND one currentHandle (with its transient Table), and resolveScanTable's transient hit works across them. schemaAtMemo is injected from the long-lived PaimonConnector into BOTH getMetadata (PaimonConnector.java:247) and getScanPlanProvider (PaimonConnector.java:311), so it is one per-catalog instance.", + "gaps": "The metadata does NOT hold a per-metadata-instance loaded-Table memo: getMetadata returns a FRESH PaimonConnectorMetadata per statement (PaimonConnector.java:244-248), and each fe-core op re-calls resolveConnectorTableHandle -> getTableHandle -> catalogOps.getTable (PaimonConnectorMetadata.java:211), building a new handle with a freshly-loaded transient Table. This is the SAME structural re-getTableHandle pattern iceberg's audit flagged, BUT for paimon it is not a remote-IO multiplier: paimon SDK CachingCatalog.tableCache makes each getTable an in-memory hit (default cache.enabled=true), and the transient handle collapses reloads within a single op. So the funnel's 'one metadata per statement' does not itself collapse table loads for paimon; the collapse is achieved by the paimon SDK cache + the transient fat-handle instead. A per-statement 'load once' table memo (iceberg PERF-07) would only save in-memory tableCache lookups + setPaimonTable churn -- negligible." + }, + "hotPathFindings": [ + { + "id": "PA-1", + "area": "partitions", + "problem": "listPartitionNames (PaimonConnectorMetadata.java:988) and listPartitionValues (PaimonConnectorMetadata.java:1057) call collectPartitions() DIRECTLY, bypassing partitionViewCache; only listPartitions() (PaimonConnectorMetadata.java:1019-1022) routes through the derived cache. So SHOW PARTITIONS and the partition_values() TVF re-run the display-name rendering (and re-issue catalogOps.listPartitions) instead of hitting the cross-query derived view. Variant C (cache exists, sibling hot paths miss it).", + "multiplier": "1x per SHOW PARTITIONS / partition_values() call (these are distinct statements, not looped); raw remote listPartitions blunted by paimon SDK partitionCache, so the residual cost is the CPU re-render only", + "cost": "cpu", + "severity": "P2", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java:988" + }, + { + "id": "PA-2", + "area": "query-plan", + "problem": "Per-statement table load: each fe-core metadata op rebuilds the handle via getTableHandle -> catalogOps.getTable (the iceberg re-getTableHandle pattern). NOT collapsed by the metadata itself, but blunted to an in-memory hit by paimon SDK CachingCatalog.tableCache and by the transient fat-handle within a single op.", + "multiplier": "~4-7 getTable per statement, each an in-memory CachingCatalog hit after the first (not remote)", + "cost": "remote-io", + "severity": "none", + "memoizedAlready": true, + "file": "fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java:211" + }, + { + "id": "PA-3", + "area": "stats-rowcount", + "problem": "getTableStatistics -> catalogOps.rowCount(table) sums split.rowCount() over table.newReadBuilder().newScan().plan().splits() -- a full manifest plan just to compute the base row count. No connector-level memo.", + "multiplier": "once per statement; caught cross-query by fe-core RowCountCache (PluginDrivenExternalTable.java:907) and manifest reads blunted by paimon SDK manifestCache", + "cost": "remote-io", + "severity": "none", + "memoizedAlready": true, + "file": "fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java:1205" + }, + { + "id": "PA-4", + "area": "schema", + "problem": "getTableSchema -> catalogOps.latestSchema(table) is a live schemaManager().latest() read of the schema directory (intentionally NOT off the frozen Table.rowType(), to catch external ALTER without a new snapshot). Not connector-cached.", + "multiplier": "once per fe-core schema-cache miss; cross-query caught by the fe-core generic schema cache (with schemaCacheTtlSecondOverride wiring meta.cache.paimon.table.ttl-second)", + "cost": "remote-io", + "severity": "none", + "memoizedAlready": true, + "file": "fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java:250" + }, + { + "id": "PA-5", + "area": "split-enum", + "problem": "planScanInternal -> planSplits(scan) reads snapshot/manifest files to enumerate the query's splits. This IS the query's data-planning read (not cacheable per-query). Per-scan/per-file invariants around it (vendedToken PaimonScanPlanProvider.java:559-560, weightDenominator :565, targetSplitSize lazy-once :593/624-626, partitionValues once per dataSplit reused across sub-splits :605/634) are already hoisted.", + "multiplier": "once per scan; repeated manifest reads across queries blunted by paimon SDK manifestCache", + "cost": "remote-io", + "severity": "none", + "memoizedAlready": true, + "file": "fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java:461" + } + ], + "authzConsistency": { + "perUserCredential": false, + "sessionUserRelevant": false, + "staleOrLeakRisk": "none — name-only caches are safe for paimon. Unlike iceberg.rest.session=user (where per-user authorization lives inside the delegated loadTable), a paimon catalog authenticates once at catalog-creation time (Kerberos plugin UGI / HMS principal / static or JDBC creds), not per Doris session identity. The REST vended credential is extracted table-level from the shared catalog's RESTTokenFileIO at scan time (PaimonScanPlanProvider.extractVendedToken :916-927), NOT keyed on the querying Doris user, so a shared cross-query cache cannot serve one user's metadata to another. Confirmed: no SUPPORTS_USER_SESSION / isUserSessionEnabled anywhere in fe-connector-paimon or fe-connector-metastore-paimon; the PaimonRestMetaStoreProvider has zero per-user/identity/delegation logic; and the authz gate tools/check-authz-cache-sharding.sh targets IcebergConnector ONLY (paimon caches are intentionally not gated).", + "notes": "The three connector caches are constructed unconditionally (never nulled), which is correct for paimon (PaimonConnector.java:138-142 documents the rationale). Write-path read+write-share-one-txn consistency (iceberg's CatalogStatementTransaction co-holder) is N/A: paimon write is not migrated, so there is no write metadata/transaction to co-hold. Read-side MVCC consistency (stable snapshot pin across a query) is provided by PaimonLatestSnapshotCache feeding beginQuerySnapshot -> applySnapshot -> scan.snapshot-id." + }, + "frameworkPiecesNeeded": [ + { + "piece": "per-statement-funnel-memoization", + "needed": "already-has", + "rationale": "PaimonConnectorMetadata is acquired only through PluginDrivenMetadata.get (funnel memoizes one metadata per statement) and PluginDrivenScanNode.cachedMetadata + resolveScanProvider memo. Routing is complete. A per-statement loaded-Table memo on the metadata is NOT additionally needed (see cross-query-table-cache)." + }, + { + "piece": "cross-query-table-cache", + "needed": "no", + "rationale": "iceberg needed IcebergTableCache because post-cutover loadTable is a live REST/HMS round-trip. Paimon's live catalog is wrapped in paimon SDK CachingCatalog (tableCache, default on), so getTable is already an in-memory cross-query hit; and PaimonLatestSnapshotCache separately provides the legacy stable-snapshot-pin semantics a bare Table object would not (Table.latestSnapshot() reads live). Porting a connector-level Table cache would duplicate the SDK cache." + }, + { + "piece": "partition-cache", + "needed": "already-has", + "rationale": "partitionViewCache (ConnectorPartitionViewCache, cross-query, keyed by db/table/snapshotId) memoizes the BUILT partition view, layered above paimon SDK partitionCache which memoizes the raw remote listPartitions. Equivalent to IcebergPartitionCache. (One gap: listPartitionNames/Values bypass it — finding PA-1.)" + }, + { + "piece": "format-cache", + "needed": "no", + "rationale": "No IcebergFormatCache analog is needed: paimon reads the default file format from table.options().getOrDefault(FILE_FORMAT,'parquet') (PaimonScanPlanProvider.java:525) — an in-memory option read off the cached Table — and per-file format by suffix (getFileFormatBySuffix). There is no unfiltered whole-table planFiles() fallback like iceberg PR #64134's." + }, + { + "piece": "comment-cache", + "needed": "no", + "rationale": "No IcebergCommentCache analog is needed: paimon's table comment/properties come from ((DataTable)table).coreOptions().toMap() off the cached Table (PaimonConnectorMetadata.java:330); there is no separate remote getComment loaded N-per-query for information_schema.tables / SHOW TABLE STATUS." + }, + { + "piece": "manifest-or-file-list-cache", + "needed": "already-has", + "rationale": "paimon SDK CachingCatalog carries a manifestCache (SegmentsCache) that caches manifest content for both planSplits and the rowCount plan. A connector-level manifest cache (iceberg PERF-04) is unnecessary." + }, + { + "piece": "per-scan-hoist", + "needed": "already-has", + "rationale": "The scan already hoists loop-invariants once per scan: vended REST token (PaimonScanPlanProvider.java:559-560), FE split-weight denominator (:565), native target split size lazily once (:593,624-626). Equivalent to iceberg PERF-06's newStorageUriNormalizer(token)." + }, + { + "piece": "per-file-split-memo", + "needed": "already-has", + "rationale": "getPartitionInfoMap is computed once per DataSplit (PaimonScanPlanProvider.java:605) and reused across all byte-slice sub-splits of the file (buildNativeRanges); the per-file deletion vector is attached to every sub-range. Equivalent to iceberg PERF-11's per-file invariant memo." + }, + { + "piece": "authz-session-user-isolation", + "needed": "no", + "rationale": "Paimon has no per-Doris-user session credential model (auth is at catalog-creation time; REST vended token is table-level from the shared catalog session, not user-keyed). Name-only caches cannot leak list!=load. No SUPPORTS_USER_SESSION; the authz gate targets iceberg only. Nulling caches under a session=user flag would be dead code." + }, + { + "piece": "shared-fe-connector-cache-adoption", + "needed": "already-has", + "rationale": "partitionViewCache uses the shared ConnectorPartitionViewCache/CacheSpec; PaimonLatestSnapshotCache is built on the shared MetaCacheEntry + CacheSpec toolkit. Only PaimonSchemaAtMemo uses a raw ConcurrentHashMap (clear-on-overflow) — it could optionally migrate to MetaCacheEntry for uniform TTL/metrics, but this is cosmetic, not a correctness or perf gap." + }, + { + "piece": "write-txn-coholder", + "needed": "no", + "rationale": "Paimon write is not migrated to the connector (no write capability declared, PaimonConnector.java:318). There is no beginWrite/commit path and thus no write transaction to co-hold next to the read metadata. N/A until paimon write is migrated." + } + ], + "overallVerdict": "Paimon is ALREADY well-cached and does NOT need the iceberg hot-path/framework port. The iceberg framework's connector-side caches were needed because the P6 cutover left iceberg's loadTable/partitions/format/manifest reads uncached and living on live REST/HMS round-trips. Paimon's situation is structurally different on two axes: (1) the live paimon catalog is wrapped in the paimon SDK CachingCatalog (default on), giving in-memory tableCache / partitionCache / manifestCache — the metastore-client cache layer iceberg's SPI path lacks; and (2) the connector already restored the three legacy-semantics caches (PaimonLatestSnapshotCache stable-snapshot pin, PaimonSchemaAtMemo, partitionViewCache) plus a transient fat-handle and fully-hoisted per-scan/per-file invariants. The per-statement funnel already routes paimon's ConnectorMetadata. Of the 11 framework pieces: 6 already-has, 4 no (format/comment/authz/write-txn are structurally inapplicable), 1 no (cross-query table cache duplicates the SDK cache). The only actionable item is one P2 cleanup — route listPartitionNames/listPartitionValues through partitionViewCache (finding PA-1) — and it is low-value because the raw remote read is already blunted by paimon's partitionCache. Recommendation: do NOT apply iceberg-style caching to paimon; optionally take PA-1 as a small consistency cleanup and optionally migrate PaimonSchemaAtMemo onto the shared MetaCacheEntry toolkit for uniformity.", + "confidence": "high", + "verify": { + "verdict": "CONFIRMED", + "confirmed": [ + "PA-1", + "PA-2", + "PA-3", + "PA-4", + "PA-5" + ], + "corrections": [ + { + "claim": "paimon SDK CachingCatalog ... CACHE_ENABLED default true so the live catalog IS wrapped (CatalogFactory.createCatalog -> CachingCatalog.tryToCreate); paimon cache.* options (cache.expiration-interval etc.)", + "issue": "Immaterial label imprecision: the actual paimon option key string is 'cache-enabled' (hyphen), and the connector calls the paimon (not Doris) org.apache.paimon.catalog.CatalogFactory at PaimonConnector.java:454. The substantive claim is fully verified in bytecode.", + "correction": "Verified: paimon-core-1.3.1 CatalogFactory.createCatalog(ctx,cl) invokestatic CachingCatalog.tryToCreate, which gates on CatalogOptions.CACHE_ENABLED and areturns the raw catalog when false; the CACHE_ENABLED field is built as booleanType().defaultValue(Boolean.valueOf(true)) (iconst_1 at offset 202), description 'Controls whether the catalog will cache databases, tables, manifests and partitions.' Default IS true; wrap covers db/table/manifest/partition. No change to any conclusion." + } + ] + } + }, + { + "connector": "maxcompute", + "migrationStatus": { + "inSpiReadyTypes": true, + "liveness": "live", + "notes": "CatalogFactory.java:57 lists \"max_compute\" in SPI_READY_TYPES, so a MaxCompute catalog is built as a PluginDrivenExternalCatalog around MaxComputeDorisConnector (CatalogFactory.java:110-118). No legacy MaxCompute classes remain in fe-core (find fe/fe-core -iname '*MaxCompute*' returns nothing); the only fe-core references are the display-compat engine-name switch (PluginDrivenExternalTable.java:1227-1230,1260-1261) and GSON migration (GsonUtils). Read path is fully cut over and live. Write path is also wired live (PhysicalPlanTranslator.visitPhysicalConnectorTableSink -> getWritePlanProvider(handle) -> planWrite, PhysicalPlanTranslator.java:676-...); the \"gate-closed/dormant until cutover\" javadoc in MaxComputeConnectorMetadata.beginTransaction:332 and MaxComputeWritePlanProvider:74 is stale relative to HEAD's SPI_READY_TYPES." + }, + "currentCaches": [ + { + "name": "MaxComputePartitionCache", + "scope": "cross-query", + "keyedBy": "(dbName, tableName); ODPS project is constant per catalog so excluded from key (PartitionKey, MaxComputePartitionCache.java:138)", + "ttl": "600s (DEFAULT_TTL_SECOND, MaxComputePartitionCache.java:74); capacity 10000 (line 75); overridable via meta.cache.max_compute.partition.* (CacheSpec.fromProperties, line 92)", + "file": "MaxComputePartitionCache.java:60-173; held final on the long-lived connector (MaxComputeDorisConnector.java:61,79-80); getPartitions:107; invalidation invalidateTable:113 / invalidateDb:118 / invalidateAll:123, wired to REFRESH via MaxComputeDorisConnector.invalidateTable:189 / invalidateDb:198 / invalidateAll:204 / invalidatePartition:213 (whole-table flush)" + }, + { + "name": "ODPS Table in-object lazy memo", + "scope": "per-scan", + "keyedBy": "the single com.aliyun.odps.Table object carried by one MaxComputeTableHandle (transient field, MaxComputeTableHandle.java:37)", + "ttl": "life of the handle; ODPS SDK Table.reload() populates schema/fileNum/isExternal on first access then caches in-object", + "file": "MaxComputeConnectorMetadata.getTableHandle:124 builds it via structureHelper.getOdpsTable (McStructureHelper.java:156-160,294-298 = lazy, no RPC); first metadata access reloads (MaxComputeScanPlanProvider.java:187,189,194). NOT shared across handle resolutions -> each fresh handle reloads again." + }, + { + "name": "fe-core SchemaCacheValue (PluginDrivenSchemaCacheValue)", + "scope": "cross-query", + "keyedBy": "fe-core ExternalTable schema cache (per table), not connector-owned", + "ttl": "fe-core external meta cache TTL / REFRESH", + "file": "PluginDrivenExternalTable.initSchema:430-471 -> getTableSchema runs once per schema-cache refresh, not per query; the connector rides this so getTableSchema/getColumnHandles are not re-issued per query." + }, + { + "name": "PluginDrivenScanNode cachedMetadata + resolvedScanProvider", + "scope": "per-scan", + "keyedBy": "per scan-node; metadata via funnel (catalogId), provider by currentHandle identity", + "ttl": "life of the scan node", + "file": "PluginDrivenScanNode.java:189 (cachedMetadata), 184/245-260 (resolvedScanProvider memo). fe-core infrastructure, connector-agnostic." + }, + { + "name": "EnvironmentSettings / SplitOptions / scan+write provider", + "scope": "cross-query", + "keyedBy": "per-connector one-time lazy init (config, not remote data)", + "ttl": "connector lifetime", + "file": "MaxComputeDorisConnector.doInit:94-128 (settings buildSettings:138, providers), MaxComputeScanPlanProvider.initFromProperties:117-161. Shared by scan and write planning (mirrors legacy catalog.getSettings)." + } + ], + "funnelParticipation": { + "funneled": true, + "metadataMemoizesLoadedTable": "no", + "evidence": "MaxComputeConnectorMetadata is obtained only via MaxComputeDorisConnector.getMetadata(session) (MaxComputeDorisConnector.java:177-181), which is routed through PluginDrivenMetadata.get -> scope.getOrCreateMetadata(\"metadata:\"+catalogId) (PluginDrivenMetadata.java:70), so a statement uses exactly ONE MaxComputeConnectorMetadata per catalog (enforced build-wide by tools/check-fecore-metadata-funnel.sh). BUT that instance memoizes NOTHING: getTableHandle (MaxComputeConnectorMetadata.java:118-130) re-invokes structureHelper.tableExist (line 121 -> ODPS tables().exists() remote probe, McStructureHelper.java:129-137/218-225) AND structureHelper.getOdpsTable (line 124 -> a FRESH lazy Table) on every call. getMetadata itself is cheap (just wraps already-init odps/structureHelper/partitionCache), so the funnel's metadata memo saves no remote IO for MaxCompute.", + "gaps": "The funnel collapses metadata construction, not table resolution. Because the ConnectorMetadata holds no (db,table)->handle map, each of the ~13 resolveConnectorTableHandle sites (PluginDrivenExternalTable.java:160,463,607,717,821,882,955,1026,1061,1128 + resolveWriteCapabilityHandle 205/222/375/410) and each translator getTableHandle (PhysicalPlanTranslator.java:624,676; BindSink.java:675,714) pays its own ODPS exists() probe within one statement; every schema-accessing path (planScan, initSchema) reloads a fresh Table. This is the iceberg PERF-07 gap, unaddressed for MaxCompute." + }, + "hotPathFindings": [ + { + "id": "MC-1", + "area": "schema", + "problem": "getTableHandle issues a redundant remote ODPS tables().exists() probe on EVERY handle resolution (MaxComputeConnectorMetadata.java:121 -> McStructureHelper.tableExist), and neither the per-statement ConnectorMetadata nor the funnel memoizes the resolved handle. A single statement resolves the handle at many independent sites (scan translate, partition prune, row-count, per-column stats, write-capability probes), each paying one exists() RPC. The table is already a resolved catalog ExternalTable, so the existence check is pure waste (variants B single-chain-repeated-k-times + C funnel-memoizes-metadata-not-handle).", + "multiplier": "k = number of resolveConnectorTableHandle / getTableHandle sites reached per statement; ~2 for a warm-cache partitioned SELECT (scan create + getNameToPartitionItems), higher on cold fe-core stats caches (fetchRowCount + per-column getColumnStatistic) and on the write path (several write-capability probes each resolve a fresh handle)", + "cost": "remote-io", + "severity": "P1", + "memoizedAlready": false, + "file": "MaxComputeConnectorMetadata.java:118-130; McStructureHelper.java:129-137,218-225" + }, + { + "id": "MC-2", + "area": "schema", + "problem": "Repeated lazy-Table reload: because handles are not memoized per statement, each fresh MaxComputeTableHandle carries its own unloaded ODPS Table, and each schema-accessing path (planScan checkOperationSupported/getFileNum/getSchema at MaxComputeScanPlanProvider.java:187,189,194; initSchema getTableSchema when the schema cache is cold) triggers its own Table.reload() RPC. Modest amplification (most non-scan paths use only db/table names, not the Table object), so ~1-2 reloads per statement rather than iceberg's 3-7x (variant B).", + "multiplier": "~1-2x per statement (vs 1 ideal)", + "cost": "remote-io", + "severity": "P2", + "memoizedAlready": true, + "file": "MaxComputeConnectorMetadata.java:124; MaxComputeScanPlanProvider.java:183-194" + }, + { + "id": "MC-3", + "area": "split-enum", + "problem": "NON-finding / already clean: split enumeration has no per-split remote amplification. The ODPS read session is built once (buildBatchReadSession, the inherent planning cost), serialized once before the loop (serializeSession, MaxComputeScanPlanProvider.java:329), and the per-split loop only constructs MaxComputeScanRange objects sharing the one serialized-session string by reference (lines 337-372). getSplits calls planScan once. No per-split loadTable / getSchema.", + "multiplier": "1x (no amplification)", + "cost": "cpu", + "severity": "none", + "memoizedAlready": true, + "file": "MaxComputeScanPlanProvider.java:326-377" + }, + { + "id": "MC-4", + "area": "partitions", + "problem": "NON-finding for the data, subsumed by MC-1 for the handle: partition listing (listPartitions / listPartitionNames / listPartitionValues + fe-core getNameToPartitionItems/getNameToPartitionValues) is served by the cross-query MaxComputePartitionCache (TTL 600s), so the per-table ODPS getPartitions() is not re-issued per query. Only the getTableHandle exists() probe inside getNameToPartitionItems remains uncached (counted in MC-1). listPartitions deliberately ignores the pushdown filter (parity with legacy SHOW PARTITIONS) — full-set listing, but cached.", + "multiplier": "1x remote getPartitions per (db,table) per TTL window", + "cost": "remote-io", + "severity": "none", + "memoizedAlready": true, + "file": "MaxComputeConnectorMetadata.java:239-296; MaxComputePartitionCache.java:107; PluginDrivenExternalTable.java:830-831" + } + ], + "authzConsistency": { + "perUserCredential": false, + "sessionUserRelevant": false, + "staleOrLeakRisk": "low", + "notes": "MaxCompute authenticates with CATALOG-STATIC credentials (AK/SK, RAM-Role-ARN, or ECS-RAM-Role) baked into catalog properties at CREATE CATALOG and materialized once in MaxComputeDorisConnector.doInit -> MCConnectorClientFactory.createClient (MaxComputeDorisConnector.java:102; MCConnectorClientFactory.java:86-127). There is NO per-user credential vending, NO session=user analog, NO REST/OIDC session credential, NO kerberos doAs, NO per-identity metastore — all users of a catalog share one ODPS identity. Therefore the Layer-3 authz cache-isolation concern does NOT apply: the name-only MaxComputePartitionCache (keyed by db+table) cannot leak a can-LIST-cannot-LOAD user's metadata, and any per-statement handle memo (the MC-1 fix) would likewise be authz-safe (one statement = one identity, already pinned by PluginDrivenMetadata.java:63-69). tools/check-authz-cache-sharding.sh applies but MaxCompute is trivially compliant. Metadata staleness is bounded and correctness-safe: partition adds become visible ~<=600s without REFRESH (matches legacy TTL); drops may serve stale for up to TTL, self-healing on next miss. Write consistency: MaxCompute is NON-MVCC (PluginDrivenExternalTable base class, no MvccTable), so no read+write snapshot pin is needed; write correctness only requires the ODPS write session bound to the per-statement MaxComputeConnectorTransaction (setWriteSession) with block allocation keyed by txn_id — already provided." + }, + "frameworkPiecesNeeded": [ + { + "piece": "per-statement-funnel-memoization", + "needed": "yes", + "rationale": "Highest-value fix. The funnel already gives ONE MaxComputeConnectorMetadata per statement (PluginDrivenMetadata.java:70), but getTableHandle re-probes ODPS existence and hands back a fresh lazy Table each call (MaxComputeConnectorMetadata.java:118-130). Adding a per-instance Map<(db,table),MaxComputeTableHandle> so getTableHandle returns the same handle (one exists() probe, one Table.reload()) for the whole statement collapses MC-1 and MC-2 at once. Simpler than iceberg's scope-keyed load memo because the metadata instance is already per-statement. Also drop the redundant tableExist() probe for already-resolved reads." + }, + { + "piece": "cross-query-table-cache", + "needed": "partial", + "rationale": "fe-core SchemaCacheValue already caches the schema cross-query (initSchema not re-run per query), so the main cross-query need is covered. A connector-side cached ODPS Table (iceberg IcebergTableCache analog, 24h/REFRESH) would additionally save the per-query planScan reload (MC-2 cross-query dimension), but with staleness/TTL cost and only modest benefit. Optional, lower priority than the per-statement memo." + }, + { + "piece": "partition-cache", + "needed": "already-has", + "rationale": "MaxComputePartitionCache is a complete cross-query partition-listing cache (keyed db+table, TTL 600s, capacity 10000, REFRESH-invalidated), shared by all three partition-listing SPI methods; MC-4 confirms it catches the hot partition read." + }, + { + "piece": "format-cache", + "needed": "no", + "rationale": "N/A: MaxCompute reads through the ODPS Storage API (arrow batches); there is no per-table file-format inference (no getFileFormat fallback / planFiles equivalent), so the iceberg IcebergFormatCache / PERF-03 problem does not exist here." + }, + { + "piece": "comment-cache", + "needed": "no", + "rationale": "MaxCompute does not override getTableComment; the SPI default returns \"\" (ConnectorTableOps.java:336) with no remote call, so there is no N-per-query getComment load to cache (iceberg PERF-05 does not apply)." + }, + { + "piece": "manifest-or-file-list-cache", + "needed": "no", + "rationale": "N/A: no manifest/snapshot model. The read session (buildBatchReadSession) and split assigner are the inherent per-query planning cost via the Storage API; estimateDataSizeByListingFiles/listFileSizes are not overridden (default -1/empty), so there is no file-listing hot path to memoize." + }, + { + "piece": "per-scan-hoist", + "needed": "already-has", + "rationale": "Scan-invariant config (EnvironmentSettings, SplitOptions, timeouts) is hoisted to one-time lazy init on the connector/provider (MaxComputeDorisConnector.doInit, MaxComputeScanPlanProvider.initFromProperties), and the serialized read session is computed once before the split loop. The iceberg PERF-06 per-scan storage-URI normalizer does not apply (no per-file vended storage config)." + }, + { + "piece": "per-file-split-memo", + "needed": "no", + "rationale": "N/A: MaxCompute splits carry only a split index/row-offset plus the one shared serialized-session string (already shared by reference across all ranges); there is no per-file partition JSON / partition-values / delete-carrier computation to memoize (iceberg PERF-11 C12/C15a/C13 do not apply)." + }, + { + "piece": "authz-session-user-isolation", + "needed": "no", + "rationale": "MaxCompute uses catalog-static AK/SK credentials with one shared ODPS identity per catalog; no session=user / OIDC / doAs. Name-only caches cannot leak or serve stale-authz across users, so the Layer-3 isolation (iceberg isUserSessionEnabled null-out / shouldBypassSchemaCache) is unnecessary." + }, + { + "piece": "shared-fe-connector-cache-adoption", + "needed": "already-has", + "rationale": "MaxComputePartitionCache is built on the shared fe-connector-cache toolkit (CacheSpec.fromProperties + MetaCacheEntry, MaxComputePartitionCache.java:20-21,92-97), mirroring CachingHmsClient/HiveFileListingCache. It does not use the higher-level ConnectorPartitionViewCache/PartitionViewCacheKey substrate, but that targets partition-derived sorted-range VIEWS (a separate CACHE-P1 concern), not the base partition listing." + }, + { + "piece": "write-txn-coholder", + "needed": "already-has", + "rationale": "MaxComputeConnectorTransaction holds the per-statement write state (write session id, TableIdentifier, settings, accumulated TMCCommitData, block-id high-water mark), bound by MaxComputeWritePlanProvider.planWrite via setWriteSession and committed keyed by txn_id (MaxComputeConnectorTransaction.java:61-131). Non-MVCC, so no read+write snapshot sharing is required; the funnel's one-write-txn-per-statement (CatalogStatementTransaction) already coordinates it." + } + ], + "overallVerdict": "LIVE and already reasonably cached — NOT a P0 target like iceberg. MaxCompute has a proper cross-query partition cache on the shared toolkit, clean split enumeration with no per-split remote amplification, a per-statement write transaction, and rides fe-core's cross-query schema cache; it is non-MVCC and uses catalog-static credentials so Layer-3 authz isolation does not apply. The ONE real, applicable framework piece is per-statement handle/table memoization: getTableHandle performs a redundant remote ODPS tables().exists() probe on every resolution and returns a fresh lazy Table, and the funnel memoizes only the metadata, not the handle — so k independent handle-resolution sites in one statement each pay an exists() RPC (MC-1, P1) and schema-accessing paths reload the Table more than once (MC-2, P2). Recommend a small connector-side fix: memoize the resolved MaxComputeTableHandle per (db,table) inside the per-statement MaxComputeConnectorMetadata instance and drop the redundant existence probe for already-resolved reads; optionally add a cross-query ODPS Table cache later. This is a low-risk, high-leverage change, far smaller than the iceberg buildout.", + "confidence": "high", + "verify": { + "verdict": "CONFIRMED", + "confirmed": [ + "MC-1", + "MC-2", + "MC-3", + "MC-4" + ], + "corrections": [ + { + "claim": "find fe/fe-core -iname '*MaxCompute*' returns nothing", + "issue": "Literally false: it returns two compiled dirs fe/fe-core/target/classes/org/apache/doris/datasource/maxcompute and target/test-classes/.../maxcompute (stale build artifacts).", + "correction": "The SOURCE tree fe/fe-core/src is clean (no MaxCompute .java). The read/write cutover conclusion is unaffected; only the stated grep evidence is imprecise. Verify with: find fe/fe-core/src -iname '*MaxCompute*' (empty)." + }, + { + "claim": "PluginDrivenScanNode.cachedMetadata + resolvedScanProvider ... PluginDrivenScanNode.java:189/184/245-260", + "issue": "File path implied by the funnel narrative (datasource/plugin/) is wrong.", + "correction": "PluginDrivenScanNode is at fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java. All cited line numbers are correct there: currentHandle:152, resolvedScanProvider:184, cachedMetadata:189, metadata():203, resolveScanProvider():245-260." + }, + { + "claim": "~13 resolveConnectorTableHandle / getTableHandle sites (PluginDrivenExternalTable.java:160,463,607,717,821,882,955,1026,1061,1128 + resolveWriteCapabilityHandle 205/222/375/410)", + "issue": "Direct resolveConnectorTableHandle count is 10, not 13; the extra 205/222/375/410 line refs point at resolveWriteCapabilityHandle callers which I did not individually confirm (the method definition is at 187).", + "correction": "Exactly 10 direct resolveConnectorTableHandle call sites confirmed at the listed lines, each in a distinct real planning entrypoint (getSyntheticScanPredicates:152, initSchema:430, getShowCreateTableDdl:599, fetchSyntheticWriteColumns:700, getNameToPartitionItems:807, getNameToPartitionValues:870, getSupportedSysTables:946, getColumnStatistic:1014, getChunkSizes:1049, fetchRowCount:1121). The write-capability probes add a few more. The 'k independent sites, none memoized' core of MC-1 holds regardless of the exact count." + }, + { + "claim": "MC-2: ~1-2 reloads per statement", + "issue": "Slight understatement risk on the cold per-column stats path was checked and cleared, worth recording.", + "correction": "getColumnStatistic (fe-core:1014-1035) resolves a FRESH handle per column (so MC-1 exists()-probe amplifies to N-per-column on cold stats), BUT MaxComputeConnectorMetadata does NOT override getColumnStatistics (SPI default no-op, no remote call and no odpsTable access), so it triggers NO extra Table.reload(). MC-2's ~1-2x reload multiplier therefore holds; the per-column amplification is exists()-probe only, which the audit already flags qualitatively under MC-1 ('higher on cold stats caches')." + } + ] + } + }, + { + "connector": "jdbc", + "migrationStatus": { + "inSpiReadyTypes": true, + "liveness": "live", + "notes": "jdbc 在 CatalogFactory.SPI_READY_TYPES 中 (fe/fe-core/.../datasource/CatalogFactory.java:57)。JdbcConnectorProvider.getType()=\"jdbc\" (JdbcConnectorProvider.java:41-42) 经 SPI 路由到 PluginDrivenExternalCatalog(CatalogFactory.java:110-118)。fe-core 侧 legacy JdbcExternalCatalog/JdbcExternalTable 已删除(find 无结果)。残留的 fe-core datasource/jdbc/client/JdbcClient 仅被 CDC/streaming-job 校验器引用(PostgresResourceValidator/StreamingJobUtils/CdcStreamTableValuedFunction),不在 catalog 查询热路径上。查询 TVF(JdbcQueryTableValueFunction:53-54)同样经 PluginDrivenMetadata.get 走连接器 getColumnsFromQuery。连接器整体 LIVE。" + }, + "currentCaches": [ + { + "name": "HikariDataSource 连接池(唯一实质性复用)", + "scope": "cross-query", + "keyedBy": "每 catalog 一个 client 一个池", + "ttl": "connector 生命周期,close() 时销毁", + "file": "fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/client/JdbcConnectorClient.java:89" + }, + { + "name": "CLASS_LOADER_MAP 驱动 classloader 缓存(泄漏修复)", + "scope": "cross-query", + "keyedBy": "driver URL", + "ttl": "进程级,永不淘汰(external regression 986696 Metaspace 泄漏修复)", + "file": "fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/client/JdbcConnectorClient.java:78" + }, + { + "name": "OceanBase compat-mode delegate 记忆(方言探测)", + "scope": "cross-query", + "keyedBy": "单 delegate/client, volatile double-checked", + "ttl": "client 生命周期", + "file": "fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/client/JdbcOceanBaseConnectorClient.java:45" + }, + { + "name": "ExternalSchemaCache(fe-core,前置在连接器 getTableSchema 之前)", + "scope": "cross-query", + "keyedBy": "SchemaCacheKey(table)", + "ttl": "expire-after-access(external_cache_expire_time_minutes_after_access)+ refresh/DDL 失效", + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java:365" + }, + { + "name": "ExternalRowCountCache(fe-core,前置在 getTableStatistics 之前)", + "scope": "cross-query", + "keyedBy": "RowCountKey(catalog,db,table)", + "ttl": "expireAfterWrite ~1 天,异步刷新", + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java:45" + }, + { + "name": "MetaCache 名录缓存(fe-core,前置在 listDatabaseNames/listTableNames 之前)", + "scope": "cross-query", + "keyedBy": "db 名 / (db,table) 名", + "ttl": "expire-after-access + refresh 失效", + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java:65" + } + ], + "funnelParticipation": { + "funneled": true, + "metadataMemoizesLoadedTable": "no", + "evidence": "所有 fe-core 读路径经 PluginDrivenMetadata.get 获取 metadata:initSchema(PluginDrivenExternalTable.java:449)、fetchRowCount(:1126)、getComment(:923)、PluginDrivenScanNode.buildColumnHandles(:1918-1920 via metadata())、JdbcQueryTableValueFunction(:53)。JdbcConnectorMetadata 无状态,仅持有 client+properties 引用,每次 getTableSchema/getColumnHandles/getTableStatistics 都原样转发 JdbcConnectorClient 做一次全新远程 round-trip,内部不做任何 memo(JdbcConnectorMetadata.java:119 getTableSchema、:162 getColumnHandles、:144 getTableStatistics)。JDBC 没有 iceberg 那种昂贵的 loaded Table 对象。", + "gaps": "funnel 保证每语句一个 JdbcConnectorMetadata,但该实例构造成本近乎为零,而真正昂贵的资源(HikariCP 连接池)已经是 per-catalog 单例(JdbcDorisConnector.getOrCreateClient:188),所以 funnel 的\"一实例合并\"对 JDBC 几乎无收益。扫描路径(buildConnectorSession,PluginDrivenExternalCatalog.java:1120)带有活的 per-statement scope,但 getColumnHandles 未在其上 memo,导致 scan 期仍重复远程取列。写路径 JdbcWritePlanProvider.buildInsertSql(:136)自行 new JdbcConnectorMetadata,完全绕开 funnel 记忆的实例。" + }, + "hotPathFindings": [ + { + "id": "HP-1", + "area": "schema", + "problem": "扫描规划期 buildColumnHandles→metadata.getColumnHandles→client.getJdbcColumnsInfo 触发一次全新远程 JDBC 元数据 round-trip(DatabaseMetaData.getColumns,MySQL 还可能追加 information_schema 查询),而这些列名+remote 名已被 fe-core ExternalSchemaCache 跨查询缓存(initSchema→getTableSchema 同样来自 getJdbcColumnsInfo)。属 variant C:缓存存在但列句柄热路径绕过它重新远程取。", + "multiplier": "每 scan node 约 1-2 次(getSplits/startSplit 二选一 + getOrLoadPropertiesResult 各一次;每查询 × jdbc 扫描数)", + "cost": "remote-io", + "severity": "P2", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadata.java:162" + }, + { + "id": "HP-2", + "area": "write", + "problem": "JdbcWritePlanProvider.buildInsertSql 用 new JdbcConnectorMetadata(client,properties) 现构一个绕开 funnel 的实例,再调 getColumnHandles→getJdbcColumnsInfo 做远程取列以解析 local→remote 列名映射;这些映射已在 ExternalSchemaCache 中。variant C + 绕过 funnel。", + "multiplier": "每 INSERT 1 次(planWrite);EXPLAIN INSERT 追加 appendExplainInfo 共 2 次", + "cost": "remote-io", + "severity": "P2", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcWritePlanProvider.java:136" + }, + { + "id": "HP-3", + "area": "show", + "problem": "PluginDrivenExternalTable.getComment 每次直接调 metadata.getTableComment(无缓存);MySQL 覆写为远程 INFORMATION_SCHEMA 查询。仅 SHOW CREATE TABLE / DESC / information_schema 展示路径命中,不在查询规划热路径上。", + "multiplier": "每次 SHOW/DESC 1 次", + "cost": "remote-io", + "severity": "P2", + "memoizedAlready": false, + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java:925" + }, + { + "id": "HP-4", + "area": "split-enum", + "problem": "(已最优,阴性对照)JdbcScanPlanProvider.planScan 零远程 IO:恒返回 1 个 scan range,纯 SQL 字符串拼装,BE JdbcJniReader 直接执行远程 SQL;estimateScanRangeCount()==1。无 split/file/manifest/partition 层,故 iceberg 式 per-split/per-file 放大问题在 JDBC 不存在。", + "multiplier": "1(常量)", + "cost": "cpu", + "severity": "none", + "memoizedAlready": true, + "file": "fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcScanPlanProvider.java:66" + }, + { + "id": "HP-5", + "area": "stats-rowcount", + "problem": "(已缓存,阴性对照)row-count 经 fetchRowCount→getTableStatistics→client.getRowCount,前置 ExternalRowCountCache 跨查询缓存;base getRowCount 返回 -1,MySQL/PG/Oracle/SQLServer 覆写为系统目录查询,均被缓存吸收。无 fetchRowCountFromFileList(连接器 dataSize 恒 -1)。", + "multiplier": "缓存命中后每查询 0 次远程", + "cost": "remote-io", + "severity": "none", + "memoizedAlready": true, + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java:1133" + } + ], + "authzConsistency": { + "perUserCredential": false, + "sessionUserRelevant": false, + "staleOrLeakRisk": "无。JDBC 使用 catalog 属性里固定的 user/password(JdbcConnectorProperties.USER/PASSWORD),烘焙进 HikariCP 数据源与 BE 的 TJdbcTable。所有 Doris 身份共享同一个远程 JDBC 用户,远程侧无 per-identity 元数据分歧,因此 name-only 缓存(fe-core schema/rowcount/name 缓存)不会泄漏 per-user 元数据,也不存在 can-LIST-cannot-LOAD 式 stale-authz。无 kerberos doAs / REST OIDC / per-identity metastore。", + "notes": "JdbcDorisConnector.getCapabilities 只声明 SUPPORTS_PASSTHROUGH_QUERY + SUPPORTS_METADATA_PRELOAD,未声明 SUPPORTS_USER_SESSION(JdbcDorisConnector.java:102-105);buildConnectorSession 仅当 supportsUserSession() 为真时注入委派凭证,此处为假,故不需要 iceberg 式 isUserSessionEnabled() 缓存置空 / shouldBypassSchemaCache 分片。写侧为 BE per-row auto-commit,beginTransaction 返回 NoOpConnectorTransaction(JdbcConnectorMetadata.java:286),读写无需共享同一 metadata/txn/snapshot。" + }, + "frameworkPiecesNeeded": [ + { + "piece": "per-statement-funnel-memoization", + "needed": "partial", + "rationale": "唯一真正适用的一块。scan 路径 buildColumnHandles 与 write 路径 buildInsertSql 各以 getColumnHandles 重复远程取列(getJdbcColumnsInfo),既无 per-statement memo、写侧还绕开 funnel 实例。在活的 statement scope 上 memo(或更优:让 fe-core buildColumnHandles 直接从已缓存的 PluginDrivenSchemaCacheValue 派生列句柄)即可合并每语句 1-2 次冗余 round-trip。优先级低(P2、remote-io 但廉价、无 split/file 放大)。" + }, + { + "piece": "cross-query-table-cache", + "needed": "no", + "rationale": "无 iceberg 式昂贵 loaded Table 对象可缓存;远程 schema/rowcount/名录已被 fe-core 跨查询缓存(ExternalSchemaCache/ExternalRowCountCache/MetaCache)前置,连接器侧无需自建跨查询表缓存。" + }, + { + "piece": "partition-cache", + "needed": "no", + "rationale": "JDBC 不暴露分区;planScan 恒发一个 scan range,无分区列举/裁剪。" + }, + { + "piece": "format-cache", + "needed": "no", + "rationale": "关系型透传,无文件格式解析;BE JdbcJniReader 直接跑远程 SQL。" + }, + { + "piece": "comment-cache", + "needed": "no", + "rationale": "getTableComment 未缓存且 MySQL 为远程查询(HP-3),但仅 SHOW/DESC 展示路径命中,不在查询规划热路径,缓存价值边际。" + }, + { + "piece": "manifest-or-file-list-cache", + "needed": "no", + "rationale": "无 manifest/文件清单;无 fetchRowCountFromFileList 路径(连接器 dataSize 恒 -1)。" + }, + { + "piece": "per-scan-hoist", + "needed": "no", + "rationale": "planScan 已是 O(1) 纯 SQL 拼装、零远程 IO,无 loop-invariant 远程操作可外提。" + }, + { + "piece": "per-file-split-memo", + "needed": "no", + "rationale": "恰好一个 split,estimateScanRangeCount()==1,无 per-file 不变量可 memo。" + }, + { + "piece": "authz-session-user-isolation", + "needed": "no", + "rationale": "固定 catalog 凭证、无 SUPPORTS_USER_SESSION;name-only 缓存不会泄漏 per-user 元数据,无需 session=user 隔离。" + }, + { + "piece": "shared-fe-connector-cache-adoption", + "needed": "no", + "rationale": "连接器无可迁移到 fe-connector-cache 工具箱的跨查询元数据缓存,仅有驱动 classloader map 与连接池等资源单例,非该工具箱职责。" + }, + { + "piece": "write-txn-coholder", + "needed": "no", + "rationale": "JDBC 写为 BE 侧逐行 auto-commit,beginTransaction 返回 NoOpConnectorTransaction;无 FE 侧读写共享 txn/snapshot 需求。" + } + ], + "overallVerdict": "不要把 iceberg 式跨查询连接器缓存框架套到 jdbc 上——那会是投机式(违反 Rule 2)。JDBC 是关系型透传:planScan 零远程 IO、恒一个 scan range,没有 split/file/manifest/partition/snapshot 层,因此 iceberg 主要痛点(重复 loadTable、PARTITIONS 重扫、planFiles 无过滤全表)在 JDBC 结构性不存在。其昂贵的远程元数据(schema/columns、row-count、db/table 名录)已被 fe-core 跨查询缓存前置,连接池是 per-catalog 单例,驱动 classloader 进程级永驻。唯一真实缺口是扫描规划与写整形期 getColumnHandles 的冗余远程取列(HP-1/HP-2,均 P2、绕过已缓存 schema),恰当补救是小范围 per-statement memo 或让 fe-core 从已缓存 schema 派生列句柄,而非新增连接器缓存。", + "confidence": "high", + "verify": { + "verdict": "CONFIRMED", + "confirmed": [ + "HP-1", + "HP-2", + "HP-3", + "HP-4", + "HP-5" + ], + "corrections": [ + { + "claim": "jdbc 在 CatalogFactory.SPI_READY_TYPES 中 (CatalogFactory.java:57)", + "issue": "Line-number anchor slightly off. The claim itself is TRUE, but the ImmutableSet containing \"jdbc\" is not at line 57.", + "correction": "At HEAD the declaration `private static final Set SPI_READY_TYPES =` is on line 56 and the `ImmutableSet.of(\"jdbc\", \"es\", \"trino-connector\", \"max_compute\", \"paimon\", \"iceberg\", \"hms\")` literal is on line 63-64 of fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java. Membership is confirmed either way." + }, + { + "claim": "ExternalSchemaCache TTL = expire-after-access(external_cache_expire_time_minutes_after_access)", + "issue": "Config key name imprecise.", + "correction": "The actual Config field seen at ExternalRowCountCache.java:46 is `external_cache_expire_time_seconds_after_access` (seconds, not minutes) and `external_cache_refresh_time_minutes`. This is a cosmetic naming imprecision; the expire-after-access + refresh semantics claimed are correct." + }, + { + "claim": "ExternalSchemaCache file: ExternalMetaCacheMgr.java:365", + "issue": "Anchor points to the accessor, not a distinct cache class.", + "correction": "Line 365 of ExternalMetaCacheMgr.java is `public Optional getSchemaCacheValue(ExternalTable table, SchemaCacheKey key)` — the fe-core schema-cache accessor. Valid anchor for the mechanism; the schema-cache front-loading of getTableSchema is real and confirmed (initSchema at PluginDrivenExternalTable.java:449→getTableSchema :469)." + } + ] + } + }, + { + "connector": "es", + "migrationStatus": { + "inSpiReadyTypes": true, + "liveness": "live", + "notes": "\"es\" is in SPI_READY_TYPES at fe/fe-core/.../datasource/CatalogFactory.java:56-57 (ImmutableSet.of(\"jdbc\",\"es\",...)); createCatalog routes it through the SPI path (line 110 -> ConnectorFactory.createConnector -> PluginDrivenExternalCatalog). EsConnectorProvider.getType()==\"es\" (EsConnectorProvider.java:34) is discovered via META-INF/services. The legacy fe-core EsExternalCatalog/EsExternalDatabase/EsExternalTable/EsScanNode were deleted (commit 4f455da5950) and BUILD_CONNECTOR_ES flipped default 0->1; only the unrelated internal-catalog EsTable.java/EsResource.java and the EsQuery function remain in fe-core. So the connector is the sole live ES path." + }, + "currentCaches": [ + { + "name": "EsConnector.restClient (per-catalog REST connection memo)", + "scope": "metastore-client", + "keyedBy": "none (single client per catalog, double-checked-locking lazy init)", + "ttl": "connector/catalog lifetime; never invalidated (rebuilt only if field is null)", + "file": "fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnector.java:43,82-107" + }, + { + "name": "EsConnectorRestClient.PLAIN_CLIENT / sslClient (static shared OkHttp clients)", + "scope": "metastore-client", + "keyedBy": "none (JVM-global; ssl vs plain)", + "ttl": "process lifetime", + "file": "fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorRestClient.java:61-63,310-319" + }, + { + "name": "Layer-2 per-statement funnel: PluginDrivenMetadata memoizes ONE EsConnectorMetadata per (statement,catalog)", + "scope": "per-statement", + "keyedBy": "\"metadata:\"+catalogId on the ConnectorStatementScope, identity-pinned by getUser", + "ttl": "one statement (closed at statement end); NONE scope memoizes nothing", + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenMetadata.java:64,70" + }, + { + "name": "ResolvedScanProvider memo: ONE EsScanPlanProvider per scan node", + "scope": "per-scan", + "keyedBy": "currentHandle identity", + "ttl": "scan-node lifetime", + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java:254-259" + }, + { + "name": "fe-core ExternalSchemaCache wrapping initSchema() -> getTableSchema() -> getMapping()", + "scope": "cross-query", + "keyedBy": "SchemaCacheKey (db.table/index)", + "ttl": "ExternalMetaCacheMgr schema cache (expire-after-write, REFRESH-invalidated)", + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java:430-470; fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java:385-445" + }, + { + "name": "fe-core RowCountCache wrapping fetchRowCount() (ES has no stats -> returns UNKNOWN, no ES-side remote heavy op)", + "scope": "cross-query", + "keyedBy": "table id", + "ttl": "row-count cache expiry", + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java:907,1121-1133" + }, + { + "name": "Connector-side dedicated scan-metadata cache (mapping field-context / shard routing / node topology)", + "scope": "none", + "keyedBy": "N/A — does not exist", + "ttl": "N/A", + "file": "fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java:99,169,273-294 (re-fetched every call, no memo)" + } + ], + "funnelParticipation": { + "funneled": true, + "metadataMemoizesLoadedTable": "no", + "evidence": "EsConnectorMetadata is obtained ONLY through PluginDrivenMetadata.get(session, connector) at every fe-core seam (PluginDrivenExternalTable.java:449,565,583,606,716,820,881,923,954,1025,1060,1126,1277; PluginDrivenScanNode buildColumnHandles metadata() at :1918-1920), so it IS routed through the Layer-2 funnel (one instance per statement). BUT the instance memoizes nothing: EsConnectorMetadata.getTableSchema (EsConnectorMetadata.java:81-94) calls restClient.getMapping() remotely on every call, and getColumnHandles (line 97-106) calls getTableSchema again -> a second remote mapping GET. The heavy scan-path metadata (shard routing + node topology + field-context mapping) is NOT on the funneled metadata at all: it lives on EsScanPlanProvider, a SEPARATE object (EsConnector.getScanPlanProvider() returns new EsScanPlanProvider each call, EsConnector.java:56-58; memoized only per-scan-node by ResolvedScanProvider). EsScanPlanProvider.planScan (line 99) and buildScanNodeProperties (line 169) each independently call fetchMetadataState (line 273-294) -> new EsMetadataState -> EsMetadataFetcher.fetch() -> getMapping + searchShards + getHttpNodes. So the funnel's \"one metadata per statement\" does NOT collapse the repeated ES remote loads.", + "gaps": "The funnel is present but unexploited. Neither EsConnectorMetadata (schema) nor EsScanPlanProvider (state) hangs a per-statement/per-scan memo, so a single SELECT re-fetches the mapping ~4x and shards/nodes ~2x. Fix is to hoist EsMetadataState once per scan (provider is already a single instance per scan node) and/or memoize schema on the per-statement EsConnectorMetadata." + }, + "hotPathFindings": [ + { + "id": "ES-F1", + "area": "split-enum", + "problem": "EsScanPlanProvider.fetchMetadataState() is invoked twice per query — once from planScan() (EsScanPlanProvider.java:99, via PluginDrivenScanNode getSplits:1303) and once from buildScanNodeProperties() (line 169, via getScanNodePropertiesResult, PluginDrivenScanNode:1863). Each call does EsMetadataFetcher.fetch() = searchShards (index/_search_shards GET) + getHttpNodes (_nodes/http GET, because NODES_DISCOVERY_DEFAULT=true) + getMapping. So the SAME shard routing and node topology are resolved from ES twice within one planning pass with no memo. Provider is already a single instance per scan node (ResolvedScanProvider), so a field memo keyed by (index,columnNames) collapses 2->1 with zero staleness risk.", + "multiplier": "2x per query (constant, not loop-amplified)", + "cost": "remote-io", + "severity": "P1", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java:99,169,273-294; EsMetadataFetcher.java:51-89" + }, + { + "id": "ES-F2", + "area": "predicate", + "problem": "Inside each fetchMetadataState, EsMetadataFetcher.fetchMapping() (EsMetadataFetcher.java:57-58) calls restClient.getMapping(index) to build the field-context (keyword sniff / doc_value / date-compat) used for query-DSL pushdown. That exact index mapping was ALREADY fetched and cached cross-query at schema resolution (initSchema -> getTableSchema -> getMapping, behind ExternalSchemaCache). The scan path re-fetches it remotely instead of reusing the cached schema, ~2x per query (once per fetchMetadataState).", + "multiplier": "~2x per query (on top of the schema-cache copy)", + "cost": "remote-io", + "severity": "P1", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsMetadataFetcher.java:57-58; EsConnectorRestClient.java:161-168" + }, + { + "id": "ES-F3", + "area": "schema", + "problem": "PluginDrivenScanNode.buildColumnHandles() calls metadata.getColumnHandles() -> EsConnectorMetadata.getColumnHandles (EsConnectorMetadata.java:97-106) -> getTableSchema -> restClient.getMapping() (raw remote GET, bypassing ExternalSchemaCache). buildColumnHandles runs at least twice per query (PluginDrivenScanNode.java:1263 in getSplits, and :1841 in getOrLoadPropertiesResult), so the mapping is re-fetched ~2x more. The funneled EsConnectorMetadata is a single per-statement instance, so memoizing the resolved schema on that instance would collapse this to 1x per statement.", + "multiplier": "2x per query", + "cost": "remote-io", + "severity": "P2", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java:81-106; fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java:1263,1841,1917-1920" + }, + { + "id": "ES-F4", + "area": "file-list", + "problem": "getTableHandle() -> restClient.existIndex() issues a remote index/_mapping GET every time a handle is resolved (EsConnectorMetadata.java:74; EsConnectorRestClient.java:104-114). Called from schema/stats/handle-resolution seams. Low cost (single lightweight GET) and the row-count seam is behind RowCountCache; flagged for completeness, not a hot loop.", + "multiplier": "1x per handle resolution", + "cost": "remote-io", + "severity": "P2", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java:71-78; EsConnectorRestClient.java:104-114" + } + ], + "authzConsistency": { + "perUserCredential": false, + "sessionUserRelevant": false, + "staleOrLeakRisk": "No authorization-leak risk. The ES connector authenticates with a SINGLE catalog-level user/password baked into the shared EsConnectorRestClient at connector construction (EsConnector.java:95-100; EsConnectorRestClient.java:76-78 builds one Basic auth header). There is no per-user credential, no iceberg.rest.session=user analog, no kerberos doAs, no REST/OIDC per-identity delegation — all Doris users hit ES with the same identity. So name-only caches (by index) would NOT cause a list-not-equal-load metadata disclosure; the authz-cache-session-user isolation gate (tools/check-authz-cache-sharding.sh) is N/A here. The only consistency concern is DATA FRESHNESS: ES shard routing rebalances over time (ES's own refresh model), so shard/node topology must be resolved per statement and must NOT be cached cross-query — matching the legacy EsScanNode/EsMetaStateTracker which re-resolved shards at each scan.", + "notes": "No write path exists in the connector (EsConnectorMetadata overrides only list/exists/getTableHandle/getTableSchema/getColumnHandles/buildTableDescriptor; no beginWrite/commit/insert/sink), so there is no read+write one-metadata/one-txn consistency requirement. ES is read-only and non-partitioned (no listPartitions override), so no snapshot/version pinning is needed." + }, + "frameworkPiecesNeeded": [ + { + "piece": "per-scan-hoist", + "needed": "yes", + "rationale": "The single genuine win. Hoist EsMetadataState (shards+nodes+field-context) to resolve ONCE per scan instead of twice (ES-F1). The provider is already one instance per scan node (ResolvedScanProvider), so a field memo keyed by (index,columnNames) is a small surgical change with zero staleness risk (same statement) and directly collapses planScan + getScanNodePropertiesResult double-fetch." + }, + { + "piece": "per-statement-funnel-memoization", + "needed": "partial", + "rationale": "The Layer-2 funnel already routes EsConnectorMetadata (one instance per statement) and the arch gate is satisfied, but the connector does not exploit it: getTableSchema/getColumnHandles re-fetch the mapping remotely on each call (ES-F3), and the heavy scan state lives on a separate provider object the funnel does not cover. Adding a per-statement schema memo on EsConnectorMetadata (and hoisting state on the provider) would let the existing funnel actually collapse repeated loads." + }, + { + "piece": "cross-query-table-cache", + "needed": "partial", + "rationale": "Schema (mapping) is ALREADY cross-query cached by fe-core ExternalSchemaCache; the scan path just fails to reuse it (ES-F2). Serving the scan-path field-context from the already-cached schema removes the redundant cross-query mapping fetch without a new cache. A dedicated connector-side Table cache is not warranted. Shard/node topology must NOT be cross-query cached (freshness/rebalance)." + }, + { + "piece": "shared-fe-connector-cache-adoption", + "needed": "no", + "rationale": "The connector uses none of fe-connector-cache (CacheFactory/CacheSpec/MetaCacheEntry) and has no cross-query cache that needs to exist. The recommended fixes are per-statement/per-scan memos, not managed cross-query caches, so adopting the shared toolkit is unnecessary. Optional only if a mapping cache is later added." + }, + { + "piece": "partition-cache", + "needed": "no", + "rationale": "ES tables are non-partitioned single indices; EsConnectorMetadata does not override listPartitions/getNameToPartitionItems. No PARTITIONS scan, no SHOW PARTITIONS heavy path." + }, + { + "piece": "format-cache", + "needed": "no", + "rationale": "File format is a fixed constant es_http (EsScanPlanProvider.java:174, mapFileFormatType es_http at PluginDrivenScanNode.java:1905). No per-query format inference / whole-table planFiles fallback like iceberg PERF-03." + }, + { + "piece": "comment-cache", + "needed": "no", + "rationale": "EsConnectorMetadata does not override getComment; there is no per-table remote comment load driven N-per-query by information_schema/SHOW TABLE STATUS." + }, + { + "piece": "manifest-or-file-list-cache", + "needed": "no", + "rationale": "No manifests or file listings. The shard-routing fetch is the structural analog but is freshness-sensitive (must stay per-statement), so it is a per-scan-hoist target, not a cross-query manifest cache." + }, + { + "piece": "per-file-split-memo", + "needed": "no", + "rationale": "planScan builds scan ranges from an in-memory shard-routing map in a simple loop (EsScanPlanProvider.java:113-138) with no per-range/per-file remote call, so there is no per-split loop-invariant remote op to memoize." + }, + { + "piece": "authz-session-user-isolation", + "needed": "no", + "rationale": "Single catalog-level credential, no session=user / OIDC / kerberos doAs (EsConnector.java:95-100). Name-only caches cannot leak across users. The session-user shard gate is N/A." + }, + { + "piece": "write-txn-coholder", + "needed": "no", + "rationale": "The ES connector is read-only: no beginWrite/commit/insert/sink in EsConnectorMetadata. No read+write shared-metadata/one-txn consistency requirement." + } + ], + "overallVerdict": "Do NOT apply the iceberg heavy framework to es. ES is read-only, non-partitioned, single-credential, with no snapshots/manifests/files/stats/writes, so almost every iceberg piece (table/partition/format/comment/manifest caches, authz session-user isolation, write-txn co-holder, per-file memo) is inapplicable. The connector currently has ZERO dedicated scan-metadata caching and re-fetches the index mapping ~4x and shard/node topology ~2x per single SELECT. The ONE worthwhile, low-risk fix is a per-scan hoist of EsMetadataState (resolve shards+nodes+field-context once per scan instead of twice, ES-F1) plus reusing the already-schema-cached mapping in the scan path (ES-F2) and memoizing schema on the per-statement EsConnectorMetadata for getColumnHandles (ES-F3). All are P1/P2 constant-factor (2-4x) wins, not loop-amplified P0s. Shard routing must stay per-statement (ES rebalance/refresh model) — do not add a cross-query shard cache.", + "confidence": "high", + "verify": { + "verdict": "ADJUSTED", + "confirmed": [ + "ES-F1", + "ES-F2", + "ES-F3", + "ES-F4" + ], + "corrections": [ + { + "claim": "frameworkPiecesNeeded.cross-query-table-cache + ES-F2 fix rationale: 'Serving the scan-path field-context from the already-cached schema removes the redundant cross-query mapping fetch WITHOUT a new cache' / 'reuse the cached schema'.", + "issue": "The fe-core ExternalSchemaCache stores only the parsed ConnectorTableSchema (column list). EsConnectorMetadata.getTableSchema builds `new ConnectorTableSchema(indexName, columns, \"ELASTICSEARCH\", Collections.emptyMap())` (EsConnectorMetadata.java:90-93) — an EMPTY properties map; the raw mapping JSON is discarded. But EsMetadataFetcher.fetchMapping needs the raw mapping (keyword-sniff / doc_value / date-compat) via EsMappingUtils.resolveFieldContext(columnNames, sourceIndex, indexMapping, mappingType) (EsMetadataFetcher.java:57-63). So the scan-path field context CANNOT be derived from the currently cached schema value.", + "correction": "The redundant remote mapping fetch (ES-F2) is REAL and the multiplier holds, but the proposed fix is understated: eliminating it requires either enriching ConnectorTableSchema to carry the raw mapping/field-context, or adding a per-statement/cross-query mapping (field-context) cache — i.e. it is NOT a zero-new-cache 'just reuse the schema' change. Files: fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java:90-93; EsMetadataFetcher.java:57-63; EsMappingUtils.resolveFieldContext.", + "file": "fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java" + }, + { + "claim": "currentCaches RowCountCache row: 'ES has no stats -> returns UNKNOWN, no ES-side remote heavy op'.", + "issue": "fetchRowCount (PluginDrivenExternalTable.java:1121-1133) first calls resolveConnectorTableHandle -> EsConnectorMetadata.getTableHandle -> restClient.existIndex (a remote `index/_mapping` GET, EsConnectorRestClient.java:104-114) BEFORE it can return UNKNOWN. EsConnectorMetadata does not override getTableStatistics (confirmed by grep), so getTableStatistics itself makes no ES RPC, but the handle-resolution step does perform one lightweight remote GET.", + "correction": "The phrase is defensible for 'heavy' op (there is none), but 'no ES-side remote op' would be wrong: the row-count seam still issues one lightweight existIndex `_mapping` GET during handle resolution (this is the same op as ES-F4). It sits behind RowCountCache so it is not per-query hot; precision note only. File: fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java:1121-1133.", + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java" + } + ] + } + }, + { + "connector": "trino", + "migrationStatus": { + "inSpiReadyTypes": true, + "liveness": "live", + "notes": "\"trino-connector\" is in SPI_READY_TYPES (CatalogFactory.java:57) and is routed through the SPI/PluginDriven path at CatalogFactory.java:110-118 -> PluginDrivenExternalCatalog. It is a LIVE, fully-wired connector: TrinoConnectorProvider (getType()=\"trino-connector\") -> TrinoDorisConnector (implements Connector) -> TrinoConnectorDorisMetadata (implements ConnectorMetadata) + TrinoScanPlanProvider. Architecturally unique among the 8 SPI connectors: it is a pass-through BRIDGE that embeds the REAL Trino connector SPI. TrinoBootstrap loads Trino plugins and creates one live io.trino.spi.connector.Connector per Doris catalog (TrinoDorisConnector.java:53,143-188); every Doris metadata/scan call delegates to that embedded Trino Connector's own ConnectorMetadata / ConnectorSplitManager. Not sibling-only, not dormant, not legacy." + }, + "currentCaches": [ + { + "name": "TrinoBootstrap.instance (Trino plugin-infra singleton: typeRegistry / handleResolver / pluginManager)", + "scope": "cross-query", + "keyedBy": "pluginDir (process-global, one per FE)", + "ttl": "process lifetime (never invalidated)", + "file": "fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoBootstrap.java:100,141-156" + }, + { + "name": "TrinoDorisConnector.trinoConnector (+trinoSession/catalogHandle) — the long-lived EMBEDDED Trino Connector object per catalog", + "scope": "cross-query", + "keyedBy": "Doris catalog (one embedded Trino Connector per catalog, double-checked-locking init)", + "ttl": "catalog lifetime; invalidated on catalog close via trinoConnector.shutdown()", + "file": "fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoDorisConnector.java:53-57,133-141,95-102" + }, + { + "name": "Embedded Trino connector's OWN metadata cache (e.g. CachingHiveMetastore / iceberg metadata cache / CachingJdbcClient) — the ACTUAL cross-query metadata cache for Trino tables", + "scope": "metastore-client", + "keyedBy": "Trino-connector-owned key; enabled+sized by connector config (e.g. hive.metastore-cache-ttl); NOT Doris-managed", + "ttl": "Trino config-driven (0/disabled by default in several Trino connectors)", + "file": "lives inside TrinoDorisConnector.trinoConnector; no source in this module — read through on every trinoConnector.getMetadata(...).getTableHandle(...)" + }, + { + "name": "TrinoServicesProvider.catalogs (Trino-internal CatalogConnector holder)", + "scope": "cross-query", + "keyedBy": "CatalogHandle (one entry per Doris catalog)", + "ttl": "catalog lifetime", + "file": "fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoServicesProvider.java:63" + }, + { + "name": "TrinoPluginManager.connectorFactories (Trino ConnectorFactory registry)", + "scope": "cross-query", + "keyedBy": "ConnectorName", + "ttl": "process lifetime", + "file": "fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoPluginManager.java:64" + }, + { + "name": "Doris-connector-layer metadata/table/handle/schema cache", + "scope": "none", + "keyedBy": "n/a — NONE exists. TrinoConnectorDorisMetadata memoizes nothing; no fe-connector-cache toolkit dep (pom.xml has no fe-connector-cache); no ConcurrentHashMap/LoadingCache for metadata", + "ttl": "n/a", + "file": "fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConnectorDorisMetadata.java (whole class)" + }, + { + "name": "(fe-core generic, NOT trino-specific) ExternalSchemaCache + RowCountCache", + "scope": "cross-query", + "keyedBy": "catalog.db.table name; shared by ALL external connectors", + "ttl": "external_cache_expire_time (name-keyed, REFRESH-invalidated)", + "file": "fe/fe-core/.../datasource/plugin/PluginDrivenExternalTable.java:430 (initSchema fills schema cache), :1121 (fetchRowCount fills rowcount cache)" + } + ], + "funnelParticipation": { + "funneled": true, + "metadataMemoizesLoadedTable": "no", + "evidence": "Routed through Layer-2 funnel: PluginDrivenScanNode.metadata()/create() and PluginDrivenExternalTable.initSchema()/fetchRowCount() all acquire the connector metadata via PluginDrivenMetadata.get(session, connector) (PluginDrivenScanNode.java:206,222; PluginDrivenExternalTable.java:449,1126), which memoizes ONE ConnectorMetadata per (statement,catalog) on ConnectorStatementScope and is enforced build-wide by tools/check-fecore-metadata-funnel.sh. So the statement holds exactly one TrinoConnectorDorisMetadata. HOWEVER that wrapper memoizes NOTHING internally: every method — listDatabaseNames:100-101, listTableNames:120-121, getTableHandle:152-153, getTableSchema:194-195, applyFilter:272-273, applyProjection:337-338 — opens a FRESH Trino transaction (beginTransaction READ_UNCOMMITTED) + a fresh trinoConnector.getMetadata(connSession,txn) and commits it in finally; TrinoScanPlanProvider.planScan opens yet another (TrinoScanPlanProvider.java:111-113). getTableHandle additionally eagerly resolves all column handles + N x getColumnMetadata (TrinoConnectorDorisMetadata.java:166-176). No resolved TrinoTableHandle / Trino Table is stashed on the scope. So the funnel gives 'one Doris wrapper per statement' but does NOT collapse repeated loads inside it — that is left to the embedded Trino connector's own cache.", + "gaps": "The funnel does not fix the 'load each table once per statement' concern at the connector layer the way iceberg PERF-07 does. But the residual multiplier is LOW because (a) fe-core's cross-query ExternalSchemaCache/RowCountCache absorb the repeated getTableHandle/getTableSchema across queries, and (b) each getTableHandle reads through the embedded Trino connector's own long-lived metastore cache. There is no partition/stats/MVCC/write surface for the funnel to protect (all default no-ops)." + }, + "hotPathFindings": [ + { + "id": "TRINO-H1", + "area": "query-plan", + "problem": "getTableHandle is the only remote-ish metadata op and is invoked from 3 distinct fe-core sites per cold statement — initSchema (PluginDrivenExternalTable.java:463), fetchRowCount (:1128) and scan-node create (PluginDrivenScanNode.java:228). Each opens a fresh Trino transaction + trinoConnector.getMetadata + getTableHandle + N x getColumnMetadata (TrinoConnectorDorisMetadata.java:144-185). No per-statement memo of the resolved handle in the wrapper.", + "multiplier": "~1x per SELECT on a warm fe-core schema+rowcount cache; ~3x cold — but the schema call is behind ExternalSchemaCache and the rowcount call behind RowCountCache (both cross-query), and each getTableHandle reads through the embedded Trino connector's own metastore cache", + "cost": "mixed", + "severity": "P2", + "memoizedAlready": true, + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java:228" + }, + { + "id": "TRINO-H2", + "area": "schema", + "problem": "During one cold schema load, getTableHandle already builds columnMetadataMap by calling getColumnMetadata for every column (TrinoConnectorDorisMetadata.java:170-176); getTableSchema then IGNORES that map and re-loops getColumnMetadata per column (:207-209). 2N getColumnMetadata for one initSchema, plus a second Trino transaction.", + "multiplier": "2N getColumnMetadata per cold schema load (N=columns); once per ExternalSchemaCache miss, not per query", + "cost": "cpu", + "severity": "P2", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConnectorDorisMetadata.java:207" + }, + { + "id": "TRINO-H3", + "area": "predicate", + "problem": "applyFilter runs twice per scan: once in fe-core convertPredicate (PluginDrivenScanNode.java:824 -> TrinoConnectorDorisMetadata.applyFilter, its own Trino txn) and again inside TrinoScanPlanProvider.planScan (:126, another Trino txn). Same for projection (tryPushDownProjection vs planScan applyProjection). Predicate re-conversion (TrinoPredicateConverter) and pushdown recomputed on the already-loaded handle.", + "multiplier": "2x applyFilter + 2x applyProjection per scan", + "cost": "cpu", + "severity": "P2", + "memoizedAlready": false, + "file": "fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoScanPlanProvider.java:126" + }, + { + "id": "TRINO-H4", + "area": "split-enum", + "problem": "Split enumeration is fully delegated to Trino's ConnectorSplitManager.getSplits + BufferingSplitSource; the per-split loop only JSON-serializes the ConnectorSplit and extracts host addresses. Shared scan-invariant fields (tableHandle/txn/columnHandles/columnMetadata/options JSON) are already pre-serialized ONCE per scan before the split loop (TrinoScanPlanProvider.java:222-227), not per split.", + "multiplier": "shared fields 1x/scan (already hoisted); only unavoidable per-split JSON of the split itself remains", + "cost": "cpu", + "severity": "none", + "memoizedAlready": true, + "file": "fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoScanPlanProvider.java:221" + }, + { + "id": "TRINO-H5", + "area": "partitions", + "problem": "No hot path: listPartitions, getTableStatistics, estimateDataSizeByListingFiles, getMvccPartitionView, getTableFreshness, getSyntheticScanPredicates, and the entire write path are NOT overridden -> interface defaults (empty / -1 / handle-unchanged). getNameToPartitionItems sees empty partitions; fetchRowCount returns UNKNOWN. So iceberg PERF-02 (PARTITIONS rescan), PERF-03 (format fallback), PERF-04 (manifest), PERF-07/R6 (write) have NO analog to fix here.", + "multiplier": "0 (no such call)", + "cost": "cpu", + "severity": "none", + "memoizedAlready": true, + "file": "fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConnectorDorisMetadata.java:64" + } + ], + "authzConsistency": { + "perUserCredential": false, + "sessionUserRelevant": false, + "staleOrLeakRisk": "none — there is no per-user dimension to leak. TrinoBootstrap bakes a single static identity Identity.ofUser(\"user\") into the per-catalog Trino Session at CREATE CATALOG time (TrinoBootstrap.java:264-265) and reuses it for every Doris user. No REST OIDC, no kerberos doAs, no per-identity metastore, no vended per-user credentials. The embedded Trino connector authenticates to the underlying store with the STATIC credentials in the catalog properties (metastore auth / object-store keys). Doris's own privilege system governs who may query. A name-only cache would therefore NOT serve one user another user's metadata (no per-user authz to shard on); iceberg's session=user 'list != load' disclosure and the Layer-3 isolation / shouldBypassSchemaCache machinery are N/A here.", + "notes": "The PluginDrivenMetadata builder-identity pin (PluginDrivenMetadata.java:63-69) still runs but is vacuous for Trino since the identity is constant. Write-path consistency (read+write share one metadata/txn) is also N/A: TrinoConnectorDorisMetadata overrides zero write/DDL methods (createTable/dropTable/beginWrite/finishInsert/executeStmt count = 0), so trino-connector is read-only from Doris — no write txn co-holder requirement." + }, + "frameworkPiecesNeeded": [ + { + "piece": "per-statement-funnel-memoization", + "needed": "partial", + "rationale": "Already routed through the funnel (one wrapper/statement). Adding an internal per-statement memo of the resolved TrinoTableHandle would collapse the ~3 fresh Trino transactions/SELECT into one, but the payoff is small: fe-core's cross-query schema/rowcount caches already absorb most repeats, and the embedded Trino connector caches the getTable read-through. Low-priority nicety, not a correctness or scaling fix." + }, + { + "piece": "cross-query-table-cache", + "needed": "no", + "rationale": "Would DOUBLE-CACHE with the embedded Trino connector's own metadata cache (CachingHiveMetastore / iceberg / CachingJdbcClient), which owns invalidation. A Doris-side Table/handle cache would freeze schema across external ALTER and reintroduce the stale-metadata class of bug Trino already solves; the TrinoTableHandle is also transaction-derived + transient/non-serializable. Do NOT add." + }, + { + "piece": "partition-cache", + "needed": "no", + "rationale": "listPartitions is the interface default (empty); Trino exposes no partition view to Doris. Nothing to cache." + }, + { + "piece": "format-cache", + "needed": "no", + "rationale": "No getFileFormat / file-format inference path (split+format handling is inside the embedded Trino connector and BE JNI scanner). iceberg PERF-03 has no analog." + }, + { + "piece": "comment-cache", + "needed": "no", + "rationale": "getTableComment defaults to \"\"; comments ride on the columns already held by the cross-query ExternalSchemaCache. No N-per-query getComment load." + }, + { + "piece": "manifest-or-file-list-cache", + "needed": "no", + "rationale": "No manifest/file-listing on the FE side; estimateDataSizeByListingFiles defaults to -1 and split enumeration is delegated to Trino's ConnectorSplitSource." + }, + { + "piece": "per-scan-hoist", + "needed": "already-has", + "rationale": "TrinoScanPlanProvider already pre-serializes all scan-invariant fields once per scan before the split loop (TrinoScanPlanProvider.java:221-227); no per-file/per-split invariant is recomputed." + }, + { + "piece": "per-file-split-memo", + "needed": "no", + "rationale": "The per-split loop only serializes each split's own JSON + host list (genuinely per-split); there is no shared per-file invariant (partition JSON / delete carriers) to memoize as in iceberg PERF-11." + }, + { + "piece": "authz-session-user-isolation", + "needed": "no", + "rationale": "Static single identity, no per-user credential; no shared cache exists to shard, and no per-user leak is possible." + }, + { + "piece": "shared-fe-connector-cache-adoption", + "needed": "no", + "rationale": "There is nothing safe to cache at the Doris connector layer (Trino owns metadata caching + invalidation), so adopting CacheFactory/ConnectorPartitionViewCache would only create a double-cache correctness hazard." + }, + { + "piece": "write-txn-coholder", + "needed": "no", + "rationale": "Read-only connector: zero write/DDL method overrides. No write transaction to co-hold with the read metadata." + } + ], + "overallVerdict": "Do NOT apply the iceberg hot-path cache framework to trino-connector. It is a live pass-through BRIDGE that embeds the real Trino connector and delegates ALL metadata caching, partition/stats/split enumeration, and cache invalidation to that embedded connector. From Doris's view it has no partition/stats/MVCC/write/comment surface (all interface defaults), no per-user credentials, and no session=user analog. Its cross-query repeated-load protection already comes from two layers Doris does not need to duplicate: (1) fe-core's generic ExternalSchemaCache + RowCountCache, and (2) the embedded Trino connector's own metastore/metadata cache. Layer-2 funnel: already wired and gate-enforced; the wrapper itself memoizes nothing, but the residual per-statement multiplier is low (~1x warm). Layer-1 connector caches and Layer-3 authz isolation are actively contraindicated (double-caching + stale-metadata risk; no per-user dimension). The only optional, low-value improvements are P2 CPU cleanups: memoize the resolved TrinoTableHandle per statement to shed the extra Trino transactions, drop the getTableHandle/getTableSchema 2N getColumnMetadata re-fetch on cold schema load, and avoid the double applyFilter/applyProjection between convertPredicate and planScan. Recommended action: leave as-is (correctly delegated); optionally schedule the three P2 cleanups if a hot-catalog BI profile ever shows them.", + "confidence": "high", + "verify": { + "verdict": "CONFIRMED", + "confirmed": [ + "TRINO-H1", + "TRINO-H2", + "TRINO-H3", + "TRINO-H4", + "TRINO-H5" + ], + "corrections": [ + { + "claim": "Cache #4 'TrinoServicesProvider.catalogs' keyedBy: CatalogHandle", + "issue": "The map is actually keyed by the catalog-name String, not a CatalogHandle object; lookups derive the name from the handle.", + "correction": "TrinoServicesProvider.java:63 declares `ConcurrentMap catalogs`; getConnectorServices(CatalogHandle) looks up via `catalogs.get(catalogHandle.getCatalogName())` (line 126). One entry per catalog (correct), but keyed by name String. Non-material.", + "file": "fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoServicesProvider.java:63,126" + }, + { + "claim": "TRINO-H3: '2x applyFilter per scan'", + "issue": "Precisely 2x only for a scan carrying a pushable predicate. For a filterless scan the fe-core convertPredicate arm short-circuits (guarded by empty conjuncts, and by TupleDomain.isAll() in the wrapper), so only planScan's applyFilter (with Constraint.alwaysTrue) runs = 1x.", + "correction": "convertPredicate returns early when conjuncts empty (PluginDrivenScanNode.java:819-820); TrinoConnectorDorisMetadata.applyFilter returns empty before opening a txn when tupleDomain.isAll() (TrinoConnectorDorisMetadata.java:266-268); planScan always calls Trino applyFilter (TrinoScanPlanProvider.java:126). So '2x' holds for the WHERE-filtered case the finding targets; overstated only for filterless scans. Finding stands.", + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java:819" + }, + { + "claim": "authzConsistency: the PluginDrivenMetadata builder-identity pin is 'vacuous for Trino since the identity is constant'", + "issue": "The pin keys on the Doris principal session.getUser(), not the constant Trino Identity.ofUser(\"user\"); it does vary per Doris user across statements. It is trivially satisfied WITHIN one statement (one statement = one user), which is why it never fires for trino — not because the identity is constant.", + "correction": "PluginDrivenMetadata.java:63-69 uses session.getUser() (Doris principal). The static Trino identity is a separate layer (TrinoBootstrap.java:264-265). Conclusion (no per-user leak) is correct; the stated reason conflates the two identity layers. Non-material.", + "file": "fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenMetadata.java:63" + } + ] + } + } +] \ No newline at end of file diff --git a/plan-doc/connector-cache-unification/designs/foundation-design-FINAL.md b/plan-doc/connector-cache-unification/designs/foundation-design-FINAL.md new file mode 100644 index 00000000000000..5e04ce2144044d --- /dev/null +++ b/plan-doc/connector-cache-unification/designs/foundation-design-FINAL.md @@ -0,0 +1,195 @@ +Verification complete. All disputed load-bearing claims confirmed at HEAD: +- **BLOCKER real**: iceberg cache fields exist on BOTH `IcebergConnector.java` (marked, constructed) AND `IcebergConnectorMetadata.java:152,156,159,163,170,171` (unmarked, injected). Gate scans only `*Connector.java`. Hive constructs `partitionViewCache` unconditionally at `HiveConnector.java:134`, unmarked, and is excluded from the gate by the `.contains()` filter (`:543`). +- invalidateDb divergence real: all 5 iceberg caches use `Namespace.of(dbName)` + `id.namespace().equals(ns)` (`IcebergTableCache.java:102-103` et al.), not String equality. +- `ConnectorStatementScopeImpl.closeAll` pass-2 closes every `AutoCloseable` (`:65,82`); scope is `StatementContext`-owned with a reset+close path (`StatementContext.java:656,673-676,995-996`). +- mc/es have zero `getCapabilities`/`SUPPORTS_USER_SESSION` in main src (static identity, rule-D safe). mc's `partitionCache` is constructed on the Connector (`MaxComputeDorisConnector.java:79`). + +Below is the final synthesized design. + +--- + +## Final design summary + +The owner-signed, foundation-first plan is buildable, and the adversarial pass **shrinks it further**: the recommended path adds **zero fe-core source lines**, so iron rule A is literally untouched, not merely "minimally crossed." Three reviewer objections are accepted as behavior-changing and force revisions; two proposals are struck; one reviewer either/or is rejected with reasoning. + +Accepted revisions (most consequential first): +1. **Strike B2 entirely** (all three reviews). The maxcompute fan-out is collapsed connector-side (B1). No "sanctioned fe-core exception" is left on the table — a signed-off but avoidable fe-core mutation is how avoidable mutations get merged (Review 1 #1), and wrapping the shared `resolveConnectorTableHandle` funnel changes the call-count contract for jdbc/paimon/trino/hive that are out of this round's test scope (Review 3 #1). **fe-core grows by 0 lines. Iron rule A is untouched, not spent.** +2. **Redesign gate D3 from owner-file field-marker scan to module-wide construction-site scan** (Review 2 BLOCKER, Review 3 #3). The generic wrapper's real holders already live unmarked on `*ConnectorMetadata` today (verified) and can move anywhere tomorrow; a field-marker-on-`*Connector.java` gate is structurally blind to them and verifies a *comment*, not the *null-gate*. The gate must key off `new …Cache(` construction expressions and assert the capability null-gate at the construction site. +3. **Soften the iceberg retrofit from "byte-identical" to "functionally equivalent for Doris single-level namespaces," gated by an explicit `invalidateDb` parity test** (Review 1 #4, Review 3 #2). The retrofit relocates db-extraction from iceberg `Namespace` equality to String equality; keep db-extraction in the connector's key builder. +4. **Make the hudi non-closeable projection mandatory, not optional** (Review 3 #5). Hudi is the first `AutoCloseable`-risk value in the scope. +5. **Resolve the hudi freshness/(A)-cache tension** (Review 3 #6) rather than deferring: re-derive the latest *completed* instant fresh per statement (cheap, from the memoized metaClient's timeline), cache the *expensive partition list* cross-query under `(table, instant)`. Hits occur precisely when the table has not committed (the common case); a new commit mints a new instant → new key → miss → fresh load. Freshness is preserved and the win is real — the either/or framing is rejected, but its "latest-completed + fresh-per-statement" pin is adopted. +6. **Delete the `ConnectorPartitionViewCache[V]` compat subclass** (Review 1 #3): its three consumers are already rewritten for the key rename in PR-1, so keep one type (`ConnectorMetadataCache[V]`) and drop the inheritance layer. +7. **Verify the prepared-EXECUTE scope lifecycle before PR-2** (Review 3 #4) and centralize the `keyNamespace` set (Review 3 #7) and legacy telemetry names (Review 3 #8). + +The design targets only the two real cross-query consumers (iceberg + hudi) plus two per-statement-only consumers (mc + es). No `[K]` type parameter, identity-sharding dimension, or credential abstraction is added (Rule 2). + +--- + +## A/B/C/D components (revised) + +### A — Generic cross-query cache wrapper (fe-connector-cache) + +**Layer**: `fe/fe-connector/fe-connector-cache`, package `org.apache.doris.connector.cache` (JDK+Caffeine only, verified zero fe-core imports — the fe-core `datasource/metacache/` `CacheSpec`/`MetaCacheEntry` is an independent duplicate this track never touches; iron rule A clean for the whole A track). + +**What is generalized (grounded).** `ConnectorPartitionViewCache[V]` already is the octet iceberg hand-rolls: `new MetaCacheEntry[...](engine + "." + entryName, null, spec, ForkJoinPool.commonPool(), false, true, 0L, true)` (`ConnectorPartitionViewCache.java:70-71`), `matches`/`matchesDb` on the key (`PartitionViewCacheKey.java:66-72`). The five iceberg caches differ only in **key type**, **value type**, and the **db-match predicate**. It hardcodes exactly two axes: `ENTRY_PARTITION_VIEW="partition_view"` and the key class. The manifest cache is **excluded** (path-keyed, no-TTL, default-off, `DataFile`-typed, `authz-cache-exempt` — read only after a per-user `resolveTable`; `IcebergConnector.java:203`). + +**Class shape.** Promote to `ConnectorMetadataCache[V]`: +- **Value stays opaque `[V]`** — load-bearing: values are `String` (comment/format), raw credentialed `org.apache.iceberg.Table` (table-handle), 2-long `CachedSnapshot` (latest-snapshot), `List[…]` (partition). A concrete value type would be useless to a second connector. +- **Key**: rename `PartitionViewCacheKey` → `ConnectorTableKey(db, table, snapshotId, schemaId)` (body already generic; only its name is partition-flavored). Unused axes pass `-1`. One key type for all six cases. +- **Two ctors**: framework-native per-entry (`meta.cache.[engine].[entry].{enable|ttl-second|capacity}`, recommended for hudi/mc/es) **and** pre-resolved `CacheSpec` (so iceberg keeps its single shared `meta.cache.iceberg.table.ttl-second` knob across all five caches — operator back-compat). The `ttl<=0 -> CACHE_TTL_DISABLE_CACHE` mapping that is copy-pasted five times today moves into `CacheSpec.of`/`fromProperties` (already there). +- **Delete `ConnectorPartitionViewCache[V]`** (revision #6). Iceberg/hive/paimon are touched by the rename in PR-1 anyway; have them construct `ConnectorMetadataCache[V]` directly (hive currently does `new ConnectorPartitionViewCache[](\"hive\", props)` at `HiveConnector.java:134` → `new ConnectorMetadataCache[](\"hive\", \"partition_view\", props)`). One type, no inheritance layer for a non-SPI toolkit class. +- **Fix the stale javadoc** `"this class has NO consumers yet"` (`ConnectorPartitionViewCache.java:33`, confirmed false — iceberg `:197,199`, hive `:134`, paimon consume it). +- **Pin legacy telemetry names in the iceberg retrofit** (Review 3 #8): the framework-native ctor derives `name = engine + \".\" + entryName` (→ `iceberg.table`), but iceberg's live entry names are `iceberg-table`, `iceberg-partition`, `iceberg-latest-snapshot`, etc. (`IcebergTableCache.java:70`). The iceberg retrofit MUST pass the exact legacy `name` via the pre-resolved `CacheSpec` ctor so dashboards/log-greps/`*ForTest` accessors don't silently break. + +**Credential policy stays connector-side (iron rule D).** The wrapper is value-opaque and knows nothing about credentials. The connector nulls the field under its gate exactly as today (verified `IcebergConnector.java:231,243,254,260` — `isUserSessionEnabled()`/`restVendedCredentialsEnabled` ternaries): table-handle null when `isUserSessionEnabled() || restVendedCredentialsEnabled`; comment null unless `restVended && !userSession`; format/partition/latest-snapshot null under `isUserSessionEnabled()`. A null field ⇒ `get()` never called ⇒ live bypass. **Trino alignment**: shared low-level toolkit + per-connector caches, no unified cache; we deliberately do NOT port Trino's per-identity `LoadingCache[user,…]` sharding — Doris keeps the binary null-field cut (correct for D1, no connector needs identity keying). + +### B — Statement-scope seam (fe-connector-api ONLY; fe-core untouched) + +**Premise correction (verified).** The memo substrate is already built and already in **fe-connector-api**: `ConnectorStatementScope.computeIfAbsent(key, Supplier)` (`:45`), `ConnectorSession.getStatementScope()` defaulting to `NONE` (`:142`), backed by `ConnectorStatementScopeImpl` (ConcurrentHashMap, idempotent `closeAll`) hung on `StatementContext` (`:209,656`). Iceberg and hudi depend on fe-connector-api and on **neither fe-core nor fe-connector-cache** — a literal fe-core class would be *unreachable* by the retrofit. **The D4 mandate is satisfied with 0 fe-core source lines; iron rule A is not crossed at all.** + +**The seam: one bare static helper + a namespace registry in fe-connector-api** (beside `ConnectorStatementScope`): + +``` +public final class ConnectorStatementScopes { + private ConnectorStatementScopes() {} + // resolve db.table once per statement; keyNamespace namespaces the value TYPE so a heterogeneous + // gateway statement touching two connectors cannot collide on (db,table) -> ClassCastException. + public static [T] T resolveInStatement(ConnectorSession session, String keyNamespace, + String db, String table, Supplier[T] loader) { + if (session == null) { + return loader.get(); // == ConnectorStatementScope.NONE + } + String key = keyNamespace + ":" + session.getCatalogId() + ":" + db + ":" + table + + ":" + session.getQueryId(); + return session.getStatementScope().computeIfAbsent(key, loader); + } +} +``` + +- No new interface, no new type on `ConnectorSession`, no new SPI method — it reuses the existing primitive and standardizes the **security-critical** key convention (dropping `queryId` leaks cross-statement; dropping `catalogId` collides across a cross-catalog MERGE). Centralizing this once is a real correctness win over two connectors re-deriving it (Review 1 #2 grants this). +- **Namespace registry** (Review 3 #7): define the `keyNamespace` constants (`iceberg.table`, `hudi.metaclient`, `maxcompute.handle`, `es.metadata_state`) in one small holder beside `ConnectorStatementScopes`, documented as a reviewed-uniqueness invariant mirroring the existing metadata-funnel key convention (`ConnectorStatementScope.java:51`). `catalogId` already sits inside the key, so two connectors under different catalogs can't collide even on an equal namespace — but the registry prevents same-catalog namespace reuse. + +**Retrofit + consumption:** +- **Iceberg (retrofit)**: `IcebergStatementScope.sharedTable` collapses to `return ConnectorStatementScopes.resolveInStatement(session, \"iceberg.table\", db, table, loader);` — namespace `iceberg.table` reproduces the existing `\"iceberg.table:\"` prefix byte-for-byte, so the funnel sites keep identical hits/misses/`NONE` fallthrough. `rewritableDeleteSupply` stays iceberg-private (a `(catalogId,queryId)`-keyed scan→write delete bridge, not table resolution — out of the seam). **Residual (Review 1 #2): whether to land this retrofit in PR-2 or defer it** — retrofitting audited iceberg widens the diff for no functional gain, but it is also the parity proof. Recommendation: keep it in PR-2 guarded by iceberg's existing memo tests; if the owner wants a narrower blast radius, hudi can be the sole first consumer and iceberg converges later. +- **Hudi (consumes)**: `HoodieTableMetaClient mc = ConnectorStatementScopes.resolveInStatement(session, \"hudi.metaclient\", db, table, () -> buildMetaClient(...))` collapses the ~6 builds/query → 1. Derived `TableSchemaResolver`/Avro/`InternalSchema` compute once off the single memoized client (~4x re-parse → 1). **MANDATORY (Review 3 #5, revision #4)**: memoize a **non-closeable projection** (the resolved schema/timeline snapshot callers need), NOT the raw `HoodieTableMetaClient`, because `closeAll` pass-2 closes every stored `AutoCloseable` at statement end (`ConnectorStatementScopeImpl.java:82`) and hudi is the first connector data-object that may be closeable / hold a `HoodieStorage`/`FileSystem`. This ordering was validated for iceberg's non-closeable `Table`, never for a closeable client. +- **Maxcompute (consumes, B1 only)**: memoize inside `MaxComputeConnectorMetadata.getTableHandle` via `resolveInStatement(session, \"maxcompute.handle\", db, table, …)`. All fan-out sites in `resolveConnectorTableHandle` then hit the connector memo; the remote `exists()` HEAD probe + lazy `get()` fires once. **fe-core untouched. B2 struck.** Precondition verified: `ConnectorSessionBuilder.captureStatementScope()` reads `sc.getOrCreateConnectorStatementScope()` (`:202`), so all in-statement `buildConnectorSession()` calls share one scope instance — B1 genuinely fires. +- **mc claim scoped (Review 1 #5)**: the collapse applies to the in-statement `buildConnectorSession()` subset only; `buildCrossStatementSession()` callers (refresh/cross-statement paths) bind a non-per-statement scope and by design do not share the memo. Do not "fix" them into it. + +**Rule D safety.** The L1 memo is single-identity by construction — a statement resolves one catalog under one identity (`PluginDrivenMetadata` fail-loud pin), scope per-statement + GC'd at end. It never crosses users even when the L2 §A cache is disabled for a vended catalog. + +### C — Authz gate generalization (tools/check-authz-cache-sharding.sh) — REDESIGNED + +> **SUPERSEDED (2026-07-24, owner decision): the gate was REMOVED, not generalized.** Two rounds of adversarial +> clean-room review showed that structurally verifying "this cache is null under session=user" in a shell +> script requires understanding arbitrary Java boolean/multi-line syntax (compound `&&`/`||` guards, multi-line +> lambda loaders, string literals, nested ternaries) — intractable and fragile (false positives break builds on +> legit code; false negatives silently miss leaks). The runtime `IcebergConnectorCacheTest` already proves the +> invariant for the real caches; the gate (`tools/check-authz-cache-sharding.sh` + self-test + the +> `fe-connector/pom.xml` execution) was deleted and replaced by an explicit `ATTN` comment at the iceberg cache +> sites. See [`../progress.md`](../progress.md) "2026-07-24 (5)". The redesign notes below are retained for history only. + +Pure tooling; touches no product source. The original owner-file field-marker scan is **rejected** (Review 2 BLOCKER, Review 3 #3): verified that iceberg holds cache fields on both `IcebergConnector.java` (marked) and `IcebergConnectorMetadata.java:152-171` (unmarked injected), and hive constructs an unmarked cross-query `partitionViewCache` at `HiveConnector.java:134` that the `.contains()` filter excludes from scope. A field-marker gate verifies a comment near a declaration, not the actual null-gate, and is blind to any holder off `*Connector.java`. + +**Redesigned gate — construction-site scan, location-independent:** + +- **Stage 1 — enumerate declaring modules**: `find` all `*Connector.java` under `*/src/main/java/*` (exclude `target/`); keep a module whose `*Connector.java` **declares** the capability via `\.add\([^)]*ConnectorCapability\.SUPPORTS_USER_SESSION` OR `(EnumSet|Set|ImmutableSet)\.of\([^)]*SUPPORTS_USER_SESSION`, after stripping comment lines (`^\s*\*`, `^\s*//`). This admits `IcebergConnector.java:860` (`.add(...)`). +- **Stage 1b — gateway modules in scope (Review 2 major)**: ALSO keep any module whose `*Connector.java` **reads** a sibling capability via `\.getCapabilities\(\)\.contains\([^)]*SUPPORTS_USER_SESSION` — this is the gateway signature (`HiveConnector.java:543`). Gateways delegate to per-user siblings and hold live cross-query caches, so they are in-scope, not excluded. Additionally assert the fail-loud front-door guard is present (the `throw` at `HiveConnector.java:546`); if a gateway reads the sibling capability but lacks the fail-loud assertion, FAIL. +- **Stage 2 — scan construction sites module-wide** (the key change): within each in-scope module, grep ALL `src/main/java` files for construction expressions `new (ConnectorMetadataCache|[A-Za-z_][A-Za-z0-9_]*Cache)[ <(]` (the holder idiom), distinguishing them from injected ctor-param reference fields (which are not `new`). Each construction site must either: + - sit inside a capability null-gate — its assignment RHS is a `isUserSessionEnabled() ? null : new …` (or equivalent capability-gated) ternary — the `authz-cache-session-user-disabled` contract, **now checked at construction, not just as a field comment**; OR + - carry an on-line/line-above `authz-cache-exempt` marker with justification. +- **Stage 3 — hive's `partitionViewCache` (Review 2 major)**: require it to carry an explicit `authz-cache-exempt` marker documenting "front door never declares SUPPORTS_USER_SESSION + fail-loud sibling guard," so the exemption is a reviewed claim, not an invisible gap. (It is unconditionally constructed and unmarked today — `HiveConnector.java:112,134`.) +- **Anti-no-op floor (Review 2, design §C.2)**: if Stage 1+1b yields **zero** in-scope modules, `exit` non-zero — a capability rename or grep drift must fail loud, never silently turn the gate into pass-everything. + +**Self-test additions**: (a) fixture with `.contains(SUPPORTS_USER_SESSION)` + fail-loud guard + an `authz-cache-exempt` cache → PASS (gateway path); (b) same gateway but MISSING the fail-loud guard → FAIL; (c) fixture declaring via `.add`/`EnumSet.of` with a `new …Cache(` constructed unconditionally (no null-gate, no marker) → FAIL (proves construction-site + null-gate check); (d) fixture with a marked field but constructed unconditionally → FAIL (proves the marker-detached-from-guard hole is closed). Keep iceberg as the primary declarer fixture. + +**Scope reality**: today Stage 1 resolves to `{iceberg}` and Stage 1b to `{hive}`; hudi/mc/es have no capability override (verified) and vend static credentials, so their caches are authz-safe without the gate and correctly fall outside it. D3 is **forward-insurance**, orthogonal to D1/D2, and does not block them. The fe-core defense layer (`PluginDrivenExternalCatalog.shouldBypass*Cache`, keyed off runtime `.contains(SUPPORTS_USER_SESSION)`, `:1289`) is already connector-agnostic and needs no change. + +### D — Connector consumption + +**D.1 Hudi (flagship) — consumes B (primary) + A (secondary).** +- **metaClient + schema memo (B)**: route both `HoodieTableMetaClient.builder()` sites through `resolveInStatement(session, \"hudi.metaclient\", …)` — memoizing a **non-closeable schema/timeline projection** (mandatory, §B). `buildMetaClient` reloads the active timeline for freshness, so the per-statement tier is the correct one for the raw client; a cross-query cache of the raw metaClient would be wrong. +- **HMS layer**: hudi is HMS-backed; its metastore access sits behind the shared `CachingHmsClient` (Trino `CachingHiveMetastore` shape, coarse REFRESH+TTL, TCCL-pinned per the `ThriftHmsClient.doAs` memory) exactly as iceberg-on-HMS/hive, so `getTable`/listing round-trips are shared cross-query beneath the per-statement memo. +- **`(table, instant)` partition cache (A) — tension resolved (revision #5, rejecting Review 3 #6's either/or)**: a `ConnectorMetadataCache[List[HudiPartition]]`, `entryName=\"partition\"`, key `ConnectorTableKey(db, table, instant-as-snapshotId, -1)`. **Pin (adopting Review 2 minor + Review 3 #6's fresh requirement)**: the instant MUST be `metaClient.getActiveTimeline().filterCompletedInstants().lastInstant()` (latest *completed*, never inflight/requested), **re-derived fresh each statement** from the statement-scoped metaClient. This is cheap (timeline read); the *expensive* partition-listing is what the cache saves. Cross-query hits occur when the table has not committed between queries (the common case — a stable table's latest-completed-instant is the same value each statement, so the key is identical); a new commit mints a new instant → new key → miss → fresh load. Freshness is therefore preserved AND hit rate is real — Review 3's "near-zero hit rate" holds only for a table committing on every query, which is not the steady state. TTL+REFRESH is the backstop (iron rule E). The cached value is a pure `List[HudiPartition]` projection with **no live metaClient/FileSystem reference** (assert in test). +- **Pom**: add `fe-connector-cache` dependency (absent today) and **verify hudi's shade/assembly self-bundles Caffeine ≥ 2.9.3** — hudi does not receive it transitively as iceberg does via iceberg-core; otherwise the child-loaded `MetaCacheEntry` `NoClassDefFound`s at runtime though it compiles (framework-coherence memory note). +- Iron rules: A honored (all memo/caches connector-side); D honored (hudi vends no per-user credentials, no `SUPPORTS_USER_SESSION`); E honored (instant-in-key + TTL + fresh derivation). + +**D.2 Maxcompute — consumes B only (B1).** +- Memoize `MaxComputeConnectorMetadata.getTableHandle` per statement (`maxcompute.handle`); collapses the ~14–17-site `resolveConnectorTableHandle` fan-out → 1, dropping the redundant remote `tables().exists()` HEAD probe + lazy `tables().get()` to once. Within one resolved handle the ODPS metadata already amortizes (`odpsTable.getSchema()`). +- **No new cross-query cache**: mc already owns `MaxComputePartitionCache` keyed `(db,table)`, constructed on the Connector (`MaxComputeDorisConnector.java:79` — the §C construction-site scan covers it). It uses static AK/SK, declares no `SUPPORTS_USER_SESSION` (verified) → rule-D-compliant as-is, no §A wrapper this round. +- Verify: ODPS static identity confirmed → handle safe to share per-statement (residual: whether it could later be promoted cross-query). Confirm `getTableHandle` has `ConnectorSession` access for the B1 wrap. + +**D.3 Es — consumes B only, must NOT get a cross-query cache (iron rule E).** +- **per-scan hoist of `EsMetadataState` (B)**: `fetchMetadataState` fires 2x/query (planScan + buildScanNodeProperties). Memoize the **mapping/schema half** per statement via `resolveInStatement(session, \"es.metadata_state\", …)` → 1 fetch. Es has no `*Cache` field at HEAD (grep empty), so there is nothing to regress yet — PR-6 introduces the memo. +- **mapping/field-context carrier**: `getTableSchema` discards the raw mapping string `resolveFieldContext` later needs (`EsConnectorMetadata.java:85-93`), forcing ~4 remote `getMapping`/query. Attach the raw mapping to the **statement-scoped `EsMetadataState`** (or `EsTableHandle`), NOT to the generic `ConnectorTableSchema` (must stay connector-agnostic — iron rule B). Collapses ~4 `getMapping` → 1. +- **CRITICAL split (iron rule E)**: `EsMetadataFetcher.fetch()` bundles `fetchMapping()` (reuse-safe) with `fetchShards()` (live shard routing). The memoized `EsMetadataState` carries **only** mapping/field-context; `fetchShards` is **always re-run per statement** and is **never** a cross-query name-keyed cache. Es gets no §A cache. + +--- + +## Sequencing + +Foundation → flagship → two small PRs → gate. Each PR independently landable + verified. + +1. **PR-0 (verification gate, blocks PR-2)** — confirm the prepared-EXECUTE scope lifecycle (Review 3 #4). Verified: `connectorStatementScope` is `StatementContext`-owned with a reset+close path (`StatementContext.java:672-677`) and closes at result return (`:995-996`). **Mechanism correction (2026-07-23 recon):** a prepared `EXECUTE` does NOT get a fresh `StatementContext` — it deliberately *reuses* one across executions (`ExecuteCommand.java:89`). Isolation is provided not by a fresh context but by an explicit `statementContext.resetConnectorStatementScope()` at the top of every `ExecuteCommand.run()` (`ExecuteCommand.java:95`, before all branches) plus a per-retry reset (`StmtExecutor.java:1077`). Recon also confirmed external/connector tables genuinely reach this path (they never take the OLAP short-circuit fast path — that rule matches `logicalOlapScan()` only, `LogicalResultSinkToShortCircuitPointQuery.java:88,97` — so an external-table prepared query always runs the normal `executor.execute()` planner, re-resolving the table each execution). The reset call is therefore reachable and load-bearing, but had NO test proving `ExecuteCommand` invokes it (existing tests cover only the reset primitive). Encode as a **wiring test** that drives `ExecuteCommand.run()` (two EXECUTEs of one prepared stmt must NOT share a memoized table). Cheap; do it before shipping the shared helper. +2. **PR-1 — Generic cache wrapper (fe-connector-cache).** Add `ConnectorMetadataCache[V]` (both ctors); rename `PartitionViewCacheKey` → `ConnectorTableKey`; **delete** `ConnectorPartitionViewCache[V]` and repoint iceberg/hive/paimon to construct `ConnectorMetadataCache[V]` directly; fix the stale javadoc. Pure addition + rename + deletion; no behavior change. Verify: reactor test-compile + existing partition-view tests. +3. **PR-2 — Statement-scope seam (fe-connector-api) + iceberg retrofit.** Add `ConnectorStatementScopes.resolveInStatement` + namespace registry; collapse `IcebergStatementScope.sharedTable` to delegate (byte-identical key). **fe-core: 0 lines. B2 does not exist.** Verify: iceberg per-statement memo tests + full iceberg suite prove cost-invariance. (Residual: defer iceberg retrofit to narrow blast radius — Review 1 #2.) +4. **PR-3 (foundation-first) — Iceberg cache convergence.** Retrofit the five hand-rolled caches onto `ConnectorMetadataCache[V]` via the pre-resolved-`CacheSpec` ctor (shared knob + **pinned legacy names**), keeping connector-side credential null-gates. **Gated on the new `invalidateDb` parity test** (revision #3). Residual #2: now vs later. +5. **PR-4 — Flagship hudi.** Pom dep + Caffeine-bundling verification; metaClient + schema memo via B (**non-closeable projection**); HMS behind `CachingHmsClient`; the `(table, completed-instant, fresh-per-statement)` cross-query partition cache via A; per-scan hoist. **E2E regression required** (heterogeneous + standalone hudi-on-HMS, per the HMS-delegated-capability memory rule), including a concurrent-commit partition-listing test. +6. **PR-5 — Maxcompute (small).** B1 memo inside `getTableHandle`; drop the redundant `exists()` probe. Verify: remote-op call-count on a SELECT. +7. **PR-6 — Es (small).** Per-scan hoist `EsMetadataState` via B; attach raw mapping to statement-scoped state; split `fetch()` so shard routing stays per-statement. Verify: `getMapping` → 1; shard routing re-runs per statement. +8. **PR-7 — Gate generalization (D3, redesigned).** Construction-site module-wide scan + gateway handling + anti-no-op floor + null-gate assertion; extend self-test fixtures. Orthogonal; recommended after PR-1/PR-2 (marker/`*Cache` conventions stable) and before any of hudi/mc/es would declare session=user. + +PR-1 + PR-2 are the foundation and unblock PR-3/4/5/6. PR-7 is independent. PR-0 blocks PR-2. + +--- + +## Risk register + +| # | Risk | Severity | Mitigation | Owner of proof | +|---|------|----------|------------|----------------| +| R1 | **Gate blind spot** — a future per-user connector holds a `ConnectorMetadataCache` off `*Connector.java` and leaks cross-user | High (was BLOCKER) | Construction-site module-wide scan (§C Stage 2) + null-gate assertion + anti-no-op floor; self-test fixtures (c)/(d) | PR-7 self-test | +| R2 | **Marker/guard drift** — copy-paste keeps the `authz-cache-session-user-disabled` comment but drops the `isUserSessionEnabled() ? null :` guard | High | Gate asserts the capability-gated ternary at the construction RHS, not just the field comment; self-test fixture (d) | PR-7 self-test | +| R3 | **Hive gateway silent hole** — live unmarked `partitionViewCache` + per-user sibling delegation fall outside gate | High | Gateway modules in-scope (Stage 1b); require `authz-cache-exempt` on `partitionViewCache` + assert fail-loud front-door guard | PR-7 self-test (a)/(b) | +| R4 | **Iceberg `invalidateDb` semantic drift** — `Namespace` equality → String equality diverges for multi-level namespaces | Med | Parity test asserting identical eviction (incl. multi-level case or a proof Doris iceberg namespaces are single-level); keep db-extraction in connector key builder | PR-3 | +| R5 | **Hudi metaClient use-after-close** — `closeAll` pass-2 closes a closeable client whose derived objects escape the statement | Med | Memoize a non-closeable projection (mandatory); test a scan pump reading memoized hudi state after nominal statement end | PR-4 | +| R6 | **Hudi Caffeine bundling** — compiles, `NoClassDefFound`s at runtime | Med | Verify hudi shade/assembly self-carries Caffeine ≥ 2.9.3; add explicit dep if absent; redeploy smoke test | PR-4 | +| R7 | **Prepared-EXECUTE stale memo** — scope + queryId both reused → cross-execution stale table | Med | PR-0 lifecycle verification + test (two EXECUTEs must not share memoized table) | PR-0 | +| R8 | **Es shard-routing regression** — reuse layer memoizes `fetchShards` output | Med | `EsMetadataState` holds only mapping/field-context by construction; `fetchShards` always fresh; assert routing re-runs | PR-6 | +| R9 | **Cross-connector `keyNamespace` collision** — gateway statement → `ClassCastException` | Low | Required distinct `keyNamespace` param + central registry; `catalogId` inside key | PR-2 | +| R10 | **Iceberg telemetry name drift** — `iceberg-table` → `iceberg.table` breaks dashboards/`*ForTest` | Low | Pin exact legacy `name` strings via pre-resolved ctor; assert names in cache tests | PR-3 | +| R11 | **PR-3 re-touches audited caches** | Low | Provably-identical octet/`CacheSpec`/invalidation (except R4); gate on `*ForTest`/`loadCountForTest`; deferrable if convergence not worth the diff | PR-3 | +| R12 | **Hudi (A) cache staleness** if latest-instant is itself cross-query cached instead of fresh-derived | Low | Pin latest-*completed*-instant, re-derived fresh per statement from the memoized timeline; only the partition list is cross-query cached | PR-4 | + +--- + +## Verification plan (UT/e2e/call-count gates) + +**Behavior-invariance gates (iceberg retrofit — must prove zero change):** +- **UT / call-count**: existing iceberg `*ForTest` accessors + `loadCountForTest` on all five caches — hits/misses/evictions unchanged pre/post PR-3. (`IcebergTableCache.java:70` legacy names asserted.) +- **UT parity (R4)**: `invalidateDb(db)` on the retrofitted `ConnectorTableKey` path evicts exactly the entries the `Namespace.of(db) + id.namespace().equals` path evicts — including a multi-level-namespace fixture OR an explicit assertion that Doris iceberg namespaces are provably single-level. +- **UT (R7)**: two `EXECUTE`s of one prepared statement do NOT share a memoized `Table` (fresh scope per execution). +- **Symbol-level**: full fe-connector reactor `test-compile` BUILD SUCCESS after PR-1's rename + `ConnectorPartitionViewCache` deletion (strongest single invariance signal, per the zero-conflict-rebase memory). +- **Byte-key check**: assert the retrofitted iceberg statement-scope key string equals the pre-retrofit `\"iceberg.table:\" + catalogId + \":\" + db + \":\" + table + \":\" + queryId`. + +**Perf-win gates (new consumers — must prove the redundancy collapses):** +- **Hudi (call-count UT)**: one `SELECT` builds `HoodieTableMetaClient` exactly 1x (was ~6) and parses schema/`InternalSchema` exactly 1x (was ~4); assert via a counting test double on `buildMetaClient`. +- **Hudi (e2e, required)**: heterogeneous HMS gateway + standalone hudi-on-HMS run the same SELECT/partition-listing and assert identical results (HMS-delegated-capability memory rule); plus a **concurrent-commit** e2e — list partitions across a live hudi commit, assert no partial/stale set (validates the latest-completed + fresh-per-statement pin, R12). +- **Hudi (UT, R5)**: the `(table,instant)` cached value holds no live metaClient/`FileSystem` reference (pure `List[HudiPartition]`); a scan pump reading memoized hudi state after nominal statement end does not hit a closed resource. +- **Maxcompute (call-count UT)**: one `SELECT` resolves `getTableHandle` 1x (was 14–17) and fires the remote `tables().exists()` HEAD probe 1x; assert via a counting ODPS client double, scoped to the in-statement `buildConnectorSession()` path (not `buildCrossStatementSession`, Review 1 #5). +- **Es (call-count UT)**: one `SELECT` calls `getMapping` 1x (was ~4) via the statement-scoped `EsMetadataState`; **and** `fetchShards()` still runs per statement (routing freshness) — assert both counts. + +**Gate self-tests (PR-7):** fixtures (a) gateway with guard → PASS, (b) gateway missing guard → FAIL, (c) declarer with unconditional `new …Cache(` → FAIL, (d) marked field constructed unconditionally → FAIL, plus (e) zero-declarer input → non-zero exit (anti-no-op). Iceberg remains the primary declarer fixture and must still PASS unchanged. + +**e2e note:** groovy regression requires a live cluster and does not run in this environment; the hudi e2e is specified for CI, not local. + +--- + +## Residual owner decisions + +1. **D4 placement (confirm the re-read).** Recon proves the reachable home is **fe-connector-api**, not fe-core (iceberg/hudi depend on neither fe-core nor fe-connector-cache; the memo primitive already exists in fe-connector-api). Accept re-reading "fe-core seam" as "fe-connector-api helper," leaving **fe-core source flat and iron rule A untouched (0 lines)**? All three reviews recommend this and recommend **striking B2 outright** rather than keeping it as a signed-off fe-core option. Recommendation: accept; strike B2; mc uses B1. +2. **Iceberg statement-scope + cache retrofit timing.** Retrofit iceberg in PR-2/PR-3 now (foundation-first, max convergence, proves parity, wider diff over audited code) — or let hudi be the sole first consumer and converge iceberg later/never (narrower blast radius, forgoes the mandate's convergence)? (Review 1 #2 / Risk R11.) +3. **Config namespace for hudi/mc/es caches.** Framework-native per-entry knobs (`meta.cache.[engine].[entry].*`, recommended) or a single shared per-catalog knob like iceberg's `table.ttl-second`? Affects operator surface and any `CacheSpec.applyCompatibilityMap` back-compat for existing mc/es TTL properties. +4. **Comment cache as a first-class entry?** Iceberg builds `commentCache` only for vended catalogs (plain catalogs serve comments from the table-handle cache). None of hudi/mc/es requested a comment cache — leave "comment" per-connector (recommended) rather than modeling it in the wrapper? +5. **MVCC-pin shape.** Keep the pin as opaque `[V]` over the 4-tuple `ConnectorTableKey` with `-1` sentinels (recommended — iceberg `(snapshotId,schemaId)`, paimon single long, hudi instant, mc/es none all fit one key), rather than a fixed 2-axis value type? +6. **Credential-value hardening: contract vs convention.** Keep the current convention (connector nulls the field + §C construction-site null-gate assertion, now enforced) — the smaller design matching today — or add a type/marker contract that makes the wrapper *refuse* credentialed values? Given the redesigned gate now checks the null-gate at construction (R2 closed), convention is likely sufficient; a contract is extra insurance. +7. **Gate strictness + gateway posture.** Hard build FAIL on zero declarers (recommended) — confirmed. And confirm the hive gateway is **in-gate-scope with a required fail-loud-guard assertion** (this design's position, Review 2 major) rather than documented as out-of-scope. +8. **Maxcompute cross-query shareability.** Confirm ODPS uses a static (non-per-user) identity so a resolved handle could later be promoted cross-query — this round it is statement-scoped only (safe default). Verified: mc declares no `SUPPORTS_USER_SESSION`; owner to confirm the identity model for any future promotion. \ No newline at end of file diff --git a/plan-doc/connector-cache-unification/designs/foundation-design-draft.md b/plan-doc/connector-cache-unification/designs/foundation-design-draft.md new file mode 100644 index 00000000000000..d4d078a7961176 --- /dev/null +++ b/plan-doc/connector-cache-unification/designs/foundation-design-draft.md @@ -0,0 +1,259 @@ +Verification confirms the load-bearing recon claims: `ConnectorStatementScope.computeIfAbsent` lives in fe-connector-api (not fe-core); `ConnectorPartitionViewCache[V]` + `PartitionViewCacheKey(db,table,snapshotId,schemaId)` already exist with the exact contextual-only MetaCacheEntry octet; the "NO consumers yet" javadoc is confirmed stale (iceberg, hive, paimon all wire it — IcebergConnector.java:281,284, HiveConnector.java:134, PaimonConnector.java:158); and the gate hardcodes `TARGET_REL` to IcebergConnector.java with the `FIELD_DECL` + two-marker logic. Design follows. + +--- + +## Summary + +The owner-signed plan is buildable with a strikingly small footprint because **most of the foundation already exists** and the recon overturns one premise of the mandate: + +- **(A) Generic cache wrapper** is a near-trivial extension of an already-shipped, already-consumed class. `ConnectorPartitionViewCache[V]` is byte-for-byte the octet that iceberg's five cross-query caches hand-roll; the five differ only in value type (all can be opaque `[V]`), the MVCC axis (already covered by the 4-tuple key), and the property-namespace entry name (the one axis the current class hardcodes). Generalizing = promote it to `ConnectorMetadataCache[V]` with an `entryName` ctor arg + a neutral key name. Iceberg retrofits with zero behavior/cost change; hudi/mc/es instantiate typed copies. + +- **(B) The D4 seam does NOT need to grow fe-core.** The per-statement memo arena (`ConnectorStatementScope.computeIfAbsent`, `ConnectorSession.getStatementScope()`, backed by `ConnectorStatementScopeImpl` on `StatementContext`) is **already complete and already connector-agnostic, and lives in fe-connector-api** — which iceberg and hudi depend on, and fe-core does **not**. Iceberg's only connector-private piece is a 9-line key-convention wrapper (`IcebergStatementScope.sharedTable`). The minimal D4 seam is a single static convenience helper in fe-connector-api that standardizes that key convention. This means **iron rule A is not actually crossed** — the sanctioned fe-core exception can be declined. This is the #1 residual owner decision. + +- **(C) The gate generalization** is a discovery change, not a new mechanism: replace the hardcoded `TARGET_REL` with a two-stage scan (enumerate `*Connector.java`, keep only files whose `getCapabilities()` *declares* `SUPPORTS_USER_SESSION` via `.add(...)`/`EnumSet.of(...)`, excluding `.contains(...)` guards and comments), then run the existing field+marker logic verbatim on each. Today it resolves to exactly `{iceberg}`, so behavior is unchanged and immediately testable. + +- **(D) Consumption**: hudi (flagship) consumes both B (metaClient + schema memo per statement — the ~6x/~4x redundancy) and A (a `(table, instant)` cross-query partition cache); maxcompute consumes B (collapse the 14–17-site `getTableHandle` fan-out, killing the repeated remote `exists()` probe) — it already has a cross-query partition cache; es consumes B only (per-scan hoist of `EsMetadataState`, split mapping from shard routing), and must **not** get a cross-query cache (iron rule E). + +- **(E) Sequencing** is foundation-first: cache wrapper PR → api seam PR (+ iceberg retrofit, proven by existing tests) → flagship hudi PR (with e2e) → small mc PR → small es PR → gate PR (orthogonal, land any time). + +The design deliberately targets **only the two real cross-query consumers (iceberg + hudi)** plus the two per-statement-only consumers (mc + es). It does not add a `[K]` type parameter, an identity-sharding dimension, or a credential abstraction that no D1 connector needs (Rule 2). + +--- + +## A. Generic cache wrapper (fe-connector-cache) + +### A.1 What is being generalized (grounded) + +The five iceberg cross-query caches (`IcebergTableCache`, `IcebergLatestSnapshotCache`, `IcebergPartitionCache`, `IcebergFormatCache`, `IcebergCommentCache`) each construct the identical contextual-only entry — `new MetaCacheEntry[...]("iceberg-", null, spec, ForkJoinPool.commonPool(), false, true, 0L, true)` — expose `getOrLoad(key, Supplier)` = `entry.get(key, ignored -> loader.get())`, `isEnabled()` = `entry.stats().isEffectiveEnabled()`, and `invalidate`/`invalidateDb`/`invalidateAll` via `invalidateKey`/`invalidateIf`/`invalidateAll`. They differ only in **key type**, **value type**, and the **db-match predicate**. `ConnectorPartitionViewCache[V]` (fe-connector-cache) already captures this exact shape for one case, keyed by `PartitionViewCacheKey(db, table, snapshotId, schemaId)` with `matches`/`matchesDb` predicates on the key — and is already a live consumer in iceberg/hive/paimon. It hardcodes exactly two axes: `ENTRY_PARTITION_VIEW = "partition_view"` (the property namespace + entry name) and the key class. + +The manifest cache is explicitly **excluded** — it is path-keyed, no-TTL, default-off, iceberg-`DataFile`-typed, authz-exempt (read only after a per-user `resolveTable`), and carries a per-scan `ScanStats` stash. It is not a cross-query generalization target. + +### A.2 Generic class shape + +**Layer**: fe-connector-cache, package `org.apache.doris.connector.cache` (the connector-side copy — JDK+Caffeine only, zero fe-core deps; NOT fe-core's `datasource/metacache`). + +Promote `ConnectorPartitionViewCache[V]` to a general `ConnectorMetadataCache[V]`: + +``` +public final class ConnectorMetadataCache[V] { + private final MetaCacheEntry[ConnectorTableKey, V] entry; + + // Framework-native ctor: per-entry namespace meta.cache...(enable|ttl-second|capacity) + public ConnectorMetadataCache(String engine, String entryName, Map[String,String] props) { + CacheSpec spec = CacheSpec.fromProperties( + props == null ? Map.of() : props, engine, entryName, CacheSpec.of(true, 86400L, 1000L)); + this.entry = new MetaCacheEntry[](engine + "." + entryName, null, spec, + ForkJoinPool.commonPool(), false, true, 0L, true); + } + + // Pre-resolved ctor: lets a connector share ONE catalog-level knob across N caches (iceberg back-compat) + public ConnectorMetadataCache(String name, CacheSpec spec) { + this.entry = new MetaCacheEntry[](name, null, spec, + ForkJoinPool.commonPool(), false, true, 0L, true); + } + + public boolean isEnabled() { return entry.stats().isEffectiveEnabled(); } + public V get(ConnectorTableKey key, Supplier[V] loader) { return entry.get(key, ignored -> loader.get()); } + public void invalidateTable(String db, String table) { entry.invalidateIf(k -> k.matches(db, table)); } + public void invalidateDb(String db) { entry.invalidateIf(k -> k.matchesDb(db)); } + public void invalidateAll() { entry.invalidateAll(); } +} +``` + +- **Key**: rename `PartitionViewCacheKey` → **`ConnectorTableKey(db, table, snapshotId, schemaId)`** (the class body is already generic; only its name is partition-flavored). Unused axes pass `-1`. Every one of the five iceberg caches maps onto it: + - table-handle / comment / latest-snapshot: `(db, table, -1, -1)` + - file-format: `(db, table, snapshotId, -1)` + - partition (raw): `(db, table, snapshotId, -1)` + - partition-view (existing): `(db, table, snapshotId, schemaId)` + + This is one key type for all six. Iceberg's caches currently key by `TableIdentifier` / composite `(TableIdentifier, snapshotId)`; retrofit swaps them to `ConnectorTableKey`, functionally identical (a `TableIdentifier` is `db.table`). + +- **Value**: stays **opaque `[V]`**. This is load-bearing: `comment`/`file-format` are `String`, but `table-handle` is a raw `org.apache.iceberg.Table` (credentialed), `latest-snapshot` is a 2-long `CachedSnapshot`, `partition` is `List[IcebergRawPartition]`. A wrapper that owned a concrete value type would be useless to any second connector. Opaque `[V]` serves all. + +- **Config — expose BOTH styles**: the framework-native per-entry ctor (`meta.cache...*`, recommended for hudi/mc/es) **and** the pre-resolved `CacheSpec` ctor so iceberg keeps its single shared knob `meta.cache.iceberg.table.ttl-second` across all five caches (operator back-compat). This also retires the `ttl<=0 -> CACHE_TTL_DISABLE_CACHE` mapping that is copy-pasted five times today (it moves into `CacheSpec.of`/`fromProperties`, already there). + +- Keep the existing `ConnectorPartitionViewCache[V]` as a 3-line thin subclass/factory that calls the new ctor with `entryName="partition_view"`, so its three current consumers (iceberg/hive/paimon) are untouched byte-for-byte. **Fix the stale "NO consumers yet" javadoc** (confirmed false at HEAD). + +### A.3 How iceberg is retrofitted without behavior/cost change + +Delete the five hand-rolled cache classes; replace each field with a typed `ConnectorMetadataCache[V]` built from the **same pre-resolved `CacheSpec`** iceberg already derives from `resolveTableCacheTtlSecond` (default 86400, capacity 1000). Because the octet, the loader wiring, the `isEffectiveEnabled` gate, and the invalidation semantics are identical, cache hits/misses/eviction are unchanged; existing iceberg cache tests (`*ForTest` accessors, `loadCountForTest`) prove invariance. + +**Iron rule D at the fault line**: the wrapper stays value-opaque and **knows nothing about credentials**. The credential policy stays exactly where it is today — in the *connector*, which nulls the field under its gate: +- table-handle: null when `isUserSessionEnabled() || restVendedCredentialsEnabled(props)` (stricter, because the value carries FileIO credentials). +- comment: null unless `restVendedCredentialsEnabled && !isUserSessionEnabled()`. +- format/partition/latest-snapshot: null under `isUserSessionEnabled()`. + +A null field ⇒ `get()` never called ⇒ live bypass (exactly today's behavior). The wrapper never needs a credential flag; the gate script (§C) enforces the marker on the field declaration. + +- **Layer**: fe-connector-cache (connector-side, child-loaded). **Iron-rule impact**: none on A — fe-core source does not grow; the new memo is connector-side (rule A honored); the class holds no property parsing beyond `CacheSpec.fromProperties`, which already lives here (rule C honored). +- **Trino alignment**: matches Trino's "shared low-level toolkit (`io.trino.cache`), per-connector caches, NO single unified cache." **Divergence**: we do not port Trino's per-identity `LoadingCache[user, ...]` sharding — Doris keeps the binary enable/disable (null field) cut, which is correct for D1 (no connector needs identity keying). +- **Consumers**: iceberg (retrofit, 5 caches), hudi (`(table,instant)` partition cache), and — if they ever add a cross-query cache — mc/es. Not needed by es this round (rule E). + +--- + +## B. Minimal fe-core statement-scope seam + +### B.1 The premise correction + +D4 was framed as extracting a **fe-core** seam that crosses iron rule A. Recon (unanimous across agents, verified above) shows the substrate is **already built and already in fe-connector-api**: + +- `ConnectorStatementScope.computeIfAbsent(String key, Supplier[T])` — the whole memo primitive, opaque `Object` values under string keys; `NONE` runs the loader every call (fe-connector-api). +- `ConnectorSession.getStatementScope()` — neutral access point, defaults to `NONE` (fe-connector-api). +- `ConnectorStatementScopeImpl` (ConcurrentHashMap, two-pass idempotent `closeAll`) hung on `StatementContext` with per-execution reset — **already in fe-core, needs zero change**. +- Iceberg's only connector-private code is `IcebergStatementScope.sharedTable(session, db, table, Supplier[Table])`: null-session ⇒ load; else key = `"iceberg.table:" + catalogId + ":" + db + ":" + table + ":" + queryId` ⇒ `computeIfAbsent`. +- Iceberg and hudi both depend on **fe-connector-api**, and on **neither fe-core nor fe-connector-cache** (a literal fe-core class would be unreachable by the retrofit; fe-connector-cache has no dep on ConnectorSession). + +### B.2 The seam: one static helper in fe-connector-api + +**Layer**: fe-connector-api, beside `ConnectorStatementScope` (connector-side SPI module). + +``` +public final class ConnectorStatementScopes { + private ConnectorStatementScopes() {} + + /** Resolve db.table once per statement and share the one object across every resolver. + * keyNamespace namespaces the value TYPE so a heterogeneous gateway statement touching + * two connectors cannot collide on the same (db,table) and hit a ClassCastException. */ + public static [T] T resolveInStatement(ConnectorSession session, String keyNamespace, + String db, String table, Supplier[T] loader) { + if (session == null) { + return loader.get(); // == ConnectorStatementScope.NONE + } + String key = keyNamespace + ":" + session.getCatalogId() + ":" + db + ":" + table + + ":" + session.getQueryId(); + return session.getStatementScope().computeIfAbsent(key, loader); + } +} +``` + +This adds **no new interface, no new type on the SPI, no fe-core source** — it reuses the existing `computeIfAbsent` primitive and standardizes the key convention iceberg hand-rolls. `keyNamespace` is a **required** parameter (not a hardcoded prefix): it both reproduces iceberg's exact existing key and prevents cross-connector value-type collisions in a gateway statement. + +### B.3 Retrofit + consumption + +- **Iceberg (retrofit)**: `IcebergStatementScope.sharedTable` collapses to `return ConnectorStatementScopes.resolveInStatement(session, "iceberg.table", db, table, loader);` — byte-identical key (namespace `"iceberg.table"` reproduces the `"iceberg.table:"` prefix), so the 4 funnel sites (`resolveTableForRead:646`, scan `:2360`, write `:704`, txn `:212`) keep identical hits/misses and `NONE` fallthrough. `rewritableDeleteSupply` stays iceberg-private (it is a `(catalogId, queryId)`-keyed scan→write delete bridge, not a table resolution, and is **out of the seam**). +- **Hudi (consumes)**: wrap the metaClient build — `HoodieTableMetaClient mc = ConnectorStatementScopes.resolveInStatement(session, "hudi.metaclient", db, table, () -> buildMetaClient(...));` — collapsing the ~6 builds/query to 1. Hudi has no per-statement memo today (two independent `HoodieTableMetaClient.builder()` sites), so this is a clean new consumer. Derived state (`TableSchemaResolver`/Avro/`InternalSchema`) is then computed once off the single memoized metaClient, killing the ~4x re-parse. Hudi needs **no new module dependency** for the seam (already depends on fe-connector-api). + +### B.4 The sanctioned fe-core exception (only if the owner insists / for the mc fan-out) + +The maxcompute redundancy is a *fe-core artifact*: `PluginDrivenExternalTable.resolveConnectorTableHandle` (fe-core:116) calls `metadata.getTableHandle` fresh at 14–17 sites. Two placements collapse it: + +- **B1 (recommended, fe-core flat)**: memoize *inside* the connector's `getTableHandle` via `resolveInStatement(session, "maxcompute.handle", db, table, ...)`. All 14–17 fe-core calls then hit the connector's per-statement memo; the remote `exists()`+`get` fires once. fe-core untouched. +- **B2 (sanctioned D4 exception, generic)**: wrap the one fe-core call site in the *existing* primitive — `session.getStatementScope().computeIfAbsent("tablehandle:" + catalogId + ":" + db + ":" + remoteName + ":" + queryId, () -> metadata.getTableHandle(...))`. This is the **smallest possible reading** of the sanctioned exception: **no new fe-core type, no new SPI method** — just a memoized call site reusing `computeIfAbsent`, benefiting every connector (including non-migrated ones) uniformly. fe-core source grows by ~3 lines in an existing method. + +**Recommendation**: use **B1** for the two named seam consumers (iceberg + hudi resolve raw `Table`/metaClient objects connector-side). Use **B2** for maxcompute if the owner wants the generic fan-out collapsed without touching mc's `getTableHandle` — mc is the connector that most justifies the sanctioned exception, and even then it costs zero new SPI surface. + +- **Iron-rule impact**: B1 = none (rule A intact). B2 = the sanctioned minimal exception, ~3 lines, reusing an existing primitive. +- **Trino alignment**: this recreates, at *statement* granularity, Trino's per-*transaction* "load each table once / read+write share one metadata." **Divergence**: Doris's unit is the statement (scope keyed by queryId, reset per prepared EXECUTE), so nothing may assume cross-statement reuse. +- **Rule D safety**: the L1 memo is single-identity by construction — a statement resolves one catalog under one identity (`PluginDrivenMetadata` fail-loud pin), scope is per-statement + GC'd at end. So the memo never crosses users even when L2 (the §A cross-query cache) is disabled for a vended catalog. **Lifecycle caveat for hudi**: `ConnectorStatementScopeImpl.closeAll` closes any stored `AutoCloseable` at statement end; if a hudi version's `HoodieTableMetaClient` (or a wrapper) is `AutoCloseable`/holds a `HoodieStorage`/`FileSystem`, verify auto-close-at-statement-end is desired, else memoize a non-closeable projection. +- **Consumers**: iceberg (retrofit), hudi (metaClient), maxcompute (handle, B1 or B2), es (metadata state, §D). + +--- + +## C. Authz gate generalization + +**Layer**: `tools/check-authz-cache-sharding.sh` + its self-test — pure tooling, touches no product source (rule A trivially honored). Wired at fe-connector reactor `validate` phase with `${project.basedir}` as root (already reaches all connector modules; no pom change). + +### C.1 The change: discovery replaces the hardcoded target + +Replace `TARGET_REL='fe-connector-iceberg/.../IcebergConnector.java'` with a three-stage scan: + +- **Stage 1 — enumerate**: `find "${ROOT}" -path '*/src/main/java/*' -name '*Connector.java'` (excluding `target/`). +- **Stage 2 — filter to declarers**: keep a file only if it **declares** the capability, matching the declaration form and excluding references/comments: + - INCLUDE lines matching `\.add\([^)]*ConnectorCapability\.SUPPORTS_USER_SESSION` OR `(EnumSet|Set|ImmutableSet)\.of\([^)]*SUPPORTS_USER_SESSION` + - after stripping comment lines (`^\s*\*`, `^\s*//`) and **excluding** any line containing `.contains(`. + + This is the single load-bearing distinction: it admits `IcebergConnector.java:860` (`capabilities.add(...SUPPORTS_USER_SESSION)`) and rejects `HiveConnector.java:543` (`sibling.getCapabilities().contains(...SUPPORTS_USER_SESSION)` — a fail-loud guard, not a declaration) and the fe-connector-api javadoc mentions. A static grep does **not** try to evaluate iceberg's `if (isUserSessionEnabled())` runtime guard — any source-level `.add` means the connector *can* carry the capability, so its caches are in scope unconditionally (correct conservative altitude). +- **Stage 3 — classify verbatim**: run the **existing** `FIELD_DECL='final ([A-Za-z_][A-Za-z0-9_]*)?Cache[ <]'` + two-marker (`authz-cache-session-user-disabled` / `authz-cache-exempt`, on-line-or-line-above) logic on each declaring file. No change to marker vocabulary or RED/GREEN semantics — iceberg keeps passing unchanged, and the existing self-test stays valid. + +### C.2 Invariants and guards + +- **Scan the OWNER file only** (`*Connector.java`), never the whole module: the constructor-injected reference fields in `*ConnectorMetadata`/`*ScanPlanProvider` are unmarked *by design*; a whole-module scan would false-positive on all of them. State this as an explicit invariant. +- **Anti-no-op floor**: if Stage 2 yields **zero** declarers, `exit` non-zero (mirroring today's `exit 2` for missing target). Without this, a capability rename or grep drift silently turns the gate into pass-everything — the highest-risk failure of any discovery gate. +- **Self-test additions**: (a) a fixture connector with a `.contains(SUPPORTS_USER_SESSION)` guard + comment + an unmarked cache field MUST NOT be scanned (proves REFERENCE excluded); (b) a second fixture that DECLARES via `.add`/`EnumSet.of` with an unmarked cache MUST fail (proves multi-connector discovery). Keep the iceberg fixture as the primary declarer. + +### C.3 Scope reality + +Today only iceberg declares the capability; hudi/mc/es have no `getCapabilities` override and vend static (not per-user) credentials, so their new caches are authz-safe **without** the gate and correctly fall outside it (rule D). D3 is therefore **forward-insurance**: it changes behavior for exactly one connector now (no-op vs iceberg) but auto-enrolls any future session=user adopter before an unguarded cross-query cache can leak. It is **orthogonal** to D1/D2 and does not block them. Documented residual: whether the hive-gateway path (a sibling could declare the capability, guarded at `HiveConnector.java:543`) is in-gate-scope or explicitly out (the front door never declares it). + +Note: the fe-core defense layer (`PluginDrivenExternalCatalog.shouldBypass{TableName,DbName,Schema}Cache`, keyed off `getCapabilities().contains(SUPPORTS_USER_SESSION)` at runtime) is **already** connector-agnostic and needs no change — it auto-covers any future declarer. The gate only closes the connector-side hole where a new `*Cache` field is added and forgotten. + +--- + +## D. Connector consumption (hudi / maxcompute / es) + +### D.1 Hudi (flagship) — consumes B (primary) + A (secondary) + +- **metaClient + schema memo (B)**: route both `HoodieTableMetaClient.builder()` sites through `resolveInStatement(session, "hudi.metaclient", db, table, () -> buildMetaClient(...))`. Collapses ~6 builds → 1/statement and, since `TableSchemaResolver`/Avro/`InternalSchema` are derived off the single memoized client, ~4 schema re-parses → 1. This is the flagship win, and the **per-statement** tier is the correct one: `buildMetaClient` reloads the active timeline for freshness, so a cross-query cache of the raw metaClient would be wrong; the per-statement memo is exactly right. +- **Wrap `CachingHmsClient`**: hudi is HMS-backed (depends on fe-connector-hms/metastore-hms). Its HMS access should sit behind the shared `CachingHmsClient` (Trino `CachingHiveMetastore` shape, coarse REFRESH+TTL, TCCL-pinned per the memory notes on `ThriftHmsClient.doAs`) exactly as iceberg-on-HMS and hive do — so metastore round-trips (`getTable`, listing) are shared cross-query at the HMS layer, beneath the per-statement metaClient memo. +- **`(table, instant)` partition cache (A)**: a `ConnectorMetadataCache[List[HudiPartition]]` with `entryName="partition"`, key `ConnectorTableKey(db, table, instant-as-snapshotId, -1)`. The instant (commit time) is hudi's MVCC coordinate, so pinning it into the key makes the cache always-correct (rule E: new instant ⇒ new key; plus bounded TTL + REFRESH invalidation). This is the secondary, TTL-bounded win — build it after the per-statement memo. +- **per-scan hoist**: the two scan-planning sites reuse the single memoized metaClient/schema via B. +- **Pom**: add `fe-connector-cache` dependency (currently absent). **Verify** hudi's assembly/shade bundles **Caffeine ≥ 2.9.3** — hudi does not receive it transitively the way iceberg does via iceberg-core; otherwise the child-loaded `MetaCacheEntry` `NoClassDefFound`s at runtime though it compiles (per the framework-coherence memory note). +- **Iron rules**: A honored (all new memo/caches connector-side); D honored (hudi vends no per-user credentials, no `SUPPORTS_USER_SESSION`, so the `(table,instant)` name-keyed cache needs no per-user disabling); E honored (instant in key + TTL). **Trino**: mirrors Trino keeping the fresh per-transaction view (timeline) distinct from the long-lived metastore cache. + +### D.2 Maxcompute — consumes B only + +- **per-metadata handle memo (B)**: the seam target is `resolveConnectorTableHandle`'s 14–17-site fan-out (recon corrects the report's "10"). Use **B1** (wrap `MaxComputeConnectorMetadata.getTableHandle` in `resolveInStatement(session, "maxcompute.handle", db, table, ...)`) or the sanctioned **B2** (memoize the fe-core call site). Either collapses to one `getTableHandle`/statement, which **drops the redundant remote `tables().exists()` HEAD probe + lazy `tables().get()`** to once. Within one resolved handle the ODPS metadata already amortizes (`odpsTable.getSchema()` reused by schema/columns/scan); the fan-out is the only leak. +- **No new cross-query cache**: maxcompute already owns a cross-query `MaxComputePartitionCache` keyed `(db, table)` (owned in the Connector, so the §C owner-file scan covers it). It uses a static AK/SK identity and does not declare `SUPPORTS_USER_SESSION` — so it is rule-D-compliant as-is and needs no §A wrapper this round. +- **Verify**: ODPS uses a static (not per-user) identity, so the resolved handle is safe to share per-statement (and would even be cross-query-safe) — confirm before treating it as shareable (rule D). Confirm `getTableHandle` has `ConnectorSession` access for the B1 wrap. +- **Iron rules**: A honored (B1) or sanctioned-minimal (B2); D honored. **Trino**: same "resolve once per unit" as iceberg. + +### D.3 Es — consumes B only, must NOT get a cross-query cache + +- **per-scan hoist of `EsMetadataState` (B)**: `fetchMetadataState` fires 2x/query (planScan + buildScanNodeProperties). Memoize the **mapping/schema half** per statement via `resolveInStatement(session, "es.metadata_state", db, table, ...)` → 1 fetch. +- **mapping/field-context carrier**: `getTableSchema` currently discards the raw mapping string that `resolveFieldContext` later needs (`EsConnectorMetadata.java:85-93`), forcing ~4 remote `getMapping` calls/query. Attach the raw mapping to a **statement-scoped `EsMetadataState`** (or the `EsTableHandle`), not to the generic `ConnectorTableSchema` (which must stay connector-agnostic — rule B). `resolveFieldContext` then reuses it, collapsing ~4 `getMapping` → 1. +- **CRITICAL split (rule E)**: `EsMetadataFetcher.fetch()` currently bundles `fetchMapping()` (reuse-safe) with `fetchShards()` (live shard routing). The reuse layer may memoize the **mapping** within a statement but **shard routing must stay per-statement fresh and must NEVER be a cross-query name-keyed cache**. So the memoized `EsMetadataState` carries only mapping/field-context; `fetchShards` is always re-run. Es gets **no §A cross-query cache**. +- **Iron rules**: A honored (state connector-side); B honored (raw mapping rides the handle/state, not the generic schema); E honored (shard routing stays per-statement). **Trino**: mirrors Trino keeping `CachingDirectoryLister`/shard state as a fresh per-transaction layer distinct from the metadata cache. + +--- + +## E. Sequencing + +Foundation first, then flagship, then the two small PRs, then the gate. Each PR is independently landable and independently verified. + +1. **PR-1 — Generic cache wrapper (fe-connector-cache).** Add `ConnectorMetadataCache[V]` (both ctors); rename `PartitionViewCacheKey` → `ConnectorTableKey` (mechanical, updates iceberg/hive/paimon imports); make `ConnectorPartitionViewCache[V]` a thin `entryName="partition_view"` specialization; **fix the stale javadoc**. Pure addition + rename; no behavior change. Verify: reactor test-compile + existing partition-view tests. + +2. **PR-2 — Statement-scope seam (fe-connector-api) + iceberg retrofit.** Add `ConnectorStatementScopes.resolveInStatement`; collapse `IcebergStatementScope.sharedTable` to delegate (byte-identical key). Verify: iceberg's existing per-statement memo tests + full iceberg suite prove cost-invariance. **This is where the owner's D4 decision is exercised** — land it as the fe-connector-api helper (rule A intact); only if the owner insists on the literal fe-core exception, add the B2 memoized call site as a separate, clearly-flagged commit. + +3. **PR-3 (optional, foundation-first) — Iceberg cache convergence.** Retrofit iceberg's five hand-rolled caches onto `ConnectorMetadataCache[V]` using the pre-resolved-`CacheSpec` ctor (shared `table.ttl-second` knob), keeping the connector-side credential null-gates. Guarded by existing iceberg cache tests + `loadCountForTest`. **Residual**: do this now (max convergence, foundation-first mandate) vs defer (narrower diff, avoids re-touching audited caches). + +4. **PR-4 — Flagship hudi.** Pom dep on fe-connector-cache + Caffeine-bundling verification; wire metaClient + schema memo via the seam (B); ensure HMS access sits behind `CachingHmsClient`; add the `(table, instant)` cross-query partition cache via A; per-scan hoist. **E2E regression required** (heterogeneous + standalone hudi-on-HMS, per the memory rule that new HMS-delegated capabilities need e2e). Largest PR. + +5. **PR-5 — Maxcompute (small).** Memoize `getTableHandle` per statement (B1) — or the fe-core B2 site if chosen in PR-2; drop the redundant `exists()` probe. Verify: query-count/remote-op assertions on a SELECT. + +6. **PR-6 — Es (small).** Per-scan hoist `EsMetadataState` via B; attach raw mapping to the statement-scoped state; split `fetch()` so shard routing stays per-statement. Verify: `getMapping` call-count drops to 1; shard routing still re-runs per statement. + +7. **PR-7 — Gate generalization (D3).** Rewrite `check-authz-cache-sharding.sh` to two-stage discovery + anti-no-op floor; extend the self-test fixtures. Orthogonal — can land any time, but recommended **after PR-1/PR-2** so the marker contract and `*Cache` type convention are stable, and **before** hudi/mc/es would ever declare session=user. + +PR-1 and PR-2 are the foundation and unblock everything. PR-4/5/6 depend on both. PR-7 is independent. + +--- + +## Risks + +- **Iceberg retrofit re-touches audited caches.** Converging the five caches (PR-3) widens the diff over previously-audited code. Mitigation: the octet + `CacheSpec` + invalidation are provably identical; gate on existing `*ForTest`/`loadCountForTest` assertions; keep the shared-knob ctor so operator-facing config is byte-identical. If the owner prefers a narrower blast radius, PR-3 can be deferred and iceberg's five caches left as-is (they already work), with `ConnectorMetadataCache` used only by new consumers — but that forgoes the foundation-first convergence the mandate asks for. +- **Hudi Caffeine bundling.** Compiles but `NoClassDefFound`s at runtime if the hudi assembly doesn't self-carry Caffeine ≥ 2.9.3. Mitigation: verify hudi's shade/assembly config (not just ``) and add an explicit Caffeine dep if not transitively present; smoke-test with a redeploy. +- **Hudi metaClient lifecycle in the statement scope.** If a hudi version's `HoodieTableMetaClient` (or a memoized wrapper) is `AutoCloseable`/holds a `FileSystem`/`HoodieStorage`, `closeAll` auto-closes it at statement end — desirable only if nothing outside the statement retains it. Mitigation: verify per hudi version; if not, memoize a non-closeable projection, not the client. +- **Gate discovery drift → silent no-op.** A capability rename or grep-form change could empty the declarer set. Mitigation: the anti-no-op floor (`exit` non-zero on zero declarers) + the two new self-test fixtures. +- **Es shard-routing regression.** Any reuse layer that accidentally memoizes `fetchShards` output cross-query (or even reuses stale routing across a statement in a way that misroutes) breaks correctness. Mitigation: the memoized `EsMetadataState` must, by construction, hold only mapping/field-context; keep `fetchShards` a separate always-fresh call; assert routing re-runs. +- **Cross-connector key collision in a gateway statement.** Two connectors memoizing the same `(db, table)` under the same namespace would collide on the opaque `Object` and `ClassCastException`. Mitigation: `keyNamespace` is a required, connector-distinct parameter in `resolveInStatement` (and the §A cache is per-connector-instance, so no cross-connector sharing there). +- **Credential leak via a future generic cache.** If a future connector caches a credentialed value under only the session=user gate, the name-keyed cache could leak (list≠load). Mitigation: the wrapper stays value-opaque and the credential gate stays in the connector (null the field); the §C gate enforces the marker on every `*Cache` field. Residual: whether to harden this from convention to a type/marker contract (see below). + +--- + +## Residual owner decisions + +1. **D4 placement (highest priority).** Recon shows the reachable home for the seam is **fe-connector-api**, not fe-core — iceberg/hudi cannot depend on fe-core, and the memo primitive already exists there. Do you accept re-reading "fe-core seam" as "fe-connector-api helper" (`ConnectorStatementScopes.resolveInStatement`), leaving **fe-core source flat and iron rule A untouched**? If you still want the literal fe-core exception, its minimal form is the B2 memoized call site in `resolveConnectorTableHandle` — **zero new fe-core type, ~3 lines reusing the existing `computeIfAbsent`** — used chiefly to collapse maxcompute's fan-out generically. (Recommendation: fe-connector-api helper for iceberg+hudi; B2 only if you want mc's fan-out collapsed without touching the mc connector.) + +2. **Iceberg cache convergence now vs later (PR-3).** Retrofit all five iceberg caches onto `ConnectorMetadataCache` this round (foundation-first, max convergence, wider diff over audited code) — or add the generic class for new consumers only and leave iceberg's five as-is (narrower, less convergence)? + +3. **Config namespace for hudi/mc/es caches.** Framework-native per-entry knobs (`meta.cache...*`, recommended) or a single shared per-catalog knob like iceberg's `table.ttl-second`? Affects operator surface and any back-compat mapping (`CacheSpec.applyCompatibilityMap`) for existing mc/es TTL properties. + +4. **Comment cache as a first-class entry?** Iceberg builds `commentCache` only for vended catalogs because plain catalogs serve the comment path from the table-handle cache. Do hudi/mc/es have a table-handle cache to piggyback, or does each need a standalone comment path? Decides whether the wrapper models "comment" as a first-class entry or leaves it per-connector. (Given D1 scope, likely leave per-connector — none of hudi/mc/es requested a comment cache.) + +5. **Latest-snapshot / MVCC-pin shape.** Iceberg's pin is `(snapshotId, schemaId)`, paimon's is a single long, hudi's is an instant, mc/es have no snapshot axis. Keep the pin as opaque `[V]` over the 4-tuple key (recommended — one key, `-1` sentinels) rather than a fixed 2-axis value type? + +6. **Credential-value hardening: contract vs convention.** Should the wrapper *refuse* credentialed values by a type/marker constraint, or keep the current convention (connector nulls the field under its gate + the §C marker enforces it)? Convention is the smaller design and matches today; a contract is insurance against a future connector caching a credentialed value under only the session=user gate. + +7. **Gate strictness on empty declarer set.** Hard build FAIL (recommended) vs warning when Stage 2 finds zero declarers? And is the hive-gateway sibling path (guarded at `HiveConnector.java:543`) in-gate-scope or explicitly documented as out? + +8. **maxcompute cross-query shareability.** Confirm ODPS uses a static (non-per-user) identity so a resolved handle is safe beyond a single statement — this decides whether mc's handle memo may be statement-scoped only (safe default) or could later be promoted cross-query (rule D). \ No newline at end of file diff --git a/plan-doc/connector-cache-unification/designs/review-1.md b/plan-doc/connector-cache-unification/designs/review-1.md new file mode 100644 index 00000000000000..e819e400f59420 --- /dev/null +++ b/plan-doc/connector-cache-unification/designs/review-1.md @@ -0,0 +1,103 @@ +I have enough grounding. Writing the adversarial review. + +--- + +# Adversarial design review — catalog-SPI cache unification (foundation-first) + +Lens: behavior parity + Trino portability. I verified the load-bearing recon claims against HEAD; they largely hold (`ConnectorStatementScope.computeIfAbsent` is in fe-connector-api `ConnectorStatementScope.java:45`; `getStatementScope()` defaults to `NONE` at `ConnectorSession.java:142`; `ConnectorPartitionViewCache`/`PartitionViewCacheKey` exist with the 4-tuple and the stale "NO consumers yet" javadoc at `ConnectorPartitionViewCache.java:34`; iceberg wires it at `IcebergConnector.java:281,283`; only `IcebergConnector.java:860` declares `SUPPORTS_USER_SESSION` via `.add`, `HiveConnector.java:543` is a `.contains` guard; the gate hardcodes `TARGET_REL` at `check-authz-cache-sharding.sh:57`). The design is buildable, but several parity/portability claims are overstated. Objections most-severe-first. + +--- + +## 1. [blocker — for the B2 variant] The sanctioned fe-core exception (B2) memoizes `getTableHandle` for EVERY connector, not just maxcompute + +**Issue.** §B.4-B2 and PR-5 wrap the *generic* fe-core funnel `PluginDrivenExternalTable.resolveConnectorTableHandle` (fe-core) in `computeIfAbsent`. That funnel is the single site all connectors flow through. Memoizing it changes the call-count contract of `getTableHandle` for jdbc/paimon/trino/hive/iceberg — every connector outside this round's D1 scope and outside its test coverage. + +**Reasoning.** The design frames "benefits every connector uniformly" (§B.4-B2) as an advantage, but that is precisely the parity hazard: today `resolveConnectorTableHandle` is called fresh at 14–17 sites, and any connector that relies on that — a remote `exists()` probe used as a liveness/authorization check, a freshness re-read, a side-effecting handle build — silently loses those repeats. You cannot assert parity for connectors you are not testing this round. This also sits at the exact fault line of iron rule A (fe-core grows) with an unbounded blast radius, which is the opposite of "minimal sanctioned exception." + +**Fix.** Drop B2. Use **B1 (connector-local memo inside `MaxComputeConnectorMetadata.getTableHandle`)** for maxcompute too — it collapses the same fan-out with zero fe-core growth and zero cross-connector reach. If the owner insists on a fe-core site, scope it so only the maxcompute path is memoized (e.g. gate on the connector already opting in), never the shared funnel unconditionally. The design's own recommendation already leans B1; make it the only option. + +--- + +## 2. [major] Iceberg retrofit is NOT byte-identical: it changes db-invalidation from iceberg `Namespace` equality to plain `String` equality + +**Issue.** §A.2/§A.3 claim the retrofit onto `ConnectorTableKey(db,table,…)` is "functionally identical (a `TableIdentifier` is `db.table`)." But four of the five caches key by `TableIdentifier` and invalidate a db via **iceberg Namespace equality**, not string equality: +- `IcebergTableCache.java:102-103`: `Namespace ns = Namespace.of(dbName); entry.invalidateIf(id -> id.namespace().equals(ns))` +- same in `IcebergLatestSnapshotCache.java:108-110`, `IcebergPartitionCache.java:121-123`, and format cache. + +`ConnectorTableKey`/`PartitionViewCacheKey` invalidate via `Objects.equals(this.db, db)` (`PartitionViewCacheKey.java:71-73`). + +**Reasoning.** For single-level namespaces these coincide, but the retrofit relocates db extraction from "iceberg's `Namespace` object at invalidation time" to "a `String db` frozen into the key at build time." That diverges for any multi-level namespace, and it moves a piece of iceberg-specific identity logic into the generic key construction — exactly the kind of silent semantic drift on already-audited, already-shipped code the mandate says to avoid. "Same cache keys, same call counts" is not established for the invalidateDb path. + +**Fix.** Before PR-3, add an explicit parity test that `invalidateDb` on the retrofitted key evicts exactly the same entries as the `Namespace.equals` path (including a multi-level-namespace case, or an assertion that iceberg-on-Doris namespaces are provably always single-level). Keep the db-extraction in the connector's key builder, not in the generic class. Treat PR-3 (§ residual #2) as deferrable if this parity can't be cheaply proven. + +--- + +## 3. [major] The generalized gate silently protects only caches placed on `*Connector.java`; nothing enforces that a `SUPPORTS_USER_SESSION` connector puts its cross-query caches there + +**Issue.** §C.2 keeps "scan the OWNER file only (`*Connector.java`)" and treats constructor-injected `*Cache` fields on `*ConnectorMetadata`/`*ScanPlanProvider` as "unmarked by design." Generalizing Stage-1/2 to "any declarer" makes this file-scoping a load-bearing, N-connector invariant with zero enforcement. + +**Reasoning.** Today it is safe only by luck: iceberg happens to hold all five caches on `IcebergConnector` (`IcebergConnector.java:169-200`). A future (or even this round's hudi) session=user adopter that holds its `(table,instant)` cache on `HudiConnectorMetadata` or a scan provider would declare the capability, pass the gate (the declaring `*Connector.java` has no unmarked `*Cache` field), and leak. The gate's whole purpose — "added a new cross-query cache and forgot to isolate it becomes a build failure" — is defeated for any cache not physically on the Connector class. + +**Fix.** When a connector is in the declarer set, scan its whole module for `*Cache` **holder** fields (final `…Cache` type), not just `*Connector.java`; OR add a companion invariant check that a declaring connector may hold authz-sensitive `*Cache` fields only on the Connector class and fail otherwise. The §C.2 "false-positive on injected reference fields" concern is real, but the fix is to distinguish holder (`new …Cache(`) from injected reference (ctor param) — not to narrow the scan to one file. + +--- + +## 4. [major] The statement-scope seam's correctness rests entirely on `queryId` lifecycle, which the design extends to three new connectors without verifying it + +**Issue.** `resolveInStatement` (and iceberg's existing `IcebergStatementScope.sharedTable`, `IcebergStatementScope.java:64`) key by `session.getQueryId()`. The whole cross-statement-isolation and prepared-EXECUTE-reuse safety claim (§B.2, §B "Rule D safety," `ConnectorStatementScopeImpl.java:64-98` idempotent `closeAll`) depends on: (a) `queryId` is stable within one statement across the request thread and all off-thread scan pumps, and (b) `queryId` differs per prepared `EXECUTE` re-execution. + +**Reasoning.** Iceberg already depends on this, so it is pre-existing for iceberg — but the design generalizes the dependency to hudi/mc/es and to a shared helper, and asserts "reset per prepared EXECUTE" (§B.3, "reused prepared context sees each execution's own table") as fact without grounding it. If `queryId` is NOT refreshed per EXECUTE while the `StatementContext`/scope IS reset, the seam is safe (scope cleared); if the scope is reused but `queryId` is refreshed, also safe; but if BOTH are reused (scope + queryId), you get cross-execution stale memo — a correctness bug. This is unverified and it is the foundation everything else builds on. Residual #8 asks about mc identity but nobody verified the queryId/scope reset interaction. + +**Fix.** Before PR-2, verify against `ExecuteCommand`/`StatementContext` that either the scope is reset OR `queryId` changes on every prepared re-execution (ideally both), and encode it as a test: two EXECUTEs of one prepared statement must NOT share a memoized table. Don't ship the shared helper on an assumed lifecycle. + +--- + +## 5. [major] Hudi is the first `AutoCloseable` connector value in the scope; `closeAll` use-after-close is a hard requirement, not a "caveat" + +**Issue.** §B.3/§D.1 route `HoodieTableMetaClient` (and derived `TableSchemaResolver`/`InternalSchema`) through the scope. `ConnectorStatementScopeImpl.closeAll` (`:84-96`) closes every stored `AutoCloseable` at statement end. Until now the only closeable scope value was `ConnectorMetadata`; the shared iceberg `Table` is not closeable, so this path was never exercised for a connector data object. + +**Reasoning.** If any hudi version's `HoodieTableMetaClient` is `AutoCloseable` or holds a `HoodieStorage`/`FileSystem`, and any consumer (BE-thrift schema assembly, an off-thread scan pump, a derived schema object that lazily touches the FS) retains it past the point `closeAll` fires, you get use-after-close on the flagship connector. The impl comment claims `closeAll` "runs after the scan off-thread pumps have quiesced," but that ordering was validated for iceberg's non-closeable `Table`, not for a closeable metaClient whose derived objects may outlive the store slot. The design lists this as risk #3 with mitigation "memoize a non-closeable projection" — but leaves it optional. + +**Fix.** Make it mandatory for the flagship: memoize a **non-closeable projection** (the resolved schema/timeline snapshot the callers actually need), never the raw closeable client, unless close-at-statement-end is provably safe for that hudi version AND no derived object escapes. Add a test that exercises a scan pump reading the memoized hudi state after nominal statement end. + +--- + +## 6. [major] Hudi's secondary `(table, instant)` cross-query cache (A) conflicts with the freshness claim — it delivers either ~zero cross-query benefit or staleness + +**Issue.** §D.1 says `buildMetaClient` "reloads the active timeline for freshness" (so the per-statement metaClient memo is correct), then also proposes a cross-query `(table, instant)` partition cache keyed by `instant-as-snapshotId`. + +**Reasoning.** These pull opposite directions. If the latest instant is resolved fresh each statement (the stated freshness goal), then each statement's `(table,instant)` key is a fresh instant with near-zero cross-query hit rate — the cache buys almost nothing. If instead the latest-instant resolution is itself cached cross-query (à la iceberg `latestSnapshotCache`) to get hits, you reintroduce the TTL staleness window the freshness claim was avoiding, and rule E's "new instant ⇒ new key" is vacuous because you're pinning a stale instant. The design asserts both benefits without resolving the tension. + +**Fix.** Decide explicitly: either (a) hudi latest-instant is cross-query cached under bounded TTL+REFRESH (accept iceberg-equivalent staleness, real hit rate) — then document the staleness parity; or (b) latest-instant is per-statement fresh — then drop the (A) cross-query cache for hudi this round (per-statement memo already captures the win) rather than shipping a cache that can't hit. Don't claim both. + +--- + +## 7. [minor] Cross-connector `keyNamespace` collision is a runtime `ClassCastException`, mitigated only by hand-assigned strings with no registry + +**Issue.** §B.2 makes `keyNamespace` a required param and §Risks notes the collision, but the namespaces are free-form strings (`"iceberg.table"`, `"hudi.metaclient"`, `"maxcompute.handle"`, `"es.metadata_state"`) stored as opaque `Object` in one `ConcurrentHashMap` (`ConnectorStatementScopeImpl.java:44`). A duplicate namespace across two connectors in a heterogeneous gateway statement is caught only at runtime as a `ClassCastException`. + +**Reasoning.** The metadata-funnel already treats its key convention (`"metadata:" + catalogId` plus owner label, `ConnectorStatementScope.java:51`) as a reviewed invariant. The new seam adds a parallel, larger key space with no central definition — easy to collide under refactor, invisible until a gateway query hits it. + +**Fix.** Put the namespaces in one place (a small constants holder / enum next to `ConnectorStatementScopes` in fe-connector-api) and document uniqueness as a reviewed invariant, mirroring the funnel-key convention. Optionally fold `catalogId` before `keyNamespace` so two connectors under different catalog ids can't collide even with an equal namespace. + +--- + +## 8. [minor] `ConnectorMetadataCache` telemetry/cache names change unless the retrofit is pinned to the old names + +**Issue.** Each iceberg cache today has a distinct entry name (`"iceberg-table"`, `"iceberg-partition"`, `"iceberg-latest-snapshot"`, … at `IcebergTableCache.java:70`, `IcebergPartitionCache.java:97`, etc.), surfaced through `MetaCacheEntry.stats()`. The framework-native ctor derives `name = engine + "." + entryName` (§A.2) → `"iceberg.table"`, a different string. + +**Reasoning.** Any operator dashboard, log grep, or `*ForTest`/`loadCountForTest` assertion keyed on the old names breaks silently. The design says iceberg uses the pre-resolved `CacheSpec` ctor (which takes `name`), so it *can* keep old names — but that must be stated as a retrofit requirement, not left to the framework-native derivation. + +**Fix.** Pin the retrofit to pass the exact legacy `name` strings; add it to the PR-3 checklist and assert names in the existing cache tests so a drift fails. + +--- + +## 9. [nit] "Iron rule A is not actually crossed" is true only if the owner accepts re-reading the mandate; the fe-core B2 form still exists in the doc as a live recommendation + +Residual #1 correctly surfaces that the reachable home is fe-connector-api (iceberg/hudi can't depend on fe-core, confirmed: they import `org.apache.doris.connector.api.*`). But the doc simultaneously recommends B2 (fe-core growth) for maxcompute. Pick one posture: if the design's thesis is "rule A stays intact," then B2 should be removed, not recommended — see objection #1. Leaving both makes the "rule A untouched" headline conditional on a decision the doc elsewhere pre-empts. + +--- + +## Overall verdict + +**Request changes.** The foundation genuinely mostly exists and the direction is sound, but the design overstates parity in three shipped-code-touching places: the fe-core B2 memo (blocker for that variant — unbounded cross-connector blast radius, use B1), the iceberg db-invalidation semantics change (major — not byte-identical, `Namespace` vs `String` equality), and the gate's unenforced "caches live on `*Connector`" invariant (major — defeats the gate's purpose for any cache held off the Connector class). Two more majors are unverified-foundation risks (queryId/prepared-EXECUTE lifecycle; hudi `AutoCloseable` use-after-close) and one is an internal contradiction (hudi freshness vs the (A) cross-query cache). None are unfixable; all are cheap to close before the respective PRs, and PR-1 (generic wrapper) plus a lifecycle-verified PR-2 can proceed once objections #4 and the #2 parity test are settled. \ No newline at end of file diff --git a/plan-doc/connector-cache-unification/designs/review-2.md b/plan-doc/connector-cache-unification/designs/review-2.md new file mode 100644 index 00000000000000..2d29205395ca30 --- /dev/null +++ b/plan-doc/connector-cache-unification/designs/review-2.md @@ -0,0 +1,44 @@ +Verification complete against HEAD. All load-bearing recon claims confirmed: the octet in `ConnectorPartitionViewCache` (line 70-71) is byte-identical to iceberg's hand-rolled caches (`IcebergTableCache:70`, `IcebergCommentCache:65`); the "NO consumers yet" javadoc (`ConnectorPartitionViewCache.java:34`) is stale — iceberg (2 fields, `IcebergConnector:197,199`), hive (`HiveConnector:112`), paimon (`PaimonConnector:143`) all consume it; `ConnectorStatementScope.computeIfAbsent` is in fe-connector-api; `fe-connector-cache` has ZERO fe-core imports (verified — the two CacheSpec/MetaCacheEntry copies are independent duplicates, so D2 lands entirely outside fe-core); gate hardcodes `TARGET_REL` to IcebergConnector.java; `IcebergConnector:860` declares via `.add(...SUPPORTS_USER_SESSION)`, `HiveConnector:543` and fe-core `PluginDrivenExternalCatalog:1289` use `.contains(...)`; mc/es have no `getCapabilities` override. + +Additionally verified the seam-efficacy substrate: `ConnectorSessionBuilder.captureStatementScope()` (line 190-202) reads `sc.getOrCreateConnectorStatementScope()` off the live `ConnectContext`, so every in-statement `buildConnectorSession()` shares ONE scope instance — B1 genuinely collapses the mc fan-out with no fe-core change. + +--- + +# Adversarial design review — fe-core seam (D4) + generic wrapper (D2) + +## 1. [major] Strike B2 entirely — do not leave a "sanctioned fe-core exception" on the table + +The design proves B1 (memoize inside mc's connector-side `getTableHandle` via the statement scope) collapses the 14–17-site `getTableHandle` fan-out. I verified the precondition B1 depends on: all in-statement `buildConnectorSession()` calls resolve to the *same* `ConnectorStatementScopeImpl` (`ConnectorSessionBuilder.java:202`), so B1's memo actually fires. That means **B2 delivers nothing B1 doesn't** — it only adds ~3 lines to the fe-core `resolveConnectorTableHandle` method (`PluginDrivenExternalTable.java:116-119`). + +Under Iron Rule A, an exception that buys zero capability over a connector-side alternative is not "minimal," it's *avoidable*. Keeping B2 described as "the smallest possible reading of the sanctioned exception" is exactly how an avoidable fe-core mutation gets merged: a later executor sees a signed-off option and takes it. **Fix:** delete B2 from the plan; state affirmatively that the D4 mandate is satisfied with **0 fe-core source lines** and iron rule A is literally untouched. The owner's D4 sign-off is honored by the fe-connector-api helper alone (or by objection #2's private-helper alternative). + +## 2. [major] The fe-connector-api helper is a Rule-2 net-add that the 2 real consumers don't structurally require + +`ConnectorStatementScopes.resolveInStatement` is offered as "no new fe-core source" — true, fe-connector-api ≠ fe-core, so **iron rule A is not breached**. But the mandate asks whether the SPI surface is truly minimal, and this adds a new public type to the SPI module to serve exactly two consumers, one of which (iceberg) *already has a working private helper* (`IcebergStatementScope.sharedTable`, 9 lines, `IcebergStatementScope.java:59`). Hudi can mirror it as a connector-private `HudiStatementScope.sharedMetaClient` — zero shared surface, each connector owns its own key convention, which the design itself concedes must be connector-distinct (the `keyNamespace` param is *required* precisely because the convention can't be shared). The helper deduplicates only ~4 lines (null-guard + `catalogId:db:table:queryId` assembly). + +Counter-argument I'll grant honestly: that assembled key is **security-critical** — dropping `queryId` leaks across statements, dropping `catalogId` collides across a cross-catalog MERGE. Centralizing it once is a legitimate correctness win over two connectors re-deriving it and one getting it wrong. So this is a real tension, not a clean cut. + +**Fix (if centralized):** keep it a bare static util (no new interface, no new method on `ConnectorSession`) — the design already does this — but **do NOT retrofit iceberg in the same PR (PR-2)**. Retrofitting working, audited iceberg code to delegate widens the blast radius over previously-signed-off code for no functional gain. Let hudi be the sole first consumer; converge iceberg later or never. If the owner prefers absolute minimum SPI surface, decline the helper and give hudi its own private mirror. + +## 3. [minor] The compat subclass `ConnectorPartitionViewCache extends ConnectorMetadataCache` is speculative preservation + +The design keeps `ConnectorPartitionViewCache` as a thin `entryName="partition_view"` subclass "so its three consumers stay untouched byte-for-byte." But those three consumers (iceberg/hive/paimon) are *already* being rewritten in PR-1 for the `PartitionViewCacheKey → ConnectorTableKey` rename. Since they're touched anyway, have them call `ConnectorMetadataCache("...","partition_view", props)` directly and **delete `ConnectorPartitionViewCache`** — one fewer type, no inheritance layer retained for a back-compat nobody external depends on (it's a connector-side toolkit class, not an SPI contract). Rule 2: don't keep an abstraction to avoid a rename you're already doing. + +## 4. [minor] "Byte-for-byte identical" iceberg retrofit overstates `invalidateDb` equivalence + +`IcebergTableCache.invalidateDb`/`IcebergCommentCache.invalidateDb` use `id.namespace().equals(Namespace.of(dbName))` on a `TableIdentifier` (`IcebergTableCache.java:101-104`). Rekeying to `ConnectorTableKey(db,table,-1,-1)` swaps this to string `matchesDb(db)`. These are equivalent *only* because Doris iceberg namespaces are single-level `[db]`; a multi-level namespace would diverge. Soften the retrofit claim from "byte-identical" to "functionally equivalent for Doris's single-level db namespace," and gate explicitly on the existing `invalidateDb` tests rather than asserting invariance by inspection. (Connector-side, not an iron-rule issue.) + +## 5. [nit] mc "14–17 → 1" over-claims: `buildCrossStatementSession` callers don't share the memo + +Several `resolveConnectorTableHandle` callers use `buildCrossStatementSession()` (`PluginDrivenExternalTable.java:460,1036,1071`; `PluginDrivenExternalCatalog.java:297,313`), which by design bind a non-per-statement scope. Those won't share B1's per-statement memo (nor should they — they're refresh/cross-statement paths). The fan-out collapse applies to the `buildConnectorSession()` in-statement subset only. State this so the win isn't over-claimed and so no one "fixes" the cross-statement paths into the memo. + +--- + +## Positive verifications (not objections) +- **D2 does not touch fe-core.** The wrapper lands in `fe-connector-cache`, which has zero fe-core imports; the fe-core `datasource/metacache/` copy is a separate duplicate the design never touches. Iron rule A is clean for the entire cache-wrapper track. +- **The generic wrapper is justified by concrete consumers, not hypotheticals.** 6 real consumers (iceberg ×5 + hudi ×1). The opaque `[V]` and the 4-tuple key with `-1` sentinels are driven by the actual value/key shapes at HEAD (raw `Table`, `String`, `CachedSnapshot`, `List<...>`; `TableIdentifier` vs `(id,snapshotId)`). The two-ctor split is backed by iceberg's real shared-knob pattern (`meta.cache.iceberg.table.ttl-second`, `IcebergTableCache.java:44`). No speculative `[K]` param, identity-sharding, or credential abstraction is added. Rule 2 satisfied. +- **The credential-null-gate stays connector-side.** Verified iceberg's session-user-disabled fields (`IcebergConnector.java:197,199,279-284`); the wrapper stays value-opaque and credential-agnostic. Iron rule D discipline is preserved by construction. + +--- + +**Overall verdict:** No blockers. The design, under its *recommended* path (fe-connector-api helper + B1), already respects iron rule A — it adds **0 fe-core source lines** and the D2 wrapper never touches fe-core. The two avoidable-byte risks are (a) B2 being left as a live fe-core option that should be struck outright (#1), and (b) the SPI helper + in-PR iceberg retrofit being wider than the 2 consumers strictly need (#2). Both are "minimize the surface," not "the surface is wrong." Adopt: strike B2, keep the helper connector-private OR at minimum defer the iceberg retrofit, and delete the compat subclass. \ No newline at end of file diff --git a/plan-doc/connector-cache-unification/designs/review-3.md b/plan-doc/connector-cache-unification/designs/review-3.md new file mode 100644 index 00000000000000..79be78edf7ebc6 --- /dev/null +++ b/plan-doc/connector-cache-unification/designs/review-3.md @@ -0,0 +1,59 @@ +I have enough grounding. Writing the adversarial review. + +--- + +# Adversarial design review — cache-unification (AUTHZ + FRESHNESS lens) + +Verification anchored at HEAD. The recon's structural claims hold: `ConnectorPartitionViewCache` is the octet the design describes (fe-connector-cache/…/ConnectorPartitionViewCache.java:46-110), it is live in iceberg/hive/paimon, the "NO consumers yet" javadoc is stale (same file:33-34), the gate hardcodes `TARGET_REL` to IcebergConnector.java (tools/check-authz-cache-sharding.sh:52-54), and hudi/mc/es declare no `SUPPORTS_USER_SESSION`. But three cache-safety objections survive, and the most severe one attacks the exact question in my mandate. + +## [BLOCKER] D3's generalized gate cannot see the generic wrapper's real instantiation sites — a future per-user connector that builds its `ConnectorMetadataCache` outside `*Connector.java` gets zero coverage + +**Issue.** The gate scans **field declarations on the owner `*Connector.java` file only** (script:52-54, `FIELD_DECL` at :80, and §C.2 enshrines "Scan the OWNER file only … never the whole module" as an invariant). But the generic wrapper is already declared in **two** places per connector — the Connector (marked) *and* the `*ConnectorMetadata` (unmarked, injected): +- iceberg: IcebergConnector.java:197-200 (marked) vs IcebergConnectorMetadata.java:170-171 (unmarked) +- hive: HiveConnector.java:112 vs HiveConnectorMetadata.java:241 +- paimon: PaimonConnector.java:143 vs PaimonConnectorMetadata.java:108 +- maxcompute: MaxComputeDorisConnector.java:61 vs MaxComputeConnectorMetadata.java:79 + +Today this is safe only because every connector *also* constructs and holds the instance on the Connector, where the marker sits. The gate's guarantee is therefore "every `*Cache` field textually present in `*Connector.java` of a declaring connector is marked" — **not** "every cross-query cache instance reachable by a declaring connector is isolated." + +**Reasoning.** Generalization actively worsens this. Today's `Iceberg*Cache` are dedicated classes the author naturally constructs in the Connector. A generic, injectable `ConnectorMetadataCache` is trivially constructed *anywhere* — a metadata object, a `*ScanPlanProvider`, a factory. A future connector that declares `SUPPORTS_USER_SESSION` and does `new ConnectorMetadataCache<>(...)` inside its `*ConnectorMetadata` (never touching `*Connector.java`, exactly as the injected field already lives there today) produces a name-keyed cross-query cache the gate structurally cannot see. That is precisely the "future per-user connector silently gets a name-keyed cross-query cache the gate fails to catch" failure I was told to hunt. The design bakes the hole in by declaring owner-file-only an invariant. + +**Fix.** Gate on the **construction expression**, location-independently: within any connector module whose `*Connector.java` declares `SUPPORTS_USER_SESSION`, grep module-wide for `new ConnectorMetadataCache<` / `new ConnectorPartitionViewCache<` (and the residual `new *Cache(` holders) and require each such construction site to carry a marker or sit under a capability null-gate. Keep the owner-file field-decl scan as a secondary net. A construction-site gate is inherently immune to where the field is *declared*. + +## [MAJOR] The marker is an unverified, copy-pasteable comment; the opaque wrapper moves the only real safety (the null-gate) into a construction idiom the gate never checks + +**Issue.** Safety is entirely convention: the connector writes `isUserSessionEnabled() ? null : new ConnectorPartitionViewCache<>(...)` (IcebergConnector.java:279-284) and the wrapper is value-opaque (ConnectorPartitionViewCache.java:46, "holding an opaque value V"). The gate only checks that the *marker string* is present near the field decl — it never checks that construction is actually null-gated. + +**Reasoning.** With a generic wrapper, PR-1/PR-4/PR-5 will copy both the field+marker line *and* the ternary idiom into new connectors. A mis-paste that keeps the marker but drops the `isUserSessionEnabled() ? null :` guard (or guards on the wrong flag) is a silent cross-user leak that passes the build GREEN — the marker asserts a discipline the code no longer honors. This is a pre-existing weakness, but generalization multiplies the copy-paste surface and detaches the marker (on the field) from the guard (at construction), making drift easier. + +**Fix.** Tie the gate to the construction guard, not just the field marker: for a `authz-cache-session-user-disabled` field, assert its assignment RHS is a capability-gated ternary (or the field is provably null under the capability). Add a RED self-test fixture: field marked disabled, constructed unconditionally → must FAIL. Without this, D3 verifies a comment, not a behavior. + +## [MAJOR] Stage-2's `.contains(` exclusion drops HiveConnector, which already holds an unmarked, unconditional, name-keyed cross-query cache AND is the gateway delegating to per-user iceberg/hudi + +**Issue.** §C.2 excludes lines with `.contains(`, so HiveConnector (whose only `SUPPORTS_USER_SESSION` reference is `sibling.getCapabilities().contains(...)` at HiveConnector.java:543) is classified a non-declarer and never scanned. Yet HiveConnector.java:112/134 holds `partitionViewCache`, an **unmarked, unconditionally-constructed, (db,table,-1,-1)-keyed cross-query cache** (comment at :108-111 says hive has "NO session=user … cache-disabling convention … constructed unconditionally"). And hive is the HMS gateway that delegates to per-user iceberg/hudi siblings. + +**Reasoning.** Today this is safe by a *runtime* guarantee, not the gate: the front door "never declares SUPPORTS_USER_SESSION" (comment :537) and fail-louds if a sibling does (:543-546). The design's residual #7 frames hive as merely "in-gate-scope or documented out." That undersells it: hive has a *live* cross-query cache that the generalized gate is *structurally blind to* by construction (the `.contains(` exclusion), so the entire safety of the gateway path rests on one runtime assertion the gate does not verify. If a future refactor makes the hive front door per-user (or the fail-loud assert is weakened), the gate stays GREEN while hive's `partitionViewCache` leaks. + +**Fix.** Treat a connector that *reads* a sibling's `SUPPORTS_USER_SESSION` (the gateway signature) as in-scope, and gate the presence of the fail-loud front-door assertion. At minimum, require hive's `partitionViewCache` to carry an explicit `authz-cache-exempt` marker with the "front door never per-user + fail-loud guard" justification, so the exemption is a reviewed claim rather than an invisible gap. + +## [MINOR] Hudi `(table, instant)` key: correctness needs "latest **completed** instant, re-read fresh per statement" — the design asserts correctness without pinning either + +**Issue.** §D.1 keys the cross-query partition cache by `(db, table, instant-as-snapshotId, -1)` and claims "always-correct (new instant ⇒ new key)." But correctness depends on two unstated properties: (a) the instant is the latest **completed** instant (not a requested/inflight one), and (b) it is re-derived **fresh each statement** from the per-statement metaClient that reloads the active timeline. + +**Reasoning.** If the key uses an inflight/requested instant, a concurrent writer can bind the entry to a partial partition set. If the instant is sourced from anything longer-lived than the per-statement metaClient, a stale instant can serve a stale partition list while still labeled "latest." The instant-in-key argument is only as strong as the freshness of the instant that forms it. + +**Fix.** Specify: the key uses `metaClient.getActiveTimeline().filterCompletedInstants().lastInstant()` (or equivalent), re-derived from the statement-scoped metaClient each statement. Add an e2e that lists partitions across a concurrent hudi commit and asserts no partial/stale set (this also satisfies the memory rule that HMS-delegated capabilities need e2e). + +## [MINOR] Enforce, don't just intend, the hudi/es freshness boundaries + +**Issue/Reasoning.** Two intent-level safeties need a test to be real: (1) es — the memoized `EsMetadataState` must hold only mapping/field-context and `fetchShards()` must re-run every statement (es has **no** `*Cache` field at HEAD — grep empty — so there is nothing to regress *yet*, but PR-6 introduces the memo); (2) hudi — the raw metaClient must stay in the per-statement seam and must never be promoted into the cross-query `ConnectorMetadataCache` layer, and the `closeAll` AutoCloseable caveat (§B.3 / Risks) must be checked so a statement-end close cannot corrupt the derived `(table,instant)` list (which stores `List`, not the client — so OK, but assert it). + +**Fix.** Add assertions/tests: es shard routing re-runs per statement; the hudi cross-query cache value type is a pure projection with no live metaClient/FileSystem reference. No design change, but make these gate-tested rather than convention. + +## [NIT] Rename churn + stale javadoc + +`PartitionViewCacheKey → ConnectorTableKey` re-touches iceberg/hive/paimon main+test imports (mechanical, low risk). Fix the confirmed-stale "this class has NO consumers yet" javadoc (ConnectorPartitionViewCache.java:33-34) in PR-1 as the design already notes. + +--- + +**Overall verdict: DO NOT SHIP D3 AS DESIGNED.** The A/B/D/E cache *placements* survive the authz+freshness lens (all D1 connectors use static creds; hudi's instant-in-key is sound once "completed + fresh" is pinned; es shard routing stays per-statement). But the gate generalization (D3) is the weak seam: as specified, it verifies owner-file field markers, while the generic wrapper's real instantiation sites already live unmarked in `*ConnectorMetadata` today and can move anywhere tomorrow — so a future per-user connector can obtain a name-keyed cross-query cache the gate cannot see (BLOCKER), the marker is an unverified comment detached from the actual null-gate (MAJOR), and hive's live unmarked cross-query cache plus its per-user delegation path fall structurally outside the gate (MAJOR). Re-scope D3 to gate cache **construction expressions** module-wide within capability-declaring (and gateway) connectors, and assert the null-gate, before the wrapper is generalized — otherwise the generalization ships a wider attack surface with a narrower gate. \ No newline at end of file diff --git a/plan-doc/connector-cache-unification/progress.md b/plan-doc/connector-cache-unification/progress.md new file mode 100644 index 00000000000000..02041cb38739a9 --- /dev/null +++ b/plan-doc/connector-cache-unification/progress.md @@ -0,0 +1,186 @@ +# Progress Log — 连接器缓存框架统一 (connector cache unification) + +> **append-only**：每 session 追加一条（日期 / 做了什么 / 结论 / 下一步）；不覆盖、不删旧条。 +> 滚动上下文在 [`HANDOFF.md`](./HANDOFF.md)，进度总览在 [`tasklist.md`](./tasklist.md)。 + +--- + +## 2026-07-23 — 搭建伞形追踪空间（未启动执行） + +- **做了什么**：读 `00-research-report.md` + `data/connector-audits.json`（hive/hudi/paimon 全文）+ 参考空间 `perf-hotpath-iceberg/`（README/HANDOFF/tasklist）；据此在本目录建 `HANDOFF.md` + `tasklist.md` + 本 `progress.md`。 +- **结论**： + - 本空间定位为**伞形协调空间**——追踪 workstream 级（WS-HUDI / WS-MC / WS-ES / WS-DOC / WS-P2）+ 4 个 owner 决策（D1–D4）；逐连接器执行各自另开 `perf-hotpath-/` 兄弟空间（报告 §9）。 + - 未改 `README.md`（它是完成态的调研交付物）；稳定流程/铁律/build 坑放进 HANDOFF 底部稳定区。 + - **未做任何拍板、未启动任何执行、未改产品代码。** +- **下一步（交下个 session）**：先向用户讲清 D1–D4 拿签字（中文，见 HANDOFF「下一步」），默认推荐先启动 WS-HUDI（唯一 P1 真缺口）并新开 `plan-doc/perf-hotpath-hudi/`。动码前按 HEAD 重侦察（行号信 grep）。 + +--- + +## 2026-07-23 (2) — owner 4 决策签字 + 设计定稿（仍 0 产品代码改动） + +- **做了什么**： + 1. 向 owner 讲清 D1–D4 并拿签字：D1 hudi+mc+es 都做；D2 先建底座；D3 现在就通用化门禁；D4 原选"提炼进 fe-core"。 + 2. 跑 11-agent 只读设计调研 workflow（6 路 HEAD 侦察 + Trino 参考 → 设计综合 → 3 路对抗评审 → 终稿）。产物存 `designs/`（`foundation-design-FINAL.md` + draft + review-1..3）。 + - 注：首跑综合步因"大嵌套结构化输出被截断"失败；改成 prose 输出 + 断点续跑（6 侦察命中缓存），成功。 + 3. 设计定稿后向 owner 二次确认，两处更正/确认拍板。 +- **结论（关键）**： + - **D4 前提被侦察推翻**：per-statement memo 底座**早已在 `fe-connector-api`**（`ConnectorStatementScope.computeIfAbsent`/`ConnectorSession.getStatementScope`；iceberg/hudi 均依赖该层、均不依赖 fe-core）→ helper 只是该层一个小静态方法 → **fe-core 0 行、铁律 A 不碰**。owner 二次确认接受、删掉"往 fe-core 塞"备选（含 mc 的 B2）。又一次"侦察推翻已签字蓝图"（memory `execution-blueprint-overestimates-recon-first`）。 + - **底座几乎现成**：通用缓存封装 = 升格已存在的 `ConnectorPartitionViewCache[V]`（iceberg/hive/paimon 已用）→ `ConnectorMetadataCache[V]`；key `PartitionViewCacheKey`→`ConnectorTableKey` 四元组。**整套 A/B/C + hudi/mc/es = 0 行 fe-core 改动。** + - **门禁原方案有 BLOCKER**：iceberg 缓存字段也在 metadata 类上、hive 网关无标记缓存 → 评审重设计为"模块内扫缓存构造点 + 断言凭证置空 + 网关纳入 + 零声明者硬失败"。 + - owner 确认：**iceberg 本轮就改挂**（真正先建底座、证 parity）。 + - 评审强制若干安全约束：hudi 记忆"不可关闭投影"、instant 每语句重取只缓存分区列表、iceberg `invalidateDb` parity 测试、es 分片路由拆开保持每语句。 +- **下一步（交下个 session）**：进入实施，按 HANDOFF「实施路线」8-PR（PR-0 先验 → PR-1 封装 → PR-2 helper+iceberg 改挂 → PR-3 iceberg 收敛 → PR-4 hudi → PR-5 mc → PR-6 es → PR-7 门禁）。**动每个文件前按 HEAD 重侦察**；全程守 fe-core 0 行，遇"不得不碰 fe-core"停手交 review。 + +--- + +## 2026-07-23 (3) — PR-0 完成：预编译执行的连接器作用域重置回归守门（含外表可达性侦察） + +- **做了什么**： + 1. 进实施前按 HEAD 重侦察 PR-0/1/2 承重事实（6 路只读 workflow + 综合 drift 报告）：**设计成立、有更正、更正均缩小工作量**。 + 2. 用户质疑"预编译语句是否支持外表"→ 专项 Explore 侦察**查实：外表确经预编译执行**——走普通 `executor.execute()` 全量规划路、每次执行重新经连接器解析外表；**永进不了 OLAP 短路点查快路**（短路规则只匹配 `logicalOlapScan()`，`LogicalResultSinkToShortCircuitPointQuery.java:88,97`）。故 `ExecuteCommand.java:95` 的 `resetConnectorStatementScope()` 真实可达、承重，只是此前**零测试守门**（现有测试只覆盖 reset 原语，不证 `ExecuteCommand` 调它）。 + 3. 加 FE 单测 `ConnectorStatementScopeTest.executeCommandResetsConnectorScopePerExecution`：往复用的语句上下文放哨兵值 → 驱动 `ExecuteCommand.run()`（执行器 stub 空操作、`enableGroupCommitFullPrepare=false` 走普通路）→ 断言执行后作用域被替换、哨兵不残留。 + 4. 改正 FINAL 设计里被证伪的机制描述（"每次执行拿新 `StatementContext`" → "复用上下文 + 每次执行显式 `resetConnectorStatementScope()`"，并记录外表可达性侦察结论）。 +- **验证**：`mvn test -pl fe-core -am -Dtest=ConnectorStatementScopeTest -Dmaven.build.cache.enabled=false`：`Tests run: 9, Failures: 0`，BUILD SUCCESS，checkstyle 过。**变异验证**（注释掉 `ExecuteCommand.java:95` 的 reset）：`Failures: 1` 且仅新测试变红（`expected: not same`），其余 8 测试仍绿 → 证明测试能失败且精确针对该 reset（Rule 9）。变异后已**逐字节还原** `ExecuteCommand.java`（`git diff` 空）。 +- **侦察更正（供后续 PR，动码前仍须再 grep 确认）**： + - ①**无"兼容子类"可删**——连接器直接构造 `ConnectorPartitionViewCache`（iceberg 构造两次 `:281/:284`，hive `:134`，paimon `:158`），`git grep "extends ConnectorPartitionViewCache"` 空 → PR-1 删掉"删兼容子类"这一步；勿把 `IcebergPartitionCache`（独立 PERF-02 层）误当子类。 + - ②`ttl≤0→CACHE_TTL_DISABLE_CACHE` 映射复制在 **6** 处（设计写 5）：`IcebergComment/Format/LatestSnapshot/Partition/Table` + `PaimonLatestSnapshotCache` → PR-1 收进 `CacheSpec` 动 6 处。 + - ③`formatCache` 挂在 `IcebergScanPlanProvider`（`IcebergConnector.java:782` 注入）**非** metadata 对象 → PR-3 触 format 缓存须对扫描规划器。 + - ④iceberg 5 缓存**全独立 `final class`**、均建于 `MetaCacheEntry`、**无一** extends `ConnectorPartitionViewCache`；entry 名 hyphen（`iceberg-table` 等，非 `iceberg.table`）→ PR-3 钉死 legacy 名。 + - ⑤`ConnectorStatementScopeImpl` 在 **fe-core**（`org.apache.doris.connector`，引用 fe-core `CatalogStatementTransaction`），interface 在 fe-connector-api；iceberg 经 fe-connector-spi **传递**依赖 api、hudi 直接依赖 → PR-2 的 `ConnectorStatementScopes` helper 放 fe-connector-api 仍**0 行 fe-core**。 +- **下一步**：PR-1 通用缓存封装升格（`ConnectorPartitionViewCache[V]`→`ConnectorMetadataCache[V]`、`PartitionViewCacheKey`→`ConnectorTableKey`、6 处 ttl 映射收进 `CacheSpec`、修 stale javadoc "no consumers yet"、iceberg/hive/paimon 改挂）；纯加+改名+删，反应堆 test-compile + 现有 partition-view 测试证零变化。**动每个文件前按 HEAD 重侦察**。 + +--- + +## 2026-07-23 (4) — PR-1 完成：通用缓存封装升格为 ConnectorMetadataCache（纯重命名，行为不变） + +- **做了什么**： + 1. 动码前按 HEAD 重侦察全部改名点（`ConnectorPartitionViewCache` / `PartitionViewCacheKey` 的所有引用，15 文件 4 模块），确认无外部脚本/配置引用、新名无冲突。 + 2. 把已经通用的缓存封装正式升格：`ConnectorPartitionViewCache`→`ConnectorMetadataCache`、`PartitionViewCacheKey`→`ConnectorTableKey`（含文件改名，`git mv` 保留历史）；构造器由硬编码 `"partition_view"` 改为显式传 `(engine, entryName, props)`，供后续连接器注册独立命名的缓存条目。 + 3. hive/iceberg/paimon（生产+测试）共 12 文件改挂新名；三连接器构造点显式传 `"partition_view"` → 条目名、`meta.cache..partition_view.*` 配置项、缓存键**逐字节不变**。修 stale "no consumers yet" javadoc。 + 4. **收窄设计原 bundling**（Rule 2/3）：TTL≤0 禁用映射去重（6 处复制）+ 预解析 CacheSpec 构造器**推迟到 iceberg 收敛那步**做（那批 6 处里 5 个是 iceberg 手写缓存类，下一步本就重写它们，避免二次翻动）。 +- **验证**：`mvn install -pl cache,hive,iceberg,paimon -am`（**install 非 test**——hive/iceberg/paimon 经 fe-connector-hms 依赖 hive-shade jar，`-am test` 不产 shade jar 会在 hms 编译期挂，见 build 坑 1）：BUILD SUCCESS，四模块全过；7 个分区视图缓存测试类共 **66 测试 0 失败**（ConnectorMetadataCacheTest 11 + hive 5+4 + paimon 7+7 + iceberg 25+7）。 +- **踩坑记录（供后续机械改名复用）**：`sed 's/ConnectorPartitionViewCache/ConnectorMetadataCache/g'` **子串过匹配**——把测试类名 `HiveConnectorPartitionViewCacheTest` 也改成 `HiveConnectorMetadataCacheTest`（但文件名没改）→ checkstyle `OuterTypeFilename` 报错。教训：跨文件类名机械改名用**词边界** `\b`（`Hive`+`ConnectorPartitionViewCache` 间无边界，`\b` 可避免误伤）；或改后用"文件名 vs public 类名"扫描兜底（本轮已用该扫描定位唯一误伤）。 +- **下一步**：PR-2 语句作用域通用 helper（`ConnectorStatementScopes.resolveInStatement` + namespace 注册表，放 `fe-connector-api`，**0 行 fe-core**）+ iceberg 私有 `IcebergStatementScope.sharedTable` 改委派（key 逐字节不变，须 byte-identical parity 测试）。动码前按 HEAD 重侦察。 + +--- + +## 2026-07-24 — PR-2 完成：语句作用域通用 helper `ConnectorStatementScopes`（0 行 fe-core，iceberg 改挂 byte-identical） + +- **做了什么**（commit `ae8c925074d`，严格 4 文件、零 fe-core 源码）： + 1. 动码前按 HEAD 重侦察全部承重事实：`ConnectorStatementScopes`(复数)不存在须新建；iceberg 现键 `"iceberg.table:" + catalogId + ":" + db + ":" + table + ":" + queryId`；`ConnectorStatementScope.computeIfAbsent(String,Supplier)`/`ConnectorSession.getCatalogId():long`/`getQueryId():String`/`getStatementScope():default NONE` 逐一核对；`rewritableDeleteSupply` 是 `(catalogId,queryId)`-keyed scan→write 累加器（非表解析）→留 iceberg 私有；4 个 `sharedTable` 调用方（metadata/scan/write/transaction）签名不变、仅 body 委派。 + 2. 新增 `fe-connector-api` 的 `ConnectorStatementScopes.resolveInStatement(session, keyNamespace, db, table, loader)`：复用已存在的 `ConnectorStatementScope.computeIfAbsent` 原语，统一"每语句解析一次 db.table"的**安全关键键约定**（丢 queryId=跨执行泄漏、丢 catalogId=跨目录 MERGE 撞车、丢 namespace=异构网关下值类型撞车→ClassCastException）；null session / NONE scope 每次跑 loader（load-every-time 不变）。namespace 注册表以 `ICEBERG_TABLE="iceberg.table"` 落地（hudi/mc/es 保留、各自 PR 接入时声明）。 + 3. iceberg `IcebergStatementScope.sharedTable` 改为委派该 helper，用 `ICEBERG_TABLE` 命名空间**逐字节复现**历史键前缀 → 4 resolver 命中/未命中/NONE 回落全等。 +- **验证**： + - `install -pl fe-connector-api,fe-connector-iceberg -am` BUILD SUCCESS，全模块 0 checkstyle。 + - 新 `ConnectorStatementScopesTest` **8 测试**（memo-once / 5 轴逐一隔离 / namespace 值类型隔离(否则 CCE) / null+NONE load-every-time / 键逐字节断言）；`IcebergStatementScopeTest` **7 测试**（+ byte-key parity `"iceberg.table:7:db1:t:q1"` + iceberg 级 null-session）；**iceberg 全模块 1133 测试 0 失败**（4 个 sharedTable 调用方测试类全绿：Transaction 66/Metadata 51/Scan 114/Write 42）。 + - **铁律 A 核实**：`git diff --numstat -- 'fe/fe-core/**'` 空 → 0 行 fe-core。 + - **4 路对抗净室复审**（byte-parity / callers / iron-rules-leak / test-quality）全判 **PARITY_HOLDS**、无一 refutes_parity；据其两条反馈**加固测试**：①测试替身 `getSessionId()` 改为 ≠ `getQueryId()`（否则 queryId→sessionId 误改会跨查询泄漏却无测试能红）②补 iceberg 级 null-session 测试。 + - **Rule 9 变异验证**：把 helper 键 `getQueryId()`→`getSessionId()`，**恰好两个 byte-key 测试变红**（api 1/8、iceberg 1/7）、其余全绿；已逐字节还原（`git diff` 该行回 `getQueryId()`）。 +- **一条 surface 给 owner（非阻塞）**：`ICEBERG_TABLE` 常量放在中立 SPI 层 `fe-connector-api` 上，两名评审标为 minor 层次瑕疵（中立 SPI 里出现连接器名），但同时判定"可接受的命名空间注册表、非有害泄漏"——这是设计 §B 修订#7 owner 已签的**中心化 uniqueness 注册表**（防 R9 跨连接器 namespace 撞车的唯一审计点）；若移到各连接器自持则失去中心审计点。保持设计原样，如 owner 更偏好各连接器自持常量可轻量改。 +- **下一步**：iceberg 5 缓存收敛（原计划下一步），动码前重侦察 + 向 owner 讲清成本后由 owner 重新定范围。 + +--- + +## 2026-07-24 (2) — owner 重定范围：只做安全 ttl 去重，全量收敛延后（行为字节级不变） + +- **背景/决策**：原计划下一步是把 iceberg 5 个手写缓存全量收敛到通用 `ConnectorMetadataCache`。动码前重侦察后向 owner 如实讲清成本：该收敛**零功能收益**、改动面宽（~19 文件 / 重写 ~37 测试 / 6 个重载构造函数签名波及 / 收敛后调用点从强类型 `TableIdentifier` 退化成字符串四元组键 / 每缓存独立注释退化成字段注释），且通用框架**已被证明可用**（已在 hive/iceberg/paimon 三连接器分区视图缓存跑着）、有性能收益的 hudi/mc/es **不依赖**它。**参考 Trino**（共享底层原语 + 各连接器自持缓存、不强制统一封装）→ iceberg 这 5 个缓存已是 Trino 式。**owner 拍板：只做安全 DRY、全量收敛延后。** +- **做了什么**（严格 8 文件、0 行 fe-core、纯 `[refactor]` 行为不变）： + 1. 新增 `CacheSpec.ofConnectorTtl(ttlSecond, capacity)`：把连接器"`ttl≤0` 禁用"契约折叠进 `CacheSpec` 的禁用哨兵（0）——负 ttl 走禁用而非被 `CacheSpec` 读成 `-1`「不过期(启用)」。逐字节等于原三元式 `ttl>0 ? of(true,ttl,cap) : of(true,DISABLE,cap)`。 + 2. **6 处**复制的三元式改调该工厂：`IcebergTable/LatestSnapshot/Comment/Format/Partition` + `PaimonLatestSnapshot`（PR-1 侦察更正②点名的 6 处）。 + 3. 补 `CacheSpecTest.ofConnectorTtlFoldsNonPositiveToDisabled`（Rule 9）：钉死承重的负-ttl-必禁用——`-1`/`-2` 折叠成禁用哨兵、`isCacheEnabled` 恒 false（否则负值静默变永不过期缓存）。 +- **验证**：`mvn install -pl :fe-connector-cache,:fe-connector-iceberg,:fe-connector-paimon -am -Dmaven.build.cache.enabled=false`（install 非 test，见 build 坑）：**BUILD SUCCESS**。`CacheSpecTest` 15（+1 新）、`ConnectorMetadataCacheTest` 11、6 个受影响缓存的既有测试**原样全绿**（Iceberg Table 7 / Comment 8 / Partition 8 / Format 8 / LatestSnapshot 6 + Paimon LatestSnapshot 6）、iceberg 全模块 1134、paimon 379，**0 失败 / 0 错误**。checkstyle 过。fe-core `git diff` 空。 +- **未做（明确延后）**：5 个 iceberg 缓存类**未收敛**、其类型/键/`*ForTest` 访问器/5 个测试文件**原样保留**；通用缓存的 pre-resolved-名/loadCount 访问器、`invalidateDb` Namespace→String parity 等收敛相关改动**一并延后**（框架已被证明可用、消费者不依赖；出现真实需要再上收）。 +- **下一步**：转入有性能收益的连接器工作——旗舰 **hudi**（新开 `plan-doc/perf-hotpath-hudi/`，镜像 iceberg 布局；前置改 pom 引 `fe-connector-cache` 工具箱）或先做 **mc / es** 两个小 PR。动码前按 HEAD 重侦察。 + +--- + +## 2026-07-24 (3) — WS-MC maxcompute round-1：每语句表句柄记忆化（commit `58daadd10e0`） + +> 承 hudi round-1 完成（见 `plan-doc/perf-hotpath-hudi/`，commit `26690775c81`）后，转入 mc/es 两个小 PR 中的 **maxcompute**。详见兄弟空间 `plan-doc/perf-hotpath-maxcompute/`。 + +- **病灶**：`MaxComputeConnectorMetadata.getTableHandle` 每次解析都发一次冗余 ODPS `tables().exists()` 远程探测 + 新建惰性 `Table`；funnel 只 memo metadata 不 memo handle → 一条语句 ~17 个解析点（fe-core `resolveConnectorTableHandle` 13 + translator 2 + BindSink 2）各付一次探测（冷统计逐列放大 O(列数)），各自 Table 首访各触发一次 reload。= 审计 MC-1（P1）+ MC-2（P2）。 +- **做了什么**（1 连接器文件、+28/-9、**0 fe-core**）：per-statement `MaxComputeConnectorMetadata` 实例加 `Map, MaxComputeTableHandle> tableHandleMemo`（`ConcurrentHashMap`），`getTableHandle` 经 `computeIfAbsent` 解析。**present-only**（mapping 返回 null → 不记录 → 缺表每次重探、`Optional.empty()` 逐字节不变）；**保留 exists() 只去重不删**（删会把干净 not-found 退化成后续 `OdpsException`）；handle 无 equals/hashCode → 按 `(db,table)` 值 key（`List.of`）；CHM 因 off-thread scan 池复用同 per-statement session/instance。`createTable`/`tableExists` 直调 helper 不经 memo；跨查询 `MaxComputePartitionCache` 正交。 +- **验证**：全 `MaxCompute*` **120 测试绿**、checkstyle **0 违规**。守门单测 `MaxComputeConnectorMetadataHandleMemoTest`（仿 `DropDbTest` 手写记录 fake、无 Mockito、null odps 离线）4 例：同表 2 解析→探测/build 各 1 + handle `assertSame`；不同表独立；**同名跨 db 不撞**；缺表每次重探不 memo。**Rule 9 变异两处**：去 memo（`memo.clear()`）→ 同表用例红（探测 1→2）；db-blind key（`List.of(tableName)`）→ 跨-db 用例红（`assertNotSame` 失败）。 +- **净室对抗复审**（4 lens + 2 verify workflow）：parity **PARITY_HOLDS**、staleness **PARITY_HOLDS**；concurrency CONCERN（共享 Table 并发首次 reload）经 verify **REFUTED**——off-thread scan 任务用 scan-node ctor 一次解析的 `currentHandle`、**不调** getTableHandle，共享 Table 是 baseline 既有属性，memo 反而让单线程分析先暖 schema、**降** reload 争用；test-quality CONCERN（无跨-db 用例，db-blind key 变异能漏网）经 verify **CONFIRMED_REAL** → **已补**跨-db 守门用例并变异验证。 +- **未做/延后**：PERF-MC02 陈旧注释（doc-only，随手/并 WS-DOC）、PERF-MC03 可选跨查询 `Table` 缓存（热点触发）。**e2e 需集群本地未跑**（异构 + 独立 max_compute catalog 的 SELECT/分区裁剪/写路径解析计数），留标注。 +- **下一步**：WS-ES（es 连接器 `fetchMetadataState` per-scan hoist 2×→1× + raw mapping 承载决策；**shard routing 绝不 cross-query 缓存**）。 + +--- + +## 2026-07-24 (4) — WS-ES es round-1 三件全做（commits `7d74ba1161b` F1+F3、`7466b354901` F2） + +> owner 拍板"三件全做"。详见兄弟空间 `plan-doc/perf-hotpath-es/`。至此三个 P1 连接器（hudi/mc/es）round-1 全部收尾。 + +- **病灶**：es 一条过滤 SELECT 约 `~4× getMapping + 2× search_shards + 2× _nodes` 远程往返（多为冗余）；连接器无扫描元数据缓存。**硬约束**：分片路由/节点拓扑绝不跨查询缓存（ES rebalance）。 +- **做了什么**（三件，按新鲜度分层；**0 fe-core**）： + 1. **per-scan hoist**（ES-F1，`EsScanPlanProvider`）：memo `EsMetadataState` 标量字段（guard on (index,columns)，**plain**——ES 从不进 batch mode 故单线程），`planScan`/`buildScanNodeProperties` 共用一次 fetch。分片随 provider per-scan 新鲜。 + 2. **per-statement schema memo**（ES-F3，`EsConnectorMetadata`）：`Map`（CHM）`computeIfAbsent`，`getColumnHandles→getTableSchema` 折叠。镜像 maxcompute。 + 3. **cross-path mapping**（ES-F2，走"每语句共享作用域"=owner 选的乙案）：`fe-connector-api` 加命名空间 `ES_INDEX_MAPPING`（iceberg/hudi 既定模式），`EsStatementScope` 把**原始 mapping 字符串**存每语句 `ConnectorStatementScope`；schema 路径 + `EsMetadataFetcher.fetchMapping` 两路共享、各自派生产物；session 穿到 fetcher/provider。**只共享原始 mapping；分片/节点绝不入作用域**。（甲案=塞 schema 缓存须改 fe-core 是坑；丙案=延后；owner 选乙。） +- **总账**：每语句 `~4×/2×/2×` → **1× getMapping + 1× search_shards + 1× _nodes**。`git diff fe/fe-core` 空。 +- **验证**：全 `Es*` **94 测试绿** + checkstyle 0；`EsScanPlanProviderTest` 12 例。**Rule 9 变异三处**（去 F1 store / F3 clear / F2 绕作用域 → 各对应门禁红、其余绿）。**净室 4-lens + verify**：parity/concurrency（schemaMemo↔scope 嵌套 computeIfAbsent 不同 map 无重入）/freshness 全 `PARITY_HOLDS`；唯一 CONFIRMED = "无门禁钉分片不入作用域" → **补 `ShardRoutingNeverSharedViaScope`**（两 provider 共享 live scope → shard/node 各 2、mapping 1）；nits（List.equals 顺序、死代码 threading）评估接受。 +- **未做/延后**：ES-F4（`existIndex` `_mapping` GET 去重，低优先）；死代码 `EsConnectorMetadata.fetchMetadataState` 清理（保留）。**e2e 需集群本地未跑**，留标注。 +- **下一步（伞形）**：三个 P1 连接器收尾完毕 → P2 backlog（热点触发）+ PR-7 门禁通用化（独立）+ WS-DOC 陈旧注释 + 各连接器 e2e 统一补。 + +--- + +## 2026-07-24 (5) — 授权缓存门禁：尝试通用化 → 两轮对抗复审证伪结构化验证不可行 → owner 拍板"删门禁 + 加 ATTN 注释" + +> 承接 (4) 剩余的独立项"门禁通用化"。结论是**删除**整个授权缓存门禁（非通用化），改用代码内 `ATTN` 注释 + 现有运行时行为单测兜底。 + +- **起点**：`tools/check-authz-cache-sharding.sh` 原只扫 `IcebergConnector.java` 字段声明标记。设计（`designs/foundation-design-FINAL.md` §C，owner 签字 D3）要改成"模块内构造点扫描 + 结构化验证每处 `new *Cache(` 是否正确置空"。 +- **做了什么（先按设计实现，再对抗复审）**： + 1. 实现构造点扫描版门禁（阶段1声明者/1b网关fail-loud/2结构化置空判定/反no-op兜底）+ 16 例自测 + hive/iceberg 4 处豁免注释；对真实树 exit 0、`mvn validate` 通过。 + 2. **第一轮净室对抗复审**（4 lens + 逐条 verify）：确认 10 项，含"反向极性三元式漏报""构造器参数内 `? null :` 误判""跨行 `//` 注释边界串味""委派构造注入缓存漏检""网关 `throw` 8 行内任意即放行"等。逐条加固。 + 3. **第二轮对抗复审（专打加固后逻辑）**：又实测复现 **11 项**——复合 `||`/`&&` 守卫漏报、多行 lambda loader 里 `;` 截断语句致**误报挡合法构建**、字符串字面量里能力名致误报、跨行 `if` 条件网关误判、`throw` 子串蒙混、嵌套 `)` 漏检声明者等。 +- **根本认识**：让 shell 判断"缓存在每用户模式下是否真置空"= 让它读懂任意 Java 布尔/多行语法，**不可行且脆**（误报挡构建、漏报放走泄漏，每修一个冒新边界）。而"置空正确性"**已被运行时行为单测 `IcebergConnectorCacheTest` 在真实实例上证明**（比任何静态检查强），门禁的机器验证是越界。 +- **owner 拍板**：**删掉整个门禁检查，改在代码注释里加显式 `ATTN` 说明**（否决"甲=简化成纯标记门禁""丙=搬到 Java 解析器""乙=继续修正则"三选项）。 +- **落地（严格 4 文件 + 3 doc）**： + - 删 `tools/check-authz-cache-sharding.sh` + `tools/check-authz-cache-sharding.test.sh`；移除 `fe/fe-connector/pom.xml` 里 `check-authz-cache-sharding` 的 exec 执行块（保留同款 `check-connector-imports`）。 + - `IcebergConnector.java`：把原门禁标记注释（`authz-cache-*` token）转成**显式 `ATTN` 段**——讲清 list!=load 越权风险、每个跨查询缓存已在构造函数 `? null : new …` 置空、manifest 豁免、**新增缓存必须置空且被 `IcebergConnectorCacheTest` 覆盖**、"无构建门禁，靠评审+单测"。7 处字段尾标记 → `// null under session=user`。 + - 本 session 期间加的 hive/iceberg-meta 4 处门禁脚手架注释一并 `git checkout` 还原（它们只为门禁服务）。 +- **验证**：`grep` 全树无残留 gate 引用；`mvn validate -pl fe-connector` **通过**（同款 import 门禁仍跑）；`checkstyle -pl :fe-connector-iceberg` **0 违规**；行长 ≤112。变更集严格 4 文件（IcebergConnector 注释、pom 去执行、删 2 脚本）。 +- **通用教训（进 memory 候选）**：shell/正则门禁只适合"存在性/前缀"类不变量（对齐 `check-connector-imports`/`check-fecore-metadata-funnel`）；一旦需要理解语言语义（布尔逻辑、多行、字符串），就不是 shell 的活——有运行时行为单测时，"注释警示 + 单测 + 评审"胜过脆弱的静态门禁。 +- **下一步（伞形）**：授权缓存门禁项**关闭**（已删+ATTN）。剩余 = P2 backlog（热点触发）+ WS-DOC 陈旧注释 + 各连接器 e2e 统一补。 + +--- + +## 2026-07-24 (6) — WS-DOC 陈旧注释清理：原目标已自动完成，另清 iceberg+maxcompute 写/过程"翻闸前休眠"陈旧注释 + +> 承 (5) 后做剩余独立项"陈旧注释清理"。动手前按 HEAD 重侦察（信 grep 不信计划快照行号），结论与计划描述不同。 + +- **侦察结论（关键）**： + 1. **计划原点名的那批（"HMS 翻 live 后没更新的注释"：hive/hudi/mc 的 "dormant / hms 未进 SPI_READY_TYPES / getTableHandle never called / no consumers yet"）——早已在前几轮（hudi/mc/es round-1 触碰 hms/hive 时）顺手更新**：现在写着 "Live since the hms flip"/"Live since the HMS cutover"，`no consumers yet` javadoc 已消失，`getTableHandle` 有真实调用方（`HiveConnectorMetadata:413/416` 路由兄弟、iceberg 写路径调用）。**计划这一项基本已完成**。 + 2. **另发现一批不同的陈旧注释**（计划未列）：iceberg 写/事务/存储过程一批注释仍写 "iceberg 未进 SPI_READY_TYPES、dormant 直到 P6.6 翻闸"——但**该翻闸 2026-07-05 已完成**（`plan-doc/PROGRESS.md:162`："iceberg 已入 SPI_READY_TYPES，FE 走 SPI 路径"），iceberg 写/DML/存储过程都有回归测试在跑（`iceberg/dml`、`write`、`branch_insert`、`position_delete`…）；maxcompute 写路径注释仍写 "dormant 直到 max_compute cutover"，但 fe-core 有**专给 mc 的写块分配基础设施 + BE→FE 回调 RPC**（`WriteBlockAllocatingTransaction`"only maxcompute today"、`getMaxComputeBlockIdRange`）→ mc 写已 live（owner 确认）。 + 3. **判活/死纪律（避免误判）**：`SPI_READY_TYPES` 成员只让**读** live，**写/过程**各连接器单独接线——故不能一律把 "dormant" 当陈旧。**hudi `IncrementalRelation` 的 "DORMANT" 是真休眠**（增量查询未接进 `HudiScanPlanProvider.planScan`）→ **正确留存不动**。 +- **owner 拍板范围**：清 iceberg + maxcompute（都已确认 live）；hudi 真休眠不动；"翻闸前/后措辞过时但描述正确"的注释本轮不顺（选 A 非 C）。 +- **做了什么**（严格 9 文件、**纯注释、0 代码行**，`git diff` 证每一改动行都是注释）：把 13 处"Gate-closed / dormant / 未进 SPI_READY_TYPES / nothing routes … yet / 直到翻闸"的陈旧前提，逐处改成反映 live 现状（镜像 hive 已有的 "Live since the … cutover" 模板），保留各注释的机制描述；顺带去掉现已完成的未来时任务码前指（T04/T05/T06/P4-T04 "land in later tasks" 等）。 + - iceberg（6 文件）：`IcebergWritePlanProvider`/`IcebergConnectorTransaction`(×4)/`IcebergRewriteDataFilesAction`/`IcebergExecuteActionFactory`/`IcebergScanPlanProvider`/`IcebergConnectorMetadata`。 + - maxcompute（3 文件）：`MaxComputeWritePlanProvider`/`MaxComputeConnectorTransaction`/`MaxComputeConnectorMetadata`。 +- **验证**：`checkstyle -pl :fe-connector-iceberg,:fe-connector-maxcompute` **0 违规**；行长全 ≤120；`git diff` 确认**全部改动行都是注释**（非注释改动行 grep 为空）→ 编译/行为零影响。 +- **下一步（伞形）**：剩 P2 backlog（热点触发）+ 各连接器 e2e 统一补（需集群）。 + +--- + +## 2026-07-24 (7) — owner 排期的三项 P2：paimon 分区列举去重 ✅ / jdbc 两处取列去重 ✅ / hive 写后读一致性 = 虚惊（非 bug） + +> 承 (6) 后做 owner 2026-07-24 点名排期的三项（非"等热点"）。5-agent 并行按 HEAD `d8e2541e567` 重侦察（旧快照 `aaab68ef474` 行号已漂移），逐项立项。全程守铁律 A（fe-core 0 行）。**结论：两项落地、一项经侦察证伪。** + +### 一、paimon 分区列举去重（PA-1）— ✅ commit `59b65912104` +- **病灶确认**：`listPartitionNames`（SHOW PARTITIONS）/`listPartitionValues`（`partition_values()` TVF）绕过 `partitionViewCache` 直连 `collectPartitions`，每次重渲染整表分区列表；`listPartitions`（裁剪）已走缓存。远程已被 paimon SDK `CachingCatalog.partitionCache` 挡下 → 省本地 CPU 重渲染。 +- **修法**：抽私有 `cachedPartitions()`（= `listPartitions` 无过滤分支等价体），三入口统一走它；`listPartitions` 保留 `filter → collectPartitions`（带过滤不写缓存）。连接器侧、0 fe-core。 +- **owner 语义决策（问后拍板"走缓存"）**：`SHOW PARTITIONS`/`partition_values()` 新鲜度从"每次重算"→"24h TTL + REFRESH"，与已缓存的裁剪路径自洽、与 Trino 一致；代价是与 hive"故意实时"不一致（hive 刻意分 fresh 方法，paimon 只读无写后读问题）。 +- **验证**：`PartitionViewCacheTest` +4、`PartitionTest`（null-cache 字节 parity）原样绿，26/26 + checkstyle 0；3-lens 净室（aliasing/freshness CLEAN，parity 仅标"有意新鲜度变化"，iron-rules CLEAN 修一处测试头注释）。**e2e 待集群**。详见兄弟空间 [`plan-doc/perf-hotpath-paimon/README.md`](../perf-hotpath-paimon/README.md)。 + +### 二、jdbc 两处冗余取列（HP-1 读 + HP-2 写）— ✅ commit `7df22cd1c71` +- **病灶确认**：本地→远端列名映射靠 `client.getJdbcColumnsInfo`（真实远程往返）；读侧 `getColumnHandles`（~2×/scan node）+ 缓存 miss 的 `getTableSchema`、写侧 `buildInsertSql` 新建实例再取（`EXPLAIN INSERT` 2×），全无记忆化。 +- **owner 拍板路线**：**语句作用域统一去重**（对齐 es cross-path 先例）。关键洞见=记忆化放**作用域**（非实例）→ 写侧新建实例的 `getColumnHandles` 也命中同条目 → 一个改动点修两处。 +- **修法**：加 `ConnectorStatementScopes.JDBC_COLUMNS` 命名空间（fe-connector-api 连接器 SPI，非 fe-core）；`JdbcConnectorMetadata` 两处取列包 `resolveInStatement`，记忆化**原始** `List`（session 无关，各消费者再套自己的变换）；写 provider 仅 javadoc。NONE 作用域逐字节与改前一致。0 fe-core。 +- **净室发现并修复**：`jdbcTypeToConnectorType` 对某些日期类型**原地** `setAllowNull(true)`→现共享同一 raw list；修正 javadoc 如实描述该幂等原地改写 + 加"会变换的类型转换 double"测试钉住不变量。 +- **验证**：全模块 **199/199 绿** + checkstyle 0；3-lens 净室（session-safety / key-scope-composition CLEAN，iron-rules 仅上述 mutation 项已修）。**e2e 待集群**。详见 [`plan-doc/perf-hotpath-jdbc/README.md`](../perf-hotpath-jdbc/README.md)。 + +### 三、hive 写后读一致性 — 🚫 前提被推翻，**非 bug，不做** +- 调研文档称"缓存只由 REFRESH/TTL 失效、写路径不失效 → 存在 TTL 有界的读旧窗口"。**两个 hive agent 独立证伪，且逐行亲验**：每次 `hms` INSERT 提交后，`PluginDrivenInsertExecutor.onComplete → doAfterCommit → RefreshManager.handleRefreshTable → refreshTableInternal` 在协调 FE 上**同步全表失效** `connector.invalidateTable` = `CachingHmsClient.flush` + `HiveFileListingCache.invalidateTable` + `partitionViewCache.invalidateTable`（RefreshManager.java:242/246-248 标 "FIX-4"；HiveConnector.java:353-371），并写 refresh 编辑日志供 follower 回放。 +- **同 FE/同 session 写后读到的就是新数据**，无 TTL 窗口。唯一残留 = 跨 FE 编辑日志回放毫秒级窗口（通用 Doris 多 FE 行为，非 hive 专有、连接器侧不可修）+ 过度失效（每次 INSERT 全表刷，属 fe-core 侧 perf 非正确性）。owner 拍板**关闭此项、记为非 bug**。又一次印证记忆 `execution-blueprint-overestimates-recon-first`。 + +### 收尾 +- **下一步（伞形）**：owner 点名的三项已收（两做一证伪）。剩 = trino P2 纯 CPU 清理（未点名 + 受"禁加 L1 缓存"硬约束，暂不排期）+ 各连接器 e2e 统一补（需集群）+ iceberg 5 缓存全量收敛（远期，PR-3 已只做安全 ttl 去重）。 diff --git a/plan-doc/connector-cache-unification/tasklist.md b/plan-doc/connector-cache-unification/tasklist.md new file mode 100644 index 00000000000000..be32cb9d14eb46 --- /dev/null +++ b/plan-doc/connector-cache-unification/tasklist.md @@ -0,0 +1,114 @@ +# Task List — 连接器缓存框架统一 (connector cache unification) + +> **伞形进度总览（program 级）**。本文件只追踪 **workstream 状态 + owner 决策**，**不复制分析**。 +> 权威分析：[`00-research-report.md`](./00-research-report.md)（§3 矩阵 / §5 路线图 / §6 授权 / §7 反指 / §8 决策 / §9 workspaces） +> · [`connectors/*.md`](./connectors/)（7 份 per-connector 审计 + 对抗复核 verdict） +> · [`data/connector-audits.json`](./data/connector-audits.json)（结构化审计 + `verify.corrections`）。 +> 逐连接器的 `PERF-NN` 细粒度勾选在**各自兄弟空间** `plan-doc/perf-hotpath-/tasklist.md`，不在本文件。 +> **ID 一旦分配永不复用**；删除标 `[deleted YYYY-MM-DD <原因>]` 保留占位。 +> 状态图例：⏳ 待启动 · 🚧 进行中 · ✅ 完成 · ❌ 阻塞 · 🔬 复核中 · 🅿 待用户拍板 · 🚫 反指/不立项 + +--- + +## 0. 待拍板决策（程序启动前置 — 见 HANDOFF「下一步」） + +> ✅ **已于 2026-07-23 拿到 owner 签字**（下方「用户拍板」列），**并在设计定稿后二次确认**（见下）。 +> **设计已定稿** → [`designs/foundation-design-FINAL.md`](./designs/foundation-design-FINAL.md)（11-agent 只读设计调研 + 3 路对抗评审，全程逐符号核对 HEAD）。 +> ⚠ **D4 重大更正**：原选"提炼进 fe-core"，但设计侦察**推翻前提**——per-statement memo 底座**早已存在于 `fe-connector-api`**（`ConnectorStatementScope.computeIfAbsent` / `ConnectorSession.getStatementScope`；iceberg、hudi 均依赖该层、均**不**依赖 fe-core）。故 helper 只是该层**一个小静态方法**（`ConnectorStatementScopes.resolveInStatement`，统一 key 约定）→ **fe-core 零改动、铁律 A 根本不碰**。owner 2026-07-23 **二次确认接受此复读**、并**删掉"往 fe-core 塞"的备选**（三方评审一致：签字了但可避免的 fe-core 改动正是不必要改动混入主干的方式）。 +> **结论：本轮整套底座（A 通用缓存封装 + B 语句作用域 helper + C 门禁）+ hudi/mc/es 消费 = 全程 0 行 fe-core 改动。** iceberg 本轮就改挂（owner 确认，真正"先建底座"）。 +> ⚠ **D2 后续更正（owner 2026-07-24 拍板）**：D2 里"iceberg **5 个手写缓存**上收为通用封装"这一子项**经动码前重侦察改为延后**——语句作用域 helper 的 iceberg 改挂（B 部分）**已完成**（commit `ae8c925074d`），但 **5 个手写缓存的全量收敛延后**：收敛零功能收益、改动面宽（~19 文件/重写 ~37 测试），框架已被 3 连接器分区视图缓存证明可用、hudi/mc/es 不依赖它（Trino 亦"共享原语+各连接器自持缓存"）。本轮只落地**安全 DRY**：`CacheSpec.ofConnectorTtl` 折叠"`ttl≤0` 禁用"契约、6 处三元式改调（见 [`progress.md`](./progress.md) "2026-07-24 (2)"）。收敛出现真实需要再上收。 + +| ID | 决策 | 报告建议（默认） | 用户拍板（2026-07-23，含设计后二次确认） | 状态 | +|---|---|---|---|---| +| **D1** | **范围**：本轮只打 hudi，还是把 mc `MC-1`、es `ES-F1/F2` 一并纳入? | hudi 单独立项 + mc/es 各一小 PR；jdbc/paimon/trino P2 进 backlog | ✅ **采纳默认**：hudi + mc + es 都做（各自独立 PR），jdbc/paimon/trino 延后 | ✅ | +| **D2** | **排序**：先建共享底座泛化 vs per-connector-first? | per-connector-first（避免投机抽象，出现第 2 个消费者再上收） | ✅ **否决默认 → 先建共享底座**：把 iceberg 5 个手写缓存上收为通用封装（升格已存在的 `ConnectorPartitionViewCache`）；**iceberg 本轮就改挂** | ✅ | +| **D3** | **门禁通用化**：`check-authz-cache-sharding.sh` 现只盯 iceberg，现在就改成扫描任意 `SUPPORTS_USER_SESSION` 连接器吗? | (b) 延后无当下风险 / (a) 前瞻更稳 —— 交用户选 | ✅ **选 (a) 现在就通用化**，且**评审重设计**：从"扫主类带标记字段"改为"**模块内扫缓存构造点 + 断言按用户凭证置空**"（原方案有 BLOCKER 漏洞：iceberg 缓存字段也在 metadata 类上、hive 网关无标记缓存）。**⚠ 2026-07-24 撤销**：门禁经两轮对抗净室复审证伪——shell 无法结构化验证"缓存在每用户模式下是否真置空"（须读懂任意 Java 布尔/多行语法：复合 `&&`/`||`、多行 lambda、字符串字面量、嵌套三元式；误报挡合法构建、漏报放走泄漏，每修冒新边界）。置空正确性**已由运行时 `IcebergConnectorCacheTest` 证明**。owner 拍板**删掉整个门禁**（脚本+自测+pom 执行）+ 在 `IcebergConnector.java` 加显式 `ATTN` 注释兜底。见 [`progress.md`](./progress.md) "2026-07-24 (5)" | 🚫 撤销（删门禁改 ATTN） | +| **D4** | **共享 helper**：投入 fe-core 共享的"经 statement scope 解析表" helper 吗? | 不提炼；让 hudi 在自己模块内 memo（碰铁律 A，Rule 2） | ✅ **设计后二次确认：不动 fe-core** —— helper 放 `fe-connector-api`（memo 底座已在该层），iceberg + hudi 共用；**铁律 A 不碰**。原"提炼进 fe-core"作废；maxcompute 的"往 fe-core funnel 包一层"备选(B2)一并**删除**（评审三方否决：改到本轮不测的 jdbc/paimon/trino/hive 调用次数契约） | ✅ | + +--- + +## 1. Workstream 总览（at-a-glance） + +> 优先级 = 报告 §5/§9。**唯一 loop-amplified（iceberg-式）P1 真缺口 = hudi**；mc/es 是 constant-factor（2–4x）P1 收尾；其余结构上反指再加重缓存。 + +| ID | 优先级 | 覆盖发现 | 主题（一句话） | 目标兄弟空间 | 依赖 | 状态 | +|---|---|---|---|---|---|---| +| **WS-HUDI** | **P1 旗舰** | HD-P01/02/03/04/05 | 缓存最薄连接器：metaClient 每 pass 重建 ~5x、schema 3x、裸 `ThriftHmsClient` → 每语句投影 memo + HMS 缓存（fresh 拆分+REFRESH）| `plan-doc/perf-hotpath-hudi/` | D1 | ✅ round-1 完成：文档 + HMS缓存(183绿) + 旗舰memo(两块独立 per-statement memo，commit `26690775c81`)；round-2 延后(跨查询缓存 H04 等) | +| **WS-MC** | P1（小） | MC-1（+MC-2） | 每次 handle 解析冗余 ODPS `tables().exists()` 远程探测 → metadata 内 `Map<(db,table),Handle>` + 去冗余探测 | `plan-doc/perf-hotpath-maxcompute/` | D1 | ✅ round-1 完成 commit `58daadd10e0`（per-statement CHM handle memo，present-only；120 测试绿 + 两处变异验证 + 净室复审；e2e 待集群） | +| **WS-ES** | P1（小） | ES-F1/F2（+ES-F3） | `fetchMetadataState` 2x/查询 + mapping 重取 2x → per-scan hoist + mapping/field-context 承载决策 | `plan-doc/perf-hotpath-es/` | D1 | ✅ round-1 完成（owner 拍板"三件全做"）commits `7d74ba1161b`（per-scan hoist + schema memo）+ `7466b354901`（cross-path mapping via 每语句作用域，0 fe-core）；每语句 getMapping/shard/node 各→1；94 测试绿 + 变异 + 净室复审；e2e 待集群 | +| **WS-DOC** | 低（doc-only） | 陈旧注释一批 | 修陈旧注释 | 随手做 | — | ✅ 完成（2026-07-24）：原点名的 HMS-flip 注释**早已在前几轮更新**（现"Live since the hms flip"/`no consumers yet` 已消/`getTableHandle` 有真调用方）；侦察另发现并已清 **iceberg+maxcompute 写/事务/存储过程**的"未进 SPI_READY_TYPES/dormant 直到翻闸"陈旧注释（9 文件、纯注释、checkstyle 0）；hudi 增量关系是**真休眠**（正确留存）。见 [`progress.md`](./progress.md) "2026-07-24 (6)" | +| **WS-P2** | P2 | PA-1 ✅ · HP-1/HP-2 ✅ · hive 写后读=非bug 🚫 · TRINO-H1/H2/H3 ⏳ | owner 2026-07-24 点名三项：paimon 分区去重（`59b65912104`）+ jdbc 两处取列去重（`7df22cd1c71`）已落地；hive 一致性经侦察证伪（fe-core 提交后已同步全表失效）；trino 纯 CPU 未排期 | `perf-hotpath-{paimon,jdbc}/` | owner 2026-07-24 | 🚧 部分完成 | + +> **共享底座泛化（deferred，绑 D2/D3）**：iceberg 3 个格式中立缓存（table-handle / comment / file-format）上收为 `ConnectorXViewCache` 泛型封装、`check-authz-cache-sharding.sh` 通用化 —— **仅在出现第 2 个消费者时启动**，当前不投机抽象。 + +--- + +## 2. Workstream 详情 + +> 详情只记「交付概要 + 约束 + 依赖 + 权威指针」，不复制病灶分析（在报告/审计/JSON 里）。 + +### [ ] WS-HUDI — P1 旗舰（HD-P01/02/03/04/05） +- **权威**：报告 §5.1 + §9；[`connectors/hudi.md`](./connectors/hudi.md)；JSON hudi 节（含 3 条 `verify.corrections`）。 +- **交付概要**（报告 §9）： + 1. **per-statement resolved-metaClient + schema memo**，由 `HudiConnectorMetadata` 与 `HudiScanPlanProvider` 经同一 `ConnectorStatementScope` 共享 → 杀 `HD-P01`（metaClient 5–6x→1x）+ `HD-P02`（schema 4x→1x）。**旗舰**。 + 2. **包 `CachingHmsClient`**（一行 parity，仿 `HiveConnector.wrapWithCache`）→ 修 `HD-P04`（裸 `ThriftHmsClient`）。 + 3. **cross-query metaClient/table 缓存 + `(table,instant)`-keyed 分区缓存**（用 `ConnectorPartitionViewCache`）→ 修 `HD-P03`（MTMV/SHOW PARTITIONS 重枚举）。 + 4. **per-scan hoist**（metaClient/schema/storage-config/uri-normalizer 各一次）→ 修 `HD-P05`。 +- **前置**：`fe-connector-hudi` 目前**零 `fe-connector-cache` 依赖** → 先改 pom 引入工具箱（见 HANDOFF build 坑 5）。 +- **约束**：**无 authz 工作**（hudi 单一 catalog 身份、非 session=user，名字为 key 缓存安全）；**无写事务工作**（只读，`beginTransaction` throws）。cross-query metaClient/分区缓存须 **TTL + REFRESH 有界**（freshness）。 +- **⚠ 复核先行**：`HD-P03` 对**过滤查询已去重到 ~1 次**（真放大在 fe-core 多次独立调 `listPartitions*` / 无过滤扫 / 一次 MTMV refresh 4–6 次）；`HD-P01` 无条件下界 ~3–4x（含条件站点达 5–6x）—— 动码前按 HEAD 重侦察确认乘数。 +- **依赖**：D1（是否本轮）。旗舰、先做、收益最大。 + +### [x] WS-MC — P1 小（MC-1，+MC-2）✅ round-1 完成 commit `58daadd10e0`（详见 `plan-doc/perf-hotpath-maxcompute/`） +- **权威**：报告 §5.1 + §9；[`connectors/maxcompute.md`](./connectors/maxcompute.md)；JSON maxcompute 节。 +- **交付概要**：在**已经是每语句一个**的 `MaxComputeConnectorMetadata` 实例内加 `Map<(db,table),Handle>`，并对已解析读路径去掉多余 ODPS `tables().exists()` 远程探测 → `MC-1` k×/语句 → 1×/语句；顺带收 `MC-2`（lazy `Table` reload）。低风险高杠杆。 +- **约束**：连接器侧改动；无 authz（静态 AK/SK 单身份）、无写事务共享需求（`MaxComputeConnectorTransaction` 非-MVCC）。 +- **依赖**：D1。可与 WS-ES 并行。 + +### [x] WS-ES — P1 小（ES-F1/F2/F3）✅ round-1 完成 commits `7d74ba1161b` + `7466b354901`（详见 `plan-doc/perf-hotpath-es/`） +- **权威**：报告 §5.1 + §9；[`connectors/es.md`](./connectors/es.md)；JSON es 节（含 ES-F2 的 ADJUSTED 更正）。 +- **交付概要**： + - `ES-F1`：per-scan hoist `EsMetadataState`（`fetchMetadataState` 每查询 2 次 → 1 次），provider 已是每 scan-node 单例，加 `(index,columnNames)` 字段 memo，零 staleness（同语句）。 + - `ES-F2`：**注意不是"零新增缓存直接复用 schema"**——fe-core `ExternalSchemaCache` 只存解析后列，`EsConnectorMetadata.getTableSchema` 丢弃了原始 mapping，而 `resolveFieldContext` 需要它。消除它须**让 `ConnectorTableSchema` 携带 field-context** 或**加 per-statement/cross-query mapping 缓存**——立项时定承载方式。 + - `ES-F3`（P2）：在每语句一个的 metadata 上 memo schema。 +- **⚠ 约束（硬）**：**ES shard routing 必须留 per-statement，绝不能 cross-query 缓存**（ES rebalance/refresh 模型）。 +- **依赖**：D1。可与 WS-MC 并行。 + +### [ ] WS-DOC — doc-only（随手可做） +- **权威**：报告 §4(4) + §9；JSON 各连接器 `migrationStatus.notes` / `verify.corrections`。 +- **交付概要**（调研时快照行号，执行时以 grep 为准）： + - `ConnectorPartitionViewCache` 类 javadoc "NO consumers yet" → 已有 ≥2 消费者（hive、paimon）。 + - hive：`CachingHmsClient.java:85-88`、`HiveFileListingCache.java:72-74`、`HiveConnector.wrapWithCache:667-668`、`HiveConnector.java:118,126-127`（sibling 字段注释）的 "Dormant / hms is not in SPI_READY_TYPES"。 + - hudi：`HudiConnectorMetadata.java:391`、`HiveConnectorMetadata.java:410,1802,1942,1972,2007` 的 "dormant until hms enters SPI_READY_TYPES / today getTableHandle is never called"。 + - maxcompute：`beginTransaction:332`、`MaxComputeWritePlanProvider:74` 的陈旧注释。 + - 起因：`#65473`/`6e521aa64b2`（2026-07-16）把 hms 翻为 live 但没更新这批 class 注释。**纯 doc，运行时无害**。 +- **依赖**：无。可任何时候随手做（甚至可并进 WS-HUDI 的 hudi 部分）。 + +### [~] WS-P2 — owner 2026-07-24 点名三项：paimon ✅ / jdbc ✅ / hive = 非bug 🚫；trino 未排期 +- **权威**：报告 §5.2 + §7 + §9；执行明细见 [`progress.md`](./progress.md) "2026-07-24 (7)" + 兄弟空间 [`perf-hotpath-paimon`](../perf-hotpath-paimon/README.md) / [`perf-hotpath-jdbc`](../perf-hotpath-jdbc/README.md)。 +- **条目**： + - **PA-1**（paimon，P2/CPU）✅ commit `59b65912104`：`listPartitionNames/Values` 绕过 `partitionViewCache` → 抽 `cachedPartitions` 收敛三入口。owner 拍板"走缓存"（SHOW PARTITIONS/`partition_values()` 新鲜度→24h TTL+REFRESH，与裁剪路径/Trino 自洽；与 hive 故意实时不一致但只读无写后读问题）。26 测试 + 3-lens 净室。**e2e 待集群**。 + - **HP-1/HP-2**（jdbc，P2）✅ commit `7df22cd1c71`：读侧 scan `getColumnHandles`×2 + `getTableSchema`、写侧 `buildInsertSql` 各自远程取列 → 统一经 `ConnectorStatementScopes.JDBC_COLUMNS` 每语句作用域记忆化原始列（读写自动共享，一个改动点修两处）。owner 拍板"语句作用域统一去重"。199 测试 + 3-lens 净室（修 mutation 不变量：`jdbcTypeToConnectorType` 原地改 allowNull）。**e2e 待集群**。 + - **hive 写后读一致性**（P2）🚫 **非 bug（侦察证伪）**：调研称"缓存只 REFRESH/TTL 失效、写不失效"错——每次 `hms` INSERT 提交后 `PluginDrivenInsertExecutor.doAfterCommit → RefreshManager.handleRefreshTable → connector.invalidateTable` 同步全表失效（CachingHmsClient/FileListing/partitionView），协调 FE 同 session 写后读到新数据；残留仅跨 FE 编辑日志回放毫秒窗口（通用 Doris 非 hive 专有）+ 过度失效（fe-core 侧 perf 非正确性）。owner 拍板关闭。印证 `execution-blueprint-overestimates-recon-first`。 + - **TRINO-H1/H2/H3**（trino，P2 / **仅 CPU 清理**）⏳：cold 3x `getTableHandle` / 2N `getColumnMetadata` / 2x `applyFilter`。**禁加 L1 缓存**（反指，见下）。**未排期**（owner 未点名 + 硬约束）。见 [`connectors/trino.md`](./connectors/trino.md)。 +- **剩余**：trino 纯 CPU（未排期）+ 各连接器 e2e 统一补（需集群）。 +- **⚠ owner 定调**：**本地 CPU（重复渲染 / 解析 / 对象构造）同样是真实性能开销**——"远程已被缓存挡住、只省 CPU"**不是延后理由**。见记忆 [`perf-local-cpu-not-only-remote-matters`]。 + +--- + +## 3. 反指 / 不立项（记录，勿重开 — 报告 §7） + +> 这些**结构上不该**再加 iceberg-式重缓存；记在此防止后续 session 按包名/惯性重新提案。 + +- **hive** 🚫（重缓存）：已 framework-aligned —— `CachingHmsClient`（表/分区名/分区/列统计 4 缓存）+ `HiveFileListingCache` + `ConnectorPartitionViewCache` + sibling memo + 写事务快照，无 P0/P1 重复远程加载缺口。`HMS-H4`（ACID `getAcidState` 未缓存）是 snapshot/write-id 依赖、legacy-parity 正确，**应保持不缓存**；`HMS-H6` belt-and-suspenders per-statement memo **不划算**（HMS `getTable` 远比 iceberg `loadTable` 便宜）。hive 唯一实活 = doc 修（→ WS-DOC）。 +- **trino** 🚫（**主动反指**）：bridge，元数据缓存/失效/split 枚举全委托嵌入式 Trino 连接器（自带 `CachingHiveMetastore` 等）。加 Doris 侧 table/handle 缓存会**双重缓存**并把 schema 跨外部 ALTER 冻结，重引 Trino 已解决的 stale-metadata bug 类；`TrinoTableHandle` 事务派生、transient/不可序列化。只做 P2 CPU 清理（H1/H2/H3）。 +- **paimon** 🚫（连接器级 table 缓存）：SDK `CachingCatalog`（默认开）已提供 tableCache/partitionCache/manifestCache，连接器又已恢复 3 个 legacy 语义缓存。加连接器级 table 缓存与 SDK 缓存重复。只有 `PA-1` 一个 P2 一致性清理值得做。 +- **jdbc** 🚫（连接器侧 table 缓存）：关系型透传，`planScan` 零远程 IO、恒一个 scan range，无 split/file/manifest/partition/snapshot 层；昂贵元数据（schema/columns/row-count/名录）已被 fe-core cross-query 缓存前置，连接池 per-catalog 单例。加连接器侧 table 缓存**无对象可缓存**。只有 `HP-1/HP-2` 两个 P2。 + +--- + +## 4. 已知全局事实（供各 workstream 复用，勿再各自重查） + +- **Layer-2 已通用、已承重、无需再做**：per-statement `ConnectorMetadata` funnel（`PluginDrivenMetadata.get`，key `"metadata:"+catalogId`，身份钉 fail-loud）对全部 7 个 `SPI_READY_TYPES = {jdbc, es, trino-connector, max_compute, paimon, iceberg, hms}` 一体适用，由 `tools/check-fecore-metadata-funnel.sh` 强制（0 exempt）。hudi 作为 hms sibling 骑同一 scope。 +- **共享底座 `fe-connector-cache` = 统一的天然归宿**：`MetaCacheEntry` / `CacheSpec`（`meta.cache...*`）/ `CacheFactory` / 泛型 `ConnectorPartitionViewCache`；已被 hive/hms/iceberg/maxcompute/paimon 采用，**hudi 尚未采用**（WS-HUDI 前置）。 +- **authz 现状**：8 连接器中**只有 iceberg-REST 声明 `SUPPORTS_USER_SESSION`** 并做 per-user 凭证 vending；其余 7 个均单一 catalog 身份 → 今天加名字为 key 的 cross-query 缓存都 authz-安全（但见铁律 4 的"结构性 vs 当前"警告 + D3）。 diff --git a/plan-doc/perf-hotpath-es/HANDOFF.md b/plan-doc/perf-hotpath-es/HANDOFF.md new file mode 100644 index 00000000000000..5342f5de8326fe --- /dev/null +++ b/plan-doc/perf-hotpath-es/HANDOFF.md @@ -0,0 +1,47 @@ +# 🤝 Session Handoff — perf-hotpath-es + +> 滚动文档：顶部「下一步」覆盖式更新；底部稳定区勿覆盖。 +> 完成明细进 `git log` + 本空间 `progress.md`；回填伞形 `plan-doc/connector-cache-unification/`。 + +--- + +# 🆕 下一步（覆盖区） + +## 当前状态（2026-07-24） + +- **Round 1 三件全做：✅ 完成**。commits `7d74ba1161b`（per-scan hoist + per-statement schema memo）、`7466b354901`（cross-path mapping via 每语句作用域，0 fe-core）。 +- 每语句远程 `~4× getMapping + 2× search_shards + 2× _nodes` → **1×/1×/1×**。全 `Es*` 94 测试绿 + checkstyle 0 + 三处变异验证 + 4-lens 净室复审（parity/concurrency/freshness HOLD，补分片-不入作用域门禁）。 +- 设计权威：[`designs/round-1-design.md`](./designs/round-1-design.md)；日志 [`progress.md`](./progress.md)。 + +## 之后 + +- **PERF-ES04**（低优先，热点触发）：`getTableHandle→existIndex` 的 `_mapping` GET 与首次 schema getMapping 同 endpoint，可再去重。 +- **死代码清理**（随手）：`EsConnectorMetadata.fetchMetadataState` 两个重载无调用者（被 provider 版取代），本轮保留、穿 session 行为中性。 +- **e2e**（需集群）：独立 es catalog 过滤 SELECT 断言远程计数 + 分片再平衡后新鲜度。 +- 伞形层面：mc/es 两小 PR 均落地 → 剩 P2 backlog（热点触发）+ 门禁通用化 + 陈旧注释清理 + 各连接器 e2e 统一补。 + +--- + +# 🧱 稳定区（勿覆盖） + +## 铁律（继承伞形） +1. **fe-core 源只出不进**：本轮 memo 在连接器侧（provider 字段 / metadata CHM）；F2 命名空间常量在 `fe-connector-api`（iceberg/hudi 既定模式），**0 fe-core**。 +2. **es 硬约束**：分片路由 + 节点拓扑**每 scan/每语句重解析、绝不跨查询缓存**（ES rebalance/refresh）；只有**原始 mapping**（语句内稳定）可进每语句作用域。`ShardRoutingNeverSharedViaScope` 门禁钉死。 +3. **authz**：es 单一 catalog Basic 凭证、无 per-user vending → 名字为 key 的 memo 安全。 +4. **parity 只减不改**：NONE scope / null session 下逐次加载（byte-identical）。 + +## build / 验证坑（继承伞形） +1. maven 绝对 `-f /fe/pom.xml`；`${revision}` → `-am`；F2 动 api → `-pl :fe-connector-api,:fe-connector-es -am`。 +2. 测试必加 `-Dmaven.build.cache.enabled=false`；别 `-q`；grep `BUILD SUCCESS`+`Tests run:`。 +3. checkstyle 绑 `validate`（`install`/`test` 都跑），扫 test 源。 +4. 后台跑直接 `mvn … >> log 2>&1`，读日志非通知 exit code。 + +## 并发探测（动码前） +- 本 worktree `find fe -newermt '-120 sec'`（滤 `/target/`）+ `pgrep -a -f maven`；`/mnt/disk1/gq` 是别的用户 worktree（独立 ~/.m2），不冲突。 +- 提交只精确 `git add` 本任务文件，禁 `git add -A`。 + +--- + +# 🔗 关系 +- **伞形**：[`plan-doc/connector-cache-unification/`](../connector-cache-unification/)（WS-ES 行）。 +- **模板**：[`perf-hotpath-maxcompute/`](../perf-hotpath-maxcompute/)（同期 mc）/ [`perf-hotpath-iceberg/`](../perf-hotpath-iceberg/) / [`perf-hotpath-hudi/`](../perf-hotpath-hudi/)。 diff --git a/plan-doc/perf-hotpath-es/README.md b/plan-doc/perf-hotpath-es/README.md new file mode 100644 index 00000000000000..b1ff91f913cc99 --- /dev/null +++ b/plan-doc/perf-hotpath-es/README.md @@ -0,0 +1,19 @@ +# 📦 任务空间 — fe-connector-es 热路径元数据抓取去重 + +> 连接器缓存框架统一的兄弟空间（伞形 = `plan-doc/connector-cache-unification/`，行 WS-ES）。 +> 镜像 `perf-hotpath-iceberg/` / `perf-hotpath-maxcompute/` 布局。协作规范沿用 `../AGENT-PLAYBOOK.md`。 + +## 一句话背景 +es **已 live、读路径为主、无写、无分区、单一凭证**，连接器自身**无扫描元数据缓存**。一条普通过滤 SELECT 约发 ~4× getMapping + 2× search_shards + 2× _nodes 远程往返（多为冗余）。**非 iceberg 那种 per-file 循环放大**——是常数倍（2–4×）冗余。硬约束：**分片路由/节点拓扑绝不跨查询缓存**（ES rebalance）。 + +## 本空间文件 +| 文件 | 用途 | +|---|---| +| [`HANDOFF.md`](./HANDOFF.md) | 下一步 + 稳定区规则（覆盖式更新） | +| [`tasklist.md`](./tasklist.md) | PERF-ESxx 勾选 + 状态 | +| [`progress.md`](./progress.md) | append-only 日志 | +| [`designs/`](./designs/) | per-round 设计蓝图（`round-1-design.md` = 当前） | + +## 流程（对齐 step-by-step-fix + iceberg 立项流程） +侦察（HEAD 重核）→ 设计 + 红队 → 实现（最小改动/守铁律/连接器侧）→ 验证（parity + build-count 守门 + 变异）→ 独立 commit → 更新 tasklist/HANDOFF/progress + 回填伞形。 +**行号信 HEAD 不信文档**（设计里 file:line 是 2026-07-24 快照）。 diff --git a/plan-doc/perf-hotpath-es/designs/round-1-design.md b/plan-doc/perf-hotpath-es/designs/round-1-design.md new file mode 100644 index 00000000000000..3d899466a7fdd3 --- /dev/null +++ b/plan-doc/perf-hotpath-es/designs/round-1-design.md @@ -0,0 +1,51 @@ +# Round 1 设计 — es 连接器元数据抓取去重（per-scan hoist + per-statement schema memo + cross-path mapping scope） + +> 兄弟空间 = `plan-doc/perf-hotpath-es/`（伞形 = `plan-doc/connector-cache-unification/`，行 WS-ES）。 +> 镜像 `perf-hotpath-iceberg/` / `perf-hotpath-maxcompute/` 布局。铁律：0 fe-core、纯连接器侧、parity 只减不改。 +> **行号是 2026-07-24 HEAD 侦察快照**（6-agent 只读侦察）。 + +## 1. 病灶（root cause） + +es 一条普通 `SELECT ... WHERE ...`（schema cache 暖态）对同一 ES 集群约发 **~4× getMapping + 2× search_shards + 2× _nodes/http** 远程往返（每次 OkHttp 10s read timeout），大部分冗余： + +- **ES-F1（P1，2×）**：`EsScanPlanProvider.planScan`（`:99`）与 `buildScanNodeProperties`（`:169`）各自完整 `fetchMetadataState`（`:273`→`EsMetadataFetcher.fetch` = getMapping + search_shards + _nodes）。二者同属**每 scan-node 单实例**的 provider（fe-core `ResolvedScanProvider` 按 currentHandle identity memo），却各抓一遍。 +- **ES-F3（P2，2×）**：`EsConnectorMetadata.getTableSchema`（`:85`）每次远程 getMapping；`getColumnHandles` 再调 `getTableSchema` → 每语句 2× 冗余 mapping。metadata 是每语句单实例，未 memo。 +- **ES-F2（P1，~2×，跨路径）**：schema 路径与 scan 路径**各自**远程 getMapping 同一份 index mapping、派生不同产物（列 vs field-context），且 fe-core 已缓存的 `ConnectorTableSchema` **丢弃原始 mapping**（`Collections.emptyMap()` properties），无法直接复用（复核 ADJUSTED：须让原始 mapping 有承载方）。 + +**硬约束**：分片路由 + 节点拓扑**必须每语句/每 scan 重解析、绝不跨查询缓存**（ES rebalance/refresh 模型）。**无写路径**（只读）、**单一凭证**（authz 安全）。 + +## 2. 修复（三件，按新鲜度分层） + +| 件 | 位置 | 机制 | 消除 | +|---|---|---|---| +| **F1** per-scan hoist | `EsScanPlanProvider` | 加 `private EsMetadataState memoizedState` 标量字段，`fetchMetadataState` 命中即复用；guard on `(index, columns)`（不同请求重抓）。**plain 字段**（ES 非 batch、单线程规划；注释标注若接 batch 须 volatile）。 | scan 侧 search_shards 2→1、_nodes 2→1、mapping 2→1 | +| **F3** per-statement schema memo | `EsConnectorMetadata` | 加 `Map schemaMemo`（CHM）按 index `computeIfAbsent`。read-only metadata，无失效。镜像 maxcompute 每语句 memo。 | schema 侧 getMapping 2→1 | +| **F2** cross-path raw-mapping scope | `ConnectorStatementScopes`(api) + `EsStatementScope` + 两路 getMapping 包裹 | 新命名空间 `ES_INDEX_MAPPING`；`EsStatementScope.sharedIndexMapping(session, index, loader)` 经 `resolveInStatement` 把**原始 mapping 字符串**存每语句作用域（key=`es.index_mapping:catalogId:default_db:index:queryId`）；schema 路径与 `EsMetadataFetcher.fetchMapping` 都走它；session 穿到 fetcher/`fetchMetadataState`/`buildScanNodeProperties`。**只共享原始 mapping（语句内稳定）；分片/节点绝不入作用域**。 | 跨路径 getMapping 2→1 | + +**总账**：`~4× getMapping + 2× search_shards + 2× _nodes` → **1× getMapping + 1× search_shards + 1× _nodes**（每语句/每 scan）。**0 fe-core**（`ConnectorStatementScopes` 加命名空间常量在 `fe-connector-api`，是 iceberg/hudi 已用的既定模式）。 + +## 3. 关键取舍（诚实） + +| # | 取舍 | 决定 | 理由 | +|---|---|---|---| +| A | F2 原始 mapping 承载方式（甲 塞 ConnectorTableSchema.properties / 乙 每语句作用域 / 丙 不做） | **乙**（owner 拍板"三件全做"） | 甲=污染 schema 缓存 + 喂回 scan 须改 fe-core（碰铁律 A，坑）；乙=0 fe-core、只需 session 穿一层（连接器内部）；两路都收 `session`，经既存作用域机制共享。 | +| B | F1 memo 键：标量+guard vs Map | **标量 + `(index,columns)` guard** | 一 provider = 一 scan-node = 一 (index,columns)；guard 令复用不同请求时重抓（`testDifferentIndexes` 绿）。分片随 provider per-scan GC，新鲜。 | +| C | F1 线程安全：plain vs volatile | **plain** | ES 从不进 batch mode（不 override `supportsBatchScan`）→ planScan/buildScanNodeProperties 同步单线程；fe-core 兄弟 memo 字段亦 plain。注释标注 batch 须 volatile。 | +| D | 只共享原始 mapping，不共享 field-context/分片 | **只 mapping** | field-context 依赖投影列（每 scan 不同）；分片/节点新鲜度敏感（硬约束）。两路各自从共享 mapping 派生自己的产物。 | +| E | metadata 死方法 `fetchMetadataState`（无调用者）threading session | **保留**（穿 session 使其编译且语义正确） | 纯连接器死代码、非 SPI；删属 scope creep（Rule 3）；穿 session 行为中性。**记为未来可清理**。 | + +## 4. 验证 + +- **守门单测**（`EsScanPlanProviderTest`，复用 `CountingRestClient`，无 Mockito、离线）： + - F1：`ShareOneFetch`（一 scan-node planScan+props → mapping/shard/node 各 1）；`DifferentIndexesFetchSeparately`（一 provider 两 index → 2）；`SeparateProviderInstancesEachFetch`（两 provider 同 index → 2，per-scan 新鲜）；`DifferentColumnsRefetch`（不同投影重抓）。 + - F3：`SchemaMemoizedPerStatement`（getTableSchema+getColumnHandles → getMapping 1）；`SchemaFreshPerStatement`（新 metadata → 重抓）。 + - F2：`MappingSharedAcrossSchemaAndScanPathsWithinStatement`（共享 live scope → getMapping 1 across 两路）；`ScopedMappingNotSharedAcrossStatements`（不同 scope → 2）；**`ShardRoutingNeverSharedViaScope`（两 provider 共享 live scope → shard/node 各 2、mapping 1，钉死硬约束）**。 +- **回归**：`install -pl :fe-connector-api,:fe-connector-es -am -Dtest='Es*,ConnectorStatementScopes*'` → BUILD SUCCESS + 全绿（92+ 测试）+ checkstyle 0。 +- **Rule 9 变异**（三处，均逐一验证变红）：去 F1 store → `ShareOneFetch` 红；F3 `schemaMemo.clear()` → `SchemaMemoized` 红；F2 fetcher 绕作用域 → `MappingSharedAcross` 红。 +- **净室对抗复审**（4 lens + verify）：parity/concurrency/freshness `PARITY_HOLDS`；唯一 CONFIRMED = "无门禁钉分片不入作用域" → **已补 `ShardRoutingNeverSharedViaScope`**；nits（List.equals 顺序=错失优化非错误、死代码 E）已评估接受。 +- **e2e**：需集群，本地未跑；留标注（独立 es catalog 的过滤 SELECT 断言 getMapping/search_shards/_nodes 计数 + 分片再平衡后新鲜度）。 + +## 5. commit（两笔） + +1. `[perf](catalog) fe-connector-es: hoist per-scan metadata state and memoize schema per statement`（F1+F3）。 +2. `[perf](catalog) fe-connector-es: share one index mapping fetch per statement across schema and scan paths`（F2，含 api 命名空间 + EsStatementScope）。 diff --git a/plan-doc/perf-hotpath-es/progress.md b/plan-doc/perf-hotpath-es/progress.md new file mode 100644 index 00000000000000..ea8d92ae868ca0 --- /dev/null +++ b/plan-doc/perf-hotpath-es/progress.md @@ -0,0 +1,18 @@ +# Progress — perf-hotpath-es（append-only） + +## 2026-07-24 — Round 1 三件全做（commits `7d74ba1161b` F1+F3、`7466b354901` F2） + +**做了什么**：es 连接器一条 SELECT 对同一索引重复远程抓 mapping/分片/节点。三件收敛，按新鲜度分层： +1. **per-scan hoist**（ES-F1）：`EsScanPlanProvider` 加 `EsMetadataState memoizedState` 标量字段（guard on (index,columns)，plain——ES 非 batch 单线程），`planScan`/`buildScanNodeProperties` 共用一次 fetch。分片随 provider per-scan 新鲜、不跨查询。 +2. **per-statement schema memo**（ES-F3）：`EsConnectorMetadata` 加 `Map`（CHM）`computeIfAbsent`；`getColumnHandles→getTableSchema` 折叠为每语句 1 次。镜像 maxcompute。 +3. **cross-path mapping**（ES-F2，owner 拍板走"每语句共享作用域"）：`fe-connector-api` 加命名空间 `ES_INDEX_MAPPING`（iceberg/hudi 既定模式），`EsStatementScope.sharedIndexMapping` 把**原始 mapping 字符串**存每语句 `ConnectorStatementScope`；schema 路径与 `EsMetadataFetcher.fetchMapping` 都走它、各自派生自己产物；session 穿到 fetcher/provider fetch 路径。**只共享原始 mapping；分片/节点绝不入作用域**。 + +**总账**：每语句 `~4× getMapping + 2× search_shards + 2× _nodes` → **1×/1×/1×**。**0 fe-core**（`git diff fe/fe-core` 空）。 + +**流程**：6-agent HEAD 只读侦察（确认 provider per-scan 单例、ES 非 batch→plain 字段、承载 mapping 须原始 JSON、`ConnectorStatementScopes` 已预留 es 命名空间）→ 设计（`designs/round-1-design.md`）→ 向 owner 中文讲清三件 + F2 三条承载路（甲坑/乙净/丙延），**owner 选"三件全做（F2 走乙）"** → 实现（分两笔：F1+F3、F2）→ 验证。 + +**验证**：全 `Es*` **94 测试绿** + checkstyle 0；`EsScanPlanProviderTest` 12 例。**Rule 9 变异三处**：去 F1 store → `ShareOneFetch` 红；F3 `schemaMemo.clear()` → `SchemaMemoized` 红；F2 fetcher 绕作用域 → `MappingSharedAcross` 红（各只对应门禁红、其余绿）。**净室 4-lens + verify**：parity/concurrency（含 schemaMemo↔scope 嵌套 computeIfAbsent 不同 map 无重入）/freshness 全 `PARITY_HOLDS`；唯一 CONFIRMED = "无门禁钉分片不入作用域" → **补 `ShardRoutingNeverSharedViaScope`**（两 provider 共享 live scope → shard/node 各 2、mapping 1）；nits（List.equals 顺序=错失优化非错误、死代码 threading）评估接受。 + +**未做/延后**：ES-F4（`existIndex` 的 `_mapping` GET 去重，低优先）；死代码 `EsConnectorMetadata.fetchMetadataState` 清理（本轮保留）。**e2e 需集群本地未跑**（独立 es catalog 过滤 SELECT 断言 getMapping/search_shards/_nodes 计数 + 分片再平衡新鲜度），留标注。 + +**下一步（伞形）**：WS-ES 完成 → mc/es 两小 PR 均落地。剩 P2 backlog（热点触发）+ 门禁通用化 + 陈旧注释清理 + 各连接器 e2e 统一补。 diff --git a/plan-doc/perf-hotpath-es/tasklist.md b/plan-doc/perf-hotpath-es/tasklist.md new file mode 100644 index 00000000000000..ee39bac710ddca --- /dev/null +++ b/plan-doc/perf-hotpath-es/tasklist.md @@ -0,0 +1,22 @@ +# Task List — perf-hotpath-es + +> ID 一旦分配永不复用。状态：⏳待启动 · 🚧进行中 · ✅完成 · 🔬复核中 · 🅿待拍板 · 🚫不立项 +> 权威分析：伞形 `plan-doc/connector-cache-unification/connectors/es.md` + 本空间 `designs/round-1-design.md`。 + +## Round 1（全部完成 2026-07-24；owner 拍板"三件全做"） + +| ID | 主题 | 覆盖发现 | 依赖 | 状态 | +|---|---|---|---|---| +| **PERF-ES01** | per-scan hoist：`EsScanPlanProvider` memo `EsMetadataState`（标量 + (index,columns) guard，plain 字段/非 batch）→ scan 侧 mapping/shard/node 各 2→1 | ES-F1（P1） | — | ✅ commit `7d74ba1161b` | +| **PERF-ES02** | per-statement schema memo：`EsConnectorMetadata` 按 index memo `ConnectorTableSchema`（CHM，read-only 无失效，镜像 maxcompute）→ schema 侧 getMapping 2→1 | ES-F3（P2） | — | ✅ commit `7d74ba1161b` | +| **PERF-ES03** | cross-path mapping：`ConnectorStatementScopes.ES_INDEX_MAPPING` + `EsStatementScope` 把原始 mapping 存每语句作用域，schema/scan 两路共享 → 跨路径 getMapping 2→1（分片/节点绝不入作用域） | ES-F2（P1） | — | ✅ commit `7466b354901`（0 fe-core） | + +> 三件总账：每语句 `~4× getMapping + 2× search_shards + 2× _nodes` → **1×/1×/1×**。94 测试绿 + checkstyle 0 + 三处变异验证 + 4-lens 净室复审（parity/concurrency/freshness HOLD；补 `ShardRoutingNeverSharedViaScope` 钉硬约束）。 + +## 延后 / 不立项（记录，勿抢跑 — 伞形审计 §7） + +| ID | 主题 | 覆盖发现 | 状态 | +|---|---|---|---| +| PERF-ES04 | `getTableHandle→existIndex` 的 `_mapping` GET（与首次 schema getMapping 同 endpoint，可再去重）；轻量、RowCountCache 兜底 | ES-F4（P2） | ⏳（低优先，热点触发） | +| — | 跨查询 shard 缓存 / authz 隔离 / write-txn / partition·format·manifest·comment 缓存 | — | 🚫 不适用（ES rebalance 硬约束 / 只读 / 单凭证 / 非分区 / 格式常量 / 无 manifest） | +| — | 死代码 `EsConnectorMetadata.fetchMetadataState`（无调用者，被 provider 版取代） | — | ⏳（可随手清理；本轮保留，穿 session 行为中性，见设计 §3-E） | diff --git a/plan-doc/perf-hotpath-hudi/HANDOFF.md b/plan-doc/perf-hotpath-hudi/HANDOFF.md new file mode 100644 index 00000000000000..db81b49bbfdc36 --- /dev/null +++ b/plan-doc/perf-hotpath-hudi/HANDOFF.md @@ -0,0 +1,37 @@ +# 🤝 HANDOFF — perf-hotpath-hudi（连接器缓存统一的旗舰连接器） + +> 覆盖式更新，只留下一个 session 必需的上下文。完成明细进 git log + `progress.md`。 +> 伞形空间 = `plan-doc/connector-cache-unification/`（本兄弟空间进度回填其 tasklist WS-HUDI 行）。 + +## 开场读序 +1. Read 本 HANDOFF +2. Read [`designs/round-1-memo-hms-cache-design.md`](./designs/round-1-memo-hms-cache-design.md) ← **本轮完整设计蓝图**（3 块 + 红队 3 修正 + 验证） +3. Read `tasklist.md` ← 勾到哪、下一个 PERF 是谁 +4. 需要发现细节 → 伞形空间 `connectors/hudi.md` / `data/connector-audits.json` hudi 节 + +## 当前状态（2026-07-24，Round 1 全部完成+提交+验证） +- 调研 + 设计 + 红队 + owner 范围拍板全部完成；**Round 1 三块全部落地**。 +- **✅ Piece A（文档清理，PERF-H01）** commit `1cb0f95f8ed`（4 处 stale "dormant hms" 注释改词，纯 doc）。 +- **✅ Piece C（HMS 缓存层，PERF-H02）** commit `e26ab33b001`（wrap `CachingHmsClient` + `collectPartitions` fresh/cached 拆分 + `HudiConnector.invalidate*` flush；无 pom 改动；`HudiConnectorHmsCacheTest` 8 测试；183 测试 0 失败）。 +- **✅ Piece B（旗舰 memo，PERF-H03）** commit `26690775c81`——**两块独立 per-statement memo**（最新 schema + 最新 instant），各 loader = 现成方法（`getSchemaFromMetaClient(path,null)` / `latestInstant`），逐字节等价、零耦合。0 fe-core、无 pom、扫描规划器/at-instant 未碰；`HudiStatementMemoTest` 4 测试（去重×2 + 两 memo 独立 + NONE/null）；全模块 0 失败。详见 [`progress.md`](./progress.md) "2026-07-24 (3)"。 + +## ⚠ 重要教训（动 hudi 前必读，勿再照抄蓝图） +- 蓝图 [`designs/round-1-pieceB-flagship-impl-notes.md`](./designs/round-1-pieceB-flagship-impl-notes.md) 已被侦察**推翻多处**：`getScanNodeProperties` 在 `HudiScanPlanProvider`（异构 build 源 / 扫描线程 / 无鉴权包装，本轮不碰）；`HudiSchemaAtInstantTest` 不用改。 +- 蓝图的"变体二（schema+instant 合并一次 metaClient 构建）"虽实现过并全绿，但 4-agent 净室复审发现**两个 minor 固有耦合边角**（取 schema 多依赖 instant 可解析；取快照顺带读 schema），**owner 拍板退回两块独立 memo**（零耦合，见 progress "2026-07-24 (3)"）。这是"执行蓝图常高估 / 错位、动码前必按 HEAD 重侦察"的又一实证。 + +## 下一步（Round 1 完成，选一） +- **转 mc/es 两个小连接器**（伞形 `WS-MC` / `WS-ES`），各独立小 PR；或 +- **hudi Round 2 延后项**：`PERF-H04`（跨查询 metaClient + `(table,instant)` 分区缓存，配更宽失效设计）、`PERF-H05/06/07`（见 [`tasklist.md`](./tasklist.md)「延后」）。 +- **e2e（PERF-H08，需集群，本轮只给规格）**：异构 hudi+iceberg+hive 单-hms-catalog + 独立 hudi-on-HMS，跑分区 / 时间旅行 / schema 演进 parity。 + +## 验证（每块后） +- `mvn install -pl :fe-connector-api,:fe-connector-hudi -am -Dmaven.build.cache.enabled=false`（install 非 test）→ 抓 `BUILD SUCCESS`+`Tests run:`。 +- 旗舰守门：`HudiStatementMemoTest`（真 scope 下 loader=1 去重 + 两 memo 独立 + NONE/null 每次加载）。 +- e2e 本地跑不了（external+集群），异构三向新用例只给规格。 + +## 稳定区（勿覆盖） +- **build 坑**：绝对 `-f /fe/pom.xml`；`-am`；**install 非 test**（hudi 经 hms 依赖 shade jar）；`-Dmaven.build.cache.enabled=false`；别 `-q`；别 `nohup &` 套后台，读日志 `BUILD SUCCESS` 行。 +- **无 pom 改动**（fe-connector-hms/api 已是直接依赖；Caffeine 3.2.3 已随 hudi-common 打包，**勿加 2.9.3**——nearest-wins 会降级）。 +- **fe-core 0 行**（`HUDI_METACLIENT` 在 fe-connector-api 非 fe-core）；遇"不得不碰 fe-core"停手交 review。 +- 提交只精确 `git add` 本任务文件（工作树大量无关 untracked，禁 `git add -A`）；`.git` 是文件（linked worktree）。 +- 并发探测：`find fe -newermt '-120 sec'`（滤 target）+ `pgrep -a -f maven`。 diff --git a/plan-doc/perf-hotpath-hudi/README.md b/plan-doc/perf-hotpath-hudi/README.md new file mode 100644 index 00000000000000..2d748b99cb0d89 --- /dev/null +++ b/plan-doc/perf-hotpath-hudi/README.md @@ -0,0 +1,19 @@ +# 📦 任务空间 — fe-connector-hudi 热路径缓存修复 + +> 连接器缓存框架统一的**旗舰连接器**兄弟空间（伞形 = `plan-doc/connector-cache-unification/`，行 WS-HUDI）。 +> 镜像 `plan-doc/perf-hotpath-iceberg/` 布局。协作规范沿用 `../AGENT-PLAYBOOK.md`。 + +## 一句话背景 +hudi 是所有 SPI 连接器里**缓存最薄**的一个：零跨查询元数据缓存、零语句内 metaClient/schema memo、HMS 客户端未包缓存层。一条普通过滤 SELECT 的规划里，同表 metaClient 重建 ~5 次、schema 重解析 3 次。iceberg 框架模式直接适用。 + +## 本空间文件 +| 文件 | 用途 | +|---|---| +| [`HANDOFF.md`](./HANDOFF.md) | 下一步 + 稳定区规则（每 session 覆盖式更新） | +| [`tasklist.md`](./tasklist.md) | PERF-Hxx 勾选 + 状态 | +| [`progress.md`](./progress.md) | append-only 日志 | +| [`designs/`](./designs/) | per-round 设计蓝图（`round-1-memo-hms-cache-design.md` = 当前） | + +## 流程（对齐 step-by-step-fix + iceberg 立项流程） +侦察（HEAD 重核）→ 设计 + 红队 → 实现（最小改动/守铁律/连接器侧）→ 验证（parity + build-count 守门）→ 独立 commit → 更新 tasklist/HANDOFF/progress + 回填伞形。 +**行号信 HEAD 不信文档**（设计里 file:line 是 2026-07-24 快照）。 diff --git a/plan-doc/perf-hotpath-hudi/designs/round-1-memo-hms-cache-design.md b/plan-doc/perf-hotpath-hudi/designs/round-1-memo-hms-cache-design.md new file mode 100644 index 00000000000000..997d49e4448518 --- /dev/null +++ b/plan-doc/perf-hotpath-hudi/designs/round-1-memo-hms-cache-design.md @@ -0,0 +1,104 @@ +# Round-1 设计 — Hudi 每语句投影 memo + HMS 缓存层（新鲜读拆分 + REFRESH 失效） + +> 基线 HEAD `f2e9706df5a`（动每个文件前仍按符号重 grep，行号会漂）。 +> 调研+设计+3 路红队：workflow `wf_3a0b9aca-966`（transcript 在 session subagents/workflows/）。 +> owner 已拍板范围：**旗舰 memo + 文档清理 + HMS 缓存层（按 hive 补齐 fresh/cached 拆分 + REFRESH 失效）**。 +> 铁律：fe-core 源只出不进；通用节点 connector-agnostic；freshness-敏感跨查询缓存必 TTL+REFRESH 有界。 + +--- + +## 0. 侦察确认（HEAD 核实的真实倍数） + +一条**带分区谓词的普通 SELECT**，单次规划 pass 对同一张表： +- **metaClient 重建 ~5 次**：3 处铁定（`planScan`@148、`getColumnHandles→getSchemaFromMetaClient`@802、`beginQuerySnapshot→latestInstant`@751）+ 1 处默认开（`getScanNodeProperties`@363，`!force_jni` 门控）；@incr 再 +1。每次远程读时间线。 +- **最新 schema 重解析 3 次**（`HudiConnectorMetadata`@820、`HudiScanPlanProvider`@188+@382）。 +- 分区列举：过滤查询已去重到 ~1 次（`resolvePartitions` 命中 `prunedPartitionPaths` 短路 @635-638）；放大在 MTMV。 +- `HudiConnectorMetadata` 字段仅 `{hmsClient, properties, metaClientExecutor, storageHadoopConfig}`（@154-162）→ **零 memo**；`HudiScanPlanProvider` 字段仅 `{properties, context}`（@107-108）。两者各自独立建 metaClient。 + +生命周期关键事实（自测已核）： +- `ConnectorStatementScopeImpl` **语句末关闭所有 AutoCloseable 缓存值**（pass 2 @82-90）。 +- `HoodieTableMetaClient`（本工作树编译的 1.0.2）只 `implements Serializable`，**非 AutoCloseable** → 放进作用域不会被关。但它可变（`reloadActiveTimeline`）、且 `planScan` 的**逐文件** schema_id resolver（@232-241）闭包捕获活 metaClient → 不宜跨 metadata→off-thread 扫描边界共享活对象。 +- `HoodieTableFileSystemView` **是** AutoCloseable → **绝不能** memo（会被 pass-2 关，后续 use-after-close）。 + +→ **memo 只存不可变投影，绝不存活 metaClient / fsView / 可变 Configuration。** + +--- + +## 1. 本轮三块（各自独立 commit） + +### Piece A — 陈旧注释清理（纯 doc，零风险，先做） + +`#65473`/`6e521aa64b2`（2026-07-16）把 hms 翻 live，但一批注释仍写"dormant until hms enters SPI_READY_TYPES / getTableHandle is never called"。删/改（动码前 grep 定位当前行）： +- 删：`HudiConnectorMetadata.java`（~392）、`HudiReadOnlyWriteRejectTest.java`、`HudiConnectorOwnsHandleTest.java` 里的过时从句。 +- 改词（非删）：`HiveConnectorMetadata.java`、`HiveConnectorMetadataFreshnessTest.java`（hms live + MVCC/MTMV freshness 已落地）。 +- `HudiScanPlanProvider.java`（~250）只删"while dormant"借口，**保留** @incr over-read 真实 caveat。 +- **勿动**：`HiveConnector.java`（~756）"dormant"=sibling 懒建从未构建（正确）、`HudiScanPlanProvider.java`（~216）"dormant"=BE history-schema-dict 轴（不同义，正确）。 + +### Piece B — 旗舰：每语句"不可变投影" memo（红队收紧后） + +**新增（0 fe-core）**： +1. `ConnectorStatementScopes`（fe-connector-**api**）加常量 `HUDI_METACLIENT = "hudi.metaclient"`（镜像 `ICEBERG_TABLE`；该类 javadoc 已预留该名）。**非 fe-core**。 +2. `HudiStatementScope.sharedTableFacts(session, db, table, loader)`（新，fe-connector-hudi）→ 委派 `ConnectorStatementScopes.resolveInStatement(session, HUDI_METACLIENT, db, table, loader)`。 +3. 连接器内投影 builder：`HudiMetaClientExecutor.execute` 内建 **一次** metaClient，导出不可变事实，**丢弃**活 metaClient。 + +**memo 值（不可变投影，key = `hudi.metaclient:catalogId:db:table:queryId`）**： +- `latestCompletedInstant`（String） +- 最新 schema：`List` + latest Avro `Schema` + `ResolvedInternalSchema`（字段 ID） +- `allHistoricalSchemas`（喂 schema-evolution dict） +- **不含**：Configuration（每消费点各自 `buildHadoopConf()` 重建，纯 CPU、无远程）；活 metaClient；fsView；at-instant schema。 + +**消费点改挂**： +- `getSchemaFromMetaClient`（2-arg 最新路径 @798）→ 读 memo schema。 +- `latestInstant`（beginQuerySnapshot @748）→ 读 memo instant。 +- `getScanNodeProperties` 普通路径 dict（@361-386）→ 读 memo 的 latest Avro + allHistoricalSchemas（把 `buildSchemaEvolutionProp` 重构成接受这两者，而非从 metaClient 现取）→ **消除其 build**。 +- `planScan`（@148）→ **只读 memo schema**（@188/@220）；**保留自己的** `lastInstant`（@163-177，字节一致的既有语义，勿改）+ 自己的活 metaClient（fsView + 逐文件 resolver）。 + +**明确不改挂（keep 今天的独立 build）**：at-instant / FOR TIME AS OF（3-arg `getTableSchema` @336、`getScanNodeProperties` pinned 分支 @373-380）、@incr `resolveIncremental`。罕见路径，不改进不回退。 + +**效果**：~5 build → ~2-3；schema 3×→1×；输出逐字节不变；并发下更一致（消除今日跨 metaClient 的 latest-commit 竞态）。 + +**loader 放置（诚实措辞）**：loader 跑在 `HudiMetaClientExecutor.execute`（TCCL pin + plugin doAs）；正常规划 metadata 先于 scan，故首触在 metadata 侧；若 scan 侧 miss 则在扫描线程按幂等重算，**逐字节相同**（`computeIfAbsent` 原子，不重复 build）。build-count 断言 metadata 侧计数。 + +### Piece C — HMS 缓存层（wrap + 新鲜读拆分 + REFRESH 失效） + +**wrap**：`HudiConnector.createClient`（@186）→ `new CachingHmsClient(new ThriftHmsClient(config, authAction), properties)`。无 pom 改动（fe-connector-hms 已是直接依赖，Caffeine 3.2.3 已随 hudi-common 打包——**勿加 2.9.3，否则 nearest-wins 降级 hudi-common 的 3.2.3**）。 + +**fresh/cached 拆分**（镜像 hive `collectPartitionNames(handle, bypassCache)`）：给 `HudiConnectorMetadata.collectPartitions` 加 `bypassCache` 参数，只影响 **hive-sync 分支**的 HMS 列举（非-hive-sync 走 metaClient，与 HMS 缓存无关）： +- `listPartitionNames`（SHOW PARTITIONS + partitions TVF）→ `bypassCache=true` → hive-sync 分支用 `hmsClient.listPartitionNamesFresh`。 +- `listPartitionValues`（partition_values TVF）→ `bypassCache=true` → fresh。 +- `listPartitions`（查询剪枝 / MTMV）→ `bypassCache=false` → 走缓存（这是 parity 收益点）。 +- `applyFilter` hive-sync 剪枝（@279）→ 走缓存（剪枝路径，`bypassCache=false` 语义）。 + +**REFRESH 失效**：`HudiConnector` override `invalidateTable/invalidateDb/invalidateAll` → 读 volatile `hmsClient` 字段（**不 force-build**：null=从未建=无缓存可刷）→ `instanceof CachingHmsClient` 则 `flush(db,table)`/`flushDb(db)`/`flushAll()`。网关 `HiveConnector.forEachBuiltSibling` 已把 REFRESH 转发给 hudi sibling（@358），故 override 后 REFRESH 生效。hudi 无 fileListingCache/partitionViewCache，故仅 flush HMS 缓存。 + +**getTableHandle**：`tableExists`（@205，CachingHmsClient pass-through）+ `getTable`（@208，缓存）→ 每次重复查询省 1 RPC；getTable 24h 缓存与 hive 一致，REFRESH 可清。 + +--- + +## 2. 验证计划 + +- 全反应堆 `mvn install -pl :fe-connector-api,:fe-connector-hudi,:fe-connector-hive -am -Dmaven.build.cache.enabled=false`（install 非 test；见 build 坑）→ BUILD SUCCESS。 +- **旗舰 build-count 守门（单测）**：注入 `DirectHudiMetaClientExecutor`（现有测试替身，无 auth/TCCL），在**真（非 NONE）** `ConnectorStatementScope` 下驱动 `getColumnHandles + beginQuerySnapshot + getTableSchema(2-arg)`，断言 **metadata 侧 build=1**（非 3）+ 输出 schema/instant 与 NONE-scope 路径**逐字节相同**（Rule 9：编码"同值、更少 build"）。⚠ counter 须覆盖 `buildMetaClient`@660 **和** planScan inline@148（或先把@148 路由过@660）。 +- 扩现有 `HudiSchemaParityTest`/`HudiSchemaAtInstantTest`/`HudiTimeTravelTest`/`HudiConnectorPartitionListingTest`：加 memo-on 变体断 parity + NONE-scope 变体断字节一致。 +- **HMS wrap 单测**：断 `createClient` 返回 `CachingHmsClient` 包 `ThriftHmsClient`；mock delegate 下第二次 `getTable` 命中缓存不重打、`tableExists` pass-through；`listPartitionNames` 走 fresh（bypass）、`listPartitions` 走缓存；`invalidateTable`→`flush` 生效。 +- **e2e 本地跑不了**（p2/external + 集群 + docker）。现有套件已覆盖网关路（全 type=hms）：分区剪枝/快照/MTMV/SHOW PARTITIONS/时间旅行/增量/schema 演进——证 parity。异构三向目录新用例本轮只给规格（放 `test_hive_hudi.groovy` 旁），交上游集群跑。 + +--- + +## 3. 红队三条 load-bearing 结论（实现须守） + +1. **不 memo Configuration**（reviewer-1）：`HadoopStorageConfiguration(conf)` by-reference 包裹 + 可变 + 非线程安全 + off-thread 并发读 → 共享会引入今日不存在的竞态。只 memo 不可变 map，各消费点重建 conf。 +2. **planScan 只读 memo schema、不读 memo instant**（reviewer-1/3）：planScan 普通读的 `timeline.lastInstant()`（@166/@175）是**文档化的字节一致语义**（`HudiConnectorMetadata`@558-560）；喂 memo instant 会在并发写下选到旧提交、且省 0 build。 +3. **旗舰 latest-only、at-instant 延后**（reviewer-3）：at-instant memo 会让投影退化成惰性/带锁/摸 metaClient，违"不可变投影"初衷；罕见路径保持独立 build。 + +授权/线程 HOLDS（三红队一致）：hudi 无 `SUPPORTS_USER_SESSION`（网关 fail-loud 守卫覆盖）、单身份 → name-key 缓存 authz-安全；作用域构造期捕获 → off-thread 扫描复用同 session；memo 非 AutoCloseable 不被 pass-2 关。 + +--- + +## 4. 延后（记录，勿抢跑） + +- 跨查询 metaClient + `(table,instant)` 分区缓存（MTMV/重复查询延迟）——下一轮，与更宽失效设计一起。 +- at-instant / FOR TIME AS OF 的 memo。 +- 跨方法 `buildHadoopConf` 去重 + 逐文件 normalize（HD-P05 余项，纯 CPU，且 Configuration 别名 caveat）。 +- @incr `(begin,end]` 行级过滤功能缺口（是功能 bug 非缓存，单独立项）。 +- 异构 hudi+iceberg+hive 单-hms-catalog e2e（需集群）。 diff --git a/plan-doc/perf-hotpath-hudi/designs/round-1-pieceB-flagship-impl-notes.md b/plan-doc/perf-hotpath-hudi/designs/round-1-pieceB-flagship-impl-notes.md new file mode 100644 index 00000000000000..5c70598615d7f5 --- /dev/null +++ b/plan-doc/perf-hotpath-hudi/designs/round-1-pieceB-flagship-impl-notes.md @@ -0,0 +1,141 @@ +# Piece B (旗舰 memo) — 实现级蓝图（动码前按 HEAD 再 grep 行号） + +> 前置：Piece A（文档）+ Piece C（HMS 缓存）已完成 + 提交 + 全模块 183 测试绿。 +> 本文件把 `round-1-memo-hms-cache-design.md` §1-B 细化到可直接编码，含**已侦察的测试交互坑**。 +> 铁律：0 fe-core、不 memo Configuration、planScan 一律不碰、latest-only（at-instant 保持独立 build）。 + +## 决定：**Scope B — planScan 完全不碰** + +planScan 本就必须建自己的 metaClient（fsView + 逐文件 schema_id resolver），故它读不读 memo 只省"schema 读"不省 build，且它是刚稳定的读热路径 + 红队 Issue-2 focus。**本轮 planScan 一行不改**（连 schema 也不从 memo 取）。build 收敛同样达成（见下），风险最低。 + +## memo 收敛的 3 个元数据侧消费点 + +一条普通过滤 SELECT 今天建 ~5 个 metaClient。本轮把**元数据侧 3 个**收敛到 1 个 loader： + +| 消费点 | 今天 | 改后 | +|---|---|---| +| `getColumnHandles`→`getTableSchema(2-arg)`→`getSchemaFromMetaClient(basePath,null)` build@802 | 建 | 读 memo.latestColumns | +| `beginQuerySnapshot`→`latestInstant(handle)` build@751 | 建 | 读 memo.instant | +| `getScanNodeProperties`（plain, `!force_jni`）build@363 | 建 | 读 memo.{resolvedInternalSchema, historicalSchemas} | +| `planScan` build@148 | 建 | **不改**（fsView 必需） | +| `applyFilter`（过滤时）partition build@292 | 建 | **不改**（分区，Piece 无关） | + +→ **~5 build → ~3**（memo-loader 1 + planScan 1 + applyFilter 1）。全部冗余 schema/instant 解析消除。 + +## 新增（0 fe-core） + +### 1. `ConnectorStatementScopes`（fe-connector-api）加常量 +- 加 `public static final String HUDI_METACLIENT = "hudi.metaclient";`（紧跟 `ICEBERG_TABLE`）。 +- 更新类 javadoc：把 `hudi.metaclient` 从"Reserved"行移到"Declared today"（现写 `{@link #ICEBERG_TABLE}`；改成 `{@link #ICEBERG_TABLE}、{@link #HUDI_METACLIENT}`）。 +- 这是 fe-connector-**api** 非 fe-core → 铁律 A 不碰。 + +### 2. `HudiStatementScope`（新，fe-connector-hudi，包私有 final class） +镜像 `IcebergStatementScope.sharedTable`： +```java +static HudiTableFacts sharedTableFacts(ConnectorSession session, String db, String table, + Supplier loader) { + return ConnectorStatementScopes.resolveInStatement( + session, ConnectorStatementScopes.HUDI_METACLIENT, db, table, loader); +} +``` +- 值类型 = `HudiTableFacts`（不可变投影，**非** AutoCloseable → 不被 scope close-pass 关）。 +- null session / NONE scope → loader 每次跑（byte-identical 到今天逐次）。 + +### 3. `HudiTableFacts`（新，fe-connector-hudi，不可变） +字段（全 final）： +- `long latestCompletedInstant` +- `List latestColumns`（已 attach field-id 的最新列） +- `HudiSchemaUtils.ResolvedInternalSchema resolvedInternalSchema`（mode-aware base + evolution flag） +- `Collection historicalSchemas`（evolution 时的历史版本，否则 emptyList） +- **不含** Configuration、活 metaClient、latestAvro（中间量）、at-instant schema。 + +### 4. loader（`HudiConnectorMetadata` 私有方法） +```java +private HudiTableFacts loadTableFacts(String basePath) { + return metaClientExecutor.execute(() -> { + HoodieTableMetaClient mc = HudiScanPlanProvider.buildMetaClient(buildHadoopConf(), basePath); + TableSchemaResolver r = new TableSchemaResolver(mc); + long instant = HudiScanPlanProvider.latestCompletedInstant(mc); + Schema latestAvro = r.getTableAvroSchema(true); + List cols = attachHudiFieldIds(r, latestAvro, avroSchemaToColumns(latestAvro)); + HudiSchemaUtils.ResolvedInternalSchema resolved = HudiSchemaUtils.resolveTableInternalSchema(r, latestAvro); + Collection hist = resolved.enableSchemaEvolution + ? HudiSchemaUtils.allHistoricalSchemas(mc) : Collections.emptyList(); + return new HudiTableFacts(instant, cols, resolved, hist); + }); +} + +private HudiTableFacts sharedFacts(ConnectorSession session, HudiTableHandle h) { + return HudiStatementScope.sharedTableFacts(session, h.getDbName(), h.getTableName(), + () -> loadTableFacts(h.getBasePath())); +} +``` +- **parity**：loader 每个 fact 的算法与今天各消费点逐字节相同，只是共享一个 mc（并发下更一致，红队 approved）。 +- 需把 `HudiSchemaUtils.allHistoricalSchemas` 由 `private` 改 **package-private** static（第 442 行）。`resolveTableInternalSchema` 已是 package static。 + +## 改挂（消费点，全部有 session） + +### A. `getTableSchema(2-arg)` @315 → 走 memo（latest） +今天：`return buildTableSchema((HudiTableHandle) handle, null);` +改：basePath 非空时读 `sharedFacts(session, h).latestColumns`；basePath 空走原 `getSchemaFromHms`；然后组装 `ConnectorTableSchema`。把 `buildTableSchema` 尾部（357-368 的 tableProperties + ConnectorTableSchema 组装）抽成 `assembleTableSchema(handle, columns)` 供两处复用。 +- `getTableSchema(3-arg, at-instant)` @335 **不改**：仍走 `buildTableSchema(handle, queryInstant)` → `getSchemaFromMetaClient`（at-instant 独立 build）。 +- `getSchemaFromMetaClient` 方法**保留**（3-arg at-instant 仍用它）；只是 2-arg latest 不再经它。 + +### B. `beginQuerySnapshot` @~418 → 读 `sharedFacts(session, h).latestCompletedInstant`（替 `latestInstant(handle)`） +- `latestInstant(handle)` 方法**保留**（collectPartitions hive-sync 分支 @699 仍用它；那条不在本轮收敛范围，Piece C 已定 collectPartitions 独立）。 + +### C. `getScanNodeProperties` @~361-391 → plain 路径走 memo +把 metaClient build（@363）挪进 `if (pin != null)` 分支；plain（pin==null）读 memo： +```java +if (!isForceJniScannerEnabled(session)) { + try { + String pin = hudiHandle.getQueryInstant(); + Optional dict; + if (pin == null) { + HudiTableFacts f = sharedFacts(session, hudiHandle); + dict = HudiSchemaUtils.buildSchemaEvolutionProp( + f.resolvedInternalSchema, f.historicalSchemas, castHudiColumns(columns)); + } else { + // 原 at-instant 逻辑原样（build mc / resolveInternalSchemaAtInstant / buildSchemaEvolutionDictAtInstant + // / 非 evolution 回退 buildSchemaEvolutionProp(mc,resolver,latestAvro,columns)） + } + dict.ifPresent(v -> props.put(SCHEMA_EVOLUTION_PROP, v)); + } catch (Exception e) { /* 原样 */ } +} +``` + +### D. `HudiSchemaUtils.buildSchemaEvolutionProp` 拆重载 +保留旧 `(HoodieTableMetaClient, TableSchemaResolver, Schema, List)`（at-instant 非 evolution 回退用），改为**委派**新重载： +```java +static Optional buildSchemaEvolutionProp(ResolvedInternalSchema resolved, + Collection historical, List requestedColumns) { + for (HudiColumnHandle h : requestedColumns) { if (h.getFieldId() < 0) return Optional.empty(); } + Collection eff = resolved.enableSchemaEvolution ? historical : Collections.emptyList(); + return Optional.of(buildSchemaEvolutionDict(requestedColumns, resolved.internalSchema, + resolved.enableSchemaEvolution, eff)); +} +// 旧重载 body 改为：resolved=resolveTableInternalSchema(r,latestAvro); hist=evolution?allHistoricalSchemas(mc):empty; +// return buildSchemaEvolutionProp(resolved, hist, requestedColumns); +``` +（byte-identical：旧 body 逻辑不变，只是抽出。） + +## ⚠ 测试交互（已侦察，动码前复核） + +- **`HudiSchemaAtInstantTest`（会挂，须改）**：`RecordingMetadata extends HudiConnectorMetadata` override `getSchemaFromMetaClient(String,String)`。其 **2-arg 测试**（`m.getTableSchema(null, handle(null))` → 断 `lastInstant==null`，即"2-arg 走 getSchemaFromMetaClient(null)"）在改挂后**不再经 getSchemaFromMetaClient**（走 memo）→ 该断言失效。**改法**：2-arg control 改成断 memo 路径（RecordingMetadata 另 stub executor 返回 canned `HudiTableFacts`，或断 2-arg 返回 latest 列）；**3-arg at-instant 测试原样保留**（仍经 getSchemaFromMetaClient）。 +- **`HudiColumnFieldIdTest`（不受影响）**：测 `avroSchemaToColumns`/field-id 纯函数（line 161 注释明确 getColumnHandles 需 live metaClient、未在此单测）。不经 getTableSchema。**复核确认后无需改**。 +- **`HudiSchemaParityTest`（大概率不受影响）**：测 schema 列派生纯函数（无 `new HudiConnectorMetadata`/`getTableSchema(session,...)` 调用命中）。**动码前 grep 确认**它不驱动 live getTableSchema(2-arg)；若驱动则同 AtInstant 处理。 + +## 新增守门测试 + +- **build-count 守门**（新 `HudiStatementMemoTest`，Rule 9）：注入计数版 executor（或计数 `buildMetaClient`——注意 planScan@148 inline build 不经 `buildMetaClient`，本轮 planScan 不改故不计），在**真（非 NONE）** `ConnectorStatementScope` 下驱动 `getColumnHandles`+`beginQuerySnapshot`+`getScanNodeProperties`，断言**元数据侧 loader 只跑 1 次**（memo 命中）；NONE-scope 对照跑 3 次（load-every-time）；两路输出（schema 列 / instant / dict）**逐字节相同**。变异（把 memo key 去掉 queryId 或让 loader 每次跑）须让计数断言变红。 +- 需要真 scope：用 `ConnectorStatementScope` 的 CHM 背书实现（api 层有 NONE；真实现见 iceberg 测试怎么造 scope——`IcebergStatementScopeTest` 有先例，镜像它构造一个 test scope + session stub 返回 getStatementScope）。 + +## 验证 +- `mvn install -pl :fe-connector-api,:fe-connector-hudi -am -Dmaven.build.cache.enabled=false -Dtest='Hudi*,ConnectorStatementScopes*' -DfailIfNoTests=false -f /fe/pom.xml` → BUILD SUCCESS + Tests run 0 fail。 +- fe-core `git diff` 必空。 +- clean-room 对抗净室复审（parity）：重点核 loader 各 fact 与今天逐字节等价 + at-instant 未退化 + planScan 未碰。 +- e2e 交集群（分区/时间旅行/schema 演进 parity）。 + +## commit +`[perf](catalog) fe-connector-hudi: per-statement metaClient/schema projection memo` +正文写：root cause（元数据侧 3 build/pass 冗余）+ 修法（不可变投影 memo，复用 ConnectorStatementScopes，0 fe-core，planScan 不碰）+ 度量（~5→~3 build，schema 3×→1×）+ 守门（build-count + parity + NONE 对照）+ 红队 3 修正落实。 diff --git a/plan-doc/perf-hotpath-hudi/progress.md b/plan-doc/perf-hotpath-hudi/progress.md new file mode 100644 index 00000000000000..445ddfb7795a0d --- /dev/null +++ b/plan-doc/perf-hotpath-hudi/progress.md @@ -0,0 +1,49 @@ +# Progress Log — perf-hotpath-hudi + +> append-only：每 session 追加一条（日期 / 做了什么 / 结论 / 下一步）。滚动上下文在 HANDOFF.md。 + +--- + +## 2026-07-24 — 立项 + 调研 + 设计 + 红队复核（未动产品代码） + +- **做了什么**： + 1. owner 选定旗舰 hudi 为下一个连接器（连接器缓存统一伞形 WS-HUDI）。 + 2. 动码前按 HEAD（`f2e9706df5a`）重侦察：13-agent workflow（`wf_3a0b9aca-966`）——9 路并行核对 5 个热点真实倍数/当前行号 + 框架接入缝 + 授权/生命周期 → 综合设计 → 3 路对抗红队。 + 3. 自测独立核实 4 个 crux：`ConnectorStatementScopeImpl` 语句末关 AutoCloseable；`HoodieTableMetaClient`(1.0.2) 非 AutoCloseable（memo 活对象 close-pass-safe）；`CachingHmsClient`/fresh/flush API + hive fresh/cached 模板；hudi 三分区入口 + pom 已依赖 hms/api（无 pom 改动）。 + 4. 写设计蓝图 `designs/round-1-memo-hms-cache-design.md`（3 块 + 红队 3 修正 + 验证 + 延后清单）。 +- **结论（关键）**： + - **侦察更正**：metaClient build ~5（非 5-6），schema 3×（非 4），过滤查询分区已去重~1（复核成立）。 + - **旗舰 = 每语句"不可变投影" memo**（{instant, 最新 schema, allHistoricalSchemas}），复用现成 `ConnectorStatementScopes.resolveInStatement`（iceberg 同款）→ **0 fe-core**。红队 3 修正：①不 memo Configuration（可变+非线程安全）②planScan 只读 memo schema、保留自己 lastInstant（文档化字节语义）③latest-only，at-instant 延后。~5 build→~2-3、schema 3×→1×、字节不变。 + - **HMS 缓存层 owner 拍板做全套**：wrap `CachingHmsClient` + 按 hive 补 fresh/cached 拆分（SHOW/TVF→`listPartitionNamesFresh`、剪枝/MTMV→cached）+ override `HudiConnector.invalidate*` flush（网关已 forEachBuiltSibling 转发 REFRESH）——否则 hive-sync 表 SHOW PARTITIONS 陈旧最多 24h 且 REFRESH 清不掉（红队 BLOCKER）。 + - **文档清理**：清 stale "dormant hms" 注释一批。 +- **下一步**：实施 3 块（序 A→C→B，各独立 commit），动每个文件前按 HEAD 重 grep，守红队 3 修正 + 铁律 A（0 fe-core）+ 无 pom 改动。 + +--- + +## 2026-07-24 (2) — Piece A（文档）+ Piece C（HMS 缓存）实现+提交+验证；Piece B checkpoint 交下轮 + +- **做了什么**： + - **Piece A**（commit `1cb0f95f8ed`）：4 处 stale "dormant until hms enters SPI_READY_TYPES" 注释改词（HudiConnectorMetadata / HudiScanPlanProvider / HudiConnectorOwnsHandleTest / HudiReadOnlyWriteRejectTest），纯 doc；保留同词异义两处（schema_id "dormant/inert"、"never add hudi"）；hive-side stale 注释留其自身模块 cleanup（不扩 blast radius）。 + - **Piece C**（commit `e26ab33b001`）：`HudiConnector.createClient` 经新 package-private `wrapWithCache` 包 `CachingHmsClient`；`collectPartitions` 加 `bypassCache`——`listPartitionNames`/`listPartitionValues`（SHOW/TVF 展示）走 `listPartitionNamesFresh` 绕缓存、`listPartitions`（剪枝/MTMV）走缓存；override `invalidateTable/Db/All` no-force-build flush（网关 forEachBuiltSibling 已转发 REFRESH 到 sibling）。**无 pom 改动**（hms 已直接依赖 + Caffeine 3.2.3 随 hudi-common）。新 `HudiConnectorHmsCacheTest` 8 测试（wrap / fresh-vs-cached 三入口 / 三 flush 钩子 / unbuilt no-op）。`mvn install -pl :fe-connector-hudi -am` **BUILD SUCCESS，hudi 全模块 183 测试 0 失败**（既有分区列举测试因 default listPartitionNamesFresh→listPartitionNames 仍绿）。 +- **Piece B（旗舰）checkpoint 未动**：动码前把它细化到可编码（`designs/round-1-pieceB-flagship-impl-notes.md`），并侦察出关键测试交互坑（`HudiSchemaAtInstantTest` 2-arg control 因改挂会挂须改）。因它触及连接器最易 SIGABRT 的 schema/field-id/演进派生 + 需 session 穿线 + 测试改造，判定在长 session 尾部收尾风险高 → 按 owner"caution over speed"+ checkpoint 纪律交下一轮/新 session。 +- **结论/决定**：**Scope B 定案**——planScan 一行不碰（省 build 不受影响、避开读热路径 + 红队 Issue-2）。旗舰 build 收敛 ~5→~3 由元数据侧 3 消费点收敛达成。 +- **下一步**：按 `round-1-pieceB-flagship-impl-notes.md` 实现旗舰 memo（新 session 保证预算），动码前按 HEAD 重 grep + 复核 HudiColumnFieldIdTest/HudiSchemaParityTest 是否纯函数不受影响。 + +--- + +## 2026-07-24 (3) — Piece B（旗舰 memo, PERF-H03）实现+提交；重侦察推翻蓝图多处、净室复审后按 owner 拍板改为"两块独立 memo" + +- **做了什么**： + - 动码前按 HEAD 重侦察（自读 + 5-agent 净室 workflow），**推翻蓝图多处**： + - 蓝图称会挂的 `HudiSchemaAtInstantTest` 2-arg 用例在"空会话每次加载"契约下其实不会挂（是否受影响取决于 loader 是否经 `getSchemaFromMetaClient`）。 + - 蓝图列为"元数据侧第 3 个消费点"的 `getScanNodeProperties` 实际在**另一个类** `HudiScanPlanProvider`：用不同存储配置源、**不经** `metaClientExecutor` 鉴权包装、跑在扫描线程 → 并入共享 loader 非字节等价且有重开 auth/TCCL 缝的风险，判定**本轮不碰扫描规划器**。 + - 多处行号/方法归属漂移已校正（分区构建在元数据类而非扫描器；`attachHudiFieldIds`/`avroSchemaToColumns` 在元数据类而非工具类等）。 + - 先按 owner 选的"变体二（schema+instant 合并一次 metaClient 构建）"实现并全绿；**4-agent 净室对抗复审**确认核心等价全部成立，但指出该合并的**两个 minor 固有边角**（取 schema 多依赖 instant 可解析；取快照顺带读 schema + 可能多打一条 WARN）。 + - owner 拍板**退回"两块独立 memo"**（零耦合）：schema 一块、instant 一块，各自 loader = 现成方法，字节等价白送、两个边角消失；代价 = 不再共用那一次构建（正常 SELECT 元数据侧 metaClient 构建 3→2，而非合并版的 →1）。 +- **落地**（commit `26690775c81`，4 文件）： + - `ConnectorStatementScopes`（fe-connector-api，**0 fe-core**）加两常量 `HUDI_LATEST_SCHEMA` / `HUDI_LATEST_INSTANT`（+ javadoc 从 Reserved 移入 Declared）。 + - 新 `HudiStatementScope`（包私有，两 helper `sharedLatestColumns`/`sharedLatestInstant`）；`HudiConnectorMetadata`：`getTableSchema` 最新分支走 `sharedLatestColumns`、`beginQuerySnapshot` 走 `sharedLatestInstant`、`latestInstant` 改包私有（供守门测试覆写）。**`getSchemaFromMetaClient`/`latestInstant` 方法体零改动；`HudiScanPlanProvider` 零改动；at-instant 路径未碰**。 + - 新守门测试 `HudiStatementMemoTest`（4 测试：schema 去重、instant 去重、**两 memo 独立**、NONE/null 每次加载）；`HudiSchemaAtInstantTest`/`HudiConnectorPartitionListingTest` **回到原样**（两块独立 memo 下最新路径仍经 `getSchemaFromMetaClient`，原断言成立）。 + - 验证：`install -pl :fe-connector-api,:fe-connector-hudi -am -Dmaven.build.cache.enabled=false` **BUILD SUCCESS**；hudi 全模块 + api scope 测试 0 失败；`git diff` 对 fe-core 空。 +- **结论/收益**：一条语句内重复的"最新 schema"解析（getColumnHandles 2-arg + 3-arg 空 instant MVCC）合成 1 次；重复"最新 instant"合成 1 次；正常 SELECT 元数据侧 metaClient 构建 ~3→2、schema 解析去重；行为字节不变、零耦合。 +- **下一步**：round-1 三块（文档 / HMS 缓存 / 旗舰 memo）**全部完成**。转 mc/es 两个小连接器，或 hudi round-2 延后项（跨查询缓存 PERF-H04 等）。e2e 需集群本地未跑（分区/时间旅行/schema 演进 parity 交集群回归）。 diff --git a/plan-doc/perf-hotpath-hudi/tasklist.md b/plan-doc/perf-hotpath-hudi/tasklist.md new file mode 100644 index 00000000000000..e57cf5c9f6c672 --- /dev/null +++ b/plan-doc/perf-hotpath-hudi/tasklist.md @@ -0,0 +1,25 @@ +# Task List — perf-hotpath-hudi + +> ID 一旦分配永不复用。状态：⏳待启动 · 🚧进行中 · ✅完成 · 🔬复核中 · 🅿待拍板 · 🚫不立项 +> 权威分析：伞形 `plan-doc/connector-cache-unification/connectors/hudi.md` + 本空间 `designs/round-1-memo-hms-cache-design.md`。 + +## Round 1（全部完成 2026-07-24：H01 文档 / H02 HMS 缓存 / H03 旗舰 memo） + +| ID | 主题 | 覆盖发现 | 依赖 | 状态 | +|---|---|---|---|---| +| **PERF-H01** | 陈旧 "dormant hms" 注释清理（纯 doc） | 陈旧注释 | — | ✅ commit `1cb0f95f8ed` | +| **PERF-H02** | HMS 缓存层：wrap `CachingHmsClient` + fresh/cached 拆分 + REFRESH flush | HD-P04（+新鲜度修正） | — | ✅ commit `e26ab33b001`（183 测试绿） | +| **PERF-H03** | 旗舰：每语句 memo（最新 schema + 最新 instant） | HD-P01 + HD-P02 | — | ✅ commit `26690775c81`（HudiStatementMemoTest 4 测试绿，全模块 0 失败） | + +> 三项独立，实施序 A(H01)→C(H02)→B(H03)（风险从低到高）。全程 0 fe-core、无 pom 改动。 +> **PERF-H03 最终实现（owner 拍板"两块独立 memo"）**：重侦察推翻蓝图——`getScanNodeProperties` 在 `HudiScanPlanProvider`（异构 build/线程/鉴权）→**本轮不碰扫描规划器**；`HudiSchemaAtInstantTest` 无须改（两块独立 memo 下最新路径仍经 `getSchemaFromMetaClient`）。落地=`ConnectorStatementScopes` 加 `HUDI_LATEST_SCHEMA`/`HUDI_LATEST_INSTANT` 两命名空间、`HudiStatementScope` 两 helper，`getTableSchema`(最新)/`beginQuerySnapshot` 各走一块 memo（loader=现成方法，字节等价、零耦合）。合并版（变体二）经 4-agent 净室复审有两 minor 固有边角，已按 owner 拍板退回两块独立 memo。 + +## 延后（记录，勿抢跑 — 设计 §4） + +| ID | 主题 | 覆盖发现 | 状态 | +|---|---|---|---| +| PERF-H04 | 跨查询 metaClient + `(table,instant)` 分区缓存 | HD-P03 | ⏳（下一轮，与更宽失效设计一起） | +| PERF-H05 | at-instant / FOR TIME AS OF 的 memo | HD-P02 子集 | ⏳ | +| PERF-H06 | 跨方法 buildHadoopConf 去重 + 逐文件 normalize | HD-P05 余项 | ⏳（纯 CPU，低优先） | +| PERF-H07 | @incr `(begin,end]` 行级过滤功能缺口 | STALE-6 | ⏳（功能 bug，单独立项） | +| PERF-H08 | 异构 hudi+iceberg+hive 单-hms-catalog e2e | — | ⏳（需集群，本轮只给规格） | diff --git a/plan-doc/perf-hotpath-jdbc/README.md b/plan-doc/perf-hotpath-jdbc/README.md new file mode 100644 index 00000000000000..05da6a955c061a --- /dev/null +++ b/plan-doc/perf-hotpath-jdbc/README.md @@ -0,0 +1,29 @@ +# perf-hotpath-jdbc — HP-1 + HP-2 (per-statement column-fetch dedup) + +> 兄弟空间（镜像 `perf-hotpath-iceberg` 布局，两项同根合并为一页）。 +> 权威病灶分析在伞形空间 `plan-doc/connector-cache-unification/connectors/jdbc.md`。 + +## 一句话 +本地→远端列名映射靠 `client.getJdbcColumnsInfo`（真实 `DatabaseMetaData.getColumns` 远程往返）。一条语句内多处重复取：读侧 `getColumnHandles`（每 scan node ~2 次）+ 缓存 miss 的 `getTableSchema`；写侧 `buildInsertSql` 新建实例再取（`EXPLAIN INSERT` 触发 2 次）。全都没记忆化。修法 = 把两处远程取列收进**每语句作用域**记忆化，读写两条路自动共享。 + +## 动码前重侦察（HEAD d8e2541e567） +- `JdbcConnectorMetadata.getTableSchema` :109 / `getColumnHandles` :152 都收 `session`，都调同一 `client.getJdbcColumnsInfo(db,table)`（:119/:162）。 +- 元数据实例每语句经漏斗单例共享（`PluginDrivenMetadata.get`）→ 读侧多次调用共享同一实例。 +- 写侧 `JdbcWritePlanProvider.buildInsertSql` :136 `new JdbcConnectorMetadata(...).getColumnHandles(session, handle)`——**新建实例绕漏斗**，但实例无状态、取列恒远程。 +- 关键洞见：记忆化放**语句作用域**（非实例），则写侧新建实例的 `getColumnHandles` 也命中同一条目 → **一个改动点修两处**。owner 拍板"语句作用域统一去重"（对齐 es 的 cross-path 先例，非 mc 的实例记忆化）。 + +## 实现（commit `7df22cd1c71`，连接器侧，0 fe-core） +- `fe-connector-api/ConnectorStatementScopes`：加命名空间常量 `JDBC_COLUMNS`（与 `ICEBERG_TABLE`/`HUDI_*`/`ES_INDEX_MAPPING` 并列；连接器 SPI，非 fe-core）。 +- `JdbcConnectorMetadata`：两处 `client.getJdbcColumnsInfo(db,table)` 包进 `ConnectorStatementScopes.resolveInStatement(session, JDBC_COLUMNS, db, table, loader)`，记忆化**原始** `List`（session 无关），键 `(catalogId,db,table,queryId)`。各消费者按语义各自再套变换（handle 走 identifier mapper、schema 走类型转换）。 +- `JdbcWritePlanProvider.buildInsertSql`：**仅 javadoc**——写路径 `getColumnHandles` 现共享该作用域条目（`EXPLAIN INSERT` 不再双取），无逻辑改动。 +- NONE 作用域（离线/fe-core 跨查询 schema 缓存 loader）下 loader 每次跑 → 逐字节与改前一致。 + +## 净室复审发现（已在提交前修复） +`getTableSchema` 的 `jdbcTypeToConnectorType` 对某些日期类型**原地** `setAllowNull(true)`；现在共享同一 raw list → 修正 `JDBC_COLUMNS` javadoc 如实描述"幂等原地改写、仅 `getTableSchema` 读 allowNull 且总先重导"，并加一个**会变换的类型转换 double** 测试钉住该不变量（共享 memo 上两次 `getTableSchema` + 中间 `getColumnHandles` 输出稳定）。 + +## 验证 +- `JdbcConnectorMetadataTest` +3（读侧去重、NONE 每次取、共享-mutable 不变量）。 +- `JdbcWritePlanProviderTest` +2（写侧 planWrite+appendExplainInfo 去重、scan+write cross-path 去重证"作用域键非实例键"）；既有 NONE-作用域字节 parity 测试原样绿。 +- 全模块 **199/199 绿**，checkstyle 0，BUILD SUCCESS。 +- 3-lens 净室对抗复审：session-safety CLEAN、key/scope/composition CLEAN、iron-rules/tests 仅上述 mutation 不变量项（已修）。 +- **e2e 需集群本地未跑**（留标注：jdbc 目录 INSERT 端到端 SQL 不变）。 diff --git a/plan-doc/perf-hotpath-maxcompute/HANDOFF.md b/plan-doc/perf-hotpath-maxcompute/HANDOFF.md new file mode 100644 index 00000000000000..7271645cdab777 --- /dev/null +++ b/plan-doc/perf-hotpath-maxcompute/HANDOFF.md @@ -0,0 +1,46 @@ +# 🤝 Session Handoff — perf-hotpath-maxcompute + +> 滚动文档：顶部「下一步」每 session 覆盖式更新；底部稳定区勿覆盖。 +> 完成明细进 `git log` + 本空间 `progress.md`；回填伞形 `plan-doc/connector-cache-unification/`。 + +--- + +# 🆕 下一步（覆盖区） + +## 当前状态（2026-07-24） + +- **Round 1（PERF-MC01）= 每语句表句柄记忆化：✅ 完成，commit `58daadd10e0`**。全模块 120 测试绿、checkstyle 0 违规、两处变异验证（去 memo + db-blind key 均能令守门测试变红）、净室对抗复审（parity/staleness HOLD、concurrency 经 verify REFUTED、test-quality 补跨-db 用例后 HOLD）。 +- **e2e 未跑**（需集群）：异构 + 独立 max_compute catalog 的 SELECT/分区裁剪/写路径解析计数，留标注。 +- 设计权威：[`designs/round-1-handle-memo-design.md`](./designs/round-1-handle-memo-design.md)；日志见 [`progress.md`](./progress.md)。 + +## 之后 + +- **PERF-MC02**（doc-only，随手）：清 `beginTransaction` / `MaxComputeWritePlanProvider` 的过时 "dormant until cutover" 注释（写路径早已 live）。可并进伞形 WS-DOC 统一做。 +- **PERF-MC03**（可选、低优先）：跨查询 ODPS `Table` 缓存——出现真实热点 profile 再上。 +- 伞形层面：WS-MC 完成后，下一个 workstream = **WS-ES**（es 连接器 per-scan hoist + mapping 承载）。 + +--- + +# 🧱 稳定区（勿覆盖） + +## 铁律（继承伞形） +1. **fe-core 源只出不进 + 禁 scaffolding 搬迁**：新缓存/memo 一律连接器侧。本轮 memo 在 `MaxComputeConnectorMetadata` 实例上，天然合规。 +2. **authz 准入**：maxcompute 一 catalog 一套**静态凭证**、单一 ODPS 身份（无 per-user vending）→ 名字为 key 的 memo authz-安全。 +3. **parity 只减不改**：性能修复必须行为不变；present 路径 memo、absent 路径逐字节等价今天（每次重探）。 + +## build / 验证坑（继承伞形） +1. maven 用绝对 `-f /fe/pom.xml`；`${revision}` 未 flatten → `-am`；测试用 `install` 或单模块 `test`（deps 已 install 后）。 +2. 测试必加 `-Dmaven.build.cache.enabled=false`；别用 `-q`；grep `BUILD SUCCESS` + `Tests run:`。 +3. 后台跑直接 `mvn … >> log 2>&1`（别 nohup&），读日志非通知 exit code。 +4. checkstyle 绑 `validate` 阶段（`install`/`test` 都会跑），扫 test 源；主源 ≤120。 + +## 并发探测（动码前） +- 本 worktree `find fe -newermt '-120 sec'`（滤 `/target/`）+ `pgrep -a -f maven`；`/mnt/disk1/gq` 是**别的用户**的 worktree（独立 ~/.m2），不冲突。 +- 提交只精确 `git add` 本任务文件，禁 `git add -A`（工作树有大量无关 untracked）。 + +--- + +# 🔗 关系 +- **伞形**：[`plan-doc/connector-cache-unification/`](../connector-cache-unification/)（WS-MC 行）。 +- **模板**：[`plan-doc/perf-hotpath-iceberg/`](../perf-hotpath-iceberg/) / [`perf-hotpath-hudi/`](../perf-hotpath-hudi/)。 +- **问题类**：[`perf-heavy-op-hot-path-problem-class.md`](../perf-heavy-op-hot-path-problem-class.md)。 diff --git a/plan-doc/perf-hotpath-maxcompute/README.md b/plan-doc/perf-hotpath-maxcompute/README.md new file mode 100644 index 00000000000000..f3b48681eadd51 --- /dev/null +++ b/plan-doc/perf-hotpath-maxcompute/README.md @@ -0,0 +1,19 @@ +# 📦 任务空间 — fe-connector-maxcompute 热路径缓存修复 + +> 连接器缓存框架统一的兄弟空间（伞形 = `plan-doc/connector-cache-unification/`，行 WS-MC）。 +> 镜像 `plan-doc/perf-hotpath-iceberg/` / `perf-hotpath-hudi/` 布局。协作规范沿用 `../AGENT-PLAYBOOK.md`。 + +## 一句话背景 +maxcompute **已 live、已相当好地缓存**（跨查询分区缓存 + 干净 split 枚举 + per-statement 写事务 + 复用 fe-core 跨查询 schema 缓存），**不是 iceberg 那样的 P0 目标**。唯一真正适用的框架件 = **每语句表句柄记忆化**：`getTableHandle` 每次解析都发一次冗余 ODPS `tables().exists()` 远程探测并新建惰性表，funnel 只 memo metadata 不 memo handle → 一条语句 ~17 个解析点各付一次探测（冷统计逐列放大）。 + +## 本空间文件 +| 文件 | 用途 | +|---|---| +| [`HANDOFF.md`](./HANDOFF.md) | 下一步 + 稳定区规则（每 session 覆盖式更新） | +| [`tasklist.md`](./tasklist.md) | PERF-MCxx 勾选 + 状态 | +| [`progress.md`](./progress.md) | append-only 日志 | +| [`designs/`](./designs/) | per-round 设计蓝图（`round-1-handle-memo-design.md` = 当前） | + +## 流程（对齐 step-by-step-fix + iceberg 立项流程） +侦察（HEAD 重核）→ 设计 + 红队 → 实现（最小改动/守铁律/连接器侧）→ 验证（parity + build-count 守门）→ 独立 commit → 更新 tasklist/HANDOFF/progress + 回填伞形。 +**行号信 HEAD 不信文档**（设计里 file:line 是 2026-07-24 快照）。 diff --git a/plan-doc/perf-hotpath-maxcompute/designs/round-1-handle-memo-design.md b/plan-doc/perf-hotpath-maxcompute/designs/round-1-handle-memo-design.md new file mode 100644 index 00000000000000..ef544b2cfd9088 --- /dev/null +++ b/plan-doc/perf-hotpath-maxcompute/designs/round-1-handle-memo-design.md @@ -0,0 +1,70 @@ +# Round 1 设计 — maxcompute 每语句表句柄记忆化 + +> 兄弟空间 = `plan-doc/perf-hotpath-maxcompute/`（伞形 = `plan-doc/connector-cache-unification/`）。 +> 镜像 `plan-doc/perf-hotpath-iceberg/` 布局。铁律：0 fe-core、纯连接器侧、parity 只减不改。 +> **行号是 2026-07-24 HEAD `b505b3d7c89` 侦察快照**（6-agent 只读侦察，见伞形 `connectors/maxcompute.md` + 本轮侦察）。 + +## 1. 病灶（root cause） + +`MaxComputeConnectorMetadata.getTableHandle(session, db, table)`（`:118-130`，改前）每次调用都： +1. `structureHelper.tableExist(odps, db, table)` → ODPS `tables().exists()` **远程 HTTP 探测**（`McStructureHelper.java:129-137,218-225`，两个 helper 实现都是裸远程、无缓存）；不存在返回 `Optional.empty()`； +2. `structureHelper.getOdpsTable(...)` → `tables().get(...)` 返回一个**惰性 `com.aliyun.odps.Table`**（get 本身不发网络，首次 `getSchema()/getFileNum()/isExternalTable()` 才触发一次远程 reload）； +3. 新建 `MaxComputeTableHandle` 携带该惰性 Table。 + +funnel 保证一条语句 + 一个 catalog **恰有一个** `MaxComputeConnectorMetadata` 实例（`PluginDrivenMetadata.get` 按 `"metadata:"+catalogId` memo 在 per-statement `ConnectorStatementScope` 上，`:53-71`），但该 metadata **不 memo 已解析的表**。于是一条语句里 `resolveConnectorTableHandle`（fe-core `PluginDrivenExternalTable.java:116-120`，无缓存）的 **13 个站点** + `PhysicalPlanTranslator`（`:624,676`）+ `BindSink`（`:675,714`）共 **~17 个解析点**各调一次 `getTableHandle`： +- 每次一个冗余 `exists()` 远程往返（**冷统计逐列解析放大到 O(列数)**：`getColumnStatistic` fe-core `:1026` 每列解析一次新 handle）； +- 每次新建一个惰性 Table，各自首次访问 schema 时各触发一次远程 reload。 + +这就是伞形审计的 **MC-1（P1，冗余 exists 探测）+ MC-2（P2，冗余 Table reload）**。表在 SQL 分析期早已解析、确定存在，这些重复全是浪费。 + +## 2. 修复（fix） + +在 per-statement `MaxComputeConnectorMetadata` 实例上加一个 `Map, MaxComputeTableHandle> tableHandleMemo`（`ConcurrentHashMap`），`getTableHandle` 改为经它 `computeIfAbsent`： + +```java +MaxComputeTableHandle handle = tableHandleMemo.computeIfAbsent( + List.of(dbName, tableName), k -> { + if (!structureHelper.tableExist(odps, dbName, tableName)) { + return null; // 缺表：不记录映射 → 每次重探、语义不变 + } + Table odpsTable = structureHelper.getOdpsTable(odps, dbName, tableName); + TableIdentifier tableId = structureHelper.getTableIdentifier(dbName, tableName); + return new MaxComputeTableHandle(dbName, tableName, odpsTable, tableId); + }); +return Optional.ofNullable(handle); +``` + +- 第一次解析：照常 `exists()` + build，存进 memo；第 2..N 次同表：命中 memo，**零远程往返**。→ MC-1：k×/语句 → 1×。 +- memo 命中返回**同一个 handle（同一个惰性 Table）**，首次 reload 后后续所有消费点（读扫描 `MaxComputeScanPlanProvider`、写 `MaxComputeWritePlanProvider`、`getTableSchema`/`getColumnHandles`）复用已 reload 的 schema。→ MC-2 **随 memo 自然消解，无需单独改动**。 +- **改动只 1 个连接器文件、约 20 行**；0 fe-core。 + +## 3. 关键取舍（诚实） + +| # | 取舍 | 决定 | 理由 / 证据 | +|---|---|---|---| +| A | 只 memo 命中（present）还是也 memo 缺表（`Optional.empty()`）? | **只 memo present**（mapping fn 返回 `null` → CHM 不记录 → 缺表每次重探） | 缺表路径**逐字节等价今天**（每次探测、返回 empty），零行为变化；单语句内 create-then-resolve 同名今天不存在，memo 负结果是"没必要的脆弱"（侦察一致建议）。 | +| B | 是否顺手**删除** `exists()` 探测（更激进的 MC-1）? | **不删**，只去重第 2..N 次 | `tables().get()` 惰性、不验存在性；删 `exists()` 会把今天干净的 `Optional.empty()`（"table not found"）退化成后续 `getSchema()` 时抛的底层 `OdpsException`——**改错误语义**，超范围。目标 = k→1，非 k→0。 | +| C | `HashMap` vs `ConcurrentHashMap`? | **ConcurrentHashMap** | 同一 metadata 实例**跨线程可达**：`PluginDrivenScanNode` 在请求线程建 1 个 `connectorSession`（`:149`）解析 metadata 一次（`:206`），再把**同一 session** 丢进 off-thread 池任务（`scheduleExecutor` runAsync，分区批 `:1615-1628` / 流式 `:1700-1704`）→ 同 session=同 scope=同 metadata。memo 是该实例上**第一个可变跨线程字段**（其余 7 字段全 final），plain HashMap = 数据竞争。对齐 `ConnectorStatementScopeImpl`（CHM）+ iceberg FIX-PERF-07（CHM）先例。`computeIfAbsent` 原子性安全：supplier（tableExist+lazy get）**不回环** memo（两步都落到 ODPS SDK，无 Doris 回调），CHM 单键 computeIfAbsent 唯一禁忌=重入同 map，此处不触发。 | +| D | key 形态? | **`List.of(db, table)`**（List 值相等，无分隔符碰撞） | handle 无 `equals/hashCode`（只有 `serialVersionUID`），不能按 handle identity 做 key；`List.of` 值相等、无 import 新增（`List` 已导入）、非空实参安全；避免 `db+sep+table` 字符串分隔符碰撞推理。 | + +## 4. 为何无需失效 / authz / 一致性工作 + +- **失效**：memo 是 per-statement，随 metadata 实例语句末 GC；无语句内 DDL 改已解析 handle 身份（DDL/DML 是分开的 Nereids 语句）。`createTable`（`:360-374`）/ `tableExists`（`:113-116`）/ `dropDatabase` **各自直调** `structureHelper.tableExist`、**不经 memo**（保持 DDL 存在性检查权威、不被读 memo 短路）。跨查询 `MaxComputePartitionCache`（长寿命 connector 上、TTL 600s、REFRESH 失效）与 memo **生命周期/key/载荷全正交**，无协调代码。 +- **authz**：MaxCompute 一 catalog 一套**静态凭证**（AK/SK / RAM-Role-ARN / ECS-RAM-Role，`MCConnectorClientFactory:86-127`），无 per-user 凭证下发、无 session=user；名字为 key 的 memo **不跨用户泄漏**；`PluginDrivenMetadata` 另钉一语句=一身份。Layer-3 授权隔离不适用（伞形铁律 4 已核）。 +- **写一致性**：写路径不自解析 handle——`planWrite` 复用上游已解析、经 memo 的同一 `MaxComputeTableHandle`（`MaxComputeWritePlanProvider:91-92,105`）；memo 令读/写共享**同一 Table 对象** → **严格提升**语句内一致性，非放松。 + +## 5. 验证 + +- **守门单测**（新 `MaxComputeConnectorMetadataHandleMemoTest`，仿 `DropDbTest` 手写记录 fake、无 Mockito、null odps 离线）： + 1. `sameTableResolvedTwiceProbesAndBuildsOnce`：同表解析 2 次 → `exists()` 探测 1 次 + `getOdpsTable` build 1 次 + 两 handle `assertSame`（MC-1+MC-2 守门；去掉 memo → 计数变 2 + handle 不同 → 红）。 + 2. `distinctTablesAreResolvedIndependently`：不同表各解析一次、handle 不同（守 List 值相等 key）。 + 3. `absentTableReprobesEachCallAndIsNeverMemoized`：缺表每次重探、恒 empty、从不 build（守取舍 A；若 memo 负结果 → 探测计数变 1 → 红）。 +- **回归**：`mvn install -pl :fe-connector-maxcompute -am -Dmaven.build.cache.enabled=false -Dtest='MaxCompute*' -f /fe/pom.xml` → BUILD SUCCESS + 全 MaxCompute* 测试 0 失败。 +- **净室对抗复审**：改动逐字节 parity（只减不改）+ 取舍 A/B/C/D 无坑。 +- **e2e**：需集群，本地不跑，留标注（异构 + 独立 max_compute catalog 的 SELECT/分区裁剪/写路径解析计数）。 + +## 6. commit + +`[perf](catalog) fe-connector-maxcompute: memoize resolved table handle per statement` + +正文：root cause（~17 解析点各付一次冗余 ODPS `exists()` + 各自 Table reload）+ 修法（per-statement metadata 实例 CHM handle memo，computeIfAbsent，present-only）+ 度量（exists k×→1×、Table reload 每表 1×）+ 守门（build-count 单测 + parity + NONE 缺表对照）+ 0 fe-core。 diff --git a/plan-doc/perf-hotpath-maxcompute/progress.md b/plan-doc/perf-hotpath-maxcompute/progress.md new file mode 100644 index 00000000000000..956c6f909f0b71 --- /dev/null +++ b/plan-doc/perf-hotpath-maxcompute/progress.md @@ -0,0 +1,15 @@ +# Progress — perf-hotpath-maxcompute（append-only） + +## 2026-07-24 — PERF-MC01 每语句表句柄记忆化（commit `58daadd10e0`） + +**做了什么**：`MaxComputeConnectorMetadata.getTableHandle` 改为经 per-statement 实例上的 `Map, MaxComputeTableHandle> tableHandleMemo`（`ConcurrentHashMap`）`computeIfAbsent` 解析。收敛 MC-1（冗余 ODPS `exists()` 探测 k×/语句 → 1×）+ MC-2（每表 Table reload 收敛为 1）。**改 1 个连接器文件、+28/-9 行、0 fe-core**。 + +**流程**： +1. **HEAD 重侦察**（6-agent 只读 workflow）：确认 metadata 每语句单例（funnel `PluginDrivenMetadata.get`）、`getTableHandle` 是唯一 handle 生产点且覆盖 fe-core 13 + translator 2 + BindSink 2 = 17 解析点、handle 无 equals/hashCode（须按 (db,table) 值 key）、跨线程可达（off-thread scan 池复用同 session）→ CHM；`createTable`/`tableExists` 直调 helper 不经 memo；partitionCache 正交。 +2. **设计**（`designs/round-1-handle-memo-design.md`）：present-only memo、保留 exists() 只去重、CHM、`List.of` 值 key；owner 中文讲清后确认。 +3. **实现 + 守门单测** `MaxComputeConnectorMetadataHandleMemoTest`（仿 `DropDbTest` 手写记录 fake、无 Mockito、null odps 离线）。 +4. **验证**：全 `MaxCompute*` 120 测试绿、checkstyle 0 违规。 +5. **变异验证（Rule 9）两处**：① 去 memo（`memo.clear()`）→ `sameTableResolvedTwice...` 变红（探测计数 1→2）；② db-blind key（`List.of(tableName)`）→ `sameTableNameInDifferentDatabasesDoNotCollide` 变红（`assertNotSame` 失败）。 +6. **净室对抗复审**（4 lens + 2 verify workflow）：parity `PARITY_HOLDS`、staleness `PARITY_HOLDS`；concurrency CONCERN（共享 Table 并发 reload）经 verify **REFUTED**（off-thread 任务用 ctor 一次解析的 `currentHandle`、不调 getTableHandle，共享 Table 是 baseline 既有属性，memo 反而降 reload 争用）；test-quality CONCERN（无跨 db 用例）经 verify **CONFIRMED_REAL** → **已补** `sameTableNameInDifferentDatabasesDoNotCollide` 并变异验证。 + +**下一步**：PERF-MC02（doc-only 陈旧注释，随手/并 WS-DOC）、PERF-MC03（可选跨查询 Table 缓存，热点触发）。**e2e** 需集群，本地未跑（异构 + 独立 max_compute catalog 的 SELECT/分区裁剪/写路径解析计数），留标注。 diff --git a/plan-doc/perf-hotpath-maxcompute/tasklist.md b/plan-doc/perf-hotpath-maxcompute/tasklist.md new file mode 100644 index 00000000000000..d113429052890d --- /dev/null +++ b/plan-doc/perf-hotpath-maxcompute/tasklist.md @@ -0,0 +1,20 @@ +# Task List — perf-hotpath-maxcompute + +> ID 一旦分配永不复用。状态：⏳待启动 · 🚧进行中 · ✅完成 · 🔬复核中 · 🅿待拍板 · 🚫不立项 +> 权威分析：伞形 `plan-doc/connector-cache-unification/connectors/maxcompute.md` + 本空间 `designs/round-1-handle-memo-design.md`。 + +## Round 1 + +| ID | 主题 | 覆盖发现 | 依赖 | 状态 | +|---|---|---|---|---| +| **PERF-MC01** | 每语句表句柄记忆化（per-statement metadata 实例上 `Map<(db,table),handle>`，`computeIfAbsent`，present-only）→ 冗余 ODPS `exists()` 探测 k×→1×、Table reload 每表 1× | MC-1（P1）+ MC-2（P2） | — | ✅ commit `58daadd10e0`（全模块 120 测试绿 + checkstyle 0 + 两处变异验证 + 净室复审 parity/staleness HOLD、concurrency REFUTED、test-quality 补跨-db 用例） | + +> 0 fe-core、无 pom 改动、纯连接器侧、authz 天然合规（静态凭证单身份）。 + +## 延后 / 不立项（记录，勿抢跑 — 伞形审计 §6/§7） + +| ID | 主题 | 覆盖发现 | 状态 | +|---|---|---|---| +| PERF-MC02 | 陈旧注释清理（`beginTransaction` / `MaxComputeWritePlanProvider` 的 "gate-closed / dormant until cutover" 已过时；写路径早已 live） | 陈旧注释 | ⏳（doc-only，随手可做，可并进伞形 WS-DOC） | +| PERF-MC03 | 跨查询 ODPS `Table` 缓存（iceberg `IcebergTableCache` 类比，24h/REFRESH）省每查询 `planScan` reload 的跨查询维度 | MC-2 跨查询维度 | ⏳（可选、低优先；有陈旧/TTL 代价、收益温和；fe-core `SchemaCacheValue` 已覆盖跨查询主需求） | +| — | 连接器侧 table/handle 重缓存、format/comment/manifest 缓存 | — | 🚫 不适用（伞形审计 §6：ODPS Storage API 无格式推断、comment 默认空、无 manifest 模型、split 枚举已干净） | diff --git a/plan-doc/perf-hotpath-paimon/README.md b/plan-doc/perf-hotpath-paimon/README.md new file mode 100644 index 00000000000000..d201cbda02e97b --- /dev/null +++ b/plan-doc/perf-hotpath-paimon/README.md @@ -0,0 +1,30 @@ +# perf-hotpath-paimon — PA-1 (partition-enumeration cache routing) + +> 兄弟空间（镜像 `perf-hotpath-iceberg` 布局，但本项为单一常数倍/CPU 清理，故合并为一页）。 +> 权威病灶分析在伞形空间 `plan-doc/connector-cache-unification/connectors/paimon.md`。 + +## 一句话 +`listPartitionNames()`（SHOW PARTITIONS）与 `listPartitionValues()`（`partition_values()` TVF）此前**绕过** `partitionViewCache`、直接调 `collectPartitions()`，每次都把整张表的分区列表重新渲染成 `ConnectorPartitionInfo`；`listPartitions()`（裁剪路径）却已走缓存。本项把三个入口收敛到同一个"带缓存收集器" `cachedPartitions()`。 + +## 动码前重侦察（HEAD d8e2541e567，行号 vs 旧快照已漂移） +- `listPartitionNames` :1030 → `collectPartitions` 直连（旁路，确认）。 +- `listPartitionValues` :1098 → `collectPartitions` 直连（旁路，确认）。 +- `listPartitions` :1056 → `partitionViewCache.get(key, collectPartitions)`（走缓存）。 +- key = `ConnectorTableKey(db,table,snapshotId,-1)`，`snapshotId` 经 `latestSnapshotCache`（`beginQuerySnapshot` 已预热）。 +- 远程 `catalogOps.listPartitions` 已被 paimon SDK `CachingCatalog.partitionCache` 挡下 → 省的是**本地 CPU 重渲染**（名称转义/日期格式化/空值归一化/查重），非远程 IO。 + +## 实现（commit `59b65912104`，连接器侧，0 fe-core） +- 抽私有 `cachedPartitions(handle)` = `listPartitions` 无过滤分支的等价体（`cache==null || 无分区键 → collectPartitions`；否则 `cache.get(key, collectPartitions)`）。 +- `listPartitions` 保留 `filter.isPresent() → collectPartitions`（带过滤不写缓存），其余委派 `cachedPartitions`。 +- `listPartitionNames`/`listPartitionValues` 改调 `cachedPartitions`，派生循环（`getPartitionName()` / 按请求列序 `getPartitionValues()`）逐字不变。 +- 更新 4 处会变陈旧的注释（两处字段注释、`collectPartitions`/`listPartitions` javadoc、测试类头 javadoc）。 + +## owner 拍板的语义变化（2026-07-24，选"走缓存"） +`SHOW PARTITIONS` / `partition_values()` 的新鲜度从"每次重算"变为"24h TTL + REFRESH 失效"，与已缓存的裁剪路径自洽、与 Trino 的带缓存分区列举一致；`REFRESH` 是逃生口。代价：与 hive"故意实时 SHOW PARTITIONS"不一致（但 hive 是刻意分出 fresh 方法，paimon 只读无写后读问题）。owner 明确选此方案。 + +## 验证 +- `PaimonConnectorMetadataPartitionViewCacheTest` +4（名/值跨查询命中、三入口共享一条目 + 派生 parity、非分区表不碰快照缝）。 +- `PaimonConnectorMetadataPartitionTest`（null-cache 路径字节 parity）原样全绿。 +- 26/26 绿，checkstyle 0，BUILD SUCCESS。 +- 3-lens 净室对抗复审：aliasing/freshness CLEAN；parity 只标"有意的新鲜度变化"（非 bug，已 owner 确认）；iron-rules/style CLEAN（修一处测试头陈旧注释）。 +- **e2e 需集群本地未跑**（留标注）。 diff --git a/tools/check-authz-cache-sharding.sh b/tools/check-authz-cache-sharding.sh deleted file mode 100755 index eba2a2ce6cf163..00000000000000 --- a/tools/check-authz-cache-sharding.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Arch gate: authorization-sensitive cross-query cache isolation on IcebergConnector. -# -# Invariant: under iceberg.rest.session=user the per-user authorization lives INSIDE the delegated -# loadTable round-trip, so any SHARED cross-query cache on IcebergConnector that can return a value -# without that per-user load bypasses authorization — a "list != load" metadata disclosure (a user who -# can list a table but is not authorized to load it receives a cached projection produced for another -# user). Every cache HOLDER field on IcebergConnector must therefore declare its isolation discipline -# with a marker on the field-declaration line or the line directly above it: -# -# authz-cache-session-user-disabled — the field is null under isUserSessionEnabled() (constructor gate) -# authz-cache-exempt — justified as not authz-bypassing (e.g. default-off + read ONLY -# after a preceding per-user resolveTable/loadTable) -# -# A cache holder field carrying NEITHER marker fails the build, so "added a new cross-query cache and -# forgot to isolate it for session=user" becomes a build failure instead of a silent leak. -# -# This gate is marker-based (like tools/check-fecore-metadata-funnel.sh): the marker is a reviewed claim -# and the self-test (tools/check-authz-cache-sharding.test.sh) locks the RED/GREEN behavior. The fe-core -# generic schema cache is a different, name-keyed cache protected separately by -# ExternalCatalog.shouldBypassSchemaCache, not by this gate. -# -# Usage: -# tools/check-authz-cache-sharding.sh # default root fe/fe-connector -# tools/check-authz-cache-sharding.sh # supplied root (the dir containing fe-connector-iceberg/) -# -# Exit code: -# 0 — every cache holder field carries a marker -# 1 — at least one cache holder field is unmarked (offending lines printed) -# 2 — the cache-holder file was not found under the root - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -DEFAULT_ROOT="${SCRIPT_DIR}/../fe/fe-connector" -ROOT="${1:-${DEFAULT_ROOT}}" - -# The single connector that owns the authorization-sensitive cross-query caches. -TARGET_REL='fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java' -TARGET="${ROOT}/${TARGET_REL}" - -if [ ! -f "${TARGET}" ]; then - echo "check-authz-cache-sharding: cache-holder file not found: ${TARGET}" >&2 - exit 2 -fi - -MARKER_DISABLED='authz-cache-session-user-disabled' -MARKER_EXEMPT='authz-cache-exempt' - -# A cache holder field declaration: any "final Cache " / "final Cache<" field, -# regardless of visibility (private/protected/public/package) or static-ness. This deliberately catches -# more than the shipped "private final Iceberg*Cache" convention — a future authz-sensitive cross-query -# cache added as a raw Caffeine/Guava Cache<...> or LoadingCache<...>, or with a different visibility, -# must still be classified. The internal MetaCacheEntry plumbing inside the *Cache classes lives in other -# files (this gate scans only IcebergConnector), so it never matches here. A holder whose type name does -# NOT end in "Cache" is out of scope by convention (cross-query caches are dedicated *Cache types). -FIELD_DECL='final ([A-Za-z_][A-Za-z0-9_]*)?Cache[ <]' - -CANDIDATES=$(grep -nE "${FIELD_DECL}" "${TARGET}" 2>/dev/null || true) - -RESULT="" -if [ -n "${CANDIDATES}" ]; then - while IFS= read -r line; do - [ -z "${line}" ] && continue - # line = : - lineno="${line%%:*}" - code="${line#*:}" - - # marker on the declaration line ... - case "${code}" in *"${MARKER_DISABLED}"*|*"${MARKER_EXEMPT}"*) continue ;; esac - # ... or on the line immediately above it (long decls / block-comment carriers). - if [ "${lineno}" -gt 1 ]; then - prev="$(sed -n "$((lineno - 1))p" "${TARGET}" 2>/dev/null || true)" - case "${prev}" in *"${MARKER_DISABLED}"*|*"${MARKER_EXEMPT}"*) continue ;; esac - fi - - RESULT="${RESULT}${TARGET}:${line}"$'\n' - done <<< "${CANDIDATES}" -fi -RESULT=$(printf '%s' "${RESULT}" | sed '/^$/d') - -if [ -n "${RESULT}" ]; then - echo "AUTHZ-SENSITIVE cache holder field(s) in IcebergConnector without an isolation marker:" >&2 - echo "${RESULT}" >&2 - echo "" >&2 - echo "Under iceberg.rest.session=user a shared, un-partitioned cross-query cache bypasses the per-user" >&2 - echo "loadTable authorization (a metadata disclosure). Each cache holder field must carry, on its" >&2 - echo "declaration line or the line directly above it, exactly ONE of:" >&2 - echo " ${MARKER_DISABLED} — the field is null under isUserSessionEnabled()" >&2 - echo " ${MARKER_EXEMPT} — justified as not authz-bypassing (e.g. read only after a per-user load)" >&2 - exit 1 -fi diff --git a/tools/check-authz-cache-sharding.test.sh b/tools/check-authz-cache-sharding.test.sh deleted file mode 100755 index e9f5bc976a5974..00000000000000 --- a/tools/check-authz-cache-sharding.test.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Self-test for tools/check-authz-cache-sharding.sh. -# -# The gate exits 0 on the real (already-marked) tree, so a controlled RED/GREEN fixture is the only way -# to prove it catches what it must. Each seeded field targets one gate property: -# SILENT — a cache field with 'authz-cache-session-user-disabled' on the decl line (trailing marker) -# SILENT — a cache field with 'authz-cache-exempt' on the line ABOVE (above-line marker) -# RED — an unmarked 'private final Iceberg*Cache' field (the core violation) -# RED — an unmarked raw 'private final Cache<...>' field (raw Caffeine form) -# RED — an unmarked 'protected static final LoadingCache<...>' field (visibility/static form) -# SILENT — a non-cache field (type does not end in Cache) (type boundary) -# Plus: exit 0 on a fully-marked tree, and the marker is load-bearing (strip it => the field is flagged). -# -# Usage: bash tools/check-authz-cache-sharding.test.sh # exit 0 = pass, 1 = fail - -set -u - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GATE="${SCRIPT_DIR}/check-authz-cache-sharding.sh" - -FX="$(mktemp -d)" -trap 'rm -rf "${FX}"' EXIT - -DIR="${FX}/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg" -mkdir -p "${DIR}" -TARGET="${DIR}/IcebergConnector.java" - -# Fixture: three marked caches (trailing + above), three UNMARKED caches in varied forms (RED), one -# non-cache field. The raw Cache<...> and the protected-static LoadingCache<...> exercise the broadened -# type/visibility coverage (a future authz cache need not be spelled "private final Iceberg*Cache"). -cat > "${TARGET}" <<'EOF' -package org.apache.doris.connector.iceberg; -public class IcebergConnector { - private final IcebergLatestSnapshotCache latestSnapshotCache; // authz-cache-session-user-disabled - private final IcebergTableCache tableCache; // authz-cache-session-user-disabled - // authz-cache-exempt: pure metadata, read only after a per-user load - private final IcebergManifestCache manifestCache = new IcebergManifestCache(); - private final IcebergBrokenCache brokenCache; - private final Cache rawCache; - protected static final LoadingCache loadingCache; - private final Object notACache = null; -} -EOF - -FAILED=0 -fail() { echo "FAIL: $1"; FAILED=1; } - -# ---- run 1: mixed fixture -> the three UNMARKED caches (all forms) flagged ---- -OUT="$(bash "${GATE}" "${FX}" 2>&1)"; EC=$? -REPORTED="$(printf '%s\n' "${OUT}" | grep -E "^${FX}.*:[0-9]+:" || true)" -N="$(printf '%s\n' "${REPORTED}" | grep -c 'Cache' || true)" - -[ "${EC}" -eq 1 ] || fail "expected exit 1 (violations present), got ${EC}" -[ "${N}" -eq 3 ] || fail "expected exactly 3 reported violations, got ${N}"$'\n'"${REPORTED}" - -printf '%s\n' "${REPORTED}" | grep -qF 'brokenCache' || fail "unmarked Iceberg*Cache NOT reported: brokenCache" -printf '%s\n' "${REPORTED}" | grep -qF 'rawCache' || fail "unmarked raw Cache<...> NOT reported: rawCache" -printf '%s\n' "${REPORTED}" | grep -qF 'loadingCache' || fail "unmarked static LoadingCache NOT reported: loadingCache" - -must_not_report() { - printf '%s\n' "${REPORTED}" | grep -qF "$1" && fail "should NOT be reported: $1" || true -} -must_not_report 'latestSnapshotCache' # trailing marker -must_not_report 'tableCache' # trailing marker -must_not_report 'manifestCache' # marker on line above -must_not_report 'notACache' # type does not end in Cache - -# ---- run 2: remove the three unmarked caches -> fully-marked tree exits 0 ---- -sed -i '/IcebergBrokenCache brokenCache;/d; /Cache rawCache;/d; /LoadingCache loadingCache;/d' "${TARGET}" -bash "${GATE}" "${FX}" >/dev/null 2>&1 && CLEAN_EC=0 || CLEAN_EC=$? -[ "${CLEAN_EC}" -eq 0 ] || fail "expected exit 0 on a fully-marked tree, got ${CLEAN_EC}" - -# ---- run 3: the 'disabled' marker is load-bearing -> strip it and its two caches are flagged ---- -# (the 'exempt' manifest stays silent, proving the two markers are independent.) -sed -i 's/authz-cache-session-user-disabled/marker-removed-here/g' "${TARGET}" -OUT3="$(bash "${GATE}" "${FX}" 2>&1)"; EC3=$? -REP3="$(printf '%s\n' "${OUT3}" | grep -E "^${FX}.*:[0-9]+:" || true)" -N3="$(printf '%s\n' "${REP3}" | grep -c 'Cache' || true)" -[ "${EC3}" -eq 1 ] || fail "expected exit 1 after stripping the disabled marker, got ${EC3}" -# latestSnapshotCache + tableCache (both were 'disabled') = 2 flagged; the exempt manifest stays silent. -[ "${N3}" -eq 2 ] || fail "expected 2 flagged after stripping the disabled marker, got ${N3}"$'\n'"${REP3}" -printf '%s\n' "${REP3}" | grep -qF 'latestSnapshotCache' || fail "disabled cache not flagged after marker strip" -printf '%s\n' "${REP3}" | grep -qF 'manifestCache' && fail "exempt cache wrongly flagged (markers not independent)" || true - -if [ "${FAILED}" -eq 0 ]; then - echo "PASS: authz-cache gate flags unmarked cache fields; disabled/exempt markers stay silent and are load-bearing." - exit 0 -fi -echo "---- run1 output ----"; printf '%s\n' "${OUT}" -exit 1