From 40aea36edf13cc9e13ef0e30e078984807a0413a Mon Sep 17 00:00:00 2001 From: daidai Date: Tue, 7 Jul 2026 16:05:43 +0800 Subject: [PATCH 01/32] [feature](iceberg) Support nested column schema change Support nested Iceberg column paths in external schema change operations and wire parser/analyzer metadata ops through ColumnPath. --- .../java/org/apache/doris/alter/Alter.java | 14 +- .../org/apache/doris/analysis/ColumnPath.java | 90 +++++ .../apache/doris/datasource/CatalogIf.java | 43 +++ .../doris/datasource/ExternalCatalog.java | 60 ++- .../iceberg/IcebergMetadataOps.java | 344 +++++++++++++++++- .../operations/ExternalMetadataOps.java | 52 +++ .../nereids/parser/LogicalPlanBuilder.java | 135 ++++++- .../plans/commands/AlterTableCommand.java | 43 ++- .../plans/commands/info/AddColumnOp.java | 17 +- .../plans/commands/info/ColumnDefinition.java | 9 +- .../plans/commands/info/DropColumnOp.java | 15 +- .../commands/info/ModifyColumnCommentOp.java | 21 +- .../plans/commands/info/ModifyColumnOp.java | 17 +- .../plans/commands/info/RenameColumnOp.java | 15 +- .../IcebergMetadataOpsValidationTest.java | 109 ++++++ ...cebergNestedSchemaEvolutionParserTest.java | 94 +++++ .../plans/commands/AlterTableCommandTest.java | 38 ++ .../org/apache/doris/nereids/DorisParser.g4 | 23 +- ...st_iceberg_nested_schema_evolution_ddl.out | 10 + ...iceberg_nested_schema_evolution_ddl.groovy | 159 ++++++++ ...chema_evolution_spark_doris_interop.groovy | 324 +++++++++++++++++ 21 files changed, 1572 insertions(+), 60 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPath.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out create mode 100644 regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 9f9eb03ceec676..85b2433092ecb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -415,20 +415,26 @@ private void processAlterTableForExternalTable( table.getDbName(), table.getName(), renameTableOp.getNewTableName()); } else if (alterOp instanceof AddColumnOp) { AddColumnOp addColumn = (AddColumnOp) alterOp; - table.getCatalog().addColumn(table, addColumn.getColumn(), addColumn.getColPos()); + table.getCatalog().addColumn( + table, addColumn.getColumnPath(), addColumn.getColumn(), addColumn.getColPos()); } else if (alterOp instanceof AddColumnsOp) { AddColumnsOp addColumns = (AddColumnsOp) alterOp; table.getCatalog().addColumns(table, addColumns.getColumns()); } else if (alterOp instanceof DropColumnOp) { DropColumnOp dropColumn = (DropColumnOp) alterOp; - table.getCatalog().dropColumn(table, dropColumn.getColName()); + table.getCatalog().dropColumn(table, dropColumn.getColumnPath()); } else if (alterOp instanceof RenameColumnOp) { RenameColumnOp columnRename = (RenameColumnOp) alterOp; table.getCatalog().renameColumn( - table, columnRename.getColName(), columnRename.getNewColName()); + table, columnRename.getColumnPath(), columnRename.getNewColName()); } else if (alterOp instanceof ModifyColumnOp) { ModifyColumnOp modifyColumn = (ModifyColumnOp) alterOp; - table.getCatalog().modifyColumn(table, modifyColumn.getColumn(), modifyColumn.getColPos()); + table.getCatalog().modifyColumn( + table, modifyColumn.getColumnPath(), modifyColumn.getColumn(), modifyColumn.getColPos()); + } else if (alterOp instanceof ModifyColumnCommentOp) { + ModifyColumnCommentOp modifyColumnComment = (ModifyColumnCommentOp) alterOp; + table.getCatalog().modifyColumnComment( + table, modifyColumnComment.getColumnPath(), modifyColumnComment.getComment()); } else if (alterOp instanceof ReorderColumnsOp) { ReorderColumnsOp reorderColumns = (ReorderColumnsOp) alterOp; table.getCatalog().reorderColumns(table, reorderColumns.getColumnsByPos()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPath.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPath.java new file mode 100644 index 00000000000000..efda051b3d8e97 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPath.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Represents a column path used by schema change statements. + */ +public class ColumnPath { + private final ImmutableList parts; + + private ColumnPath(List parts) { + Preconditions.checkArgument(parts != null && !parts.isEmpty(), "column path is empty"); + for (String part : parts) { + Preconditions.checkArgument(part != null && !part.isEmpty(), "column path contains empty part"); + } + this.parts = ImmutableList.copyOf(parts); + } + + public static ColumnPath of(List parts) { + return new ColumnPath(parts); + } + + public static ColumnPath of(String name) { + return new ColumnPath(ImmutableList.of(name)); + } + + public static ColumnPath fromDotName(String name) { + return new ColumnPath(Arrays.asList(name.split("\\."))); + } + + public List getParts() { + return parts; + } + + public boolean isNested() { + return parts.size() > 1; + } + + public String getTopLevelName() { + return parts.get(0); + } + + public String getLeafName() { + return parts.get(parts.size() - 1); + } + + public ColumnPath getParentPath() { + Preconditions.checkState(isNested(), "top-level column path has no parent"); + return new ColumnPath(parts.subList(0, parts.size() - 1)); + } + + public String getParentPathString() { + return getParentPath().getFullPath(); + } + + public String getFullPath() { + return String.join(".", parts); + } + + public String toSql() { + return parts.stream().map(part -> "`" + part + "`").collect(Collectors.joining(".")); + } + + @Override + public String toString() { + return getFullPath(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index c598ef520f5ef6..5c4decd93a30b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.TableIf; @@ -249,6 +250,15 @@ default void addColumn(TableIf table, Column column, ColumnPosition columnPositi throw new UserException("Not support add column operation"); } + default void addColumn(TableIf table, ColumnPath columnPath, Column column, ColumnPosition columnPosition) + throws UserException { + if (!columnPath.isNested()) { + addColumn(table, column, columnPosition); + return; + } + throw new UserException("Not support nested add column operation"); + } + default void addColumns(TableIf table, List columns) throws UserException { throw new UserException("Not support add columns operation"); } @@ -257,14 +267,47 @@ default void dropColumn(TableIf table, String name) throws UserException { throw new UserException("Not support drop column operation"); } + default void dropColumn(TableIf table, ColumnPath columnPath) throws UserException { + if (!columnPath.isNested()) { + dropColumn(table, columnPath.getTopLevelName()); + return; + } + throw new UserException("Not support nested drop column operation"); + } + default void renameColumn(TableIf table, String oldName, String newName) throws UserException { throw new UserException("Not support rename column operation"); } + default void renameColumn(TableIf table, ColumnPath columnPath, String newName) throws UserException { + if (!columnPath.isNested()) { + renameColumn(table, columnPath.getTopLevelName(), newName); + return; + } + throw new UserException("Not support nested rename column operation"); + } + default void modifyColumn(TableIf table, Column column, ColumnPosition columnPosition) throws UserException { throw new UserException("Not support update column operation"); } + default void modifyColumn(TableIf table, ColumnPath columnPath, Column column, ColumnPosition columnPosition) + throws UserException { + if (!columnPath.isNested()) { + modifyColumn(table, column, columnPosition); + return; + } + throw new UserException("Not support nested modify column operation"); + } + + default void modifyColumnComment(TableIf table, String name, String comment) throws UserException { + modifyColumnComment(table, ColumnPath.of(name), comment); + } + + default void modifyColumnComment(TableIf table, ColumnPath columnPath, String comment) throws UserException { + throw new UserException("Not support modify column comment operation"); + } + default void reorderColumns(TableIf table, List newOrder) throws UserException { throw new UserException("Not support reorder columns operation"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 44de21858698b6..8e114d1b5b43e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; @@ -1523,6 +1524,12 @@ private void logRefreshExternalTable(ExternalTable dorisTable, long updateTime) @Override public void addColumn(TableIf dorisTable, Column column, ColumnPosition position) throws UserException { + addColumn(dorisTable, ColumnPath.of(column.getName()), column, position); + } + + @Override + public void addColumn(TableIf dorisTable, ColumnPath columnPath, Column column, ColumnPosition position) + throws UserException { makeSureInitialized(); Preconditions.checkState(dorisTable instanceof ExternalTable, dorisTable.getName()); ExternalTable externalTable = (ExternalTable) dorisTable; @@ -1531,11 +1538,11 @@ public void addColumn(TableIf dorisTable, Column column, ColumnPosition position } try { long updateTime = System.currentTimeMillis(); - metadataOps.addColumn(externalTable, column, position, updateTime); + metadataOps.addColumn(externalTable, columnPath, column, position, updateTime); logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to add column {} to table {}.{} in catalog {}", - column.getName(), externalTable.getDbName(), externalTable.getName(), getName(), e); + columnPath.getFullPath(), externalTable.getDbName(), externalTable.getName(), getName(), e); throw e; } } @@ -1561,6 +1568,11 @@ public void addColumns(TableIf dorisTable, List columns) throws UserExce @Override public void dropColumn(TableIf dorisTable, String columnName) throws UserException { + dropColumn(dorisTable, ColumnPath.of(columnName)); + } + + @Override + public void dropColumn(TableIf dorisTable, ColumnPath columnPath) throws UserException { makeSureInitialized(); Preconditions.checkState(dorisTable instanceof ExternalTable, dorisTable.getName()); ExternalTable externalTable = (ExternalTable) dorisTable; @@ -1569,17 +1581,22 @@ public void dropColumn(TableIf dorisTable, String columnName) throws UserExcepti } try { long updateTime = System.currentTimeMillis(); - metadataOps.dropColumn(externalTable, columnName, updateTime); + metadataOps.dropColumn(externalTable, columnPath, updateTime); logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to drop column {} from table {}.{} in catalog {}", - columnName, externalTable.getDbName(), externalTable.getName(), getName(), e); + columnPath.getFullPath(), externalTable.getDbName(), externalTable.getName(), getName(), e); throw e; } } @Override public void renameColumn(TableIf dorisTable, String oldName, String newName) throws UserException { + renameColumn(dorisTable, ColumnPath.of(oldName), newName); + } + + @Override + public void renameColumn(TableIf dorisTable, ColumnPath columnPath, String newName) throws UserException { makeSureInitialized(); Preconditions.checkState(dorisTable instanceof ExternalTable, dorisTable.getName()); ExternalTable externalTable = (ExternalTable) dorisTable; @@ -1588,17 +1605,23 @@ public void renameColumn(TableIf dorisTable, String oldName, String newName) thr } try { long updateTime = System.currentTimeMillis(); - metadataOps.renameColumn(externalTable, oldName, newName, updateTime); + metadataOps.renameColumn(externalTable, columnPath, newName, updateTime); logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { - LOG.warn("Failed to rename column {} to {} in table {}.{} in catalog {}", - oldName, newName, externalTable.getDbName(), externalTable.getName(), getName(), e); + LOG.warn("Failed to rename column {} to {} in table {}.{} in catalog {}", columnPath.getFullPath(), + newName, externalTable.getDbName(), externalTable.getName(), getName(), e); throw e; } } @Override public void modifyColumn(TableIf dorisTable, Column column, ColumnPosition columnPosition) throws UserException { + modifyColumn(dorisTable, ColumnPath.of(column.getName()), column, columnPosition); + } + + @Override + public void modifyColumn(TableIf dorisTable, ColumnPath columnPath, Column column, ColumnPosition columnPosition) + throws UserException { makeSureInitialized(); Preconditions.checkState(dorisTable instanceof ExternalTable, dorisTable.getName()); ExternalTable externalTable = (ExternalTable) dorisTable; @@ -1607,11 +1630,30 @@ public void modifyColumn(TableIf dorisTable, Column column, ColumnPosition colum } try { long updateTime = System.currentTimeMillis(); - metadataOps.modifyColumn(externalTable, column, columnPosition, updateTime); + metadataOps.modifyColumn(externalTable, columnPath, column, columnPosition, updateTime); logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to modify column {} in table {}.{} in catalog {}", - column.getName(), externalTable.getDbName(), externalTable.getName(), getName(), e); + columnPath.getFullPath(), externalTable.getDbName(), externalTable.getName(), getName(), e); + throw e; + } + } + + @Override + public void modifyColumnComment(TableIf dorisTable, ColumnPath columnPath, String comment) throws UserException { + makeSureInitialized(); + Preconditions.checkState(dorisTable instanceof ExternalTable, dorisTable.getName()); + ExternalTable externalTable = (ExternalTable) dorisTable; + if (metadataOps == null) { + throw new DdlException("Modify column comment operation is not supported for catalog: " + getName()); + } + try { + long updateTime = System.currentTimeMillis(); + metadataOps.modifyColumnComment(externalTable, columnPath, comment, updateTime); + logRefreshExternalTable(externalTable, updateTime); + } catch (Exception e) { + LOG.warn("Failed to modify column comment {} in table {}.{} in catalog {}", + columnPath.getFullPath(), externalTable.getDbName(), externalTable.getName(), getName(), e); throw e; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 7bc4cf7d35e94f..f0fa21530e200b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.ColumnType; @@ -765,13 +766,48 @@ private void addOneColumn(UpdateSchema updateSchema, Column column) throws UserE } private void applyPosition(UpdateSchema updateSchema, ColumnPosition position, String columnName) { + applyPosition(updateSchema, position, ColumnPath.of(columnName)); + } + + private void applyPosition(UpdateSchema updateSchema, ColumnPosition position, ColumnPath columnPath) { + String columnName = columnPath.getFullPath(); if (position.isFirst()) { updateSchema.moveFirst(columnName); } else { - updateSchema.moveAfter(columnName, position.getLastCol()); + updateSchema.moveAfter(columnName, getPositionReferencePath(columnPath, position)); } } + private void applyPosition(UpdateSchema updateSchema, ColumnPosition position, ColumnPath columnPath, Schema schema, + String operation) throws UserException { + String columnName = columnPath.getFullPath(); + if (position.isFirst()) { + updateSchema.moveFirst(columnName); + } else { + updateSchema.moveAfter(columnName, getPositionReferencePath(schema, columnPath, position, operation)); + } + } + + @VisibleForTesting + String getPositionReferencePath(ColumnPath columnPath, ColumnPosition position) { + if (position == null || position.isFirst() || !columnPath.isNested()) { + return position == null || position.isFirst() ? null : position.getLastCol(); + } + return columnPath.getParentPathString() + "." + position.getLastCol(); + } + + @VisibleForTesting + String getPositionReferencePath(Schema schema, ColumnPath columnPath, ColumnPosition position, String operation) + throws UserException { + if (position == null || position.isFirst()) { + return null; + } + ColumnPath referencePath = columnPath.isNested() + ? childPath(columnPath.getParentPath(), position.getLastCol()) + : ColumnPath.of(position.getLastCol()); + return resolveColumnPath(schema, referencePath, operation).getFullPath(); + } + private void refreshTable(ExternalTable dorisTable, long updateTime) { Optional> db = dorisCatalog.getDbForReplay(dorisTable.getRemoteDbName()); if (db.isPresent()) { @@ -791,7 +827,7 @@ public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition po UpdateSchema updateSchema = icebergTable.updateSchema(); addOneColumn(updateSchema, column); if (position != null) { - applyPosition(updateSchema, position, column.getName()); + applyPosition(updateSchema, position, ColumnPath.of(column.getName()), icebergTable.schema(), "add"); } try { executionAuthenticator.execute(() -> updateSchema.commit()); @@ -802,6 +838,42 @@ public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition po refreshTable(dorisTable, updateTime); } + @Override + public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, + long updateTime) throws UserException { + if (!columnPath.isNested()) { + addColumn(dorisTable, column, position, updateTime); + return; + } + validateCommonColumnInfo(column); + if (!column.isAllowNull()) { + throw new UserException("New nested field '" + columnPath.getFullPath() + "' must be nullable"); + } + Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "add"); + if (!parentPath.getType().isStructType()) { + throw new UserException("Parent column path '" + columnPath.getParentPathString() + + "' is not a struct in Iceberg table: " + icebergTable.name()); + } + + UpdateSchema updateSchema = icebergTable.updateSchema(); + org.apache.iceberg.types.Type dorisType = IcebergUtils.dorisTypeToIcebergType(column.getType()); + Literal defaultValue = IcebergUtils.parseIcebergLiteral(column.getDefaultValue(), dorisType); + updateSchema.addColumn(parentPath.getFullPath(), columnPath.getLeafName(), dorisType, + column.getComment(), defaultValue); + if (position != null) { + applyPosition(updateSchema, position, childPath(parentPath.getColumnPath(), columnPath.getLeafName()), + icebergTable.schema(), "add"); + } + try { + executionAuthenticator.execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to add nested column: " + columnPath.getFullPath() + " to table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } + refreshTable(dorisTable, updateTime); + } + @Override public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); @@ -822,8 +894,9 @@ public void addColumns(ExternalTable dorisTable, List columns, long upda @Override public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(columnName), "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); - updateSchema.deleteColumn(columnName); + updateSchema.deleteColumn(columnPath.getFullPath()); try { executionAuthenticator.execute(() -> updateSchema.commit()); } catch (Exception e) { @@ -833,12 +906,33 @@ public void dropColumn(ExternalTable dorisTable, String columnName, long updateT refreshTable(dorisTable, updateTime); } + @Override + public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long updateTime) throws UserException { + if (!columnPath.isNested()) { + dropColumn(dorisTable, columnPath.getTopLevelName(), updateTime); + return; + } + Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "drop"); + + UpdateSchema updateSchema = icebergTable.updateSchema(); + updateSchema.deleteColumn(resolvedPath.getFullPath()); + try { + executionAuthenticator.execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to drop nested column: " + columnPath.getFullPath() + " from table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } + refreshTable(dorisTable, updateTime); + } + @Override public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + ResolvedColumnPath oldPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(oldName), "rename"); UpdateSchema updateSchema = icebergTable.updateSchema(); - updateSchema.renameColumn(oldName, newName); + updateSchema.renameColumn(oldPath.getFullPath(), newName); try { executionAuthenticator.execute(() -> updateSchema.commit()); } catch (Exception e) { @@ -849,13 +943,33 @@ public void renameColumn(ExternalTable dorisTable, String oldName, String newNam } @Override - public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) + public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String newName, long updateTime) throws UserException { + if (!columnPath.isNested()) { + renameColumn(dorisTable, columnPath.getTopLevelName(), newName, updateTime); + return; + } Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); - NestedField currentCol = icebergTable.schema().findField(column.getName()); - if (currentCol == null) { - throw new UserException("Column " + column.getName() + " does not exist"); + ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "rename"); + + UpdateSchema updateSchema = icebergTable.updateSchema(); + updateSchema.renameColumn(resolvedPath.getFullPath(), newName); + try { + executionAuthenticator.execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to rename nested column: " + columnPath.getFullPath() + " to " + newName + + " in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } + refreshTable(dorisTable, updateTime); + } + + @Override + public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) + throws UserException { + Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(column.getName()), + "modify"); + NestedField currentCol = columnPath.getField(); validateCommonColumnInfo(column); UpdateSchema updateSchema = icebergTable.updateSchema(); @@ -863,27 +977,27 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition if (column.getType().isComplexType()) { // Complex type processing branch validateForModifyComplexColumn(column, currentCol); - applyComplexTypeChange(updateSchema, column.getName(), currentCol.type(), column.getType()); + applyComplexTypeChange(updateSchema, columnPath.getFullPath(), currentCol.type(), column.getType()); if (column.isAllowNull()) { - updateSchema.makeColumnOptional(column.getName()); + updateSchema.makeColumnOptional(columnPath.getFullPath()); } if (!Objects.equals(currentCol.doc(), column.getComment())) { - updateSchema.updateColumnDoc(column.getName(), column.getComment()); + updateSchema.updateColumnDoc(columnPath.getFullPath(), column.getComment()); } } else { // Primitive type processing (existing logic) validateForModifyColumn(column, currentCol); Type icebergType = IcebergUtils.dorisTypeToIcebergType(column.getType()); - updateSchema.updateColumn(column.getName(), icebergType.asPrimitiveType(), column.getComment()); + updateSchema.updateColumn(columnPath.getFullPath(), icebergType.asPrimitiveType(), column.getComment()); if (column.isAllowNull()) { // we can change a required column to optional, but not the other way around // because we don't know whether there is existing data with null values. - updateSchema.makeColumnOptional(column.getName()); + updateSchema.makeColumnOptional(columnPath.getFullPath()); } } if (position != null) { - applyPosition(updateSchema, position, column.getName()); + applyPosition(updateSchema, position, columnPath.getColumnPath(), icebergTable.schema(), "modify"); } try { executionAuthenticator.execute(() -> updateSchema.commit()); @@ -894,18 +1008,96 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition refreshTable(dorisTable, updateTime); } + @Override + public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, + long updateTime) throws UserException { + if (!columnPath.isNested()) { + modifyColumn(dorisTable, column, position, updateTime); + return; + } + + Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "modify"); + NestedField currentCol = resolvedPath.getField(); + + validateCommonColumnInfo(column); + UpdateSchema updateSchema = icebergTable.updateSchema(); + + if (column.getType().isComplexType()) { + validateForModifyComplexColumn(column, currentCol, columnPath.getFullPath()); + applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), column.getType()); + if (column.isAllowNull()) { + updateSchema.makeColumnOptional(resolvedPath.getFullPath()); + } + if (!Objects.equals(currentCol.doc(), column.getComment())) { + updateSchema.updateColumnDoc(resolvedPath.getFullPath(), column.getComment()); + } + } else { + validateForModifyColumn(column, currentCol, columnPath.getFullPath()); + Type icebergType = IcebergUtils.dorisTypeToIcebergType(column.getType()); + updateSchema.updateColumn(resolvedPath.getFullPath(), icebergType.asPrimitiveType(), column.getComment()); + if (column.isAllowNull()) { + updateSchema.makeColumnOptional(resolvedPath.getFullPath()); + } + } + + if (position != null) { + applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); + } + + try { + executionAuthenticator.execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to modify nested column: " + columnPath.getFullPath() + " in table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } + refreshTable(dorisTable, updateTime); + } + + @Override + public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) + throws UserException { + Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + ResolvedColumnPath resolvedPath; + if (columnPath.isNested()) { + resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "modify comment"); + } else { + resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify comment"); + } + + UpdateSchema updateSchema = icebergTable.updateSchema(); + updateSchema.updateColumnDoc(resolvedPath.getFullPath(), StringUtils.defaultString(comment)); + try { + executionAuthenticator.execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to modify column comment: " + columnPath.getFullPath() + " in table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } + refreshTable(dorisTable, updateTime); + } + private void validateForModifyColumn(Column column, NestedField currentCol) throws UserException { + validateForModifyColumn(column, currentCol, column.getName()); + } + + private void validateForModifyColumn(Column column, NestedField currentCol, String columnPath) + throws UserException { // check complex type if (column.getType().isComplexType()) { throw new UserException("Modify column type to non-primitive type is not supported: " + column.getType()); } // check nullable if (currentCol.isOptional() && !column.isAllowNull()) { - throw new UserException("Can not change nullable column " + column.getName() + " to not null"); + throw new UserException("Can not change nullable column " + columnPath + " to not null"); } } private void validateForModifyComplexColumn(Column column, NestedField currentCol) throws UserException { + validateForModifyComplexColumn(column, currentCol, column.getName()); + } + + private void validateForModifyComplexColumn(Column column, NestedField currentCol, String columnPath) + throws UserException { if (!column.getType().isComplexType()) { throw new UserException("Modify column type to non-complex type is not supported: " + column.getType()); } @@ -927,13 +1119,133 @@ private void validateForModifyComplexColumn(Column column, NestedField currentCo throw new UserException(e.getMessage(), e); } if (currentCol.isOptional() && !column.isAllowNull()) { - throw new UserException("Cannot change nullable column " + column.getName() + " to not null"); + throw new UserException("Cannot change nullable column " + columnPath + " to not null"); } if (column.getDefaultValue() != null || column.getDefaultValueExprDef() != null) { throw new UserException("Complex type default value only supports NULL"); } } + @VisibleForTesting + org.apache.iceberg.types.Type resolveNestedColumnPath(Schema schema, ColumnPath columnPath, String operation) + throws UserException { + return resolveColumnPath(schema, columnPath, operation).getType(); + } + + @VisibleForTesting + String getCanonicalColumnPath(Schema schema, ColumnPath columnPath, String operation) throws UserException { + return resolveColumnPath(schema, columnPath, operation).getFullPath(); + } + + private ResolvedColumnPath resolveColumnPath(Schema schema, ColumnPath columnPath, String operation) + throws UserException { + org.apache.iceberg.types.Type currentType = schema.asStruct(); + NestedField currentField = null; + String currentPath = ""; + List canonicalParts = new ArrayList<>(); + for (String part : columnPath.getParts()) { + if (!currentPath.isEmpty()) { + currentPath += "."; + } + currentPath += part; + + if (currentType.isStructType()) { + NestedField field = currentType.asStructType().caseInsensitiveField(part); + if (field == null) { + throw new UserException("Column path does not exist in Iceberg schema: " + + columnPath.getFullPath()); + } + canonicalParts.add(field.name()); + currentField = field; + currentType = field.type(); + } else if (currentType.isListType()) { + Types.ListType listType = currentType.asListType(); + NestedField elementField = listType.field(listType.elementId()); + if (!elementField.name().equalsIgnoreCase(part)) { + throw new UserException("Expected array element path at '" + currentPath + + "' for Iceberg column path: " + columnPath.getFullPath()); + } + canonicalParts.add(elementField.name()); + currentField = elementField; + currentType = listType.elementType(); + } else if (currentType.isMapType()) { + Types.MapType mapType = currentType.asMapType(); + NestedField keyField = mapType.field(mapType.keyId()); + if (keyField.name().equalsIgnoreCase(part)) { + throw new UserException("Cannot " + operation + " MAP key nested column: " + + columnPath.getFullPath()); + } + NestedField valueField = mapType.field(mapType.valueId()); + if (!valueField.name().equalsIgnoreCase(part)) { + throw new UserException("Expected map value path at '" + currentPath + + "' for Iceberg column path: " + columnPath.getFullPath()); + } + canonicalParts.add(valueField.name()); + currentField = valueField; + currentType = mapType.valueType(); + } else { + throw new UserException("Cannot resolve nested field under primitive column path: " + + columnPath.getFullPath()); + } + } + return new ResolvedColumnPath(ColumnPath.of(canonicalParts), currentType, currentField); + } + + @VisibleForTesting + org.apache.iceberg.types.Type validateNestedStructField(Schema schema, ColumnPath columnPath, String operation) + throws UserException { + return validateNestedStructFieldPath(schema, columnPath, operation).getType(); + } + + private ResolvedColumnPath validateNestedStructFieldPath(Schema schema, ColumnPath columnPath, String operation) + throws UserException { + ResolvedColumnPath parentPath = resolveColumnPath(schema, columnPath.getParentPath(), operation); + org.apache.iceberg.types.Type parentType = parentPath.getType(); + if (!parentType.isStructType()) { + throw new UserException("Parent column path '" + columnPath.getParentPathString() + + "' is not a struct for Iceberg nested " + operation + ": " + columnPath.getFullPath()); + } + NestedField field = parentType.asStructType().caseInsensitiveField(columnPath.getLeafName()); + if (field == null) { + throw new UserException("Column path does not exist in Iceberg schema: " + columnPath.getFullPath()); + } + return new ResolvedColumnPath(childPath(parentPath.getColumnPath(), field.name()), field.type(), field); + } + + private ColumnPath childPath(ColumnPath parentPath, String childName) { + List parts = new ArrayList<>(parentPath.getParts()); + parts.add(childName); + return ColumnPath.of(parts); + } + + private static class ResolvedColumnPath { + private final ColumnPath columnPath; + private final org.apache.iceberg.types.Type type; + private final NestedField field; + + private ResolvedColumnPath(ColumnPath columnPath, org.apache.iceberg.types.Type type, NestedField field) { + this.columnPath = columnPath; + this.type = type; + this.field = field; + } + + private ColumnPath getColumnPath() { + return columnPath; + } + + private String getFullPath() { + return columnPath.getFullPath(); + } + + private org.apache.iceberg.types.Type getType() { + return type; + } + + private NestedField getField() { + return field; + } + } + private boolean isSameComplexCategory(Type oldIcebergType, org.apache.doris.catalog.Type newDorisType) { switch (oldIcebergType.typeId()) { case STRUCT: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java index b7f6331306861c..f513eb10fca7a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.operations; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.info.ColumnPosition; import org.apache.doris.catalog.info.CreateOrReplaceBranchInfo; @@ -226,6 +227,15 @@ default void addColumn(ExternalTable dorisTable, Column column, ColumnPosition p throw new UnsupportedOperationException("Add column operation is not supported for this table type."); } + default void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, + long updateTime) throws UserException { + if (!columnPath.isNested()) { + addColumn(dorisTable, column, position, updateTime); + return; + } + throw new UnsupportedOperationException("Nested add column operation is not supported for this table type."); + } + /** * add columns for external table * @@ -250,6 +260,15 @@ default void dropColumn(ExternalTable dorisTable, String columnName, long update throw new UnsupportedOperationException("Drop column operation is not supported for this table type."); } + default void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long updateTime) + throws UserException { + if (!columnPath.isNested()) { + dropColumn(dorisTable, columnPath.getTopLevelName(), updateTime); + return; + } + throw new UnsupportedOperationException("Nested drop column operation is not supported for this table type."); + } + /** * rename column for external table * @@ -263,6 +282,15 @@ default void renameColumn(ExternalTable dorisTable, String oldName, String newNa throw new UnsupportedOperationException("Rename column operation is not supported for this table type."); } + default void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String newName, long updateTime) + throws UserException { + if (!columnPath.isNested()) { + renameColumn(dorisTable, columnPath.getTopLevelName(), newName, updateTime); + return; + } + throw new UnsupportedOperationException("Nested rename column operation is not supported for this table type."); + } + /** * update column for external table * @@ -276,6 +304,30 @@ default void modifyColumn(ExternalTable dorisTable, Column column, ColumnPositio throw new UnsupportedOperationException("Modify column operation is not supported for this table type."); } + default void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, + long updateTime) throws UserException { + if (!columnPath.isNested()) { + modifyColumn(dorisTable, column, position, updateTime); + return; + } + throw new UnsupportedOperationException("Nested modify column operation is not supported for this table type."); + } + + /** + * modify column comment for external table + * + * @param dorisTable + * @param columnPath + * @param comment + * @param updateTime + * @throws UserException + */ + default void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) + throws UserException { + throw new UnsupportedOperationException( + "Modify column comment operation is not supported for this table type."); + } + /** * reorder columns for external table * diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c470bc789d9e78..74254cc2c5860d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.ArithmeticExpr.Operator; import org.apache.doris.analysis.BrokerDesc; import org.apache.doris.analysis.ColumnNullableType; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.analysis.DbName; import org.apache.doris.analysis.EncryptKeyName; import org.apache.doris.analysis.PassVar; @@ -140,6 +141,7 @@ import org.apache.doris.nereids.DorisParser.CleanLabelContext; import org.apache.doris.nereids.DorisParser.CollateContext; import org.apache.doris.nereids.DorisParser.ColumnDefContext; +import org.apache.doris.nereids.DorisParser.ColumnDefWithPathContext; import org.apache.doris.nereids.DorisParser.ColumnDefsContext; import org.apache.doris.nereids.DorisParser.ColumnReferenceContext; import org.apache.doris.nereids.DorisParser.CommentRelationHintContext; @@ -1154,6 +1156,16 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { private static String DEFAULT_NESTED_COLUMN_NAME = "unnest"; private static String DEFAULT_ORDINALITY_COLUMN_NAME = "ordinality"; + private static class ColumnDefinitionWithPath { + private final ColumnDefinition columnDefinition; + private final ColumnPath columnPath; + + private ColumnDefinitionWithPath(ColumnDefinition columnDefinition, ColumnPath columnPath) { + this.columnDefinition = columnDefinition; + this.columnPath = columnPath; + } + } + // Sort the parameters with token position to keep the order with original placeholders // in prepared statement.Otherwise, the order maybe broken private final Map tokenPosToParameters = Maps.newTreeMap((pos1, pos2) -> { @@ -3906,6 +3918,13 @@ public List visitIdentifierSeq(IdentifierSeqContext ctx) { .collect(ImmutableList.toImmutableList()); } + @Override + public List visitQualifiedName(QualifiedNameContext ctx) { + return ctx.identifier().stream() + .map(RuleContext::getText) + .collect(ImmutableList.toImmutableList()); + } + @Override public EqualTo visitUpdateAssignment(UpdateAssignmentContext ctx) { return new EqualTo(new UnboundSlot( @@ -4234,6 +4253,102 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { onUpdateDefaultValue, comment, desc); } + @Override + public ColumnDefinitionWithPath visitColumnDefWithPath(ColumnDefWithPathContext ctx) { + ColumnPath columnPath = ColumnPath.of(visitQualifiedName(ctx.colName)); + String colName = columnPath.getLeafName(); + DataType colType = ctx.type instanceof PrimitiveDataTypeContext + ? visitPrimitiveDataType(((PrimitiveDataTypeContext) ctx.type)) + : ctx.type instanceof ComplexDataTypeContext + ? visitComplexDataType((ComplexDataTypeContext) ctx.type) + : ctx.type instanceof VariantPredefinedFieldsContext + ? visitVariantPredefinedFields((VariantPredefinedFieldsContext) ctx.type) + : visitAggStateDataType((AggStateDataTypeContext) ctx.type); + colType = colType.conversion(); + boolean isKey = ctx.KEY() != null; + ColumnNullableType nullableType = ColumnNullableType.DEFAULT; + if (ctx.NOT() != null) { + nullableType = ColumnNullableType.NOT_NULLABLE; + } else if (ctx.nullable != null) { + nullableType = ColumnNullableType.NULLABLE; + } + String aggTypeString = ctx.aggType != null ? ctx.aggType.getText() : null; + Optional defaultValue = Optional.empty(); + Optional onUpdateDefaultValue = Optional.empty(); + if (ctx.DEFAULT() != null) { + if (ctx.INTEGER_VALUE() != null) { + if (ctx.SUBTRACT() == null) { + defaultValue = Optional.of(new DefaultValue(ctx.INTEGER_VALUE().getText())); + } else { + defaultValue = Optional.of(new DefaultValue("-" + ctx.INTEGER_VALUE().getText())); + } + } else if (ctx.DECIMAL_VALUE() != null) { + if (ctx.SUBTRACT() == null) { + defaultValue = Optional.of(new DefaultValue(ctx.DECIMAL_VALUE().getText())); + } else { + defaultValue = Optional.of(new DefaultValue("-" + ctx.DECIMAL_VALUE().getText())); + } + } else if (ctx.stringValue != null) { + defaultValue = Optional.of(new DefaultValue(toStringValue(ctx.stringValue.getText()))); + } else if (ctx.nullValue != null) { + defaultValue = Optional.of(DefaultValue.NULL_DEFAULT_VALUE); + } else if (ctx.defaultTimestamp != null) { + if (ctx.defaultValuePrecision == null) { + defaultValue = Optional.of(DefaultValue.CURRENT_TIMESTAMP_DEFAULT_VALUE); + } else { + defaultValue = Optional.of(DefaultValue + .currentTimeStampDefaultValueWithPrecision( + Long.valueOf(ctx.defaultValuePrecision.getText()))); + } + } else if (ctx.CURRENT_DATE() != null) { + defaultValue = Optional.of(DefaultValue.CURRENT_DATE_DEFAULT_VALUE); + } else if (ctx.PI() != null) { + defaultValue = Optional.of(DefaultValue.PI_DEFAULT_VALUE); + } else if (ctx.E() != null) { + defaultValue = Optional.of(DefaultValue.E_NUM_DEFAULT_VALUE); + } else if (ctx.BITMAP_EMPTY() != null) { + defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); + } + } + if (ctx.UPDATE() != null) { + if (ctx.onUpdateValuePrecision == null) { + onUpdateDefaultValue = Optional.of(DefaultValue.CURRENT_TIMESTAMP_DEFAULT_VALUE); + } else { + onUpdateDefaultValue = Optional.of(DefaultValue + .currentTimeStampDefaultValueWithPrecision( + Long.valueOf(ctx.onUpdateValuePrecision.getText()))); + } + } + AggregateType aggType = null; + if (aggTypeString != null) { + try { + aggType = AggregateType.valueOf(aggTypeString.toUpperCase()); + } catch (Exception e) { + throw new AnalysisException(String.format("Aggregate type %s is unsupported", aggTypeString), + e.getCause()); + } + } + String comment = ctx.comment != null ? ctx.comment.getText().substring(1, ctx.comment.getText().length() - 1) + .replace("\\", "") : ""; + long autoIncInitValue = -1; + if (ctx.AUTO_INCREMENT() != null) { + if (ctx.autoIncInitValue != null) { + autoIncInitValue = Long.valueOf(ctx.autoIncInitValue.getText()); + if (autoIncInitValue < 0) { + throw new AnalysisException("AUTO_INCREMENT start value can not be negative."); + } + } else { + autoIncInitValue = Long.valueOf(1); + } + } + Optional desc = ctx.generatedExpr != null + ? Optional.of(new GeneratedColumnDesc(ctx.generatedExpr.getText(), getExpression(ctx.generatedExpr))) + : Optional.empty(); + ColumnDefinition columnDefinition = new ColumnDefinition(colName, colType, isKey, aggType, nullableType, + autoIncInitValue, defaultValue, onUpdateDefaultValue, comment, desc); + return new ColumnDefinitionWithPath(columnDefinition, columnPath); + } + @Override public List visitIndexDefs(IndexDefsContext ctx) { return ctx.indexes.stream().map(this::visitIndexDef).collect(Collectors.toList()); @@ -6105,7 +6220,7 @@ public AlterTableCommand visitAlterTableProperties(DorisParser.AlterTablePropert @Override public AlterTableOp visitAddColumnClause(AddColumnClauseContext ctx) { - ColumnDefinition columnDefinition = visitColumnDef(ctx.columnDef()); + ColumnDefinitionWithPath columnDefinitionWithPath = visitColumnDefWithPath(ctx.columnDefWithPath()); ColumnPosition columnPosition = null; if (ctx.columnPosition() != null) { if (ctx.columnPosition().FIRST() != null) { @@ -6118,7 +6233,8 @@ public AlterTableOp visitAddColumnClause(AddColumnClauseContext ctx) { Map properties = ctx.properties != null ? Maps.newHashMap(visitPropertyClause(ctx.properties)) : Maps.newHashMap(); - return new AddColumnOp(columnDefinition, columnPosition, rollupName, properties); + return new AddColumnOp(columnDefinitionWithPath.columnDefinition, columnDefinitionWithPath.columnPath, + columnPosition, rollupName, properties); } @Override @@ -6133,17 +6249,17 @@ public AlterTableOp visitAddColumnsClause(AddColumnsClauseContext ctx) { @Override public AlterTableOp visitDropColumnClause(DropColumnClauseContext ctx) { - String columnName = ctx.name.getText(); + ColumnPath columnPath = ColumnPath.of(visitQualifiedName(ctx.name)); String rollupName = ctx.fromRollup() != null ? ctx.fromRollup().rollup.getText() : null; Map properties = ctx.properties != null ? Maps.newHashMap(visitPropertyClause(ctx.properties)) : Maps.newHashMap(); - return new DropColumnOp(columnName, rollupName, properties); + return new DropColumnOp(columnPath, rollupName, properties); } @Override public AlterTableOp visitModifyColumnClause(ModifyColumnClauseContext ctx) { - ColumnDefinition columnDefinition = visitColumnDef(ctx.columnDef()); + ColumnDefinitionWithPath columnDefinitionWithPath = visitColumnDefWithPath(ctx.columnDefWithPath()); ColumnPosition columnPosition = null; if (ctx.columnPosition() != null) { if (ctx.columnPosition().FIRST() != null) { @@ -6156,7 +6272,8 @@ public AlterTableOp visitModifyColumnClause(ModifyColumnClauseContext ctx) { Map properties = ctx.properties != null ? Maps.newHashMap(visitPropertyClause(ctx.properties)) : Maps.newHashMap(); - return new ModifyColumnOp(columnDefinition, columnPosition, rollupName, properties); + return new ModifyColumnOp(columnDefinitionWithPath.columnDefinition, columnDefinitionWithPath.columnPath, + columnPosition, rollupName, properties); } @Override @@ -6359,7 +6476,7 @@ public AlterTableOp visitRenamePartitionClause(RenamePartitionClauseContext ctx) @Override public AlterTableOp visitRenameColumnClause(RenameColumnClauseContext ctx) { - return new RenameColumnOp(ctx.name.getText(), ctx.newName.getText()); + return new RenameColumnOp(ColumnPath.of(visitQualifiedName(ctx.name)), ctx.newName.getText()); } @Override @@ -6467,9 +6584,9 @@ public AlterTableOp visitModifyTableCommentClause(ModifyTableCommentClauseContex @Override public AlterTableOp visitModifyColumnCommentClause(ModifyColumnCommentClauseContext ctx) { - String columnName = ctx.name.getText(); + ColumnPath columnPath = ColumnPath.of(visitQualifiedName(ctx.name)); String comment = stripQuotes(ctx.STRING_LITERAL().getText()); - return new ModifyColumnCommentOp(columnName, comment); + return new ModifyColumnCommentOp(columnPath, comment); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java index 86b085740e2925..2db06ba7c12b62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.analysis.StmtType; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; @@ -36,6 +37,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.InternalDatabaseUtil; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.info.AddColumnOp; @@ -52,6 +54,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.DropRollupOp; import org.apache.doris.nereids.trees.plans.commands.info.DropTagOp; import org.apache.doris.nereids.trees.plans.commands.info.EnableFeatureOp; +import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnCommentOp; import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnOp; import org.apache.doris.nereids.trees.plans.commands.info.ModifyEngineOp; import org.apache.doris.nereids.trees.plans.commands.info.ModifyTablePropertiesOp; @@ -115,10 +118,6 @@ private void validate(ConnectContext ctx) throws UserException { if (ops == null || ops.isEmpty()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_ALTER_OPERATION); } - for (AlterTableOp op : ops) { - op.setTableName(tbl); - op.validate(ctx); - } String ctlName = tbl.getCtl(); String dbName = tbl.getDb(); String tableName = tbl.getTbl(); @@ -129,6 +128,11 @@ private void validate(ConnectContext ctx) throws UserException { if (tableIf.isTemporary()) { throw new AnalysisException("Do not support alter temporary table[" + tableName + "]"); } + checkNestedColumnPathSupported(tableIf, ops); + for (AlterTableOp op : ops) { + op.setTableName(tbl); + op.validate(ctx); + } if (tableIf instanceof OlapTable) { rewriteAlterOpForOlapTable(ctx, (OlapTable) tableIf); } else { @@ -136,6 +140,36 @@ private void validate(ConnectContext ctx) throws UserException { } } + static void checkNestedColumnPathSupported(TableIf table, List alterTableOps) + throws AnalysisException { + if (table instanceof IcebergExternalTable) { + return; + } + for (AlterTableOp alterTableOp : alterTableOps) { + ColumnPath columnPath = getNestedColumnPath(alterTableOp); + if (columnPath != null) { + throw new AnalysisException("Nested column path is only supported for Iceberg tables: " + + columnPath.getFullPath()); + } + } + } + + private static ColumnPath getNestedColumnPath(AlterTableOp alterTableOp) { + ColumnPath columnPath = null; + if (alterTableOp instanceof AddColumnOp) { + columnPath = ((AddColumnOp) alterTableOp).getColumnPath(); + } else if (alterTableOp instanceof DropColumnOp) { + columnPath = ((DropColumnOp) alterTableOp).getColumnPath(); + } else if (alterTableOp instanceof RenameColumnOp) { + columnPath = ((RenameColumnOp) alterTableOp).getColumnPath(); + } else if (alterTableOp instanceof ModifyColumnOp) { + columnPath = ((ModifyColumnOp) alterTableOp).getColumnPath(); + } else if (alterTableOp instanceof ModifyColumnCommentOp) { + columnPath = ((ModifyColumnCommentOp) alterTableOp).getColumnPath(); + } + return columnPath != null && columnPath.isNested() ? columnPath : null; + } + private void rewriteAlterOpForOlapTable(ConnectContext ctx, OlapTable table) throws UserException { List alterTableOps = new ArrayList<>(); for (AlterTableOp alterClause : ops) { @@ -242,6 +276,7 @@ private void checkExternalTableOperationAllow(TableIf table) throws UserExceptio || alterTableOp instanceof DropColumnOp || alterTableOp instanceof RenameColumnOp || alterTableOp instanceof ModifyColumnOp + || (alterTableOp instanceof ModifyColumnCommentOp && table instanceof IcebergExternalTable) || alterTableOp instanceof ReorderColumnsOp || alterTableOp instanceof ModifyEngineOp || alterTableOp instanceof ModifyTablePropertiesOp diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java index 09c686779a862c..10ced240ee50ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; @@ -44,6 +45,7 @@ */ public class AddColumnOp extends AlterTableOp { private ColumnDefinition columnDef; + private ColumnPath columnPath; // Column position private ColumnPosition colPos; // if rollupName is null, add to column to base index. @@ -56,8 +58,17 @@ public class AddColumnOp extends AlterTableOp { public AddColumnOp(ColumnDefinition columnDef, ColumnPosition colPos, String rollupName, Map properties) { + this(columnDef, ColumnPath.of(columnDef.getName()), colPos, rollupName, properties); + } + + /** + * Create add-column operation with the original nested column path. + */ + public AddColumnOp(ColumnDefinition columnDef, ColumnPath columnPath, ColumnPosition colPos, + String rollupName, Map properties) { super(AlterOpType.SCHEMA_CHANGE); this.columnDef = columnDef; + this.columnPath = columnPath; this.colPos = colPos; this.rollupName = rollupName; this.properties = properties; @@ -67,6 +78,10 @@ public Column getColumn() { return column; } + public ColumnPath getColumnPath() { + return columnPath; + } + public ColumnPosition getColPos() { return colPos; } @@ -111,7 +126,7 @@ public boolean allowOpRowBinlog() { @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("ADD COLUMN ").append(columnDef.toSql()); + sb.append("ADD COLUMN ").append(columnDef.toSql(columnPath.toSql())); if (colPos != null) { sb.append(" ").append(colPos.toSql()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 4be6315f079fdb..dbd37b4886e927 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -207,8 +207,15 @@ public String getComment(boolean escapeQuota) { * toSql */ public String toSql() { + return toSql("`" + name + "`"); + } + + /** + * Convert this column definition to SQL with a caller-provided column name. + */ + public String toSql(String columnNameSql) { StringBuilder sb = new StringBuilder(); - sb.append("`").append(name).append("` "); + sb.append(columnNameSql).append(" "); sb.append(type.toSql()).append(" "); if (aggType != null && aggType != AggregateType.NONE) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java index be5e91c8a0b7d4..3a95246790941d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; @@ -42,6 +43,7 @@ */ public class DropColumnOp extends AlterTableOp { private String colName; + private ColumnPath columnPath; private String rollupName; private Map properties; @@ -50,8 +52,13 @@ public class DropColumnOp extends AlterTableOp { * DropColumnOp */ public DropColumnOp(String colName, String rollupName, Map properties) { + this(ColumnPath.fromDotName(colName), rollupName, properties); + } + + public DropColumnOp(ColumnPath columnPath, String rollupName, Map properties) { super(AlterOpType.SCHEMA_CHANGE); - this.colName = colName; + this.colName = columnPath.getLeafName(); + this.columnPath = columnPath; this.rollupName = rollupName; this.properties = properties; } @@ -60,6 +67,10 @@ public String getColName() { return colName; } + public ColumnPath getColumnPath() { + return columnPath; + } + public String getRollupName() { return rollupName; } @@ -171,7 +182,7 @@ public boolean allowOpRowBinlog() { @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("DROP COLUMN `").append(colName).append("`"); + sb.append("DROP COLUMN ").append(columnPath.toSql()); if (rollupName != null) { sb.append(" FROM `").append(rollupName).append("`"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java index 0b1c6ac49558e2..49ea3d8bcb8f27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.qe.ConnectContext; @@ -31,17 +32,25 @@ * ModifyColumnCommentOp */ public class ModifyColumnCommentOp extends AlterTableOp { - private String colName; + private ColumnPath columnPath; private String comment; public ModifyColumnCommentOp(String colName, String comment) { + this(ColumnPath.fromDotName(colName), comment); + } + + public ModifyColumnCommentOp(ColumnPath columnPath, String comment) { super(AlterOpType.MODIFY_COLUMN_COMMENT); - this.colName = colName; + this.columnPath = columnPath; this.comment = Strings.nullToEmpty(comment); } public String getColName() { - return colName; + return columnPath.getFullPath(); + } + + public ColumnPath getColumnPath() { + return columnPath; } public String getComment() { @@ -55,7 +64,7 @@ public Map getProperties() { @Override public void validate(ConnectContext ctx) throws UserException { - if (Strings.isNullOrEmpty(colName)) { + if (columnPath == null || Strings.isNullOrEmpty(columnPath.getFullPath())) { throw new AnalysisException("Empty column name"); } } @@ -79,8 +88,8 @@ public boolean allowOpRowBinlog() { @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("MODIFY COLUMN COMMENT ").append(colName); - sb.append(" '").append(comment).append("'"); + sb.append("MODIFY COLUMN ").append(columnPath.toSql()); + sb.append(" COMMENT '").append(comment).append("'"); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnOp.java index 049d12bc325ae4..94b6c5986e698e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnOp.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; @@ -43,6 +44,7 @@ */ public class ModifyColumnOp extends AlterTableOp { private ColumnDefinition columnDef; + private ColumnPath columnPath; private ColumnPosition colPos; // which rollup is to be modify, if rollup is null, modify base table. private String rollupName; @@ -54,8 +56,17 @@ public class ModifyColumnOp extends AlterTableOp { public ModifyColumnOp(ColumnDefinition columnDef, ColumnPosition colPos, String rollup, Map properties) { + this(columnDef, ColumnPath.of(columnDef.getName()), colPos, rollup, properties); + } + + /** + * Create modify-column operation with the original nested column path. + */ + public ModifyColumnOp(ColumnDefinition columnDef, ColumnPath columnPath, ColumnPosition colPos, + String rollup, Map properties) { super(AlterOpType.SCHEMA_CHANGE); this.columnDef = columnDef; + this.columnPath = columnPath; this.colPos = colPos; this.rollupName = rollup; this.properties = properties; @@ -65,6 +76,10 @@ public Column getColumn() { return column; } + public ColumnPath getColumnPath() { + return columnPath; + } + public ColumnPosition getColPos() { return colPos; } @@ -187,7 +202,7 @@ public boolean needChangeMTMVState() { @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("MODIFY COLUMN ").append(columnDef.toSql()); + sb.append("MODIFY COLUMN ").append(columnDef.toSql(columnPath.toSql())); if (colPos != null) { sb.append(" ").append(colPos); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java index e66ae02c15337b..976fe258e15192 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.Column; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeNameFormat; @@ -33,11 +34,17 @@ */ public class RenameColumnOp extends AlterTableOp { private String colName; + private ColumnPath columnPath; private String newColName; public RenameColumnOp(String colName, String newColName) { + this(ColumnPath.fromDotName(colName), newColName); + } + + public RenameColumnOp(ColumnPath columnPath, String newColName) { super(AlterOpType.RENAME); - this.colName = colName; + this.colName = columnPath.getLeafName(); + this.columnPath = columnPath; this.newColName = newColName; } @@ -45,6 +52,10 @@ public String getColName() { return colName; } + public ColumnPath getColumnPath() { + return columnPath; + } + public String getNewColName() { return newColName; } @@ -83,7 +94,7 @@ public boolean needChangeMTMVState() { @Override public String toSql() { - return "RENAME COLUMN " + colName + " " + newColName; + return "RENAME COLUMN " + columnPath.toSql() + " " + newColName; } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 937167c34c351d..3c906df8e0ffbd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MapType; @@ -24,10 +25,12 @@ import org.apache.doris.catalog.StructField; import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.Type; +import org.apache.doris.catalog.info.ColumnPosition; import org.apache.doris.common.UserException; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.iceberg.Schema; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.types.Types; @@ -182,6 +185,87 @@ public void testValidateForModifyComplexColumnSuccess() throws Throwable { invokeValidateForModifyComplexColumn(column, currentCol); } + @Test + public void testResolveNestedColumnPathSupportsStructArrayElementAndMapValue() throws Throwable { + Schema schema = nestedSchema(); + Assert.assertTrue(ops.resolveNestedColumnPath(schema, ColumnPath.fromDotName("s"), "add").isStructType()); + Assert.assertTrue(ops.resolveNestedColumnPath(schema, ColumnPath.fromDotName("arr.element"), "add") + .isStructType()); + Assert.assertTrue(ops.resolveNestedColumnPath(schema, ColumnPath.fromDotName("m.value"), "add") + .isStructType()); + } + + @Test + public void testResolveNestedColumnPathUsesCaseInsensitiveCanonicalIcebergPath() throws Throwable { + Schema schema = mixedCaseNestedSchema(); + Assert.assertEquals("Info.Metric", + ops.getCanonicalColumnPath(schema, ColumnPath.fromDotName("info.metric"), "modify")); + Assert.assertEquals("Events.element.Score", + ops.getCanonicalColumnPath(schema, ColumnPath.fromDotName("events.element.score"), "modify")); + Assert.assertEquals("Attrs.value.Code", + ops.getCanonicalColumnPath(schema, ColumnPath.fromDotName("attrs.value.code"), "modify")); + Assert.assertEquals("Info.Label", + ops.getPositionReferencePath(schema, ColumnPath.fromDotName("Info.NewField"), + new ColumnPosition("label"), "add")); + } + + @Test + public void testResolveNestedColumnPathRejectsMapKey() { + assertUserException(() -> ops.resolveNestedColumnPath(nestedSchema(), ColumnPath.fromDotName("m.key.x"), + "modify"), + "Cannot modify MAP key nested column"); + } + + @Test + public void testResolveNestedColumnPathRejectsPrimitiveParent() { + assertUserException(() -> ops.resolveNestedColumnPath(nestedSchema(), ColumnPath.fromDotName("id.x"), + "modify"), + "Cannot resolve nested field under primitive column path"); + } + + @Test + public void testGetPositionReferencePathForNestedColumn() { + Assert.assertEquals("s.a", ops.getPositionReferencePath(ColumnPath.fromDotName("s.new_col"), + new ColumnPosition("a"))); + Assert.assertEquals("arr.element.x", ops.getPositionReferencePath( + ColumnPath.fromDotName("arr.element.new_col"), new ColumnPosition("x"))); + Assert.assertEquals("m.value.v", ops.getPositionReferencePath( + ColumnPath.fromDotName("m.value.new_col"), new ColumnPosition("v"))); + Assert.assertEquals("id", ops.getPositionReferencePath(ColumnPath.fromDotName("new_col"), + new ColumnPosition("id"))); + } + + @Test + public void testValidateNestedStructFieldSupportsStructArrayElementAndMapValueFields() throws Throwable { + Schema schema = nestedSchema(); + Assert.assertTrue(ops.validateNestedStructField(schema, ColumnPath.fromDotName("s.a"), "drop") + .isPrimitiveType()); + Assert.assertTrue(ops.validateNestedStructField(schema, ColumnPath.fromDotName("arr.element.x"), "drop") + .isPrimitiveType()); + Assert.assertTrue(ops.validateNestedStructField(schema, ColumnPath.fromDotName("m.value.v"), "rename") + .isPrimitiveType()); + } + + @Test + public void testValidateNestedStructFieldRejectsArrayElementAndMapValuePseudoFields() { + assertUserException(() -> ops.validateNestedStructField(nestedSchema(), ColumnPath.fromDotName("arr.element"), + "drop"), + "Parent column path 'arr' is not a struct"); + assertUserException(() -> ops.validateNestedStructField(nestedSchema(), ColumnPath.fromDotName("m.value"), + "rename"), + "Parent column path 'm' is not a struct"); + } + + @Test + public void testValidateNestedStructFieldRejectsMapKeyAndMissingField() { + assertUserException(() -> ops.validateNestedStructField(nestedSchema(), ColumnPath.fromDotName("m.key.k"), + "drop"), + "Cannot drop MAP key nested column"); + assertUserException(() -> ops.validateNestedStructField(nestedSchema(), ColumnPath.fromDotName("s.missing"), + "rename"), + "Column path does not exist in Iceberg schema"); + } + private void invokeValidateForModifyColumn(Column column, NestedField currentCol) throws Throwable { invokeValidationMethod(validateForModifyColumnMethod, column, currentCol); } @@ -212,4 +296,29 @@ private void assertUserException(ThrowingRunnable runnable, String expectedMessa private interface ThrowingRunnable { void run() throws Throwable; } + + private Schema nestedSchema() { + return new Schema( + Types.NestedField.optional(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "s", Types.StructType.of( + Types.NestedField.optional(3, "a", Types.IntegerType.get()))), + Types.NestedField.optional(4, "arr", Types.ListType.ofOptional(5, + Types.StructType.of(Types.NestedField.optional(6, "x", Types.IntegerType.get())))), + Types.NestedField.optional(7, "m", Types.MapType.ofOptional(8, 9, + Types.StringType.get(), + Types.StructType.of(Types.NestedField.optional(10, "v", Types.IntegerType.get()))))); + } + + private Schema mixedCaseNestedSchema() { + return new Schema( + Types.NestedField.optional(1, "Id", Types.IntegerType.get()), + Types.NestedField.optional(2, "Info", Types.StructType.of( + Types.NestedField.optional(3, "Metric", Types.IntegerType.get()), + Types.NestedField.optional(4, "Label", Types.StringType.get()))), + Types.NestedField.optional(5, "Events", Types.ListType.ofOptional(6, + Types.StructType.of(Types.NestedField.optional(7, "Score", Types.IntegerType.get())))), + Types.NestedField.optional(8, "Attrs", Types.MapType.ofOptional(9, 10, + Types.StringType.get(), + Types.StructType.of(Types.NestedField.optional(11, "Code", Types.IntegerType.get()))))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java new file mode 100644 index 00000000000000..dbf7d49a21e0a5 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand; +import org.apache.doris.nereids.trees.plans.commands.info.AddColumnOp; +import org.apache.doris.nereids.trees.plans.commands.info.AlterTableOp; +import org.apache.doris.nereids.trees.plans.commands.info.DropColumnOp; +import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnCommentOp; +import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnOp; +import org.apache.doris.nereids.trees.plans.commands.info.RenameColumnOp; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class IcebergNestedSchemaEvolutionParserTest { + + private final NereidsParser parser = new NereidsParser(); + + @Test + public void testParseNestedAddColumnPaths() { + assertSingleClausePath("ALTER TABLE t ADD COLUMN s.c STRING NULL", + AddColumnOp.class, "s.c"); + assertSingleClausePath("ALTER TABLE t ADD COLUMN s.first_col INT NULL FIRST", + AddColumnOp.class, "s.first_col"); + assertSingleClausePath("ALTER TABLE t ADD COLUMN s.after_col INT NULL AFTER a", + AddColumnOp.class, "s.after_col"); + assertSingleClausePath("ALTER TABLE t ADD COLUMN arr.element.y INT NULL", + AddColumnOp.class, "arr.element.y"); + assertSingleClausePath("ALTER TABLE t ADD COLUMN m.value.y INT NULL", + AddColumnOp.class, "m.value.y"); + } + + @Test + public void testParseNestedModifyDropRenamePaths() { + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN s.a BIGINT", + ModifyColumnOp.class, "s.a"); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN s.a BIGINT AFTER b", + ModifyColumnOp.class, "s.a"); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN s.a COMMENT 'nested comment'", + ModifyColumnCommentOp.class, "s.a"); + assertSingleClausePath("ALTER TABLE t DROP COLUMN s.c", + DropColumnOp.class, "s.c"); + assertSingleClausePath("ALTER TABLE t DROP COLUMN arr.element.y", + DropColumnOp.class, "arr.element.y"); + assertSingleClausePath("ALTER TABLE t DROP COLUMN m.value.y", + DropColumnOp.class, "m.value.y"); + assertSingleClausePath("ALTER TABLE t RENAME COLUMN s.c TO c2", + RenameColumnOp.class, "s.c"); + assertSingleClausePath("ALTER TABLE t RENAME COLUMN arr.element.y TO y2", + RenameColumnOp.class, "arr.element.y"); + assertSingleClausePath("ALTER TABLE t RENAME COLUMN m.value.y TO y2", + RenameColumnOp.class, "m.value.y"); + } + + private void assertSingleClausePath(String sql, Class clauseClass, + String expectedPath) { + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(AlterTableCommand.class, plan); + List clauses = ((AlterTableCommand) plan).getOps(); + Assertions.assertEquals(1, clauses.size()); + AlterTableOp clause = clauses.get(0); + Assertions.assertInstanceOf(clauseClass, clause); + if (clause instanceof AddColumnOp) { + Assertions.assertEquals(expectedPath, ((AddColumnOp) clause).getColumnPath().getFullPath()); + } else if (clause instanceof ModifyColumnOp) { + Assertions.assertEquals(expectedPath, ((ModifyColumnOp) clause).getColumnPath().getFullPath()); + } else if (clause instanceof ModifyColumnCommentOp) { + Assertions.assertEquals(expectedPath, ((ModifyColumnCommentOp) clause).getColumnPath().getFullPath()); + } else if (clause instanceof DropColumnOp) { + Assertions.assertEquals(expectedPath, ((DropColumnOp) clause).getColumnPath().getFullPath()); + } else if (clause instanceof RenameColumnOp) { + Assertions.assertEquals(expectedPath, ((RenameColumnOp) clause).getColumnPath().getFullPath()); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index db64fe354d5cc3..7e0151e25fe383 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -17,7 +17,12 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.info.TableNameInfo; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.info.AddPartitionFieldOp; import org.apache.doris.nereids.trees.plans.commands.info.AlterTableOp; import org.apache.doris.nereids.trees.plans.commands.info.DropPartitionFieldOp; @@ -26,13 +31,17 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; public class AlterTableCommandTest { + private final NereidsParser parser = new NereidsParser(); + @Test void testEnableFeatureOp() { List ops = new ArrayList<>(); @@ -155,4 +164,33 @@ void testReplacePartitionFieldOp() { "ALTER TABLE `internal`.`db`.`test` REPLACE PARTITION KEY bucket(16, id) WITH truncate(5, code) AS code_trunc", alterTableCommand.toSql()); } + + @Test + void testRejectNestedColumnPathForNonIcebergTable() { + TableIf table = Mockito.mock(TableIf.class); + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN s.c STRING NULL", + "ALTER TABLE t MODIFY COLUMN s.a BIGINT", + "ALTER TABLE t MODIFY COLUMN s.a COMMENT 'nested comment'", + "ALTER TABLE t DROP COLUMN s.c", + "ALTER TABLE t RENAME COLUMN s.c TO c2")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkNestedColumnPathSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("Nested column path is only supported for Iceberg tables")); + } + } + + @Test + void testAllowNestedColumnPathForIcebergTable() throws AnalysisException { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + AlterTableCommand.checkNestedColumnPathSupported(table, + parseAlter("ALTER TABLE t ADD COLUMN s.c STRING NULL").getOps()); + } + + private AlterTableCommand parseAlter(String sql) { + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(AlterTableCommand.class, plan); + return (AlterTableCommand) plan; + } } diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index cd2391b1775f1e..d85f99c48597c5 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -771,11 +771,11 @@ addRollupClause ; alterTableClause - : ADD COLUMN columnDef columnPosition? toRollup? properties=propertyClause? #addColumnClause + : ADD COLUMN columnDefWithPath columnPosition? toRollup? properties=propertyClause? #addColumnClause | ADD COLUMN LEFT_PAREN columnDefs RIGHT_PAREN toRollup? properties=propertyClause? #addColumnsClause - | DROP COLUMN name=identifier fromRollup? properties=propertyClause? #dropColumnClause - | MODIFY COLUMN columnDef columnPosition? fromRollup? + | DROP COLUMN name=qualifiedName fromRollup? properties=propertyClause? #dropColumnClause + | MODIFY COLUMN columnDefWithPath columnPosition? fromRollup? properties=propertyClause? #modifyColumnClause | ORDER BY identifierList fromRollup? properties=propertyClause? #reorderColumnsClause | ADD TEMPORARY? partitionDef @@ -794,14 +794,14 @@ alterTableClause | RENAME newName=identifier #renameClause | RENAME ROLLUP name=identifier newName=identifier #renameRollupClause | RENAME PARTITION name=identifier newName=identifier #renamePartitionClause - | RENAME COLUMN name=identifier newName=identifier #renameColumnClause + | RENAME COLUMN name=qualifiedName TO? newName=identifier #renameColumnClause | ADD indexDef #addIndexClause | DROP INDEX (IF EXISTS)? name=identifier partitionSpec? #dropIndexClause | ENABLE FEATURE name=STRING_LITERAL (WITH properties=propertyClause)? #enableFeatureClause | MODIFY DISTRIBUTION (DISTRIBUTED BY (HASH hashKeys=identifierList | RANDOM) (BUCKETS (INTEGER_VALUE | autoBucket=AUTO))?)? #modifyDistributionClause | MODIFY COMMENT comment=STRING_LITERAL #modifyTableCommentClause - | MODIFY COLUMN name=identifier COMMENT comment=STRING_LITERAL #modifyColumnCommentClause + | MODIFY COLUMN name=qualifiedName COMMENT comment=STRING_LITERAL #modifyColumnCommentClause | MODIFY ENGINE TO name=identifier properties=propertyClause? #modifyEngineClause | ADD TEMPORARY? PARTITIONS FROM from=partitionValueList TO to=partitionValueList @@ -1554,6 +1554,19 @@ columnDef (COMMENT comment=STRING_LITERAL)? ; +columnDefWithPath + : colName=qualifiedName type=dataType + KEY? + (aggType=aggTypeDef)? + ((GENERATED ALWAYS)? AS LEFT_PAREN generatedExpr=expression RIGHT_PAREN)? + ((NOT)? nullable=NULL)? + (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)? + (DEFAULT (nullValue=NULL | SUBTRACT? INTEGER_VALUE | SUBTRACT? DECIMAL_VALUE | PI | E | BITMAP_EMPTY | stringValue=STRING_LITERAL + | CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))? + (ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)? + (COMMENT comment=STRING_LITERAL)? + ; + indexDefs : indexes+=indexDef (COMMA indexes+=indexDef)* ; diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out new file mode 100644 index 00000000000000..9f0b31155e9c66 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc -- +id int Yes true \N +s struct Yes true \N +arr array> Yes true \N +m map> Yes true \N + +-- !query_rows -- +1 \N 10 \N \N 100 \N \N 1000 \N \N +2 first 20 after_a c2 200 202 201 2000 2002 2001 diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy new file mode 100644 index 00000000000000..c75a8dbffbdcd7 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_docker,external_docker_doris") { + + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_nested_schema_evolution_ddl" + String dbName = "iceberg_nested_schema_evolution_db" + String tableName = "iceberg_nested_evolution" + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${restPort}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1" + );""" + + sql """switch ${catalogName};""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName};""" + + sql """set enable_fallback_to_original_planner=false;""" + sql """set show_column_comment_in_describe=true;""" + + sql """drop table if exists ${tableName}""" + sql """ + CREATE TABLE ${tableName} ( + id INT NOT NULL, + s STRUCT, + arr ARRAY>, + m MAP> + ); + """ + + sql """ + INSERT INTO ${tableName} VALUES ( + 1, + STRUCT(10, 'old'), + ARRAY(STRUCT(100)), + MAP('k', STRUCT(1000)) + ) + """ + + sql """ALTER TABLE ${tableName} ADD COLUMN s.c STRING NULL COMMENT 'new nested field'""" + sql """ALTER TABLE ${tableName} ADD COLUMN s.first_pos STRING NULL FIRST""" + sql """ALTER TABLE ${tableName} ADD COLUMN s.after_a STRING NULL AFTER a""" + sql """ALTER TABLE ${tableName} ADD COLUMN arr.element.y INT NULL""" + sql """ALTER TABLE ${tableName} ADD COLUMN arr.element.after_x INT NULL AFTER x""" + sql """ALTER TABLE ${tableName} ADD COLUMN m.value.y INT NULL""" + sql """ALTER TABLE ${tableName} ADD COLUMN m.value.after_v INT NULL AFTER v""" + sql """ALTER TABLE ${tableName} ADD COLUMN s.drop_me STRING NULL""" + sql """ALTER TABLE ${tableName} ADD COLUMN arr.element.drop_me INT NULL""" + sql """ALTER TABLE ${tableName} ADD COLUMN m.value.drop_me INT NULL""" + + test { + sql """ALTER TABLE ${tableName} ADD COLUMN s.required_field INT NOT NULL""" + exception "Field 'required_field' doesn't have a default value" + } + + sql """ALTER TABLE ${tableName} MODIFY COLUMN s.a BIGINT""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN arr.element.x BIGINT""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN m.value.v BIGINT""" + + sql """ALTER TABLE ${tableName} RENAME COLUMN s.c TO c2""" + sql """ALTER TABLE ${tableName} RENAME COLUMN arr.element.y TO y2""" + sql """ALTER TABLE ${tableName} RENAME COLUMN m.value.y TO y2""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN s.c2 COMMENT 'renamed struct field'""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN arr.element.y2 COMMENT 'renamed array element field'""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN m.value.y2 COMMENT 'renamed map value field'""" + sql """ALTER TABLE ${tableName} DROP COLUMN s.drop_me""" + sql """ALTER TABLE ${tableName} DROP COLUMN arr.element.drop_me""" + sql """ALTER TABLE ${tableName} DROP COLUMN m.value.drop_me""" + + sql """ + INSERT INTO ${tableName} VALUES ( + 2, + STRUCT('first', 20, 'after_a', 'new', 'c2'), + ARRAY(STRUCT(200, 202, 201)), + MAP('k', STRUCT(2000, 2002, 2001)) + ) + """ + + qt_desc """DESC ${tableName}""" + + def descRows = sql """DESC ${tableName}""" + def normalizeType = { String s -> s.toLowerCase().replaceAll("\\s+", "") } + def typeOf = { String name -> + def row = descRows.find { it[0].toString().equalsIgnoreCase(name) } + assertTrue(row != null, "column not found: ${name}") + return normalizeType(row[1].toString()) + } + + def sType = typeOf("s") + assertTrue(sType.contains("first_pos:text"), sType) + assertTrue(sType.contains("a:bigint"), sType) + assertTrue(sType.contains("after_a:text"), sType) + assertTrue(sType.contains("c2:text"), sType) + assertTrue(!sType.contains("drop_me"), sType) + + def arrType = typeOf("arr") + assertTrue(arrType.contains("x:bigint"), arrType) + assertTrue(arrType.contains("after_x:int"), arrType) + assertTrue(arrType.contains("y2:int"), arrType) + assertTrue(!arrType.contains("drop_me"), arrType) + + def mType = typeOf("m") + assertTrue(mType.contains("v:bigint"), mType) + assertTrue(mType.contains("after_v:int"), mType) + assertTrue(mType.contains("y2:int"), mType) + assertTrue(!mType.contains("drop_me"), mType) + + order_qt_query_rows """ + SELECT id, + element_at(s, 'first_pos'), + element_at(s, 'a'), + element_at(s, 'after_a'), + element_at(s, 'c2'), + element_at(arr[1], 'x'), + element_at(arr[1], 'after_x'), + element_at(arr[1], 'y2'), + element_at(m['k'], 'v'), + element_at(m['k'], 'after_v'), + element_at(m['k'], 'y2') + FROM ${tableName} + ORDER BY id + """ + + sql """drop table if exists ${tableName}""" + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy new file mode 100644 index 00000000000000..89d3c463c6e9b0 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy @@ -0,0 +1,324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String catalogName = "test_iceberg_nested_schema_evolution_spark_doris_interop" + String dbName = "iceberg_nested_schema_evolution_interop_db" + String dorisTable = "doris_nested_evolution_to_spark" + String sparkTable = "spark_nested_evolution_to_doris" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${restPort}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0" + ); + """ + + try { + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner=false""" + + spark_iceberg """CREATE DATABASE IF NOT EXISTS demo.${dbName}""" + + sql """ + CREATE TABLE ${dorisTable} ( + id INT NOT NULL, + info STRUCT, + events ARRAY>, + attrs MAP> + ) + """ + sql """ + INSERT INTO ${dorisTable} VALUES ( + 1, + STRUCT(10, 'doris_before'), + ARRAY(STRUCT(100)), + MAP('k', STRUCT(1000)) + ) + """ + + sql """ALTER TABLE ${dorisTable} ADD COLUMN info.doris_added STRING NULL COMMENT 'added by doris'""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN info.doris_first STRING NULL FIRST""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN info.doris_after_metric STRING NULL AFTER metric""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.doris_score INT NULL""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.doris_first_score INT NULL FIRST""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.doris_after_score INT NULL AFTER score""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.doris_code INT NULL""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.doris_first_code INT NULL FIRST""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.doris_after_code INT NULL AFTER code""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN info.drop_me STRING NULL""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.drop_me INT NULL""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.drop_me INT NULL""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN info.metric BIGINT""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN events.element.score BIGINT""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN attrs.value.code BIGINT""" + sql """ALTER TABLE ${dorisTable} RENAME COLUMN info.doris_added TO doris_renamed""" + sql """ALTER TABLE ${dorisTable} RENAME COLUMN events.element.doris_score TO doris_score_renamed""" + sql """ALTER TABLE ${dorisTable} RENAME COLUMN attrs.value.doris_code TO doris_code_renamed""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN info.doris_renamed COMMENT 'renamed by doris'""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN events.element.doris_score_renamed COMMENT 'renamed by doris'""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN attrs.value.doris_code_renamed COMMENT 'renamed by doris'""" + sql """ALTER TABLE ${dorisTable} DROP COLUMN info.drop_me""" + sql """ALTER TABLE ${dorisTable} DROP COLUMN events.element.drop_me""" + sql """ALTER TABLE ${dorisTable} DROP COLUMN attrs.value.drop_me""" + + spark_iceberg """REFRESH TABLE demo.${dbName}.${dorisTable}""" + spark_iceberg """ + INSERT INTO demo.${dbName}.${dorisTable} VALUES ( + 2, + NAMED_STRUCT('doris_first', 'spark_first', + 'metric', CAST(20 AS BIGINT), + 'doris_after_metric', 'spark_after_metric', + 'label', 'spark_after_doris', + 'doris_renamed', 'spark_can_write_doris_field'), + ARRAY(NAMED_STRUCT('doris_first_score', 202, + 'score', CAST(200 AS BIGINT), + 'doris_after_score', 203, + 'doris_score_renamed', 201)), + MAP('k', NAMED_STRUCT('doris_first_code', 2002, + 'code', CAST(2000 AS BIGINT), + 'doris_after_code', 2003, + 'doris_code_renamed', 2001)) + ) + """ + + sql """refresh table ${dbName}.${dorisTable}""" + def dorisDrivenSparkRows = spark_iceberg """ + SELECT id, + info.doris_first, + info.metric, + info.doris_after_metric, + info.label, + info.doris_renamed, + events[0].doris_first_score, + events[0].score, + events[0].doris_after_score, + events[0].doris_score_renamed, + attrs['k'].doris_first_code, + attrs['k'].code, + attrs['k'].doris_after_code, + attrs['k'].doris_code_renamed + FROM demo.${dbName}.${dorisTable} + ORDER BY id + """ + def dorisDrivenDorisRows = sql """ + SELECT id, + element_at(info, 'doris_first'), + element_at(info, 'metric'), + element_at(info, 'doris_after_metric'), + element_at(info, 'label'), + element_at(info, 'doris_renamed'), + element_at(events[1], 'doris_first_score'), + element_at(events[1], 'score'), + element_at(events[1], 'doris_after_score'), + element_at(events[1], 'doris_score_renamed'), + element_at(attrs['k'], 'doris_first_code'), + element_at(attrs['k'], 'code'), + element_at(attrs['k'], 'doris_after_code'), + element_at(attrs['k'], 'doris_code_renamed') + FROM ${dorisTable} + ORDER BY id + """ + assertSparkDorisResultEquals(dorisDrivenSparkRows, dorisDrivenDorisRows) + + spark_iceberg_multi """ + DROP TABLE IF EXISTS demo.${dbName}.${sparkTable}; + CREATE TABLE demo.${dbName}.${sparkTable} ( + id INT, + info STRUCT, + events ARRAY>, + attrs MAP> + ) USING iceberg; + INSERT INTO demo.${dbName}.${sparkTable} VALUES ( + 1, + NAMED_STRUCT('metric', 10, 'label', 'spark_before'), + ARRAY(NAMED_STRUCT('score', 100)), + MAP('k', NAMED_STRUCT('code', 1000)) + ); + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.spark_added STRING; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.spark_first STRING FIRST; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.spark_after_metric STRING AFTER metric; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.spark_score INT; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.spark_first_score INT FIRST; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.spark_after_score INT AFTER score; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.spark_code INT; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.spark_first_code INT FIRST; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.spark_after_code INT AFTER code; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.drop_me STRING; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.drop_me INT; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.drop_me INT; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN info.metric TYPE BIGINT; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN events.element.score TYPE BIGINT; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN attrs.value.code TYPE BIGINT; + ALTER TABLE demo.${dbName}.${sparkTable} RENAME COLUMN info.spark_added TO spark_renamed; + ALTER TABLE demo.${dbName}.${sparkTable} RENAME COLUMN events.element.spark_score TO spark_score_renamed; + ALTER TABLE demo.${dbName}.${sparkTable} RENAME COLUMN attrs.value.spark_code TO spark_code_renamed; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN info.spark_renamed COMMENT 'renamed by spark'; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN events.element.spark_score_renamed COMMENT 'renamed by spark'; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN attrs.value.spark_code_renamed COMMENT 'renamed by spark'; + ALTER TABLE demo.${dbName}.${sparkTable} DROP COLUMN info.drop_me; + ALTER TABLE demo.${dbName}.${sparkTable} DROP COLUMN events.element.drop_me; + ALTER TABLE demo.${dbName}.${sparkTable} DROP COLUMN attrs.value.drop_me; + INSERT INTO demo.${dbName}.${sparkTable} VALUES ( + 2, + NAMED_STRUCT('spark_first', 'spark_first_field', + 'metric', CAST(20 AS BIGINT), + 'spark_after_metric', 'spark_after_metric_field', + 'label', 'spark_after', + 'spark_renamed', 'spark_new_field'), + ARRAY(NAMED_STRUCT('spark_first_score', 202, + 'score', CAST(200 AS BIGINT), + 'spark_after_score', 203, + 'spark_score_renamed', 201)), + MAP('k', NAMED_STRUCT('spark_first_code', 2002, + 'code', CAST(2000 AS BIGINT), + 'spark_after_code', 2003, + 'spark_code_renamed', 2001)) + ); + """ + + sql """refresh catalog ${catalogName}""" + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + def sparkDrivenDescRows = sql """DESC ${sparkTable}""" + String sparkDrivenDesc = sparkDrivenDescRows.toString().toLowerCase().replaceAll("\\s+", "") + assertTrue(sparkDrivenDesc.contains("spark_first"), sparkDrivenDesc) + assertTrue(sparkDrivenDesc.contains("metric:bigint"), sparkDrivenDesc) + assertTrue(sparkDrivenDesc.contains("spark_after_metric"), sparkDrivenDesc) + assertTrue(sparkDrivenDesc.contains("spark_renamed"), sparkDrivenDesc) + assertTrue(!sparkDrivenDesc.contains("drop_me"), sparkDrivenDesc) + assertTrue(sparkDrivenDesc.contains("spark_first_score:int"), sparkDrivenDesc) + assertTrue(sparkDrivenDesc.contains("score:bigint"), sparkDrivenDesc) + assertTrue(sparkDrivenDesc.contains("spark_after_score:int"), sparkDrivenDesc) + assertTrue(sparkDrivenDesc.contains("spark_score_renamed:int"), sparkDrivenDesc) + assertTrue(sparkDrivenDesc.contains("spark_first_code:int"), sparkDrivenDesc) + assertTrue(sparkDrivenDesc.contains("code:bigint"), sparkDrivenDesc) + assertTrue(sparkDrivenDesc.contains("spark_after_code:int"), sparkDrivenDesc) + assertTrue(sparkDrivenDesc.contains("spark_code_renamed:int"), sparkDrivenDesc) + + def sparkDrivenDorisRowsBeforeWrite = sql """ + SELECT id, + element_at(info, 'spark_first'), + element_at(info, 'metric'), + element_at(info, 'spark_after_metric'), + element_at(info, 'label'), + element_at(info, 'spark_renamed'), + element_at(events[1], 'spark_first_score'), + element_at(events[1], 'score'), + element_at(events[1], 'spark_after_score'), + element_at(events[1], 'spark_score_renamed'), + element_at(attrs['k'], 'spark_first_code'), + element_at(attrs['k'], 'code'), + element_at(attrs['k'], 'spark_after_code'), + element_at(attrs['k'], 'spark_code_renamed') + FROM ${sparkTable} + ORDER BY id + """ + assertTrue(sparkDrivenDorisRowsBeforeWrite.size() == 2, sparkDrivenDorisRowsBeforeWrite.toString()) + assertTrue(sparkDrivenDorisRowsBeforeWrite[0][1] == null, sparkDrivenDorisRowsBeforeWrite.toString()) + assertTrue(sparkDrivenDorisRowsBeforeWrite[0][3] == null, sparkDrivenDorisRowsBeforeWrite.toString()) + assertTrue(sparkDrivenDorisRowsBeforeWrite[0][5] == null, sparkDrivenDorisRowsBeforeWrite.toString()) + assertTrue(sparkDrivenDorisRowsBeforeWrite[0][6] == null, sparkDrivenDorisRowsBeforeWrite.toString()) + assertTrue(sparkDrivenDorisRowsBeforeWrite[0][8] == null, sparkDrivenDorisRowsBeforeWrite.toString()) + assertTrue(sparkDrivenDorisRowsBeforeWrite[0][9] == null, sparkDrivenDorisRowsBeforeWrite.toString()) + assertTrue(sparkDrivenDorisRowsBeforeWrite[0][11] == null, sparkDrivenDorisRowsBeforeWrite.toString()) + assertTrue(sparkDrivenDorisRowsBeforeWrite[0][12] == null, sparkDrivenDorisRowsBeforeWrite.toString()) + + sql """ + INSERT INTO ${sparkTable} VALUES ( + 3, + STRUCT('doris_first_field', 30, 'doris_after_metric_field', + 'doris_after_spark', 'doris_can_write_spark_field'), + ARRAY(STRUCT(302, 300, 303, 301)), + MAP('k', STRUCT(3002, 3000, 3003, 3001)) + ) + """ + + spark_iceberg """REFRESH TABLE demo.${dbName}.${sparkTable}""" + def sparkDrivenSparkRows = spark_iceberg """ + SELECT id, + info.spark_first, + info.metric, + info.spark_after_metric, + info.label, + info.spark_renamed, + events[0].spark_first_score, + events[0].score, + events[0].spark_after_score, + events[0].spark_score_renamed, + attrs['k'].spark_first_code, + attrs['k'].code, + attrs['k'].spark_after_code, + attrs['k'].spark_code_renamed + FROM demo.${dbName}.${sparkTable} + ORDER BY id + """ + def sparkDrivenDorisRows = sql """ + SELECT id, + element_at(info, 'spark_first'), + element_at(info, 'metric'), + element_at(info, 'spark_after_metric'), + element_at(info, 'label'), + element_at(info, 'spark_renamed'), + element_at(events[1], 'spark_first_score'), + element_at(events[1], 'score'), + element_at(events[1], 'spark_after_score'), + element_at(events[1], 'spark_score_renamed'), + element_at(attrs['k'], 'spark_first_code'), + element_at(attrs['k'], 'code'), + element_at(attrs['k'], 'spark_after_code'), + element_at(attrs['k'], 'spark_code_renamed') + FROM ${sparkTable} + ORDER BY id + """ + assertSparkDorisResultEquals(sparkDrivenSparkRows, sparkDrivenDorisRows) + } finally { + try { + spark_iceberg """DROP TABLE IF EXISTS demo.${dbName}.${dorisTable}""" + spark_iceberg """DROP TABLE IF EXISTS demo.${dbName}.${sparkTable}""" + } catch (Throwable t) { + logger.warn("failed to clean Spark Iceberg tables", t) + } + try { + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + } catch (Throwable t) { + logger.warn("failed to clean Doris Iceberg database", t) + } + sql """drop catalog if exists ${catalogName}""" + } +} From 6dbe65c4a955d737335c1d1d6f9d5e25d016b284 Mon Sep 17 00:00:00 2001 From: daidai Date: Sun, 12 Jul 2026 16:18:16 +0800 Subject: [PATCH 02/32] [fix](iceberg) Fix nested schema evolution CI failures ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Nested schema evolution changed the missing top-level MODIFY COLUMN error contract and two new regression tests asserted an incorrect DESCRIBE shape and wrong field indexes. Restore case-insensitive top-level lookup with the legacy error message, use the default DESCRIBE shape with generated output, and validate the actual newly added map fields. ### Release note None ### Check List (For Author) - Test: Unit Test, FE build, and Regression test - Unit Test: IcebergMetadataOpsValidationTest, IcebergNestedSchemaEvolutionParserTest, AlterTableCommandTest - Regression test: test_iceberg_nested_schema_evolution_ddl, iceberg_schema_change_ddl, test_iceberg_nested_schema_evolution_spark_doris_interop - Behavior changed: Yes. Missing top-level Iceberg columns preserve the existing error contract. - Does this need documentation: No --- .../doris/datasource/iceberg/IcebergMetadataOps.java | 9 ++++++--- .../iceberg/test_iceberg_nested_schema_evolution_ddl.out | 9 +++++---- .../test_iceberg_nested_schema_evolution_ddl.groovy | 1 - ...rg_nested_schema_evolution_spark_doris_interop.groovy | 3 ++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index f0fa21530e200b..a0127a2da39752 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -967,9 +967,12 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); - ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(column.getName()), - "modify"); - NestedField currentCol = columnPath.getField(); + NestedField currentCol = icebergTable.schema().caseInsensitiveFindField(column.getName()); + if (currentCol == null) { + throw new UserException("Column " + column.getName() + " does not exist"); + } + ResolvedColumnPath columnPath = new ResolvedColumnPath(ColumnPath.of(currentCol.name()), + currentCol.type(), currentCol); validateCommonColumnInfo(column); UpdateSchema updateSchema = icebergTable.updateSchema(); diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out index 9f0b31155e9c66..e00b1b5b67b88e 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out @@ -1,10 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !desc -- -id int Yes true \N -s struct Yes true \N -arr array> Yes true \N -m map> Yes true \N +id int Yes true \N +s struct Yes true \N +arr array> Yes true \N +m map> Yes true \N -- !query_rows -- 1 \N 10 \N \N 100 \N \N 1000 \N \N 2 first 20 after_a c2 200 202 201 2000 2002 2001 + diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy index c75a8dbffbdcd7..b04eb1c5d14bf5 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy @@ -48,7 +48,6 @@ suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_do sql """use ${dbName};""" sql """set enable_fallback_to_original_planner=false;""" - sql """set show_column_comment_in_describe=true;""" sql """drop table if exists ${tableName}""" sql """ diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy index 89d3c463c6e9b0..0122ad94d88f45 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy @@ -255,8 +255,9 @@ suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,i assertTrue(sparkDrivenDorisRowsBeforeWrite[0][6] == null, sparkDrivenDorisRowsBeforeWrite.toString()) assertTrue(sparkDrivenDorisRowsBeforeWrite[0][8] == null, sparkDrivenDorisRowsBeforeWrite.toString()) assertTrue(sparkDrivenDorisRowsBeforeWrite[0][9] == null, sparkDrivenDorisRowsBeforeWrite.toString()) - assertTrue(sparkDrivenDorisRowsBeforeWrite[0][11] == null, sparkDrivenDorisRowsBeforeWrite.toString()) + assertTrue(sparkDrivenDorisRowsBeforeWrite[0][10] == null, sparkDrivenDorisRowsBeforeWrite.toString()) assertTrue(sparkDrivenDorisRowsBeforeWrite[0][12] == null, sparkDrivenDorisRowsBeforeWrite.toString()) + assertTrue(sparkDrivenDorisRowsBeforeWrite[0][13] == null, sparkDrivenDorisRowsBeforeWrite.toString()) sql """ INSERT INTO ${sparkTable} VALUES ( From 546467519193eda75c7dbe2724c221437cb49e2c Mon Sep 17 00:00:00 2001 From: daidai Date: Sun, 12 Jul 2026 16:49:08 +0800 Subject: [PATCH 03/32] [fix](iceberg) Reject case-insensitive nested field collisions ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Iceberg permits sibling fields whose names differ only by case, while Doris exposes and resolves Iceberg paths case-insensitively. Nested ADD or RENAME could therefore create an Iceberg schema that Doris cannot resolve after refresh. Validate the canonical parent struct before UpdateSchema, reject collisions with existing siblings, and preserve case-only rename of the same field. ### Release note Reject case-insensitive name collisions when adding or renaming nested Iceberg fields. ### Check List (For Author) - Test: Unit Test, FE build, and Regression test - Unit Test: IcebergMetadataOpsValidationTest - Regression test: test_iceberg_nested_schema_evolution_spark_doris_interop and test_iceberg_nested_schema_evolution_ddl - Behavior changed: Yes. Invalid case-insensitive nested field collisions are rejected before updating Iceberg metadata. - Does this need documentation: No --- .../iceberg/IcebergMetadataOps.java | 18 ++++++++++++++++++ .../IcebergMetadataOpsValidationTest.java | 19 +++++++++++++++++++ ...chema_evolution_spark_doris_interop.groovy | 16 ++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index a0127a2da39752..e4530591d9735b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -855,6 +855,8 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co throw new UserException("Parent column path '" + columnPath.getParentPathString() + "' is not a struct in Iceberg table: " + icebergTable.name()); } + validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), + parentPath.getColumnPath(), columnPath.getLeafName(), null, "add"); UpdateSchema updateSchema = icebergTable.updateSchema(); org.apache.iceberg.types.Type dorisType = IcebergUtils.dorisTypeToIcebergType(column.getType()); @@ -951,6 +953,9 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String } Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "rename"); + ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "rename"); + validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), + parentPath.getColumnPath(), newName, resolvedPath.getField(), "rename"); UpdateSchema updateSchema = icebergTable.updateSchema(); updateSchema.renameColumn(resolvedPath.getFullPath(), newName); @@ -1221,6 +1226,19 @@ private ColumnPath childPath(ColumnPath parentPath, String childName) { return ColumnPath.of(parts); } + @VisibleForTesting + void validateNoCaseInsensitiveSiblingCollision(Types.StructType parentType, ColumnPath parentPath, + String targetName, NestedField sourceField, String operation) throws UserException { + NestedField conflictingField = parentType.caseInsensitiveField(targetName); + if (conflictingField != null + && (sourceField == null || conflictingField.fieldId() != sourceField.fieldId())) { + String targetPath = childPath(parentPath, targetName).getFullPath(); + String conflictingPath = childPath(parentPath, conflictingField.name()).getFullPath(); + throw new UserException("Cannot " + operation + " nested column '" + targetPath + + "': conflicts with existing Iceberg field '" + conflictingPath + "' (case-insensitive)"); + } + } + private static class ResolvedColumnPath { private final ColumnPath columnPath; private final org.apache.iceberg.types.Type type; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 3c906df8e0ffbd..a62b3f943b9a8c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -209,6 +209,25 @@ public void testResolveNestedColumnPathUsesCaseInsensitiveCanonicalIcebergPath() new ColumnPosition("label"), "add")); } + @Test + public void testValidateNoCaseInsensitiveSiblingCollisionRejectsAddAndRenameTargets() { + Types.StructType parentType = mixedCaseNestedSchema().findField("Info").type().asStructType(); + ColumnPath parentPath = ColumnPath.fromDotName("Info"); + assertUserException(() -> ops.validateNoCaseInsensitiveSiblingCollision( + parentType, parentPath, "metric", null, "add"), + "Cannot add nested column 'Info.metric': conflicts with existing Iceberg field 'Info.Metric'"); + assertUserException(() -> ops.validateNoCaseInsensitiveSiblingCollision( + parentType, parentPath, "metric", parentType.field("Label"), "rename"), + "Cannot rename nested column 'Info.metric': conflicts with existing Iceberg field 'Info.Metric'"); + } + + @Test + public void testValidateNoCaseInsensitiveSiblingCollisionAllowsCaseOnlyRename() throws Throwable { + Types.StructType parentType = mixedCaseNestedSchema().findField("Info").type().asStructType(); + ops.validateNoCaseInsensitiveSiblingCollision(parentType, ColumnPath.fromDotName("Info"), + "metric", parentType.field("Metric"), "rename"); + } + @Test public void testResolveNestedColumnPathRejectsMapKey() { assertUserException(() -> ops.resolveNestedColumnPath(nestedSchema(), ColumnPath.fromDotName("m.key.x"), diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy index 0122ad94d88f45..0437f5e4b12680 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy @@ -26,6 +26,7 @@ suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,i String dbName = "iceberg_nested_schema_evolution_interop_db" String dorisTable = "doris_nested_evolution_to_spark" String sparkTable = "spark_nested_evolution_to_doris" + String mixedCaseTable = "spark_mixed_case_nested_collision" String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") String minioPort = context.config.otherConfigs.get("iceberg_minio_port") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") @@ -156,6 +157,11 @@ suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,i spark_iceberg_multi """ DROP TABLE IF EXISTS demo.${dbName}.${sparkTable}; + DROP TABLE IF EXISTS demo.${dbName}.${mixedCaseTable}; + CREATE TABLE demo.${dbName}.${mixedCaseTable} ( + id INT, + Info STRUCT + ) USING iceberg; CREATE TABLE demo.${dbName}.${sparkTable} ( id INT, info STRUCT, @@ -214,6 +220,15 @@ suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,i sql """switch ${catalogName}""" sql """use ${dbName}""" + test { + sql """ALTER TABLE ${mixedCaseTable} ADD COLUMN info.metric STRING NULL""" + exception "conflicts with existing Iceberg field 'Info.Metric' (case-insensitive)" + } + test { + sql """ALTER TABLE ${mixedCaseTable} RENAME COLUMN info.label TO metric""" + exception "conflicts with existing Iceberg field 'Info.Metric' (case-insensitive)" + } + def sparkDrivenDescRows = sql """DESC ${sparkTable}""" String sparkDrivenDesc = sparkDrivenDescRows.toString().toLowerCase().replaceAll("\\s+", "") assertTrue(sparkDrivenDesc.contains("spark_first"), sparkDrivenDesc) @@ -311,6 +326,7 @@ suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,i try { spark_iceberg """DROP TABLE IF EXISTS demo.${dbName}.${dorisTable}""" spark_iceberg """DROP TABLE IF EXISTS demo.${dbName}.${sparkTable}""" + spark_iceberg """DROP TABLE IF EXISTS demo.${dbName}.${mixedCaseTable}""" } catch (Throwable t) { logger.warn("failed to clean Spark Iceberg tables", t) } From 4641e4ac2d1dfe36318c5c1be70129593ef5f2c0 Mon Sep 17 00:00:00 2001 From: daidai Date: Mon, 13 Jul 2026 18:31:02 +0800 Subject: [PATCH 04/32] [fix](iceberg) Address nested schema evolution review gaps ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Iceberg nested schema evolution allowed case-insensitive top-level field collisions in root ADD, multi-ADD, and RENAME operations, and direct array element or map value type updates were rejected by the struct-only path validator. The regression suites also lacked deterministic output baselines for Spark/Doris interoperability. This change validates root and intra-statement collisions before schema commits, resolves direct element/value MODIFY paths through the canonical path resolver while retaining MAP-key rejection, preserves dotted top-level names for legacy String constructors, and adds generated regression baselines. ### Release note Prevent case-insensitive Iceberg top-level field collisions and support direct array element and map value type promotions. ### Check List (For Author) - Test: Regression test and Unit Test - ./build.sh --fe - IcebergMetadataOpsValidationTest - IcebergNestedSchemaEvolutionParserTest - test_iceberg_nested_schema_evolution_ddl - test_iceberg_nested_schema_evolution_spark_doris_interop - Behavior changed: Yes. Iceberg root field collisions are rejected case-insensitively, and direct array element/map value type promotions are supported. - Does this need documentation: No --- .../iceberg/IcebergMetadataOps.java | 47 +- .../plans/commands/info/DropColumnOp.java | 2 +- .../commands/info/ModifyColumnCommentOp.java | 2 +- .../plans/commands/info/RenameColumnOp.java | 2 +- .../IcebergMetadataOpsValidationTest.java | 81 ++- ...cebergNestedSchemaEvolutionParserTest.java | 19 + ...st_iceberg_nested_schema_evolution_ddl.out | 14 +- ...d_schema_evolution_spark_doris_interop.out | 20 + ...iceberg_nested_schema_evolution_ddl.groovy | 60 +- ...chema_evolution_spark_doris_interop.groovy | 532 +++++++++--------- 10 files changed, 445 insertions(+), 334 deletions(-) create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index e4530591d9735b..4b4728cc40fbfe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -94,6 +94,8 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -824,10 +826,13 @@ public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition po throws UserException { validateCommonColumnInfo(column); Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Schema schema = icebergTable.schema(); + validateNoCaseInsensitiveSiblingCollision( + schema.asStruct(), "", column.getName(), null, "add"); UpdateSchema updateSchema = icebergTable.updateSchema(); addOneColumn(updateSchema, column); if (position != null) { - applyPosition(updateSchema, position, ColumnPath.of(column.getName()), icebergTable.schema(), "add"); + applyPosition(updateSchema, position, ColumnPath.of(column.getName()), schema, "add"); } try { executionAuthenticator.execute(() -> updateSchema.commit()); @@ -879,9 +884,13 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co @Override public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); - UpdateSchema updateSchema = icebergTable.updateSchema(); for (Column column : columns) { validateCommonColumnInfo(column); + } + validateNoCaseInsensitiveTopLevelCollisions(icebergTable.schema(), columns); + + UpdateSchema updateSchema = icebergTable.updateSchema(); + for (Column column : columns) { addOneColumn(updateSchema, column); } try { @@ -932,7 +941,10 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); - ResolvedColumnPath oldPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(oldName), "rename"); + Schema schema = icebergTable.schema(); + ResolvedColumnPath oldPath = resolveColumnPath(schema, ColumnPath.of(oldName), "rename"); + validateNoCaseInsensitiveSiblingCollision( + schema.asStruct(), "", newName, oldPath.getField(), "rename"); UpdateSchema updateSchema = icebergTable.updateSchema(); updateSchema.renameColumn(oldPath.getFullPath(), newName); try { @@ -1025,7 +1037,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column } Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); - ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "modify"); + ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); NestedField currentCol = resolvedPath.getField(); validateCommonColumnInfo(column); @@ -1229,16 +1241,37 @@ private ColumnPath childPath(ColumnPath parentPath, String childName) { @VisibleForTesting void validateNoCaseInsensitiveSiblingCollision(Types.StructType parentType, ColumnPath parentPath, String targetName, NestedField sourceField, String operation) throws UserException { + validateNoCaseInsensitiveSiblingCollision( + parentType, parentPath.getFullPath(), targetName, sourceField, operation); + } + + private void validateNoCaseInsensitiveSiblingCollision(Types.StructType parentType, String parentPath, + String targetName, NestedField sourceField, String operation) throws UserException { NestedField conflictingField = parentType.caseInsensitiveField(targetName); if (conflictingField != null && (sourceField == null || conflictingField.fieldId() != sourceField.fieldId())) { - String targetPath = childPath(parentPath, targetName).getFullPath(); - String conflictingPath = childPath(parentPath, conflictingField.name()).getFullPath(); - throw new UserException("Cannot " + operation + " nested column '" + targetPath + String targetPath = parentPath.isEmpty() ? targetName : parentPath + "." + targetName; + String conflictingPath = parentPath.isEmpty() + ? conflictingField.name() : parentPath + "." + conflictingField.name(); + String columnDescription = parentPath.isEmpty() ? "column" : "nested column"; + throw new UserException("Cannot " + operation + " " + columnDescription + " '" + targetPath + "': conflicts with existing Iceberg field '" + conflictingPath + "' (case-insensitive)"); } } + private void validateNoCaseInsensitiveTopLevelCollisions(Schema schema, List columns) + throws UserException { + Set requestedNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (Column column : columns) { + validateNoCaseInsensitiveSiblingCollision( + schema.asStruct(), "", column.getName(), null, "add"); + if (!requestedNames.add(column.getName())) { + throw new UserException("Cannot add column '" + column.getName() + + "': conflicts with another requested column (case-insensitive)"); + } + } + } + private static class ResolvedColumnPath { private final ColumnPath columnPath; private final org.apache.iceberg.types.Type type; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java index 3a95246790941d..3068e4858de291 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java @@ -52,7 +52,7 @@ public class DropColumnOp extends AlterTableOp { * DropColumnOp */ public DropColumnOp(String colName, String rollupName, Map properties) { - this(ColumnPath.fromDotName(colName), rollupName, properties); + this(ColumnPath.of(colName), rollupName, properties); } public DropColumnOp(ColumnPath columnPath, String rollupName, Map properties) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java index 49ea3d8bcb8f27..c8be93d4259485 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java @@ -36,7 +36,7 @@ public class ModifyColumnCommentOp extends AlterTableOp { private String comment; public ModifyColumnCommentOp(String colName, String comment) { - this(ColumnPath.fromDotName(colName), comment); + this(ColumnPath.of(colName), comment); } public ModifyColumnCommentOp(ColumnPath columnPath, String comment) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java index 976fe258e15192..d375e545ac7a5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java @@ -38,7 +38,7 @@ public class RenameColumnOp extends AlterTableOp { private String newColName; public RenameColumnOp(String colName, String newColName) { - this(ColumnPath.fromDotName(colName), newColName); + this(ColumnPath.of(colName), newColName); } public RenameColumnOp(ColumnPath columnPath, String newColName) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index a62b3f943b9a8c..e6286f48f767d0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -29,8 +29,11 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalTable; import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.UpdateSchema; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.types.Types; @@ -38,26 +41,31 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.MockedStatic; import org.mockito.Mockito; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.Arrays; import java.util.Collections; +import java.util.Optional; public class IcebergMetadataOpsValidationTest { private IcebergMetadataOps ops; + private ExternalCatalog dorisCatalog; private Method validateForModifyColumnMethod; private Method validateForModifyComplexColumnMethod; @Before public void setUp() throws Exception { - ExternalCatalog dorisCatalog = Mockito.mock(ExternalCatalog.class); + dorisCatalog = Mockito.mock(ExternalCatalog.class); Catalog icebergCatalog = Mockito.mock(Catalog.class, Mockito.withSettings().extraInterfaces(SupportsNamespaces.class)); Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { }); Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.emptyMap()); + Mockito.doReturn(Optional.empty()).when(dorisCatalog).getDbForReplay(Mockito.anyString()); ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog); validateForModifyColumnMethod = IcebergMetadataOps.class.getDeclaredMethod( @@ -228,6 +236,69 @@ public void testValidateNoCaseInsensitiveSiblingCollisionAllowsCaseOnlyRename() "metric", parentType.field("Metric"), "rename"); } + @Test + public void testTopLevelCaseInsensitiveCollisionsAndCaseOnlyRename() throws Throwable { + Schema schema = new Schema( + Types.NestedField.optional(1, "Id", Types.IntegerType.get()), + Types.NestedField.optional(2, "Label", Types.StringType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.addColumn( + dorisTable, new Column("id", Type.STRING, true), null, 1L), + "Cannot add column 'id': conflicts with existing Iceberg field 'Id'"); + assertUserException(() -> ops.addColumns(dorisTable, + Collections.singletonList(new Column("id", Type.STRING, true)), 1L), + "Cannot add column 'id': conflicts with existing Iceberg field 'Id'"); + assertUserException(() -> ops.addColumns(dorisTable, Arrays.asList( + new Column("new_field", Type.STRING, true), + new Column("NEW_FIELD", Type.STRING, true)), 1L), + "conflicts with another requested column (case-insensitive)"); + assertUserException(() -> ops.renameColumn(dorisTable, "label", "id", 1L), + "Cannot rename column 'id': conflicts with existing Iceberg field 'Id'"); + + ops.renameColumn(dorisTable, "id", "id", 1L); + } + + Mockito.verify(updateSchema).renameColumn("Id", "id"); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throwable { + Schema schema = primitiveContainerSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), + new Column("element", Type.BIGINT, true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("m.value"), + new Column("value", Type.BIGINT, true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("arr.element", Types.LongType.get(), ""); + Mockito.verify(updateSchema).updateColumn("m.value", Types.LongType.get(), ""); + Mockito.verify(updateSchema).makeColumnOptional("arr.element"); + Mockito.verify(updateSchema).makeColumnOptional("m.value"); + Mockito.verify(updateSchema, Mockito.times(2)).commit(); + } + @Test public void testResolveNestedColumnPathRejectsMapKey() { assertUserException(() -> ops.resolveNestedColumnPath(nestedSchema(), ColumnPath.fromDotName("m.key.x"), @@ -340,4 +411,12 @@ private Schema mixedCaseNestedSchema() { Types.StringType.get(), Types.StructType.of(Types.NestedField.optional(11, "Code", Types.IntegerType.get()))))); } + + private Schema primitiveContainerSchema() { + return new Schema( + Types.NestedField.optional(1, "arr", + Types.ListType.ofOptional(2, Types.IntegerType.get())), + Types.NestedField.optional(3, "m", Types.MapType.ofOptional( + 4, 5, Types.StringType.get(), Types.IntegerType.get()))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index dbf7d49a21e0a5..fab9a4da1280fb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -29,6 +29,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.List; public class IcebergNestedSchemaEvolutionParserTest { @@ -55,6 +56,10 @@ public void testParseNestedModifyDropRenamePaths() { ModifyColumnOp.class, "s.a"); assertSingleClausePath("ALTER TABLE t MODIFY COLUMN s.a BIGINT AFTER b", ModifyColumnOp.class, "s.a"); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN arr.element BIGINT", + ModifyColumnOp.class, "arr.element"); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN m.value BIGINT", + ModifyColumnOp.class, "m.value"); assertSingleClausePath("ALTER TABLE t MODIFY COLUMN s.a COMMENT 'nested comment'", ModifyColumnCommentOp.class, "s.a"); assertSingleClausePath("ALTER TABLE t DROP COLUMN s.c", @@ -71,6 +76,20 @@ public void testParseNestedModifyDropRenamePaths() { RenameColumnOp.class, "m.value.y"); } + @Test + public void testLegacyStringConstructorsKeepDottedTopLevelNames() { + DropColumnOp drop = new DropColumnOp("top.level", null, Collections.emptyMap()); + RenameColumnOp rename = new RenameColumnOp("top.level", "renamed"); + ModifyColumnCommentOp comment = new ModifyColumnCommentOp("top.level", "comment"); + + Assertions.assertFalse(drop.getColumnPath().isNested()); + Assertions.assertFalse(rename.getColumnPath().isNested()); + Assertions.assertFalse(comment.getColumnPath().isNested()); + Assertions.assertEquals("top.level", drop.getColumnPath().getFullPath()); + Assertions.assertEquals("top.level", rename.getColumnPath().getFullPath()); + Assertions.assertEquals("top.level", comment.getColumnPath().getFullPath()); + } + private void assertSingleClausePath(String sql, Class clauseClass, String expectedPath) { Plan plan = parser.parseSingle(sql); diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out index e00b1b5b67b88e..1249ff6cf2b19c 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out @@ -1,11 +1,13 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !desc -- -id int Yes true \N -s struct Yes true \N -arr array> Yes true \N -m map> Yes true \N +id int(11) +s struct +arr array> +m map> +arr_scalar array +m_scalar map -- !query_rows -- -1 \N 10 \N \N 100 \N \N 1000 \N \N -2 first 20 after_a c2 200 202 201 2000 2002 2001 +1 \N 10 \N \N 100 \N \N 1000 \N \N 7 70 +2 first 20 after_a c2 200 202 201 2000 2002 2001 8 80 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out new file mode 100644 index 00000000000000..0119135c2ee878 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out @@ -0,0 +1,20 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !doris_driven_rows -- +1 \N 10 \N doris_before \N \N 100 \N \N \N 1000 \N \N +2 spark_first 20 spark_after_metric spark_after_doris spark_can_write_doris_field 202 200 203 201 2002 2000 2003 2001 + +-- !spark_driven_schema -- +id int(11) +info struct +events array> +attrs map> + +-- !spark_driven_rows_before_write -- +1 \N 10 \N spark_before \N \N 100 \N \N \N 1000 \N \N +2 spark_first_field 20 spark_after_metric_field spark_after spark_new_field 202 200 203 201 2002 2000 2003 2001 + +-- !spark_driven_rows_after_write -- +1 \N 10 \N spark_before \N \N 100 \N \N \N 1000 \N \N +2 spark_first_field 20 spark_after_metric_field spark_after spark_new_field 202 200 203 201 2002 2000 2003 2001 +3 doris_first_field 30 doris_after_metric_field doris_after_spark doris_can_write_spark_field 302 300 303 301 3002 3000 3003 3001 + diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy index b04eb1c5d14bf5..e497b33995a19a 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy @@ -55,7 +55,9 @@ suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_do id INT NOT NULL, s STRUCT, arr ARRAY>, - m MAP> + m MAP>, + arr_scalar ARRAY, + m_scalar MAP ); """ @@ -64,7 +66,9 @@ suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_do 1, STRUCT(10, 'old'), ARRAY(STRUCT(100)), - MAP('k', STRUCT(1000)) + MAP('k', STRUCT(1000)), + ARRAY(7), + MAP('k', 70) ) """ @@ -87,6 +91,12 @@ suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_do sql """ALTER TABLE ${tableName} MODIFY COLUMN s.a BIGINT""" sql """ALTER TABLE ${tableName} MODIFY COLUMN arr.element.x BIGINT""" sql """ALTER TABLE ${tableName} MODIFY COLUMN m.value.v BIGINT""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN arr_scalar.element BIGINT""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.value BIGINT""" + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.`key` BIGINT""" + exception "Cannot modify MAP key nested column" + } sql """ALTER TABLE ${tableName} RENAME COLUMN s.c TO c2""" sql """ALTER TABLE ${tableName} RENAME COLUMN arr.element.y TO y2""" @@ -103,38 +113,18 @@ suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_do 2, STRUCT('first', 20, 'after_a', 'new', 'c2'), ARRAY(STRUCT(200, 202, 201)), - MAP('k', STRUCT(2000, 2002, 2001)) + MAP('k', STRUCT(2000, 2002, 2001)), + ARRAY(8), + MAP('k', 80) ) """ - qt_desc """DESC ${tableName}""" - - def descRows = sql """DESC ${tableName}""" - def normalizeType = { String s -> s.toLowerCase().replaceAll("\\s+", "") } - def typeOf = { String name -> - def row = descRows.find { it[0].toString().equalsIgnoreCase(name) } - assertTrue(row != null, "column not found: ${name}") - return normalizeType(row[1].toString()) - } - - def sType = typeOf("s") - assertTrue(sType.contains("first_pos:text"), sType) - assertTrue(sType.contains("a:bigint"), sType) - assertTrue(sType.contains("after_a:text"), sType) - assertTrue(sType.contains("c2:text"), sType) - assertTrue(!sType.contains("drop_me"), sType) - - def arrType = typeOf("arr") - assertTrue(arrType.contains("x:bigint"), arrType) - assertTrue(arrType.contains("after_x:int"), arrType) - assertTrue(arrType.contains("y2:int"), arrType) - assertTrue(!arrType.contains("drop_me"), arrType) - - def mType = typeOf("m") - assertTrue(mType.contains("v:bigint"), mType) - assertTrue(mType.contains("after_v:int"), mType) - assertTrue(mType.contains("y2:int"), mType) - assertTrue(!mType.contains("drop_me"), mType) + qt_desc """ + SELECT COLUMN_NAME, COLUMN_TYPE + FROM ${catalogName}.information_schema.columns + WHERE TABLE_SCHEMA = '${dbName}' AND TABLE_NAME = '${tableName}' + ORDER BY ORDINAL_POSITION + """ order_qt_query_rows """ SELECT id, @@ -147,12 +137,10 @@ suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_do element_at(arr[1], 'y2'), element_at(m['k'], 'v'), element_at(m['k'], 'after_v'), - element_at(m['k'], 'y2') + element_at(m['k'], 'y2'), + arr_scalar[1], + m_scalar['k'] FROM ${tableName} ORDER BY id """ - - sql """drop table if exists ${tableName}""" - sql """drop database if exists ${dbName} force""" - sql """drop catalog if exists ${catalogName}""" } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy index 0437f5e4b12680..c6cdc524d9ebc4 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy @@ -45,297 +45,267 @@ suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,i ); """ - try { - sql """switch ${catalogName}""" - sql """drop database if exists ${dbName} force""" - sql """create database ${dbName}""" - sql """use ${dbName}""" - sql """set enable_fallback_to_original_planner=false""" + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner=false""" - spark_iceberg """CREATE DATABASE IF NOT EXISTS demo.${dbName}""" + spark_iceberg """CREATE DATABASE IF NOT EXISTS demo.${dbName}""" - sql """ - CREATE TABLE ${dorisTable} ( - id INT NOT NULL, - info STRUCT, - events ARRAY>, - attrs MAP> - ) - """ - sql """ - INSERT INTO ${dorisTable} VALUES ( - 1, - STRUCT(10, 'doris_before'), - ARRAY(STRUCT(100)), - MAP('k', STRUCT(1000)) - ) - """ + sql """ + CREATE TABLE ${dorisTable} ( + id INT NOT NULL, + info STRUCT, + events ARRAY>, + attrs MAP> + ) + """ + sql """ + INSERT INTO ${dorisTable} VALUES ( + 1, + STRUCT(10, 'doris_before'), + ARRAY(STRUCT(100)), + MAP('k', STRUCT(1000)) + ) + """ - sql """ALTER TABLE ${dorisTable} ADD COLUMN info.doris_added STRING NULL COMMENT 'added by doris'""" - sql """ALTER TABLE ${dorisTable} ADD COLUMN info.doris_first STRING NULL FIRST""" - sql """ALTER TABLE ${dorisTable} ADD COLUMN info.doris_after_metric STRING NULL AFTER metric""" - sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.doris_score INT NULL""" - sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.doris_first_score INT NULL FIRST""" - sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.doris_after_score INT NULL AFTER score""" - sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.doris_code INT NULL""" - sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.doris_first_code INT NULL FIRST""" - sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.doris_after_code INT NULL AFTER code""" - sql """ALTER TABLE ${dorisTable} ADD COLUMN info.drop_me STRING NULL""" - sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.drop_me INT NULL""" - sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.drop_me INT NULL""" - sql """ALTER TABLE ${dorisTable} MODIFY COLUMN info.metric BIGINT""" - sql """ALTER TABLE ${dorisTable} MODIFY COLUMN events.element.score BIGINT""" - sql """ALTER TABLE ${dorisTable} MODIFY COLUMN attrs.value.code BIGINT""" - sql """ALTER TABLE ${dorisTable} RENAME COLUMN info.doris_added TO doris_renamed""" - sql """ALTER TABLE ${dorisTable} RENAME COLUMN events.element.doris_score TO doris_score_renamed""" - sql """ALTER TABLE ${dorisTable} RENAME COLUMN attrs.value.doris_code TO doris_code_renamed""" - sql """ALTER TABLE ${dorisTable} MODIFY COLUMN info.doris_renamed COMMENT 'renamed by doris'""" - sql """ALTER TABLE ${dorisTable} MODIFY COLUMN events.element.doris_score_renamed COMMENT 'renamed by doris'""" - sql """ALTER TABLE ${dorisTable} MODIFY COLUMN attrs.value.doris_code_renamed COMMENT 'renamed by doris'""" - sql """ALTER TABLE ${dorisTable} DROP COLUMN info.drop_me""" - sql """ALTER TABLE ${dorisTable} DROP COLUMN events.element.drop_me""" - sql """ALTER TABLE ${dorisTable} DROP COLUMN attrs.value.drop_me""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN info.doris_added STRING NULL COMMENT 'added by doris'""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN info.doris_first STRING NULL FIRST""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN info.doris_after_metric STRING NULL AFTER metric""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.doris_score INT NULL""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.doris_first_score INT NULL FIRST""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.doris_after_score INT NULL AFTER score""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.doris_code INT NULL""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.doris_first_code INT NULL FIRST""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.doris_after_code INT NULL AFTER code""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN info.drop_me STRING NULL""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.drop_me INT NULL""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.drop_me INT NULL""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN info.metric BIGINT""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN events.element.score BIGINT""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN attrs.value.code BIGINT""" + sql """ALTER TABLE ${dorisTable} RENAME COLUMN info.doris_added TO doris_renamed""" + sql """ALTER TABLE ${dorisTable} RENAME COLUMN events.element.doris_score TO doris_score_renamed""" + sql """ALTER TABLE ${dorisTable} RENAME COLUMN attrs.value.doris_code TO doris_code_renamed""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN info.doris_renamed COMMENT 'renamed by doris'""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN events.element.doris_score_renamed COMMENT 'renamed by doris'""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN attrs.value.doris_code_renamed COMMENT 'renamed by doris'""" + sql """ALTER TABLE ${dorisTable} DROP COLUMN info.drop_me""" + sql """ALTER TABLE ${dorisTable} DROP COLUMN events.element.drop_me""" + sql """ALTER TABLE ${dorisTable} DROP COLUMN attrs.value.drop_me""" - spark_iceberg """REFRESH TABLE demo.${dbName}.${dorisTable}""" - spark_iceberg """ - INSERT INTO demo.${dbName}.${dorisTable} VALUES ( - 2, - NAMED_STRUCT('doris_first', 'spark_first', - 'metric', CAST(20 AS BIGINT), - 'doris_after_metric', 'spark_after_metric', - 'label', 'spark_after_doris', - 'doris_renamed', 'spark_can_write_doris_field'), - ARRAY(NAMED_STRUCT('doris_first_score', 202, - 'score', CAST(200 AS BIGINT), - 'doris_after_score', 203, - 'doris_score_renamed', 201)), - MAP('k', NAMED_STRUCT('doris_first_code', 2002, - 'code', CAST(2000 AS BIGINT), - 'doris_after_code', 2003, - 'doris_code_renamed', 2001)) - ) - """ + spark_iceberg """REFRESH TABLE demo.${dbName}.${dorisTable}""" + spark_iceberg """ + INSERT INTO demo.${dbName}.${dorisTable} VALUES ( + 2, + NAMED_STRUCT('doris_first', 'spark_first', + 'metric', CAST(20 AS BIGINT), + 'doris_after_metric', 'spark_after_metric', + 'label', 'spark_after_doris', + 'doris_renamed', 'spark_can_write_doris_field'), + ARRAY(NAMED_STRUCT('doris_first_score', 202, + 'score', CAST(200 AS BIGINT), + 'doris_after_score', 203, + 'doris_score_renamed', 201)), + MAP('k', NAMED_STRUCT('doris_first_code', 2002, + 'code', CAST(2000 AS BIGINT), + 'doris_after_code', 2003, + 'doris_code_renamed', 2001)) + ) + """ - sql """refresh table ${dbName}.${dorisTable}""" - def dorisDrivenSparkRows = spark_iceberg """ - SELECT id, - info.doris_first, - info.metric, - info.doris_after_metric, - info.label, - info.doris_renamed, - events[0].doris_first_score, - events[0].score, - events[0].doris_after_score, - events[0].doris_score_renamed, - attrs['k'].doris_first_code, - attrs['k'].code, - attrs['k'].doris_after_code, - attrs['k'].doris_code_renamed - FROM demo.${dbName}.${dorisTable} - ORDER BY id - """ - def dorisDrivenDorisRows = sql """ - SELECT id, - element_at(info, 'doris_first'), - element_at(info, 'metric'), - element_at(info, 'doris_after_metric'), - element_at(info, 'label'), - element_at(info, 'doris_renamed'), - element_at(events[1], 'doris_first_score'), - element_at(events[1], 'score'), - element_at(events[1], 'doris_after_score'), - element_at(events[1], 'doris_score_renamed'), - element_at(attrs['k'], 'doris_first_code'), - element_at(attrs['k'], 'code'), - element_at(attrs['k'], 'doris_after_code'), - element_at(attrs['k'], 'doris_code_renamed') - FROM ${dorisTable} - ORDER BY id - """ - assertSparkDorisResultEquals(dorisDrivenSparkRows, dorisDrivenDorisRows) + sql """refresh table ${dbName}.${dorisTable}""" + def dorisDrivenSparkRows = spark_iceberg """ + SELECT id, + info.doris_first, + info.metric, + info.doris_after_metric, + info.label, + info.doris_renamed, + events[0].doris_first_score, + events[0].score, + events[0].doris_after_score, + events[0].doris_score_renamed, + attrs['k'].doris_first_code, + attrs['k'].code, + attrs['k'].doris_after_code, + attrs['k'].doris_code_renamed + FROM demo.${dbName}.${dorisTable} + ORDER BY id + """ + String dorisDrivenDorisQuery = """ + SELECT id, + element_at(info, 'doris_first'), + element_at(info, 'metric'), + element_at(info, 'doris_after_metric'), + element_at(info, 'label'), + element_at(info, 'doris_renamed'), + element_at(events[1], 'doris_first_score'), + element_at(events[1], 'score'), + element_at(events[1], 'doris_after_score'), + element_at(events[1], 'doris_score_renamed'), + element_at(attrs['k'], 'doris_first_code'), + element_at(attrs['k'], 'code'), + element_at(attrs['k'], 'doris_after_code'), + element_at(attrs['k'], 'doris_code_renamed') + FROM ${dorisTable} + ORDER BY id + """ + order_qt_doris_driven_rows dorisDrivenDorisQuery + def dorisDrivenDorisRows = sql dorisDrivenDorisQuery + assertSparkDorisResultEquals(dorisDrivenSparkRows, dorisDrivenDorisRows) - spark_iceberg_multi """ - DROP TABLE IF EXISTS demo.${dbName}.${sparkTable}; - DROP TABLE IF EXISTS demo.${dbName}.${mixedCaseTable}; - CREATE TABLE demo.${dbName}.${mixedCaseTable} ( - id INT, - Info STRUCT - ) USING iceberg; - CREATE TABLE demo.${dbName}.${sparkTable} ( - id INT, - info STRUCT, - events ARRAY>, - attrs MAP> - ) USING iceberg; - INSERT INTO demo.${dbName}.${sparkTable} VALUES ( - 1, - NAMED_STRUCT('metric', 10, 'label', 'spark_before'), - ARRAY(NAMED_STRUCT('score', 100)), - MAP('k', NAMED_STRUCT('code', 1000)) - ); - ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.spark_added STRING; - ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.spark_first STRING FIRST; - ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.spark_after_metric STRING AFTER metric; - ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.spark_score INT; - ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.spark_first_score INT FIRST; - ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.spark_after_score INT AFTER score; - ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.spark_code INT; - ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.spark_first_code INT FIRST; - ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.spark_after_code INT AFTER code; - ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.drop_me STRING; - ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.drop_me INT; - ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.drop_me INT; - ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN info.metric TYPE BIGINT; - ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN events.element.score TYPE BIGINT; - ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN attrs.value.code TYPE BIGINT; - ALTER TABLE demo.${dbName}.${sparkTable} RENAME COLUMN info.spark_added TO spark_renamed; - ALTER TABLE demo.${dbName}.${sparkTable} RENAME COLUMN events.element.spark_score TO spark_score_renamed; - ALTER TABLE demo.${dbName}.${sparkTable} RENAME COLUMN attrs.value.spark_code TO spark_code_renamed; - ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN info.spark_renamed COMMENT 'renamed by spark'; - ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN events.element.spark_score_renamed COMMENT 'renamed by spark'; - ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN attrs.value.spark_code_renamed COMMENT 'renamed by spark'; - ALTER TABLE demo.${dbName}.${sparkTable} DROP COLUMN info.drop_me; - ALTER TABLE demo.${dbName}.${sparkTable} DROP COLUMN events.element.drop_me; - ALTER TABLE demo.${dbName}.${sparkTable} DROP COLUMN attrs.value.drop_me; - INSERT INTO demo.${dbName}.${sparkTable} VALUES ( - 2, - NAMED_STRUCT('spark_first', 'spark_first_field', - 'metric', CAST(20 AS BIGINT), - 'spark_after_metric', 'spark_after_metric_field', - 'label', 'spark_after', - 'spark_renamed', 'spark_new_field'), - ARRAY(NAMED_STRUCT('spark_first_score', 202, - 'score', CAST(200 AS BIGINT), - 'spark_after_score', 203, - 'spark_score_renamed', 201)), - MAP('k', NAMED_STRUCT('spark_first_code', 2002, - 'code', CAST(2000 AS BIGINT), - 'spark_after_code', 2003, - 'spark_code_renamed', 2001)) - ); - """ + spark_iceberg_multi """ + DROP TABLE IF EXISTS demo.${dbName}.${sparkTable}; + DROP TABLE IF EXISTS demo.${dbName}.${mixedCaseTable}; + CREATE TABLE demo.${dbName}.${mixedCaseTable} ( + Id INT, + Label STRING, + Info STRUCT + ) USING iceberg; + CREATE TABLE demo.${dbName}.${sparkTable} ( + id INT, + info STRUCT, + events ARRAY>, + attrs MAP> + ) USING iceberg; + INSERT INTO demo.${dbName}.${sparkTable} VALUES ( + 1, + NAMED_STRUCT('metric', 10, 'label', 'spark_before'), + ARRAY(NAMED_STRUCT('score', 100)), + MAP('k', NAMED_STRUCT('code', 1000)) + ); + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.spark_added STRING; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.spark_first STRING FIRST; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.spark_after_metric STRING AFTER metric; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.spark_score INT; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.spark_first_score INT FIRST; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.spark_after_score INT AFTER score; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.spark_code INT; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.spark_first_code INT FIRST; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.spark_after_code INT AFTER code; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.drop_me STRING; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.drop_me INT; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.drop_me INT; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN info.metric TYPE BIGINT; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN events.element.score TYPE BIGINT; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN attrs.value.code TYPE BIGINT; + ALTER TABLE demo.${dbName}.${sparkTable} RENAME COLUMN info.spark_added TO spark_renamed; + ALTER TABLE demo.${dbName}.${sparkTable} RENAME COLUMN events.element.spark_score TO spark_score_renamed; + ALTER TABLE demo.${dbName}.${sparkTable} RENAME COLUMN attrs.value.spark_code TO spark_code_renamed; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN info.spark_renamed COMMENT 'renamed by spark'; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN events.element.spark_score_renamed COMMENT 'renamed by spark'; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN attrs.value.spark_code_renamed COMMENT 'renamed by spark'; + ALTER TABLE demo.${dbName}.${sparkTable} DROP COLUMN info.drop_me; + ALTER TABLE demo.${dbName}.${sparkTable} DROP COLUMN events.element.drop_me; + ALTER TABLE demo.${dbName}.${sparkTable} DROP COLUMN attrs.value.drop_me; + INSERT INTO demo.${dbName}.${sparkTable} VALUES ( + 2, + NAMED_STRUCT('spark_first', 'spark_first_field', + 'metric', CAST(20 AS BIGINT), + 'spark_after_metric', 'spark_after_metric_field', + 'label', 'spark_after', + 'spark_renamed', 'spark_new_field'), + ARRAY(NAMED_STRUCT('spark_first_score', 202, + 'score', CAST(200 AS BIGINT), + 'spark_after_score', 203, + 'spark_score_renamed', 201)), + MAP('k', NAMED_STRUCT('spark_first_code', 2002, + 'code', CAST(2000 AS BIGINT), + 'spark_after_code', 2003, + 'spark_code_renamed', 2001)) + ); + """ - sql """refresh catalog ${catalogName}""" - sql """switch ${catalogName}""" - sql """use ${dbName}""" + sql """refresh catalog ${catalogName}""" + sql """switch ${catalogName}""" + sql """use ${dbName}""" - test { - sql """ALTER TABLE ${mixedCaseTable} ADD COLUMN info.metric STRING NULL""" - exception "conflicts with existing Iceberg field 'Info.Metric' (case-insensitive)" - } - test { - sql """ALTER TABLE ${mixedCaseTable} RENAME COLUMN info.label TO metric""" - exception "conflicts with existing Iceberg field 'Info.Metric' (case-insensitive)" - } + test { + sql """ALTER TABLE ${mixedCaseTable} ADD COLUMN id STRING NULL""" + exception "conflicts with existing Iceberg field 'Id' (case-insensitive)" + } + test { + sql """ALTER TABLE ${mixedCaseTable} ADD COLUMN (id STRING)""" + exception "conflicts with existing Iceberg field 'Id' (case-insensitive)" + } + test { + sql """ALTER TABLE ${mixedCaseTable} ADD COLUMN (new_field STRING, NEW_FIELD STRING)""" + exception "conflicts with another requested column (case-insensitive)" + } + test { + sql """ALTER TABLE ${mixedCaseTable} RENAME COLUMN label TO id""" + exception "conflicts with existing Iceberg field 'Id' (case-insensitive)" + } + sql """ALTER TABLE ${mixedCaseTable} RENAME COLUMN label TO label""" - def sparkDrivenDescRows = sql """DESC ${sparkTable}""" - String sparkDrivenDesc = sparkDrivenDescRows.toString().toLowerCase().replaceAll("\\s+", "") - assertTrue(sparkDrivenDesc.contains("spark_first"), sparkDrivenDesc) - assertTrue(sparkDrivenDesc.contains("metric:bigint"), sparkDrivenDesc) - assertTrue(sparkDrivenDesc.contains("spark_after_metric"), sparkDrivenDesc) - assertTrue(sparkDrivenDesc.contains("spark_renamed"), sparkDrivenDesc) - assertTrue(!sparkDrivenDesc.contains("drop_me"), sparkDrivenDesc) - assertTrue(sparkDrivenDesc.contains("spark_first_score:int"), sparkDrivenDesc) - assertTrue(sparkDrivenDesc.contains("score:bigint"), sparkDrivenDesc) - assertTrue(sparkDrivenDesc.contains("spark_after_score:int"), sparkDrivenDesc) - assertTrue(sparkDrivenDesc.contains("spark_score_renamed:int"), sparkDrivenDesc) - assertTrue(sparkDrivenDesc.contains("spark_first_code:int"), sparkDrivenDesc) - assertTrue(sparkDrivenDesc.contains("code:bigint"), sparkDrivenDesc) - assertTrue(sparkDrivenDesc.contains("spark_after_code:int"), sparkDrivenDesc) - assertTrue(sparkDrivenDesc.contains("spark_code_renamed:int"), sparkDrivenDesc) + test { + sql """ALTER TABLE ${mixedCaseTable} ADD COLUMN info.metric STRING NULL""" + exception "conflicts with existing Iceberg field 'Info.Metric' (case-insensitive)" + } + test { + sql """ALTER TABLE ${mixedCaseTable} RENAME COLUMN info.label TO metric""" + exception "conflicts with existing Iceberg field 'Info.Metric' (case-insensitive)" + } - def sparkDrivenDorisRowsBeforeWrite = sql """ - SELECT id, - element_at(info, 'spark_first'), - element_at(info, 'metric'), - element_at(info, 'spark_after_metric'), - element_at(info, 'label'), - element_at(info, 'spark_renamed'), - element_at(events[1], 'spark_first_score'), - element_at(events[1], 'score'), - element_at(events[1], 'spark_after_score'), - element_at(events[1], 'spark_score_renamed'), - element_at(attrs['k'], 'spark_first_code'), - element_at(attrs['k'], 'code'), - element_at(attrs['k'], 'spark_after_code'), - element_at(attrs['k'], 'spark_code_renamed') - FROM ${sparkTable} - ORDER BY id - """ - assertTrue(sparkDrivenDorisRowsBeforeWrite.size() == 2, sparkDrivenDorisRowsBeforeWrite.toString()) - assertTrue(sparkDrivenDorisRowsBeforeWrite[0][1] == null, sparkDrivenDorisRowsBeforeWrite.toString()) - assertTrue(sparkDrivenDorisRowsBeforeWrite[0][3] == null, sparkDrivenDorisRowsBeforeWrite.toString()) - assertTrue(sparkDrivenDorisRowsBeforeWrite[0][5] == null, sparkDrivenDorisRowsBeforeWrite.toString()) - assertTrue(sparkDrivenDorisRowsBeforeWrite[0][6] == null, sparkDrivenDorisRowsBeforeWrite.toString()) - assertTrue(sparkDrivenDorisRowsBeforeWrite[0][8] == null, sparkDrivenDorisRowsBeforeWrite.toString()) - assertTrue(sparkDrivenDorisRowsBeforeWrite[0][9] == null, sparkDrivenDorisRowsBeforeWrite.toString()) - assertTrue(sparkDrivenDorisRowsBeforeWrite[0][10] == null, sparkDrivenDorisRowsBeforeWrite.toString()) - assertTrue(sparkDrivenDorisRowsBeforeWrite[0][12] == null, sparkDrivenDorisRowsBeforeWrite.toString()) - assertTrue(sparkDrivenDorisRowsBeforeWrite[0][13] == null, sparkDrivenDorisRowsBeforeWrite.toString()) + qt_spark_driven_schema """ + SELECT COLUMN_NAME, COLUMN_TYPE + FROM ${catalogName}.information_schema.columns + WHERE TABLE_SCHEMA = '${dbName}' AND TABLE_NAME = '${sparkTable}' + ORDER BY ORDINAL_POSITION + """ - sql """ - INSERT INTO ${sparkTable} VALUES ( - 3, - STRUCT('doris_first_field', 30, 'doris_after_metric_field', - 'doris_after_spark', 'doris_can_write_spark_field'), - ARRAY(STRUCT(302, 300, 303, 301)), - MAP('k', STRUCT(3002, 3000, 3003, 3001)) - ) - """ + String sparkDrivenDorisQuery = """ + SELECT id, + element_at(info, 'spark_first'), + element_at(info, 'metric'), + element_at(info, 'spark_after_metric'), + element_at(info, 'label'), + element_at(info, 'spark_renamed'), + element_at(events[1], 'spark_first_score'), + element_at(events[1], 'score'), + element_at(events[1], 'spark_after_score'), + element_at(events[1], 'spark_score_renamed'), + element_at(attrs['k'], 'spark_first_code'), + element_at(attrs['k'], 'code'), + element_at(attrs['k'], 'spark_after_code'), + element_at(attrs['k'], 'spark_code_renamed') + FROM ${sparkTable} + ORDER BY id + """ + order_qt_spark_driven_rows_before_write sparkDrivenDorisQuery - spark_iceberg """REFRESH TABLE demo.${dbName}.${sparkTable}""" - def sparkDrivenSparkRows = spark_iceberg """ - SELECT id, - info.spark_first, - info.metric, - info.spark_after_metric, - info.label, - info.spark_renamed, - events[0].spark_first_score, - events[0].score, - events[0].spark_after_score, - events[0].spark_score_renamed, - attrs['k'].spark_first_code, - attrs['k'].code, - attrs['k'].spark_after_code, - attrs['k'].spark_code_renamed - FROM demo.${dbName}.${sparkTable} - ORDER BY id - """ - def sparkDrivenDorisRows = sql """ - SELECT id, - element_at(info, 'spark_first'), - element_at(info, 'metric'), - element_at(info, 'spark_after_metric'), - element_at(info, 'label'), - element_at(info, 'spark_renamed'), - element_at(events[1], 'spark_first_score'), - element_at(events[1], 'score'), - element_at(events[1], 'spark_after_score'), - element_at(events[1], 'spark_score_renamed'), - element_at(attrs['k'], 'spark_first_code'), - element_at(attrs['k'], 'code'), - element_at(attrs['k'], 'spark_after_code'), - element_at(attrs['k'], 'spark_code_renamed') - FROM ${sparkTable} - ORDER BY id - """ - assertSparkDorisResultEquals(sparkDrivenSparkRows, sparkDrivenDorisRows) - } finally { - try { - spark_iceberg """DROP TABLE IF EXISTS demo.${dbName}.${dorisTable}""" - spark_iceberg """DROP TABLE IF EXISTS demo.${dbName}.${sparkTable}""" - spark_iceberg """DROP TABLE IF EXISTS demo.${dbName}.${mixedCaseTable}""" - } catch (Throwable t) { - logger.warn("failed to clean Spark Iceberg tables", t) - } - try { - sql """switch ${catalogName}""" - sql """drop database if exists ${dbName} force""" - } catch (Throwable t) { - logger.warn("failed to clean Doris Iceberg database", t) - } - sql """drop catalog if exists ${catalogName}""" - } + sql """ + INSERT INTO ${sparkTable} VALUES ( + 3, + STRUCT('doris_first_field', 30, 'doris_after_metric_field', + 'doris_after_spark', 'doris_can_write_spark_field'), + ARRAY(STRUCT(302, 300, 303, 301)), + MAP('k', STRUCT(3002, 3000, 3003, 3001)) + ) + """ + + spark_iceberg """REFRESH TABLE demo.${dbName}.${sparkTable}""" + def sparkDrivenSparkRows = spark_iceberg """ + SELECT id, + info.spark_first, + info.metric, + info.spark_after_metric, + info.label, + info.spark_renamed, + events[0].spark_first_score, + events[0].score, + events[0].spark_after_score, + events[0].spark_score_renamed, + attrs['k'].spark_first_code, + attrs['k'].code, + attrs['k'].spark_after_code, + attrs['k'].spark_code_renamed + FROM demo.${dbName}.${sparkTable} + ORDER BY id + """ + order_qt_spark_driven_rows_after_write sparkDrivenDorisQuery + def sparkDrivenDorisRows = sql sparkDrivenDorisQuery + assertSparkDorisResultEquals(sparkDrivenSparkRows, sparkDrivenDorisRows) } From a16c6b2509e9672954fea25a9092b488b62448b5 Mon Sep 17 00:00:00 2001 From: daidai Date: Tue, 14 Jul 2026 00:12:33 +0800 Subject: [PATCH 05/32] [test](iceberg) Verify nested column comment updates ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: The Iceberg nested schema evolution regression executes comment-only changes for struct, array-element, and map-value nested fields, but existing tests did not verify that Doris passes the canonical Iceberg path and requested comment to UpdateSchema. Add focused FE coverage through the public modifyColumnComment path for all three nested shapes and verify each update is committed. ### Release note None ### Check List (For Author) - Test: Unit Test - IcebergMetadataOpsValidationTest (26 tests, 0 failures) - Behavior changed: No - Does this need documentation: No --- .../IcebergMetadataOpsValidationTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index e6286f48f767d0..bf0838fa872c05 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -299,6 +299,34 @@ public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throw Mockito.verify(updateSchema, Mockito.times(2)).commit(); } + @Test + public void testModifyColumnCommentUsesCanonicalNestedPaths() throws Throwable { + Schema schema = mixedCaseNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("info.metric"), + "struct comment", 1L); + ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("events.element.score"), + "array element comment", 1L); + ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("attrs.value.code"), + "map value comment", 1L); + } + + Mockito.verify(updateSchema).updateColumnDoc("Info.Metric", "struct comment"); + Mockito.verify(updateSchema).updateColumnDoc("Events.element.Score", "array element comment"); + Mockito.verify(updateSchema).updateColumnDoc("Attrs.value.Code", "map value comment"); + Mockito.verify(updateSchema, Mockito.times(3)).commit(); + } + @Test public void testResolveNestedColumnPathRejectsMapKey() { assertUserException(() -> ops.resolveNestedColumnPath(nestedSchema(), ColumnPath.fromDotName("m.key.x"), From 5831ec0f61896ec4cb14ee23df634595c7fad912 Mon Sep 17 00:00:00 2001 From: daidai Date: Tue, 14 Jul 2026 01:22:51 +0800 Subject: [PATCH 06/32] [fix](iceberg) Support direct nested element comments ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Direct Iceberg list-element and map-value comment updates were routed through struct-child validation, so they failed before reaching Iceberg even though the adjacent type-modification path supports those targets. Route comment updates through the canonical Iceberg column-path resolver, retain MAP key rejection, and add focused unit, parser, and regression coverage. The parser coverage also proves that quoted nested identifiers, escaped backticks, quoted AFTER references, and rename targets are already normalized by the parser postprocessor. ### Release note Support modifying comments on direct Iceberg array elements and map values. ### Check List (For Author) - Test: Regression test and Unit Test - ./run-fe-ut.sh --run org.apache.doris.datasource.iceberg.IcebergMetadataOpsValidationTest,org.apache.doris.nereids.parser.IcebergNestedSchemaEvolutionParserTest - ./run-regression-test.sh --run -d external_table_p0/iceberg -s test_iceberg_nested_schema_evolution_ddl - ./build.sh --fe - Behavior changed: Yes. Direct Iceberg array-element and map-value comments are now supported while MAP key comments remain rejected. - Does this need documentation: No --- .../iceberg/IcebergMetadataOps.java | 8 ++---- .../IcebergMetadataOpsValidationTest.java | 28 +++++++++++++++++++ ...cebergNestedSchemaEvolutionParserTest.java | 23 ++++++++++++++- ...iceberg_nested_schema_evolution_ddl.groovy | 9 +++++- 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 4b4728cc40fbfe..6436e0a2a416fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -1078,12 +1078,8 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); - ResolvedColumnPath resolvedPath; - if (columnPath.isNested()) { - resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "modify comment"); - } else { - resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify comment"); - } + ResolvedColumnPath resolvedPath = resolveColumnPath( + icebergTable.schema(), columnPath, "modify comment"); UpdateSchema updateSchema = icebergTable.updateSchema(); updateSchema.updateColumnDoc(resolvedPath.getFullPath(), StringUtils.defaultString(comment)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index bf0838fa872c05..86fa7414665432 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -327,6 +327,34 @@ public void testModifyColumnCommentUsesCanonicalNestedPaths() throws Throwable { Mockito.verify(updateSchema, Mockito.times(3)).commit(); } + @Test + public void testModifyColumnCommentSupportsDirectArrayElementAndMapValue() throws Throwable { + Schema schema = primitiveContainerSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("arr.element"), + "array element comment", 1L); + ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("m.value"), + "map value comment", 1L); + assertUserException(() -> ops.modifyColumnComment( + dorisTable, ColumnPath.fromDotName("m.key"), "map key comment", 1L), + "Cannot modify comment MAP key nested column"); + } + + Mockito.verify(updateSchema).updateColumnDoc("arr.element", "array element comment"); + Mockito.verify(updateSchema).updateColumnDoc("m.value", "map value comment"); + Mockito.verify(updateSchema, Mockito.times(2)).commit(); + } + @Test public void testResolveNestedColumnPathRejectsMapKey() { assertUserException(() -> ops.resolveNestedColumnPath(nestedSchema(), ColumnPath.fromDotName("m.key.x"), diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index fab9a4da1280fb..ff7322ca9487e6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -90,7 +90,27 @@ public void testLegacyStringConstructorsKeepDottedTopLevelNames() { Assertions.assertEquals("top.level", comment.getColumnPath().getFullPath()); } - private void assertSingleClausePath(String sql, Class clauseClass, + @Test + public void testQuotedNestedIdentifiersAreNormalized() { + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN info.`Metric` BIGINT", + ModifyColumnOp.class, "info.Metric"); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN m_scalar.`key` BIGINT", + ModifyColumnOp.class, "m_scalar.key"); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN info.`Metric``Name` COMMENT 'quoted'", + ModifyColumnCommentOp.class, "info.Metric`Name"); + + AddColumnOp add = assertSingleClausePath( + "ALTER TABLE t ADD COLUMN info.`New``Field` INT NULL AFTER `Old``Field`", + AddColumnOp.class, "info.New`Field"); + Assertions.assertEquals("Old`Field", add.getColPos().getLastCol()); + + RenameColumnOp rename = assertSingleClausePath( + "ALTER TABLE t RENAME COLUMN info.`Metric` TO `New``Metric`", + RenameColumnOp.class, "info.Metric"); + Assertions.assertEquals("New`Metric", rename.getNewColName()); + } + + private T assertSingleClausePath(String sql, Class clauseClass, String expectedPath) { Plan plan = parser.parseSingle(sql); Assertions.assertInstanceOf(AlterTableCommand.class, plan); @@ -109,5 +129,6 @@ private void assertSingleClausePath(String sql, Class Date: Tue, 14 Jul 2026 10:13:49 +0800 Subject: [PATCH 07/32] [fix](iceberg) Reject duplicate appended struct fields ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Complex Iceberg MODIFY validation already canonicalizes struct field names and rejects a field that conflicts with an existing sibling before UpdateSchema is mutated. However, the validator did not record each accepted appended field, so a programmatic request containing two appended siblings with the same canonical name could reach UpdateSchema. Record appended names during validation and add public-entry coverage for mixed-case collisions in structs, list element structs, map value structs, and between appended siblings. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.datasource.iceberg.IcebergMetadataOpsValidationTest,org.apache.doris.nereids.parser.IcebergNestedSchemaEvolutionParserTest - Behavior changed: Yes. Invalid complex type modifications with duplicate appended sibling names are rejected before UpdateSchema is mutated. - Does this need documentation: No --- .../org/apache/doris/catalog/ColumnType.java | 5 +- .../IcebergMetadataOpsValidationTest.java | 46 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java index 015fecfd28a263..483f22e2425f16 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java @@ -322,11 +322,10 @@ private static void checkSupportSchemaChangeForComplexType(Type checkType, Type existingNames.add(originalField.getName()); } - // check new field name is not conflict with old field name + // check appended field names do not conflict with existing or earlier appended fields for (int i = originalFields.size(); i < otherStructType.getFields().size(); i++) { - // to check new field name is not conflict with old field name String newFieldName = otherStructType.getFields().get(i).getName(); - if (existingNames.contains(newFieldName)) { + if (!existingNames.add(newFieldName)) { throw new DdlException("Added struct field '" + newFieldName + "' conflicts with existing field"); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 86fa7414665432..eeb83c28ec96b3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -193,6 +193,52 @@ public void testValidateForModifyComplexColumnSuccess() throws Throwable { invokeValidateForModifyComplexColumn(column, currentCol); } + @Test + public void testModifyComplexColumnRejectsCaseInsensitiveStructFieldAdditions() { + Schema schema = mixedCaseNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + StructType infoType = new StructType( + new StructField("Metric", Type.INT), + new StructField("Label", Type.STRING), + new StructField("metric", Type.INT)); + ArrayType eventsType = ArrayType.create(new StructType( + new StructField("Score", Type.INT), + new StructField("score", Type.INT)), true); + MapType attrsType = new MapType(Type.STRING, new StructType( + new StructField("Code", Type.INT), + new StructField("code", Type.INT))); + StructType duplicateNewFieldsType = new StructType( + new StructField("Metric", Type.INT), + new StructField("Label", Type.STRING), + new StructField("Extra", Type.INT), + new StructField("EXTRA", Type.STRING)); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn( + dorisTable, new Column("info", infoType, true), null, 1L), + "Added struct field 'metric' conflicts with existing field"); + assertUserException(() -> ops.modifyColumn( + dorisTable, new Column("events", eventsType, true), null, 1L), + "Added struct field 'score' conflicts with existing field"); + assertUserException(() -> ops.modifyColumn( + dorisTable, new Column("attrs", attrsType, true), null, 1L), + "Added struct field 'code' conflicts with existing field"); + assertUserException(() -> ops.modifyColumn( + dorisTable, new Column("info", duplicateNewFieldsType, true), null, 1L), + "Added struct field 'extra' conflicts with existing field"); + } + + Mockito.verifyNoInteractions(updateSchema); + } + @Test public void testResolveNestedColumnPathSupportsStructArrayElementAndMapValue() throws Throwable { Schema schema = nestedSchema(); From d91fb5339522f5330ca4ec8c7f0d153a3ad8b301 Mon Sep 17 00:00:00 2001 From: daidai Date: Tue, 14 Jul 2026 15:22:01 +0800 Subject: [PATCH 08/32] [fix](iceberg) Address nested schema review findings ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Nested Iceberg schema evolution had four correctness gaps found during review. Generated SQL did not escape embedded backticks, full complex modifications could silently relax required nested fields, unsupported Doris target types could escape validation as runtime exceptions, and nested MODIFY default metadata could be accepted but ignored. Use the shared identifier renderer, preserve existing requiredness for full complex nested modifications, validate Iceberg type encodability before mutating UpdateSchema, and reject nested MODIFY DEFAULT or ON UPDATE clauses before their explicit presence is lost. Add focused parser, command, and metadata-operation coverage for each path. ### Release note Preserve existing Iceberg requiredness during nested complex modifications and reject unsupported target types or ignored default changes. ### Check List (For Author) - Test: Unit Test (not completed; stopped at user request) - Added focused parser, command, and Iceberg metadata operation tests - git diff --cached --check - Behavior changed: Yes. Nested Iceberg schema modifications now preserve requiredness, reject unsupported target types with UserException, reject ignored default metadata, and render quoted identifiers safely. - Does this need documentation: No --- .../doris/catalog/info/ColumnPosition.java | 3 +- .../org/apache/doris/analysis/ColumnPath.java | 4 +- .../iceberg/IcebergMetadataOps.java | 24 ++-- .../plans/commands/AlterTableCommand.java | 11 ++ .../plans/commands/info/ColumnDefinition.java | 4 + .../plans/commands/info/RenameColumnOp.java | 3 +- .../IcebergMetadataOpsValidationTest.java | 108 +++++++++++++++++- ...cebergNestedSchemaEvolutionParserTest.java | 10 +- .../plans/commands/AlterTableCommandTest.java | 20 ++++ 9 files changed, 167 insertions(+), 20 deletions(-) diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/info/ColumnPosition.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/info/ColumnPosition.java index 7c30c725d6c3a4..5b6b53918ee59c 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/info/ColumnPosition.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/info/ColumnPosition.java @@ -18,6 +18,7 @@ package org.apache.doris.catalog.info; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.SqlUtils; import com.google.common.base.Strings; @@ -57,7 +58,7 @@ public String toSql() { if (this == FIRST) { sb.append("FIRST"); } else { - sb.append("AFTER `").append(lastCol).append("`"); + sb.append("AFTER ").append(SqlUtils.getIdentSql(lastCol)); } return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPath.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPath.java index efda051b3d8e97..0e28f38de41ef5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPath.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPath.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.common.util.SqlUtils; + import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -80,7 +82,7 @@ public String getFullPath() { } public String toSql() { - return parts.stream().map(part -> "`" + part + "`").collect(Collectors.joining(".")); + return parts.stream().map(SqlUtils::getIdentSql).collect(Collectors.joining(".")); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 6436e0a2a416fa..e83c062e4798fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -1041,14 +1041,16 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column NestedField currentCol = resolvedPath.getField(); validateCommonColumnInfo(column); + if (column.hasDefaultValue() || column.hasOnUpdateDefaultValue()) { + throw new UserException("Modifying default values is not supported for Iceberg columns: " + columnPath); + } UpdateSchema updateSchema = icebergTable.updateSchema(); if (column.getType().isComplexType()) { validateForModifyComplexColumn(column, currentCol, columnPath.getFullPath()); applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), column.getType()); - if (column.isAllowNull()) { - updateSchema.makeColumnOptional(resolvedPath.getFullPath()); - } + // Column does not preserve whether NULL was explicit or omitted for MODIFY. + // Keep the existing target requiredness instead of silently relaxing it. if (!Objects.equals(currentCol.doc(), column.getComment())) { updateSchema.updateColumnDoc(resolvedPath.getFullPath(), column.getComment()); } @@ -1360,10 +1362,6 @@ private void applyStructChange(UpdateSchema updateSchema, String path, updateSchema.updateColumnDoc(fieldPath, newField.getComment()); } } - - if (!oldField.isOptional() && newField.getContainsNull()) { - updateSchema.makeColumnOptional(fieldPath); - } } for (int i = oldFields.size(); i < newFields.size(); i++) { @@ -1396,9 +1394,6 @@ private void applyListChange(UpdateSchema updateSchema, String path, } else { applyComplexTypeChange(updateSchema, elementPath, oldElementType, newElementType); } - if (!oldListType.isElementOptional() && newArrayType.getContainsNull()) { - updateSchema.makeColumnOptional(elementPath); - } } private void applyMapChange(UpdateSchema updateSchema, String path, @@ -1429,9 +1424,6 @@ private void applyMapChange(UpdateSchema updateSchema, String path, } else { applyComplexTypeChange(updateSchema, valuePath, oldValueType, newValueType); } - if (!oldMapType.isValueOptional() && newMapType.getIsValueContainsNull()) { - updateSchema.makeColumnOptional(valuePath); - } } private void validateCommonColumnInfo(Column column) throws UserException { @@ -1443,6 +1435,12 @@ private void validateCommonColumnInfo(Column column) throws UserException { if (column.isAutoInc()) { throw new UserException("Can not specify auto incremental iceberg table column"); } + try { + IcebergUtils.dorisTypeToIcebergType(column.getType()); + } catch (UnsupportedOperationException | IllegalArgumentException e) { + throw new UserException("Type " + column.getType().toSql() + + " is not supported for Iceberg column " + column.getName(), e); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java index 2db06ba7c12b62..a56bf1b2e07f90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java @@ -143,6 +143,17 @@ private void validate(ConnectContext ctx) throws UserException { static void checkNestedColumnPathSupported(TableIf table, List alterTableOps) throws AnalysisException { if (table instanceof IcebergExternalTable) { + for (AlterTableOp alterTableOp : alterTableOps) { + if (alterTableOp instanceof ModifyColumnOp) { + ModifyColumnOp modifyColumnOp = (ModifyColumnOp) alterTableOp; + if (modifyColumnOp.getColumnPath().isNested() + && (modifyColumnOp.getColumnDef().hasDefaultValue() + || modifyColumnOp.getColumnDef().hasOnUpdateDefaultValue())) { + throw new AnalysisException("DEFAULT and ON UPDATE are not supported for nested Iceberg " + + "MODIFY COLUMN: " + modifyColumnOp.getColumnPath().getFullPath()); + } + } + } return; } for (AlterTableOp alterTableOp : alterTableOps) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index dbd37b4886e927..46ef11e8130a04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -183,6 +183,10 @@ public boolean hasDefaultValue() { return defaultValue.isPresent(); } + public boolean hasOnUpdateDefaultValue() { + return onUpdateDefaultValue.isPresent(); + } + public boolean isVisible() { return isVisible; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java index d375e545ac7a5b..2a1e7a65ab133a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java @@ -23,6 +23,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; @@ -94,7 +95,7 @@ public boolean needChangeMTMVState() { @Override public String toSql() { - return "RENAME COLUMN " + columnPath.toSql() + " " + newColName; + return "RENAME COLUMN " + columnPath.toSql() + " " + SqlUtils.getIdentSql(newColName); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index eeb83c28ec96b3..2638ea8b0b83f2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -193,6 +193,98 @@ public void testValidateForModifyComplexColumnSuccess() throws Throwable { invokeValidateForModifyComplexColumn(column, currentCol); } + @Test + public void testRejectUnsupportedIcebergTargetTypesBeforeUpdateSchema() { + Schema schema = requiredNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + StructType unsupportedStruct = new StructType(new StructField("value", Type.LARGEINT)); + ArrayType unsupportedArray = ArrayType.create(Type.LARGEINT, true); + MapType unsupportedMap = new MapType(Type.STRING, Type.LARGEINT); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("info.new_field"), + new Column("new_field", Type.LARGEINT, true), null, 1L), + "is not supported for Iceberg column new_field"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), + new Column("metric", Type.LARGEINT, true), null, 1L), + "is not supported for Iceberg column metric"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), + new Column("child", unsupportedStruct, false), null, 1L), + "is not supported for Iceberg column child"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.events"), + new Column("events", unsupportedArray, false), null, 1L), + "is not supported for Iceberg column events"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.attrs"), + new Column("attrs", unsupportedMap, false), null, 1L), + "is not supported for Iceberg column attrs"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testComplexModifyPreservesRequiredNestedFields() throws Throwable { + Schema schema = requiredNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), + new Column("child", new StructType(new StructField("value", Type.BIGINT)), true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.events"), + new Column("events", ArrayType.create(Type.BIGINT, true), true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.attrs"), + new Column("attrs", new MapType(Type.STRING, Type.BIGINT), true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("info.child.value", Types.LongType.get(), null); + Mockito.verify(updateSchema).updateColumn("info.events.element", Types.LongType.get(), null); + Mockito.verify(updateSchema).updateColumn("info.attrs.value", Types.LongType.get(), null); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional(Mockito.anyString()); + Mockito.verify(updateSchema, Mockito.times(3)).commit(); + } + + @Test + public void testModifyColumnRejectsDefaultMetadata() { + Schema schema = nestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + Column defaultColumn = new Column("a", Type.BIGINT, false, null, true, "7", ""); + Column onUpdateColumn = Mockito.spy(new Column("a", Type.BIGINT, true)); + Mockito.doReturn(true).when(onUpdateColumn).hasOnUpdateDefaultValue(); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("s.a"), + defaultColumn, null, 1L), + "Modifying default values is not supported for Iceberg columns: s.a"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("s.a"), + onUpdateColumn, null, 1L), + "Modifying default values is not supported for Iceberg columns: s.a"); + } + + Mockito.verifyNoInteractions(updateSchema); + } + @Test public void testModifyComplexColumnRejectsCaseInsensitiveStructFieldAdditions() { Schema schema = mixedCaseNestedSchema(); @@ -479,8 +571,9 @@ private void assertUserException(ThrowingRunnable runnable, String expectedMessa runnable.run(); Assert.fail("expected UserException"); } catch (Throwable t) { - Assert.assertTrue(t instanceof UserException); - Assert.assertTrue(t.getMessage().contains(expectedMessage)); + Assert.assertTrue("expected UserException but got " + t, t instanceof UserException); + Assert.assertTrue("expected message containing '" + expectedMessage + "' but was '" + + t.getMessage() + "'", t.getMessage().contains(expectedMessage)); } } @@ -521,4 +614,15 @@ private Schema primitiveContainerSchema() { Types.NestedField.optional(3, "m", Types.MapType.ofOptional( 4, 5, Types.StringType.get(), Types.IntegerType.get()))); } + + private Schema requiredNestedSchema() { + return new Schema(Types.NestedField.optional(1, "info", Types.StructType.of( + Types.NestedField.optional(2, "metric", Types.IntegerType.get()), + Types.NestedField.required(3, "child", Types.StructType.of( + Types.NestedField.required(4, "value", Types.IntegerType.get()))), + Types.NestedField.required(5, "events", Types.ListType.ofRequired( + 6, Types.IntegerType.get())), + Types.NestedField.required(7, "attrs", Types.MapType.ofRequired( + 8, 9, Types.StringType.get(), Types.IntegerType.get()))))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index ff7322ca9487e6..de3cd3fb31d20c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -103,11 +103,17 @@ public void testQuotedNestedIdentifiersAreNormalized() { "ALTER TABLE t ADD COLUMN info.`New``Field` INT NULL AFTER `Old``Field`", AddColumnOp.class, "info.New`Field"); Assertions.assertEquals("Old`Field", add.getColPos().getLastCol()); + AddColumnOp reparsedAdd = assertSingleClausePath( + "ALTER TABLE t " + add.toSql(), AddColumnOp.class, "info.New`Field"); + Assertions.assertEquals("Old`Field", reparsedAdd.getColPos().getLastCol()); RenameColumnOp rename = assertSingleClausePath( - "ALTER TABLE t RENAME COLUMN info.`Metric` TO `New``Metric`", - RenameColumnOp.class, "info.Metric"); + "ALTER TABLE t RENAME COLUMN info.`Metric``Name` TO `New``Metric`", + RenameColumnOp.class, "info.Metric`Name"); Assertions.assertEquals("New`Metric", rename.getNewColName()); + RenameColumnOp reparsedRename = assertSingleClausePath( + "ALTER TABLE t " + rename.toSql(), RenameColumnOp.class, "info.Metric`Name"); + Assertions.assertEquals("New`Metric", reparsedRename.getNewColName()); } private T assertSingleClausePath(String sql, Class clauseClass, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index 7e0151e25fe383..5b46a6d5feed46 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -188,6 +188,26 @@ void testAllowNestedColumnPathForIcebergTable() throws AnalysisException { parseAlter("ALTER TABLE t ADD COLUMN s.c STRING NULL").getOps()); } + @Test + void testRejectDefaultMetadataForNestedIcebergModify() throws AnalysisException { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t MODIFY COLUMN s.a BIGINT DEFAULT 7", + "ALTER TABLE t MODIFY COLUMN s.a BIGINT DEFAULT NULL", + "ALTER TABLE t MODIFY COLUMN s.ts DATETIME DEFAULT CURRENT_TIMESTAMP " + + "ON UPDATE CURRENT_TIMESTAMP")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkNestedColumnPathSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("DEFAULT and ON UPDATE are not supported for nested Iceberg MODIFY COLUMN")); + } + + AlterTableCommand.checkNestedColumnPathSupported(table, + parseAlter("ALTER TABLE t MODIFY COLUMN s.a BIGINT").getOps()); + AlterTableCommand.checkNestedColumnPathSupported(table, + parseAlter("ALTER TABLE t ADD COLUMN s.b BIGINT NULL DEFAULT 7").getOps()); + } + private AlterTableCommand parseAlter(String sql) { Plan plan = parser.parseSingle(sql); Assertions.assertInstanceOf(AlterTableCommand.class, plan); From ca388a465f2fb2bdfaf726949f10d32e57baea26 Mon Sep 17 00:00:00 2001 From: daidai Date: Tue, 14 Jul 2026 17:12:18 +0800 Subject: [PATCH 09/32] [fix](iceberg) Preserve requiredness in column modifications ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Iceberg MODIFY COLUMN translated omitted nullability to a nullable Column, so top-level primitive and complex fields as well as nested primitive fields could be silently changed from required to optional during a type or comment update. Preserve whether NULL or NOT NULL was explicitly specified as runtime-only schema-change intent, and call Iceberg makeColumnOptional only for an explicit NULL clause. This keeps requiredness unchanged when nullability is omitted while retaining the existing supported required-to-optional behavior for explicit NULL. Add parser and metadata-operation coverage for nested required primitives, mixed-case top-level primitive and complex fields, explicit NULL, and direct list/map targets. ### Release note Iceberg MODIFY COLUMN now preserves existing requiredness when nullability is omitted and only relaxes it for an explicit NULL clause. ### Check List (For Author) - Test: Unit Test (not run at user request) - Added focused parser and Iceberg metadata operation tests - git diff --cached --check - Behavior changed: Yes. Omitted nullability no longer silently relaxes Iceberg required fields; explicit NULL remains supported. - Does this need documentation: No --- .../java/org/apache/doris/catalog/Column.java | 11 +++ .../iceberg/IcebergMetadataOps.java | 21 ++--- .../plans/commands/info/ColumnDefinition.java | 7 ++ .../IcebergMetadataOpsValidationTest.java | 91 ++++++++++++++++++- .../plans/commands/AlterTableCommandTest.java | 18 ++++ 5 files changed, 131 insertions(+), 17 deletions(-) diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java index f3acb582001073..a689682f694cee 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java @@ -137,6 +137,8 @@ public static Column generateBeforeValueColumn(Column column) { private boolean isKey; @SerializedName(value = "isAllowNull") private boolean isAllowNull; + // Runtime-only schema change intent; do not persist it as part of the table schema. + private transient boolean nullableSpecified; @SerializedName(value = "isAutoInc") private boolean isAutoInc; @@ -363,6 +365,7 @@ public Column(Column column) { this.isKey = column.isKey(); this.isCompoundKey = column.isCompoundKey(); this.isAllowNull = column.isAllowNull(); + this.nullableSpecified = column.isNullableSpecified(); this.isAutoInc = column.isAutoInc(); this.defaultValue = column.getDefaultValue(); this.realDefaultValue = column.realDefaultValue; @@ -578,6 +581,10 @@ public boolean isAllowNull() { return isAllowNull; } + public boolean isNullableSpecified() { + return nullableSpecified; + } + public boolean isAutoInc() { return isAutoInc; } @@ -590,6 +597,10 @@ public void setIsAllowNull(boolean isAllowNull) { this.isAllowNull = isAllowNull; } + public void setNullableSpecified(boolean nullableSpecified) { + this.nullableSpecified = nullableSpecified; + } + public String getDefaultValue() { return this.defaultValue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index e83c062e4798fc..759f08d7af4d84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -998,9 +998,6 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition // Complex type processing branch validateForModifyComplexColumn(column, currentCol); applyComplexTypeChange(updateSchema, columnPath.getFullPath(), currentCol.type(), column.getType()); - if (column.isAllowNull()) { - updateSchema.makeColumnOptional(columnPath.getFullPath()); - } if (!Objects.equals(currentCol.doc(), column.getComment())) { updateSchema.updateColumnDoc(columnPath.getFullPath(), column.getComment()); } @@ -1009,12 +1006,8 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition validateForModifyColumn(column, currentCol); Type icebergType = IcebergUtils.dorisTypeToIcebergType(column.getType()); updateSchema.updateColumn(columnPath.getFullPath(), icebergType.asPrimitiveType(), column.getComment()); - if (column.isAllowNull()) { - // we can change a required column to optional, but not the other way around - // because we don't know whether there is existing data with null values. - updateSchema.makeColumnOptional(columnPath.getFullPath()); - } } + applyExplicitNullableChange(updateSchema, columnPath.getFullPath(), column); if (position != null) { applyPosition(updateSchema, position, columnPath.getColumnPath(), icebergTable.schema(), "modify"); @@ -1049,8 +1042,6 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column if (column.getType().isComplexType()) { validateForModifyComplexColumn(column, currentCol, columnPath.getFullPath()); applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), column.getType()); - // Column does not preserve whether NULL was explicit or omitted for MODIFY. - // Keep the existing target requiredness instead of silently relaxing it. if (!Objects.equals(currentCol.doc(), column.getComment())) { updateSchema.updateColumnDoc(resolvedPath.getFullPath(), column.getComment()); } @@ -1058,10 +1049,8 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column validateForModifyColumn(column, currentCol, columnPath.getFullPath()); Type icebergType = IcebergUtils.dorisTypeToIcebergType(column.getType()); updateSchema.updateColumn(resolvedPath.getFullPath(), icebergType.asPrimitiveType(), column.getComment()); - if (column.isAllowNull()) { - updateSchema.makeColumnOptional(resolvedPath.getFullPath()); - } } + applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column); if (position != null) { applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); @@ -1094,6 +1083,12 @@ public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, refreshTable(dorisTable, updateTime); } + private void applyExplicitNullableChange(UpdateSchema updateSchema, String columnPath, Column column) { + if (column.isNullableSpecified() && column.isAllowNull()) { + updateSchema.makeColumnOptional(columnPath); + } + } + private void validateForModifyColumn(Column column, NestedField currentCol) throws UserException { validateForModifyColumn(column, currentCol, column.getName()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 46ef11e8130a04..a0a2cc66e4f067 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -61,6 +61,8 @@ public class ColumnDefinition { private boolean isKey; private AggregateType aggType; private boolean isNullable; + // Distinguishes an explicit NULL/NOT NULL clause from the parser's default nullability. + private final boolean nullableSpecified; private Optional defaultValue; private Optional onUpdateDefaultValue = Optional.empty(); private final String comment; @@ -96,6 +98,7 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType this.isKey = isKey; this.aggType = aggType; this.isNullable = isNullable; + this.nullableSpecified = true; this.defaultValue = defaultValue; this.comment = comment; this.isVisible = isVisible; @@ -112,6 +115,7 @@ private ColumnDefinition(String name, DataType type, boolean isKey, AggregateTyp this.isKey = isKey; this.aggType = aggType; this.isNullable = isNullable; + this.nullableSpecified = true; this.autoIncInitValue = autoIncInitValue; this.defaultValue = defaultValue; this.onUpdateDefaultValue = onUpdateDefaultValue; @@ -131,6 +135,8 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType this.isKey = isKey; this.aggType = aggType; this.isNullable = nullableType.getNullable(type.toCatalogDataType().getPrimitiveType()); + this.nullableSpecified = nullableType == ColumnNullableType.NULLABLE + || nullableType == ColumnNullableType.NOT_NULLABLE; this.autoIncInitValue = autoIncInitValue; this.defaultValue = defaultValue; this.onUpdateDefaultValue = onUpdateDefaultValue; @@ -565,6 +571,7 @@ public Column translateToCatalogStyleForSchemaChange() { generatedColumnDesc.map(desc -> ConnectContextUtil.getAffectQueryResultInPlanVariables(ConnectContext.get())) .orElse(null)); + column.setNullableSpecified(nullableSpecified); column.setAggregationTypeImplicit(aggTypeImplicit); return column; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 2638ea8b0b83f2..dc1d7d325c887e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -257,6 +257,90 @@ public void testComplexModifyPreservesRequiredNestedFields() throws Throwable { Mockito.verify(updateSchema, Mockito.times(3)).commit(); } + @Test + public void testPrimitiveModifyPreservesRequiredNestedField() throws Throwable { + Schema schema = requiredNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), + new Column("metric", Type.BIGINT, true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("info.metric", Types.LongType.get(), ""); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional(Mockito.anyString()); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testTopLevelModifyPreservesRequiredMixedCaseFields() throws Throwable { + Schema schema = new Schema( + Types.NestedField.required(1, "Id", Types.IntegerType.get()), + Types.NestedField.required(2, "Payload", Types.StructType.of( + Types.NestedField.required(3, "Value", Types.IntegerType.get())))); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.of("id"), + new Column("id", Type.BIGINT, true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.of("payload"), new Column("payload", + new StructType(new StructField("Value", Type.BIGINT)), true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("Id", Types.LongType.get(), ""); + Mockito.verify(updateSchema).updateColumn("Payload.Value", Types.LongType.get(), null); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional(Mockito.anyString()); + Mockito.verify(updateSchema, Mockito.times(2)).commit(); + } + + @Test + public void testExplicitNullableModifyMakesRequiredFieldsOptional() throws Throwable { + Schema schema = requiredNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + Column topLevelColumn = new Column("info", new StructType( + new StructField("metric", Type.BIGINT), + new StructField("child", new StructType(new StructField("value", Type.INT))), + new StructField("events", ArrayType.create(Type.INT, true)), + new StructField("attrs", new MapType(Type.STRING, Type.INT))), true); + topLevelColumn.setNullableSpecified(true); + Column nestedColumn = new Column("metric", Type.BIGINT, true); + nestedColumn.setNullableSpecified(true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.of("info"), topLevelColumn, null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), nestedColumn, null, 1L); + } + + Mockito.verify(updateSchema).makeColumnOptional("info"); + Mockito.verify(updateSchema).makeColumnOptional("info.metric"); + Mockito.verify(updateSchema, Mockito.times(2)).commit(); + } + @Test public void testModifyColumnRejectsDefaultMetadata() { Schema schema = nestedSchema(); @@ -432,8 +516,7 @@ public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throw Mockito.verify(updateSchema).updateColumn("arr.element", Types.LongType.get(), ""); Mockito.verify(updateSchema).updateColumn("m.value", Types.LongType.get(), ""); - Mockito.verify(updateSchema).makeColumnOptional("arr.element"); - Mockito.verify(updateSchema).makeColumnOptional("m.value"); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional(Mockito.anyString()); Mockito.verify(updateSchema, Mockito.times(2)).commit(); } @@ -616,8 +699,8 @@ private Schema primitiveContainerSchema() { } private Schema requiredNestedSchema() { - return new Schema(Types.NestedField.optional(1, "info", Types.StructType.of( - Types.NestedField.optional(2, "metric", Types.IntegerType.get()), + return new Schema(Types.NestedField.required(1, "info", Types.StructType.of( + Types.NestedField.required(2, "metric", Types.IntegerType.get()), Types.NestedField.required(3, "child", Types.StructType.of( Types.NestedField.required(4, "value", Types.IntegerType.get()))), Types.NestedField.required(5, "events", Types.ListType.ofRequired( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index 5b46a6d5feed46..c6e7a92104db58 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.AlterTableOp; import org.apache.doris.nereids.trees.plans.commands.info.DropPartitionFieldOp; import org.apache.doris.nereids.trees.plans.commands.info.EnableFeatureOp; +import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnOp; import org.apache.doris.nereids.trees.plans.commands.info.ReplacePartitionFieldOp; import org.junit.jupiter.api.Assertions; @@ -208,6 +209,23 @@ void testRejectDefaultMetadataForNestedIcebergModify() throws AnalysisException parseAlter("ALTER TABLE t ADD COLUMN s.b BIGINT NULL DEFAULT 7").getOps()); } + @Test + void testModifyColumnTracksExplicitNullability() { + ModifyColumnOp omitted = (ModifyColumnOp) parseAlter( + "ALTER TABLE t MODIFY COLUMN s.a BIGINT").getOps().get(0); + ModifyColumnOp nullable = (ModifyColumnOp) parseAlter( + "ALTER TABLE t MODIFY COLUMN s.a BIGINT NULL").getOps().get(0); + ModifyColumnOp notNullable = (ModifyColumnOp) parseAlter( + "ALTER TABLE t MODIFY COLUMN s.a BIGINT NOT NULL").getOps().get(0); + + Assertions.assertFalse(omitted.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + Assertions.assertTrue(nullable.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + Assertions.assertTrue(notNullable.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + } + private AlterTableCommand parseAlter(String sql) { Plan plan = parser.parseSingle(sql); Assertions.assertInstanceOf(AlterTableCommand.class, plan); From 013a3b0472761acfaeade62a842169c9b4f2f561 Mon Sep 17 00:00:00 2001 From: daidai Date: Wed, 15 Jul 2026 00:30:30 +0800 Subject: [PATCH 10/32] [fix](iceberg) Reject unsupported nested column clauses ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Nested Iceberg ADD, DROP, and MODIFY COLUMN clauses could carry rollup targets that are not propagated to external catalog operations. Nested ADD and MODIFY could also carry KEY or generated-column metadata that Iceberg schema updates ignore. Reject these unsupported clauses before dispatch so ALTER TABLE cannot report success after silently dropping user intent. ### Release note Reject unsupported rollup, KEY, and generated-column clauses in nested Iceberg column schema changes. ### Check List (For Author) - Test: Unit Test (not run at user request) - Added focused AlterTableCommand parser and validation tests - git diff --cached --check - Behavior changed: Yes. Unsupported nested Iceberg ALTER COLUMN clauses now fail analysis instead of being silently ignored. - Does this need documentation: No --- .../plans/commands/AlterTableCommand.java | 53 ++++++++++++++++--- .../plans/commands/info/AddColumnOp.java | 4 ++ .../plans/commands/AlterTableCommandTest.java | 41 ++++++++++++++ 3 files changed, 90 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java index a56bf1b2e07f90..76e45d1dce7d62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java @@ -144,14 +144,28 @@ static void checkNestedColumnPathSupported(TableIf table, List alt throws AnalysisException { if (table instanceof IcebergExternalTable) { for (AlterTableOp alterTableOp : alterTableOps) { - if (alterTableOp instanceof ModifyColumnOp) { - ModifyColumnOp modifyColumnOp = (ModifyColumnOp) alterTableOp; - if (modifyColumnOp.getColumnPath().isNested() - && (modifyColumnOp.getColumnDef().hasDefaultValue() - || modifyColumnOp.getColumnDef().hasOnUpdateDefaultValue())) { - throw new AnalysisException("DEFAULT and ON UPDATE are not supported for nested Iceberg " - + "MODIFY COLUMN: " + modifyColumnOp.getColumnPath().getFullPath()); - } + ColumnPath columnPath = getNestedColumnPath(alterTableOp); + if (columnPath == null) { + continue; + } + if (getRollupName(alterTableOp) != null) { + throw new AnalysisException("Rollup is not supported for nested Iceberg column operation: " + + columnPath.getFullPath()); + } + ColumnDefinition columnDefinition = getColumnDefinition(alterTableOp); + if (columnDefinition != null && columnDefinition.isKey()) { + throw new AnalysisException("KEY is not supported for nested Iceberg ADD/MODIFY COLUMN: " + + columnPath.getFullPath()); + } + if (columnDefinition != null && columnDefinition.getGeneratedColumnDesc().isPresent()) { + throw new AnalysisException("Generated columns are not supported for nested Iceberg " + + "ADD/MODIFY COLUMN: " + columnPath.getFullPath()); + } + if (alterTableOp instanceof ModifyColumnOp && columnDefinition != null + && (columnDefinition.hasDefaultValue() + || columnDefinition.hasOnUpdateDefaultValue())) { + throw new AnalysisException("DEFAULT and ON UPDATE are not supported for nested Iceberg " + + "MODIFY COLUMN: " + columnPath.getFullPath()); } } return; @@ -181,6 +195,29 @@ private static ColumnPath getNestedColumnPath(AlterTableOp alterTableOp) { return columnPath != null && columnPath.isNested() ? columnPath : null; } + private static ColumnDefinition getColumnDefinition(AlterTableOp alterTableOp) { + if (alterTableOp instanceof AddColumnOp) { + return ((AddColumnOp) alterTableOp).getColumnDef(); + } + if (alterTableOp instanceof ModifyColumnOp) { + return ((ModifyColumnOp) alterTableOp).getColumnDef(); + } + return null; + } + + private static String getRollupName(AlterTableOp alterTableOp) { + if (alterTableOp instanceof AddColumnOp) { + return ((AddColumnOp) alterTableOp).getRollupName(); + } + if (alterTableOp instanceof DropColumnOp) { + return ((DropColumnOp) alterTableOp).getRollupName(); + } + if (alterTableOp instanceof ModifyColumnOp) { + return ((ModifyColumnOp) alterTableOp).getRollupName(); + } + return null; + } + private void rewriteAlterOpForOlapTable(ConnectContext ctx, OlapTable table) throws UserException { List alterTableOps = new ArrayList<>(); for (AlterTableOp alterClause : ops) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java index 10ced240ee50ed..153030536f48b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java @@ -78,6 +78,10 @@ public Column getColumn() { return column; } + public ColumnDefinition getColumnDef() { + return columnDef; + } + public ColumnPath getColumnPath() { return columnPath; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index c6e7a92104db58..50e640781bb2e3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -209,6 +209,47 @@ void testRejectDefaultMetadataForNestedIcebergModify() throws AnalysisException parseAlter("ALTER TABLE t ADD COLUMN s.b BIGINT NULL DEFAULT 7").getOps()); } + @Test + void testRejectRollupForNestedIcebergColumnOperations() { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN s.c STRING NULL TO r1", + "ALTER TABLE t ADD COLUMN s.c STRING NULL IN r1", + "ALTER TABLE t DROP COLUMN s.c FROM r1", + "ALTER TABLE t MODIFY COLUMN s.c STRING FROM r1")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkNestedColumnPathSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("Rollup is not supported for nested Iceberg column operation")); + } + } + + @Test + void testRejectKeyForNestedIcebergAddAndModify() { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN s.c INT KEY NULL", + "ALTER TABLE t MODIFY COLUMN s.c BIGINT KEY")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkNestedColumnPathSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("KEY is not supported for nested Iceberg ADD/MODIFY COLUMN")); + } + } + + @Test + void testRejectGeneratedColumnForNestedIcebergAddAndModify() { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN s.c INT AS (id + 1) NULL", + "ALTER TABLE t MODIFY COLUMN s.c BIGINT AS (id + 1)")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkNestedColumnPathSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("Generated columns are not supported for nested Iceberg ADD/MODIFY COLUMN")); + } + } + @Test void testModifyColumnTracksExplicitNullability() { ModifyColumnOp omitted = (ModifyColumnOp) parseAlter( From e6911fa64114cbe4df533ff80fd8ba7f110a8014 Mon Sep 17 00:00:00 2001 From: daidai Date: Wed, 15 Jul 2026 14:56:17 +0800 Subject: [PATCH 11/32] [fix](iceberg) Reject unsupported column default metadata ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Top-level Iceberg MODIFY COLUMN accepted DEFAULT and ON UPDATE clauses even though the metadata update only applies type, comment, position, and nullability, so the requested default semantics could be silently discarded. Reject default metadata for every Iceberg MODIFY before ColumnDefinition intent is lost, and enforce the same rule at the Iceberg metadata boundary for programmatic callers. Also reject unsupported ON UPDATE metadata for single and multi-column ADD operations while preserving supported ordinary ADD defaults. ### Release note Iceberg column schema changes now reject unsupported DEFAULT and ON UPDATE metadata instead of ignoring it. ### Check List (For Author) - Test: Unit Test (not run at user request) - Added command validation coverage for top-level and nested MODIFY defaults - Added metadata validation coverage for MODIFY defaults and ADD ON UPDATE - git diff --cached --check - Behavior changed: Yes. Unsupported Iceberg DEFAULT and ON UPDATE clauses now fail before schema mutation. - Does this need documentation: No --- .../iceberg/IcebergMetadataOps.java | 27 ++++++++---- .../plans/commands/AlterTableCommand.java | 19 ++++---- .../IcebergMetadataOpsValidationTest.java | 43 ++++++++++++++++--- .../plans/commands/AlterTableCommandTest.java | 28 +++++++----- 4 files changed, 85 insertions(+), 32 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 759f08d7af4d84..ccae09ad7dc01f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -824,7 +824,7 @@ private void refreshTable(ExternalTable dorisTable, long updateTime) { @Override public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { - validateCommonColumnInfo(column); + validateAddColumnMetadata(column); Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); Schema schema = icebergTable.schema(); validateNoCaseInsensitiveSiblingCollision( @@ -850,7 +850,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co addColumn(dorisTable, column, position, updateTime); return; } - validateCommonColumnInfo(column); + validateAddColumnMetadata(column); if (!column.isAllowNull()) { throw new UserException("New nested field '" + columnPath.getFullPath() + "' must be nullable"); } @@ -885,7 +885,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); for (Column column : columns) { - validateCommonColumnInfo(column); + validateAddColumnMetadata(column); } validateNoCaseInsensitiveTopLevelCollisions(icebergTable.schema(), columns); @@ -991,7 +991,7 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition ResolvedColumnPath columnPath = new ResolvedColumnPath(ColumnPath.of(currentCol.name()), currentCol.type(), currentCol); - validateCommonColumnInfo(column); + validateModifyColumnMetadata(column, columnPath.getFullPath()); UpdateSchema updateSchema = icebergTable.updateSchema(); if (column.getType().isComplexType()) { @@ -1033,10 +1033,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); NestedField currentCol = resolvedPath.getField(); - validateCommonColumnInfo(column); - if (column.hasDefaultValue() || column.hasOnUpdateDefaultValue()) { - throw new UserException("Modifying default values is not supported for Iceberg columns: " + columnPath); - } + validateModifyColumnMetadata(column, resolvedPath.getFullPath()); UpdateSchema updateSchema = icebergTable.updateSchema(); if (column.getType().isComplexType()) { @@ -1438,6 +1435,20 @@ private void validateCommonColumnInfo(Column column) throws UserException { } } + private void validateAddColumnMetadata(Column column) throws UserException { + validateCommonColumnInfo(column); + if (column.hasOnUpdateDefaultValue()) { + throw new UserException("ON UPDATE is not supported for Iceberg ADD COLUMN: " + column.getName()); + } + } + + private void validateModifyColumnMetadata(Column column, String columnPath) throws UserException { + validateCommonColumnInfo(column); + if (column.hasDefaultValue() || column.hasOnUpdateDefaultValue()) { + throw new UserException("Modifying default values is not supported for Iceberg columns: " + columnPath); + } + } + @Override public void reorderColumns(ExternalTable dorisTable, List newOrder, long updateTime) throws UserException { if (newOrder == null || newOrder.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java index 76e45d1dce7d62..e97c4fd322deb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java @@ -128,7 +128,7 @@ private void validate(ConnectContext ctx) throws UserException { if (tableIf.isTemporary()) { throw new AnalysisException("Do not support alter temporary table[" + tableName + "]"); } - checkNestedColumnPathSupported(tableIf, ops); + checkColumnOperationsSupported(tableIf, ops); for (AlterTableOp op : ops) { op.setTableName(tbl); op.validate(ctx); @@ -140,10 +140,18 @@ private void validate(ConnectContext ctx) throws UserException { } } - static void checkNestedColumnPathSupported(TableIf table, List alterTableOps) + static void checkColumnOperationsSupported(TableIf table, List alterTableOps) throws AnalysisException { if (table instanceof IcebergExternalTable) { for (AlterTableOp alterTableOp : alterTableOps) { + ColumnDefinition columnDefinition = getColumnDefinition(alterTableOp); + // Column translation cannot distinguish an omitted default from an explicit DEFAULT NULL. + if (alterTableOp instanceof ModifyColumnOp && columnDefinition != null + && (columnDefinition.hasDefaultValue() + || columnDefinition.hasOnUpdateDefaultValue())) { + throw new AnalysisException( + "DEFAULT and ON UPDATE are not supported for Iceberg MODIFY COLUMN"); + } ColumnPath columnPath = getNestedColumnPath(alterTableOp); if (columnPath == null) { continue; @@ -152,7 +160,6 @@ static void checkNestedColumnPathSupported(TableIf table, List alt throw new AnalysisException("Rollup is not supported for nested Iceberg column operation: " + columnPath.getFullPath()); } - ColumnDefinition columnDefinition = getColumnDefinition(alterTableOp); if (columnDefinition != null && columnDefinition.isKey()) { throw new AnalysisException("KEY is not supported for nested Iceberg ADD/MODIFY COLUMN: " + columnPath.getFullPath()); @@ -161,12 +168,6 @@ static void checkNestedColumnPathSupported(TableIf table, List alt throw new AnalysisException("Generated columns are not supported for nested Iceberg " + "ADD/MODIFY COLUMN: " + columnPath.getFullPath()); } - if (alterTableOp instanceof ModifyColumnOp && columnDefinition != null - && (columnDefinition.hasDefaultValue() - || columnDefinition.hasOnUpdateDefaultValue())) { - throw new AnalysisException("DEFAULT and ON UPDATE are not supported for nested Iceberg " - + "MODIFY COLUMN: " + columnPath.getFullPath()); - } } return; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index dc1d7d325c887e..b592a10959c9c4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -350,25 +350,58 @@ public void testModifyColumnRejectsDefaultMetadata() { Mockito.when(icebergTable.schema()).thenReturn(schema); Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); - Column defaultColumn = new Column("a", Type.BIGINT, false, null, true, "7", ""); - Column onUpdateColumn = Mockito.spy(new Column("a", Type.BIGINT, true)); - Mockito.doReturn(true).when(onUpdateColumn).hasOnUpdateDefaultValue(); + Column topLevelDefaultColumn = new Column("id", Type.BIGINT, false, null, true, "7", ""); + Column nestedDefaultColumn = new Column("a", Type.BIGINT, false, null, true, "7", ""); + Column topLevelOnUpdateColumn = Mockito.spy(new Column("id", Type.BIGINT, true)); + Column nestedOnUpdateColumn = Mockito.spy(new Column("a", Type.BIGINT, true)); + Mockito.doReturn(true).when(topLevelOnUpdateColumn).hasOnUpdateDefaultValue(); + Mockito.doReturn(true).when(nestedOnUpdateColumn).hasOnUpdateDefaultValue(); try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("id"), + topLevelDefaultColumn, null, 1L), + "Modifying default values is not supported for Iceberg columns: id"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("id"), + topLevelOnUpdateColumn, null, 1L), + "Modifying default values is not supported for Iceberg columns: id"); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("s.a"), - defaultColumn, null, 1L), + nestedDefaultColumn, null, 1L), "Modifying default values is not supported for Iceberg columns: s.a"); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("s.a"), - onUpdateColumn, null, 1L), + nestedOnUpdateColumn, null, 1L), "Modifying default values is not supported for Iceberg columns: s.a"); } Mockito.verifyNoInteractions(updateSchema); } + @Test + public void testAddColumnRejectsOnUpdateMetadata() { + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Column onUpdateColumn = Mockito.spy(new Column("new_col", Type.DATETIME, true)); + Mockito.doReturn(true).when(onUpdateColumn).hasOnUpdateDefaultValue(); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.of("new_col"), + onUpdateColumn, null, 1L), + "ON UPDATE is not supported for Iceberg ADD COLUMN: new_col"); + assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("s.new_col"), + onUpdateColumn, null, 1L), + "ON UPDATE is not supported for Iceberg ADD COLUMN: new_col"); + assertUserException(() -> ops.addColumns(dorisTable, Collections.singletonList(onUpdateColumn), 1L), + "ON UPDATE is not supported for Iceberg ADD COLUMN: new_col"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + @Test public void testModifyComplexColumnRejectsCaseInsensitiveStructFieldAdditions() { Schema schema = mixedCaseNestedSchema(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index 50e640781bb2e3..5bd066a378d816 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -176,7 +176,7 @@ void testRejectNestedColumnPathForNonIcebergTable() { "ALTER TABLE t DROP COLUMN s.c", "ALTER TABLE t RENAME COLUMN s.c TO c2")) { AnalysisException exception = Assertions.assertThrows(AnalysisException.class, - () -> AlterTableCommand.checkNestedColumnPathSupported(table, parseAlter(sql).getOps())); + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); Assertions.assertTrue(exception.getMessage() .contains("Nested column path is only supported for Iceberg tables")); } @@ -185,27 +185,35 @@ void testRejectNestedColumnPathForNonIcebergTable() { @Test void testAllowNestedColumnPathForIcebergTable() throws AnalysisException { IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); - AlterTableCommand.checkNestedColumnPathSupported(table, + AlterTableCommand.checkColumnOperationsSupported(table, parseAlter("ALTER TABLE t ADD COLUMN s.c STRING NULL").getOps()); } @Test - void testRejectDefaultMetadataForNestedIcebergModify() throws AnalysisException { + void testRejectDefaultMetadataForIcebergModify() throws AnalysisException { IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); for (String sql : Arrays.asList( + "ALTER TABLE t MODIFY COLUMN a BIGINT DEFAULT 7", + "ALTER TABLE t MODIFY COLUMN a BIGINT DEFAULT NULL", + "ALTER TABLE t MODIFY COLUMN ts DATETIME DEFAULT CURRENT_TIMESTAMP " + + "ON UPDATE CURRENT_TIMESTAMP", "ALTER TABLE t MODIFY COLUMN s.a BIGINT DEFAULT 7", "ALTER TABLE t MODIFY COLUMN s.a BIGINT DEFAULT NULL", "ALTER TABLE t MODIFY COLUMN s.ts DATETIME DEFAULT CURRENT_TIMESTAMP " + "ON UPDATE CURRENT_TIMESTAMP")) { AnalysisException exception = Assertions.assertThrows(AnalysisException.class, - () -> AlterTableCommand.checkNestedColumnPathSupported(table, parseAlter(sql).getOps())); + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); Assertions.assertTrue(exception.getMessage() - .contains("DEFAULT and ON UPDATE are not supported for nested Iceberg MODIFY COLUMN")); + .contains("DEFAULT and ON UPDATE are not supported for Iceberg MODIFY COLUMN")); } - AlterTableCommand.checkNestedColumnPathSupported(table, + AlterTableCommand.checkColumnOperationsSupported(table, + parseAlter("ALTER TABLE t MODIFY COLUMN a BIGINT").getOps()); + AlterTableCommand.checkColumnOperationsSupported(table, parseAlter("ALTER TABLE t MODIFY COLUMN s.a BIGINT").getOps()); - AlterTableCommand.checkNestedColumnPathSupported(table, + AlterTableCommand.checkColumnOperationsSupported(table, + parseAlter("ALTER TABLE t ADD COLUMN b BIGINT NULL DEFAULT 7").getOps()); + AlterTableCommand.checkColumnOperationsSupported(table, parseAlter("ALTER TABLE t ADD COLUMN s.b BIGINT NULL DEFAULT 7").getOps()); } @@ -218,7 +226,7 @@ void testRejectRollupForNestedIcebergColumnOperations() { "ALTER TABLE t DROP COLUMN s.c FROM r1", "ALTER TABLE t MODIFY COLUMN s.c STRING FROM r1")) { AnalysisException exception = Assertions.assertThrows(AnalysisException.class, - () -> AlterTableCommand.checkNestedColumnPathSupported(table, parseAlter(sql).getOps())); + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); Assertions.assertTrue(exception.getMessage() .contains("Rollup is not supported for nested Iceberg column operation")); } @@ -231,7 +239,7 @@ void testRejectKeyForNestedIcebergAddAndModify() { "ALTER TABLE t ADD COLUMN s.c INT KEY NULL", "ALTER TABLE t MODIFY COLUMN s.c BIGINT KEY")) { AnalysisException exception = Assertions.assertThrows(AnalysisException.class, - () -> AlterTableCommand.checkNestedColumnPathSupported(table, parseAlter(sql).getOps())); + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); Assertions.assertTrue(exception.getMessage() .contains("KEY is not supported for nested Iceberg ADD/MODIFY COLUMN")); } @@ -244,7 +252,7 @@ void testRejectGeneratedColumnForNestedIcebergAddAndModify() { "ALTER TABLE t ADD COLUMN s.c INT AS (id + 1) NULL", "ALTER TABLE t MODIFY COLUMN s.c BIGINT AS (id + 1)")) { AnalysisException exception = Assertions.assertThrows(AnalysisException.class, - () -> AlterTableCommand.checkNestedColumnPathSupported(table, parseAlter(sql).getOps())); + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); Assertions.assertTrue(exception.getMessage() .contains("Generated columns are not supported for nested Iceberg ADD/MODIFY COLUMN")); } From 8ef56e75206da02158f369ecd577fb66d9f8a122 Mon Sep 17 00:00:00 2001 From: daidai Date: Wed, 15 Jul 2026 18:19:42 +0800 Subject: [PATCH 12/32] [fix](iceberg) Reject ignored column clauses ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Iceberg column schema changes accepted rollup targets, properties, key markers, and generated-column expressions that external dispatch did not preserve. Reject those unsupported clauses for single, nested, grouped, and reorder operations, and align the nested NOT NULL regression expectation with Doris validation. ### Release note Reject unsupported Iceberg column clauses instead of silently ignoring them. ### Check List (For Author) - Test: Not run per request; static review and git diff --check completed - Behavior changed: Yes, unsupported Iceberg column clauses now return an analysis error - Does this need documentation: No --- .../plans/commands/AlterTableCommand.java | 47 +++++++++++---- .../plans/commands/info/AddColumnsOp.java | 4 ++ .../plans/commands/AlterTableCommandTest.java | 57 ++++++++++++++++--- ...iceberg_nested_schema_evolution_ddl.groovy | 2 +- 4 files changed, 91 insertions(+), 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java index e97c4fd322deb2..c258ba692b1023 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java @@ -152,21 +152,21 @@ static void checkColumnOperationsSupported(TableIf table, List alt throw new AnalysisException( "DEFAULT and ON UPDATE are not supported for Iceberg MODIFY COLUMN"); } - ColumnPath columnPath = getNestedColumnPath(alterTableOp); - if (columnPath == null) { + if (!isIcebergColumnSchemaOperation(alterTableOp)) { continue; } if (getRollupName(alterTableOp) != null) { - throw new AnalysisException("Rollup is not supported for nested Iceberg column operation: " - + columnPath.getFullPath()); + throw new AnalysisException("Rollup is not supported for Iceberg column operations"); } - if (columnDefinition != null && columnDefinition.isKey()) { - throw new AnalysisException("KEY is not supported for nested Iceberg ADD/MODIFY COLUMN: " - + columnPath.getFullPath()); + Map properties = alterTableOp.getProperties(); + if (properties != null && !properties.isEmpty()) { + throw new AnalysisException("PROPERTIES are not supported for Iceberg column operations"); } - if (columnDefinition != null && columnDefinition.getGeneratedColumnDesc().isPresent()) { - throw new AnalysisException("Generated columns are not supported for nested Iceberg " - + "ADD/MODIFY COLUMN: " + columnPath.getFullPath()); + checkIcebergColumnDefinition(columnDefinition); + if (alterTableOp instanceof AddColumnsOp) { + for (ColumnDefinition definition : ((AddColumnsOp) alterTableOp).getColumnDefinitions()) { + checkIcebergColumnDefinition(definition); + } } } return; @@ -206,16 +206,43 @@ private static ColumnDefinition getColumnDefinition(AlterTableOp alterTableOp) { return null; } + private static boolean isIcebergColumnSchemaOperation(AlterTableOp alterTableOp) { + return alterTableOp instanceof AddColumnOp + || alterTableOp instanceof AddColumnsOp + || alterTableOp instanceof DropColumnOp + || alterTableOp instanceof ModifyColumnOp + || alterTableOp instanceof ReorderColumnsOp; + } + + private static void checkIcebergColumnDefinition(ColumnDefinition columnDefinition) + throws AnalysisException { + if (columnDefinition == null) { + return; + } + if (columnDefinition.isKey()) { + throw new AnalysisException("KEY is not supported for Iceberg ADD/MODIFY COLUMN"); + } + if (columnDefinition.getGeneratedColumnDesc().isPresent()) { + throw new AnalysisException("Generated columns are not supported for Iceberg ADD/MODIFY COLUMN"); + } + } + private static String getRollupName(AlterTableOp alterTableOp) { if (alterTableOp instanceof AddColumnOp) { return ((AddColumnOp) alterTableOp).getRollupName(); } + if (alterTableOp instanceof AddColumnsOp) { + return ((AddColumnsOp) alterTableOp).getRollupName(); + } if (alterTableOp instanceof DropColumnOp) { return ((DropColumnOp) alterTableOp).getRollupName(); } if (alterTableOp instanceof ModifyColumnOp) { return ((ModifyColumnOp) alterTableOp).getRollupName(); } + if (alterTableOp instanceof ReorderColumnsOp) { + return ((ReorderColumnsOp) alterTableOp).getRollupName(); + } return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnsOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnsOp.java index 418185ed7557f2..9ff326ffb385de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnsOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnsOp.java @@ -62,6 +62,10 @@ public List getColumns() { return columns; } + public List getColumnDefinitions() { + return columnDefs == null ? Collections.emptyList() : columnDefs; + } + public String getRollupName() { return rollupName; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index 5bd066a378d816..c6c2ced503379a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -23,6 +23,7 @@ import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.info.AddColumnsOp; import org.apache.doris.nereids.trees.plans.commands.info.AddPartitionFieldOp; import org.apache.doris.nereids.trees.plans.commands.info.AlterTableOp; import org.apache.doris.nereids.trees.plans.commands.info.DropPartitionFieldOp; @@ -218,46 +219,86 @@ void testRejectDefaultMetadataForIcebergModify() throws AnalysisException { } @Test - void testRejectRollupForNestedIcebergColumnOperations() { + void testRejectRollupForIcebergColumnOperations() { IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN c STRING NULL TO r1", "ALTER TABLE t ADD COLUMN s.c STRING NULL TO r1", - "ALTER TABLE t ADD COLUMN s.c STRING NULL IN r1", + "ALTER TABLE t ADD COLUMN (c1 STRING NULL, c2 INT NULL) IN r1", + "ALTER TABLE t DROP COLUMN c FROM r1", "ALTER TABLE t DROP COLUMN s.c FROM r1", - "ALTER TABLE t MODIFY COLUMN s.c STRING FROM r1")) { + "ALTER TABLE t MODIFY COLUMN c STRING FROM r1", + "ALTER TABLE t MODIFY COLUMN s.c STRING FROM r1", + "ALTER TABLE t ORDER BY (c1, c2) FROM r1")) { AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); Assertions.assertTrue(exception.getMessage() - .contains("Rollup is not supported for nested Iceberg column operation")); + .contains("Rollup is not supported for Iceberg column operations")); } } @Test - void testRejectKeyForNestedIcebergAddAndModify() { + void testRejectPropertiesForIcebergColumnOperations() { IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN c STRING NULL PROPERTIES ('k' = 'v')", + "ALTER TABLE t ADD COLUMN s.c STRING NULL PROPERTIES ('k' = 'v')", + "ALTER TABLE t ADD COLUMN (c1 STRING NULL, c2 INT NULL) PROPERTIES ('k' = 'v')", + "ALTER TABLE t DROP COLUMN c PROPERTIES ('k' = 'v')", + "ALTER TABLE t DROP COLUMN s.c PROPERTIES ('k' = 'v')", + "ALTER TABLE t MODIFY COLUMN c STRING PROPERTIES ('k' = 'v')", + "ALTER TABLE t MODIFY COLUMN s.c STRING PROPERTIES ('k' = 'v')", + "ALTER TABLE t ORDER BY (c1, c2) PROPERTIES ('k' = 'v')")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("PROPERTIES are not supported for Iceberg column operations")); + } + } + + @Test + void testRejectKeyForIcebergAddAndModify() { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN c INT KEY NULL", "ALTER TABLE t ADD COLUMN s.c INT KEY NULL", + "ALTER TABLE t ADD COLUMN (c1 INT KEY NULL, c2 INT NULL)", + "ALTER TABLE t MODIFY COLUMN c BIGINT KEY", "ALTER TABLE t MODIFY COLUMN s.c BIGINT KEY")) { AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); Assertions.assertTrue(exception.getMessage() - .contains("KEY is not supported for nested Iceberg ADD/MODIFY COLUMN")); + .contains("KEY is not supported for Iceberg ADD/MODIFY COLUMN")); } } @Test - void testRejectGeneratedColumnForNestedIcebergAddAndModify() { + void testRejectGeneratedColumnForIcebergAddAndModify() { IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN c INT AS (id + 1) NULL", "ALTER TABLE t ADD COLUMN s.c INT AS (id + 1) NULL", + "ALTER TABLE t ADD COLUMN (c1 INT AS (id + 1) NULL, c2 INT NULL)", + "ALTER TABLE t MODIFY COLUMN c BIGINT AS (id + 1)", "ALTER TABLE t MODIFY COLUMN s.c BIGINT AS (id + 1)")) { AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); Assertions.assertTrue(exception.getMessage() - .contains("Generated columns are not supported for nested Iceberg ADD/MODIFY COLUMN")); + .contains("Generated columns are not supported for Iceberg ADD/MODIFY COLUMN")); } } + @Test + void testPreserveEmptyAddColumnsValidationForIcebergTable() throws AnalysisException { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + AddColumnsOp addColumnsOp = new AddColumnsOp(null, null, new HashMap<>()); + + AlterTableCommand.checkColumnOperationsSupported(table, Arrays.asList(addColumnsOp)); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> addColumnsOp.validate(null)); + Assertions.assertTrue(exception.getMessage().contains("Columns is empty in add columns clause")); + } + @Test void testModifyColumnTracksExplicitNullability() { ModifyColumnOp omitted = (ModifyColumnOp) parseAlter( diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy index d33b31cafed7aa..ccf3dd16a8e7ac 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy @@ -85,7 +85,7 @@ suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_do test { sql """ALTER TABLE ${tableName} ADD COLUMN s.required_field INT NOT NULL""" - exception "Field 'required_field' doesn't have a default value" + exception "New nested field 's.required_field' must be nullable" } sql """ALTER TABLE ${tableName} MODIFY COLUMN s.a BIGINT""" From 3515e1d72caf34661caab53a1f415797cff6279a Mon Sep 17 00:00:00 2001 From: daidai Date: Wed, 15 Jul 2026 21:19:08 +0800 Subject: [PATCH 13/32] [fix](iceberg) Harden nested schema evolution validation ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: A full review of Iceberg nested schema evolution found several compatibility and validation gaps. The legacy programmatic modify-column API no longer preserved its historical nullable behavior, direct list/map pseudo-fields accepted unsupported position clauses, and new metadata paths could silently ignore KEY or generated-column intent. Doris boolean/date/timestamp defaults were also converted with the wrong semantics, dynamic and complex defaults reached unchecked Iceberg parsing, and mixed-case reorder names were not canonicalized. Preserve legacy behavior only on the legacy API path, reject unsupported metadata and positions before schema mutation, convert constant defaults through Doris literal semantics, and use canonical Iceberg names for reorder operations. Add focused validation coverage and clean generated expected-output endings. ### Release note Harden Iceberg schema evolution validation and preserve legacy modify-column compatibility. ### Check List (For Author) - Test: Unit Test (added but not run at user request); static diff checks completed - Behavior changed: Yes. Unsupported Iceberg column metadata, dynamic or complex defaults, and pseudo-field positions now fail early; constant defaults use Doris semantics; mixed-case reorder names resolve canonically; and legacy nullable behavior is preserved. - Does this need documentation: No --- .../doris/datasource/ExternalCatalog.java | 16 +- .../iceberg/IcebergMetadataOps.java | 140 +++++++---- .../datasource/iceberg/IcebergUtils.java | 46 +++- .../IcebergMetadataOpsValidationTest.java | 220 ++++++++++++++++++ ...st_iceberg_nested_schema_evolution_ddl.out | 1 - ...d_schema_evolution_spark_doris_interop.out | 1 - 6 files changed, 374 insertions(+), 50 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index b30c31355eeeac..341477c9911253 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -1629,7 +1629,21 @@ public void renameColumn(TableIf dorisTable, ColumnPath columnPath, String newNa @Override public void modifyColumn(TableIf dorisTable, Column column, ColumnPosition columnPosition) throws UserException { - modifyColumn(dorisTable, ColumnPath.of(column.getName()), column, columnPosition); + makeSureInitialized(); + Preconditions.checkState(dorisTable instanceof ExternalTable, dorisTable.getName()); + ExternalTable externalTable = (ExternalTable) dorisTable; + if (metadataOps == null) { + throw new DdlException("Modify column operation is not supported for catalog: " + getName()); + } + try { + long updateTime = System.currentTimeMillis(); + metadataOps.modifyColumn(externalTable, column, columnPosition, updateTime); + logRefreshExternalTable(externalTable, updateTime); + } catch (Exception e) { + LOG.warn("Failed to modify column {} in table {}.{} in catalog {}", + column.getName(), externalTable.getDbName(), externalTable.getName(), getName(), e); + throw e; + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index b66a18895d4d89..72d689053eebee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.analysis.ColumnPath; +import org.apache.doris.analysis.DefaultValueExprDef; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.ColumnType; @@ -816,23 +817,11 @@ private void addOneColumn(UpdateSchema updateSchema, Column column) throws UserE throw new UserException("can't add a non-nullable column to an Iceberg table"); } org.apache.iceberg.types.Type dorisType = IcebergUtils.dorisTypeToIcebergType(column.getType()); - Literal defaultValue = IcebergUtils.parseIcebergLiteral(column.getDefaultValue(), dorisType); + Literal defaultValue = IcebergUtils.parseIcebergLiteral( + column.getDefaultValue(), column.getType(), dorisType); updateSchema.addColumn(column.getName(), dorisType, column.getComment(), defaultValue); } - private void applyPosition(UpdateSchema updateSchema, ColumnPosition position, String columnName) { - applyPosition(updateSchema, position, ColumnPath.of(columnName)); - } - - private void applyPosition(UpdateSchema updateSchema, ColumnPosition position, ColumnPath columnPath) { - String columnName = columnPath.getFullPath(); - if (position.isFirst()) { - updateSchema.moveFirst(columnName); - } else { - updateSchema.moveAfter(columnName, getPositionReferencePath(columnPath, position)); - } - } - private void applyPosition(UpdateSchema updateSchema, ColumnPosition position, ColumnPath columnPath, Schema schema, String operation) throws UserException { String columnName = columnPath.getFullPath(); @@ -843,6 +832,18 @@ private void applyPosition(UpdateSchema updateSchema, ColumnPosition position, C } } + private void validatePositionTarget(Schema schema, ColumnPath columnPath, String operation) + throws UserException { + if (!columnPath.isNested()) { + return; + } + ResolvedColumnPath parentPath = resolveColumnPath(schema, columnPath.getParentPath(), operation); + if (!parentPath.getType().isStructType()) { + throw new UserException("Cannot apply column position to '" + columnPath.getFullPath() + + "': parent column path '" + parentPath.getFullPath() + "' is not a struct"); + } + } + @VisibleForTesting String getPositionReferencePath(ColumnPath columnPath, ColumnPosition position) { if (position == null || position.isFirst() || !columnPath.isNested()) { @@ -918,7 +919,8 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co UpdateSchema updateSchema = icebergTable.updateSchema(); org.apache.iceberg.types.Type dorisType = IcebergUtils.dorisTypeToIcebergType(column.getType()); - Literal defaultValue = IcebergUtils.parseIcebergLiteral(column.getDefaultValue(), dorisType); + Literal defaultValue = IcebergUtils.parseIcebergLiteral( + column.getDefaultValue(), column.getType(), dorisType); updateSchema.addColumn(parentPath.getFullPath(), columnPath.getLeafName(), dorisType, column.getComment(), defaultValue); if (position != null) { @@ -1036,6 +1038,11 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String @Override public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { + modifyTopLevelColumn(dorisTable, column, position, updateTime, true); + } + + private void modifyTopLevelColumn(ExternalTable dorisTable, Column column, ColumnPosition position, + long updateTime, boolean legacyMode) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); NestedField currentCol = icebergTable.schema().caseInsensitiveFindField(column.getName()); if (currentCol == null) { @@ -1044,13 +1051,14 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition ResolvedColumnPath columnPath = new ResolvedColumnPath(ColumnPath.of(currentCol.name()), currentCol.type(), currentCol); - validateModifyColumnMetadata(column, columnPath.getFullPath()); + validateModifyColumnMetadata(column, columnPath.getFullPath(), !legacyMode); UpdateSchema updateSchema = icebergTable.updateSchema(); if (column.getType().isComplexType()) { // Complex type processing branch validateForModifyComplexColumn(column, currentCol); - applyComplexTypeChange(updateSchema, columnPath.getFullPath(), currentCol.type(), column.getType()); + applyComplexTypeChange(updateSchema, columnPath.getFullPath(), currentCol.type(), column.getType(), + legacyMode); if (!Objects.equals(currentCol.doc(), column.getComment())) { updateSchema.updateColumnDoc(columnPath.getFullPath(), column.getComment()); } @@ -1060,7 +1068,7 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition Type icebergType = IcebergUtils.dorisTypeToIcebergType(column.getType()); updateSchema.updateColumn(columnPath.getFullPath(), icebergType.asPrimitiveType(), column.getComment()); } - applyExplicitNullableChange(updateSchema, columnPath.getFullPath(), column); + applyExplicitNullableChange(updateSchema, columnPath.getFullPath(), column, legacyMode); if (position != null) { applyPosition(updateSchema, position, columnPath.getColumnPath(), icebergTable.schema(), "modify"); @@ -1078,20 +1086,24 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, long updateTime) throws UserException { if (!columnPath.isNested()) { - modifyColumn(dorisTable, column, position, updateTime); + modifyTopLevelColumn(dorisTable, column, position, updateTime, false); return; } Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); NestedField currentCol = resolvedPath.getField(); + if (position != null) { + validatePositionTarget(icebergTable.schema(), resolvedPath.getColumnPath(), "modify"); + } - validateModifyColumnMetadata(column, resolvedPath.getFullPath()); + validateModifyColumnMetadata(column, resolvedPath.getFullPath(), true); UpdateSchema updateSchema = icebergTable.updateSchema(); if (column.getType().isComplexType()) { validateForModifyComplexColumn(column, currentCol, columnPath.getFullPath()); - applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), column.getType()); + applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), column.getType(), + false); if (!Objects.equals(currentCol.doc(), column.getComment())) { updateSchema.updateColumnDoc(resolvedPath.getFullPath(), column.getComment()); } @@ -1100,7 +1112,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column Type icebergType = IcebergUtils.dorisTypeToIcebergType(column.getType()); updateSchema.updateColumn(resolvedPath.getFullPath(), icebergType.asPrimitiveType(), column.getComment()); } - applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column); + applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column, false); if (position != null) { applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); @@ -1133,8 +1145,9 @@ public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, refreshTable(dorisTable, updateTime); } - private void applyExplicitNullableChange(UpdateSchema updateSchema, String columnPath, Column column) { - if (column.isNullableSpecified() && column.isAllowNull()) { + private void applyExplicitNullableChange(UpdateSchema updateSchema, String columnPath, Column column, + boolean legacyNullableExplicit) { + if ((legacyNullableExplicit || column.isNullableSpecified()) && column.isAllowNull()) { updateSchema.makeColumnOptional(columnPath); } } @@ -1358,16 +1371,19 @@ private boolean isSameComplexCategory(Type oldIcebergType, org.apache.doris.cata private void applyComplexTypeChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldIcebergType, - org.apache.doris.catalog.Type newDorisType) throws UserException { + org.apache.doris.catalog.Type newDorisType, boolean legacyNullableExplicit) throws UserException { switch (oldIcebergType.typeId()) { case STRUCT: - applyStructChange(updateSchema, path, oldIcebergType.asStructType(), (StructType) newDorisType); + applyStructChange(updateSchema, path, oldIcebergType.asStructType(), (StructType) newDorisType, + legacyNullableExplicit); break; case LIST: - applyListChange(updateSchema, path, (Types.ListType) oldIcebergType, (ArrayType) newDorisType); + applyListChange(updateSchema, path, (Types.ListType) oldIcebergType, (ArrayType) newDorisType, + legacyNullableExplicit); break; case MAP: - applyMapChange(updateSchema, path, (Types.MapType) oldIcebergType, (MapType) newDorisType); + applyMapChange(updateSchema, path, (Types.MapType) oldIcebergType, (MapType) newDorisType, + legacyNullableExplicit); break; default: throw new UserException("Unsupported complex type for modify: " + oldIcebergType); @@ -1375,7 +1391,8 @@ private void applyComplexTypeChange(UpdateSchema updateSchema, String path, } private void applyStructChange(UpdateSchema updateSchema, String path, - Types.StructType oldStructType, StructType newStructType) throws UserException { + Types.StructType oldStructType, StructType newStructType, boolean legacyNullableExplicit) + throws UserException { List oldFields = oldStructType.fields(); List newFields = newStructType.getFields(); @@ -1402,11 +1419,15 @@ private void applyStructChange(UpdateSchema updateSchema, String path, newField.getComment()); } } else { - applyComplexTypeChange(updateSchema, fieldPath, oldFieldType, newFieldType); + applyComplexTypeChange(updateSchema, fieldPath, oldFieldType, newFieldType, + legacyNullableExplicit); if (!Objects.equals(oldField.doc(), newField.getComment())) { updateSchema.updateColumnDoc(fieldPath, newField.getComment()); } } + if (legacyNullableExplicit && !oldField.isOptional() && newField.getContainsNull()) { + updateSchema.makeColumnOptional(fieldPath); + } } for (int i = oldFields.size(); i < newFields.size(); i++) { @@ -1421,7 +1442,8 @@ private void applyStructChange(UpdateSchema updateSchema, String path, } private void applyListChange(UpdateSchema updateSchema, String path, - Types.ListType oldListType, ArrayType newArrayType) throws UserException { + Types.ListType oldListType, ArrayType newArrayType, boolean legacyNullableExplicit) + throws UserException { String elementPath = path + "." + oldListType.field(oldListType.elementId()).name(); if (oldListType.isElementOptional() && !newArrayType.getContainsNull()) { throw new UserException("Cannot change nullable column " + elementPath + " to not null"); @@ -1437,12 +1459,17 @@ private void applyListChange(UpdateSchema updateSchema, String path, updateSchema.updateColumn(elementPath, newIcebergElementType.asPrimitiveType(), null); } } else { - applyComplexTypeChange(updateSchema, elementPath, oldElementType, newElementType); + applyComplexTypeChange(updateSchema, elementPath, oldElementType, newElementType, + legacyNullableExplicit); + } + if (legacyNullableExplicit && !oldListType.isElementOptional() && newArrayType.getContainsNull()) { + updateSchema.makeColumnOptional(elementPath); } } private void applyMapChange(UpdateSchema updateSchema, String path, - Types.MapType oldMapType, MapType newMapType) throws UserException { + Types.MapType oldMapType, MapType newMapType, boolean legacyNullableExplicit) + throws UserException { org.apache.iceberg.types.Type oldKeyType = oldMapType.keyType(); org.apache.doris.catalog.Type newKeyType = newMapType.getKeyType(); org.apache.doris.catalog.Type oldDorisKeyType = @@ -1467,11 +1494,21 @@ private void applyMapChange(UpdateSchema updateSchema, String path, updateSchema.updateColumn(valuePath, newIcebergValueType.asPrimitiveType(), null); } } else { - applyComplexTypeChange(updateSchema, valuePath, oldValueType, newValueType); + applyComplexTypeChange(updateSchema, valuePath, oldValueType, newValueType, + legacyNullableExplicit); + } + if (legacyNullableExplicit && !oldMapType.isValueOptional() && newMapType.getIsValueContainsNull()) { + updateSchema.makeColumnOptional(valuePath); } } - private void validateCommonColumnInfo(Column column) throws UserException { + private void validateCommonColumnInfo(Column column, boolean rejectKey) throws UserException { + if (rejectKey && column.isKey()) { + throw new UserException("KEY is not supported for Iceberg ADD/MODIFY COLUMN"); + } + if (column.isGeneratedColumn()) { + throw new UserException("Generated columns are not supported for Iceberg ADD/MODIFY COLUMN"); + } // check aggregation method if (column.isAggregated()) { throw new UserException("Can not specify aggregation method for iceberg table column"); @@ -1489,14 +1526,25 @@ private void validateCommonColumnInfo(Column column) throws UserException { } private void validateAddColumnMetadata(Column column) throws UserException { - validateCommonColumnInfo(column); + validateCommonColumnInfo(column, true); if (column.hasOnUpdateDefaultValue()) { throw new UserException("ON UPDATE is not supported for Iceberg ADD COLUMN: " + column.getName()); } + if (column.getType().isComplexType() && column.getDefaultValue() != null) { + throw new UserException("Complex type default value only supports NULL"); + } + DefaultValueExprDef defaultValueExpr = column.getDefaultValueExprDef(); + if (defaultValueExpr != null + && ("now".equalsIgnoreCase(defaultValueExpr.getExprName()) + || "current_date".equalsIgnoreCase(defaultValueExpr.getExprName()))) { + throw new UserException("Dynamic default value " + column.getDefaultValue() + + " is not supported for Iceberg ADD COLUMN: " + column.getName()); + } } - private void validateModifyColumnMetadata(Column column, String columnPath) throws UserException { - validateCommonColumnInfo(column); + private void validateModifyColumnMetadata(Column column, String columnPath, boolean rejectKey) + throws UserException { + validateCommonColumnInfo(column, rejectKey); if (column.hasDefaultValue() || column.hasOnUpdateDefaultValue()) { throw new UserException("Modifying default values is not supported for Iceberg columns: " + columnPath); } @@ -1508,10 +1556,20 @@ public void reorderColumns(ExternalTable dorisTable, List newOrder, long throw new UserException("Reorder column failed, new order is empty."); } Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + List canonicalOrder = new ArrayList<>(newOrder.size()); + Set canonicalNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (String columnName : newOrder) { + String canonicalName = resolveColumnPath( + icebergTable.schema(), ColumnPath.of(columnName), "reorder").getFullPath(); + if (!canonicalNames.add(canonicalName)) { + throw new UserException("Duplicate column in reorder columns: " + columnName); + } + canonicalOrder.add(canonicalName); + } UpdateSchema updateSchema = icebergTable.updateSchema(); - updateSchema.moveFirst(newOrder.get(0)); - for (int i = 1; i < newOrder.size(); i++) { - updateSchema.moveAfter(newOrder.get(i), newOrder.get(i - 1)); + updateSchema.moveFirst(canonicalOrder.get(0)); + for (int i = 1; i < canonicalOrder.size(); i++) { + updateSchema.moveAfter(canonicalOrder.get(i), canonicalOrder.get(i - 1)); } try { executionAuthenticator.execute(() -> updateSchema.commit()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 12b5dce11395fa..4c3fb1622213ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.CompoundPredicate; import org.apache.doris.analysis.DateLiteral; +import org.apache.doris.analysis.DateLiteralUtils; import org.apache.doris.analysis.DecimalLiteral; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToExprNameVisitor; @@ -950,32 +951,39 @@ public static org.apache.iceberg.types.Type dorisTypeToIcebergType(Type type) { } public static Literal parseIcebergLiteral(String value, org.apache.iceberg.types.Type type) { + return parseIcebergLiteral(value, null, type); + } + + public static Literal parseIcebergLiteral( + String value, Type dorisType, org.apache.iceberg.types.Type type) { if (value == null) { return null; } switch (type.typeId()) { case BOOLEAN: try { - return Literal.of(Boolean.parseBoolean(value)); - } catch (IllegalArgumentException e) { + return Literal.of(new BoolLiteral(value).getValue()); + } catch (AnalysisException e) { throw new IllegalArgumentException("Invalid Boolean string: " + value, e); } case INTEGER: - case DATE: try { return Literal.of(Integer.parseInt(value)); } catch (NumberFormatException e) { throw new IllegalArgumentException("Invalid Int string: " + value, e); } case LONG: - case TIME: - case TIMESTAMP: - case TIMESTAMP_NANO: try { return Literal.of(Long.parseLong(value)); } catch (NumberFormatException e) { throw new IllegalArgumentException("Invalid Long string: " + value, e); } + case DATE: + case TIMESTAMP: + case TIMESTAMP_NANO: + return parseIcebergDateLiteral(value, dorisType, type); + case TIME: + return parseIcebergTemporalLiteral(value, value, type); case FLOAT: try { return Literal.of(Float.parseFloat(value)); @@ -1012,6 +1020,32 @@ public static Literal parseIcebergLiteral(String value, org.apache.iceberg.ty } } + private static Literal parseIcebergDateLiteral( + String value, Type dorisType, org.apache.iceberg.types.Type type) { + String canonicalValue = value; + if (dorisType != null) { + try { + canonicalValue = DateLiteralUtils.createDateLiteral(value, dorisType).getStringValue(); + } catch (AnalysisException | IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid date or timestamp string: " + value, e); + } + } + return parseIcebergTemporalLiteral(value, canonicalValue.replace(' ', 'T'), type); + } + + private static Literal parseIcebergTemporalLiteral( + String originalValue, String canonicalValue, org.apache.iceberg.types.Type type) { + try { + Literal literal = Literal.of(canonicalValue).to(type); + if (literal == null) { + throw new IllegalArgumentException("Cannot convert value to Iceberg type " + type); + } + return literal; + } catch (RuntimeException e) { + throw new IllegalArgumentException("Invalid temporal string: " + originalValue, e); + } + } + /** * Convert human-readable partition value string to appropriate Java type for * Iceberg expression. diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index b592a10959c9c4..518ce50cad5431 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.analysis.ColumnPath; +import org.apache.doris.analysis.DefaultValueExprDef; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MapType; @@ -36,6 +37,7 @@ import org.apache.iceberg.UpdateSchema; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; import org.junit.Assert; @@ -46,6 +48,9 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Collections; import java.util.Optional; @@ -309,6 +314,62 @@ public void testTopLevelModifyPreservesRequiredMixedCaseFields() throws Throwabl Mockito.verify(updateSchema, Mockito.times(2)).commit(); } + @Test + public void testLegacyModifyColumnTreatsNullabilityAsExplicit() throws Throwable { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + // Iceberg schema columns are represented as keys in Doris, so the legacy API must not + // interpret isKey as an explicit KEY clause. + ops.modifyColumn(dorisTable, + new Column("id", Type.BIGINT, true, null, true, null, ""), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("id", Types.LongType.get(), ""); + Mockito.verify(updateSchema).makeColumnOptional("id"); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testLegacyComplexModifyRestoresRecursiveNullableSemantics() throws Throwable { + Schema schema = requiredNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + Column column = new Column("info", new StructType( + new StructField("metric", Type.INT), + new StructField("child", new StructType(new StructField("value", Type.INT))), + new StructField("events", ArrayType.create(Type.INT, true)), + new StructField("attrs", new MapType(Type.STRING, Type.INT))), true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, column, null, 1L); + } + + Mockito.verify(updateSchema).makeColumnOptional("info"); + Mockito.verify(updateSchema).makeColumnOptional("info.metric"); + Mockito.verify(updateSchema).makeColumnOptional("info.child.value"); + Mockito.verify(updateSchema).makeColumnOptional("info.events.element"); + Mockito.verify(updateSchema).makeColumnOptional("info.attrs.value"); + Mockito.verify(updateSchema).commit(); + } + @Test public void testExplicitNullableModifyMakesRequiredFieldsOptional() throws Throwable { Schema schema = requiredNestedSchema(); @@ -402,6 +463,113 @@ public void testAddColumnRejectsOnUpdateMetadata() { Mockito.verify(icebergTable, Mockito.never()).updateSchema(); } + @Test + public void testAddColumnRejectsUnsupportedDefaultsBeforeUpdateSchema() { + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Column currentTimestamp = Mockito.spy(new Column("event_time", Type.DATETIME, false, null, true, + "CURRENT_TIMESTAMP", "")); + Column currentDate = Mockito.spy(new Column("event_date", Type.DATEV2, false, null, true, + "CURRENT_DATE", "")); + Column complexDefault = new Column("items", ArrayType.create(Type.INT, true), false, null, true, + "[]", ""); + Mockito.doReturn(new DefaultValueExprDef("now")) + .when(currentTimestamp).getDefaultValueExprDef(); + Mockito.doReturn(new DefaultValueExprDef("current_date")) + .when(currentDate).getDefaultValueExprDef(); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.addColumn(dorisTable, currentTimestamp, null, 1L), + "Dynamic default value CURRENT_TIMESTAMP is not supported for Iceberg ADD COLUMN: event_time"); + assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("s.event_date"), + currentDate, null, 1L), + "Dynamic default value CURRENT_DATE is not supported for Iceberg ADD COLUMN: event_date"); + assertUserException(() -> ops.addColumn(dorisTable, complexDefault, null, 1L), + "Complex type default value only supports NULL"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testRejectKeyAndGeneratedMetadataBeforeUpdateSchema() { + Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + Column keyColumn = new Column("id", Type.BIGINT, true, null, true, null, ""); + Column generatedColumn = Mockito.mock(Column.class); + Mockito.when(generatedColumn.getName()).thenReturn("id"); + Mockito.when(generatedColumn.isGeneratedColumn()).thenReturn(true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.addColumn(dorisTable, keyColumn, null, 1L), + "KEY is not supported for Iceberg ADD/MODIFY COLUMN"); + assertUserException(() -> ops.addColumns(dorisTable, Collections.singletonList(keyColumn), 1L), + "KEY is not supported for Iceberg ADD/MODIFY COLUMN"); + assertUserException(() -> ops.modifyColumn( + dorisTable, ColumnPath.of("id"), keyColumn, null, 1L), + "KEY is not supported for Iceberg ADD/MODIFY COLUMN"); + assertUserException(() -> ops.addColumns(dorisTable, + Collections.singletonList(generatedColumn), 1L), + "Generated columns are not supported for Iceberg ADD/MODIFY COLUMN"); + assertUserException(() -> ops.modifyColumn( + dorisTable, ColumnPath.of("id"), generatedColumn, null, 1L), + "Generated columns are not supported for Iceberg ADD/MODIFY COLUMN"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testNestedAddColumnConvertsDorisDefaultsToIcebergLiterals() throws Throwable { + Schema schema = new Schema(Types.NestedField.optional(1, "s", Types.StructType.of( + Types.NestedField.optional(2, "existing", Types.IntegerType.get())))); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + Column flag = new Column("flag", Type.BOOLEAN, false, null, true, "1", ""); + Column eventDate = new Column("event_date", Type.DATEV2, false, null, true, + "2024-01-02", ""); + Column eventTime = new Column("event_time", ScalarType.createDatetimeV2Type(6), false, null, true, + "2024-01-02 03:04:05.123456", ""); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.addColumn(dorisTable, ColumnPath.fromDotName("s.flag"), flag, null, 1L); + ops.addColumn(dorisTable, ColumnPath.fromDotName("s.event_date"), eventDate, null, 1L); + ops.addColumn(dorisTable, ColumnPath.fromDotName("s.event_time"), eventTime, null, 1L); + } + + int expectedDate = Math.toIntExact(LocalDate.of(2024, 1, 2).toEpochDay()); + long expectedTimestamp = ChronoUnit.MICROS.between( + LocalDateTime.of(1970, 1, 1, 0, 0), + LocalDateTime.of(2024, 1, 2, 3, 4, 5, 123456000)); + Mockito.verify(updateSchema).addColumn(Mockito.eq("s"), Mockito.eq("flag"), + Mockito.eq(Types.BooleanType.get()), Mockito.eq(""), + Mockito.>argThat(literal -> Boolean.TRUE.equals(literal.value()))); + Mockito.verify(updateSchema).addColumn(Mockito.eq("s"), Mockito.eq("event_date"), + Mockito.eq(Types.DateType.get()), Mockito.eq(""), + Mockito.>argThat(literal -> Integer.valueOf(expectedDate).equals(literal.value()))); + Mockito.verify(updateSchema).addColumn(Mockito.eq("s"), Mockito.eq("event_time"), + Mockito.eq(Types.TimestampType.withoutZone()), Mockito.eq(""), + Mockito.>argThat(literal -> Long.valueOf(expectedTimestamp).equals(literal.value()))); + Mockito.verify(updateSchema, Mockito.times(3)).commit(); + } + @Test public void testModifyComplexColumnRejectsCaseInsensitiveStructFieldAdditions() { Schema schema = mixedCaseNestedSchema(); @@ -527,6 +695,30 @@ public void testTopLevelCaseInsensitiveCollisionsAndCaseOnlyRename() throws Thro Mockito.verify(updateSchema).commit(); } + @Test + public void testReorderColumnsUsesCanonicalIcebergNames() throws Throwable { + Schema schema = new Schema( + Types.NestedField.optional(1, "Id", Types.IntegerType.get()), + Types.NestedField.optional(2, "Label", Types.StringType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.reorderColumns(dorisTable, Arrays.asList("label", "id"), 1L); + } + + Mockito.verify(updateSchema).moveFirst("Label"); + Mockito.verify(updateSchema).moveAfter("Id", "Label"); + Mockito.verify(updateSchema).commit(); + } + @Test public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throwable { Schema schema = primitiveContainerSchema(); @@ -553,6 +745,34 @@ public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throw Mockito.verify(updateSchema, Mockito.times(2)).commit(); } + @Test + public void testModifyColumnRejectsPositionForDirectArrayElementAndMapValue() { + Schema schema = primitiveContainerSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), + new Column("element", Type.BIGINT, true), ColumnPosition.FIRST, 1L), + "Cannot apply column position to 'arr.element': parent column path 'arr' is not a struct"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), + new Column("element", Type.BIGINT, true), new ColumnPosition("element"), 1L), + "Cannot apply column position to 'arr.element': parent column path 'arr' is not a struct"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("m.value"), + new Column("value", Type.BIGINT, true), ColumnPosition.FIRST, 1L), + "Cannot apply column position to 'm.value': parent column path 'm' is not a struct"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("m.value"), + new Column("value", Type.BIGINT, true), new ColumnPosition("value"), 1L), + "Cannot apply column position to 'm.value': parent column path 'm' is not a struct"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + @Test public void testModifyColumnCommentUsesCanonicalNestedPaths() throws Throwable { Schema schema = mixedCaseNestedSchema(); diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out index 1249ff6cf2b19c..93826ea80b7925 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out @@ -10,4 +10,3 @@ m_scalar map -- !query_rows -- 1 \N 10 \N \N 100 \N \N 1000 \N \N 7 70 2 first 20 after_a c2 200 202 201 2000 2002 2001 8 80 - diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out index 0119135c2ee878..6aef898cfc7950 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out @@ -17,4 +17,3 @@ attrs map> 1 \N 10 \N spark_before \N \N 100 \N \N \N 1000 \N \N 2 spark_first_field 20 spark_after_metric_field spark_after spark_new_field 202 200 203 201 2002 2000 2003 2001 3 doris_first_field 30 doris_after_metric_field doris_after_spark doris_can_write_spark_field 302 300 303 301 3002 3000 3003 3001 - From 9c2496abf95d116e951d55e678c008e1dc5fe2a8 Mon Sep 17 00:00:00 2001 From: daidai Date: Thu, 16 Jul 2026 01:29:16 +0800 Subject: [PATCH 14/32] [fix](iceberg) Fix schema evolution validation failures ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: PR #65329 was blocked by deterministic TeamCity failures. Required nested Iceberg fields were rejected by the generic no-default validation before the dedicated Iceberg invariant could report its intended error. Existing Iceberg regression expectations also predated the PR's explicit default-metadata and unsupported-type validation. Separately, the table-level compaction policy test placed CREATE, sync, and the expected-failure ALTER in one TestAction, which retained only the final SQL and produced Unknown table. Run the Iceberg nested-field check before generic column validation, align the affected expectations with the current validation contract, and create the compaction table outside the expected-failure action. ### Release note None ### Check List (For Author) - Test: Unit Test and Regression Test - Unit Test: ./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.commands.AlterTableCommandTest - Regression Test: test_table_level_compaction_policy, test_iceberg_nested_schema_evolution_ddl, test_iceberg_schema_change_complex_types - Build: ./build.sh --fe - Behavior changed: Yes. Required nested Iceberg fields now return the dedicated nullable-field validation error before the generic no-default error; accepted operations are unchanged. - Does this need documentation: No --- .../plans/commands/AlterTableCommand.java | 8 +++++ .../plans/commands/AlterTableCommandTest.java | 10 +++++++ .../test_table_level_compaction_policy.groovy | 30 +++++++++---------- ...iceberg_schema_change_complex_types.groovy | 8 ++--- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java index c258ba692b1023..d938ccf9aaed89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java @@ -145,6 +145,14 @@ static void checkColumnOperationsSupported(TableIf table, List alt if (table instanceof IcebergExternalTable) { for (AlterTableOp alterTableOp : alterTableOps) { ColumnDefinition columnDefinition = getColumnDefinition(alterTableOp); + ColumnPath nestedColumnPath = getNestedColumnPath(alterTableOp); + // Keep this before AddColumnOp.validate(), whose generic NOT NULL check would mask + // the Iceberg nested-field invariant. Metadata validation retains the same guard for other callers. + if (alterTableOp instanceof AddColumnOp && columnDefinition != null && nestedColumnPath != null + && !columnDefinition.isNullable()) { + throw new AnalysisException("New nested field '" + nestedColumnPath.getFullPath() + + "' must be nullable"); + } // Column translation cannot distinguish an omitted default from an explicit DEFAULT NULL. if (alterTableOp instanceof ModifyColumnOp && columnDefinition != null && (columnDefinition.hasDefaultValue() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index c6c2ced503379a..b5ca2e25f1458d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -190,6 +190,16 @@ void testAllowNestedColumnPathForIcebergTable() throws AnalysisException { parseAlter("ALTER TABLE t ADD COLUMN s.c STRING NULL").getOps()); } + @Test + void testRejectRequiredNestedColumnForIcebergTable() { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, + parseAlter("ALTER TABLE t ADD COLUMN s.required_field INT NOT NULL").getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("New nested field 's.required_field' must be nullable")); + } + @Test void testRejectDefaultMetadataForIcebergModify() throws AnalysisException { IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); diff --git a/regression-test/suites/compaction/test_table_level_compaction_policy.groovy b/regression-test/suites/compaction/test_table_level_compaction_policy.groovy index 5edff4e38e0da5..0e1c70b02c9eea 100644 --- a/regression-test/suites/compaction/test_table_level_compaction_policy.groovy +++ b/regression-test/suites/compaction/test_table_level_compaction_policy.groovy @@ -219,22 +219,22 @@ suite("test_table_level_compaction_policy") { exception "only time series compaction policy support for time series config" } - test { - sql """ - CREATE TABLE ${tableName} ( - `c_custkey` int(11) NOT NULL COMMENT "", - `c_name` varchar(26) NOT NULL COMMENT "", - `c_address` varchar(41) NOT NULL COMMENT "", - `c_city` varchar(11) NOT NULL COMMENT "" - ) - DUPLICATE KEY (`c_custkey`) - DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 - PROPERTIES ( - "replication_num" = "1" - ); - """ - sql """sync""" + sql """ + CREATE TABLE ${tableName} ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "" + ) + DUPLICATE KEY (`c_custkey`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """sync""" + test { sql """ alter table ${tableName} set ("compaction_policy" = "ok") """ diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy index 567f3298adf291..b887c4de3e99ff 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy @@ -77,7 +77,7 @@ suite("test_iceberg_schema_change_complex_types", "p0,external,doris,external_do STRUCT(1, CAST(1.1 AS FLOAT), 'abc', MAP(1, 10), ARRAY(1, 2)) )""" - // Complex type default value only supports NULL + // Iceberg MODIFY COLUMN does not support default metadata test { sql """ ALTER TABLE ${table_name} MODIFY COLUMN st STRUCT< @@ -87,7 +87,7 @@ suite("test_iceberg_schema_change_complex_types", "p0,external,doris,external_do sm: MAP COMMENT 'm1', sa: ARRAY COMMENT 'a1' > DEFAULT 'x'""" - exception "just support null" + exception "DEFAULT and ON UPDATE are not supported for Iceberg MODIFY COLUMN" } // Cannot change nullable complex column to not null @@ -133,10 +133,10 @@ suite("test_iceberg_schema_change_complex_types", "p0,external,doris,external_do exception "Cannot reduce struct fields" } - // Nested type downgrade is not allowed + // Iceberg does not support SMALLINT, including nested types test { sql """ALTER TABLE ${table_name} MODIFY COLUMN arr_i ARRAY""" - exception "Cannot change int to smallint in nested types" + exception "Type array is not supported for Iceberg column arr_i" } // Array element type promotions From 7383b6b0b1a6c4bb9f7b56d101a5bba8745b0ea2 Mon Sep 17 00:00:00 2001 From: daidai Date: Thu, 16 Jul 2026 18:15:54 +0800 Subject: [PATCH 15/32] [fix](iceberg) Preserve nested schema alter semantics ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: ALTER schema SQL rendering did not preserve omitted nullability or quoted comments across parse and replay. Nested Iceberg fields also inherited Doris top-level hidden and shadow prefix restrictions, while Iceberg v3 row-lineage names still needed protection in the top-level namespace. Preserve explicit nullability intent, use a SQL-mode-aware string literal codec, scope Doris system-prefix checks to top-level columns, and reject Iceberg v3 row-lineage mutations at the metadata boundary. ### Release note None ### Check List (For Author) - Test: Unit Test and Build - Unit Test: ./run-fe-ut.sh --run org.apache.doris.nereids.parser.IcebergNestedSchemaEvolutionParserTest,org.apache.doris.nereids.trees.plans.commands.AlterTableCommandTest,org.apache.doris.datasource.iceberg.IcebergMetadataOpsValidationTest - Unit Test: ./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.commands.AlterTableCommandTest - Build: ./build.sh --fe - Behavior changed: Yes. Nested Iceberg fields may use names that overlap Doris top-level system prefixes, while Iceberg v3 top-level row-lineage columns remain immutable and ALTER SQL round-trips preserve nullability and comments. - Does this need documentation: No --- .../org/apache/doris/common/FeNameFormat.java | 9 +- .../iceberg/IcebergMetadataOps.java | 20 +++++ .../nereids/parser/LogicalPlanBuilder.java | 15 +--- .../parser/LogicalPlanBuilderAssistant.java | 23 +++++ .../plans/commands/info/AddColumnOp.java | 13 ++- .../plans/commands/info/ColumnDefinition.java | 28 +++++-- .../plans/commands/info/DropColumnOp.java | 2 +- .../commands/info/ModifyColumnCommentOp.java | 3 +- .../plans/commands/info/ModifyColumnOp.java | 6 +- .../plans/commands/info/RenameColumnOp.java | 8 +- .../IcebergMetadataOpsValidationTest.java | 83 +++++++++++++++++++ ...cebergNestedSchemaEvolutionParserTest.java | 60 ++++++++++++++ .../plans/commands/AlterTableCommandTest.java | 63 ++++++++++++++ 13 files changed, 306 insertions(+), 27 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java index 0cf43bb1757cca..9e40042d32d754 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java @@ -107,11 +107,18 @@ public static void checkColumnName(String columnName) throws AnalysisException { } public static void checkColumnNameBypassHiddenColumn(String columnName) throws AnalysisException { + checkColumnNameBypassSystemColumnPrefix(columnName); + checkColumnNamePrefix(columnName, Column.SHADOW_NAME_PREFIX); + } + + /** + * Check column name syntax without applying Doris top-level hidden/shadow column prefix rules. + */ + public static void checkColumnNameBypassSystemColumnPrefix(String columnName) throws AnalysisException { if (Strings.isNullOrEmpty(columnName) || !columnName.matches(getColumnNameRegex())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, columnName, getColumnNameRegex()); } - checkColumnNamePrefix(columnName, Column.SHADOW_NAME_PREFIX); } private static void checkColumnNamePrefix(String columnName, String prefix) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 72d689053eebee..f09d2c2c49e110 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -880,6 +880,7 @@ public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition po throws UserException { validateAddColumnMetadata(column); Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); Schema schema = icebergTable.schema(); validateNoCaseInsensitiveSiblingCollision( schema.asStruct(), "", column.getName(), null, "add"); @@ -941,6 +942,7 @@ public void addColumns(ExternalTable dorisTable, List columns, long upda Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); for (Column column : columns) { validateAddColumnMetadata(column); + validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); } validateNoCaseInsensitiveTopLevelCollisions(icebergTable.schema(), columns); @@ -960,6 +962,7 @@ public void addColumns(ExternalTable dorisTable, List columns, long upda @Override public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + validateRowLineageColumnMutation(icebergTable, columnName, "drop"); ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(columnName), "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); updateSchema.deleteColumn(columnPath.getFullPath()); @@ -996,6 +999,8 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + validateRowLineageColumnMutation(icebergTable, oldName, "rename"); + validateRowLineageColumnMutation(icebergTable, newName, "rename to"); Schema schema = icebergTable.schema(); ResolvedColumnPath oldPath = resolveColumnPath(schema, ColumnPath.of(oldName), "rename"); validateNoCaseInsensitiveSiblingCollision( @@ -1044,6 +1049,7 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition private void modifyTopLevelColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime, boolean legacyMode) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + validateRowLineageColumnMutation(icebergTable, column.getName(), "modify"); NestedField currentCol = icebergTable.schema().caseInsensitiveFindField(column.getName()); if (currentCol == null) { throw new UserException("Column " + column.getName() + " does not exist"); @@ -1131,6 +1137,9 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + if (!columnPath.isNested()) { + validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify comment for"); + } ResolvedColumnPath resolvedPath = resolveColumnPath( icebergTable.schema(), columnPath, "modify comment"); @@ -1550,6 +1559,16 @@ private void validateModifyColumnMetadata(Column column, String columnPath, bool } } + private void validateRowLineageColumnMutation(Table icebergTable, String columnName, String operation) + throws UserException { + int formatVersion = IcebergUtils.getFormatVersion(icebergTable); + if (formatVersion >= IcebergUtils.ICEBERG_ROW_LINEAGE_MIN_VERSION + && IcebergUtils.isIcebergRowLineageColumn(columnName)) { + throw new UserException("Cannot " + operation + " Iceberg v" + formatVersion + + " reserved row lineage column: " + columnName); + } + } + @Override public void reorderColumns(ExternalTable dorisTable, List newOrder, long updateTime) throws UserException { if (newOrder == null || newOrder.isEmpty()) { @@ -1559,6 +1578,7 @@ public void reorderColumns(ExternalTable dorisTable, List newOrder, long List canonicalOrder = new ArrayList<>(newOrder.size()); Set canonicalNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (String columnName : newOrder) { + validateRowLineageColumnMutation(icebergTable, columnName, "reorder"); String canonicalName = resolveColumnPath( icebergTable.schema(), ColumnPath.of(columnName), "reorder").getFullPath(); if (!canonicalNames.add(canonicalName)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 74254cc2c5860d..a817bc4e658dbe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -3740,18 +3740,7 @@ public Literal visitIntegerLiteral(IntegerLiteralContext ctx) { @Override public Literal visitStringLiteral(StringLiteralContext ctx) { - String txt = ctx.STRING_LITERAL().getText(); - String s = txt.substring(1, txt.length() - 1); - if (txt.charAt(0) == '\'') { - // for single quote string, '' should be converted to ' - s = s.replace("''", "'"); - } else if (txt.charAt(0) == '"') { - // for double quote string, "" should be converted to " - s = s.replace("\"\"", "\""); - } - if (!SqlModeHelper.hasNoBackSlashEscapes()) { - s = LogicalPlanBuilderAssistant.escapeBackSlash(s); - } + String s = LogicalPlanBuilderAssistant.parseStringLiteral(ctx.STRING_LITERAL().getText()); int strLength = Utils.containChinese(s) ? s.length() * StringLikeLiteral.CHINESE_CHAR_BYTE_LENGTH : s.length(); if (strLength > ScalarType.MAX_VARCHAR_LENGTH) { return new StringLiteral(s); @@ -6585,7 +6574,7 @@ public AlterTableOp visitModifyTableCommentClause(ModifyTableCommentClauseContex @Override public AlterTableOp visitModifyColumnCommentClause(ModifyColumnCommentClauseContext ctx) { ColumnPath columnPath = ColumnPath.of(visitQualifiedName(ctx.name)); - String comment = stripQuotes(ctx.STRING_LITERAL().getText()); + String comment = LogicalPlanBuilderAssistant.parseStringLiteral(ctx.STRING_LITERAL().getText()); return new ModifyColumnCommentOp(columnPath, comment); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java index 7806201a75dbbd..a2dde0054dc648 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.SqlModeHelper; import com.google.common.collect.ImmutableSet; @@ -83,6 +84,28 @@ public static String escapeBackSlash(String str) { return sb.toString(); } + /** + * Decode a STRING_LITERAL token according to the current SQL mode. + */ + public static String parseStringLiteral(String text) { + String value = text.substring(1, text.length() - 1); + if (text.charAt(0) == '\'') { + value = value.replace("''", "'"); + } else { + value = value.replace("\"\"", "\""); + } + return SqlModeHelper.hasNoBackSlashEscapes() ? value : escapeBackSlash(value); + } + + /** + * Quote a value as a STRING_LITERAL that can be parsed under the current SQL mode. + */ + public static String quoteStringLiteral(String value) { + String escaped = SqlModeHelper.hasNoBackSlashEscapes() + ? value : value.replace("\\", "\\\\"); + return "\"" + escaped.replace("\"", "\"\"") + "\""; + } + /** * MySQL will trim any inviable char and blank at the beginning of the string, * and stop at the first \0 if it exists. we follow the behavior. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java index 153030536f48b2..2e6c5ab3727820 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java @@ -103,7 +103,7 @@ public void validate(ConnectContext ctx) throws UserException { if (colPos != null) { colPos.analyze(); } - validateColumnDef(tableName, columnDef, colPos, rollupName); + validateColumnDef(tableName, columnDef, colPos, rollupName, columnPath.isNested()); column = columnDef.translateToCatalogStyleForSchemaChange(); } @@ -151,6 +151,11 @@ public String toString() { public static void validateColumnDef(TableNameInfo tableName, ColumnDefinition columnDef, ColumnPosition colPos, String rollupName) throws UserException { + validateColumnDef(tableName, columnDef, colPos, rollupName, false); + } + + private static void validateColumnDef(TableNameInfo tableName, ColumnDefinition columnDef, ColumnPosition colPos, + String rollupName, boolean nestedColumn) throws UserException { if (columnDef == null) { throw new AnalysisException("No column definition in add column clause."); } @@ -201,7 +206,11 @@ public static void validateColumnDef(TableNameInfo tableName, ColumnDefinition c } } - columnDef.validate(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType); + if (nestedColumn) { + columnDef.validateNestedColumn(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType); + } else { + columnDef.validate(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType); + } if (!columnDef.isNullable() && !columnDef.hasDefaultValue()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DEFAULT_FOR_FIELD, columnDef.getName()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index a0a2cc66e4f067..2a8eec3731e84f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -232,11 +232,12 @@ public String toSql(String columnNameSql) { sb.append(aggType.name()).append(" "); } - if (!isNullable) { - sb.append("NOT NULL "); - } else { - // should append NULL to make result can be executed right. - sb.append("NULL "); + if (nullableSpecified) { + if (!isNullable) { + sb.append("NOT NULL "); + } else { + sb.append("NULL "); + } } if (autoIncInitValue != -1) { @@ -323,10 +324,25 @@ private void checkKeyColumnType(boolean isOlap) { */ public void validate(boolean isOlap, Set keysSet, Set clusterKeySet, boolean isEnableMergeOnWrite, KeysType keysType) { + validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, false); + } + + /** + * Validate a nested field whose name is scoped by its parent path rather than the Doris top-level column namespace. + */ + public void validateNestedColumn(boolean isOlap, Set keysSet, Set clusterKeySet, + boolean isEnableMergeOnWrite, KeysType keysType) { + validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, true); + } + + private void validateInternal(boolean isOlap, Set keysSet, Set clusterKeySet, + boolean isEnableMergeOnWrite, KeysType keysType, boolean nestedColumn) { try { // if enableAddHiddenColumn is true, can add hidden column. // So does not check if the column name starts with __DORIS_ - if (enableAddHiddenColumn) { + if (nestedColumn) { + FeNameFormat.checkColumnNameBypassSystemColumnPrefix(name); + } else if (enableAddHiddenColumn) { FeNameFormat.checkColumnNameBypassHiddenColumn(name); } else { FeNameFormat.checkColumnName(name); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java index 3068e4858de291..f9a3444cbe0394 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java @@ -81,7 +81,7 @@ public void validate(ConnectContext ctx) throws UserException { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, colName, FeNameFormat.getColumnNameRegex()); } - if (colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) { + if (!columnPath.isNested() && colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) { throw new AnalysisException("Do not support drop hidden column"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java index c8be93d4259485..6dfeb28475d303 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.ColumnPath; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; +import org.apache.doris.nereids.parser.LogicalPlanBuilderAssistant; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; @@ -89,7 +90,7 @@ public boolean allowOpRowBinlog() { public String toSql() { StringBuilder sb = new StringBuilder(); sb.append("MODIFY COLUMN ").append(columnPath.toSql()); - sb.append(" COMMENT '").append(comment).append("'"); + sb.append(" COMMENT ").append(LogicalPlanBuilderAssistant.quoteStringLiteral(comment)); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnOp.java index 94b6c5986e698e..f5acfbed9df9cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnOp.java @@ -137,7 +137,11 @@ public void validate(ConnectContext ctx) throws UserException { } } } - columnDef.validate(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType); + if (columnPath.isNested()) { + columnDef.validateNestedColumn(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType); + } else { + columnDef.validate(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType); + } if (colPos != null) { colPos.analyze(); if (olapTable != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java index 2a1e7a65ab133a..15270360535d85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java @@ -71,11 +71,15 @@ public void validate(ConnectContext ctx) throws UserException { throw new AnalysisException("New column name is not set"); } - if (colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) { + if (!columnPath.isNested() && colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) { throw new AnalysisException("Do not support rename hidden column"); } - FeNameFormat.checkColumnName(newColName); + if (columnPath.isNested()) { + FeNameFormat.checkColumnNameBypassSystemColumnPrefix(newColName); + } else { + FeNameFormat.checkColumnName(newColName); + } } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 518ce50cad5431..df52230badf405 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -829,6 +829,89 @@ public void testModifyColumnCommentSupportsDirectArrayElementAndMapValue() throw Mockito.verify(updateSchema, Mockito.times(2)).commit(); } + @Test + public void testRejectsTopLevelRowLineageMutationsForV3Tables() { + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.properties()).thenReturn(Collections.singletonMap("format-version", "3")); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.addColumn(dorisTable, + new Column("_row_id", Type.BIGINT, true), null, 1L), + "Cannot add Iceberg v3 reserved row lineage column: _row_id"); + assertUserException(() -> ops.addColumns(dorisTable, Collections.singletonList( + new Column("_last_updated_sequence_number", Type.BIGINT, true)), 1L), + "Cannot add Iceberg v3 reserved row lineage column: _last_updated_sequence_number"); + assertUserException(() -> ops.dropColumn(dorisTable, "_ROW_ID", 1L), + "Cannot drop Iceberg v3 reserved row lineage column: _ROW_ID"); + assertUserException(() -> ops.renameColumn(dorisTable, "_row_id", "renamed", 1L), + "Cannot rename Iceberg v3 reserved row lineage column: _row_id"); + assertUserException(() -> ops.renameColumn( + dorisTable, "id", "_last_updated_sequence_number", 1L), + "Cannot rename to Iceberg v3 reserved row lineage column: _last_updated_sequence_number"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("_row_id"), + new Column("_row_id", Type.BIGINT, true), null, 1L), + "Cannot modify Iceberg v3 reserved row lineage column: _row_id"); + assertUserException(() -> ops.modifyColumnComment(dorisTable, + ColumnPath.of("_last_updated_sequence_number"), "comment", 1L), + "Cannot modify comment for Iceberg v3 reserved row lineage column: " + + "_last_updated_sequence_number"); + assertUserException(() -> ops.reorderColumns(dorisTable, + Arrays.asList("_row_id", "id"), 1L), + "Cannot reorder Iceberg v3 reserved row lineage column: _row_id"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testAllowsV3NestedAndV2TopLevelRowLineageNames() throws Throwable { + Schema v3Schema = new Schema( + Types.NestedField.optional(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "s", Types.StructType.of( + Types.NestedField.optional(3, "source", Types.LongType.get()), + Types.NestedField.optional(4, "_row_id", Types.LongType.get())))); + ExternalTable v3DorisTable = Mockito.mock(ExternalTable.class); + Table v3IcebergTable = Mockito.mock(Table.class); + UpdateSchema v3UpdateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(v3DorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(v3IcebergTable.properties()).thenReturn(Collections.singletonMap("format-version", "3")); + Mockito.when(v3IcebergTable.schema()).thenReturn(v3Schema); + Mockito.when(v3IcebergTable.updateSchema()).thenReturn(v3UpdateSchema); + + Schema v2Schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); + ExternalTable v2DorisTable = Mockito.mock(ExternalTable.class); + Table v2IcebergTable = Mockito.mock(Table.class); + UpdateSchema v2UpdateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(v2DorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(v2IcebergTable.properties()).thenReturn(Collections.singletonMap("format-version", "2")); + Mockito.when(v2IcebergTable.schema()).thenReturn(v2Schema); + Mockito.when(v2IcebergTable.updateSchema()).thenReturn(v2UpdateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(v3DorisTable)).thenReturn(v3IcebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(v2DorisTable)).thenReturn(v2IcebergTable); + + ops.addColumn(v3DorisTable, ColumnPath.fromDotName("s._last_updated_sequence_number"), + new Column("_last_updated_sequence_number", Type.BIGINT, true), null, 1L); + ops.renameColumn(v3DorisTable, ColumnPath.fromDotName("s.source"), "_last_updated_sequence_number", 1L); + ops.modifyColumn(v3DorisTable, ColumnPath.fromDotName("s._row_id"), + new Column("_row_id", Type.BIGINT, true), null, 1L); + ops.modifyColumnComment(v3DorisTable, ColumnPath.fromDotName("s._row_id"), "comment", 1L); + ops.dropColumn(v3DorisTable, ColumnPath.fromDotName("s._row_id"), 1L); + + ops.addColumn(v2DorisTable, new Column("_row_id", Type.BIGINT, true), null, 1L); + ops.renameColumn(v2DorisTable, "id", "_last_updated_sequence_number", 1L); + } + + Mockito.verify(v3UpdateSchema, Mockito.times(5)).commit(); + Mockito.verify(v2UpdateSchema, Mockito.times(2)).commit(); + } + @Test public void testResolveNestedColumnPathRejectsMapKey() { assertUserException(() -> ops.resolveNestedColumnPath(nestedSchema(), ColumnPath.fromDotName("m.key.x"), diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index de3cd3fb31d20c..5de81383fa4709 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.parser; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand; import org.apache.doris.nereids.trees.plans.commands.info.AddColumnOp; @@ -25,9 +26,12 @@ import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnCommentOp; import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnOp; import org.apache.doris.nereids.trees.plans.commands.info.RenameColumnOp; +import org.apache.doris.qe.SqlModeHelper; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import java.util.Collections; import java.util.List; @@ -116,6 +120,62 @@ public void testQuotedNestedIdentifiersAreNormalized() { Assertions.assertEquals("New`Metric", reparsedRename.getNewColName()); } + @Test + public void testModifyColumnRoundTripPreservesNullabilityIntent() { + ModifyColumnOp omitted = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.metric BIGINT", + ModifyColumnOp.class, "info.metric"); + Assertions.assertFalse(omitted.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + Assertions.assertFalse(omitted.toSql().contains(" BIGINT NULL ")); + + ModifyColumnOp reparsedOmitted = assertSingleClausePath( + "ALTER TABLE t " + omitted.toSql(), ModifyColumnOp.class, "info.metric"); + Assertions.assertFalse(reparsedOmitted.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + + ModifyColumnOp nullable = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.metric BIGINT NULL", + ModifyColumnOp.class, "info.metric"); + ModifyColumnOp reparsedNullable = assertSingleClausePath( + "ALTER TABLE t " + nullable.toSql(), ModifyColumnOp.class, "info.metric"); + Assertions.assertTrue(reparsedNullable.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + } + + @Test + public void testModifyColumnCommentRoundTripEscapesQuotesAndBackslashes() { + assertCommentRoundTrip(false); + assertCommentRoundTrip(true); + } + + private void assertCommentRoundTrip(boolean noBackslashEscapes) { + try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { + mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(noBackslashEscapes); + + String expectedComment = "owner's \"field\" C:\\tmp\\"; + ModifyColumnCommentOp comment = new ModifyColumnCommentOp( + ColumnPath.fromDotName("info.metric"), expectedComment); + String renderedSql = comment.toSql(); + Assertions.assertTrue(renderedSql.contains("\"\"field\"\"")); + Assertions.assertTrue(renderedSql.contains(noBackslashEscapes + ? "C:\\tmp\\" : "C:\\\\tmp\\\\")); + + ModifyColumnCommentOp reparsed = assertSingleClausePath( + "ALTER TABLE t " + renderedSql, ModifyColumnCommentOp.class, "info.metric"); + Assertions.assertEquals(expectedComment, reparsed.getComment()); + + ModifyColumnCommentOp doubledSingleQuote = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.metric COMMENT 'owner''s'", + ModifyColumnCommentOp.class, "info.metric"); + Assertions.assertEquals("owner's", doubledSingleQuote.getComment()); + ModifyColumnCommentOp doubledDoubleQuote = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.metric COMMENT \"a\"\"b\"", + ModifyColumnCommentOp.class, "info.metric"); + Assertions.assertEquals("a\"b", doubledDoubleQuote.getComment()); + } + } + private T assertSingleClausePath(String sql, Class clauseClass, String expectedPath) { Plan plan = parser.parseSingle(sql); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index b5ca2e25f1458d..facb5e340b272e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -17,9 +17,13 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.info.TableNameInfo; import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.Plan; @@ -33,6 +37,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; import org.mockito.Mockito; import java.util.ArrayList; @@ -326,6 +331,64 @@ void testModifyColumnTracksExplicitNullability() { .translateToCatalogStyleForSchemaChange().isNullableSpecified()); } + @Test + void testNestedIcebergColumnNamesBypassTopLevelSystemPrefixes() throws Exception { + Env env = Mockito.mock(Env.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(catalogMgr.getCatalogOrDdlException("iceberg")).thenReturn(catalog); + Mockito.when(catalog.getDbOrDdlException("db")).thenReturn(database); + Mockito.when(database.getTableOrDdlException("t")).thenReturn(table); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN s.__DORIS_metric INT NULL", + "ALTER TABLE t ADD COLUMN s.__doris_shadow_metric INT NULL", + "ALTER TABLE t MODIFY COLUMN s.__DORIS_metric BIGINT", + "ALTER TABLE t MODIFY COLUMN s.__doris_shadow_metric BIGINT", + "ALTER TABLE t RENAME COLUMN s.__DORIS_metric TO metric", + "ALTER TABLE t RENAME COLUMN s.metric TO __doris_shadow_metric", + "ALTER TABLE t ADD COLUMN s._row_id BIGINT NULL", + "ALTER TABLE t MODIFY COLUMN s._row_id BIGINT", + "ALTER TABLE t RENAME COLUMN s.metric TO _last_updated_sequence_number", + "ALTER TABLE t DROP COLUMN s.__DORIS_metric")) { + validateIcebergAlter(sql, table); + } + + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN __DORIS_metric INT NULL", + "ALTER TABLE t ADD COLUMN __doris_shadow_metric INT NULL", + "ALTER TABLE t MODIFY COLUMN __DORIS_metric BIGINT")) { + org.apache.doris.nereids.exceptions.AnalysisException exception = Assertions.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> validateIcebergAlter(sql, table)); + Assertions.assertTrue(exception.getMessage().contains("Incorrect column name")); + } + + AnalysisException renameException = Assertions.assertThrows(AnalysisException.class, + () -> validateIcebergAlter( + "ALTER TABLE t RENAME COLUMN metric TO __DORIS_metric", table)); + Assertions.assertTrue(renameException.getMessage().contains("Incorrect column name")); + AnalysisException dropException = Assertions.assertThrows(AnalysisException.class, + () -> validateIcebergAlter("ALTER TABLE t DROP COLUMN __DORIS_metric", table)); + Assertions.assertTrue(dropException.getMessage().contains("Do not support drop hidden column")); + } + } + + private void validateIcebergAlter(String sql, IcebergExternalTable table) throws Exception { + AlterTableCommand command = parseAlter(sql); + AlterTableCommand.checkColumnOperationsSupported(table, command.getOps()); + for (AlterTableOp op : command.getOps()) { + op.setTableName(new TableNameInfo("iceberg", "db", "t")); + op.validate(null); + } + } + private AlterTableCommand parseAlter(String sql) { Plan plan = parser.parseSingle(sql); Assertions.assertInstanceOf(AlterTableCommand.class, plan); From 1805aadfeaff068b9b94a55fc432c4cf3b8e6035 Mon Sep 17 00:00:00 2001 From: daidai Date: Fri, 17 Jul 2026 14:18:43 +0800 Subject: [PATCH 16/32] [fix](iceberg) Address nested schema evolution review issues ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Iceberg nested initial defaults were only preserved for top-level fields, so older files could materialize added nested children as NULL in both legacy and V2 readers. Path-aware DEFAULT and COMMENT literals also bypassed SQL-mode-aware decoding, and unzoned TIMESTAMPTZ literals were interpreted as UTC instead of the session time zone. This change recursively transports initial-default metadata, materializes it only for physically missing nested fields, shares lossless binary default parsing across readers, and uses the existing literal and TIMESTAMPTZ parsers for correct semantics. ### Release note Fix Iceberg nested schema evolution defaults, literal decoding, and TIMESTAMPTZ time-zone handling. ### Check List (For Author) - Test: Unit Test - FE targeted tests: 33 tests passed - BE modified production sources compiled successfully; full BE UT build was stopped before linking and test execution at user request - Behavior changed: Yes, physically missing Iceberg nested fields now use their initial defaults and DDL literals preserve SQL/session semantics - Does this need documentation: No --- be/src/exec/scan/access_path_parser.cpp | 3 + be/src/format/orc/vorc_reader.cpp | 13 +- .../format/parquet/vparquet_column_reader.cpp | 10 +- .../format/table/iceberg_initial_default.cpp | 59 ++++ be/src/format/table/iceberg_initial_default.h | 32 ++ be/src/format/table/iceberg_reader_mixin.h | 40 +-- .../table/table_schema_change_helper.cpp | 46 ++- .../format/table/table_schema_change_helper.h | 25 +- be/src/format_v2/column_mapper.cpp | 1 + be/src/format_v2/table/iceberg_reader.cpp | 33 +- be/src/format_v2/table_reader.cpp | 21 ++ be/src/format_v2/table_reader.h | 53 ++- be/test/exec/scan/access_path_parser_test.cpp | 29 ++ .../table/iceberg/iceberg_reader_test.cpp | 321 +++++++++++++++++- .../table/table_schema_change_helper_test.cpp | 73 ++++ be/test/format_v2/column_mapper_test.cpp | 6 + be/test/format_v2/table_reader_test.cpp | 52 ++- .../java/org/apache/doris/catalog/Column.java | 4 + .../datasource/iceberg/IcebergUtils.java | 31 +- .../nereids/parser/LogicalPlanBuilder.java | 7 +- .../doris/datasource/ExternalUtilTest.java | 25 +- .../datasource/iceberg/IcebergUtilsTest.java | 69 ++++ ...cebergNestedSchemaEvolutionParserTest.java | 22 ++ 23 files changed, 859 insertions(+), 116 deletions(-) create mode 100644 be/src/format/table/iceberg_initial_default.cpp create mode 100644 be/src/format/table/iceberg_initial_default.h diff --git a/be/src/exec/scan/access_path_parser.cpp b/be/src/exec/scan/access_path_parser.cpp index b215212b6d861b..fdf6c3e3dfad63 100644 --- a/be/src/exec/scan/access_path_parser.cpp +++ b/be/src/exec/scan/access_path_parser.cpp @@ -87,6 +87,9 @@ void inherit_schema_metadata(format::ColumnDefinition* column, return; } column->name_mapping = schema_column->name_mapping; + column->default_expr = schema_column->default_expr; + column->initial_default_value = schema_column->initial_default_value; + column->initial_default_value_is_base64 = schema_column->initial_default_value_is_base64; } const format::ColumnDefinition* find_schema_child_by_path( diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 28ef930057f3a2..f31ae922c0d031 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -2151,8 +2151,17 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name, col_name); } auto mutable_field = IColumn::mutate(std::move(doris_field)); - reinterpret_cast(mutable_field.get()) - ->insert_many_defaults(num_values); + const auto& table_column_name = doris_struct_type->get_name_by_position(missing_field); + ColumnPtr initial_default; + if (!root_node->children_column_exists(table_column_name)) { + initial_default = root_node->children_initial_default(table_column_name); + } + if (initial_default) { + mutable_field->insert_many_from(*initial_default, 0, num_values); + } else { + reinterpret_cast(mutable_field.get()) + ->insert_many_defaults(num_values); + } doris_field = std::move(mutable_field); } diff --git a/be/src/format/parquet/vparquet_column_reader.cpp b/be/src/format/parquet/vparquet_column_reader.cpp index 2726044c193b4c..1bface37fb8126 100644 --- a/be/src/format/parquet/vparquet_column_reader.cpp +++ b/be/src/format/parquet/vparquet_column_reader.cpp @@ -993,8 +993,14 @@ Status StructColumnReader::read_column_data( DCHECK(doris_type->is_nullable()); doris_field = IColumn::mutate(std::move(doris_field)); auto mutable_column = doris_field->assert_mutable(); - auto* nullable_column = static_cast(mutable_column.get()); - nullable_column->insert_many_defaults(missing_column_sz); + const auto& initial_default = + root_node->children_initial_default(doris_struct_type->get_element_name(idx)); + if (initial_default) { + mutable_column->insert_many_from(*initial_default, 0, missing_column_sz); + } else { + auto* nullable_column = static_cast(mutable_column.get()); + nullable_column->insert_many_defaults(missing_column_sz); + } } if (null_map_ptr != nullptr) { diff --git a/be/src/format/table/iceberg_initial_default.cpp b/be/src/format/table/iceberg_initial_default.cpp new file mode 100644 index 00000000000000..fc1dd615e08bb5 --- /dev/null +++ b/be/src/format/table/iceberg_initial_default.cpp @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/table/iceberg_initial_default.h" + +#include + +#include "common/consts.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/primitive_type.h" +#include "core/field.h" +#include "util/url_coding.h" + +namespace doris::iceberg { + +Status parse_initial_default(const DataTypePtr& type, std::string_view encoded_value, + bool is_base64, std::string_view field_name, ColumnPtr* result) { + DORIS_CHECK(type != nullptr); + DORIS_CHECK(result != nullptr); + const auto nested_type = remove_nullable(type); + const auto primitive_type = nested_type->get_primitive_type(); + Field value; + if (is_base64 || primitive_type == TYPE_VARBINARY) { + std::string decoded_value; + if (!base64_decode(std::string(encoded_value), &decoded_value)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + field_name); + } + if (primitive_type == TYPE_VARBINARY) { + value = Field::create_field(StringView(decoded_value)); + } else { + DORIS_CHECK(is_string_type(primitive_type)); + value = Field::create_field(decoded_value); + } + } else { + RETURN_IF_ERROR( + nested_type->get_serde()->from_fe_string(std::string(encoded_value), value)); + } + auto column = type->create_column(); + column->insert(value); + *result = std::move(column); + return Status::OK(); +} + +} // namespace doris::iceberg diff --git a/be/src/format/table/iceberg_initial_default.h b/be/src/format/table/iceberg_initial_default.h new file mode 100644 index 00000000000000..ef3e6ec2c249ee --- /dev/null +++ b/be/src/format/table/iceberg_initial_default.h @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/column/column.h" +#include "core/data_type/data_type.h" + +namespace doris::iceberg { + +// Returns a one-row column so variable-length values own their decoded bytes after this call. +Status parse_initial_default(const DataTypePtr& type, std::string_view encoded_value, + bool is_base64, std::string_view field_name, ColumnPtr* result); + +} // namespace doris::iceberg diff --git a/be/src/format/table/iceberg_reader_mixin.h b/be/src/format/table/iceberg_reader_mixin.h index aee211aee019cd..dd0e9240c4eea9 100644 --- a/be/src/format/table/iceberg_reader_mixin.h +++ b/be/src/format/table/iceberg_reader_mixin.h @@ -40,11 +40,11 @@ #include "format/generic_reader.h" #include "format/table/equality_delete.h" #include "format/table/iceberg_delete_file_reader_helper.h" +#include "format/table/iceberg_initial_default.h" #include "format/table/table_schema_change_helper.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "storage/olap_common.h" -#include "util/url_coding.h" namespace doris { class TIcebergDeleteFileDesc; @@ -703,33 +703,19 @@ Status IcebergReaderMixin::_register_missing_equality_delete_column( "Missing Iceberg schema metadata for equality-delete field id {}", field_id); } - Field value; + ColumnPtr value; if (table_field->__isset.initial_default_value) { - const auto nested_type = remove_nullable(delete_key_type); const bool default_is_base64 = (table_field->__isset.initial_default_value_is_base64 && table_field->initial_default_value_is_base64) || (table_field->__isset.type && thrift_to_type(table_field->type.type) == TYPE_VARBINARY); - if (default_is_base64) { - std::string decoded_default; - if (!base64_decode(table_field->initial_default_value, &decoded_default)) { - return Status::InvalidArgument( - "Invalid Base64 Iceberg initial default for field {}", table_field->name); - } - if (nested_type->get_primitive_type() == TYPE_VARBINARY) { - value = Field::create_field(StringView(decoded_default)); - } else { - DORIS_CHECK(is_string_type(nested_type->get_primitive_type())); - value = Field::create_field(decoded_default); - } - } else { - RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string( - table_field->initial_default_value, value)); - } + RETURN_IF_ERROR(doris::iceberg::parse_initial_default( + delete_key_type, table_field->initial_default_value, default_is_base64, + table_field->name, &value)); + } else { + value = delete_key_type->create_column_const(1, Field()); } - const bool inserted = _missing_equality_delete_values - .emplace(name, delete_key_type->create_column_const(1, value)) - .second; + const bool inserted = _missing_equality_delete_values.emplace(name, std::move(value)).second; DORIS_CHECK(inserted); this->register_synthesized_column_handler( name, [this, name](Block* block, size_t rows) -> Status { @@ -743,6 +729,10 @@ Status IcebergReaderMixin::_register_missing_equality_delete_column( template Status IcebergReaderMixin::_materialize_missing_equality_delete_column( Block* block, const std::string& name, const ColumnPtr& value, size_t rows) { + const auto one_row_value = value->convert_to_full_column_if_const(); + auto materialized_value = one_row_value->clone_empty(); + materialized_value->reserve(rows); + materialized_value->insert_many_from(*one_row_value, 0, rows); if (!this->col_name_to_block_idx_ref()->contains(name)) { // ORC must not register a key that is absent from the file as a physical child. In that // case the reader block has no slot for the synthesized key, so append one here before @@ -752,8 +742,7 @@ Status IcebergReaderMixin::_materialize_missing_equality_delete_colu [&](const ColumnWithTypeAndName& col) { return col.name == name; }); DORIS_CHECK(expand_col != _expand_columns.end()); (*this->col_name_to_block_idx_ref())[name] = block->columns(); - block->insert({value->clone_resized(rows)->convert_to_full_column_if_const(), - expand_col->type, name}); + block->insert({std::move(materialized_value), expand_col->type, name}); return Status::OK(); } const auto position = this->col_name_to_block_idx_ref()->at(name); @@ -762,8 +751,7 @@ Status IcebergReaderMixin::_materialize_missing_equality_delete_colu // MultiEqualityDelete hashes each key column directly. Materialize the repeated default so // every key has the batch row count; a ColumnConst keeps only one nested value and therefore // cannot participate in the row-wise multi-column hash contract. - block->get_by_position(position).column = - value->clone_resized(rows)->convert_to_full_column_if_const(); + block->get_by_position(position).column = std::move(materialized_value); return Status::OK(); } diff --git a/be/src/format/table/table_schema_change_helper.cpp b/be/src/format/table/table_schema_change_helper.cpp index 3d4cb6b5a6c99c..caca698c96c4eb 100644 --- a/be/src/format/table/table_schema_change_helper.cpp +++ b/be/src/format/table/table_schema_change_helper.cpp @@ -25,9 +25,11 @@ #include "common/status.h" #include "core/block/block.h" #include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_struct.h" #include "format/generic_reader.h" +#include "format/table/iceberg_initial_default.h" #include "util/string_util.h" namespace doris { @@ -78,6 +80,38 @@ bool find_file_field_idx_by_name_mapping( return table_field.__isset.name && try_match(table_field.name); } +Status parse_iceberg_initial_default(const schema::external::TField& field, + ColumnPtr* initial_default) { + DORIS_CHECK(initial_default != nullptr); + initial_default->reset(); + if (!field.__isset.initial_default_value) { + return Status::OK(); + } + DORIS_CHECK(field.__isset.type); + const auto primitive_type = thrift_to_type(field.type.type); + DORIS_CHECK(!is_complex_type(primitive_type)); + const auto type = DataTypeFactory::instance().create_data_type( + primitive_type, field.__isset.is_optional && field.is_optional, + field.type.__isset.precision ? field.type.precision : 0, + field.type.__isset.scale ? field.type.scale : 0, + field.type.__isset.len ? field.type.len : -1); + const bool is_base64 = + field.__isset.initial_default_value_is_base64 && field.initial_default_value_is_base64; + RETURN_IF_ERROR(doris::iceberg::parse_initial_default(type, field.initial_default_value, + is_base64, field.name, initial_default)); + return Status::OK(); +} + +Status add_missing_iceberg_child( + const std::shared_ptr& struct_node, + const schema::external::TField& field) { + DORIS_CHECK(struct_node != nullptr); + ColumnPtr initial_default; + RETURN_IF_ERROR(parse_iceberg_initial_default(field, &initial_default)); + struct_node->add_not_exist_children(field.name, std::move(initial_default)); + return Status::OK(); +} + } // namespace Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name( @@ -448,7 +482,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id( struct_node->add_children(table_column_name, parquet_fields_schema[file_column_idx].name, field_node); } else { - struct_node->add_not_exist_children(table_column_name); + RETURN_IF_ERROR(add_missing_iceberg_child(struct_node, *table_field.field_ptr)); } } @@ -534,7 +568,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id( exist_field_id)); struct_node->add_children(table_column_name, file_field.name, field_node); } else { - struct_node->add_not_exist_children(table_column_name); + RETURN_IF_ERROR(add_missing_iceberg_child(struct_node, *table_field.field_ptr)); } } node = struct_node; @@ -586,7 +620,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam } if (!matched) { - struct_node->add_not_exist_children(table_column_name); + RETURN_IF_ERROR(add_missing_iceberg_child(struct_node, *table_field.field_ptr)); continue; } @@ -690,7 +724,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam } if (!matched) { - struct_node->add_not_exist_children(table_column_name); + RETURN_IF_ERROR(add_missing_iceberg_child(struct_node, *table_field.field_ptr)); continue; } @@ -740,7 +774,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id( struct_node->add_children(table_column_name, orc_root->getFieldName(file_field_idx), field_node); } else { - struct_node->add_not_exist_children(table_column_name); + RETURN_IF_ERROR(add_missing_iceberg_child(struct_node, *table_field.field_ptr)); } } node = struct_node; @@ -857,7 +891,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma } if (!matched) { - struct_node->add_not_exist_children(table_column_name); + RETURN_IF_ERROR(add_missing_iceberg_child(struct_node, *table_field.field_ptr)); continue; } diff --git a/be/src/format/table/table_schema_change_helper.h b/be/src/format/table/table_schema_change_helper.h index 2d54b3216cac89..008ff3a1c6f6a6 100644 --- a/be/src/format/table/table_schema_change_helper.h +++ b/be/src/format/table/table_schema_change_helper.h @@ -67,6 +67,11 @@ class TableSchemaChangeHelper { "children_column_exists should not be called on base TableInfoNode"); } + virtual const ColumnPtr& children_initial_default(std::string table_column_name) const { + throw std::logic_error( + "children_initial_default should not be called on base TableInfoNode"); + } + virtual std::shared_ptr get_element_node() const { throw std::logic_error("get_element_node should not be called on base TableInfoNode"); } @@ -78,7 +83,8 @@ class TableSchemaChangeHelper { throw std::logic_error("get_value_node should not be called on base TableInfoNode"); } - virtual void add_not_exist_children(std::string table_column_name) { + virtual void add_not_exist_children(std::string table_column_name, + ColumnPtr initial_default = nullptr) { throw std::logic_error( "add_not_exist_children should not be called on base TableInfoNode"); }; @@ -131,9 +137,10 @@ class TableSchemaChangeHelper { const std::shared_ptr node; const std::string column_name; const bool exists; + const ColumnPtr initial_default; }; - // table column name -> { node, file_column_name, exists_in_file} + // table column name -> {node, file_column_name, exists_in_file, initial_default} std::map children; public: @@ -167,14 +174,22 @@ class TableSchemaChangeHelper { return children.at(table_column_name).exists; } - void add_not_exist_children(std::string table_column_name) override { - children.emplace(table_column_name, StructChild {nullptr, "", false}); + const ColumnPtr& children_initial_default(std::string table_column_name) const override { + DCHECK(children.contains(table_column_name)); + DCHECK(!children_column_exists(table_column_name)); + return children.at(table_column_name).initial_default; + } + + void add_not_exist_children(std::string table_column_name, + ColumnPtr initial_default = nullptr) override { + children.emplace(table_column_name, + StructChild {nullptr, "", false, std::move(initial_default)}); } void add_children(std::string table_column_name, std::string file_column_name, std::shared_ptr children_node) override { children.emplace(table_column_name, - StructChild {children_node, file_column_name, true}); + StructChild {children_node, file_column_name, true, nullptr}); } const std::map& get_children() const { return children; } diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 6a0daf903f01d4..54e5f0c2afcfc0 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -2115,6 +2115,7 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c child_mapping.table_type = table_child.type; child_mapping.file_type = table_child.type; child_mapping.filter_conversion = FilterConversionType::FINALIZE_ONLY; + child_mapping.default_expr = table_child.default_expr; mapping->child_mappings.push_back(std::move(child_mapping)); continue; } diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index b817a6d1eadbf5..92598af378d4d7 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -37,6 +37,7 @@ #include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "format/table/deletion_vector_reader.h" +#include "format/table/iceberg_initial_default.h" #include "format_v2/expr/cast.h" #include "format_v2/expr/equality_delete_predicate.h" #include "format_v2/orc/orc_reader.h" @@ -44,7 +45,6 @@ #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/table_reader.h" #include "io/file_factory.h" -#include "util/url_coding.h" namespace doris::format::iceberg { @@ -91,31 +91,14 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit return Status::OK(); } - Field initial_default; - if (table_field.initial_default_value_is_base64 || - table_field.type->get_primitive_type() == TYPE_VARBINARY) { - // New FE versions mark every Iceberg UUID/BINARY/FIXED default as Base64 regardless of its - // Doris mapping. Keep the VARBINARY fallback for scan descriptors produced before that - // marker existed. Decode before parsing so STRING/CHAR and VARBINARY all compare against - // the raw bytes stored in equality-delete files. - std::string decoded_default; - if (!base64_decode(*table_field.initial_default_value, &decoded_default)) { - return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", - table_field.name); - } - if (table_field.type->get_primitive_type() == TYPE_VARBINARY) { - initial_default = Field::create_field(StringView(decoded_default)); - } else { - DORIS_CHECK(is_string_type(table_field.type->get_primitive_type())); - initial_default = Field::create_field(decoded_default); - } - } else { - // An added field's initial default is its logical value in every older data file that lacks - // the physical column. FE normalizes the string for the current Doris table type. - RETURN_IF_ERROR(table_field.type->get_serde()->from_fe_string( - *table_field.initial_default_value, initial_default)); - } + ColumnPtr initial_default_column; + RETURN_IF_ERROR(doris::iceberg::parse_initial_default( + table_field.type, *table_field.initial_default_value, + table_field.initial_default_value_is_base64, table_field.name, + &initial_default_column)); + Field initial_default; + initial_default_column->get(0, initial_default); auto literal = VLiteral::create_shared(table_field.type, initial_default); if (table_field.type->equals(*delete_key_type)) { *key_expr = std::move(literal); diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 5f37538c1f2639..6d1780c9dcc5e7 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -38,9 +38,11 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type/primitive_type.h" #include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "format/table/deletion_vector_reader.h" #include "format/table/iceberg_delete_file_reader_helper.h" +#include "format/table/iceberg_initial_default.h" #include "format/table/paimon_reader.h" #include "format_v2/column_mapper.h" #include "format_v2/delimited_text/csv_reader.h" @@ -279,6 +281,24 @@ ColumnDefinition build_schema_column_from_external_field(const schema::external: return column; } +Status attach_initial_default_exprs(ColumnDefinition* column) { + DORIS_CHECK(column != nullptr); + if (column->initial_default_value.has_value()) { + ColumnPtr initial_default_column; + RETURN_IF_ERROR(doris::iceberg::parse_initial_default( + column->type, *column->initial_default_value, + column->initial_default_value_is_base64, column->name, &initial_default_column)); + Field initial_default; + initial_default_column->get(0, initial_default); + column->default_expr = + VExprContext::create_shared(VLiteral::create_shared(column->type, initial_default)); + } + for (auto& child : column->children) { + RETURN_IF_ERROR(attach_initial_default_exprs(&child)); + } + return Status::OK(); +} + const schema::external::TField* find_external_root_field(const TFileScanRangeParams* params, const ColumnDefinition& column) { if (params == nullptr || !params->__isset.history_schema_info || @@ -486,6 +506,7 @@ Status TableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info return Status::OK(); } context->schema_column = build_schema_column_from_external_field(*schema_field, column->type); + RETURN_IF_ERROR(attach_initial_default_exprs(&*context->schema_column)); column->identifier = context->schema_column->identifier; column->name_mapping = context->schema_column->name_mapping; return Status::OK(); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 73bade81bc4c65..8cb755ba4bdfc0 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -1296,6 +1296,28 @@ class TableReader { return column.get(); } + Status _materialize_missing_child_mapping_column(const ColumnMapping& mapping, + const size_t rows, ColumnPtr* column) { + DORIS_CHECK(mapping.table_type != nullptr); + DORIS_CHECK(column != nullptr); + if (mapping.default_expr == nullptr) { + *column = + _detach_column(mapping.table_type->create_column_const_with_default_value(rows) + ->convert_to_full_column_if_const()); + return Status::OK(); + } + Block eval_block; + eval_block.insert({mapping.table_type->create_column_const_with_default_value(rows), + mapping.table_type, "__table_reader_nested_default_rows"}); + ColumnWithTypeAndName result; + RETURN_IF_ERROR(_execute_default_expr_without_root_type_check(mapping.default_expr, + &eval_block, &result)); + ColumnPtr result_column = result.column; + RETURN_IF_ERROR(_align_column_nullability(&result_column, mapping.table_type)); + *column = _detach_column(std::move(result_column)); + return Status::OK(); + } + Status _materialize_struct_mapping_column(const ColumnMapping& mapping, const ColumnPtr& file_column, const size_t rows, ColumnPtr* column) { @@ -1318,9 +1340,10 @@ class TableReader { for (const auto* child_mapping : table_ordered_children) { DORIS_CHECK(child_mapping != nullptr); if (!child_mapping->file_local_id.has_value()) { - child_columns.push_back( - child_mapping->table_type->create_column_const_with_default_value(rows) - ->convert_to_full_column_if_const()); + ColumnPtr child_column; + RETURN_IF_ERROR(_materialize_missing_child_mapping_column(*child_mapping, rows, + &child_column)); + child_columns.push_back(std::move(child_column)); continue; } const auto file_child_idx = @@ -1440,17 +1463,25 @@ class TableReader { return Status::OK(); } + Status _open_mapping_expr(const ColumnMapping& mapping, RowDescriptor& row_desc) { + if (mapping.projection != nullptr) { + RETURN_IF_ERROR(mapping.projection->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(mapping.projection->open(_runtime_state)); + } + if (mapping.default_expr != nullptr) { + RETURN_IF_ERROR(mapping.default_expr->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(mapping.default_expr->open(_runtime_state)); + } + for (const auto& child_mapping : mapping.child_mappings) { + RETURN_IF_ERROR(_open_mapping_expr(child_mapping, row_desc)); + } + return Status::OK(); + } + Status _open_mapping_exprs() { RowDescriptor row_desc; for (const auto& mapping : _data_reader.column_mapper->mappings()) { - if (mapping.projection != nullptr) { - RETURN_IF_ERROR(mapping.projection->prepare(_runtime_state, row_desc)); - RETURN_IF_ERROR(mapping.projection->open(_runtime_state)); - } - if (mapping.default_expr != nullptr) { - RETURN_IF_ERROR(mapping.default_expr->prepare(_runtime_state, row_desc)); - RETURN_IF_ERROR(mapping.default_expr->open(_runtime_state)); - } + RETURN_IF_ERROR(_open_mapping_expr(mapping, row_desc)); } return Status::OK(); } diff --git a/be/test/exec/scan/access_path_parser_test.cpp b/be/test/exec/scan/access_path_parser_test.cpp index d4bd6ab6c06360..75007c317bc59d 100644 --- a/be/test/exec/scan/access_path_parser_test.cpp +++ b/be/test/exec/scan/access_path_parser_test.cpp @@ -32,6 +32,8 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/field.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" namespace doris { namespace { @@ -218,6 +220,33 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { } } +TEST(AccessPathParserTest, InheritsNestedInitialDefaultMetadata) { + auto string_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {string_type}, Strings {"added"}); + auto added = field(101, "added", string_type); + added.initial_default_value = "AAEC/w=="; + added.initial_default_value_is_base64 = true; + added.default_expr = VExprContext::create_shared( + VLiteral::create_shared(string_type, Field::create_field("value"))); + format::ColumnDefinition schema { + .identifier = Field::create_field(100), + .name = "s", + .type = struct_type, + .children = {added}, + }; + + for (const auto& path : std::vector> {{"s"}, {"s", "added"}}) { + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path(path)}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + EXPECT_EQ(column.children[0].default_expr, added.default_expr); + EXPECT_EQ(column.children[0].initial_default_value, added.initial_default_value); + EXPECT_TRUE(column.children[0].initial_default_value_is_base64); + } +} + // Scenario: array access paths must pass through the "*" element token, then reuse struct child // parsing under the element wrapper; invalid array tokens are rejected. TEST(AccessPathParserTest, ArrayAccessPathMatrix) { diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index 6c504e0b49a46b..c13eeb8ec111ab 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -98,6 +98,33 @@ void write_iceberg_int_orc_file(const std::string& file_path, const std::string& output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); } +void write_iceberg_struct_orc_file(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct>")); + type->getSubtype(0)->setAttribute("iceberg.id", "1"); + type->getSubtype(0)->getSubtype(0)->setAttribute("iceberg.id", "2"); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(2); + auto& root_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*root_batch.fields[0]); + auto& value_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + value_batch.data[0] = 10; + value_batch.data[1] = 20; + root_batch.numElements = 2; + struct_batch.numElements = 2; + value_batch.numElements = 2; + writer->add(*batch); + writer->close(); + + std::ofstream output(file_path, std::ios::binary); + output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + void write_iceberg_three_int_orc_file( const std::string& file_path, const std::string& first_name, std::optional first_id, const std::vector& first_values, @@ -148,6 +175,36 @@ std::shared_ptr build_iceberg_int32_array(const std::vectorWithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"2"})); + auto struct_type = arrow::struct_({child_field}); + arrow::StructBuilder builder( + struct_type, arrow::default_memory_pool(), + {std::make_shared(arrow::default_memory_pool())}); + auto* value_builder = assert_cast(builder.field_builder(0)); + DORIS_CHECK(builder.Append().ok()); + DORIS_CHECK(value_builder->Append(10).ok()); + DORIS_CHECK(builder.Append().ok()); + DORIS_CHECK(value_builder->Append(20).ok()); + + auto root_field = + arrow::field("s", struct_type, false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"1"})); + std::shared_ptr struct_array; + DORIS_CHECK(builder.Finish(&struct_array).ok()); + auto table = arrow::Table::Make(arrow::schema({root_field}), {struct_array}); + auto output = arrow::io::FileOutputStream::Open(file_path); + DORIS_CHECK(output.ok()); + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 2, properties.build())); +} + void write_iceberg_three_int_parquet_file( const std::string& file_path, const std::string& first_name, std::optional first_id, const std::vector& first_values, @@ -245,6 +302,122 @@ Status create_single_int_tuple_descriptor(ObjectPool* object_pool, const std::st return Status::OK(); } +Status create_struct_tuple_descriptor(ObjectPool* object_pool, DescriptorTbl** descriptor_table, + const TupleDescriptor** tuple_descriptor) { + TDescriptorTable thrift_table; + TTableDescriptor table_descriptor; + table_descriptor.__set_id(0); + table_descriptor.__set_tableType(TTableType::OLAP_TABLE); + table_descriptor.__set_numCols(1); + table_descriptor.__set_numClusteringCols(0); + thrift_table.__set_tableDescriptors({table_descriptor}); + + TTypeNode struct_node; + struct_node.__set_type(TTypeNodeType::STRUCT); + TStructField a_field; + a_field.__set_name("a"); + a_field.__set_contains_null(true); + TStructField b_field; + b_field.__set_name("b"); + b_field.__set_contains_null(true); + struct_node.__set_struct_fields({a_field, b_field}); + TTypeNode a_node; + a_node.__set_type(TTypeNodeType::SCALAR); + TScalarType a_scalar; + a_scalar.__set_type(TPrimitiveType::INT); + a_node.__set_scalar_type(a_scalar); + TTypeNode b_node = a_node; + TTypeDesc type; + type.__set_types({struct_node, a_node, b_node}); + + TSlotDescriptor slot_descriptor; + slot_descriptor.__set_id(0); + slot_descriptor.__set_parent(0); + slot_descriptor.__set_col_unique_id(1); + slot_descriptor.__set_slotType(type); + slot_descriptor.__set_columnPos(0); + slot_descriptor.__set_byteOffset(0); + slot_descriptor.__set_nullIndicatorByte(0); + slot_descriptor.__set_nullIndicatorBit(0); + slot_descriptor.__set_colName("s"); + slot_descriptor.__set_slotIdx(0); + slot_descriptor.__set_isMaterialized(true); + thrift_table.__set_slotDescriptors({slot_descriptor}); + + TTupleDescriptor tuple; + tuple.__set_id(0); + tuple.__set_byteSize(16); + tuple.__set_numNullBytes(1); + tuple.__set_tableId(0); + thrift_table.__set_tupleDescriptors({tuple}); + + RETURN_IF_ERROR(DescriptorTbl::create(object_pool, thrift_table, descriptor_table)); + *tuple_descriptor = (*descriptor_table)->get_tuple_descriptor(0); + return Status::OK(); +} + +schema::external::TSchema make_struct_schema_with_initial_default() { + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + auto make_int_field = [&int_type](const std::string& name, int32_t id, + std::optional initial_default) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + field->__set_is_optional(true); + field->__set_type(int_type); + if (initial_default.has_value()) { + field->__set_initial_default_value(*initial_default); + } + schema::external::TFieldPtr ptr; + ptr.field_ptr = std::move(field); + ptr.__isset.field_ptr = true; + return ptr; + }; + auto a_field = make_int_field("a", 2, std::nullopt); + auto b_field = make_int_field("b", 3, "7"); + + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(1); + struct_field->__set_is_optional(true); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + struct_field->__set_type(struct_type); + schema::external::TStructField nested_fields; + nested_fields.__set_fields({a_field, b_field}); + struct_field->nestedField.__set_struct_field(std::move(nested_fields)); + struct_field->__isset.nestedField = true; + schema::external::TFieldPtr struct_ptr; + struct_ptr.field_ptr = std::move(struct_field); + struct_ptr.__isset.field_ptr = true; + + schema::external::TStructField root; + root.__set_fields({struct_ptr}); + schema::external::TSchema schema; + schema.__set_schema_id(-1); + schema.__set_root_field(std::move(root)); + return schema; +} + +void expect_struct_initial_default(const Block& block, size_t expected_rows) { + ASSERT_EQ(block.rows(), expected_rows); + const IColumn* root_column = block.get_by_position(0).column.get(); + if (const auto* nullable = check_and_get_column(*root_column)) { + root_column = &nullable->get_nested_column(); + } + const auto& struct_column = assert_cast(*root_column); + ASSERT_EQ(struct_column.tuple_size(), 2); + const auto& default_column = assert_cast(struct_column.get_column(1)); + const auto& values = + assert_cast(default_column.get_nested_column()).get_data(); + ASSERT_EQ(values.size(), expected_rows); + for (size_t row = 0; row < expected_rows; ++row) { + EXPECT_FALSE(default_column.is_null_at(row)); + EXPECT_EQ(values[row], 7); + } +} + schema::external::TFieldPtr make_external_int_field(const std::string& name, int32_t field_id, std::optional initial_default, std::vector aliases = {}) { @@ -1049,6 +1222,124 @@ TEST_F(IcebergReaderTest, v1_position_delete_read_error_releases_cache_entry) { EXPECT_NE(status.to_string().find(delete_file.path), std::string::npos); } +TEST_F(IcebergReaderTest, v1_parquet_materializes_nested_initial_default) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_v1_parquet_nested_initial_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "data.parquet").string(); + write_iceberg_struct_parquet_file(file_path); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({make_struct_schema_with_initial_default()}); + TFileRangeDesc scan_range; + scan_range.__set_fs_name(""); + scan_range.__set_path(file_path); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(file_path))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(file_path))); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_struct_tuple_descriptor(&object_pool, &descriptor_table, &tuple_descriptor) + .ok()); + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &timezone_obj, + &io_ctx, &runtime_state, cache.get()); + io::FileReaderSPtr file_reader; + ASSERT_TRUE(io::global_local_filesystem()->open_file(file_path, &file_reader).ok()); + reader.set_file_reader(file_reader); + + std::vector column_descriptors(1); + column_descriptors[0].name = "s"; + std::unordered_map block_positions = {{"s", 0}}; + ParquetInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + ASSERT_TRUE(reader.init_reader(&context).ok()); + + const auto int_type = make_nullable(std::make_shared()); + const auto struct_type = make_nullable( + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"})); + Block block; + block.insert({struct_type->create_column(), struct_type, "s"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(read_rows, 2); + expect_struct_initial_default(block, 2); + std::filesystem::remove_all(test_dir); +} + +TEST_F(IcebergReaderTest, v1_orc_materializes_nested_initial_default) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_v1_orc_nested_initial_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "data.orc").string(); + write_iceberg_struct_orc_file(file_path); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_ORC); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({make_struct_schema_with_initial_default()}); + TFileRangeDesc scan_range; + scan_range.__set_fs_name(""); + scan_range.__set_path(file_path); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(file_path))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(file_path))); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_struct_tuple_descriptor(&object_pool, &descriptor_table, &tuple_descriptor) + .ok()); + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + runtime_state.set_timezone("UTC"); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergOrcReader reader(&kv_cache, &profile, &runtime_state, scan_params, scan_range, 1024, + "UTC", &io_ctx, cache.get()); + + std::vector column_descriptors(1); + column_descriptors[0].name = "s"; + std::unordered_map block_positions = {{"s", 0}}; + OrcInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + ASSERT_TRUE(reader.init_reader(&context).ok()); + + const auto int_type = make_nullable(std::make_shared()); + const auto struct_type = make_nullable( + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"})); + Block block; + block.insert({struct_type->create_column(), struct_type, "s"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(read_rows, 2); + expect_struct_initial_default(block, 2); + std::filesystem::remove_all(test_dir); +} + TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaults) { auto make_field = [](const std::string& name, int32_t id, TPrimitiveType::type primitive_type, const std::string& default_value, int32_t len, int32_t scale, @@ -1076,12 +1367,12 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul }; schema::external::TStructField root_field; - root_field.__set_fields( - {make_field("added_timestamp", 1, TPrimitiveType::DATETIMEV2, - "2024-01-01 00:00:00.123456", -1, 6, false), - make_field("added_binary", 2, TPrimitiveType::VARBINARY, "AAEC/w==", 4, -1, true), - make_field("added_string_binary", 3, TPrimitiveType::STRING, "AAEC/w==", -1, -1, - true)}); + root_field.__set_fields({make_field("added_timestamp", 1, TPrimitiveType::DATETIMEV2, + "2024-01-01 00:00:00.123456", -1, 6, false), + make_field("added_binary", 2, TPrimitiveType::VARBINARY, + "AAECAwQFBgcICQoLDA0ODw==", 16, -1, true), + make_field("added_string_binary", 3, TPrimitiveType::STRING, + "AAEC/w==", -1, -1, true)}); schema::external::TSchema current_schema; current_schema.__set_schema_id(-1); current_schema.__set_root_field(root_field); @@ -1102,7 +1393,7 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul &runtime_state, cache.get()); const auto timestamp_type = make_nullable(std::make_shared(6)); - const auto varbinary_type = make_nullable(std::make_shared(4)); + const auto varbinary_type = make_nullable(std::make_shared(16)); const auto string_type = make_nullable(std::make_shared()); Block block; block.insert({timestamp_type->create_column(), timestamp_type, "added_timestamp"}); @@ -1123,12 +1414,16 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul ASSERT_TRUE(reader.TEST_materialize_missing_equality_delete_columns(&block, 3).ok()); ASSERT_EQ(block.rows(), 3); - EXPECT_EQ(timestamp_type->to_string(*block.get_by_position(0).column, 0), - "2024-01-01 00:00:00.123456"); - EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(1).column, 0), - std::string("\x00\x01\x02\xff", 4)); - EXPECT_EQ(string_type->to_string(*block.get_by_position(2).column, 0), - std::string("\x00\x01\x02\xff", 4)); + for (size_t i = 0; i < block.rows(); ++i) { + SCOPED_TRACE(i); + EXPECT_EQ(timestamp_type->to_string(*block.get_by_position(0).column, i), + "2024-01-01 00:00:00.123456"); + EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(1).column, i), + std::string("\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 16)); + EXPECT_EQ(string_type->to_string(*block.get_by_position(2).column, i), + std::string("\x00\x01\x02\xff", 4)); + } EXPECT_FALSE(is_column_const(*block.get_by_position(0).column)); EXPECT_FALSE(is_column_const(*block.get_by_position(1).column)); EXPECT_FALSE(is_column_const(*block.get_by_position(2).column)); diff --git a/be/test/format/table/table_schema_change_helper_test.cpp b/be/test/format/table/table_schema_change_helper_test.cpp index 3fcb9d39878255..2a98e8422a8ae5 100644 --- a/be/test/format/table/table_schema_change_helper_test.cpp +++ b/be/test/format/table/table_schema_change_helper_test.cpp @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -425,6 +426,78 @@ TEST(MockTableSchemaChangeHelper, IcebergParquetNameMappingFallback) { " ScalarNode\n"); } +TEST(MockTableSchemaChangeHelper, IcebergNestedMissingFieldKeepsInitialDefault) { + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + + auto make_field = [](const std::string& name, int32_t id, const TColumnType& type, + std::optional initial_default = std::nullopt) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + field->__set_type(type); + field->__set_is_optional(true); + if (initial_default.has_value()) { + field->__set_initial_default_value(*initial_default); + } + schema::external::TFieldPtr ptr; + ptr.field_ptr = std::move(field); + ptr.__isset.field_ptr = true; + return ptr; + }; + + auto struct_field = make_field("s", 1, struct_type); + schema::external::TStructField nested_fields; + nested_fields.__set_fields({make_field("a", 2, int_type), make_field("b", 3, int_type, "7")}); + struct_field.field_ptr->nestedField.__set_struct_field(nested_fields); + struct_field.field_ptr->__isset.nestedField = true; + schema::external::TStructField table_schema; + table_schema.__set_fields({struct_field}); + + auto assert_initial_default = [](const std::shared_ptr& root) { + auto struct_node = root->get_children_node("s"); + ASSERT_FALSE(struct_node->children_column_exists("b")); + const auto& initial_default = struct_node->children_initial_default("b"); + ASSERT_NE(initial_default, nullptr); + Field value; + initial_default->get(0, value); + EXPECT_EQ(value.get(), 7); + }; + + FieldSchema file_a; + file_a.name = "a"; + file_a.field_id = 2; + file_a.data_type = DataTypeFactory::instance().create_data_type(TYPE_INT, true); + FieldSchema file_s; + file_s.name = "s"; + file_s.field_id = 1; + file_s.children = {file_a}; + file_s.data_type = + std::make_shared(DataTypes {file_a.data_type}, Strings {"a"}); + FieldDescriptor parquet_schema; + parquet_schema._fields = {file_s}; + parquet_schema.data_type = + std::make_shared(DataTypes {file_s.data_type}, Strings {"s"}); + std::shared_ptr parquet_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + table_schema, parquet_schema, parquet_node) + .ok()); + assert_initial_default(parquet_node); + + auto orc_schema = + std::unique_ptr(orc::Type::buildTypeFromString("struct>")); + const auto& attribute = IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE; + orc_schema->getSubtype(0)->setAttribute(attribute, "1"); + orc_schema->getSubtype(0)->getSubtype(0)->setAttribute(attribute, "2"); + std::shared_ptr orc_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + table_schema, orc_schema.get(), attribute, orc_node) + .ok()); + assert_initial_default(orc_node); +} + TEST(MockTableSchemaChangeHelper, IcebergOrcSchemaChange) { schema::external::TField test_field; TColumnType struct_type; diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index 3885deb107f072..38b9e7d6321cb5 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -3208,6 +3208,8 @@ TEST(ColumnMapperSchemaEvolutionTest, StructChildrenHandleMissingRenameReorderAn auto table_a = field_id_col("a", 1, int_type); auto table_renamed_b = field_id_col("renamed_b", 2, string_type); auto table_c = field_id_col("c", 3, int_type); + table_c.default_expr = + VExprContext::create_shared(literal(int_type, Field::create_field(7))); auto table_struct = struct_col("s", 10, {table_a, table_renamed_b, table_c}); auto v1_a = field_id_col("a", 1, int_type, 0); @@ -3229,6 +3231,9 @@ TEST(ColumnMapperSchemaEvolutionTest, StructChildrenHandleMissingRenameReorderAn EXPECT_EQ(*v1_mapping.child_mappings[0].file_local_id, 0); EXPECT_EQ(*v1_mapping.child_mappings[1].file_local_id, 1); EXPECT_FALSE(v1_mapping.child_mappings[2].file_local_id.has_value()); + EXPECT_EQ(v1_mapping.child_mappings[2].default_expr, table_c.default_expr); + EXPECT_FALSE(v1_mapping.child_mappings[2].constant_index.has_value()); + EXPECT_TRUE(v1_mapper.constant_map().empty()); ASSERT_EQ(v1_request.non_predicate_columns.size(), 1); EXPECT_EQ(v1_request.non_predicate_columns[0].column_id(), LocalColumnId(5)); EXPECT_TRUE(v1_request.non_predicate_columns[0].project_all_children); @@ -3243,6 +3248,7 @@ TEST(ColumnMapperSchemaEvolutionTest, StructChildrenHandleMissingRenameReorderAn EXPECT_EQ(*v2_mapping.child_mappings[0].file_local_id, 1); EXPECT_EQ(*v2_mapping.child_mappings[1].file_local_id, 0); EXPECT_EQ(*v2_mapping.child_mappings[2].file_local_id, 2); + EXPECT_EQ(v2_mapping.child_mappings[2].default_expr, nullptr); ASSERT_EQ(v2_request.non_predicate_columns.size(), 1); EXPECT_EQ(v2_request.non_predicate_columns[0].column_id(), LocalColumnId(8)); EXPECT_TRUE(v2_request.non_predicate_columns[0].project_all_children); diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index b5ec1dfc765053..f0931ea152d2fd 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -48,6 +48,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "exec/scan/access_path_parser.h" #include "exprs/runtime_filter_expr.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" @@ -2893,7 +2894,7 @@ TEST(TableReaderTest, ProjectedListStructReordersRenamedAndMissingElementChildre // Scenario: when every projected array-element struct child is missing/default-only, the reader // still receives a full element projection and can materialize the default child without crashing. -TEST(TableReaderTest, ProjectedListStructOnlyMissingElementChildFallsBackToFullElement) { +TEST(TableReaderTest, ProjectedListStructOnlyMissingElementChildUsesInitialDefault) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_list_only_missing_child_test"; std::filesystem::remove_all(test_dir); @@ -2904,6 +2905,8 @@ TEST(TableReaderTest, ProjectedListStructOnlyMissingElementChildFallsBackToFullE const auto string_type = std::make_shared(); auto missing_child = make_table_column(99, "missing_child", string_type); + missing_child.default_expr = VExprContext::create_shared(VLiteral::create_shared( + missing_child.type, Field::create_field("fallback"))); auto element_type = std::make_shared(DataTypes {string_type}, Strings {"missing_child"}); auto nullable_element_type = make_nullable(element_type); @@ -2943,7 +2946,12 @@ TEST(TableReaderTest, ProjectedListStructOnlyMissingElementChildFallsBackToFullE const auto& element_struct = assert_cast(nullable_elements.get_nested_column()); ASSERT_EQ(element_struct.get_columns().size(), 1); - expect_nullable_column_all_null(element_struct.get_column(0)); + const auto& missing_values = assert_cast( + expect_not_null_nullable_nested_column(element_struct.get_column(0))); + ASSERT_EQ(missing_values.size(), 4); + for (size_t row = 0; row < missing_values.size(); ++row) { + EXPECT_EQ(missing_values.get_data_at(row).to_string(), "fallback"); + } ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); @@ -3036,6 +3044,8 @@ TEST(TableReaderTest, ProjectedMapValueStructReordersRenamedAndMissingChildren) auto b_child = make_table_column(1, "renamed_b", nullable_string_type); b_child.name_mapping = {"b"}; auto missing_child = make_table_column(99, "missing_child", string_type); + missing_child.default_expr = VExprContext::create_shared(VLiteral::create_shared( + missing_child.type, Field::create_field("fallback"))); auto a_child = make_table_column(0, "renamed_a", nullable_int_type); a_child.name_mapping = {"a"}; auto value_type = std::make_shared( @@ -3084,13 +3094,17 @@ TEST(TableReaderTest, ProjectedMapValueStructReordersRenamedAndMissingChildren) ASSERT_EQ(value_struct.get_columns().size(), 3); const auto& b_values = assert_cast( expect_not_null_nullable_nested_column(value_struct.get_column(0))); - const auto& missing_values = value_struct.get_column(1); + const auto& missing_values = assert_cast( + expect_not_null_nullable_nested_column(value_struct.get_column(1))); const auto& a_values = assert_cast( expect_not_null_nullable_nested_column(value_struct.get_column(2))); EXPECT_EQ(b_values.get_data_at(0).to_string(), "ma"); EXPECT_EQ(b_values.get_data_at(1).to_string(), "mb"); EXPECT_EQ(b_values.get_data_at(2).to_string(), "mc"); - expect_nullable_column_all_null(missing_values); + ASSERT_EQ(missing_values.size(), 3); + for (size_t row = 0; row < missing_values.size(); ++row) { + EXPECT_EQ(missing_values.get_data_at(row).to_string(), "fallback"); + } EXPECT_EQ(a_values.get_element(0), 10); EXPECT_EQ(a_values.get_element(1), 20); EXPECT_EQ(a_values.get_element(2), 30); @@ -4030,7 +4044,7 @@ TEST(TableReaderTest, ProjectedColumnsFillMissingParquetColumnWithDefault) { std::filesystem::remove_all(test_dir); } -TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { +TEST(TableReaderTest, ProjectedStructFillsMissingChildWithInitialDefault) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_struct_missing_child_test"; std::filesystem::remove_all(test_dir); @@ -4041,12 +4055,28 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { const auto int_type = std::make_shared(); const auto string_type = std::make_shared(); - auto id_child = make_table_column(0, "id", int_type); - auto missing_child = make_table_column(99, "missing_child", string_type); auto struct_type = std::make_shared(DataTypes {int_type, string_type}, Strings {"id", "missing_child"}); auto struct_column = make_table_column(100, "s", struct_type); - struct_column.children = {id_child, missing_child}; + auto missing_field = external_schema_field("missing_child", 99); + missing_field.field_ptr->__set_initial_default_value("fallback"); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(200); + scan_params.__set_history_schema_info({external_schema( + 200, + {external_struct_field("s", 100, {external_schema_field("id", 0), missing_field})})}); + ProjectedColumnBuildContext build_context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader schema_reader; + ASSERT_TRUE(schema_reader.annotate_projected_column(slot_info, &build_context, &struct_column) + .ok()); + ASSERT_TRUE(build_context.schema_column.has_value()); + ASSERT_TRUE(AccessPathParser::build_nested_children(&struct_column, + std::vector {}, + &*build_context.schema_column) + .ok()); + ASSERT_EQ(struct_column.children.size(), 2); + ASSERT_NE(struct_column.children[1].default_expr, nullptr); std::vector projected_columns = {struct_column}; RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -4056,7 +4086,7 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { .projected_columns = projected_columns, .conjuncts = {}, .format = FileFormat::PARQUET, - .scan_params = nullptr, + .scan_params = &scan_params, .io_ctx = nullptr, .runtime_state = &state, .scanner_profile = nullptr, @@ -4077,7 +4107,9 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { expect_not_null_nullable_nested_column(struct_result.get_column(0))); ASSERT_EQ(struct_result.size(), 1); EXPECT_EQ(ids.get_element(0), 7); - expect_nullable_column_all_null(struct_result.get_column(1)); + const auto& missing_values = + expect_not_null_nullable_nested_column(struct_result.get_column(1)); + EXPECT_EQ(missing_values.get_data_at(0).to_string(), "fallback"); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java index acbd760a2bf057..d5ac42c6365c1f 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java @@ -610,6 +610,10 @@ public String getDefaultValue() { return this.defaultValue; } + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + public String getRealDefaultValue() { return realDefaultValue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 4c3fb1622213ea..0b791411f15623 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -65,6 +65,9 @@ import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.nereids.trees.expressions.literal.Result; +import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.VarBinaryType; import org.apache.doris.nereids.util.DateUtils; import org.apache.doris.persist.gson.GsonUtils; @@ -1025,8 +1028,13 @@ private static Literal parseIcebergDateLiteral( String canonicalValue = value; if (dorisType != null) { try { - canonicalValue = DateLiteralUtils.createDateLiteral(value, dorisType).getStringValue(); - } catch (AnalysisException | IllegalArgumentException e) { + if (dorisType.isTimeStampTz()) { + TimeStampTzType timestampTzType = (TimeStampTzType) DataType.fromCatalogType(dorisType); + canonicalValue = TimestampTzLiteral.fromSessionTimeZone(timestampTzType, value).getStringValue(); + } else { + canonicalValue = DateLiteralUtils.createDateLiteral(value, dorisType).getStringValue(); + } + } catch (AnalysisException | RuntimeException e) { throw new IllegalArgumentException("Invalid date or timestamp string: " + value, e); } } @@ -1149,8 +1157,13 @@ private static long parseTimestampToMicros(String valueStr, TimestampType timest return epochSecond * 1_000_000L + microSecond; } - private static void updateIcebergColumnUniqueId(Column column, Types.NestedField icebergField) { + private static void updateIcebergColumnMetadata(Column column, Types.NestedField icebergField, + boolean enableMappingTimestampTz) { column.setUniqueId(icebergField.fieldId()); + if (icebergField.initialDefault() != null) { + column.setDefaultValue(serializeInitialDefault( + icebergField.type(), icebergField.initialDefault(), enableMappingTimestampTz)); + } List icebergFields = Lists.newArrayList(); switch (icebergField.type().typeId()) { case LIST: @@ -1169,7 +1182,8 @@ private static void updateIcebergColumnUniqueId(Column column, Types.NestedField if (column.getChildren() != null) { List childColumns = column.getChildren(); for (int idx = 0; idx < childColumns.size(); idx++) { - updateIcebergColumnUniqueId(childColumns.get(idx), icebergFields.get(idx)); + updateIcebergColumnMetadata( + childColumns.get(idx), icebergFields.get(idx), enableMappingTimestampTz); } } } @@ -1218,15 +1232,10 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi List columns = schema.columns(); List resSchema = Lists.newArrayListWithCapacity(columns.size()); for (Types.NestedField field : columns) { - String initialDefault = null; - if (field.initialDefault() != null) { - initialDefault = serializeInitialDefault(field.type(), field.initialDefault(), - enableMappingTimestampTz); - } Column column = new Column(field.name(), IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz), - true, null, true, initialDefault, field.doc(), true, -1); - updateIcebergColumnUniqueId(column, field); + true, null, true, null, field.doc(), true, -1); + updateIcebergColumnMetadata(column, field, enableMappingTimestampTz); if (field.type().isPrimitiveType() && field.type().typeId() == TypeID.TIMESTAMP) { Types.TimestampType timestampType = (Types.TimestampType) field.type(); if (timestampType.shouldAdjustToUTC()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index a817bc4e658dbe..ef17dfc872f6c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -4278,7 +4278,8 @@ public ColumnDefinitionWithPath visitColumnDefWithPath(ColumnDefWithPathContext defaultValue = Optional.of(new DefaultValue("-" + ctx.DECIMAL_VALUE().getText())); } } else if (ctx.stringValue != null) { - defaultValue = Optional.of(new DefaultValue(toStringValue(ctx.stringValue.getText()))); + defaultValue = Optional.of(new DefaultValue( + LogicalPlanBuilderAssistant.parseStringLiteral(ctx.stringValue.getText()))); } else if (ctx.nullValue != null) { defaultValue = Optional.of(DefaultValue.NULL_DEFAULT_VALUE); } else if (ctx.defaultTimestamp != null) { @@ -4317,8 +4318,8 @@ public ColumnDefinitionWithPath visitColumnDefWithPath(ColumnDefWithPathContext e.getCause()); } } - String comment = ctx.comment != null ? ctx.comment.getText().substring(1, ctx.comment.getText().length() - 1) - .replace("\\", "") : ""; + String comment = ctx.comment != null + ? LogicalPlanBuilderAssistant.parseStringLiteral(ctx.comment.getText()) : ""; long autoIncInitValue = -1; if (ctx.AUTO_INCREMENT() != null) { if (ctx.autoIncInitValue != null) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 3ecb3a72d6b023..20e4a36d7d1f1d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -223,8 +223,17 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { col1.setUniqueId(101); Column col2 = new Column("c2", Type.VARCHAR, false); col2.setUniqueId(102); + StructType structType = new StructType( + new StructField("added_child", Type.INT), + new StructField("added_binary", Type.VARBINARY)); + Column col3 = new Column("c3", structType, true); + col3.setUniqueId(103); + col3.getChildren().get(0).setUniqueId(104); + col3.getChildren().get(0).setDefaultValue("9"); + col3.getChildren().get(1).setUniqueId(105); + col3.getChildren().get(1).setDefaultValue("ignored by Base64 transport"); - List columns = Arrays.asList(col1, col2); + List columns = Arrays.asList(col1, col2, col3); Map> nameMapping = new HashMap<>(); nameMapping.put(col1.getUniqueId(), Arrays.asList("m_c1")); @@ -232,6 +241,7 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Map base64InitialDefaults = new HashMap<>(); base64InitialDefaults.put(col2.getUniqueId(), "AAEC/w=="); + base64InitialDefaults.put(105, "BAUGBw=="); ExternalUtil.initSchemaInfoForAllColumn( params, schemaId, columns, nameMapping, base64InitialDefaults); @@ -244,10 +254,11 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { TStructField rootField = tSchema.getRootField(); Assert.assertNotNull(rootField); - Assert.assertEquals(2, rootField.getFieldsSize()); + Assert.assertEquals(3, rootField.getFieldsSize()); TField field1 = rootField.getFields().get(0).getFieldPtr(); TField field2 = rootField.getFields().get(1).getFieldPtr(); + TField field3 = rootField.getFields().get(2).getFieldPtr(); Assert.assertEquals(col1.getName(), field1.getName()); Assert.assertEquals(col1.getUniqueId(), field1.getId()); @@ -264,5 +275,15 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(Arrays.asList("m_c2_a", "m_c2_b"), field2.getNameMapping()); Assert.assertEquals("AAEC/w==", field2.getInitialDefaultValue()); Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); + + TField nestedField = field3.getNestedField().getStructField().getFields().get(0).getFieldPtr(); + Assert.assertEquals(104, nestedField.getId()); + Assert.assertEquals("9", nestedField.getInitialDefaultValue()); + Assert.assertFalse(nestedField.isSetInitialDefaultValueIsBase64()); + + TField nestedBinary = field3.getNestedField().getStructField().getFields().get(1).getFieldPtr(); + Assert.assertEquals(105, nestedBinary.getId()); + Assert.assertEquals("BAUGBw==", nestedBinary.getInitialDefaultValue()); + Assert.assertTrue(nestedBinary.isInitialDefaultValueIsBase64()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index ee34b4bee50b0c..44d7a26052a1d1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; @@ -54,6 +55,7 @@ import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.LongType; import org.apache.iceberg.types.Types.StructType; +import org.apache.iceberg.util.DateTimeUtil; import org.apache.iceberg.view.View; import org.junit.Assert; import org.junit.Test; @@ -63,6 +65,7 @@ import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.time.DateTimeException; +import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -327,6 +330,35 @@ public void testParseSchemaPreservesInitialDefault() { .withId(5) .ofType(Types.FixedType.ofLength(4)) .withInitialDefault(ByteBuffer.wrap(new byte[] {3, 2, 1, 0})) + .build(), + Types.NestedField.optional("nested") + .withId(6) + .ofType(Types.StructType.of( + Types.NestedField.optional("added_nested") + .withId(7) + .ofType(Types.IntegerType.get()) + .withInitialDefault(9) + .build())) + .build(), + Types.NestedField.optional("array_nested") + .withId(8) + .ofType(Types.ListType.ofOptional(9, Types.StructType.of( + Types.NestedField.optional("added_in_element") + .withId(10) + .ofType(Types.StringType.get()) + .withInitialDefault("x") + .build()))) + .build(), + Types.NestedField.optional("map_nested") + .withId(11) + .ofType(Types.MapType.ofOptional(12, 13, Types.StringType.get(), + Types.StructType.of( + Types.NestedField.optional("added_in_value") + .withId(14) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap( + new byte[] {4, 5, 6, 7})) + .build()))) .build()); List columns = IcebergUtils.parseSchema(schema, true, false); @@ -335,11 +367,48 @@ public void testParseSchemaPreservesInitialDefault() { Assert.assertEquals("2024-01-01 00:00:00.123456", columns.get(1).getDefaultValue()); Assert.assertEquals("AAAAAAAAAAAAAAAAAAAAAA==", columns.get(2).getDefaultValue()); Assert.assertEquals("AAEC/w==", columns.get(3).getDefaultValue()); + Assert.assertEquals("9", columns.get(5).getChildren().get(0).getDefaultValue()); + Assert.assertEquals(7, columns.get(5).getChildren().get(0).getUniqueId()); + Assert.assertEquals("x", columns.get(6).getChildren().get(0) + .getChildren().get(0).getDefaultValue()); + Assert.assertEquals(10, columns.get(6).getChildren().get(0) + .getChildren().get(0).getUniqueId()); + Assert.assertEquals("BAUGBw==", columns.get(7).getChildren().get(1) + .getChildren().get(0).getDefaultValue()); + Assert.assertEquals(14, columns.get(7).getChildren().get(1) + .getChildren().get(0).getUniqueId()); Map base64Defaults = IcebergUtils.getBase64EncodedInitialDefaults(schema); Assert.assertEquals("AAAAAAAAAAAAAAAAAAAAAA==", base64Defaults.get(3)); Assert.assertEquals("AAEC/w==", base64Defaults.get(4)); Assert.assertEquals("AwIBAA==", base64Defaults.get(5)); + Assert.assertEquals("BAUGBw==", base64Defaults.get(14)); + } + + @Test + public void testParseIcebergTimestampTzUsesSessionTimeZone() { + ConnectContext context = new ConnectContext(); + context.getSessionVariable().setTimeZone("+07:00"); + context.setThreadLocalInfo(); + try { + Types.TimestampType icebergType = Types.TimestampType.withZone(); + + org.apache.iceberg.expressions.Literal sessionLocal = + IcebergUtils.parseIcebergLiteral("2020-01-01 00:00:00.1235", + ScalarType.createTimeStampTzType(3), icebergType); + Assert.assertEquals(DateTimeUtil.microsFromInstant( + Instant.parse("2019-12-31T17:00:00.124Z")), + ((Long) sessionLocal.value()).longValue()); + + org.apache.iceberg.expressions.Literal explicitOffset = + IcebergUtils.parseIcebergLiteral("2020-01-01 00:00:00.654321+02:00", + ScalarType.createTimeStampTzType(6), icebergType); + Assert.assertEquals(DateTimeUtil.microsFromInstant( + Instant.parse("2019-12-31T22:00:00.654321Z")), + ((Long) explicitOffset.value()).longValue()); + } finally { + ConnectContext.remove(); + } } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index 5de81383fa4709..504b540265fe55 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -149,6 +149,28 @@ public void testModifyColumnCommentRoundTripEscapesQuotesAndBackslashes() { assertCommentRoundTrip(true); } + @Test + public void testColumnDefinitionWithPathDecodesDefaultAndCommentLiterals() { + assertColumnDefinitionLiteralDecoding(false); + assertColumnDefinitionLiteralDecoding(true); + } + + private void assertColumnDefinitionLiteralDecoding(boolean noBackslashEscapes) { + try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { + mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(noBackslashEscapes); + + String sqlPath = noBackslashEscapes ? "C:\\tmp\\" : "C:\\\\tmp\\\\"; + AddColumnOp add = assertSingleClausePath( + "ALTER TABLE t ADD COLUMN info.owner STRING DEFAULT 'owner''s " + sqlPath + + "' COMMENT \"a\"\"b " + sqlPath + "\"", + AddColumnOp.class, "info.owner"); + + Assertions.assertEquals("owner's C:\\tmp\\", add.getColumnDef() + .translateToCatalogStyleForSchemaChange().getDefaultValue()); + Assertions.assertEquals("a\"b C:\\tmp\\", add.getColumnDef().getComment()); + } + } + private void assertCommentRoundTrip(boolean noBackslashEscapes) { try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(noBackslashEscapes); From c1d926865da9fe832eef4715d7fb3f60c4a1bc58 Mon Sep 17 00:00:00 2001 From: daidai Date: Fri, 17 Jul 2026 15:18:42 +0800 Subject: [PATCH 17/32] [fix](iceberg) Fix nested default BE test compilation ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: The nested initial-default BE unit test used an invalid FieldDescriptor member and compared Doris ColumnPtr directly with nullptr, preventing the BE UT target from compiling. Remove the unsupported root descriptor assignment and assert through the pointer accessor. ### Release note None ### Check List (For Author) - Test: Unit Test - Compiled table_schema_change_helper_test.cpp in the ASAN BE_TEST target - Behavior changed: No - Does this need documentation: No --- be/src/exec/scan/access_path_parser.cpp | 3 - be/src/format/orc/vorc_reader.cpp | 13 +- .../format/parquet/vparquet_column_reader.cpp | 10 +- .../format/table/iceberg_initial_default.cpp | 59 ---- be/src/format/table/iceberg_initial_default.h | 32 -- be/src/format/table/iceberg_reader_mixin.h | 41 ++- .../table/table_schema_change_helper.cpp | 46 +-- .../format/table/table_schema_change_helper.h | 25 +- be/src/format_v2/column_mapper.cpp | 1 - be/src/format_v2/table/iceberg_reader.cpp | 34 +- be/src/format_v2/table_reader.cpp | 21 -- be/src/format_v2/table_reader.h | 53 +-- be/test/exec/scan/access_path_parser_test.cpp | 29 -- .../table/iceberg/iceberg_reader_test.cpp | 325 +----------------- .../table/table_schema_change_helper_test.cpp | 73 ---- be/test/format_v2/column_mapper_test.cpp | 6 - .../format_v2/table/iceberg_reader_test.cpp | 13 +- be/test/format_v2/table_reader_test.cpp | 52 +-- .../java/org/apache/doris/catalog/Column.java | 4 - .../iceberg/IcebergMetadataOps.java | 40 +-- .../datasource/iceberg/IcebergUtils.java | 73 +--- .../nereids/parser/LogicalPlanBuilder.java | 63 +--- .../plans/commands/AlterTableCommand.java | 7 - .../plans/commands/info/ColumnDefinition.java | 7 +- .../doris/datasource/ExternalUtilTest.java | 25 +- .../IcebergMetadataOpsValidationTest.java | 134 +------- .../datasource/iceberg/IcebergUtilsTest.java | 69 ---- ...cebergNestedSchemaEvolutionParserTest.java | 94 ++++- .../plans/commands/AlterTableCommandTest.java | 28 -- .../commands/info/ColumnDefinitionTest.java | 10 + .../org/apache/doris/nereids/DorisParser.g4 | 6 +- ...iceberg_schema_change_complex_types.groovy | 4 +- 32 files changed, 263 insertions(+), 1137 deletions(-) delete mode 100644 be/src/format/table/iceberg_initial_default.cpp delete mode 100644 be/src/format/table/iceberg_initial_default.h diff --git a/be/src/exec/scan/access_path_parser.cpp b/be/src/exec/scan/access_path_parser.cpp index fdf6c3e3dfad63..b215212b6d861b 100644 --- a/be/src/exec/scan/access_path_parser.cpp +++ b/be/src/exec/scan/access_path_parser.cpp @@ -87,9 +87,6 @@ void inherit_schema_metadata(format::ColumnDefinition* column, return; } column->name_mapping = schema_column->name_mapping; - column->default_expr = schema_column->default_expr; - column->initial_default_value = schema_column->initial_default_value; - column->initial_default_value_is_base64 = schema_column->initial_default_value_is_base64; } const format::ColumnDefinition* find_schema_child_by_path( diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index f31ae922c0d031..28ef930057f3a2 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -2151,17 +2151,8 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name, col_name); } auto mutable_field = IColumn::mutate(std::move(doris_field)); - const auto& table_column_name = doris_struct_type->get_name_by_position(missing_field); - ColumnPtr initial_default; - if (!root_node->children_column_exists(table_column_name)) { - initial_default = root_node->children_initial_default(table_column_name); - } - if (initial_default) { - mutable_field->insert_many_from(*initial_default, 0, num_values); - } else { - reinterpret_cast(mutable_field.get()) - ->insert_many_defaults(num_values); - } + reinterpret_cast(mutable_field.get()) + ->insert_many_defaults(num_values); doris_field = std::move(mutable_field); } diff --git a/be/src/format/parquet/vparquet_column_reader.cpp b/be/src/format/parquet/vparquet_column_reader.cpp index 1bface37fb8126..2726044c193b4c 100644 --- a/be/src/format/parquet/vparquet_column_reader.cpp +++ b/be/src/format/parquet/vparquet_column_reader.cpp @@ -993,14 +993,8 @@ Status StructColumnReader::read_column_data( DCHECK(doris_type->is_nullable()); doris_field = IColumn::mutate(std::move(doris_field)); auto mutable_column = doris_field->assert_mutable(); - const auto& initial_default = - root_node->children_initial_default(doris_struct_type->get_element_name(idx)); - if (initial_default) { - mutable_column->insert_many_from(*initial_default, 0, missing_column_sz); - } else { - auto* nullable_column = static_cast(mutable_column.get()); - nullable_column->insert_many_defaults(missing_column_sz); - } + auto* nullable_column = static_cast(mutable_column.get()); + nullable_column->insert_many_defaults(missing_column_sz); } if (null_map_ptr != nullptr) { diff --git a/be/src/format/table/iceberg_initial_default.cpp b/be/src/format/table/iceberg_initial_default.cpp deleted file mode 100644 index fc1dd615e08bb5..00000000000000 --- a/be/src/format/table/iceberg_initial_default.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/iceberg_initial_default.h" - -#include - -#include "common/consts.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/primitive_type.h" -#include "core/field.h" -#include "util/url_coding.h" - -namespace doris::iceberg { - -Status parse_initial_default(const DataTypePtr& type, std::string_view encoded_value, - bool is_base64, std::string_view field_name, ColumnPtr* result) { - DORIS_CHECK(type != nullptr); - DORIS_CHECK(result != nullptr); - const auto nested_type = remove_nullable(type); - const auto primitive_type = nested_type->get_primitive_type(); - Field value; - if (is_base64 || primitive_type == TYPE_VARBINARY) { - std::string decoded_value; - if (!base64_decode(std::string(encoded_value), &decoded_value)) { - return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", - field_name); - } - if (primitive_type == TYPE_VARBINARY) { - value = Field::create_field(StringView(decoded_value)); - } else { - DORIS_CHECK(is_string_type(primitive_type)); - value = Field::create_field(decoded_value); - } - } else { - RETURN_IF_ERROR( - nested_type->get_serde()->from_fe_string(std::string(encoded_value), value)); - } - auto column = type->create_column(); - column->insert(value); - *result = std::move(column); - return Status::OK(); -} - -} // namespace doris::iceberg diff --git a/be/src/format/table/iceberg_initial_default.h b/be/src/format/table/iceberg_initial_default.h deleted file mode 100644 index ef3e6ec2c249ee..00000000000000 --- a/be/src/format/table/iceberg_initial_default.h +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "common/status.h" -#include "core/column/column.h" -#include "core/data_type/data_type.h" - -namespace doris::iceberg { - -// Returns a one-row column so variable-length values own their decoded bytes after this call. -Status parse_initial_default(const DataTypePtr& type, std::string_view encoded_value, - bool is_base64, std::string_view field_name, ColumnPtr* result); - -} // namespace doris::iceberg diff --git a/be/src/format/table/iceberg_reader_mixin.h b/be/src/format/table/iceberg_reader_mixin.h index dd0e9240c4eea9..221158a895c344 100644 --- a/be/src/format/table/iceberg_reader_mixin.h +++ b/be/src/format/table/iceberg_reader_mixin.h @@ -40,11 +40,11 @@ #include "format/generic_reader.h" #include "format/table/equality_delete.h" #include "format/table/iceberg_delete_file_reader_helper.h" -#include "format/table/iceberg_initial_default.h" #include "format/table/table_schema_change_helper.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "storage/olap_common.h" +#include "util/url_coding.h" namespace doris { class TIcebergDeleteFileDesc; @@ -703,19 +703,34 @@ Status IcebergReaderMixin::_register_missing_equality_delete_column( "Missing Iceberg schema metadata for equality-delete field id {}", field_id); } - ColumnPtr value; + Field value; + // A VARBINARY Field keeps a non-owning StringView until create_column_const() below. + std::string decoded_default; if (table_field->__isset.initial_default_value) { + const auto nested_type = remove_nullable(delete_key_type); const bool default_is_base64 = (table_field->__isset.initial_default_value_is_base64 && table_field->initial_default_value_is_base64) || (table_field->__isset.type && thrift_to_type(table_field->type.type) == TYPE_VARBINARY); - RETURN_IF_ERROR(doris::iceberg::parse_initial_default( - delete_key_type, table_field->initial_default_value, default_is_base64, - table_field->name, &value)); - } else { - value = delete_key_type->create_column_const(1, Field()); + if (default_is_base64) { + if (!base64_decode(table_field->initial_default_value, &decoded_default)) { + return Status::InvalidArgument( + "Invalid Base64 Iceberg initial default for field {}", table_field->name); + } + if (nested_type->get_primitive_type() == TYPE_VARBINARY) { + value = Field::create_field(StringView(decoded_default)); + } else { + DORIS_CHECK(is_string_type(nested_type->get_primitive_type())); + value = Field::create_field(decoded_default); + } + } else { + RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string( + table_field->initial_default_value, value)); + } } - const bool inserted = _missing_equality_delete_values.emplace(name, std::move(value)).second; + const bool inserted = _missing_equality_delete_values + .emplace(name, delete_key_type->create_column_const(1, value)) + .second; DORIS_CHECK(inserted); this->register_synthesized_column_handler( name, [this, name](Block* block, size_t rows) -> Status { @@ -729,10 +744,6 @@ Status IcebergReaderMixin::_register_missing_equality_delete_column( template Status IcebergReaderMixin::_materialize_missing_equality_delete_column( Block* block, const std::string& name, const ColumnPtr& value, size_t rows) { - const auto one_row_value = value->convert_to_full_column_if_const(); - auto materialized_value = one_row_value->clone_empty(); - materialized_value->reserve(rows); - materialized_value->insert_many_from(*one_row_value, 0, rows); if (!this->col_name_to_block_idx_ref()->contains(name)) { // ORC must not register a key that is absent from the file as a physical child. In that // case the reader block has no slot for the synthesized key, so append one here before @@ -742,7 +753,8 @@ Status IcebergReaderMixin::_materialize_missing_equality_delete_colu [&](const ColumnWithTypeAndName& col) { return col.name == name; }); DORIS_CHECK(expand_col != _expand_columns.end()); (*this->col_name_to_block_idx_ref())[name] = block->columns(); - block->insert({std::move(materialized_value), expand_col->type, name}); + block->insert({value->clone_resized(rows)->convert_to_full_column_if_const(), + expand_col->type, name}); return Status::OK(); } const auto position = this->col_name_to_block_idx_ref()->at(name); @@ -751,7 +763,8 @@ Status IcebergReaderMixin::_materialize_missing_equality_delete_colu // MultiEqualityDelete hashes each key column directly. Materialize the repeated default so // every key has the batch row count; a ColumnConst keeps only one nested value and therefore // cannot participate in the row-wise multi-column hash contract. - block->get_by_position(position).column = std::move(materialized_value); + block->get_by_position(position).column = + value->clone_resized(rows)->convert_to_full_column_if_const(); return Status::OK(); } diff --git a/be/src/format/table/table_schema_change_helper.cpp b/be/src/format/table/table_schema_change_helper.cpp index caca698c96c4eb..3d4cb6b5a6c99c 100644 --- a/be/src/format/table/table_schema_change_helper.cpp +++ b/be/src/format/table/table_schema_change_helper.cpp @@ -25,11 +25,9 @@ #include "common/status.h" #include "core/block/block.h" #include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_struct.h" #include "format/generic_reader.h" -#include "format/table/iceberg_initial_default.h" #include "util/string_util.h" namespace doris { @@ -80,38 +78,6 @@ bool find_file_field_idx_by_name_mapping( return table_field.__isset.name && try_match(table_field.name); } -Status parse_iceberg_initial_default(const schema::external::TField& field, - ColumnPtr* initial_default) { - DORIS_CHECK(initial_default != nullptr); - initial_default->reset(); - if (!field.__isset.initial_default_value) { - return Status::OK(); - } - DORIS_CHECK(field.__isset.type); - const auto primitive_type = thrift_to_type(field.type.type); - DORIS_CHECK(!is_complex_type(primitive_type)); - const auto type = DataTypeFactory::instance().create_data_type( - primitive_type, field.__isset.is_optional && field.is_optional, - field.type.__isset.precision ? field.type.precision : 0, - field.type.__isset.scale ? field.type.scale : 0, - field.type.__isset.len ? field.type.len : -1); - const bool is_base64 = - field.__isset.initial_default_value_is_base64 && field.initial_default_value_is_base64; - RETURN_IF_ERROR(doris::iceberg::parse_initial_default(type, field.initial_default_value, - is_base64, field.name, initial_default)); - return Status::OK(); -} - -Status add_missing_iceberg_child( - const std::shared_ptr& struct_node, - const schema::external::TField& field) { - DORIS_CHECK(struct_node != nullptr); - ColumnPtr initial_default; - RETURN_IF_ERROR(parse_iceberg_initial_default(field, &initial_default)); - struct_node->add_not_exist_children(field.name, std::move(initial_default)); - return Status::OK(); -} - } // namespace Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name( @@ -482,7 +448,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id( struct_node->add_children(table_column_name, parquet_fields_schema[file_column_idx].name, field_node); } else { - RETURN_IF_ERROR(add_missing_iceberg_child(struct_node, *table_field.field_ptr)); + struct_node->add_not_exist_children(table_column_name); } } @@ -568,7 +534,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id( exist_field_id)); struct_node->add_children(table_column_name, file_field.name, field_node); } else { - RETURN_IF_ERROR(add_missing_iceberg_child(struct_node, *table_field.field_ptr)); + struct_node->add_not_exist_children(table_column_name); } } node = struct_node; @@ -620,7 +586,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam } if (!matched) { - RETURN_IF_ERROR(add_missing_iceberg_child(struct_node, *table_field.field_ptr)); + struct_node->add_not_exist_children(table_column_name); continue; } @@ -724,7 +690,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam } if (!matched) { - RETURN_IF_ERROR(add_missing_iceberg_child(struct_node, *table_field.field_ptr)); + struct_node->add_not_exist_children(table_column_name); continue; } @@ -774,7 +740,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id( struct_node->add_children(table_column_name, orc_root->getFieldName(file_field_idx), field_node); } else { - RETURN_IF_ERROR(add_missing_iceberg_child(struct_node, *table_field.field_ptr)); + struct_node->add_not_exist_children(table_column_name); } } node = struct_node; @@ -891,7 +857,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma } if (!matched) { - RETURN_IF_ERROR(add_missing_iceberg_child(struct_node, *table_field.field_ptr)); + struct_node->add_not_exist_children(table_column_name); continue; } diff --git a/be/src/format/table/table_schema_change_helper.h b/be/src/format/table/table_schema_change_helper.h index 008ff3a1c6f6a6..2d54b3216cac89 100644 --- a/be/src/format/table/table_schema_change_helper.h +++ b/be/src/format/table/table_schema_change_helper.h @@ -67,11 +67,6 @@ class TableSchemaChangeHelper { "children_column_exists should not be called on base TableInfoNode"); } - virtual const ColumnPtr& children_initial_default(std::string table_column_name) const { - throw std::logic_error( - "children_initial_default should not be called on base TableInfoNode"); - } - virtual std::shared_ptr get_element_node() const { throw std::logic_error("get_element_node should not be called on base TableInfoNode"); } @@ -83,8 +78,7 @@ class TableSchemaChangeHelper { throw std::logic_error("get_value_node should not be called on base TableInfoNode"); } - virtual void add_not_exist_children(std::string table_column_name, - ColumnPtr initial_default = nullptr) { + virtual void add_not_exist_children(std::string table_column_name) { throw std::logic_error( "add_not_exist_children should not be called on base TableInfoNode"); }; @@ -137,10 +131,9 @@ class TableSchemaChangeHelper { const std::shared_ptr node; const std::string column_name; const bool exists; - const ColumnPtr initial_default; }; - // table column name -> {node, file_column_name, exists_in_file, initial_default} + // table column name -> { node, file_column_name, exists_in_file} std::map children; public: @@ -174,22 +167,14 @@ class TableSchemaChangeHelper { return children.at(table_column_name).exists; } - const ColumnPtr& children_initial_default(std::string table_column_name) const override { - DCHECK(children.contains(table_column_name)); - DCHECK(!children_column_exists(table_column_name)); - return children.at(table_column_name).initial_default; - } - - void add_not_exist_children(std::string table_column_name, - ColumnPtr initial_default = nullptr) override { - children.emplace(table_column_name, - StructChild {nullptr, "", false, std::move(initial_default)}); + void add_not_exist_children(std::string table_column_name) override { + children.emplace(table_column_name, StructChild {nullptr, "", false}); } void add_children(std::string table_column_name, std::string file_column_name, std::shared_ptr children_node) override { children.emplace(table_column_name, - StructChild {children_node, file_column_name, true, nullptr}); + StructChild {children_node, file_column_name, true}); } const std::map& get_children() const { return children; } diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 54e5f0c2afcfc0..6a0daf903f01d4 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -2115,7 +2115,6 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c child_mapping.table_type = table_child.type; child_mapping.file_type = table_child.type; child_mapping.filter_conversion = FilterConversionType::FINALIZE_ONLY; - child_mapping.default_expr = table_child.default_expr; mapping->child_mappings.push_back(std::move(child_mapping)); continue; } diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 92598af378d4d7..b3f56909277e76 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -37,7 +37,6 @@ #include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "format/table/deletion_vector_reader.h" -#include "format/table/iceberg_initial_default.h" #include "format_v2/expr/cast.h" #include "format_v2/expr/equality_delete_predicate.h" #include "format_v2/orc/orc_reader.h" @@ -45,6 +44,7 @@ #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/table_reader.h" #include "io/file_factory.h" +#include "util/url_coding.h" namespace doris::format::iceberg { @@ -91,14 +91,32 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit return Status::OK(); } - ColumnPtr initial_default_column; - RETURN_IF_ERROR(doris::iceberg::parse_initial_default( - table_field.type, *table_field.initial_default_value, - table_field.initial_default_value_is_base64, table_field.name, - &initial_default_column)); - Field initial_default; - initial_default_column->get(0, initial_default); + // A VARBINARY Field keeps a non-owning StringView until VLiteral materializes it below. + std::string decoded_default; + if (table_field.initial_default_value_is_base64 || + table_field.type->get_primitive_type() == TYPE_VARBINARY) { + // New FE versions mark every Iceberg UUID/BINARY/FIXED default as Base64 regardless of its + // Doris mapping. Keep the VARBINARY fallback for scan descriptors produced before that + // marker existed. Decode before parsing so STRING/CHAR and VARBINARY all compare against + // the raw bytes stored in equality-delete files. + if (!base64_decode(*table_field.initial_default_value, &decoded_default)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + table_field.name); + } + if (table_field.type->get_primitive_type() == TYPE_VARBINARY) { + initial_default = Field::create_field(StringView(decoded_default)); + } else { + DORIS_CHECK(is_string_type(table_field.type->get_primitive_type())); + initial_default = Field::create_field(decoded_default); + } + } else { + // An added field's initial default is its logical value in every older data file that lacks + // the physical column. FE normalizes the string for the current Doris table type. + RETURN_IF_ERROR(table_field.type->get_serde()->from_fe_string( + *table_field.initial_default_value, initial_default)); + } + auto literal = VLiteral::create_shared(table_field.type, initial_default); if (table_field.type->equals(*delete_key_type)) { *key_expr = std::move(literal); diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 6d1780c9dcc5e7..5f37538c1f2639 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -38,11 +38,9 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type/primitive_type.h" #include "exprs/vexpr_context.h" -#include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "format/table/deletion_vector_reader.h" #include "format/table/iceberg_delete_file_reader_helper.h" -#include "format/table/iceberg_initial_default.h" #include "format/table/paimon_reader.h" #include "format_v2/column_mapper.h" #include "format_v2/delimited_text/csv_reader.h" @@ -281,24 +279,6 @@ ColumnDefinition build_schema_column_from_external_field(const schema::external: return column; } -Status attach_initial_default_exprs(ColumnDefinition* column) { - DORIS_CHECK(column != nullptr); - if (column->initial_default_value.has_value()) { - ColumnPtr initial_default_column; - RETURN_IF_ERROR(doris::iceberg::parse_initial_default( - column->type, *column->initial_default_value, - column->initial_default_value_is_base64, column->name, &initial_default_column)); - Field initial_default; - initial_default_column->get(0, initial_default); - column->default_expr = - VExprContext::create_shared(VLiteral::create_shared(column->type, initial_default)); - } - for (auto& child : column->children) { - RETURN_IF_ERROR(attach_initial_default_exprs(&child)); - } - return Status::OK(); -} - const schema::external::TField* find_external_root_field(const TFileScanRangeParams* params, const ColumnDefinition& column) { if (params == nullptr || !params->__isset.history_schema_info || @@ -506,7 +486,6 @@ Status TableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info return Status::OK(); } context->schema_column = build_schema_column_from_external_field(*schema_field, column->type); - RETURN_IF_ERROR(attach_initial_default_exprs(&*context->schema_column)); column->identifier = context->schema_column->identifier; column->name_mapping = context->schema_column->name_mapping; return Status::OK(); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 8cb755ba4bdfc0..73bade81bc4c65 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -1296,28 +1296,6 @@ class TableReader { return column.get(); } - Status _materialize_missing_child_mapping_column(const ColumnMapping& mapping, - const size_t rows, ColumnPtr* column) { - DORIS_CHECK(mapping.table_type != nullptr); - DORIS_CHECK(column != nullptr); - if (mapping.default_expr == nullptr) { - *column = - _detach_column(mapping.table_type->create_column_const_with_default_value(rows) - ->convert_to_full_column_if_const()); - return Status::OK(); - } - Block eval_block; - eval_block.insert({mapping.table_type->create_column_const_with_default_value(rows), - mapping.table_type, "__table_reader_nested_default_rows"}); - ColumnWithTypeAndName result; - RETURN_IF_ERROR(_execute_default_expr_without_root_type_check(mapping.default_expr, - &eval_block, &result)); - ColumnPtr result_column = result.column; - RETURN_IF_ERROR(_align_column_nullability(&result_column, mapping.table_type)); - *column = _detach_column(std::move(result_column)); - return Status::OK(); - } - Status _materialize_struct_mapping_column(const ColumnMapping& mapping, const ColumnPtr& file_column, const size_t rows, ColumnPtr* column) { @@ -1340,10 +1318,9 @@ class TableReader { for (const auto* child_mapping : table_ordered_children) { DORIS_CHECK(child_mapping != nullptr); if (!child_mapping->file_local_id.has_value()) { - ColumnPtr child_column; - RETURN_IF_ERROR(_materialize_missing_child_mapping_column(*child_mapping, rows, - &child_column)); - child_columns.push_back(std::move(child_column)); + child_columns.push_back( + child_mapping->table_type->create_column_const_with_default_value(rows) + ->convert_to_full_column_if_const()); continue; } const auto file_child_idx = @@ -1463,25 +1440,17 @@ class TableReader { return Status::OK(); } - Status _open_mapping_expr(const ColumnMapping& mapping, RowDescriptor& row_desc) { - if (mapping.projection != nullptr) { - RETURN_IF_ERROR(mapping.projection->prepare(_runtime_state, row_desc)); - RETURN_IF_ERROR(mapping.projection->open(_runtime_state)); - } - if (mapping.default_expr != nullptr) { - RETURN_IF_ERROR(mapping.default_expr->prepare(_runtime_state, row_desc)); - RETURN_IF_ERROR(mapping.default_expr->open(_runtime_state)); - } - for (const auto& child_mapping : mapping.child_mappings) { - RETURN_IF_ERROR(_open_mapping_expr(child_mapping, row_desc)); - } - return Status::OK(); - } - Status _open_mapping_exprs() { RowDescriptor row_desc; for (const auto& mapping : _data_reader.column_mapper->mappings()) { - RETURN_IF_ERROR(_open_mapping_expr(mapping, row_desc)); + if (mapping.projection != nullptr) { + RETURN_IF_ERROR(mapping.projection->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(mapping.projection->open(_runtime_state)); + } + if (mapping.default_expr != nullptr) { + RETURN_IF_ERROR(mapping.default_expr->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(mapping.default_expr->open(_runtime_state)); + } } return Status::OK(); } diff --git a/be/test/exec/scan/access_path_parser_test.cpp b/be/test/exec/scan/access_path_parser_test.cpp index 75007c317bc59d..d4bd6ab6c06360 100644 --- a/be/test/exec/scan/access_path_parser_test.cpp +++ b/be/test/exec/scan/access_path_parser_test.cpp @@ -32,8 +32,6 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/field.h" -#include "exprs/vexpr_context.h" -#include "exprs/vliteral.h" namespace doris { namespace { @@ -220,33 +218,6 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { } } -TEST(AccessPathParserTest, InheritsNestedInitialDefaultMetadata) { - auto string_type = std::make_shared(); - auto struct_type = std::make_shared(DataTypes {string_type}, Strings {"added"}); - auto added = field(101, "added", string_type); - added.initial_default_value = "AAEC/w=="; - added.initial_default_value_is_base64 = true; - added.default_expr = VExprContext::create_shared( - VLiteral::create_shared(string_type, Field::create_field("value"))); - format::ColumnDefinition schema { - .identifier = Field::create_field(100), - .name = "s", - .type = struct_type, - .children = {added}, - }; - - for (const auto& path : std::vector> {{"s"}, {"s", "added"}}) { - auto column = root_column(100, "s", struct_type); - auto status = AccessPathParser::build_nested_children( - &column, std::vector {data_access_path(path)}, &schema); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(column.children.size(), 1); - EXPECT_EQ(column.children[0].default_expr, added.default_expr); - EXPECT_EQ(column.children[0].initial_default_value, added.initial_default_value); - EXPECT_TRUE(column.children[0].initial_default_value_is_base64); - } -} - // Scenario: array access paths must pass through the "*" element token, then reuse struct child // parsing under the element wrapper; invalid array tokens are rejected. TEST(AccessPathParserTest, ArrayAccessPathMatrix) { diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index c13eeb8ec111ab..27271d155d0e45 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -98,33 +98,6 @@ void write_iceberg_int_orc_file(const std::string& file_path, const std::string& output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); } -void write_iceberg_struct_orc_file(const std::string& file_path) { - auto type = std::unique_ptr<::orc::Type>( - ::orc::Type::buildTypeFromString("struct>")); - type->getSubtype(0)->setAttribute("iceberg.id", "1"); - type->getSubtype(0)->getSubtype(0)->setAttribute("iceberg.id", "2"); - - MemoryOutputStream memory_stream(1024 * 1024); - ::orc::WriterOptions options; - options.setCompression(::orc::CompressionKind_NONE); - options.setMemoryPool(::orc::getDefaultPool()); - auto writer = ::orc::createWriter(*type, &memory_stream, options); - auto batch = writer->createRowBatch(2); - auto& root_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); - auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*root_batch.fields[0]); - auto& value_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); - value_batch.data[0] = 10; - value_batch.data[1] = 20; - root_batch.numElements = 2; - struct_batch.numElements = 2; - value_batch.numElements = 2; - writer->add(*batch); - writer->close(); - - std::ofstream output(file_path, std::ios::binary); - output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); -} - void write_iceberg_three_int_orc_file( const std::string& file_path, const std::string& first_name, std::optional first_id, const std::vector& first_values, @@ -175,36 +148,6 @@ std::shared_ptr build_iceberg_int32_array(const std::vectorWithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"2"})); - auto struct_type = arrow::struct_({child_field}); - arrow::StructBuilder builder( - struct_type, arrow::default_memory_pool(), - {std::make_shared(arrow::default_memory_pool())}); - auto* value_builder = assert_cast(builder.field_builder(0)); - DORIS_CHECK(builder.Append().ok()); - DORIS_CHECK(value_builder->Append(10).ok()); - DORIS_CHECK(builder.Append().ok()); - DORIS_CHECK(value_builder->Append(20).ok()); - - auto root_field = - arrow::field("s", struct_type, false) - ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"1"})); - std::shared_ptr struct_array; - DORIS_CHECK(builder.Finish(&struct_array).ok()); - auto table = arrow::Table::Make(arrow::schema({root_field}), {struct_array}); - auto output = arrow::io::FileOutputStream::Open(file_path); - DORIS_CHECK(output.ok()); - ::parquet::WriterProperties::Builder properties; - properties.version(::parquet::ParquetVersion::PARQUET_2_6); - properties.data_page_version(::parquet::ParquetDataPageVersion::V2); - properties.compression(::parquet::Compression::UNCOMPRESSED); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, - 2, properties.build())); -} - void write_iceberg_three_int_parquet_file( const std::string& file_path, const std::string& first_name, std::optional first_id, const std::vector& first_values, @@ -302,122 +245,6 @@ Status create_single_int_tuple_descriptor(ObjectPool* object_pool, const std::st return Status::OK(); } -Status create_struct_tuple_descriptor(ObjectPool* object_pool, DescriptorTbl** descriptor_table, - const TupleDescriptor** tuple_descriptor) { - TDescriptorTable thrift_table; - TTableDescriptor table_descriptor; - table_descriptor.__set_id(0); - table_descriptor.__set_tableType(TTableType::OLAP_TABLE); - table_descriptor.__set_numCols(1); - table_descriptor.__set_numClusteringCols(0); - thrift_table.__set_tableDescriptors({table_descriptor}); - - TTypeNode struct_node; - struct_node.__set_type(TTypeNodeType::STRUCT); - TStructField a_field; - a_field.__set_name("a"); - a_field.__set_contains_null(true); - TStructField b_field; - b_field.__set_name("b"); - b_field.__set_contains_null(true); - struct_node.__set_struct_fields({a_field, b_field}); - TTypeNode a_node; - a_node.__set_type(TTypeNodeType::SCALAR); - TScalarType a_scalar; - a_scalar.__set_type(TPrimitiveType::INT); - a_node.__set_scalar_type(a_scalar); - TTypeNode b_node = a_node; - TTypeDesc type; - type.__set_types({struct_node, a_node, b_node}); - - TSlotDescriptor slot_descriptor; - slot_descriptor.__set_id(0); - slot_descriptor.__set_parent(0); - slot_descriptor.__set_col_unique_id(1); - slot_descriptor.__set_slotType(type); - slot_descriptor.__set_columnPos(0); - slot_descriptor.__set_byteOffset(0); - slot_descriptor.__set_nullIndicatorByte(0); - slot_descriptor.__set_nullIndicatorBit(0); - slot_descriptor.__set_colName("s"); - slot_descriptor.__set_slotIdx(0); - slot_descriptor.__set_isMaterialized(true); - thrift_table.__set_slotDescriptors({slot_descriptor}); - - TTupleDescriptor tuple; - tuple.__set_id(0); - tuple.__set_byteSize(16); - tuple.__set_numNullBytes(1); - tuple.__set_tableId(0); - thrift_table.__set_tupleDescriptors({tuple}); - - RETURN_IF_ERROR(DescriptorTbl::create(object_pool, thrift_table, descriptor_table)); - *tuple_descriptor = (*descriptor_table)->get_tuple_descriptor(0); - return Status::OK(); -} - -schema::external::TSchema make_struct_schema_with_initial_default() { - TColumnType int_type; - int_type.__set_type(TPrimitiveType::INT); - auto make_int_field = [&int_type](const std::string& name, int32_t id, - std::optional initial_default) { - auto field = std::make_shared(); - field->__set_name(name); - field->__set_id(id); - field->__set_is_optional(true); - field->__set_type(int_type); - if (initial_default.has_value()) { - field->__set_initial_default_value(*initial_default); - } - schema::external::TFieldPtr ptr; - ptr.field_ptr = std::move(field); - ptr.__isset.field_ptr = true; - return ptr; - }; - auto a_field = make_int_field("a", 2, std::nullopt); - auto b_field = make_int_field("b", 3, "7"); - - auto struct_field = std::make_shared(); - struct_field->__set_name("s"); - struct_field->__set_id(1); - struct_field->__set_is_optional(true); - TColumnType struct_type; - struct_type.__set_type(TPrimitiveType::STRUCT); - struct_field->__set_type(struct_type); - schema::external::TStructField nested_fields; - nested_fields.__set_fields({a_field, b_field}); - struct_field->nestedField.__set_struct_field(std::move(nested_fields)); - struct_field->__isset.nestedField = true; - schema::external::TFieldPtr struct_ptr; - struct_ptr.field_ptr = std::move(struct_field); - struct_ptr.__isset.field_ptr = true; - - schema::external::TStructField root; - root.__set_fields({struct_ptr}); - schema::external::TSchema schema; - schema.__set_schema_id(-1); - schema.__set_root_field(std::move(root)); - return schema; -} - -void expect_struct_initial_default(const Block& block, size_t expected_rows) { - ASSERT_EQ(block.rows(), expected_rows); - const IColumn* root_column = block.get_by_position(0).column.get(); - if (const auto* nullable = check_and_get_column(*root_column)) { - root_column = &nullable->get_nested_column(); - } - const auto& struct_column = assert_cast(*root_column); - ASSERT_EQ(struct_column.tuple_size(), 2); - const auto& default_column = assert_cast(struct_column.get_column(1)); - const auto& values = - assert_cast(default_column.get_nested_column()).get_data(); - ASSERT_EQ(values.size(), expected_rows); - for (size_t row = 0; row < expected_rows; ++row) { - EXPECT_FALSE(default_column.is_null_at(row)); - EXPECT_EQ(values[row], 7); - } -} - schema::external::TFieldPtr make_external_int_field(const std::string& name, int32_t field_id, std::optional initial_default, std::vector aliases = {}) { @@ -1222,124 +1049,6 @@ TEST_F(IcebergReaderTest, v1_position_delete_read_error_releases_cache_entry) { EXPECT_NE(status.to_string().find(delete_file.path), std::string::npos); } -TEST_F(IcebergReaderTest, v1_parquet_materializes_nested_initial_default) { - const auto test_dir = - std::filesystem::temp_directory_path() / "doris_v1_parquet_nested_initial_default_test"; - std::filesystem::remove_all(test_dir); - std::filesystem::create_directories(test_dir); - const auto file_path = (test_dir / "data.parquet").string(); - write_iceberg_struct_parquet_file(file_path); - - TFileScanRangeParams scan_params; - scan_params.__set_file_type(TFileType::FILE_LOCAL); - scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); - scan_params.__set_current_schema_id(-1); - scan_params.__set_history_schema_info({make_struct_schema_with_initial_default()}); - TFileRangeDesc scan_range; - scan_range.__set_fs_name(""); - scan_range.__set_path(file_path); - scan_range.__set_start_offset(0); - scan_range.__set_size(static_cast(std::filesystem::file_size(file_path))); - scan_range.__set_file_size(static_cast(std::filesystem::file_size(file_path))); - - ObjectPool object_pool; - DescriptorTbl* descriptor_table = nullptr; - const TupleDescriptor* tuple_descriptor = nullptr; - ASSERT_TRUE(create_struct_tuple_descriptor(&object_pool, &descriptor_table, &tuple_descriptor) - .ok()); - RuntimeProfile profile("test_profile"); - RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; - io::IOContext io_ctx; - ShardedKVCache kv_cache(8); - IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &timezone_obj, - &io_ctx, &runtime_state, cache.get()); - io::FileReaderSPtr file_reader; - ASSERT_TRUE(io::global_local_filesystem()->open_file(file_path, &file_reader).ok()); - reader.set_file_reader(file_reader); - - std::vector column_descriptors(1); - column_descriptors[0].name = "s"; - std::unordered_map block_positions = {{"s", 0}}; - ParquetInitContext context; - context.column_descs = &column_descriptors; - context.col_name_to_block_idx = &block_positions; - context.tuple_descriptor = tuple_descriptor; - context.params = &scan_params; - context.range = &scan_range; - ASSERT_TRUE(reader.init_reader(&context).ok()); - - const auto int_type = make_nullable(std::make_shared()); - const auto struct_type = make_nullable( - std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"})); - Block block; - block.insert({struct_type->create_column(), struct_type, "s"}); - size_t read_rows = 0; - bool eof = false; - const auto status = reader.get_next_block(&block, &read_rows, &eof); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(read_rows, 2); - expect_struct_initial_default(block, 2); - std::filesystem::remove_all(test_dir); -} - -TEST_F(IcebergReaderTest, v1_orc_materializes_nested_initial_default) { - const auto test_dir = - std::filesystem::temp_directory_path() / "doris_v1_orc_nested_initial_default_test"; - std::filesystem::remove_all(test_dir); - std::filesystem::create_directories(test_dir); - const auto file_path = (test_dir / "data.orc").string(); - write_iceberg_struct_orc_file(file_path); - - TFileScanRangeParams scan_params; - scan_params.__set_file_type(TFileType::FILE_LOCAL); - scan_params.__set_format_type(TFileFormatType::FORMAT_ORC); - scan_params.__set_current_schema_id(-1); - scan_params.__set_history_schema_info({make_struct_schema_with_initial_default()}); - TFileRangeDesc scan_range; - scan_range.__set_fs_name(""); - scan_range.__set_path(file_path); - scan_range.__set_start_offset(0); - scan_range.__set_size(static_cast(std::filesystem::file_size(file_path))); - scan_range.__set_file_size(static_cast(std::filesystem::file_size(file_path))); - - ObjectPool object_pool; - DescriptorTbl* descriptor_table = nullptr; - const TupleDescriptor* tuple_descriptor = nullptr; - ASSERT_TRUE(create_struct_tuple_descriptor(&object_pool, &descriptor_table, &tuple_descriptor) - .ok()); - RuntimeProfile profile("test_profile"); - RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; - runtime_state.set_timezone("UTC"); - io::IOContext io_ctx; - ShardedKVCache kv_cache(8); - IcebergOrcReader reader(&kv_cache, &profile, &runtime_state, scan_params, scan_range, 1024, - "UTC", &io_ctx, cache.get()); - - std::vector column_descriptors(1); - column_descriptors[0].name = "s"; - std::unordered_map block_positions = {{"s", 0}}; - OrcInitContext context; - context.column_descs = &column_descriptors; - context.col_name_to_block_idx = &block_positions; - context.tuple_descriptor = tuple_descriptor; - context.params = &scan_params; - context.range = &scan_range; - ASSERT_TRUE(reader.init_reader(&context).ok()); - - const auto int_type = make_nullable(std::make_shared()); - const auto struct_type = make_nullable( - std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"})); - Block block; - block.insert({struct_type->create_column(), struct_type, "s"}); - size_t read_rows = 0; - bool eof = false; - const auto status = reader.get_next_block(&block, &read_rows, &eof); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(read_rows, 2); - expect_struct_initial_default(block, 2); - std::filesystem::remove_all(test_dir); -} - TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaults) { auto make_field = [](const std::string& name, int32_t id, TPrimitiveType::type primitive_type, const std::string& default_value, int32_t len, int32_t scale, @@ -1366,13 +1075,18 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul return field_ptr; }; + static_assert(sizeof("0123456789abcdef0123456789abcdef") - 1 > StringView::kInlineSize); + const std::string binary_default = "0123456789abcdef0123456789abcdef"; + schema::external::TStructField root_field; - root_field.__set_fields({make_field("added_timestamp", 1, TPrimitiveType::DATETIMEV2, - "2024-01-01 00:00:00.123456", -1, 6, false), - make_field("added_binary", 2, TPrimitiveType::VARBINARY, - "AAECAwQFBgcICQoLDA0ODw==", 16, -1, true), - make_field("added_string_binary", 3, TPrimitiveType::STRING, - "AAEC/w==", -1, -1, true)}); + root_field.__set_fields( + {make_field("added_timestamp", 1, TPrimitiveType::DATETIMEV2, + "2024-01-01 00:00:00.123456", -1, 6, false), + make_field("added_binary", 2, TPrimitiveType::VARBINARY, + "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", + static_cast(binary_default.size()), -1, true), + make_field("added_string_binary", 3, TPrimitiveType::STRING, + "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", -1, -1, true)}); schema::external::TSchema current_schema; current_schema.__set_schema_id(-1); current_schema.__set_root_field(root_field); @@ -1393,7 +1107,8 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul &runtime_state, cache.get()); const auto timestamp_type = make_nullable(std::make_shared(6)); - const auto varbinary_type = make_nullable(std::make_shared(16)); + const auto varbinary_type = make_nullable( + std::make_shared(static_cast(binary_default.size()))); const auto string_type = make_nullable(std::make_shared()); Block block; block.insert({timestamp_type->create_column(), timestamp_type, "added_timestamp"}); @@ -1414,16 +1129,10 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul ASSERT_TRUE(reader.TEST_materialize_missing_equality_delete_columns(&block, 3).ok()); ASSERT_EQ(block.rows(), 3); - for (size_t i = 0; i < block.rows(); ++i) { - SCOPED_TRACE(i); - EXPECT_EQ(timestamp_type->to_string(*block.get_by_position(0).column, i), - "2024-01-01 00:00:00.123456"); - EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(1).column, i), - std::string("\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - 16)); - EXPECT_EQ(string_type->to_string(*block.get_by_position(2).column, i), - std::string("\x00\x01\x02\xff", 4)); - } + EXPECT_EQ(timestamp_type->to_string(*block.get_by_position(0).column, 0), + "2024-01-01 00:00:00.123456"); + EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(1).column, 0), binary_default); + EXPECT_EQ(string_type->to_string(*block.get_by_position(2).column, 0), binary_default); EXPECT_FALSE(is_column_const(*block.get_by_position(0).column)); EXPECT_FALSE(is_column_const(*block.get_by_position(1).column)); EXPECT_FALSE(is_column_const(*block.get_by_position(2).column)); diff --git a/be/test/format/table/table_schema_change_helper_test.cpp b/be/test/format/table/table_schema_change_helper_test.cpp index 2a98e8422a8ae5..3fcb9d39878255 100644 --- a/be/test/format/table/table_schema_change_helper_test.cpp +++ b/be/test/format/table/table_schema_change_helper_test.cpp @@ -19,7 +19,6 @@ #include -#include #include #include #include @@ -426,78 +425,6 @@ TEST(MockTableSchemaChangeHelper, IcebergParquetNameMappingFallback) { " ScalarNode\n"); } -TEST(MockTableSchemaChangeHelper, IcebergNestedMissingFieldKeepsInitialDefault) { - TColumnType int_type; - int_type.__set_type(TPrimitiveType::INT); - TColumnType struct_type; - struct_type.__set_type(TPrimitiveType::STRUCT); - - auto make_field = [](const std::string& name, int32_t id, const TColumnType& type, - std::optional initial_default = std::nullopt) { - auto field = std::make_shared(); - field->__set_name(name); - field->__set_id(id); - field->__set_type(type); - field->__set_is_optional(true); - if (initial_default.has_value()) { - field->__set_initial_default_value(*initial_default); - } - schema::external::TFieldPtr ptr; - ptr.field_ptr = std::move(field); - ptr.__isset.field_ptr = true; - return ptr; - }; - - auto struct_field = make_field("s", 1, struct_type); - schema::external::TStructField nested_fields; - nested_fields.__set_fields({make_field("a", 2, int_type), make_field("b", 3, int_type, "7")}); - struct_field.field_ptr->nestedField.__set_struct_field(nested_fields); - struct_field.field_ptr->__isset.nestedField = true; - schema::external::TStructField table_schema; - table_schema.__set_fields({struct_field}); - - auto assert_initial_default = [](const std::shared_ptr& root) { - auto struct_node = root->get_children_node("s"); - ASSERT_FALSE(struct_node->children_column_exists("b")); - const auto& initial_default = struct_node->children_initial_default("b"); - ASSERT_NE(initial_default, nullptr); - Field value; - initial_default->get(0, value); - EXPECT_EQ(value.get(), 7); - }; - - FieldSchema file_a; - file_a.name = "a"; - file_a.field_id = 2; - file_a.data_type = DataTypeFactory::instance().create_data_type(TYPE_INT, true); - FieldSchema file_s; - file_s.name = "s"; - file_s.field_id = 1; - file_s.children = {file_a}; - file_s.data_type = - std::make_shared(DataTypes {file_a.data_type}, Strings {"a"}); - FieldDescriptor parquet_schema; - parquet_schema._fields = {file_s}; - parquet_schema.data_type = - std::make_shared(DataTypes {file_s.data_type}, Strings {"s"}); - std::shared_ptr parquet_node; - ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( - table_schema, parquet_schema, parquet_node) - .ok()); - assert_initial_default(parquet_node); - - auto orc_schema = - std::unique_ptr(orc::Type::buildTypeFromString("struct>")); - const auto& attribute = IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE; - orc_schema->getSubtype(0)->setAttribute(attribute, "1"); - orc_schema->getSubtype(0)->getSubtype(0)->setAttribute(attribute, "2"); - std::shared_ptr orc_node; - ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( - table_schema, orc_schema.get(), attribute, orc_node) - .ok()); - assert_initial_default(orc_node); -} - TEST(MockTableSchemaChangeHelper, IcebergOrcSchemaChange) { schema::external::TField test_field; TColumnType struct_type; diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index 38b9e7d6321cb5..3885deb107f072 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -3208,8 +3208,6 @@ TEST(ColumnMapperSchemaEvolutionTest, StructChildrenHandleMissingRenameReorderAn auto table_a = field_id_col("a", 1, int_type); auto table_renamed_b = field_id_col("renamed_b", 2, string_type); auto table_c = field_id_col("c", 3, int_type); - table_c.default_expr = - VExprContext::create_shared(literal(int_type, Field::create_field(7))); auto table_struct = struct_col("s", 10, {table_a, table_renamed_b, table_c}); auto v1_a = field_id_col("a", 1, int_type, 0); @@ -3231,9 +3229,6 @@ TEST(ColumnMapperSchemaEvolutionTest, StructChildrenHandleMissingRenameReorderAn EXPECT_EQ(*v1_mapping.child_mappings[0].file_local_id, 0); EXPECT_EQ(*v1_mapping.child_mappings[1].file_local_id, 1); EXPECT_FALSE(v1_mapping.child_mappings[2].file_local_id.has_value()); - EXPECT_EQ(v1_mapping.child_mappings[2].default_expr, table_c.default_expr); - EXPECT_FALSE(v1_mapping.child_mappings[2].constant_index.has_value()); - EXPECT_TRUE(v1_mapper.constant_map().empty()); ASSERT_EQ(v1_request.non_predicate_columns.size(), 1); EXPECT_EQ(v1_request.non_predicate_columns[0].column_id(), LocalColumnId(5)); EXPECT_TRUE(v1_request.non_predicate_columns[0].project_all_children); @@ -3248,7 +3243,6 @@ TEST(ColumnMapperSchemaEvolutionTest, StructChildrenHandleMissingRenameReorderAn EXPECT_EQ(*v2_mapping.child_mappings[0].file_local_id, 1); EXPECT_EQ(*v2_mapping.child_mappings[1].file_local_id, 0); EXPECT_EQ(*v2_mapping.child_mappings[2].file_local_id, 2); - EXPECT_EQ(v2_mapping.child_mappings[2].default_expr, nullptr); ASSERT_EQ(v2_request.non_predicate_columns.size(), 1); EXPECT_EQ(v2_request.non_predicate_columns[0].column_id(), LocalColumnId(8)); EXPECT_TRUE(v2_request.non_predicate_columns[0].project_all_children); diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 0831c98750ecbf..7b67063a7fd63c 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -2240,7 +2240,8 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor const auto file_path = (test_dir / "split.parquet").string(); const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); - const std::string binary_default("\x00\x01\x02\xff", 4); + static_assert(sizeof("0123456789abcdef0123456789abcdef") - 1 > StringView::kInlineSize); + const std::string binary_default = "0123456789abcdef0123456789abcdef"; write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, "added_binary"); @@ -2249,10 +2250,12 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor auto scan_params = make_local_parquet_scan_params(); scan_params.__set_current_schema_id(100); scan_params.__set_history_schema_info({external_schema( - 100, - {external_schema_field("id", 0), - external_schema_field("added_binary", 1, {}, "AAEC/w==", - external_primitive_type(TPrimitiveType::VARBINARY, 4), true)})}); + 100, {external_schema_field("id", 0), + external_schema_field( + "added_binary", 1, {}, "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", + external_primitive_type(TPrimitiveType::VARBINARY, + static_cast(binary_default.size())), + true)})}); RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index f0931ea152d2fd..b5ec1dfc765053 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -48,7 +48,6 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" -#include "exec/scan/access_path_parser.h" #include "exprs/runtime_filter_expr.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" @@ -2894,7 +2893,7 @@ TEST(TableReaderTest, ProjectedListStructReordersRenamedAndMissingElementChildre // Scenario: when every projected array-element struct child is missing/default-only, the reader // still receives a full element projection and can materialize the default child without crashing. -TEST(TableReaderTest, ProjectedListStructOnlyMissingElementChildUsesInitialDefault) { +TEST(TableReaderTest, ProjectedListStructOnlyMissingElementChildFallsBackToFullElement) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_list_only_missing_child_test"; std::filesystem::remove_all(test_dir); @@ -2905,8 +2904,6 @@ TEST(TableReaderTest, ProjectedListStructOnlyMissingElementChildUsesInitialDefau const auto string_type = std::make_shared(); auto missing_child = make_table_column(99, "missing_child", string_type); - missing_child.default_expr = VExprContext::create_shared(VLiteral::create_shared( - missing_child.type, Field::create_field("fallback"))); auto element_type = std::make_shared(DataTypes {string_type}, Strings {"missing_child"}); auto nullable_element_type = make_nullable(element_type); @@ -2946,12 +2943,7 @@ TEST(TableReaderTest, ProjectedListStructOnlyMissingElementChildUsesInitialDefau const auto& element_struct = assert_cast(nullable_elements.get_nested_column()); ASSERT_EQ(element_struct.get_columns().size(), 1); - const auto& missing_values = assert_cast( - expect_not_null_nullable_nested_column(element_struct.get_column(0))); - ASSERT_EQ(missing_values.size(), 4); - for (size_t row = 0; row < missing_values.size(); ++row) { - EXPECT_EQ(missing_values.get_data_at(row).to_string(), "fallback"); - } + expect_nullable_column_all_null(element_struct.get_column(0)); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); @@ -3044,8 +3036,6 @@ TEST(TableReaderTest, ProjectedMapValueStructReordersRenamedAndMissingChildren) auto b_child = make_table_column(1, "renamed_b", nullable_string_type); b_child.name_mapping = {"b"}; auto missing_child = make_table_column(99, "missing_child", string_type); - missing_child.default_expr = VExprContext::create_shared(VLiteral::create_shared( - missing_child.type, Field::create_field("fallback"))); auto a_child = make_table_column(0, "renamed_a", nullable_int_type); a_child.name_mapping = {"a"}; auto value_type = std::make_shared( @@ -3094,17 +3084,13 @@ TEST(TableReaderTest, ProjectedMapValueStructReordersRenamedAndMissingChildren) ASSERT_EQ(value_struct.get_columns().size(), 3); const auto& b_values = assert_cast( expect_not_null_nullable_nested_column(value_struct.get_column(0))); - const auto& missing_values = assert_cast( - expect_not_null_nullable_nested_column(value_struct.get_column(1))); + const auto& missing_values = value_struct.get_column(1); const auto& a_values = assert_cast( expect_not_null_nullable_nested_column(value_struct.get_column(2))); EXPECT_EQ(b_values.get_data_at(0).to_string(), "ma"); EXPECT_EQ(b_values.get_data_at(1).to_string(), "mb"); EXPECT_EQ(b_values.get_data_at(2).to_string(), "mc"); - ASSERT_EQ(missing_values.size(), 3); - for (size_t row = 0; row < missing_values.size(); ++row) { - EXPECT_EQ(missing_values.get_data_at(row).to_string(), "fallback"); - } + expect_nullable_column_all_null(missing_values); EXPECT_EQ(a_values.get_element(0), 10); EXPECT_EQ(a_values.get_element(1), 20); EXPECT_EQ(a_values.get_element(2), 30); @@ -4044,7 +4030,7 @@ TEST(TableReaderTest, ProjectedColumnsFillMissingParquetColumnWithDefault) { std::filesystem::remove_all(test_dir); } -TEST(TableReaderTest, ProjectedStructFillsMissingChildWithInitialDefault) { +TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_struct_missing_child_test"; std::filesystem::remove_all(test_dir); @@ -4055,28 +4041,12 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithInitialDefault) { const auto int_type = std::make_shared(); const auto string_type = std::make_shared(); + auto id_child = make_table_column(0, "id", int_type); + auto missing_child = make_table_column(99, "missing_child", string_type); auto struct_type = std::make_shared(DataTypes {int_type, string_type}, Strings {"id", "missing_child"}); auto struct_column = make_table_column(100, "s", struct_type); - auto missing_field = external_schema_field("missing_child", 99); - missing_field.field_ptr->__set_initial_default_value("fallback"); - TFileScanRangeParams scan_params; - scan_params.__set_current_schema_id(200); - scan_params.__set_history_schema_info({external_schema( - 200, - {external_struct_field("s", 100, {external_schema_field("id", 0), missing_field})})}); - ProjectedColumnBuildContext build_context {.scan_params = &scan_params}; - TFileScanSlotInfo slot_info; - TableReader schema_reader; - ASSERT_TRUE(schema_reader.annotate_projected_column(slot_info, &build_context, &struct_column) - .ok()); - ASSERT_TRUE(build_context.schema_column.has_value()); - ASSERT_TRUE(AccessPathParser::build_nested_children(&struct_column, - std::vector {}, - &*build_context.schema_column) - .ok()); - ASSERT_EQ(struct_column.children.size(), 2); - ASSERT_NE(struct_column.children[1].default_expr, nullptr); + struct_column.children = {id_child, missing_child}; std::vector projected_columns = {struct_column}; RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -4086,7 +4056,7 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithInitialDefault) { .projected_columns = projected_columns, .conjuncts = {}, .format = FileFormat::PARQUET, - .scan_params = &scan_params, + .scan_params = nullptr, .io_ctx = nullptr, .runtime_state = &state, .scanner_profile = nullptr, @@ -4107,9 +4077,7 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithInitialDefault) { expect_not_null_nullable_nested_column(struct_result.get_column(0))); ASSERT_EQ(struct_result.size(), 1); EXPECT_EQ(ids.get_element(0), 7); - const auto& missing_values = - expect_not_null_nullable_nested_column(struct_result.get_column(1)); - EXPECT_EQ(missing_values.get_data_at(0).to_string(), "fallback"); + expect_nullable_column_all_null(struct_result.get_column(1)); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java index d5ac42c6365c1f..acbd760a2bf057 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java @@ -610,10 +610,6 @@ public String getDefaultValue() { return this.defaultValue; } - public void setDefaultValue(String defaultValue) { - this.defaultValue = defaultValue; - } - public String getRealDefaultValue() { return realDefaultValue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index f09d2c2c49e110..c5b056b5d318f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.analysis.ColumnPath; -import org.apache.doris.analysis.DefaultValueExprDef; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.ColumnType; @@ -817,8 +816,7 @@ private void addOneColumn(UpdateSchema updateSchema, Column column) throws UserE throw new UserException("can't add a non-nullable column to an Iceberg table"); } org.apache.iceberg.types.Type dorisType = IcebergUtils.dorisTypeToIcebergType(column.getType()); - Literal defaultValue = IcebergUtils.parseIcebergLiteral( - column.getDefaultValue(), column.getType(), dorisType); + Literal defaultValue = IcebergUtils.parseIcebergLiteral(column.getDefaultValue(), dorisType); updateSchema.addColumn(column.getName(), dorisType, column.getComment(), defaultValue); } @@ -878,7 +876,7 @@ private void refreshTable(ExternalTable dorisTable, long updateTime) { @Override public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { - validateAddColumnMetadata(column); + validateCommonColumnInfo(column, true); Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); Schema schema = icebergTable.schema(); @@ -905,7 +903,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co addColumn(dorisTable, column, position, updateTime); return; } - validateAddColumnMetadata(column); + validateNestedAddColumnMetadata(column, columnPath); if (!column.isAllowNull()) { throw new UserException("New nested field '" + columnPath.getFullPath() + "' must be nullable"); } @@ -920,10 +918,8 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co UpdateSchema updateSchema = icebergTable.updateSchema(); org.apache.iceberg.types.Type dorisType = IcebergUtils.dorisTypeToIcebergType(column.getType()); - Literal defaultValue = IcebergUtils.parseIcebergLiteral( - column.getDefaultValue(), column.getType(), dorisType); updateSchema.addColumn(parentPath.getFullPath(), columnPath.getLeafName(), dorisType, - column.getComment(), defaultValue); + column.getComment()); if (position != null) { applyPosition(updateSchema, position, childPath(parentPath.getColumnPath(), columnPath.getLeafName()), icebergTable.schema(), "add"); @@ -941,7 +937,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); for (Column column : columns) { - validateAddColumnMetadata(column); + validateCommonColumnInfo(column, true); validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); } validateNoCaseInsensitiveTopLevelCollisions(icebergTable.schema(), columns); @@ -1057,7 +1053,7 @@ private void modifyTopLevelColumn(ExternalTable dorisTable, Column column, Colum ResolvedColumnPath columnPath = new ResolvedColumnPath(ColumnPath.of(currentCol.name()), currentCol.type(), currentCol); - validateModifyColumnMetadata(column, columnPath.getFullPath(), !legacyMode); + validateCommonColumnInfo(column, !legacyMode); UpdateSchema updateSchema = icebergTable.updateSchema(); if (column.getType().isComplexType()) { @@ -1103,7 +1099,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column validatePositionTarget(icebergTable.schema(), resolvedPath.getColumnPath(), "modify"); } - validateModifyColumnMetadata(column, resolvedPath.getFullPath(), true); + validateNestedModifyColumnMetadata(column, resolvedPath.getFullPath()); UpdateSchema updateSchema = icebergTable.updateSchema(); if (column.getType().isComplexType()) { @@ -1534,26 +1530,16 @@ private void validateCommonColumnInfo(Column column, boolean rejectKey) throws U } } - private void validateAddColumnMetadata(Column column) throws UserException { + private void validateNestedAddColumnMetadata(Column column, ColumnPath columnPath) throws UserException { validateCommonColumnInfo(column, true); - if (column.hasOnUpdateDefaultValue()) { - throw new UserException("ON UPDATE is not supported for Iceberg ADD COLUMN: " + column.getName()); - } - if (column.getType().isComplexType() && column.getDefaultValue() != null) { - throw new UserException("Complex type default value only supports NULL"); - } - DefaultValueExprDef defaultValueExpr = column.getDefaultValueExprDef(); - if (defaultValueExpr != null - && ("now".equalsIgnoreCase(defaultValueExpr.getExprName()) - || "current_date".equalsIgnoreCase(defaultValueExpr.getExprName()))) { - throw new UserException("Dynamic default value " + column.getDefaultValue() - + " is not supported for Iceberg ADD COLUMN: " + column.getName()); + if (column.hasDefaultValue() || column.hasOnUpdateDefaultValue()) { + throw new UserException("DEFAULT and ON UPDATE are not supported for Iceberg nested ADD COLUMN: " + + columnPath.getFullPath()); } } - private void validateModifyColumnMetadata(Column column, String columnPath, boolean rejectKey) - throws UserException { - validateCommonColumnInfo(column, rejectKey); + private void validateNestedModifyColumnMetadata(Column column, String columnPath) throws UserException { + validateCommonColumnInfo(column, true); if (column.hasDefaultValue() || column.hasOnUpdateDefaultValue()) { throw new UserException("Modifying default values is not supported for Iceberg columns: " + columnPath); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 0b791411f15623..12b5dce11395fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -22,7 +22,6 @@ import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.CompoundPredicate; import org.apache.doris.analysis.DateLiteral; -import org.apache.doris.analysis.DateLiteralUtils; import org.apache.doris.analysis.DecimalLiteral; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToExprNameVisitor; @@ -65,9 +64,6 @@ import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.nereids.trees.expressions.literal.Result; -import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; -import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.VarBinaryType; import org.apache.doris.nereids.util.DateUtils; import org.apache.doris.persist.gson.GsonUtils; @@ -954,39 +950,32 @@ public static org.apache.iceberg.types.Type dorisTypeToIcebergType(Type type) { } public static Literal parseIcebergLiteral(String value, org.apache.iceberg.types.Type type) { - return parseIcebergLiteral(value, null, type); - } - - public static Literal parseIcebergLiteral( - String value, Type dorisType, org.apache.iceberg.types.Type type) { if (value == null) { return null; } switch (type.typeId()) { case BOOLEAN: try { - return Literal.of(new BoolLiteral(value).getValue()); - } catch (AnalysisException e) { + return Literal.of(Boolean.parseBoolean(value)); + } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Invalid Boolean string: " + value, e); } case INTEGER: + case DATE: try { return Literal.of(Integer.parseInt(value)); } catch (NumberFormatException e) { throw new IllegalArgumentException("Invalid Int string: " + value, e); } case LONG: + case TIME: + case TIMESTAMP: + case TIMESTAMP_NANO: try { return Literal.of(Long.parseLong(value)); } catch (NumberFormatException e) { throw new IllegalArgumentException("Invalid Long string: " + value, e); } - case DATE: - case TIMESTAMP: - case TIMESTAMP_NANO: - return parseIcebergDateLiteral(value, dorisType, type); - case TIME: - return parseIcebergTemporalLiteral(value, value, type); case FLOAT: try { return Literal.of(Float.parseFloat(value)); @@ -1023,37 +1012,6 @@ public static Literal parseIcebergLiteral( } } - private static Literal parseIcebergDateLiteral( - String value, Type dorisType, org.apache.iceberg.types.Type type) { - String canonicalValue = value; - if (dorisType != null) { - try { - if (dorisType.isTimeStampTz()) { - TimeStampTzType timestampTzType = (TimeStampTzType) DataType.fromCatalogType(dorisType); - canonicalValue = TimestampTzLiteral.fromSessionTimeZone(timestampTzType, value).getStringValue(); - } else { - canonicalValue = DateLiteralUtils.createDateLiteral(value, dorisType).getStringValue(); - } - } catch (AnalysisException | RuntimeException e) { - throw new IllegalArgumentException("Invalid date or timestamp string: " + value, e); - } - } - return parseIcebergTemporalLiteral(value, canonicalValue.replace(' ', 'T'), type); - } - - private static Literal parseIcebergTemporalLiteral( - String originalValue, String canonicalValue, org.apache.iceberg.types.Type type) { - try { - Literal literal = Literal.of(canonicalValue).to(type); - if (literal == null) { - throw new IllegalArgumentException("Cannot convert value to Iceberg type " + type); - } - return literal; - } catch (RuntimeException e) { - throw new IllegalArgumentException("Invalid temporal string: " + originalValue, e); - } - } - /** * Convert human-readable partition value string to appropriate Java type for * Iceberg expression. @@ -1157,13 +1115,8 @@ private static long parseTimestampToMicros(String valueStr, TimestampType timest return epochSecond * 1_000_000L + microSecond; } - private static void updateIcebergColumnMetadata(Column column, Types.NestedField icebergField, - boolean enableMappingTimestampTz) { + private static void updateIcebergColumnUniqueId(Column column, Types.NestedField icebergField) { column.setUniqueId(icebergField.fieldId()); - if (icebergField.initialDefault() != null) { - column.setDefaultValue(serializeInitialDefault( - icebergField.type(), icebergField.initialDefault(), enableMappingTimestampTz)); - } List icebergFields = Lists.newArrayList(); switch (icebergField.type().typeId()) { case LIST: @@ -1182,8 +1135,7 @@ private static void updateIcebergColumnMetadata(Column column, Types.NestedField if (column.getChildren() != null) { List childColumns = column.getChildren(); for (int idx = 0; idx < childColumns.size(); idx++) { - updateIcebergColumnMetadata( - childColumns.get(idx), icebergFields.get(idx), enableMappingTimestampTz); + updateIcebergColumnUniqueId(childColumns.get(idx), icebergFields.get(idx)); } } } @@ -1232,10 +1184,15 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi List columns = schema.columns(); List resSchema = Lists.newArrayListWithCapacity(columns.size()); for (Types.NestedField field : columns) { + String initialDefault = null; + if (field.initialDefault() != null) { + initialDefault = serializeInitialDefault(field.type(), field.initialDefault(), + enableMappingTimestampTz); + } Column column = new Column(field.name(), IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz), - true, null, true, null, field.doc(), true, -1); - updateIcebergColumnMetadata(column, field, enableMappingTimestampTz); + true, null, true, initialDefault, field.doc(), true, -1); + updateIcebergColumnUniqueId(column, field); if (field.type().isPrimitiveType() && field.type().typeId() == TypeID.TIMESTAMP) { Types.TimestampType timestampType = (Types.TimestampType) field.type(); if (timestampType.shouldAdjustToUTC()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index ef17dfc872f6c7..c622e70e880879 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -4219,9 +4219,8 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { e.getCause()); } } - //comment should remove '\' and '(") at the beginning and end - String comment = ctx.comment != null ? ctx.comment.getText().substring(1, ctx.comment.getText().length() - 1) - .replace("\\", "") : ""; + String comment = ctx.comment != null + ? LogicalPlanBuilderAssistant.parseStringLiteral(ctx.comment.getText()) : ""; long autoIncInitValue = -1; if (ctx.AUTO_INCREMENT() != null) { if (ctx.autoIncInitValue != null) { @@ -4244,7 +4243,14 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { @Override public ColumnDefinitionWithPath visitColumnDefWithPath(ColumnDefWithPathContext ctx) { - ColumnPath columnPath = ColumnPath.of(visitQualifiedName(ctx.colName)); + if (ctx.columnDef() != null) { + ColumnDefinition columnDefinition = visitColumnDef(ctx.columnDef()); + return new ColumnDefinitionWithPath(columnDefinition, ColumnPath.of(columnDefinition.getName())); + } + + ColumnPath columnPath = ColumnPath.of(ctx.colNames.stream() + .map(RuleContext::getText) + .collect(Collectors.toList())); String colName = columnPath.getLeafName(); DataType colType = ctx.type instanceof PrimitiveDataTypeContext ? visitPrimitiveDataType(((PrimitiveDataTypeContext) ctx.type)) @@ -4262,53 +4268,6 @@ public ColumnDefinitionWithPath visitColumnDefWithPath(ColumnDefWithPathContext nullableType = ColumnNullableType.NULLABLE; } String aggTypeString = ctx.aggType != null ? ctx.aggType.getText() : null; - Optional defaultValue = Optional.empty(); - Optional onUpdateDefaultValue = Optional.empty(); - if (ctx.DEFAULT() != null) { - if (ctx.INTEGER_VALUE() != null) { - if (ctx.SUBTRACT() == null) { - defaultValue = Optional.of(new DefaultValue(ctx.INTEGER_VALUE().getText())); - } else { - defaultValue = Optional.of(new DefaultValue("-" + ctx.INTEGER_VALUE().getText())); - } - } else if (ctx.DECIMAL_VALUE() != null) { - if (ctx.SUBTRACT() == null) { - defaultValue = Optional.of(new DefaultValue(ctx.DECIMAL_VALUE().getText())); - } else { - defaultValue = Optional.of(new DefaultValue("-" + ctx.DECIMAL_VALUE().getText())); - } - } else if (ctx.stringValue != null) { - defaultValue = Optional.of(new DefaultValue( - LogicalPlanBuilderAssistant.parseStringLiteral(ctx.stringValue.getText()))); - } else if (ctx.nullValue != null) { - defaultValue = Optional.of(DefaultValue.NULL_DEFAULT_VALUE); - } else if (ctx.defaultTimestamp != null) { - if (ctx.defaultValuePrecision == null) { - defaultValue = Optional.of(DefaultValue.CURRENT_TIMESTAMP_DEFAULT_VALUE); - } else { - defaultValue = Optional.of(DefaultValue - .currentTimeStampDefaultValueWithPrecision( - Long.valueOf(ctx.defaultValuePrecision.getText()))); - } - } else if (ctx.CURRENT_DATE() != null) { - defaultValue = Optional.of(DefaultValue.CURRENT_DATE_DEFAULT_VALUE); - } else if (ctx.PI() != null) { - defaultValue = Optional.of(DefaultValue.PI_DEFAULT_VALUE); - } else if (ctx.E() != null) { - defaultValue = Optional.of(DefaultValue.E_NUM_DEFAULT_VALUE); - } else if (ctx.BITMAP_EMPTY() != null) { - defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); - } - } - if (ctx.UPDATE() != null) { - if (ctx.onUpdateValuePrecision == null) { - onUpdateDefaultValue = Optional.of(DefaultValue.CURRENT_TIMESTAMP_DEFAULT_VALUE); - } else { - onUpdateDefaultValue = Optional.of(DefaultValue - .currentTimeStampDefaultValueWithPrecision( - Long.valueOf(ctx.onUpdateValuePrecision.getText()))); - } - } AggregateType aggType = null; if (aggTypeString != null) { try { @@ -4335,7 +4294,7 @@ public ColumnDefinitionWithPath visitColumnDefWithPath(ColumnDefWithPathContext ? Optional.of(new GeneratedColumnDesc(ctx.generatedExpr.getText(), getExpression(ctx.generatedExpr))) : Optional.empty(); ColumnDefinition columnDefinition = new ColumnDefinition(colName, colType, isKey, aggType, nullableType, - autoIncInitValue, defaultValue, onUpdateDefaultValue, comment, desc); + autoIncInitValue, Optional.empty(), Optional.empty(), comment, desc); return new ColumnDefinitionWithPath(columnDefinition, columnPath); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java index d938ccf9aaed89..db0df4ad2ebf6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java @@ -153,13 +153,6 @@ static void checkColumnOperationsSupported(TableIf table, List alt throw new AnalysisException("New nested field '" + nestedColumnPath.getFullPath() + "' must be nullable"); } - // Column translation cannot distinguish an omitted default from an explicit DEFAULT NULL. - if (alterTableOp instanceof ModifyColumnOp && columnDefinition != null - && (columnDefinition.hasDefaultValue() - || columnDefinition.hasOnUpdateDefaultValue())) { - throw new AnalysisException( - "DEFAULT and ON UPDATE are not supported for Iceberg MODIFY COLUMN"); - } if (!isIcebergColumnSchemaOperation(alterTableOp)) { continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 2a8eec3731e84f..2c590b6e73ac36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -27,6 +27,7 @@ import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.util.SqlUtils; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.parser.LogicalPlanBuilderAssistant; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BitmapType; @@ -189,10 +190,6 @@ public boolean hasDefaultValue() { return defaultValue.isPresent(); } - public boolean hasOnUpdateDefaultValue() { - return onUpdateDefaultValue.isPresent(); - } - public boolean isVisible() { return isVisible; } @@ -265,7 +262,7 @@ public String toSql(String columnNameSql) { sb.append("DEFAULT ").append("NULL").append(" "); } } - sb.append("COMMENT \"").append(SqlUtils.escapeQuota(comment)).append("\""); + sb.append("COMMENT ").append(LogicalPlanBuilderAssistant.quoteStringLiteral(getComment())); return sb.toString(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 20e4a36d7d1f1d..3ecb3a72d6b023 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -223,17 +223,8 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { col1.setUniqueId(101); Column col2 = new Column("c2", Type.VARCHAR, false); col2.setUniqueId(102); - StructType structType = new StructType( - new StructField("added_child", Type.INT), - new StructField("added_binary", Type.VARBINARY)); - Column col3 = new Column("c3", structType, true); - col3.setUniqueId(103); - col3.getChildren().get(0).setUniqueId(104); - col3.getChildren().get(0).setDefaultValue("9"); - col3.getChildren().get(1).setUniqueId(105); - col3.getChildren().get(1).setDefaultValue("ignored by Base64 transport"); - List columns = Arrays.asList(col1, col2, col3); + List columns = Arrays.asList(col1, col2); Map> nameMapping = new HashMap<>(); nameMapping.put(col1.getUniqueId(), Arrays.asList("m_c1")); @@ -241,7 +232,6 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Map base64InitialDefaults = new HashMap<>(); base64InitialDefaults.put(col2.getUniqueId(), "AAEC/w=="); - base64InitialDefaults.put(105, "BAUGBw=="); ExternalUtil.initSchemaInfoForAllColumn( params, schemaId, columns, nameMapping, base64InitialDefaults); @@ -254,11 +244,10 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { TStructField rootField = tSchema.getRootField(); Assert.assertNotNull(rootField); - Assert.assertEquals(3, rootField.getFieldsSize()); + Assert.assertEquals(2, rootField.getFieldsSize()); TField field1 = rootField.getFields().get(0).getFieldPtr(); TField field2 = rootField.getFields().get(1).getFieldPtr(); - TField field3 = rootField.getFields().get(2).getFieldPtr(); Assert.assertEquals(col1.getName(), field1.getName()); Assert.assertEquals(col1.getUniqueId(), field1.getId()); @@ -275,15 +264,5 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(Arrays.asList("m_c2_a", "m_c2_b"), field2.getNameMapping()); Assert.assertEquals("AAEC/w==", field2.getInitialDefaultValue()); Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); - - TField nestedField = field3.getNestedField().getStructField().getFields().get(0).getFieldPtr(); - Assert.assertEquals(104, nestedField.getId()); - Assert.assertEquals("9", nestedField.getInitialDefaultValue()); - Assert.assertFalse(nestedField.isSetInitialDefaultValueIsBase64()); - - TField nestedBinary = field3.getNestedField().getStructField().getFields().get(1).getFieldPtr(); - Assert.assertEquals(105, nestedBinary.getId()); - Assert.assertEquals("BAUGBw==", nestedBinary.getInitialDefaultValue()); - Assert.assertTrue(nestedBinary.isInitialDefaultValueIsBase64()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index df52230badf405..a45f7fd93d19f3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.analysis.ColumnPath; -import org.apache.doris.analysis.DefaultValueExprDef; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MapType; @@ -37,7 +36,6 @@ import org.apache.iceberg.UpdateSchema; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.SupportsNamespaces; -import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; import org.junit.Assert; @@ -48,9 +46,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Collections; import java.util.Optional; @@ -403,92 +398,31 @@ public void testExplicitNullableModifyMakesRequiredFieldsOptional() throws Throw } @Test - public void testModifyColumnRejectsDefaultMetadata() { - Schema schema = nestedSchema(); + public void testNestedColumnOperationsRejectDefaultMetadata() { + Schema schema = new Schema(Types.NestedField.optional(1, "s", Types.StructType.of( + Types.NestedField.optional(2, "existing", Types.IntegerType.get())))); ExternalTable dorisTable = Mockito.mock(ExternalTable.class); Table icebergTable = Mockito.mock(Table.class); - UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); Mockito.when(icebergTable.schema()).thenReturn(schema); - Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); - Column topLevelDefaultColumn = new Column("id", Type.BIGINT, false, null, true, "7", ""); - Column nestedDefaultColumn = new Column("a", Type.BIGINT, false, null, true, "7", ""); - Column topLevelOnUpdateColumn = Mockito.spy(new Column("id", Type.BIGINT, true)); - Column nestedOnUpdateColumn = Mockito.spy(new Column("a", Type.BIGINT, true)); - Mockito.doReturn(true).when(topLevelOnUpdateColumn).hasOnUpdateDefaultValue(); + Column nestedAddDefaultColumn = new Column("new_col", Type.BIGINT, false, null, true, "7", ""); + Column nestedDefaultColumn = new Column("existing", Type.BIGINT, false, null, true, "7", ""); + Column nestedOnUpdateColumn = Mockito.spy(new Column("existing", Type.BIGINT, true)); Mockito.doReturn(true).when(nestedOnUpdateColumn).hasOnUpdateDefaultValue(); try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); - assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("id"), - topLevelDefaultColumn, null, 1L), - "Modifying default values is not supported for Iceberg columns: id"); - assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("id"), - topLevelOnUpdateColumn, null, 1L), - "Modifying default values is not supported for Iceberg columns: id"); - assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("s.a"), + assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("s.new_col"), + nestedAddDefaultColumn, null, 1L), + "DEFAULT and ON UPDATE are not supported for Iceberg nested ADD COLUMN: s.new_col"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("s.existing"), nestedDefaultColumn, null, 1L), - "Modifying default values is not supported for Iceberg columns: s.a"); - assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("s.a"), + "Modifying default values is not supported for Iceberg columns: s.existing"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("s.existing"), nestedOnUpdateColumn, null, 1L), - "Modifying default values is not supported for Iceberg columns: s.a"); - } - - Mockito.verifyNoInteractions(updateSchema); - } - - @Test - public void testAddColumnRejectsOnUpdateMetadata() { - ExternalTable dorisTable = Mockito.mock(ExternalTable.class); - Table icebergTable = Mockito.mock(Table.class); - Column onUpdateColumn = Mockito.spy(new Column("new_col", Type.DATETIME, true)); - Mockito.doReturn(true).when(onUpdateColumn).hasOnUpdateDefaultValue(); - - try (MockedStatic mockedIcebergUtils = - Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); - - assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.of("new_col"), - onUpdateColumn, null, 1L), - "ON UPDATE is not supported for Iceberg ADD COLUMN: new_col"); - assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("s.new_col"), - onUpdateColumn, null, 1L), - "ON UPDATE is not supported for Iceberg ADD COLUMN: new_col"); - assertUserException(() -> ops.addColumns(dorisTable, Collections.singletonList(onUpdateColumn), 1L), - "ON UPDATE is not supported for Iceberg ADD COLUMN: new_col"); - } - - Mockito.verify(icebergTable, Mockito.never()).updateSchema(); - } - - @Test - public void testAddColumnRejectsUnsupportedDefaultsBeforeUpdateSchema() { - ExternalTable dorisTable = Mockito.mock(ExternalTable.class); - Table icebergTable = Mockito.mock(Table.class); - Column currentTimestamp = Mockito.spy(new Column("event_time", Type.DATETIME, false, null, true, - "CURRENT_TIMESTAMP", "")); - Column currentDate = Mockito.spy(new Column("event_date", Type.DATEV2, false, null, true, - "CURRENT_DATE", "")); - Column complexDefault = new Column("items", ArrayType.create(Type.INT, true), false, null, true, - "[]", ""); - Mockito.doReturn(new DefaultValueExprDef("now")) - .when(currentTimestamp).getDefaultValueExprDef(); - Mockito.doReturn(new DefaultValueExprDef("current_date")) - .when(currentDate).getDefaultValueExprDef(); - - try (MockedStatic mockedIcebergUtils = - Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); - - assertUserException(() -> ops.addColumn(dorisTable, currentTimestamp, null, 1L), - "Dynamic default value CURRENT_TIMESTAMP is not supported for Iceberg ADD COLUMN: event_time"); - assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("s.event_date"), - currentDate, null, 1L), - "Dynamic default value CURRENT_DATE is not supported for Iceberg ADD COLUMN: event_date"); - assertUserException(() -> ops.addColumn(dorisTable, complexDefault, null, 1L), - "Complex type default value only supports NULL"); + "Modifying default values is not supported for Iceberg columns: s.existing"); } Mockito.verify(icebergTable, Mockito.never()).updateSchema(); @@ -528,48 +462,6 @@ public void testRejectKeyAndGeneratedMetadataBeforeUpdateSchema() { Mockito.verify(icebergTable, Mockito.never()).updateSchema(); } - @Test - public void testNestedAddColumnConvertsDorisDefaultsToIcebergLiterals() throws Throwable { - Schema schema = new Schema(Types.NestedField.optional(1, "s", Types.StructType.of( - Types.NestedField.optional(2, "existing", Types.IntegerType.get())))); - ExternalTable dorisTable = Mockito.mock(ExternalTable.class); - Table icebergTable = Mockito.mock(Table.class); - UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); - Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); - Mockito.when(icebergTable.schema()).thenReturn(schema); - Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); - - Column flag = new Column("flag", Type.BOOLEAN, false, null, true, "1", ""); - Column eventDate = new Column("event_date", Type.DATEV2, false, null, true, - "2024-01-02", ""); - Column eventTime = new Column("event_time", ScalarType.createDatetimeV2Type(6), false, null, true, - "2024-01-02 03:04:05.123456", ""); - - try (MockedStatic mockedIcebergUtils = - Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); - - ops.addColumn(dorisTable, ColumnPath.fromDotName("s.flag"), flag, null, 1L); - ops.addColumn(dorisTable, ColumnPath.fromDotName("s.event_date"), eventDate, null, 1L); - ops.addColumn(dorisTable, ColumnPath.fromDotName("s.event_time"), eventTime, null, 1L); - } - - int expectedDate = Math.toIntExact(LocalDate.of(2024, 1, 2).toEpochDay()); - long expectedTimestamp = ChronoUnit.MICROS.between( - LocalDateTime.of(1970, 1, 1, 0, 0), - LocalDateTime.of(2024, 1, 2, 3, 4, 5, 123456000)); - Mockito.verify(updateSchema).addColumn(Mockito.eq("s"), Mockito.eq("flag"), - Mockito.eq(Types.BooleanType.get()), Mockito.eq(""), - Mockito.>argThat(literal -> Boolean.TRUE.equals(literal.value()))); - Mockito.verify(updateSchema).addColumn(Mockito.eq("s"), Mockito.eq("event_date"), - Mockito.eq(Types.DateType.get()), Mockito.eq(""), - Mockito.>argThat(literal -> Integer.valueOf(expectedDate).equals(literal.value()))); - Mockito.verify(updateSchema).addColumn(Mockito.eq("s"), Mockito.eq("event_time"), - Mockito.eq(Types.TimestampType.withoutZone()), Mockito.eq(""), - Mockito.>argThat(literal -> Long.valueOf(expectedTimestamp).equals(literal.value()))); - Mockito.verify(updateSchema, Mockito.times(3)).commit(); - } - @Test public void testModifyComplexColumnRejectsCaseInsensitiveStructFieldAdditions() { Schema schema = mixedCaseNestedSchema(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index 44d7a26052a1d1..ee34b4bee50b0c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -21,7 +21,6 @@ import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; @@ -55,7 +54,6 @@ import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.LongType; import org.apache.iceberg.types.Types.StructType; -import org.apache.iceberg.util.DateTimeUtil; import org.apache.iceberg.view.View; import org.junit.Assert; import org.junit.Test; @@ -65,7 +63,6 @@ import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.time.DateTimeException; -import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -330,35 +327,6 @@ public void testParseSchemaPreservesInitialDefault() { .withId(5) .ofType(Types.FixedType.ofLength(4)) .withInitialDefault(ByteBuffer.wrap(new byte[] {3, 2, 1, 0})) - .build(), - Types.NestedField.optional("nested") - .withId(6) - .ofType(Types.StructType.of( - Types.NestedField.optional("added_nested") - .withId(7) - .ofType(Types.IntegerType.get()) - .withInitialDefault(9) - .build())) - .build(), - Types.NestedField.optional("array_nested") - .withId(8) - .ofType(Types.ListType.ofOptional(9, Types.StructType.of( - Types.NestedField.optional("added_in_element") - .withId(10) - .ofType(Types.StringType.get()) - .withInitialDefault("x") - .build()))) - .build(), - Types.NestedField.optional("map_nested") - .withId(11) - .ofType(Types.MapType.ofOptional(12, 13, Types.StringType.get(), - Types.StructType.of( - Types.NestedField.optional("added_in_value") - .withId(14) - .ofType(Types.BinaryType.get()) - .withInitialDefault(ByteBuffer.wrap( - new byte[] {4, 5, 6, 7})) - .build()))) .build()); List columns = IcebergUtils.parseSchema(schema, true, false); @@ -367,48 +335,11 @@ public void testParseSchemaPreservesInitialDefault() { Assert.assertEquals("2024-01-01 00:00:00.123456", columns.get(1).getDefaultValue()); Assert.assertEquals("AAAAAAAAAAAAAAAAAAAAAA==", columns.get(2).getDefaultValue()); Assert.assertEquals("AAEC/w==", columns.get(3).getDefaultValue()); - Assert.assertEquals("9", columns.get(5).getChildren().get(0).getDefaultValue()); - Assert.assertEquals(7, columns.get(5).getChildren().get(0).getUniqueId()); - Assert.assertEquals("x", columns.get(6).getChildren().get(0) - .getChildren().get(0).getDefaultValue()); - Assert.assertEquals(10, columns.get(6).getChildren().get(0) - .getChildren().get(0).getUniqueId()); - Assert.assertEquals("BAUGBw==", columns.get(7).getChildren().get(1) - .getChildren().get(0).getDefaultValue()); - Assert.assertEquals(14, columns.get(7).getChildren().get(1) - .getChildren().get(0).getUniqueId()); Map base64Defaults = IcebergUtils.getBase64EncodedInitialDefaults(schema); Assert.assertEquals("AAAAAAAAAAAAAAAAAAAAAA==", base64Defaults.get(3)); Assert.assertEquals("AAEC/w==", base64Defaults.get(4)); Assert.assertEquals("AwIBAA==", base64Defaults.get(5)); - Assert.assertEquals("BAUGBw==", base64Defaults.get(14)); - } - - @Test - public void testParseIcebergTimestampTzUsesSessionTimeZone() { - ConnectContext context = new ConnectContext(); - context.getSessionVariable().setTimeZone("+07:00"); - context.setThreadLocalInfo(); - try { - Types.TimestampType icebergType = Types.TimestampType.withZone(); - - org.apache.iceberg.expressions.Literal sessionLocal = - IcebergUtils.parseIcebergLiteral("2020-01-01 00:00:00.1235", - ScalarType.createTimeStampTzType(3), icebergType); - Assert.assertEquals(DateTimeUtil.microsFromInstant( - Instant.parse("2019-12-31T17:00:00.124Z")), - ((Long) sessionLocal.value()).longValue()); - - org.apache.iceberg.expressions.Literal explicitOffset = - IcebergUtils.parseIcebergLiteral("2020-01-01 00:00:00.654321+02:00", - ScalarType.createTimeStampTzType(6), icebergType); - Assert.assertEquals(DateTimeUtil.microsFromInstant( - Instant.parse("2019-12-31T22:00:00.654321Z")), - ((Long) explicitOffset.value()).longValue()); - } finally { - ConnectContext.remove(); - } } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index 504b540265fe55..a4da87271d24f5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -18,9 +18,11 @@ package org.apache.doris.nereids.parser; import org.apache.doris.analysis.ColumnPath; +import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand; import org.apache.doris.nereids.trees.plans.commands.info.AddColumnOp; +import org.apache.doris.nereids.trees.plans.commands.info.AddColumnsOp; import org.apache.doris.nereids.trees.plans.commands.info.AlterTableOp; import org.apache.doris.nereids.trees.plans.commands.info.DropColumnOp; import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnCommentOp; @@ -80,6 +82,28 @@ public void testParseNestedModifyDropRenamePaths() { RenameColumnOp.class, "m.value.y"); } + @Test + public void testNestedColumnDefaultClausesAreNotInGrammar() { + for (String sql : List.of( + "ALTER TABLE t ADD COLUMN s.b BIGINT NULL DEFAULT 7", + "ALTER TABLE t ADD COLUMN s.c BIGINT NULL DEFAULT NULL", + "ALTER TABLE t ADD COLUMN s.ts DATETIME NULL DEFAULT CURRENT_TIMESTAMP " + + "ON UPDATE CURRENT_TIMESTAMP", + "ALTER TABLE t MODIFY COLUMN s.a BIGINT DEFAULT 7", + "ALTER TABLE t MODIFY COLUMN s.a BIGINT DEFAULT NULL", + "ALTER TABLE t MODIFY COLUMN s.ts DATETIME DEFAULT CURRENT_TIMESTAMP " + + "ON UPDATE CURRENT_TIMESTAMP")) { + Assertions.assertThrows(ParseException.class, () -> parser.parseSingle(sql), sql); + } + } + + @Test + public void testTopLevelColumnKeepsExistingDefaultGrammar() { + AddColumnOp add = assertSingleClausePath( + "ALTER TABLE t ADD COLUMN b BIGINT NULL DEFAULT 7", AddColumnOp.class, "b"); + Assertions.assertTrue(add.getColumnDef().hasDefaultValue()); + } + @Test public void testLegacyStringConstructorsKeepDottedTopLevelNames() { DropColumnOp drop = new DropColumnOp("top.level", null, Collections.emptyMap()); @@ -150,24 +174,59 @@ public void testModifyColumnCommentRoundTripEscapesQuotesAndBackslashes() { } @Test - public void testColumnDefinitionWithPathDecodesDefaultAndCommentLiterals() { - assertColumnDefinitionLiteralDecoding(false); - assertColumnDefinitionLiteralDecoding(true); + public void testColumnDefinitionCommentRoundTripEscapesQuotesAndBackslashes() { + assertColumnDefinitionCommentRoundTrip(false); + assertColumnDefinitionCommentRoundTrip(true); + } + + @Test + public void testRegularColumnDefinitionCommentRoundTripEscapesQuotesAndBackslashes() { + assertRegularColumnDefinitionCommentRoundTrip(false); + assertRegularColumnDefinitionCommentRoundTrip(true); } - private void assertColumnDefinitionLiteralDecoding(boolean noBackslashEscapes) { + private void assertRegularColumnDefinitionCommentRoundTrip(boolean noBackslashEscapes) { try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(noBackslashEscapes); String sqlPath = noBackslashEscapes ? "C:\\tmp\\" : "C:\\\\tmp\\\\"; + String expectedComment = "owner's \"field\" C:\\tmp\\"; + AddColumnsOp addColumns = assertSingleClause( + "ALTER TABLE t ADD COLUMN (owner STRING NULL COMMENT 'owner''s \"field\" " + + sqlPath + "', metric INT NULL)", AddColumnsOp.class); + Assertions.assertEquals(expectedComment, + addColumns.getColumnDefinitions().get(0).getComment()); + + AddColumnsOp reparsedAddColumns = assertSingleClause( + "ALTER TABLE t " + addColumns.toSql(), AddColumnsOp.class); + Assertions.assertEquals(expectedComment, + reparsedAddColumns.getColumnDefinitions().get(0).getComment()); + } + } + + private void assertColumnDefinitionCommentRoundTrip(boolean noBackslashEscapes) { + try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { + mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(noBackslashEscapes); + + String sqlPath = noBackslashEscapes ? "C:\\tmp\\" : "C:\\\\tmp\\\\"; + String expectedComment = "owner's \"field\" C:\\tmp\\"; AddColumnOp add = assertSingleClausePath( - "ALTER TABLE t ADD COLUMN info.owner STRING DEFAULT 'owner''s " + sqlPath - + "' COMMENT \"a\"\"b " + sqlPath + "\"", + "ALTER TABLE t ADD COLUMN info.owner STRING NULL COMMENT 'owner''s \"field\" " + + sqlPath + "'", AddColumnOp.class, "info.owner"); + Assertions.assertEquals(expectedComment, add.getColumnDef().getComment()); + AddColumnOp reparsedAdd = assertSingleClausePath( + "ALTER TABLE t " + add.toSql(), AddColumnOp.class, "info.owner"); + Assertions.assertEquals(expectedComment, reparsedAdd.getColumnDef().getComment()); - Assertions.assertEquals("owner's C:\\tmp\\", add.getColumnDef() - .translateToCatalogStyleForSchemaChange().getDefaultValue()); - Assertions.assertEquals("a\"b C:\\tmp\\", add.getColumnDef().getComment()); + ModifyColumnOp modify = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.owner STRING COMMENT 'owner''s \"field\" " + + sqlPath + "'", + ModifyColumnOp.class, "info.owner"); + Assertions.assertEquals(expectedComment, modify.getColumnDef().getComment()); + ModifyColumnOp reparsedModify = assertSingleClausePath( + "ALTER TABLE t " + modify.toSql(), ModifyColumnOp.class, "info.owner"); + Assertions.assertEquals(expectedComment, reparsedModify.getColumnDef().getComment()); } } @@ -200,12 +259,7 @@ private void assertCommentRoundTrip(boolean noBackslashEscapes) { private T assertSingleClausePath(String sql, Class clauseClass, String expectedPath) { - Plan plan = parser.parseSingle(sql); - Assertions.assertInstanceOf(AlterTableCommand.class, plan); - List clauses = ((AlterTableCommand) plan).getOps(); - Assertions.assertEquals(1, clauses.size()); - AlterTableOp clause = clauses.get(0); - Assertions.assertInstanceOf(clauseClass, clause); + T clause = assertSingleClause(sql, clauseClass); if (clause instanceof AddColumnOp) { Assertions.assertEquals(expectedPath, ((AddColumnOp) clause).getColumnPath().getFullPath()); } else if (clause instanceof ModifyColumnOp) { @@ -217,6 +271,16 @@ private T assertSingleClausePath(String sql, Class c } else if (clause instanceof RenameColumnOp) { Assertions.assertEquals(expectedPath, ((RenameColumnOp) clause).getColumnPath().getFullPath()); } + return clause; + } + + private T assertSingleClause(String sql, Class clauseClass) { + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(AlterTableCommand.class, plan); + List clauses = ((AlterTableCommand) plan).getOps(); + Assertions.assertEquals(1, clauses.size()); + AlterTableOp clause = clauses.get(0); + Assertions.assertInstanceOf(clauseClass, clause); return clauseClass.cast(clause); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index facb5e340b272e..e80beec063e154 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -205,34 +205,6 @@ void testRejectRequiredNestedColumnForIcebergTable() { .contains("New nested field 's.required_field' must be nullable")); } - @Test - void testRejectDefaultMetadataForIcebergModify() throws AnalysisException { - IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); - for (String sql : Arrays.asList( - "ALTER TABLE t MODIFY COLUMN a BIGINT DEFAULT 7", - "ALTER TABLE t MODIFY COLUMN a BIGINT DEFAULT NULL", - "ALTER TABLE t MODIFY COLUMN ts DATETIME DEFAULT CURRENT_TIMESTAMP " - + "ON UPDATE CURRENT_TIMESTAMP", - "ALTER TABLE t MODIFY COLUMN s.a BIGINT DEFAULT 7", - "ALTER TABLE t MODIFY COLUMN s.a BIGINT DEFAULT NULL", - "ALTER TABLE t MODIFY COLUMN s.ts DATETIME DEFAULT CURRENT_TIMESTAMP " - + "ON UPDATE CURRENT_TIMESTAMP")) { - AnalysisException exception = Assertions.assertThrows(AnalysisException.class, - () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); - Assertions.assertTrue(exception.getMessage() - .contains("DEFAULT and ON UPDATE are not supported for Iceberg MODIFY COLUMN")); - } - - AlterTableCommand.checkColumnOperationsSupported(table, - parseAlter("ALTER TABLE t MODIFY COLUMN a BIGINT").getOps()); - AlterTableCommand.checkColumnOperationsSupported(table, - parseAlter("ALTER TABLE t MODIFY COLUMN s.a BIGINT").getOps()); - AlterTableCommand.checkColumnOperationsSupported(table, - parseAlter("ALTER TABLE t ADD COLUMN b BIGINT NULL DEFAULT 7").getOps()); - AlterTableCommand.checkColumnOperationsSupported(table, - parseAlter("ALTER TABLE t ADD COLUMN s.b BIGINT NULL DEFAULT 7").getOps()); - } - @Test void testRejectRollupForIcebergColumnOperations() { IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java index f20925011a4c44..6cbb4073df2d50 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.plans.commands.info; +import org.apache.doris.nereids.types.StringType; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -33,4 +35,12 @@ public void testNameEquals() { boolean expected2 = false; Assertions.assertEquals(expected2, columnDefinition.nameEquals(otherColName2, false)); } + + @Test + public void testToSqlHandlesNullComment() { + ColumnDefinition columnDefinition = new ColumnDefinition("col1", StringType.INSTANCE, true, null); + + String sql = columnDefinition.toSql(); + Assertions.assertTrue(sql.endsWith("COMMENT \"\"")); + } } diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index d85f99c48597c5..b300b6185c8730 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1555,15 +1555,13 @@ columnDef ; columnDefWithPath - : colName=qualifiedName type=dataType + : columnDef + | colNames+=identifier (DOT colNames+=identifier)+ type=dataType KEY? (aggType=aggTypeDef)? ((GENERATED ALWAYS)? AS LEFT_PAREN generatedExpr=expression RIGHT_PAREN)? ((NOT)? nullable=NULL)? (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)? - (DEFAULT (nullValue=NULL | SUBTRACT? INTEGER_VALUE | SUBTRACT? DECIMAL_VALUE | PI | E | BITMAP_EMPTY | stringValue=STRING_LITERAL - | CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))? - (ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)? (COMMENT comment=STRING_LITERAL)? ; diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy index b887c4de3e99ff..b70ba7f8002631 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy @@ -77,7 +77,7 @@ suite("test_iceberg_schema_change_complex_types", "p0,external,doris,external_do STRUCT(1, CAST(1.1 AS FLOAT), 'abc', MAP(1, 10), ARRAY(1, 2)) )""" - // Iceberg MODIFY COLUMN does not support default metadata + // Complex type default value only supports NULL test { sql """ ALTER TABLE ${table_name} MODIFY COLUMN st STRUCT< @@ -87,7 +87,7 @@ suite("test_iceberg_schema_change_complex_types", "p0,external,doris,external_do sm: MAP COMMENT 'm1', sa: ARRAY COMMENT 'a1' > DEFAULT 'x'""" - exception "DEFAULT and ON UPDATE are not supported for Iceberg MODIFY COLUMN" + exception "just support null" } // Cannot change nullable complex column to not null From a8705b4d861b59c797d49327adb3510961cbe47a Mon Sep 17 00:00:00 2001 From: daidai Date: Sat, 18 Jul 2026 13:01:04 +0800 Subject: [PATCH 18/32] [fix](iceberg) Preserve mapped types in column modifications ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Iceberg UUID, BINARY, FIXED, and TIMESTAMPTZ fields are exposed through mapping-aware Doris types. MODIFY COLUMN converted those lossy views back to generic Iceberg types or rejected unchanged VARBINARY leaves, causing legal primitive metadata changes and complex changes with unchanged mapped children to fail. Keep the existing Iceberg type as the source of truth, use catalog mapping options only to identify unchanged views, and convert only changed or newly added leaves. ### Release note Fix Iceberg column modifications for mapped UUID, binary, fixed, and timestamp-with-zone fields. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.datasource.iceberg.IcebergMetadataOpsValidationTest targeted mapped-type cases - Behavior changed: Yes. Legal Iceberg MODIFY COLUMN operations preserve existing mapped primitive types. - Does this need documentation: No --- .../iceberg/IcebergMetadataOps.java | 189 ++++++++++++++---- .../IcebergMetadataOpsValidationTest.java | 179 ++++++++++++++++- 2 files changed, 323 insertions(+), 45 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index c5b056b5d318f6..414cdadef6eccc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -815,7 +815,8 @@ private void addOneColumn(UpdateSchema updateSchema, Column column) throws UserE if (!column.isAllowNull()) { throw new UserException("can't add a non-nullable column to an Iceberg table"); } - org.apache.iceberg.types.Type dorisType = IcebergUtils.dorisTypeToIcebergType(column.getType()); + org.apache.iceberg.types.Type dorisType = + toIcebergTypeForSchemaChange(column.getType(), column.getName()); Literal defaultValue = IcebergUtils.parseIcebergLiteral(column.getDefaultValue(), dorisType); updateSchema.addColumn(column.getName(), dorisType, column.getComment(), defaultValue); } @@ -917,7 +918,8 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co parentPath.getColumnPath(), columnPath.getLeafName(), null, "add"); UpdateSchema updateSchema = icebergTable.updateSchema(); - org.apache.iceberg.types.Type dorisType = IcebergUtils.dorisTypeToIcebergType(column.getType()); + org.apache.iceberg.types.Type dorisType = + toIcebergTypeForSchemaChange(column.getType(), columnPath.getFullPath()); updateSchema.addColumn(parentPath.getFullPath(), columnPath.getLeafName(), dorisType, column.getComment()); if (position != null) { @@ -1053,22 +1055,27 @@ private void modifyTopLevelColumn(ExternalTable dorisTable, Column column, Colum ResolvedColumnPath columnPath = new ResolvedColumnPath(ColumnPath.of(currentCol.name()), currentCol.type(), currentCol); - validateCommonColumnInfo(column, !legacyMode); - UpdateSchema updateSchema = icebergTable.updateSchema(); - + validateCommonColumnMetadata(column, !legacyMode); + org.apache.iceberg.types.Type targetType; if (column.getType().isComplexType()) { - // Complex type processing branch validateForModifyComplexColumn(column, currentCol); - applyComplexTypeChange(updateSchema, columnPath.getFullPath(), currentCol.type(), column.getType(), - legacyMode); + targetType = currentCol.type(); + } else { + validateForModifyColumn(column, currentCol); + targetType = resolvePrimitiveTypeForModify( + currentCol.type(), column.getType(), columnPath.getFullPath()); + } + + UpdateSchema updateSchema = icebergTable.updateSchema(); + if (column.getType().isComplexType()) { + applyComplexTypeChange(updateSchema, columnPath.getFullPath(), currentCol.type(), + column.getType(), legacyMode); if (!Objects.equals(currentCol.doc(), column.getComment())) { updateSchema.updateColumnDoc(columnPath.getFullPath(), column.getComment()); } } else { - // Primitive type processing (existing logic) - validateForModifyColumn(column, currentCol); - Type icebergType = IcebergUtils.dorisTypeToIcebergType(column.getType()); - updateSchema.updateColumn(columnPath.getFullPath(), icebergType.asPrimitiveType(), column.getComment()); + applyPrimitiveColumnChange(updateSchema, columnPath.getFullPath(), currentCol, + targetType.asPrimitiveType(), column.getComment()); } applyExplicitNullableChange(updateSchema, columnPath.getFullPath(), column, legacyMode); @@ -1100,19 +1107,26 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column } validateNestedModifyColumnMetadata(column, resolvedPath.getFullPath()); - UpdateSchema updateSchema = icebergTable.updateSchema(); - + org.apache.iceberg.types.Type targetType; if (column.getType().isComplexType()) { validateForModifyComplexColumn(column, currentCol, columnPath.getFullPath()); - applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), column.getType(), - false); + targetType = currentCol.type(); + } else { + validateForModifyColumn(column, currentCol, columnPath.getFullPath()); + targetType = resolvePrimitiveTypeForModify( + currentCol.type(), column.getType(), resolvedPath.getFullPath()); + } + + UpdateSchema updateSchema = icebergTable.updateSchema(); + if (column.getType().isComplexType()) { + applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), + column.getType(), false); if (!Objects.equals(currentCol.doc(), column.getComment())) { updateSchema.updateColumnDoc(resolvedPath.getFullPath(), column.getComment()); } } else { - validateForModifyColumn(column, currentCol, columnPath.getFullPath()); - Type icebergType = IcebergUtils.dorisTypeToIcebergType(column.getType()); - updateSchema.updateColumn(resolvedPath.getFullPath(), icebergType.asPrimitiveType(), column.getComment()); + applyPrimitiveColumnChange(updateSchema, resolvedPath.getFullPath(), currentCol, + targetType.asPrimitiveType(), column.getComment()); } applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column, false); @@ -1157,6 +1171,25 @@ private void applyExplicitNullableChange(UpdateSchema updateSchema, String colum } } + private org.apache.iceberg.types.Type.PrimitiveType resolvePrimitiveTypeForModify( + org.apache.iceberg.types.Type currentIcebergType, + org.apache.doris.catalog.Type requestedDorisType, String columnPath) throws UserException { + if (isSameMappedDorisType(mappedDorisType(currentIcebergType), requestedDorisType)) { + return currentIcebergType.asPrimitiveType(); + } + return toIcebergTypeForSchemaChange(requestedDorisType, columnPath).asPrimitiveType(); + } + + private void applyPrimitiveColumnChange(UpdateSchema updateSchema, String columnPath, + NestedField currentCol, org.apache.iceberg.types.Type.PrimitiveType targetType, + String targetComment) { + if (!currentCol.type().equals(targetType)) { + updateSchema.updateColumn(columnPath, targetType, targetComment); + } else if (!Objects.equals(currentCol.doc(), targetComment)) { + updateSchema.updateColumnDoc(columnPath, targetComment); + } + } + private void validateForModifyColumn(Column column, NestedField currentCol) throws UserException { validateForModifyColumn(column, currentCol, column.getName()); } @@ -1188,7 +1221,7 @@ private void validateForModifyComplexColumn(Column column, NestedField currentCo + column.getName()); } - org.apache.doris.catalog.Type oldDorisType = IcebergUtils.icebergTypeToDorisType(oldIcebergType, false, false); + org.apache.doris.catalog.Type oldDorisType = mappedDorisType(oldIcebergType); org.apache.doris.catalog.Type newDorisType = column.getType(); if (!isSameComplexCategory(oldIcebergType, newDorisType)) { throw new UserException("Cannot change complex column type category from " @@ -1199,6 +1232,7 @@ private void validateForModifyComplexColumn(Column column, NestedField currentCo } catch (DdlException e) { throw new UserException(e.getMessage(), e); } + validateComplexTypeChanges(oldIcebergType, newDorisType, columnPath); if (currentCol.isOptional() && !column.isAllowNull()) { throw new UserException("Cannot change nullable column " + columnPath + " to not null"); } @@ -1374,6 +1408,59 @@ private boolean isSameComplexCategory(Type oldIcebergType, org.apache.doris.cata } } + private void validateComplexTypeChanges(org.apache.iceberg.types.Type oldIcebergType, + org.apache.doris.catalog.Type newDorisType, String path) throws UserException { + switch (oldIcebergType.typeId()) { + case STRUCT: + validateStructTypeChanges(oldIcebergType.asStructType(), (StructType) newDorisType, path); + break; + case LIST: + Types.ListType oldListType = oldIcebergType.asListType(); + validateExistingTypeChange(oldListType.elementType(), ((ArrayType) newDorisType).getItemType(), + path + "." + oldListType.field(oldListType.elementId()).name()); + break; + case MAP: + Types.MapType oldMapType = oldIcebergType.asMapType(); + MapType newMapType = (MapType) newDorisType; + org.apache.doris.catalog.Type oldDorisKeyType = mappedDorisType(oldMapType.keyType()); + if (!isSameMappedDorisType(oldDorisKeyType, newMapType.getKeyType())) { + throw new UserException("Cannot change MAP key type from " + + oldDorisKeyType.toSql() + " to " + newMapType.getKeyType().toSql()); + } + validateExistingTypeChange(oldMapType.valueType(), newMapType.getValueType(), + path + "." + oldMapType.field(oldMapType.valueId()).name()); + break; + default: + throw new UserException("Unsupported complex type for modify: " + oldIcebergType); + } + } + + private void validateStructTypeChanges(Types.StructType oldStructType, + StructType newStructType, String path) throws UserException { + List oldFields = oldStructType.fields(); + List newFields = newStructType.getFields(); + for (int i = 0; i < oldFields.size(); i++) { + NestedField oldField = oldFields.get(i); + validateExistingTypeChange(oldField.type(), newFields.get(i).getType(), + path + "." + oldField.name()); + } + for (int i = oldFields.size(); i < newFields.size(); i++) { + StructField newField = newFields.get(i); + toIcebergTypeForSchemaChange(newField.getType(), path + "." + newField.getName()); + } + } + + private void validateExistingTypeChange(org.apache.iceberg.types.Type oldIcebergType, + org.apache.doris.catalog.Type newDorisType, String path) throws UserException { + if (oldIcebergType.isPrimitiveType()) { + if (!isSameMappedDorisType(mappedDorisType(oldIcebergType), newDorisType)) { + toIcebergTypeForSchemaChange(newDorisType, path); + } + return; + } + validateComplexTypeChanges(oldIcebergType, newDorisType, path); + } + private void applyComplexTypeChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldIcebergType, org.apache.doris.catalog.Type newDorisType, boolean legacyNullableExplicit) throws UserException { @@ -1413,15 +1500,16 @@ private void applyStructChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldFieldType = oldField.type(); org.apache.doris.catalog.Type newFieldType = newField.getType(); if (oldFieldType.isPrimitiveType()) { - org.apache.doris.catalog.Type oldDorisFieldType = - IcebergUtils.icebergTypeToDorisType(oldFieldType, false, false); - boolean typeChanged = !oldDorisFieldType.equals(newFieldType); + org.apache.doris.catalog.Type oldDorisFieldType = mappedDorisType(oldFieldType); + boolean typeChanged = !isSameMappedDorisType(oldDorisFieldType, newFieldType); boolean commentChanged = !Objects.equals(oldField.doc(), newField.getComment()); - if (typeChanged || commentChanged) { + if (typeChanged) { org.apache.iceberg.types.Type newIcebergFieldType = - IcebergUtils.dorisTypeToIcebergType(newFieldType); + toIcebergTypeForSchemaChange(newFieldType, fieldPath); updateSchema.updateColumn(fieldPath, newIcebergFieldType.asPrimitiveType(), newField.getComment()); + } else if (commentChanged) { + updateSchema.updateColumnDoc(fieldPath, newField.getComment()); } } else { applyComplexTypeChange(updateSchema, fieldPath, oldFieldType, newFieldType, @@ -1441,7 +1529,7 @@ private void applyStructChange(UpdateSchema updateSchema, String path, throw new UserException("New struct field '" + newField.getName() + "' must be nullable"); } org.apache.iceberg.types.Type newFieldIcebergType = - IcebergUtils.dorisTypeToIcebergType(newField.getType()); + toIcebergTypeForSchemaChange(newField.getType(), path + "." + newField.getName()); updateSchema.addColumn(path, newField.getName(), newFieldIcebergType, newField.getComment()); } } @@ -1456,11 +1544,10 @@ private void applyListChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldElementType = oldListType.elementType(); org.apache.doris.catalog.Type newElementType = newArrayType.getItemType(); if (oldElementType.isPrimitiveType()) { - org.apache.doris.catalog.Type oldDorisElementType = - IcebergUtils.icebergTypeToDorisType(oldElementType, false, false); - if (!oldDorisElementType.equals(newElementType)) { + org.apache.doris.catalog.Type oldDorisElementType = mappedDorisType(oldElementType); + if (!isSameMappedDorisType(oldDorisElementType, newElementType)) { org.apache.iceberg.types.Type newIcebergElementType = - IcebergUtils.dorisTypeToIcebergType(newElementType); + toIcebergTypeForSchemaChange(newElementType, elementPath); updateSchema.updateColumn(elementPath, newIcebergElementType.asPrimitiveType(), null); } } else { @@ -1477,9 +1564,8 @@ private void applyMapChange(UpdateSchema updateSchema, String path, throws UserException { org.apache.iceberg.types.Type oldKeyType = oldMapType.keyType(); org.apache.doris.catalog.Type newKeyType = newMapType.getKeyType(); - org.apache.doris.catalog.Type oldDorisKeyType = - IcebergUtils.icebergTypeToDorisType(oldKeyType, false, false); - if (!oldDorisKeyType.equals(newKeyType)) { + org.apache.doris.catalog.Type oldDorisKeyType = mappedDorisType(oldKeyType); + if (!isSameMappedDorisType(oldDorisKeyType, newKeyType)) { throw new UserException("Cannot change MAP key type from " + oldDorisKeyType.toSql() + " to " + newKeyType.toSql()); } @@ -1491,11 +1577,10 @@ private void applyMapChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldValueType = oldMapType.valueType(); org.apache.doris.catalog.Type newValueType = newMapType.getValueType(); if (oldValueType.isPrimitiveType()) { - org.apache.doris.catalog.Type oldDorisValueType = - IcebergUtils.icebergTypeToDorisType(oldValueType, false, false); - if (!oldDorisValueType.equals(newValueType)) { + org.apache.doris.catalog.Type oldDorisValueType = mappedDorisType(oldValueType); + if (!isSameMappedDorisType(oldDorisValueType, newValueType)) { org.apache.iceberg.types.Type newIcebergValueType = - IcebergUtils.dorisTypeToIcebergType(newValueType); + toIcebergTypeForSchemaChange(newValueType, valuePath); updateSchema.updateColumn(valuePath, newIcebergValueType.asPrimitiveType(), null); } } else { @@ -1508,6 +1593,11 @@ private void applyMapChange(UpdateSchema updateSchema, String path, } private void validateCommonColumnInfo(Column column, boolean rejectKey) throws UserException { + validateCommonColumnMetadata(column, rejectKey); + toIcebergTypeForSchemaChange(column.getType(), column.getName()); + } + + private void validateCommonColumnMetadata(Column column, boolean rejectKey) throws UserException { if (rejectKey && column.isKey()) { throw new UserException("KEY is not supported for Iceberg ADD/MODIFY COLUMN"); } @@ -1522,11 +1612,28 @@ private void validateCommonColumnInfo(Column column, boolean rejectKey) throws U if (column.isAutoInc()) { throw new UserException("Can not specify auto incremental iceberg table column"); } + } + + private org.apache.doris.catalog.Type mappedDorisType(org.apache.iceberg.types.Type icebergType) { + return IcebergUtils.icebergTypeToDorisType(icebergType, + dorisCatalog.getEnableMappingVarbinary(), dorisCatalog.getEnableMappingTimestampTz()); + } + + private boolean isSameMappedDorisType(org.apache.doris.catalog.Type mappedType, + org.apache.doris.catalog.Type requestedType) { + // ScalarType.equals does not compare VARBINARY length, but Iceberg FIXED uses the + // mapped length to distinguish an unchanged view from a requested type change. + return mappedType.equals(requestedType) + && (!mappedType.isVarbinaryType() || mappedType.getLength() == requestedType.getLength()); + } + + private org.apache.iceberg.types.Type toIcebergTypeForSchemaChange( + org.apache.doris.catalog.Type dorisType, String columnPath) throws UserException { try { - IcebergUtils.dorisTypeToIcebergType(column.getType()); + return IcebergUtils.dorisTypeToIcebergType(dorisType); } catch (UnsupportedOperationException | IllegalArgumentException e) { - throw new UserException("Type " + column.getType().toSql() - + " is not supported for Iceberg column " + column.getName(), e); + throw new UserException("Type " + dorisType.toSql() + + " is not supported for Iceberg column " + columnPath, e); } } @@ -1539,7 +1646,7 @@ private void validateNestedAddColumnMetadata(Column column, ColumnPath columnPat } private void validateNestedModifyColumnMetadata(Column column, String columnPath) throws UserException { - validateCommonColumnInfo(column, true); + validateCommonColumnMetadata(column, true); if (column.hasDefaultValue() || column.hasOnUpdateDefaultValue()) { throw new UserException("Modifying default values is not supported for Iceberg columns: " + columnPath); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index a45f7fd93d19f3..3ae9b7e3885cb1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -213,16 +213,16 @@ public void testRejectUnsupportedIcebergTargetTypesBeforeUpdateSchema() { "is not supported for Iceberg column new_field"); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.LARGEINT, true), null, 1L), - "is not supported for Iceberg column metric"); + "is not supported for Iceberg column info.metric"); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), new Column("child", unsupportedStruct, false), null, 1L), - "is not supported for Iceberg column child"); + "is not supported for Iceberg column info.child.value"); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.events"), new Column("events", unsupportedArray, false), null, 1L), - "is not supported for Iceberg column events"); + "is not supported for Iceberg column info.events.element"); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.attrs"), new Column("attrs", unsupportedMap, false), null, 1L), - "is not supported for Iceberg column attrs"); + "is not supported for Iceberg column info.attrs.value"); } Mockito.verify(icebergTable, Mockito.never()).updateSchema(); @@ -309,6 +309,139 @@ public void testTopLevelModifyPreservesRequiredMixedCaseFields() throws Throwabl Mockito.verify(updateSchema, Mockito.times(2)).commit(); } + @Test + public void testPrimitiveModifyPreservesActualTypeWhenMappingDisabled() throws Throwable { + Schema schema = mappedPrimitiveSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + Mockito.when(dorisCatalog.getEnableMappingVarbinary()).thenReturn(false); + Mockito.when(dorisCatalog.getEnableMappingTimestampTz()).thenReturn(false); + + Column topUuid = new Column("top_uuid", Type.STRING, true); + topUuid.setNullableSpecified(true); + Column nestedUuid = new Column("uuid_value", Type.STRING, true); + nestedUuid.setNullableSpecified(true); + Column nestedTimestamp = new Column( + "tz_value", ScalarType.createDatetimeV2Type(6), true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), topUuid, ColumnPosition.FIRST, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.uuid_value"), nestedUuid, + new ColumnPosition("other"), 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.tz_value"), + nestedTimestamp, null, 1L); + } + + Mockito.verify(updateSchema).updateColumnDoc("top_uuid", ""); + Mockito.verify(updateSchema).updateColumnDoc("info.uuid_value", ""); + Mockito.verify(updateSchema).updateColumnDoc("info.tz_value", ""); + Mockito.verify(updateSchema).makeColumnOptional("top_uuid"); + Mockito.verify(updateSchema).makeColumnOptional("info.uuid_value"); + Mockito.verify(updateSchema).moveFirst("top_uuid"); + Mockito.verify(updateSchema).moveAfter("info.uuid_value", "info.other"); + Mockito.verify(updateSchema, Mockito.never()).updateColumn( + Mockito.anyString(), Mockito.any(org.apache.iceberg.types.Type.PrimitiveType.class), + Mockito.nullable(String.class)); + Mockito.verify(updateSchema, Mockito.times(3)).commit(); + } + + @Test + public void testPrimitiveModifyPreservesActualTypeWhenMappingEnabled() throws Throwable { + Schema schema = mappedPrimitiveSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + Mockito.when(dorisCatalog.getEnableMappingVarbinary()).thenReturn(true); + Mockito.when(dorisCatalog.getEnableMappingTimestampTz()).thenReturn(true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), + new Column("top_uuid", ScalarType.createVarbinaryType(16), true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.tz_value"), + new Column("tz_value", ScalarType.createTimeStampTzType(6), true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumnDoc("top_uuid", ""); + Mockito.verify(updateSchema).updateColumnDoc("info.tz_value", ""); + Mockito.verify(updateSchema, Mockito.never()).updateColumn( + Mockito.anyString(), Mockito.any(org.apache.iceberg.types.Type.PrimitiveType.class), + Mockito.nullable(String.class)); + Mockito.verify(updateSchema, Mockito.times(2)).commit(); + } + + @Test + public void testComplexModifyIgnoresUnchangedMappedChildren() throws Throwable { + Schema schema = mappedComplexSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + Mockito.when(dorisCatalog.getEnableMappingVarbinary()).thenReturn(true); + Mockito.when(dorisCatalog.getEnableMappingTimestampTz()).thenReturn(true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), + new Column("payload", mappedPayloadDorisType(Type.BIGINT, 8, + ScalarType.createVarbinaryType(4)), true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn( + "outer.payload.metric", Types.LongType.get(), null); + Mockito.verify(updateSchema, Mockito.times(1)).updateColumn( + Mockito.anyString(), Mockito.any(org.apache.iceberg.types.Type.PrimitiveType.class), + Mockito.nullable(String.class)); + Mockito.verify(updateSchema).updateColumnDoc("outer.payload", ""); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testComplexModifyRejectsChangedUnsupportedMappedChildrenBeforeUpdateSchema() { + Schema schema = mappedComplexSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(dorisCatalog.getEnableMappingVarbinary()).thenReturn(true); + Mockito.when(dorisCatalog.getEnableMappingTimestampTz()).thenReturn(true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), + new Column("payload", mappedPayloadDorisType(Type.LARGEINT, 8, + ScalarType.createVarbinaryType(4)), true), null, 1L), + "Type largeint is not supported for Iceberg column outer.payload.metric"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), + new Column("payload", mappedPayloadDorisType(Type.INT, 16, + ScalarType.createVarbinaryType(4)), true), null, 1L), + "Type varbinary(16) is not supported for Iceberg column outer.payload.fixed_value"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), + new Column("payload", mappedPayloadDorisType(Type.INT, 8, + ScalarType.createVarbinaryType(8)), true), null, 1L), + "Cannot change MAP key type from varbinary(4) to varbinary(8)"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + @Test public void testLegacyModifyColumnTreatsNullabilityAsExplicit() throws Throwable { Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); @@ -926,6 +1059,44 @@ private Schema primitiveContainerSchema() { 4, 5, Types.StringType.get(), Types.IntegerType.get()))); } + private Schema mappedPrimitiveSchema() { + return new Schema( + Types.NestedField.required(1, "top_uuid", Types.UUIDType.get()), + Types.NestedField.required(2, "top_other", Types.IntegerType.get()), + Types.NestedField.optional(3, "info", Types.StructType.of( + Types.NestedField.required(4, "uuid_value", Types.UUIDType.get()), + Types.NestedField.required(5, "tz_value", Types.TimestampType.withZone()), + Types.NestedField.required(6, "other", Types.IntegerType.get())))); + } + + private Schema mappedComplexSchema() { + return new Schema(Types.NestedField.optional(1, "outer", Types.StructType.of( + Types.NestedField.optional(2, "payload", Types.StructType.of( + Types.NestedField.optional(3, "uuid_value", Types.UUIDType.get()), + Types.NestedField.optional(4, "binary_value", Types.BinaryType.get()), + Types.NestedField.optional(5, "fixed_value", Types.FixedType.ofLength(8)), + Types.NestedField.optional(6, "tz_value", Types.TimestampType.withZone()), + Types.NestedField.optional(7, "metric", Types.IntegerType.get()), + Types.NestedField.optional(8, "events", + Types.ListType.ofOptional(9, Types.UUIDType.get())), + Types.NestedField.optional(10, "attrs", Types.MapType.ofOptional( + 11, 12, Types.FixedType.ofLength(4), Types.TimestampType.withZone()))))))); + } + + private StructType mappedPayloadDorisType(Type metricType, int fixedLength, Type mapKeyType) { + return new StructType( + new StructField("uuid_value", ScalarType.createVarbinaryType(16)), + new StructField("binary_value", + ScalarType.createVarbinaryType(ScalarType.MAX_VARBINARY_LENGTH)), + new StructField("fixed_value", ScalarType.createVarbinaryType(fixedLength)), + new StructField("tz_value", ScalarType.createTimeStampTzType(6)), + new StructField("metric", metricType), + new StructField("events", ArrayType.create( + ScalarType.createVarbinaryType(16), true)), + new StructField("attrs", new MapType( + mapKeyType, ScalarType.createTimeStampTzType(6)))); + } + private Schema requiredNestedSchema() { return new Schema(Types.NestedField.required(1, "info", Types.StructType.of( Types.NestedField.required(2, "metric", Types.IntegerType.get()), From 459f5968bbf6d417b59c1bc84da1b62310afbd17 Mon Sep 17 00:00:00 2001 From: daidai Date: Sat, 18 Jul 2026 21:24:03 +0800 Subject: [PATCH 19/32] [fix](iceberg) Harden column schema change validation ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: A full-PR review found four reachable validation gaps in Iceberg column schema changes. MODIFY COLUMN could silently discard DEFAULT or ON UPDATE intent, unsupported complex-to-primitive and primitive narrowing changes failed late inside Iceberg, compound ALTER statements containing nested or comment operations could partially commit before a later clause failed, and empty quoted identifiers leaked an unchecked exception from ColumnPath construction. Reject unsupported default intent before translation, validate Iceberg primitive promotions before creating UpdateSchema, fail compound nested/comment statements before catalog dispatch, and convert empty quoted identifiers into a standard parse error. ### Release note Reject unsupported or non-atomic Iceberg column schema changes before mutation and return controlled validation errors. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.datasource.iceberg.IcebergMetadataOpsValidationTest,org.apache.doris.nereids.parser.IcebergNestedSchemaEvolutionParserTest,org.apache.doris.nereids.trees.plans.commands.AlterTableCommandTest (76 tests) - Behavior changed: Yes. Unsupported Iceberg default changes, invalid type transitions, compound nested/comment ALTER statements, and empty quoted identifiers now fail before schema mutation. - Does this need documentation: No --- .../iceberg/IcebergMetadataOps.java | 33 +++++++-- .../plans/commands/AlterTableCommand.java | 31 +++++++- .../plans/commands/info/ColumnDefinition.java | 4 + .../IcebergMetadataOpsValidationTest.java | 74 +++++++++++++++++++ ...cebergNestedSchemaEvolutionParserTest.java | 15 ++++ .../plans/commands/AlterTableCommandTest.java | 41 ++++++++++ .../doris/nereids/parser/PostProcessor.java | 4 + 7 files changed, 194 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 414cdadef6eccc..8b94544e4d4a14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -82,6 +82,7 @@ import org.apache.iceberg.expressions.Term; import org.apache.iceberg.rest.RESTSessionCatalog; import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; import org.apache.logging.log4j.LogManager; @@ -877,7 +878,7 @@ private void refreshTable(ExternalTable dorisTable, long updateTime) { @Override public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { - validateCommonColumnInfo(column, true); + validateAddColumnMetadata(column, true); Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); Schema schema = icebergTable.schema(); @@ -939,7 +940,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); for (Column column : columns) { - validateCommonColumnInfo(column, true); + validateAddColumnMetadata(column, true); validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); } validateNoCaseInsensitiveTopLevelCollisions(icebergTable.schema(), columns); @@ -1055,7 +1056,7 @@ private void modifyTopLevelColumn(ExternalTable dorisTable, Column column, Colum ResolvedColumnPath columnPath = new ResolvedColumnPath(ColumnPath.of(currentCol.name()), currentCol.type(), currentCol); - validateCommonColumnMetadata(column, !legacyMode); + validateModifyColumnMetadata(column, columnPath.getFullPath(), !legacyMode); org.apache.iceberg.types.Type targetType; if (column.getType().isComplexType()) { validateForModifyComplexColumn(column, currentCol); @@ -1177,7 +1178,14 @@ private org.apache.iceberg.types.Type.PrimitiveType resolvePrimitiveTypeForModif if (isSameMappedDorisType(mappedDorisType(currentIcebergType), requestedDorisType)) { return currentIcebergType.asPrimitiveType(); } - return toIcebergTypeForSchemaChange(requestedDorisType, columnPath).asPrimitiveType(); + org.apache.iceberg.types.Type.PrimitiveType currentType = currentIcebergType.asPrimitiveType(); + org.apache.iceberg.types.Type.PrimitiveType targetType = + toIcebergTypeForSchemaChange(requestedDorisType, columnPath).asPrimitiveType(); + if (!currentType.equals(targetType) && !TypeUtil.isPromotionAllowed(currentType, targetType)) { + throw new UserException("Cannot change Iceberg column " + columnPath + " type from " + + currentType + " to " + targetType); + } + return targetType; } private void applyPrimitiveColumnChange(UpdateSchema updateSchema, String columnPath, @@ -1200,6 +1208,9 @@ private void validateForModifyColumn(Column column, NestedField currentCol, Stri if (column.getType().isComplexType()) { throw new UserException("Modify column type to non-primitive type is not supported: " + column.getType()); } + if (!currentCol.type().isPrimitiveType()) { + throw new UserException("Modify column type from complex to primitive is not supported: " + columnPath); + } // check nullable if (currentCol.isOptional() && !column.isAllowNull()) { throw new UserException("Can not change nullable column " + columnPath + " to not null"); @@ -1646,7 +1657,19 @@ private void validateNestedAddColumnMetadata(Column column, ColumnPath columnPat } private void validateNestedModifyColumnMetadata(Column column, String columnPath) throws UserException { - validateCommonColumnMetadata(column, true); + validateModifyColumnMetadata(column, columnPath, true); + } + + private void validateAddColumnMetadata(Column column, boolean rejectKey) throws UserException { + validateCommonColumnInfo(column, rejectKey); + if (column.hasOnUpdateDefaultValue()) { + throw new UserException("ON UPDATE is not supported for Iceberg ADD COLUMN: " + column.getName()); + } + } + + private void validateModifyColumnMetadata(Column column, String columnPath, boolean rejectKey) + throws UserException { + validateCommonColumnMetadata(column, rejectKey); if (column.hasDefaultValue() || column.hasOnUpdateDefaultValue()) { throw new UserException("Modifying default values is not supported for Iceberg columns: " + columnPath); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java index db0df4ad2ebf6d..b42703e9b06bdd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java @@ -143,6 +143,7 @@ private void validate(ConnectContext ctx) throws UserException { static void checkColumnOperationsSupported(TableIf table, List alterTableOps) throws AnalysisException { if (table instanceof IcebergExternalTable) { + checkIcebergCompoundColumnOperations(alterTableOps); for (AlterTableOp alterTableOp : alterTableOps) { ColumnDefinition columnDefinition = getColumnDefinition(alterTableOp); ColumnPath nestedColumnPath = getNestedColumnPath(alterTableOp); @@ -163,10 +164,10 @@ static void checkColumnOperationsSupported(TableIf table, List alt if (properties != null && !properties.isEmpty()) { throw new AnalysisException("PROPERTIES are not supported for Iceberg column operations"); } - checkIcebergColumnDefinition(columnDefinition); + checkIcebergColumnDefinition(alterTableOp, columnDefinition); if (alterTableOp instanceof AddColumnsOp) { for (ColumnDefinition definition : ((AddColumnsOp) alterTableOp).getColumnDefinitions()) { - checkIcebergColumnDefinition(definition); + checkIcebergColumnDefinition(alterTableOp, definition); } } } @@ -181,6 +182,20 @@ static void checkColumnOperationsSupported(TableIf table, List alt } } + private static void checkIcebergCompoundColumnOperations(List alterTableOps) + throws AnalysisException { + if (alterTableOps.size() <= 1) { + return; + } + for (AlterTableOp alterTableOp : alterTableOps) { + if (getNestedColumnPath(alterTableOp) != null + || alterTableOp instanceof ModifyColumnCommentOp) { + throw new AnalysisException("Multiple Iceberg column operations are not supported when a statement " + + "contains a nested column path or MODIFY COLUMN COMMENT"); + } + } + } + private static ColumnPath getNestedColumnPath(AlterTableOp alterTableOp) { ColumnPath columnPath = null; if (alterTableOp instanceof AddColumnOp) { @@ -215,7 +230,7 @@ private static boolean isIcebergColumnSchemaOperation(AlterTableOp alterTableOp) || alterTableOp instanceof ReorderColumnsOp; } - private static void checkIcebergColumnDefinition(ColumnDefinition columnDefinition) + private static void checkIcebergColumnDefinition(AlterTableOp alterTableOp, ColumnDefinition columnDefinition) throws AnalysisException { if (columnDefinition == null) { return; @@ -226,6 +241,16 @@ private static void checkIcebergColumnDefinition(ColumnDefinition columnDefiniti if (columnDefinition.getGeneratedColumnDesc().isPresent()) { throw new AnalysisException("Generated columns are not supported for Iceberg ADD/MODIFY COLUMN"); } + if (alterTableOp instanceof ModifyColumnOp + && (columnDefinition.hasDefaultValue() || columnDefinition.hasOnUpdateDefaultValue())) { + throw new AnalysisException("Modifying default values is not supported for Iceberg columns: " + + ((ModifyColumnOp) alterTableOp).getColumnPath().getFullPath()); + } + if ((alterTableOp instanceof AddColumnOp || alterTableOp instanceof AddColumnsOp) + && columnDefinition.hasOnUpdateDefaultValue()) { + throw new AnalysisException("ON UPDATE is not supported for Iceberg ADD COLUMN: " + + columnDefinition.getName()); + } } private static String getRollupName(AlterTableOp alterTableOp) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 2c590b6e73ac36..6a192eda93b5f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -190,6 +190,10 @@ public boolean hasDefaultValue() { return defaultValue.isPresent(); } + public boolean hasOnUpdateDefaultValue() { + return onUpdateDefaultValue.isPresent(); + } + public boolean isVisible() { return isVisible; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 3ae9b7e3885cb1..18251a1ada302b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -99,6 +99,15 @@ public void testValidateForModifyColumnSuccess() throws Throwable { invokeValidateForModifyColumn(column, currentCol); } + @Test + public void testValidateForModifyColumnRejectsComplexToPrimitive() { + Column column = new Column("struct_col", Type.INT, true); + NestedField currentCol = Types.NestedField.required(1, "struct_col", Types.StructType.of( + Types.NestedField.required(2, "value", Types.IntegerType.get()))); + assertUserException(() -> invokeValidateForModifyColumn(column, currentCol), + "Modify column type from complex to primitive is not supported: struct_col"); + } + @Test public void testValidateForModifyComplexColumnRejectsPrimitiveType() { Column column = new Column("arr_i", Type.INT, true); @@ -561,6 +570,71 @@ public void testNestedColumnOperationsRejectDefaultMetadata() { Mockito.verify(icebergTable, Mockito.never()).updateSchema(); } + @Test + public void testTopLevelColumnOperationsRejectUnsupportedDefaultMetadata() { + Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + Column defaultColumn = new Column("id", Type.BIGINT, false, null, true, "7", ""); + Column onUpdateColumn = Mockito.spy(new Column("id", Type.BIGINT, true)); + Column onUpdateAddColumn = Mockito.spy(new Column("new_col", Type.BIGINT, true)); + Mockito.doReturn(true).when(onUpdateColumn).hasOnUpdateDefaultValue(); + Mockito.doReturn(true).when(onUpdateAddColumn).hasOnUpdateDefaultValue(); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn(dorisTable, defaultColumn, null, 1L), + "Modifying default values is not supported for Iceberg columns: id"); + assertUserException(() -> ops.modifyColumn( + dorisTable, ColumnPath.of("id"), onUpdateColumn, null, 1L), + "Modifying default values is not supported for Iceberg columns: id"); + assertUserException(() -> ops.addColumn(dorisTable, onUpdateAddColumn, null, 1L), + "ON UPDATE is not supported for Iceberg ADD COLUMN: new_col"); + assertUserException(() -> ops.addColumns( + dorisTable, Collections.singletonList(onUpdateAddColumn), 1L), + "ON UPDATE is not supported for Iceberg ADD COLUMN: new_col"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testUnsupportedPrimitiveModifyFailsBeforeUpdateSchema() { + Schema schema = new Schema( + Types.NestedField.required(1, "top_long", Types.LongType.get()), + Types.NestedField.required(2, "info", Types.StructType.of( + Types.NestedField.required(3, "metric", Types.LongType.get()), + Types.NestedField.required(4, "child", Types.StructType.of( + Types.NestedField.required(5, "value", Types.IntegerType.get())))))); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn( + dorisTable, ColumnPath.of("info"), new Column("info", Type.INT, true), null, 1L), + "Modify column type from complex to primitive is not supported: info"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), + new Column("child", Type.INT, true), null, 1L), + "Modify column type from complex to primitive is not supported: info.child"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("top_long"), + new Column("top_long", Type.INT, true), null, 1L), + "Cannot change Iceberg column top_long type from long to int"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), + new Column("metric", Type.INT, true), null, 1L), + "Cannot change Iceberg column info.metric type from long to int"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + @Test public void testRejectKeyAndGeneratedMetadataBeforeUpdateSchema() { Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index a4da87271d24f5..a768275a9829cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -144,6 +144,21 @@ public void testQuotedNestedIdentifiersAreNormalized() { Assertions.assertEquals("New`Metric", reparsedRename.getNewColName()); } + @Test + public void testEmptyQuotedIdentifiersAreRejectedAsParseErrors() { + for (String sql : List.of( + "ALTER TABLE t ADD COLUMN `` INT NULL", + "ALTER TABLE t ADD COLUMN info.`` INT NULL", + "ALTER TABLE t MODIFY COLUMN info.`` BIGINT", + "ALTER TABLE t MODIFY COLUMN info.`` COMMENT 'comment'", + "ALTER TABLE t DROP COLUMN info.``", + "ALTER TABLE t RENAME COLUMN info.`` TO renamed")) { + ParseException exception = Assertions.assertThrows(ParseException.class, + () -> parser.parseSingle(sql), sql); + Assertions.assertTrue(exception.getMessage().contains("Quoted identifier cannot be empty"), sql); + } + } + @Test public void testModifyColumnRoundTripPreservesNullabilityIntent() { ModifyColumnOp omitted = assertSingleClausePath( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index e80beec063e154..568bdd64d6b865 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -275,6 +275,47 @@ void testRejectGeneratedColumnForIcebergAddAndModify() { } } + @Test + void testRejectUnsupportedDefaultChangesForIcebergTable() throws AnalysisException { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t MODIFY COLUMN c BIGINT DEFAULT 7", + "ALTER TABLE t MODIFY COLUMN c BIGINT DEFAULT NULL", + "ALTER TABLE t MODIFY COLUMN ts DATETIME DEFAULT CURRENT_TIMESTAMP " + + "ON UPDATE CURRENT_TIMESTAMP")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("Modifying default values is not supported for Iceberg columns")); + } + + AnalysisException onUpdateException = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter( + "ALTER TABLE t ADD COLUMN ts DATETIME NULL DEFAULT CURRENT_TIMESTAMP " + + "ON UPDATE CURRENT_TIMESTAMP").getOps())); + Assertions.assertTrue(onUpdateException.getMessage() + .contains("ON UPDATE is not supported for Iceberg ADD COLUMN")); + + AlterTableCommand.checkColumnOperationsSupported(table, + parseAlter("ALTER TABLE t ADD COLUMN c BIGINT NULL DEFAULT 7").getOps()); + } + + @Test + void testRejectCompoundNestedIcebergColumnOperations() throws AnalysisException { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN s.good INT NULL, DROP COLUMN m.value.x", + "ALTER TABLE t MODIFY COLUMN c COMMENT 'new comment', ADD COLUMN d INT NULL")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("Multiple Iceberg column operations are not supported")); + } + + AlterTableCommand.checkColumnOperationsSupported(table, + parseAlter("ALTER TABLE t ADD COLUMN c INT NULL, DROP COLUMN d").getOps()); + } + @Test void testPreserveEmptyAddColumnsValidationForIcebergTable() throws AnalysisException { IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); diff --git a/fe/fe-sql-parser/src/main/java/org/apache/doris/nereids/parser/PostProcessor.java b/fe/fe-sql-parser/src/main/java/org/apache/doris/nereids/parser/PostProcessor.java index d94fb545081f08..4927f468dd0f13 100644 --- a/fe/fe-sql-parser/src/main/java/org/apache/doris/nereids/parser/PostProcessor.java +++ b/fe/fe-sql-parser/src/main/java/org/apache/doris/nereids/parser/PostProcessor.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.DorisParser.QuotedIdentifierContext; import org.apache.doris.nereids.DorisParserBaseListener; import org.apache.doris.nereids.errors.QueryParsingErrors; +import org.apache.doris.nereids.exceptions.ParseException; import org.antlr.v4.runtime.CommonToken; import org.antlr.v4.runtime.ParserRuleContext; @@ -43,6 +44,9 @@ public void exitErrorIdent(ErrorIdentContext ctx) { @Override public void exitQuotedIdentifier(QuotedIdentifierContext ctx) { + if (ctx.getText().equals("``")) { + throw new ParseException("Quoted identifier cannot be empty", ctx); + } replaceTokenByIdentifier(ctx, 1, token -> { // Remove the double back ticks in the string. token.setText(token.getText().replace("``", "`")); From 9c5401d1307fc08554909ef7256e301c03ec69f7 Mon Sep 17 00:00:00 2001 From: daidai Date: Sun, 19 Jul 2026 08:36:59 +0800 Subject: [PATCH 20/32] [fix](iceberg) Preserve schema change validation compatibility ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Recent Iceberg schema-change validation rejected every empty quoted identifier during parsing, which broke existing GRANT wildcard and branch/tag statements that handle empty identifiers later with their own semantics. It also replaced established errors for unsupported primitive narrowing and invalid complex defaults. Scope empty-identifier rejection to column paths and column ordering, validate complex defaults before the Iceberg-specific rejection, and preserve the existing primitive type-change error contract. ### Release note Preserve existing SQL semantics and error messages while validating Iceberg nested schema changes. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.datasource.iceberg.IcebergMetadataOpsValidationTest,org.apache.doris.nereids.parser.IcebergNestedSchemaEvolutionParserTest,org.apache.doris.nereids.trees.plans.commands.AlterTableCommandTest (77 tests) - Behavior changed: Yes. Empty quoted identifiers are rejected only where a non-empty column path is required, while existing GRANT and branch/tag handling remains unchanged. - Does this need documentation: No --- .../iceberg/IcebergMetadataOps.java | 4 +-- .../nereids/parser/LogicalPlanBuilder.java | 24 +++++++++++---- .../plans/commands/AlterTableCommand.java | 1 + .../plans/commands/info/ColumnDefinition.java | 30 +++++++++++-------- .../IcebergMetadataOpsValidationTest.java | 4 +-- ...cebergNestedSchemaEvolutionParserTest.java | 10 ++++++- .../plans/commands/AlterTableCommandTest.java | 7 +++++ .../doris/nereids/parser/PostProcessor.java | 4 --- 8 files changed, 58 insertions(+), 26 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 8b94544e4d4a14..8f553535ce790d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -1182,8 +1182,8 @@ private org.apache.iceberg.types.Type.PrimitiveType resolvePrimitiveTypeForModif org.apache.iceberg.types.Type.PrimitiveType targetType = toIcebergTypeForSchemaChange(requestedDorisType, columnPath).asPrimitiveType(); if (!currentType.equals(targetType) && !TypeUtil.isPromotionAllowed(currentType, targetType)) { - throw new UserException("Cannot change Iceberg column " + columnPath + " type from " - + currentType + " to " + targetType); + throw new UserException("Cannot change column type: " + columnPath + ": " + + currentType + " -> " + targetType); } return targetType; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c622e70e880879..ad08bf3467e189 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1166,6 +1166,18 @@ private ColumnDefinitionWithPath(ColumnDefinition columnDefinition, ColumnPath c } } + private static String requireNonEmptyColumnIdentifier(ParserRuleContext ctx, String identifier) { + if (identifier.isEmpty()) { + throw new ParseException("Quoted identifier cannot be empty", ctx); + } + return identifier; + } + + private static ColumnPath parseColumnPath(ParserRuleContext ctx, List parts) { + parts.forEach(part -> requireNonEmptyColumnIdentifier(ctx, part)); + return ColumnPath.of(parts); + } + // Sort the parameters with token position to keep the order with original placeholders // in prepared statement.Otherwise, the order maybe broken private final Map tokenPosToParameters = Maps.newTreeMap((pos1, pos2) -> { @@ -4245,10 +4257,11 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { public ColumnDefinitionWithPath visitColumnDefWithPath(ColumnDefWithPathContext ctx) { if (ctx.columnDef() != null) { ColumnDefinition columnDefinition = visitColumnDef(ctx.columnDef()); - return new ColumnDefinitionWithPath(columnDefinition, ColumnPath.of(columnDefinition.getName())); + ColumnPath columnPath = parseColumnPath(ctx, Collections.singletonList(columnDefinition.getName())); + return new ColumnDefinitionWithPath(columnDefinition, columnPath); } - ColumnPath columnPath = ColumnPath.of(ctx.colNames.stream() + ColumnPath columnPath = parseColumnPath(ctx, ctx.colNames.stream() .map(RuleContext::getText) .collect(Collectors.toList())); String colName = columnPath.getLeafName(); @@ -6198,7 +6211,7 @@ public AlterTableOp visitAddColumnsClause(AddColumnsClauseContext ctx) { @Override public AlterTableOp visitDropColumnClause(DropColumnClauseContext ctx) { - ColumnPath columnPath = ColumnPath.of(visitQualifiedName(ctx.name)); + ColumnPath columnPath = parseColumnPath(ctx.name, visitQualifiedName(ctx.name)); String rollupName = ctx.fromRollup() != null ? ctx.fromRollup().rollup.getText() : null; Map properties = ctx.properties != null ? Maps.newHashMap(visitPropertyClause(ctx.properties)) @@ -6228,6 +6241,7 @@ public AlterTableOp visitModifyColumnClause(ModifyColumnClauseContext ctx) { @Override public AlterTableOp visitReorderColumnsClause(ReorderColumnsClauseContext ctx) { List columnsByPos = visitIdentifierList(ctx.identifierList()); + columnsByPos.forEach(column -> requireNonEmptyColumnIdentifier(ctx.identifierList(), column)); String rollupName = ctx.fromRollup() != null ? ctx.fromRollup().rollup.getText() : null; Map properties = ctx.properties != null ? Maps.newHashMap(visitPropertyClause(ctx.properties)) @@ -6425,7 +6439,7 @@ public AlterTableOp visitRenamePartitionClause(RenamePartitionClauseContext ctx) @Override public AlterTableOp visitRenameColumnClause(RenameColumnClauseContext ctx) { - return new RenameColumnOp(ColumnPath.of(visitQualifiedName(ctx.name)), ctx.newName.getText()); + return new RenameColumnOp(parseColumnPath(ctx.name, visitQualifiedName(ctx.name)), ctx.newName.getText()); } @Override @@ -6533,7 +6547,7 @@ public AlterTableOp visitModifyTableCommentClause(ModifyTableCommentClauseContex @Override public AlterTableOp visitModifyColumnCommentClause(ModifyColumnCommentClauseContext ctx) { - ColumnPath columnPath = ColumnPath.of(visitQualifiedName(ctx.name)); + ColumnPath columnPath = parseColumnPath(ctx.name, visitQualifiedName(ctx.name)); String comment = LogicalPlanBuilderAssistant.parseStringLiteral(ctx.STRING_LITERAL().getText()); return new ModifyColumnCommentOp(columnPath, comment); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java index b42703e9b06bdd..49fca1b1571ccd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java @@ -243,6 +243,7 @@ private static void checkIcebergColumnDefinition(AlterTableOp alterTableOp, Colu } if (alterTableOp instanceof ModifyColumnOp && (columnDefinition.hasDefaultValue() || columnDefinition.hasOnUpdateDefaultValue())) { + columnDefinition.validateComplexTypeDefaultValue(); throw new AnalysisException("Modifying default values is not supported for Iceberg columns: " + ((ModifyColumnOp) alterTableOp).getColumnPath().getFullPath()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 6a192eda93b5f7..02d5ff8f0f6a14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -462,18 +462,8 @@ private void validateInternal(boolean isOlap, Set keysSet, Set c .getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) { throw new AnalysisException("Array type column default value only support null or " + DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE); - } else if (type.isMapType()) { - if (defaultValue.isPresent() && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE) { - throw new AnalysisException("Map type column default value just support null"); - } - } else if (type.isStructType()) { - if (defaultValue.isPresent() && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE) { - throw new AnalysisException("Struct type column default value just support null"); - } - } else if (type.isJsonType() || type.isVariantType()) { - if (defaultValue.isPresent() && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE) { - throw new AnalysisException("Json or Variant type column default value just support null"); - } + } else { + validateComplexTypeDefaultValue(); } if (!isNullable && defaultValue.isPresent() @@ -555,6 +545,22 @@ private void validateInternal(boolean isOlap, Set keysSet, Set c validateGeneratedColumnInfo(); } + /** + * Validate non-null defaults for complex types before connector-specific validation. + */ + public void validateComplexTypeDefaultValue() throws AnalysisException { + if (!defaultValue.isPresent() || defaultValue.get() == DefaultValue.NULL_DEFAULT_VALUE) { + return; + } + if (type.isMapType()) { + throw new AnalysisException("Map type column default value just support null"); + } else if (type.isStructType()) { + throw new AnalysisException("Struct type column default value just support null"); + } else if (type.isJsonType() || type.isVariantType()) { + throw new AnalysisException("Json or Variant type column default value just support null"); + } + } + /** * translate to catalog create table stmt */ diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 18251a1ada302b..83571a26db87c7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -626,10 +626,10 @@ public void testUnsupportedPrimitiveModifyFailsBeforeUpdateSchema() { "Modify column type from complex to primitive is not supported: info.child"); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("top_long"), new Column("top_long", Type.INT, true), null, 1L), - "Cannot change Iceberg column top_long type from long to int"); + "Cannot change column type: top_long: long -> int"); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.INT, true), null, 1L), - "Cannot change Iceberg column info.metric type from long to int"); + "Cannot change column type: info.metric: long -> int"); } Mockito.verify(icebergTable, Mockito.never()).updateSchema(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index a768275a9829cc..8c087929863f43 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -152,13 +152,21 @@ public void testEmptyQuotedIdentifiersAreRejectedAsParseErrors() { "ALTER TABLE t MODIFY COLUMN info.`` BIGINT", "ALTER TABLE t MODIFY COLUMN info.`` COMMENT 'comment'", "ALTER TABLE t DROP COLUMN info.``", - "ALTER TABLE t RENAME COLUMN info.`` TO renamed")) { + "ALTER TABLE t RENAME COLUMN info.`` TO renamed", + "ALTER TABLE t ORDER BY (id, ``)")) { ParseException exception = Assertions.assertThrows(ParseException.class, () -> parser.parseSingle(sql), sql); Assertions.assertTrue(exception.getMessage().contains("Quoted identifier cannot be empty"), sql); } } + @Test + public void testEmptyQuotedIdentifiersOutsideColumnPathsKeepExistingParserSemantics() { + Assertions.assertDoesNotThrow(() -> parser.parseSingle("ALTER TABLE t CREATE BRANCH ``")); + Assertions.assertDoesNotThrow( + () -> parser.parseSingle("GRANT SELECT_PRIV ON `internal`.``.`` TO 'user1'")); + } + @Test public void testModifyColumnRoundTripPreservesNullabilityIntent() { ModifyColumnOp omitted = assertSingleClausePath( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index 568bdd64d6b865..234389a2b2ccb2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -289,6 +289,13 @@ void testRejectUnsupportedDefaultChangesForIcebergTable() throws AnalysisExcepti .contains("Modifying default values is not supported for Iceberg columns")); } + org.apache.doris.nereids.exceptions.AnalysisException complexDefaultException = Assertions.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter( + "ALTER TABLE t MODIFY COLUMN st STRUCT DEFAULT 'x'").getOps())); + Assertions.assertTrue(complexDefaultException.getMessage() + .contains("Struct type column default value just support null")); + AnalysisException onUpdateException = Assertions.assertThrows(AnalysisException.class, () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter( "ALTER TABLE t ADD COLUMN ts DATETIME NULL DEFAULT CURRENT_TIMESTAMP " diff --git a/fe/fe-sql-parser/src/main/java/org/apache/doris/nereids/parser/PostProcessor.java b/fe/fe-sql-parser/src/main/java/org/apache/doris/nereids/parser/PostProcessor.java index 4927f468dd0f13..d94fb545081f08 100644 --- a/fe/fe-sql-parser/src/main/java/org/apache/doris/nereids/parser/PostProcessor.java +++ b/fe/fe-sql-parser/src/main/java/org/apache/doris/nereids/parser/PostProcessor.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.DorisParser.QuotedIdentifierContext; import org.apache.doris.nereids.DorisParserBaseListener; import org.apache.doris.nereids.errors.QueryParsingErrors; -import org.apache.doris.nereids.exceptions.ParseException; import org.antlr.v4.runtime.CommonToken; import org.antlr.v4.runtime.ParserRuleContext; @@ -44,9 +43,6 @@ public void exitErrorIdent(ErrorIdentContext ctx) { @Override public void exitQuotedIdentifier(QuotedIdentifierContext ctx) { - if (ctx.getText().equals("``")) { - throw new ParseException("Quoted identifier cannot be empty", ctx); - } replaceTokenByIdentifier(ctx, 1, token -> { // Remove the double back ticks in the string. token.setText(token.getText().replace("``", "`")); From 92f882e29acfed7406a0372532ddabf71b1ba41e Mon Sep 17 00:00:00 2001 From: daidai Date: Sun, 19 Jul 2026 09:42:33 +0800 Subject: [PATCH 21/32] [fix](iceberg) Align complex schema regression expectation ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: The Iceberg complex schema-change regression expected the unsupported-type error for ARRAY to ARRAY, but validation rejects the unsafe nested downgrade first. Align the regression expectation with the actual validation contract so the suite no longer fails on the correct error. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.datasource.iceberg.IcebergMetadataOpsValidationTest (48 tests passed) - Behavior changed: No - Does this need documentation: No --- .../iceberg/test_iceberg_schema_change_complex_types.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy index b70ba7f8002631..567f3298adf291 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy @@ -133,10 +133,10 @@ suite("test_iceberg_schema_change_complex_types", "p0,external,doris,external_do exception "Cannot reduce struct fields" } - // Iceberg does not support SMALLINT, including nested types + // Nested type downgrade is not allowed test { sql """ALTER TABLE ${table_name} MODIFY COLUMN arr_i ARRAY""" - exception "Type array is not supported for Iceberg column arr_i" + exception "Cannot change int to smallint in nested types" } // Array element type promotions From 15ff18e441c7127a4630c4e9595b06fdff00f438 Mon Sep 17 00:00:00 2001 From: daidai Date: Sun, 19 Jul 2026 10:49:17 +0800 Subject: [PATCH 22/32] [fix](fe) Fix LogicalPlanBuilder declaration order ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: The FE code style checker rejected two static column-path helper methods because they were declared before existing instance fields and the constructor in LogicalPlanBuilder. Move the helpers after the constructor to satisfy DeclarationOrder without changing behavior. ### Release note None ### Check List (For Author) - Test: Manual test - ./build.sh --fe - Behavior changed: No - Does this need documentation: No --- .../nereids/parser/LogicalPlanBuilder.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index ad08bf3467e189..ae48c98c8ecaff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1166,18 +1166,6 @@ private ColumnDefinitionWithPath(ColumnDefinition columnDefinition, ColumnPath c } } - private static String requireNonEmptyColumnIdentifier(ParserRuleContext ctx, String identifier) { - if (identifier.isEmpty()) { - throw new ParseException("Quoted identifier cannot be empty", ctx); - } - return identifier; - } - - private static ColumnPath parseColumnPath(ParserRuleContext ctx, List parts) { - parts.forEach(part -> requireNonEmptyColumnIdentifier(ctx, part)); - return ColumnPath.of(parts); - } - // Sort the parameters with token position to keep the order with original placeholders // in prepared statement.Otherwise, the order maybe broken private final Map tokenPosToParameters = Maps.newTreeMap((pos1, pos2) -> { @@ -1198,6 +1186,18 @@ public LogicalPlanBuilder(Map selectHintMap) { this.selectHintMap = selectHintMap; } + private static String requireNonEmptyColumnIdentifier(ParserRuleContext ctx, String identifier) { + if (identifier.isEmpty()) { + throw new ParseException("Quoted identifier cannot be empty", ctx); + } + return identifier; + } + + private static ColumnPath parseColumnPath(ParserRuleContext ctx, List parts) { + parts.forEach(part -> requireNonEmptyColumnIdentifier(ctx, part)); + return ColumnPath.of(parts); + } + @SuppressWarnings("unchecked") protected T typedVisit(ParseTree ctx) { return (T) ctx.accept(this); From 6c7dc876bdb5623744731636a8d37449da4574e0 Mon Sep 17 00:00:00 2001 From: daidai Date: Sun, 19 Jul 2026 18:36:28 +0800 Subject: [PATCH 23/32] [fix](iceberg) Reject unsupported collection comments ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Iceberg 1.10.1 does not persist docs on list element or map value pseudo-fields, but Doris accepted those ALTER statements and reported success. Reject those unsupported comment operations before creating UpdateSchema. Also align the existing Iceberg schema-change regression with the specific complex-to-primitive validation error reported by the current implementation. ### Release note Reject unsupported comments on Iceberg list element and map value pseudo-fields. ### Check List (For Author) - Test: Unit Test - IcebergMetadataOpsValidationTest - FE Checkstyle - Behavior changed: Yes, unsupported collection pseudo-field comment ALTERs now fail explicitly instead of succeeding as no-ops. - Does this need documentation: No --- .../iceberg/IcebergMetadataOps.java | 17 +++++++++++++ .../IcebergMetadataOpsValidationTest.java | 24 +++++++++++-------- .../iceberg/iceberg_schema_change_ddl.groovy | 2 +- ...iceberg_nested_schema_evolution_ddl.groovy | 18 ++++++++++++-- 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 8f553535ce790d..d4546948286dcb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -1103,6 +1103,8 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); NestedField currentCol = resolvedPath.getField(); + validateCollectionPseudoFieldComment( + icebergTable.schema(), resolvedPath, column.getComment(), false); if (position != null) { validatePositionTarget(icebergTable.schema(), resolvedPath.getColumnPath(), "modify"); } @@ -1153,6 +1155,7 @@ public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, } ResolvedColumnPath resolvedPath = resolveColumnPath( icebergTable.schema(), columnPath, "modify comment"); + validateCollectionPseudoFieldComment(icebergTable.schema(), resolvedPath, comment, true); UpdateSchema updateSchema = icebergTable.updateSchema(); updateSchema.updateColumnDoc(resolvedPath.getFullPath(), StringUtils.defaultString(comment)); @@ -1165,6 +1168,20 @@ public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, refreshTable(dorisTable, updateTime); } + private void validateCollectionPseudoFieldComment(Schema schema, ResolvedColumnPath resolvedPath, + String comment, boolean explicitCommentOperation) throws UserException { + if (!resolvedPath.getColumnPath().isNested() + || (!explicitCommentOperation && StringUtils.isEmpty(comment))) { + return; + } + ResolvedColumnPath parentPath = resolveColumnPath( + schema, resolvedPath.getColumnPath().getParentPath(), "modify comment"); + if (parentPath.getType().isListType() || parentPath.getType().isMapType()) { + throw new UserException("Iceberg does not support comments on collection element or value fields: " + + resolvedPath.getFullPath()); + } + } + private void applyExplicitNullableChange(UpdateSchema updateSchema, String columnPath, Column column, boolean legacyNullableExplicit) { if ((legacyNullableExplicit || column.isNullableSpecified()) && column.isAllowNull()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 83571a26db87c7..412d2a348b74e9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -901,31 +901,35 @@ public void testModifyColumnCommentUsesCanonicalNestedPaths() throws Throwable { } @Test - public void testModifyColumnCommentSupportsDirectArrayElementAndMapValue() throws Throwable { + public void testRejectsCommentsOnDirectArrayElementAndMapValue() { Schema schema = primitiveContainerSchema(); ExternalTable dorisTable = Mockito.mock(ExternalTable.class); Table icebergTable = Mockito.mock(Table.class); - UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); Mockito.when(icebergTable.schema()).thenReturn(schema); - Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); - ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("arr.element"), - "array element comment", 1L); - ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("m.value"), - "map value comment", 1L); + assertUserException(() -> ops.modifyColumnComment( + dorisTable, ColumnPath.fromDotName("arr.element"), "array element comment", 1L), + "Iceberg does not support comments on collection element or value fields: arr.element"); + assertUserException(() -> ops.modifyColumnComment( + dorisTable, ColumnPath.fromDotName("m.value"), "map value comment", 1L), + "Iceberg does not support comments on collection element or value fields: m.value"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), + new Column("element", Type.BIGINT, true, "array element comment"), null, 1L), + "Iceberg does not support comments on collection element or value fields: arr.element"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("m.value"), + new Column("value", Type.BIGINT, true, "map value comment"), null, 1L), + "Iceberg does not support comments on collection element or value fields: m.value"); assertUserException(() -> ops.modifyColumnComment( dorisTable, ColumnPath.fromDotName("m.key"), "map key comment", 1L), "Cannot modify comment MAP key nested column"); } - Mockito.verify(updateSchema).updateColumnDoc("arr.element", "array element comment"); - Mockito.verify(updateSchema).updateColumnDoc("m.value", "map value comment"); - Mockito.verify(updateSchema, Mockito.times(2)).commit(); + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); } @Test diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy index 93c8faf1ce522d..8bd7e240c4d83e 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy @@ -305,7 +305,7 @@ suite("iceberg_schema_change_ddl", "p0,external") { // struct/complex type changes test { sql """ ALTER TABLE ${table_name} MODIFY COLUMN address STRING """ - exception "Cannot change column type" + exception "Modify column type from complex to primitive is not supported" } test { diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy index ccf3dd16a8e7ac..f5c8dd515c7dd7 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy @@ -98,8 +98,22 @@ suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_do exception "Cannot modify MAP key nested column" } - sql """ALTER TABLE ${tableName} MODIFY COLUMN arr_scalar.element COMMENT 'array element comment'""" - sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.value COMMENT 'map value comment'""" + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN arr_scalar.element COMMENT 'array element comment'""" + exception "Iceberg does not support comments on collection element or value fields" + } + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.value COMMENT 'map value comment'""" + exception "Iceberg does not support comments on collection element or value fields" + } + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN arr_scalar.element BIGINT COMMENT 'array element comment'""" + exception "Iceberg does not support comments on collection element or value fields" + } + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.value BIGINT COMMENT 'map value comment'""" + exception "Iceberg does not support comments on collection element or value fields" + } test { sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.`key` COMMENT 'map key comment'""" exception "Cannot modify comment MAP key nested column" From 900b4b5fdcf3bc8e3f5e42d09705ed019f9f96a7 Mon Sep 17 00:00:00 2001 From: daidai Date: Sun, 19 Jul 2026 19:22:00 +0800 Subject: [PATCH 24/32] [fix](iceberg) Preserve explicit empty comment intent ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Nested MODIFY COLUMN parsing collapsed an omitted COMMENT clause and an explicit empty COMMENT into the same empty string. This allowed COMMENT '' on Iceberg list element and map value pseudo-fields to bypass the unsupported-comment validation, so Doris could report success even though Iceberg could not persist the comment. Preserve whether COMMENT was explicitly specified through parser, command rendering, and catalog translation, then reject explicit empty comments before creating UpdateSchema. ### Release note Reject explicit empty comments on Iceberg list element and map value pseudo-fields instead of silently ignoring them. ### Check List (For Author) - Test: Unit Test - IcebergMetadataOpsValidationTest and IcebergNestedSchemaEvolutionParserTest: 61 tests passed - FE Checkstyle: 57 modules passed with zero violations - Regression coverage added; external Iceberg suite not run locally - Behavior changed: Yes, explicit COMMENT '' on Iceberg collection pseudo-fields now fails clearly while type-only changes remain supported. - Does this need documentation: No --- .../java/org/apache/doris/catalog/Column.java | 11 ++++++ .../iceberg/IcebergMetadataOps.java | 6 ++-- .../nereids/parser/LogicalPlanBuilder.java | 4 +-- .../plans/commands/info/ColumnDefinition.java | 34 +++++++++++++++++-- .../IcebergMetadataOpsValidationTest.java | 10 ++++++ ...cebergNestedSchemaEvolutionParserTest.java | 24 +++++++++++++ ...iceberg_nested_schema_evolution_ddl.groovy | 8 +++++ 7 files changed, 89 insertions(+), 8 deletions(-) diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java index acbd760a2bf057..7cac33db8089bf 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java @@ -144,6 +144,8 @@ public static Column generateBeforeValueColumn(Column column) { private boolean isAllowNull; // Runtime-only schema change intent; do not persist it as part of the table schema. private transient boolean nullableSpecified; + // Runtime-only schema change intent; do not persist it as part of the table schema. + private transient boolean commentSpecified; @SerializedName(value = "isAutoInc") private boolean isAutoInc; @@ -371,6 +373,7 @@ public Column(Column column) { this.isCompoundKey = column.isCompoundKey(); this.isAllowNull = column.isAllowNull(); this.nullableSpecified = column.isNullableSpecified(); + this.commentSpecified = column.isCommentSpecified(); this.isAutoInc = column.isAutoInc(); this.defaultValue = column.getDefaultValue(); this.realDefaultValue = column.realDefaultValue; @@ -590,6 +593,10 @@ public boolean isNullableSpecified() { return nullableSpecified; } + public boolean isCommentSpecified() { + return commentSpecified; + } + public boolean isAutoInc() { return isAutoInc; } @@ -606,6 +613,10 @@ public void setNullableSpecified(boolean nullableSpecified) { this.nullableSpecified = nullableSpecified; } + public void setCommentSpecified(boolean commentSpecified) { + this.commentSpecified = commentSpecified; + } + public String getDefaultValue() { return this.defaultValue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index d4546948286dcb..b9768bdcf5aa5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -1104,7 +1104,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); NestedField currentCol = resolvedPath.getField(); validateCollectionPseudoFieldComment( - icebergTable.schema(), resolvedPath, column.getComment(), false); + icebergTable.schema(), resolvedPath, column.getComment(), column.isCommentSpecified()); if (position != null) { validatePositionTarget(icebergTable.schema(), resolvedPath.getColumnPath(), "modify"); } @@ -1169,9 +1169,9 @@ public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, } private void validateCollectionPseudoFieldComment(Schema schema, ResolvedColumnPath resolvedPath, - String comment, boolean explicitCommentOperation) throws UserException { + String comment, boolean commentSpecified) throws UserException { if (!resolvedPath.getColumnPath().isNested() - || (!explicitCommentOperation && StringUtils.isEmpty(comment))) { + || (!commentSpecified && StringUtils.isEmpty(comment))) { return; } ResolvedColumnPath parentPath = resolveColumnPath( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index ae48c98c8ecaff..c8b76d320b71f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -4250,7 +4250,7 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { ? Optional.of(new GeneratedColumnDesc(ctx.generatedExpr.getText(), getExpression(ctx.generatedExpr))) : Optional.empty(); return new ColumnDefinition(colName, colType, isKey, aggType, nullableType, autoIncInitValue, defaultValue, - onUpdateDefaultValue, comment, desc); + onUpdateDefaultValue, comment, ctx.comment != null, true, desc); } @Override @@ -4307,7 +4307,7 @@ public ColumnDefinitionWithPath visitColumnDefWithPath(ColumnDefWithPathContext ? Optional.of(new GeneratedColumnDesc(ctx.generatedExpr.getText(), getExpression(ctx.generatedExpr))) : Optional.empty(); ColumnDefinition columnDefinition = new ColumnDefinition(colName, colType, isKey, aggType, nullableType, - autoIncInitValue, Optional.empty(), Optional.empty(), comment, desc); + autoIncInitValue, Optional.empty(), Optional.empty(), comment, ctx.comment != null, true, desc); return new ColumnDefinitionWithPath(columnDefinition, columnPath); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 02d5ff8f0f6a14..1b52218a354218 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -67,6 +67,8 @@ public class ColumnDefinition { private Optional defaultValue; private Optional onUpdateDefaultValue = Optional.empty(); private final String comment; + // Distinguishes an explicit COMMENT '' clause from an omitted COMMENT clause. + private final boolean commentSpecified; private final boolean isVisible; private boolean aggTypeImplicit = false; private long autoIncInitValue = -1; @@ -86,7 +88,7 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType Optional onUpdateDefaultValue, String comment, Optional generatedColumnDesc) { this(name, type, isKey, aggType, nullableType, autoIncInitValue, defaultValue, onUpdateDefaultValue, - comment, true, generatedColumnDesc); + comment, comment != null && !comment.isEmpty(), true, generatedColumnDesc); } /** @@ -102,6 +104,7 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType this.nullableSpecified = true; this.defaultValue = defaultValue; this.comment = comment; + this.commentSpecified = comment != null && !comment.isEmpty(); this.isVisible = isVisible; } @@ -121,6 +124,7 @@ private ColumnDefinition(String name, DataType type, boolean isKey, AggregateTyp this.defaultValue = defaultValue; this.onUpdateDefaultValue = onUpdateDefaultValue; this.comment = comment; + this.commentSpecified = comment != null && !comment.isEmpty(); this.isVisible = isVisible; } @@ -131,6 +135,18 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType ColumnNullableType nullableType, long autoIncInitValue, Optional defaultValue, Optional onUpdateDefaultValue, String comment, boolean isVisible, Optional generatedColumnDesc) { + this(name, type, isKey, aggType, nullableType, autoIncInitValue, defaultValue, onUpdateDefaultValue, + comment, comment != null && !comment.isEmpty(), isVisible, generatedColumnDesc); + } + + /** + * constructor + */ + public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, + ColumnNullableType nullableType, long autoIncInitValue, Optional defaultValue, + Optional onUpdateDefaultValue, String comment, boolean commentSpecified, + boolean isVisible, + Optional generatedColumnDesc) { this.name = name; this.type = type; this.isKey = isKey; @@ -142,6 +158,7 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType this.defaultValue = defaultValue; this.onUpdateDefaultValue = onUpdateDefaultValue; this.comment = comment; + this.commentSpecified = commentSpecified; this.isVisible = isVisible; this.generatedColumnDesc = generatedColumnDesc; } @@ -214,17 +231,25 @@ public String getComment(boolean escapeQuota) { return SqlUtils.escapeQuota(comment); } + public boolean isCommentSpecified() { + return commentSpecified; + } + /** * toSql */ public String toSql() { - return toSql("`" + name + "`"); + return toSql("`" + name + "`", true); } /** * Convert this column definition to SQL with a caller-provided column name. */ public String toSql(String columnNameSql) { + return toSql(columnNameSql, commentSpecified); + } + + private String toSql(String columnNameSql, boolean includeComment) { StringBuilder sb = new StringBuilder(); sb.append(columnNameSql).append(" "); sb.append(type.toSql()).append(" "); @@ -266,7 +291,9 @@ public String toSql(String columnNameSql) { sb.append("DEFAULT ").append("NULL").append(" "); } } - sb.append("COMMENT ").append(LogicalPlanBuilderAssistant.quoteStringLiteral(getComment())); + if (includeComment) { + sb.append("COMMENT ").append(LogicalPlanBuilderAssistant.quoteStringLiteral(getComment())); + } return sb.toString(); } @@ -595,6 +622,7 @@ public Column translateToCatalogStyleForSchemaChange() { ConnectContextUtil.getAffectQueryResultInPlanVariables(ConnectContext.get())) .orElse(null)); column.setNullableSpecified(nullableSpecified); + column.setCommentSpecified(commentSpecified); column.setAggregationTypeImplicit(aggTypeImplicit); return column; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 412d2a348b74e9..f02dcf809a4e0c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -924,6 +924,16 @@ public void testRejectsCommentsOnDirectArrayElementAndMapValue() { assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("m.value"), new Column("value", Type.BIGINT, true, "map value comment"), null, 1L), "Iceberg does not support comments on collection element or value fields: m.value"); + Column arrayElementWithEmptyComment = new Column("element", Type.BIGINT, true, ""); + arrayElementWithEmptyComment.setCommentSpecified(true); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), + arrayElementWithEmptyComment, null, 1L), + "Iceberg does not support comments on collection element or value fields: arr.element"); + Column mapValueWithEmptyComment = new Column("value", Type.BIGINT, true, ""); + mapValueWithEmptyComment.setCommentSpecified(true); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("m.value"), + mapValueWithEmptyComment, null, 1L), + "Iceberg does not support comments on collection element or value fields: m.value"); assertUserException(() -> ops.modifyColumnComment( dorisTable, ColumnPath.fromDotName("m.key"), "map key comment", 1L), "Cannot modify comment MAP key nested column"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index 8c087929863f43..66a887c5356f62 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -190,6 +190,30 @@ public void testModifyColumnRoundTripPreservesNullabilityIntent() { .translateToCatalogStyleForSchemaChange().isNullableSpecified()); } + @Test + public void testModifyColumnRoundTripPreservesCommentIntent() { + ModifyColumnOp omitted = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN arr.element BIGINT", + ModifyColumnOp.class, "arr.element"); + Assertions.assertFalse(omitted.getColumnDef() + .translateToCatalogStyleForSchemaChange().isCommentSpecified()); + Assertions.assertFalse(omitted.toSql().contains(" COMMENT ")); + + ModifyColumnOp reparsedOmitted = assertSingleClausePath( + "ALTER TABLE t " + omitted.toSql(), ModifyColumnOp.class, "arr.element"); + Assertions.assertFalse(reparsedOmitted.getColumnDef() + .translateToCatalogStyleForSchemaChange().isCommentSpecified()); + + ModifyColumnOp empty = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN arr.element BIGINT COMMENT ''", + ModifyColumnOp.class, "arr.element"); + ModifyColumnOp reparsedEmpty = assertSingleClausePath( + "ALTER TABLE t " + empty.toSql(), ModifyColumnOp.class, "arr.element"); + Assertions.assertTrue(reparsedEmpty.getColumnDef() + .translateToCatalogStyleForSchemaChange().isCommentSpecified()); + Assertions.assertEquals("", reparsedEmpty.getColumnDef().getComment()); + } + @Test public void testModifyColumnCommentRoundTripEscapesQuotesAndBackslashes() { assertCommentRoundTrip(false); diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy index f5c8dd515c7dd7..012bc88e3d9faf 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy @@ -114,6 +114,14 @@ suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_do sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.value BIGINT COMMENT 'map value comment'""" exception "Iceberg does not support comments on collection element or value fields" } + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN arr_scalar.element BIGINT COMMENT ''""" + exception "Iceberg does not support comments on collection element or value fields" + } + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.value BIGINT COMMENT ''""" + exception "Iceberg does not support comments on collection element or value fields" + } test { sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.`key` COMMENT 'map key comment'""" exception "Cannot modify comment MAP key nested column" From c000f067b06d980a7bdc6dd2c9795e36a46f15d9 Mon Sep 17 00:00:00 2001 From: daidai Date: Sun, 19 Jul 2026 23:31:24 +0800 Subject: [PATCH 25/32] [fix](iceberg) Preserve struct member comments ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Whole-complex Iceberg MODIFY statements decoded STRUCT member comments without SQL-mode-aware quote handling and rendered them without a valid string literal. Comments containing doubled quotes or backslashes could therefore be persisted incorrectly or change after SQL rendering and reparsing. Decode and quote all Nereids string literals through a shared neutral utility, and cover parser round-trip plus Iceberg metadata persistence. ### Release note Preserve quoted and escaped STRUCT member comments during Iceberg complex column schema changes. ### Check List (For Author) - Test: Unit Test - IcebergMetadataOpsValidationTest and IcebergNestedSchemaEvolutionParserTest: 63 tests passed - FE Checkstyle: all 57 modules passed with zero violations after fixing import order - External Iceberg regression suite not run locally - Behavior changed: Yes, STRUCT member comments now retain their decoded value across execution and SQL replay. - Does this need documentation: No --- .../nereids/parser/LogicalPlanBuilder.java | 19 ++-- .../parser/LogicalPlanBuilderAssistant.java | 64 +------------ .../plans/commands/info/ColumnDefinition.java | 4 +- .../commands/info/ModifyColumnCommentOp.java | 4 +- .../doris/nereids/types/StructField.java | 4 +- .../doris/nereids/util/SqlLiteralUtils.java | 96 +++++++++++++++++++ .../IcebergMetadataOpsValidationTest.java | 27 ++++++ ...cebergNestedSchemaEvolutionParserTest.java | 27 ++++++ 8 files changed, 167 insertions(+), 78 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c8b76d320b71f2..d9f964b83e113f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1096,6 +1096,7 @@ import org.apache.doris.nereids.types.coercion.CharacterType; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.RelationUtil; +import org.apache.doris.nereids.util.SqlLiteralUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.policy.FilterType; import org.apache.doris.policy.PolicyTypeEnum; @@ -3752,7 +3753,7 @@ public Literal visitIntegerLiteral(IntegerLiteralContext ctx) { @Override public Literal visitStringLiteral(StringLiteralContext ctx) { - String s = LogicalPlanBuilderAssistant.parseStringLiteral(ctx.STRING_LITERAL().getText()); + String s = SqlLiteralUtils.parseStringLiteral(ctx.STRING_LITERAL().getText()); int strLength = Utils.containChinese(s) ? s.length() * StringLikeLiteral.CHINESE_CHAR_BYTE_LENGTH : s.length(); if (strLength > ScalarType.MAX_VARCHAR_LENGTH) { return new StringLiteral(s); @@ -4232,7 +4233,7 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { } } String comment = ctx.comment != null - ? LogicalPlanBuilderAssistant.parseStringLiteral(ctx.comment.getText()) : ""; + ? SqlLiteralUtils.parseStringLiteral(ctx.comment.getText()) : ""; long autoIncInitValue = -1; if (ctx.AUTO_INCREMENT() != null) { if (ctx.autoIncInitValue != null) { @@ -4291,7 +4292,7 @@ public ColumnDefinitionWithPath visitColumnDefWithPath(ColumnDefWithPathContext } } String comment = ctx.comment != null - ? LogicalPlanBuilderAssistant.parseStringLiteral(ctx.comment.getText()) : ""; + ? SqlLiteralUtils.parseStringLiteral(ctx.comment.getText()) : ""; long autoIncInitValue = -1; if (ctx.AUTO_INCREMENT() != null) { if (ctx.autoIncInitValue != null) { @@ -5494,13 +5495,9 @@ public List visitComplexColTypeList(ComplexColTypeListContext ctx) @Override public StructField visitComplexColType(ComplexColTypeContext ctx) { - String comment; - if (ctx.commentSpec() != null) { - comment = ctx.commentSpec().STRING_LITERAL().getText(); - comment = LogicalPlanBuilderAssistant.escapeBackSlash(comment.substring(1, comment.length() - 1)); - } else { - comment = ""; - } + String comment = ctx.commentSpec() == null ? "" + : SqlLiteralUtils.parseStringLiteral( + ctx.commentSpec().STRING_LITERAL().getText()); return new StructField(ctx.identifier().getText(), typedVisit(ctx.dataType()), true, comment); } @@ -6548,7 +6545,7 @@ public AlterTableOp visitModifyTableCommentClause(ModifyTableCommentClauseContex @Override public AlterTableOp visitModifyColumnCommentClause(ModifyColumnCommentClauseContext ctx) { ColumnPath columnPath = parseColumnPath(ctx.name, visitQualifiedName(ctx.name)); - String comment = LogicalPlanBuilderAssistant.parseStringLiteral(ctx.STRING_LITERAL().getText()); + String comment = SqlLiteralUtils.parseStringLiteral(ctx.STRING_LITERAL().getText()); return new ModifyColumnCommentOp(columnPath, comment); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java index a2dde0054dc648..63a8e9557b1144 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.qe.SqlModeHelper; +import org.apache.doris.nereids.util.SqlLiteralUtils; import com.google.common.collect.ImmutableSet; @@ -43,67 +43,7 @@ private LogicalPlanBuilderAssistant() { * EscapeBackSlash such \n, \t */ public static String escapeBackSlash(String str) { - StringBuilder sb = new StringBuilder(); - int strLen = str.length(); - for (int i = 0; i < strLen; ++i) { - char c = str.charAt(i); - if (c == '\\' && (i + 1) < strLen) { - switch (str.charAt(i + 1)) { - case 'n': - sb.append('\n'); - break; - case 't': - sb.append('\t'); - break; - case 'r': - sb.append('\r'); - break; - case 'b': - sb.append('\b'); - break; - case '0': - sb.append('\0'); // Ascii null - break; - case 'Z': // ^Z must be escaped on Win32 - sb.append('\032'); - break; - case '_': - case '%': - sb.append('\\'); // remember prefix for wildcard - sb.append(str.charAt(i + 1)); - break; - default: - sb.append(str.charAt(i + 1)); - break; - } - i++; - } else { - sb.append(c); - } - } - return sb.toString(); - } - - /** - * Decode a STRING_LITERAL token according to the current SQL mode. - */ - public static String parseStringLiteral(String text) { - String value = text.substring(1, text.length() - 1); - if (text.charAt(0) == '\'') { - value = value.replace("''", "'"); - } else { - value = value.replace("\"\"", "\""); - } - return SqlModeHelper.hasNoBackSlashEscapes() ? value : escapeBackSlash(value); - } - - /** - * Quote a value as a STRING_LITERAL that can be parsed under the current SQL mode. - */ - public static String quoteStringLiteral(String value) { - String escaped = SqlModeHelper.hasNoBackSlashEscapes() - ? value : value.replace("\\", "\\\\"); - return "\"" + escaped.replace("\"", "\"\"") + "\""; + return SqlLiteralUtils.unescapeBackSlash(str); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 1b52218a354218..3513e40cb4ff39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -27,7 +27,6 @@ import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.util.SqlUtils; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.parser.LogicalPlanBuilderAssistant; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BitmapType; @@ -40,6 +39,7 @@ import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.CharacterType; +import org.apache.doris.nereids.util.SqlLiteralUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectContextUtil; import org.apache.doris.qe.SessionVariable; @@ -292,7 +292,7 @@ private String toSql(String columnNameSql, boolean includeComment) { } } if (includeComment) { - sb.append("COMMENT ").append(LogicalPlanBuilderAssistant.quoteStringLiteral(getComment())); + sb.append("COMMENT ").append(SqlLiteralUtils.quoteStringLiteral(getComment())); } return sb.toString(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java index 6dfeb28475d303..9b0e4cadab3514 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java @@ -21,7 +21,7 @@ import org.apache.doris.analysis.ColumnPath; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; -import org.apache.doris.nereids.parser.LogicalPlanBuilderAssistant; +import org.apache.doris.nereids.util.SqlLiteralUtils; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; @@ -90,7 +90,7 @@ public boolean allowOpRowBinlog() { public String toSql() { StringBuilder sb = new StringBuilder(); sb.append("MODIFY COLUMN ").append(columnPath.toSql()); - sb.append(" COMMENT ").append(LogicalPlanBuilderAssistant.quoteStringLiteral(comment)); + sb.append(" COMMENT ").append(SqlLiteralUtils.quoteStringLiteral(comment)); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java index 63e94bc369bb0c..8766581289e2de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.types; +import org.apache.doris.nereids.util.SqlLiteralUtils; import org.apache.doris.nereids.util.Utils; import java.util.Objects; @@ -85,7 +86,8 @@ public org.apache.doris.catalog.StructField toCatalogDataType() { public String toSql() { return name + ":" + dataType.toSql() + (nullable ? "" : " NOT NULL") - + (comment.isEmpty() ? "" : " COMMENT " + comment); + + (comment.isEmpty() ? "" : " COMMENT " + + SqlLiteralUtils.quoteStringLiteral(comment)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java new file mode 100644 index 00000000000000..5d87bc08f5f56d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.util; + +import org.apache.doris.qe.SqlModeHelper; + +/** + * Utilities for decoding and rendering SQL string literals. + */ +public final class SqlLiteralUtils { + + private SqlLiteralUtils() { + } + + /** + * Decode backslash escape sequences used in SQL string literals. + */ + public static String unescapeBackSlash(String value) { + StringBuilder result = new StringBuilder(); + int length = value.length(); + for (int i = 0; i < length; ++i) { + char current = value.charAt(i); + if (current == '\\' && (i + 1) < length) { + switch (value.charAt(i + 1)) { + case 'n': + result.append('\n'); + break; + case 't': + result.append('\t'); + break; + case 'r': + result.append('\r'); + break; + case 'b': + result.append('\b'); + break; + case '0': + result.append('\0'); + break; + case 'Z': + result.append('\032'); + break; + case '_': + case '%': + result.append('\\'); + result.append(value.charAt(i + 1)); + break; + default: + result.append(value.charAt(i + 1)); + break; + } + i++; + } else { + result.append(current); + } + } + return result.toString(); + } + + /** + * Decode a STRING_LITERAL token according to the current SQL mode. + */ + public static String parseStringLiteral(String text) { + String value = text.substring(1, text.length() - 1); + if (text.charAt(0) == '\'') { + value = value.replace("''", "'"); + } else { + value = value.replace("\"\"", "\""); + } + return SqlModeHelper.hasNoBackSlashEscapes() ? value : unescapeBackSlash(value); + } + + /** + * Quote a value as a STRING_LITERAL that can be parsed under the current SQL mode. + */ + public static String quoteStringLiteral(String value) { + String escaped = SqlModeHelper.hasNoBackSlashEscapes() + ? value : value.replace("\\", "\\\\"); + return "\"" + escaped.replace("\"", "\"\"") + "\""; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index f02dcf809a4e0c..d2f6ea4b0f919a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -266,6 +266,33 @@ public void testComplexModifyPreservesRequiredNestedFields() throws Throwable { Mockito.verify(updateSchema, Mockito.times(3)).commit(); } + @Test + public void testComplexModifyPersistsDecodedStructMemberComment() throws Throwable { + Schema schema = new Schema(Types.NestedField.optional(1, "info", Types.StructType.of( + Types.NestedField.optional(2, "payload", Types.StructType.of( + Types.NestedField.optional(3, "name", Types.StringType.get(), "old comment")))))); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + String decodedComment = "owner's \"field\" C:\\tmp\\"; + Column column = new Column("payload", new StructType( + new StructField("name", Type.STRING, decodedComment, true)), true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), column, null, 1L); + } + + Mockito.verify(updateSchema).updateColumnDoc("info.payload.name", decodedComment); + Mockito.verify(updateSchema).commit(); + } + @Test public void testPrimitiveModifyPreservesRequiredNestedField() throws Throwable { Schema schema = requiredNestedSchema(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index 66a887c5356f62..ae91d1b13744a1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnCommentOp; import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnOp; import org.apache.doris.nereids.trees.plans.commands.info.RenameColumnOp; +import org.apache.doris.nereids.types.StructType; import org.apache.doris.qe.SqlModeHelper; import org.junit.jupiter.api.Assertions; @@ -232,6 +233,32 @@ public void testRegularColumnDefinitionCommentRoundTripEscapesQuotesAndBackslash assertRegularColumnDefinitionCommentRoundTrip(true); } + @Test + public void testStructMemberCommentRoundTripEscapesQuotesAndBackslashes() { + assertStructMemberCommentRoundTrip(false); + assertStructMemberCommentRoundTrip(true); + } + + private void assertStructMemberCommentRoundTrip(boolean noBackslashEscapes) { + try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { + mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(noBackslashEscapes); + + String sqlPath = noBackslashEscapes ? "C:\\tmp\\" : "C:\\\\tmp\\\\"; + String expectedComment = "owner's \"field\" C:\\tmp\\"; + ModifyColumnOp modify = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.payload " + + "STRUCT", + ModifyColumnOp.class, "info.payload"); + StructType structType = (StructType) modify.getColumnDef().getType(); + Assertions.assertEquals(expectedComment, structType.getFields().get(0).getComment()); + + ModifyColumnOp reparsedModify = assertSingleClausePath( + "ALTER TABLE t " + modify.toSql(), ModifyColumnOp.class, "info.payload"); + StructType reparsedType = (StructType) reparsedModify.getColumnDef().getType(); + Assertions.assertEquals(expectedComment, reparsedType.getFields().get(0).getComment()); + } + } + private void assertRegularColumnDefinitionCommentRoundTrip(boolean noBackslashEscapes) { try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(noBackslashEscapes); From 1eb8cc8e87b9c7228cfe65024d54fc9397761b03 Mon Sep 17 00:00:00 2001 From: daidai Date: Mon, 20 Jul 2026 01:43:08 +0800 Subject: [PATCH 26/32] [fix](iceberg) Preserve column path identity in schema changes ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: A quoted one-component column name containing a dot could be resolved as an Iceberg nested path and then reduced to its leaf, allowing MODIFY COLUMN to update a different top-level column. STRUCT member SQL rendering also emitted decoded identifiers without quoting, so reserved names or names containing backticks could not survive ALTER statement serialization and reparse. Resolve top-level modifies against the root struct while preserving the original component boundary and existing missing-column error, and quote STRUCT member identifiers with the shared SQL identifier renderer. ### Release note Iceberg nested schema changes now preserve quoted top-level path identity, and generated ALTER SQL correctly escapes STRUCT member names. ### Check List (For Author) - Test: Unit Test - 115 targeted FE unit tests passed - FE Checkstyle passed for the 25-module fe-core reactor with zero violations - External Iceberg regression suites were not run locally - Behavior changed: Yes, invalid quoted dotted-column modifies no longer update an unrelated column, and STRUCT member SQL is rendered with escaped identifiers. - Does this need documentation: No --- .../iceberg/IcebergMetadataOps.java | 31 ++++++------- .../doris/nereids/types/StructField.java | 3 +- .../IcebergMetadataOpsValidationTest.java | 44 +++++++++++++++++++ ...cebergNestedSchemaEvolutionParserTest.java | 31 +++++++++++++ .../rules/rewrite/PruneNestedColumnTest.java | 33 +++++++++----- .../data-types/struct-md.groovy | 12 ++--- .../sql-function/test_array_function.groovy | 6 +-- 7 files changed, 123 insertions(+), 37 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index b9768bdcf5aa5c..5a512e4b58210f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -1042,21 +1042,22 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String @Override public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { - modifyTopLevelColumn(dorisTable, column, position, updateTime, true); + modifyTopLevelColumn(dorisTable, ColumnPath.of(column.getName()), column, position, updateTime, true); } - private void modifyTopLevelColumn(ExternalTable dorisTable, Column column, ColumnPosition position, - long updateTime, boolean legacyMode) throws UserException { + private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, + ColumnPosition position, long updateTime, boolean legacyMode) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); - validateRowLineageColumnMutation(icebergTable, column.getName(), "modify"); - NestedField currentCol = icebergTable.schema().caseInsensitiveFindField(column.getName()); + validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify"); + NestedField currentCol = icebergTable.schema().asStruct() + .caseInsensitiveField(columnPath.getTopLevelName()); if (currentCol == null) { - throw new UserException("Column " + column.getName() + " does not exist"); + throw new UserException("Column " + columnPath.getTopLevelName() + " does not exist"); } - ResolvedColumnPath columnPath = new ResolvedColumnPath(ColumnPath.of(currentCol.name()), + ResolvedColumnPath resolvedPath = new ResolvedColumnPath(ColumnPath.of(currentCol.name()), currentCol.type(), currentCol); - validateModifyColumnMetadata(column, columnPath.getFullPath(), !legacyMode); + validateModifyColumnMetadata(column, resolvedPath.getFullPath(), !legacyMode); org.apache.iceberg.types.Type targetType; if (column.getType().isComplexType()) { validateForModifyComplexColumn(column, currentCol); @@ -1064,24 +1065,24 @@ private void modifyTopLevelColumn(ExternalTable dorisTable, Column column, Colum } else { validateForModifyColumn(column, currentCol); targetType = resolvePrimitiveTypeForModify( - currentCol.type(), column.getType(), columnPath.getFullPath()); + currentCol.type(), column.getType(), resolvedPath.getFullPath()); } UpdateSchema updateSchema = icebergTable.updateSchema(); if (column.getType().isComplexType()) { - applyComplexTypeChange(updateSchema, columnPath.getFullPath(), currentCol.type(), + applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), column.getType(), legacyMode); if (!Objects.equals(currentCol.doc(), column.getComment())) { - updateSchema.updateColumnDoc(columnPath.getFullPath(), column.getComment()); + updateSchema.updateColumnDoc(resolvedPath.getFullPath(), column.getComment()); } } else { - applyPrimitiveColumnChange(updateSchema, columnPath.getFullPath(), currentCol, + applyPrimitiveColumnChange(updateSchema, resolvedPath.getFullPath(), currentCol, targetType.asPrimitiveType(), column.getComment()); } - applyExplicitNullableChange(updateSchema, columnPath.getFullPath(), column, legacyMode); + applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column, legacyMode); if (position != null) { - applyPosition(updateSchema, position, columnPath.getColumnPath(), icebergTable.schema(), "modify"); + applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); } try { executionAuthenticator.execute(() -> updateSchema.commit()); @@ -1096,7 +1097,7 @@ private void modifyTopLevelColumn(ExternalTable dorisTable, Column column, Colum public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, long updateTime) throws UserException { if (!columnPath.isNested()) { - modifyTopLevelColumn(dorisTable, column, position, updateTime, false); + modifyTopLevelColumn(dorisTable, columnPath, column, position, updateTime, false); return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java index 8766581289e2de..0b697d7aa10481 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.types; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.nereids.util.SqlLiteralUtils; import org.apache.doris.nereids.util.Utils; @@ -84,7 +85,7 @@ public org.apache.doris.catalog.StructField toCatalogDataType() { } public String toSql() { - return name + ":" + dataType.toSql() + return SqlUtils.getIdentSql(name) + ":" + dataType.toSql() + (nullable ? "" : " NOT NULL") + (comment.isEmpty() ? "" : " COMMENT " + SqlLiteralUtils.quoteStringLiteral(comment)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index d2f6ea4b0f919a..2db33dee21ea0c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -345,6 +345,50 @@ public void testTopLevelModifyPreservesRequiredMixedCaseFields() throws Throwabl Mockito.verify(updateSchema, Mockito.times(2)).commit(); } + @Test + public void testTopLevelModifyDoesNotResolveQuotedComponentAsNestedPath() { + Schema schema = new Schema( + Types.NestedField.optional(1, "a", Types.StructType.of( + Types.NestedField.optional(2, "b", Types.IntegerType.get()))), + Types.NestedField.optional(3, "b", Types.IntegerType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), + new Column("a.b", Type.BIGINT, true), null, 1L), + "Column a.b does not exist"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testTopLevelModifyPreservesDottedTopLevelName() throws Throwable { + Schema schema = new Schema(Types.NestedField.optional(1, "a.b", Types.IntegerType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), + new Column("a.b", Type.BIGINT, true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("a.b", Types.LongType.get(), ""); + Mockito.verify(updateSchema).commit(); + } + @Test public void testPrimitiveModifyPreservesActualTypeWhenMappingDisabled() throws Throwable { Schema schema = mappedPrimitiveSchema(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index ae91d1b13744a1..6f09f8734d1aff 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -121,6 +121,11 @@ public void testLegacyStringConstructorsKeepDottedTopLevelNames() { @Test public void testQuotedNestedIdentifiersAreNormalized() { + ModifyColumnOp dottedTopLevel = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN `a.b` BIGINT", + ModifyColumnOp.class, "a.b"); + Assertions.assertFalse(dottedTopLevel.getColumnPath().isNested()); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN info.`Metric` BIGINT", ModifyColumnOp.class, "info.Metric"); assertSingleClausePath("ALTER TABLE t MODIFY COLUMN m_scalar.`key` BIGINT", @@ -145,6 +150,27 @@ public void testQuotedNestedIdentifiersAreNormalized() { Assertions.assertEquals("New`Metric", reparsedRename.getNewColName()); } + @Test + public void testStructMemberIdentifiersRoundTrip() { + AddColumnOp add = assertSingleClausePath( + "ALTER TABLE t ADD COLUMN info.payload " + + "STRUCT<`key`:INT,`Metric``Name`:STRING> NULL", + AddColumnOp.class, "info.payload"); + assertStructMemberNames((StructType) add.getColumnDef().getType()); + AddColumnOp reparsedAdd = assertSingleClausePath( + "ALTER TABLE t " + add.toSql(), AddColumnOp.class, "info.payload"); + assertStructMemberNames((StructType) reparsedAdd.getColumnDef().getType()); + + ModifyColumnOp modify = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.payload " + + "STRUCT<`key`:INT,`Metric``Name`:STRING>", + ModifyColumnOp.class, "info.payload"); + assertStructMemberNames((StructType) modify.getColumnDef().getType()); + ModifyColumnOp reparsedModify = assertSingleClausePath( + "ALTER TABLE t " + modify.toSql(), ModifyColumnOp.class, "info.payload"); + assertStructMemberNames((StructType) reparsedModify.getColumnDef().getType()); + } + @Test public void testEmptyQuotedIdentifiersAreRejectedAsParseErrors() { for (String sql : List.of( @@ -259,6 +285,11 @@ private void assertStructMemberCommentRoundTrip(boolean noBackslashEscapes) { } } + private void assertStructMemberNames(StructType structType) { + Assertions.assertEquals("key", structType.getFields().get(0).getName()); + Assertions.assertEquals("metric`name", structType.getFields().get(1).getName()); + } + private void assertRegularColumnDefinitionCommentRoundTrip(boolean noBackslashEscapes) { try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(noBackslashEscapes); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index efbe72a75acecd..4309493abb3be3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -1015,29 +1015,38 @@ public void testDataTypeAccessTree() { DataTypeAccessTree tree = trees.get(0).second; Assertions.assertEquals(NullType.INSTANCE, tree.getType()); Assertions.assertEquals(1, tree.getChildren().size()); - Assertions.assertEquals("STRUCT", tree.getChildren().get("s").getType().toSql()); + Assertions.assertEquals("STRUCT<`city`:TEXT>", tree.getChildren().get("s").getType().toSql()); SlotReference slot = trees.get(0).first; Type columnType = slot.getOriginalColumn().get().getType(); Assertions.assertEquals("struct>>>", columnType.toSql()); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "city"), "STRUCT"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "KEYS"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES", "a"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES", "b"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*", "a"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*", "b"), "STRUCT>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "city"), "STRUCT<`city`:TEXT>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data"), + "STRUCT<`data`:ARRAY>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*"), + "STRUCT<`data`:ARRAY>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "KEYS"), + "STRUCT<`data`:ARRAY>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES"), + "STRUCT<`data`:ARRAY>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES", "a"), + "STRUCT<`data`:ARRAY>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES", "b"), + "STRUCT<`data`:ARRAY>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*"), + "STRUCT<`data`:ARRAY>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*", "a"), + "STRUCT<`data`:ARRAY>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*", "b"), + "STRUCT<`data`:ARRAY>>>"); setAccessPathsAndAssertType(slot, ImmutableList.of( ImmutableList.of("s", "data", "*", "*", "b"), ImmutableList.of("s", "city") ), - "STRUCT>>>" + "STRUCT<`city`:TEXT,`data`:ARRAY>>>" ); } diff --git a/regression-test/suites/doc/sql-manual/basic-elements/data-types/struct-md.groovy b/regression-test/suites/doc/sql-manual/basic-elements/data-types/struct-md.groovy index ec088e300e8b31..1ab47fd80dfdc7 100644 --- a/regression-test/suites/doc/sql-manual/basic-elements/data-types/struct-md.groovy +++ b/regression-test/suites/doc/sql-manual/basic-elements/data-types/struct-md.groovy @@ -184,7 +184,7 @@ suite("struct-md", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ - exception "Aggregate type SUM is not compatible with primitive type STRUCT" + exception "Aggregate type SUM is not compatible with primitive type STRUCT<`id`:INT,`name`:TEXT>" } test{ @@ -200,7 +200,7 @@ suite("struct-md", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ - exception "Aggregate type MIN is not compatible with primitive type STRUCT" + exception "Aggregate type MIN is not compatible with primitive type STRUCT<`id`:INT,`name`:TEXT>" } test { @@ -216,7 +216,7 @@ suite("struct-md", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ - exception "Aggregate type MAX is not compatible with primitive type STRUCT" + exception "Aggregate type MAX is not compatible with primitive type STRUCT<`id`:INT,`name`:TEXT>" } test { @@ -232,7 +232,7 @@ suite("struct-md", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ - exception "Aggregate type HLL_UNION is not compatible with primitive type STRUCT" + exception "Aggregate type HLL_UNION is not compatible with primitive type STRUCT<`id`:INT,`name`:TEXT>" } test { @@ -248,7 +248,7 @@ suite("struct-md", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ - exception "Aggregate type BITMAP_UNION is not compatible with primitive type STRUCT" + exception "Aggregate type BITMAP_UNION is not compatible with primitive type STRUCT<`id`:INT,`name`:TEXT>" } test { @@ -264,7 +264,7 @@ suite("struct-md", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ - exception "Aggregate type QUANTILE_UNION is not compatible with primitive type STRUCT" + exception "Aggregate type QUANTILE_UNION is not compatible with primitive type STRUCT<`id`:INT,`name`:TEXT>" } test { diff --git a/regression-test/suites/doc/sql-manual/sql-function/test_array_function.groovy b/regression-test/suites/doc/sql-manual/sql-function/test_array_function.groovy index 6f9f1f7b0e0ca4..c6073b28c6763b 100644 --- a/regression-test/suites/doc/sql-manual/sql-function/test_array_function.groovy +++ b/regression-test/suites/doc/sql-manual/sql-function/test_array_function.groovy @@ -126,7 +126,7 @@ suite("test_array_function_doc", "p0") { test { sql """ SELECT ARRAYS_OVERLAP(array_struct, array_struct) from ${tableName}; """ - exception "arrays_overlap does not support types: ARRAY>" + exception "arrays_overlap does not support types: ARRAY>" } test { @@ -400,7 +400,7 @@ suite("test_array_function_doc", "p0") { test { sql """ SELECT ARRAY_UNION(array_struct, array_struct) from ${tableName}; """ - exception "array_union does not support types: ARRAY>" + exception "array_union does not support types: ARRAY>" } test { @@ -520,4 +520,4 @@ suite("test_array_function_doc", "p0") { qt_sql """ SELECT ARRAY_SORTBY(x -> x[1], [[1,2],[3,4]]); """ qt_sql """ SELECT ARRAY_SORT([[1,2],[3,4]]); """ qt_sql """ SELECT ARRAY_REVERSE_SORT([[1,2],[3,4]]); """ -} \ No newline at end of file +} From 05f39ca8b7cb298ecbeac1edb13276057a059167 Mon Sep 17 00:00:00 2001 From: daidai Date: Mon, 20 Jul 2026 15:03:00 +0800 Subject: [PATCH 27/32] [fix](iceberg) Preserve unspecified nested field comments ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Path-aware Iceberg MODIFY COLUMN operations treated omitted COMMENT clauses as empty comments. Primitive type changes and whole-STRUCT changes could therefore erase existing field documentation. Preserve comment intent through nested STRUCT fields and reuse the current Iceberg documentation unless COMMENT is explicitly specified; explicit COMMENT empty still clears it. ### Release note Iceberg nested column modifications now preserve existing field comments unless COMMENT is explicitly specified. ### Check List (For Author) - Test: Unit Test - Targeted FE unit tests: 69 passed - Iceberg Spark interoperability regression coverage added; local execution was blocked before the suite body by an unavailable REST Catalog endpoint - Behavior changed: Yes, omitted COMMENT clauses no longer clear existing Iceberg field comments - Does this need documentation: No --- .../iceberg/IcebergMetadataOps.java | 56 +++++----- .../nereids/parser/LogicalPlanBuilder.java | 3 +- .../apache/doris/nereids/types/DataType.java | 3 +- .../doris/nereids/types/StructField.java | 19 +++- .../IcebergMetadataOpsValidationTest.java | 101 ++++++++++++++++-- ...cebergNestedSchemaEvolutionParserTest.java | 22 ++++ .../org/apache/doris/catalog/StructField.java | 19 +++- ...chema_evolution_spark_doris_interop.groovy | 44 ++++++++ 8 files changed, 223 insertions(+), 44 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 5a512e4b58210f..a8d3d569e3dd8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -1069,15 +1069,16 @@ private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPat } UpdateSchema updateSchema = icebergTable.updateSchema(); + String targetComment = resolveTargetComment(currentCol, column, legacyMode); if (column.getType().isComplexType()) { applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), column.getType(), legacyMode); - if (!Objects.equals(currentCol.doc(), column.getComment())) { - updateSchema.updateColumnDoc(resolvedPath.getFullPath(), column.getComment()); + if (!Objects.equals(currentCol.doc(), targetComment)) { + updateSchema.updateColumnDoc(resolvedPath.getFullPath(), targetComment); } } else { applyPrimitiveColumnChange(updateSchema, resolvedPath.getFullPath(), currentCol, - targetType.asPrimitiveType(), column.getComment()); + targetType.asPrimitiveType(), targetComment); } applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column, legacyMode); @@ -1122,15 +1123,16 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column } UpdateSchema updateSchema = icebergTable.updateSchema(); + String targetComment = resolveTargetComment(currentCol, column, false); if (column.getType().isComplexType()) { applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), column.getType(), false); - if (!Objects.equals(currentCol.doc(), column.getComment())) { - updateSchema.updateColumnDoc(resolvedPath.getFullPath(), column.getComment()); + if (!Objects.equals(currentCol.doc(), targetComment)) { + updateSchema.updateColumnDoc(resolvedPath.getFullPath(), targetComment); } } else { applyPrimitiveColumnChange(updateSchema, resolvedPath.getFullPath(), currentCol, - targetType.asPrimitiveType(), column.getComment()); + targetType.asPrimitiveType(), targetComment); } applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column, false); @@ -1190,6 +1192,10 @@ private void applyExplicitNullableChange(UpdateSchema updateSchema, String colum } } + private String resolveTargetComment(NestedField currentCol, Column column, boolean legacyMode) { + return legacyMode || column.isCommentSpecified() ? column.getComment() : currentCol.doc(); + } + private org.apache.iceberg.types.Type.PrimitiveType resolvePrimitiveTypeForModify( org.apache.iceberg.types.Type currentIcebergType, org.apache.doris.catalog.Type requestedDorisType, String columnPath) throws UserException { @@ -1492,19 +1498,19 @@ private void validateExistingTypeChange(org.apache.iceberg.types.Type oldIceberg private void applyComplexTypeChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldIcebergType, - org.apache.doris.catalog.Type newDorisType, boolean legacyNullableExplicit) throws UserException { + org.apache.doris.catalog.Type newDorisType, boolean legacyMode) throws UserException { switch (oldIcebergType.typeId()) { case STRUCT: applyStructChange(updateSchema, path, oldIcebergType.asStructType(), (StructType) newDorisType, - legacyNullableExplicit); + legacyMode); break; case LIST: applyListChange(updateSchema, path, (Types.ListType) oldIcebergType, (ArrayType) newDorisType, - legacyNullableExplicit); + legacyMode); break; case MAP: applyMapChange(updateSchema, path, (Types.MapType) oldIcebergType, (MapType) newDorisType, - legacyNullableExplicit); + legacyMode); break; default: throw new UserException("Unsupported complex type for modify: " + oldIcebergType); @@ -1512,7 +1518,7 @@ private void applyComplexTypeChange(UpdateSchema updateSchema, String path, } private void applyStructChange(UpdateSchema updateSchema, String path, - Types.StructType oldStructType, StructType newStructType, boolean legacyNullableExplicit) + Types.StructType oldStructType, StructType newStructType, boolean legacyMode) throws UserException { List oldFields = oldStructType.fields(); List newFields = newStructType.getFields(); @@ -1528,26 +1534,28 @@ private void applyStructChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldFieldType = oldField.type(); org.apache.doris.catalog.Type newFieldType = newField.getType(); + String targetComment = legacyMode || newField.isCommentSpecified() + ? newField.getComment() : oldField.doc(); if (oldFieldType.isPrimitiveType()) { org.apache.doris.catalog.Type oldDorisFieldType = mappedDorisType(oldFieldType); boolean typeChanged = !isSameMappedDorisType(oldDorisFieldType, newFieldType); - boolean commentChanged = !Objects.equals(oldField.doc(), newField.getComment()); + boolean commentChanged = !Objects.equals(oldField.doc(), targetComment); if (typeChanged) { org.apache.iceberg.types.Type newIcebergFieldType = toIcebergTypeForSchemaChange(newFieldType, fieldPath); updateSchema.updateColumn(fieldPath, newIcebergFieldType.asPrimitiveType(), - newField.getComment()); + targetComment); } else if (commentChanged) { - updateSchema.updateColumnDoc(fieldPath, newField.getComment()); + updateSchema.updateColumnDoc(fieldPath, targetComment); } } else { applyComplexTypeChange(updateSchema, fieldPath, oldFieldType, newFieldType, - legacyNullableExplicit); - if (!Objects.equals(oldField.doc(), newField.getComment())) { - updateSchema.updateColumnDoc(fieldPath, newField.getComment()); + legacyMode); + if (!Objects.equals(oldField.doc(), targetComment)) { + updateSchema.updateColumnDoc(fieldPath, targetComment); } } - if (legacyNullableExplicit && !oldField.isOptional() && newField.getContainsNull()) { + if (legacyMode && !oldField.isOptional() && newField.getContainsNull()) { updateSchema.makeColumnOptional(fieldPath); } } @@ -1564,7 +1572,7 @@ private void applyStructChange(UpdateSchema updateSchema, String path, } private void applyListChange(UpdateSchema updateSchema, String path, - Types.ListType oldListType, ArrayType newArrayType, boolean legacyNullableExplicit) + Types.ListType oldListType, ArrayType newArrayType, boolean legacyMode) throws UserException { String elementPath = path + "." + oldListType.field(oldListType.elementId()).name(); if (oldListType.isElementOptional() && !newArrayType.getContainsNull()) { @@ -1581,15 +1589,15 @@ private void applyListChange(UpdateSchema updateSchema, String path, } } else { applyComplexTypeChange(updateSchema, elementPath, oldElementType, newElementType, - legacyNullableExplicit); + legacyMode); } - if (legacyNullableExplicit && !oldListType.isElementOptional() && newArrayType.getContainsNull()) { + if (legacyMode && !oldListType.isElementOptional() && newArrayType.getContainsNull()) { updateSchema.makeColumnOptional(elementPath); } } private void applyMapChange(UpdateSchema updateSchema, String path, - Types.MapType oldMapType, MapType newMapType, boolean legacyNullableExplicit) + Types.MapType oldMapType, MapType newMapType, boolean legacyMode) throws UserException { org.apache.iceberg.types.Type oldKeyType = oldMapType.keyType(); org.apache.doris.catalog.Type newKeyType = newMapType.getKeyType(); @@ -1614,9 +1622,9 @@ private void applyMapChange(UpdateSchema updateSchema, String path, } } else { applyComplexTypeChange(updateSchema, valuePath, oldValueType, newValueType, - legacyNullableExplicit); + legacyMode); } - if (legacyNullableExplicit && !oldMapType.isValueOptional() && newMapType.getIsValueContainsNull()) { + if (legacyMode && !oldMapType.isValueOptional() && newMapType.getIsValueContainsNull()) { updateSchema.makeColumnOptional(valuePath); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index d9f964b83e113f..42695f85a8d34f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -5498,7 +5498,8 @@ public StructField visitComplexColType(ComplexColTypeContext ctx) { String comment = ctx.commentSpec() == null ? "" : SqlLiteralUtils.parseStringLiteral( ctx.commentSpec().STRING_LITERAL().getText()); - return new StructField(ctx.identifier().getText(), typedVisit(ctx.dataType()), true, comment); + return new StructField(ctx.identifier().getText(), typedVisit(ctx.dataType()), true, + comment, ctx.commentSpec() != null); } private String parseConstant(ConstantContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java index a47352a29e60cc..c6a5dc1b0285ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java @@ -468,7 +468,8 @@ public static DataType fromCatalogType(Type type) { if (type.isStructType()) { List structFields = ((org.apache.doris.catalog.StructType) (type)).getFields().stream() .map(cf -> new StructField(cf.getName(), fromCatalogType(cf.getType()), - cf.getContainsNull(), cf.getComment() == null ? "" : cf.getComment())) + cf.getContainsNull(), cf.getComment() == null ? "" : cf.getComment(), + cf.isCommentSpecified())) .collect(ImmutableList.toImmutableList()); return new StructType(structFields); } else if (type.isMapType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java index 0b697d7aa10481..c008886b4677c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java @@ -34,6 +34,7 @@ public class StructField { private final DataType dataType; private final boolean nullable; private final String comment; + private final boolean commentSpecified; /** * StructField Constructor @@ -42,10 +43,16 @@ public class StructField { * @param nullable Indicates if values of this field can be `null` values */ public StructField(String name, DataType dataType, boolean nullable, String comment) { + this(name, dataType, nullable, comment, comment != null && !comment.isEmpty()); + } + + public StructField(String name, DataType dataType, boolean nullable, String comment, + boolean commentSpecified) { this.name = Objects.requireNonNull(name, "name should not be null").toLowerCase(); this.dataType = Objects.requireNonNull(dataType, "dataType should not be null"); this.nullable = nullable; this.comment = Objects.requireNonNull(comment, "comment should not be null"); + this.commentSpecified = commentSpecified; } public String getName() { @@ -64,6 +71,10 @@ public String getComment() { return comment; } + public boolean isCommentSpecified() { + return commentSpecified; + } + public StructField conversion() { if (this.dataType.equals(dataType.conversion())) { return this; @@ -72,22 +83,22 @@ public StructField conversion() { } public StructField withDataType(DataType dataType) { - return new StructField(name, dataType, nullable, comment); + return new StructField(name, dataType, nullable, comment, commentSpecified); } public StructField withDataTypeAndNullable(DataType dataType, boolean nullable) { - return new StructField(name, dataType, nullable, comment); + return new StructField(name, dataType, nullable, comment, commentSpecified); } public org.apache.doris.catalog.StructField toCatalogDataType() { return new org.apache.doris.catalog.StructField( - name, dataType.toCatalogDataType(), comment, nullable); + name, dataType.toCatalogDataType(), comment, nullable, commentSpecified); } public String toSql() { return SqlUtils.getIdentSql(name) + ":" + dataType.toSql() + (nullable ? "" : " NOT NULL") - + (comment.isEmpty() ? "" : " COMMENT " + + (!commentSpecified ? "" : " COMMENT " + SqlLiteralUtils.quoteStringLiteral(comment)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 2db33dee21ea0c..e4b388a8796993 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -293,6 +293,85 @@ public void testComplexModifyPersistsDecodedStructMemberComment() throws Throwab Mockito.verify(updateSchema).commit(); } + @Test + public void testPrimitiveModifyPreservesOmittedCommentAndClearsExplicitEmptyComment() throws Throwable { + Schema schema = new Schema( + Types.NestedField.optional(1, "info", Types.StructType.of( + Types.NestedField.optional(2, "metric", Types.IntegerType.get(), "metric doc"), + Types.NestedField.optional(3, "clear_me", Types.StringType.get(), "clear doc"))), + Types.NestedField.optional(4, "top_metric", Types.IntegerType.get(), "top metric doc")); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + Column clearComment = new Column("clear_me", Type.STRING, true, ""); + clearComment.setCommentSpecified(true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), + new Column("metric", Type.BIGINT, true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.of("top_metric"), + new Column("top_metric", Type.BIGINT, true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.clear_me"), + clearComment, null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("info.metric", Types.LongType.get(), "metric doc"); + Mockito.verify(updateSchema).updateColumn("top_metric", Types.LongType.get(), "top metric doc"); + Mockito.verify(updateSchema).updateColumnDoc("info.clear_me", ""); + Mockito.verify(updateSchema, Mockito.times(3)).commit(); + } + + @Test + public void testFullStructModifyPreservesOmittedChildComments() throws Throwable { + Schema schema = new Schema(Types.NestedField.optional(1, "info", Types.StructType.of( + Types.NestedField.optional(2, "payload", Types.StructType.of( + Types.NestedField.optional(3, "metric", Types.IntegerType.get(), "metric doc"), + Types.NestedField.optional(4, "clear_me", Types.StringType.get(), "clear doc"), + Types.NestedField.optional(5, "details", Types.StructType.of( + Types.NestedField.optional( + 6, "count", Types.IntegerType.get(), "count doc")), "details doc")), + "payload doc")))); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + StructType detailsType = new StructType( + new StructField("count", Type.BIGINT, "", true, false)); + StructType payloadType = new StructType( + new StructField("metric", Type.BIGINT, "", true, false), + new StructField("clear_me", Type.STRING, "", true, true), + new StructField("details", detailsType, "", true, false)); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), + new Column("payload", payloadType, true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn( + "info.payload.metric", Types.LongType.get(), "metric doc"); + Mockito.verify(updateSchema).updateColumnDoc("info.payload.clear_me", ""); + Mockito.verify(updateSchema).updateColumn( + "info.payload.details.count", Types.LongType.get(), "count doc"); + Mockito.verify(updateSchema, Mockito.never()).updateColumnDoc( + Mockito.eq("info.payload"), Mockito.nullable(String.class)); + Mockito.verify(updateSchema, Mockito.never()).updateColumnDoc( + Mockito.eq("info.payload.details"), Mockito.nullable(String.class)); + Mockito.verify(updateSchema).commit(); + } + @Test public void testPrimitiveModifyPreservesRequiredNestedField() throws Throwable { Schema schema = requiredNestedSchema(); @@ -311,7 +390,7 @@ public void testPrimitiveModifyPreservesRequiredNestedField() throws Throwable { new Column("metric", Type.BIGINT, true), null, 1L); } - Mockito.verify(updateSchema).updateColumn("info.metric", Types.LongType.get(), ""); + Mockito.verify(updateSchema).updateColumn("info.metric", Types.LongType.get(), null); Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional(Mockito.anyString()); Mockito.verify(updateSchema).commit(); } @@ -339,7 +418,7 @@ public void testTopLevelModifyPreservesRequiredMixedCaseFields() throws Throwabl new StructType(new StructField("Value", Type.BIGINT)), true), null, 1L); } - Mockito.verify(updateSchema).updateColumn("Id", Types.LongType.get(), ""); + Mockito.verify(updateSchema).updateColumn("Id", Types.LongType.get(), null); Mockito.verify(updateSchema).updateColumn("Payload.Value", Types.LongType.get(), null); Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional(Mockito.anyString()); Mockito.verify(updateSchema, Mockito.times(2)).commit(); @@ -385,7 +464,7 @@ public void testTopLevelModifyPreservesDottedTopLevelName() throws Throwable { new Column("a.b", Type.BIGINT, true), null, 1L); } - Mockito.verify(updateSchema).updateColumn("a.b", Types.LongType.get(), ""); + Mockito.verify(updateSchema).updateColumn("a.b", Types.LongType.get(), null); Mockito.verify(updateSchema).commit(); } @@ -419,9 +498,8 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingDisabled() throws T nestedTimestamp, null, 1L); } - Mockito.verify(updateSchema).updateColumnDoc("top_uuid", ""); - Mockito.verify(updateSchema).updateColumnDoc("info.uuid_value", ""); - Mockito.verify(updateSchema).updateColumnDoc("info.tz_value", ""); + Mockito.verify(updateSchema, Mockito.never()).updateColumnDoc( + Mockito.anyString(), Mockito.nullable(String.class)); Mockito.verify(updateSchema).makeColumnOptional("top_uuid"); Mockito.verify(updateSchema).makeColumnOptional("info.uuid_value"); Mockito.verify(updateSchema).moveFirst("top_uuid"); @@ -454,8 +532,8 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingEnabled() throws Th new Column("tz_value", ScalarType.createTimeStampTzType(6), true), null, 1L); } - Mockito.verify(updateSchema).updateColumnDoc("top_uuid", ""); - Mockito.verify(updateSchema).updateColumnDoc("info.tz_value", ""); + Mockito.verify(updateSchema, Mockito.never()).updateColumnDoc( + Mockito.anyString(), Mockito.nullable(String.class)); Mockito.verify(updateSchema, Mockito.never()).updateColumn( Mockito.anyString(), Mockito.any(org.apache.iceberg.types.Type.PrimitiveType.class), Mockito.nullable(String.class)); @@ -488,7 +566,8 @@ public void testComplexModifyIgnoresUnchangedMappedChildren() throws Throwable { Mockito.verify(updateSchema, Mockito.times(1)).updateColumn( Mockito.anyString(), Mockito.any(org.apache.iceberg.types.Type.PrimitiveType.class), Mockito.nullable(String.class)); - Mockito.verify(updateSchema).updateColumnDoc("outer.payload", ""); + Mockito.verify(updateSchema, Mockito.never()).updateColumnDoc( + Mockito.anyString(), Mockito.nullable(String.class)); Mockito.verify(updateSchema).commit(); } @@ -909,8 +988,8 @@ public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throw new Column("value", Type.BIGINT, true), null, 1L); } - Mockito.verify(updateSchema).updateColumn("arr.element", Types.LongType.get(), ""); - Mockito.verify(updateSchema).updateColumn("m.value", Types.LongType.get(), ""); + Mockito.verify(updateSchema).updateColumn("arr.element", Types.LongType.get(), null); + Mockito.verify(updateSchema).updateColumn("m.value", Types.LongType.get(), null); Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional(Mockito.anyString()); Mockito.verify(updateSchema, Mockito.times(2)).commit(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index 6f09f8734d1aff..29caecc49101d5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -241,6 +241,28 @@ public void testModifyColumnRoundTripPreservesCommentIntent() { Assertions.assertEquals("", reparsedEmpty.getColumnDef().getComment()); } + @Test + public void testStructMemberRoundTripPreservesCommentIntent() { + ModifyColumnOp modify = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.payload " + + "STRUCT", + ModifyColumnOp.class, "info.payload"); + StructType structType = (StructType) modify.getColumnDef().getType(); + Assertions.assertFalse(structType.getFields().get(0).isCommentSpecified()); + Assertions.assertTrue(structType.getFields().get(1).isCommentSpecified()); + + ModifyColumnOp reparsed = assertSingleClausePath( + "ALTER TABLE t " + modify.toSql(), ModifyColumnOp.class, "info.payload"); + StructType reparsedType = (StructType) reparsed.getColumnDef().getType(); + Assertions.assertFalse(reparsedType.getFields().get(0).isCommentSpecified()); + Assertions.assertTrue(reparsedType.getFields().get(1).isCommentSpecified()); + Assertions.assertEquals("", reparsedType.getFields().get(1).getComment()); + org.apache.doris.catalog.StructType catalogType = (org.apache.doris.catalog.StructType) reparsed + .getColumnDef().translateToCatalogStyleForSchemaChange().getType(); + Assertions.assertFalse(catalogType.getFields().get(0).isCommentSpecified()); + Assertions.assertTrue(catalogType.getFields().get(1).isCommentSpecified()); + } + @Test public void testModifyColumnCommentRoundTripEscapesQuotesAndBackslashes() { assertCommentRoundTrip(false); diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/StructField.java b/fe/fe-type/src/main/java/org/apache/doris/catalog/StructField.java index ea8124c8aeb234..e9432c1efad1c5 100644 --- a/fe/fe-type/src/main/java/org/apache/doris/catalog/StructField.java +++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/StructField.java @@ -40,13 +40,22 @@ public class StructField { @SerializedName(value = "containsNull") private final boolean containsNull; // Now always true (nullable field) + // Runtime-only schema change intent; do not persist it as part of the table schema. + private transient boolean commentSpecified; + public static final String DEFAULT_FIELD_NAME = "col"; public StructField(String name, Type type, String comment, boolean containsNull) { + this(name, type, comment, containsNull, !Strings.isNullOrEmpty(comment)); + } + + public StructField(String name, Type type, String comment, boolean containsNull, + boolean commentSpecified) { this.name = name.toLowerCase(); this.type = type; this.comment = comment; this.containsNull = containsNull; + this.commentSpecified = commentSpecified; } public StructField(String name, Type type) { @@ -65,6 +74,10 @@ public String getComment() { return comment; } + public boolean isCommentSpecified() { + return commentSpecified || !Strings.isNullOrEmpty(comment); + } + public String getName() { return name; } @@ -96,7 +109,7 @@ public String toSql(int depth) { if (type != null) { sb.append(":").append(typeSql); } - if (!Strings.isNullOrEmpty(comment)) { + if (isCommentSpecified()) { sb.append(String.format(" comment '%s'", comment)); } return sb.toString(); @@ -116,7 +129,7 @@ public String prettyPrint(int lpad) { typeStr = typeStr.substring(lpad); sb.append(":").append(typeStr); } - if (!Strings.isNullOrEmpty(comment)) { + if (isCommentSpecified()) { sb.append(String.format(" COMMENT '%s'", comment)); } return sb.toString(); @@ -153,7 +166,7 @@ public String toString() { if (type != null) { sb.append(":").append(type); } - if (!Strings.isNullOrEmpty(comment)) { + if (isCommentSpecified()) { sb.append(String.format(" COMMENT '%s'", comment)); } return sb.toString(); diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy index c6cdc524d9ebc4..bdcb12f734b144 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +import groovy.json.JsonSlurper + suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,iceberg") { String enabled = context.config.otherConfigs.get("enableIcebergTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { @@ -25,6 +27,7 @@ suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,i String catalogName = "test_iceberg_nested_schema_evolution_spark_doris_interop" String dbName = "iceberg_nested_schema_evolution_interop_db" String dorisTable = "doris_nested_evolution_to_spark" + String commentTable = "doris_nested_comment_semantics" String sparkTable = "spark_nested_evolution_to_doris" String mixedCaseTable = "spark_mixed_case_nested_collision" String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") @@ -53,6 +56,47 @@ suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,i spark_iceberg """CREATE DATABASE IF NOT EXISTS demo.${dbName}""" + sql """ + CREATE TABLE ${commentTable} ( + id INT, + info STRUCT< + metric:INT COMMENT 'metric doc', + note:STRING COMMENT 'old note', + clear_me:STRING COMMENT 'clear doc', + payload:STRUCT< + name:STRING COMMENT 'name doc', + count:INT COMMENT 'count doc' + > COMMENT 'payload doc' + > + ) + """ + sql """ALTER TABLE ${commentTable} MODIFY COLUMN info.note COMMENT 'new note'""" + sql """ALTER TABLE ${commentTable} MODIFY COLUMN info.metric BIGINT""" + sql """ALTER TABLE ${commentTable} MODIFY COLUMN info.clear_me STRING COMMENT ''""" + sql """ + ALTER TABLE ${commentTable} MODIFY COLUMN info.payload + STRUCT + """ + + spark_iceberg """REFRESH TABLE demo.${dbName}.${commentTable}""" + def commentMetadataRows = spark_iceberg """ + DESCRIBE TABLE EXTENDED demo.${dbName}.${commentTable} AS JSON + """ + assertEquals(1, commentMetadataRows.size()) + def tableMetadata = new JsonSlurper().parseText(commentMetadataRows[0][0].toString()) + def infoColumn = tableMetadata.columns.find { it.name == "info" } + assertNotNull(infoColumn, "info column should exist in Spark's Iceberg schema") + def infoFields = infoColumn.type.fields.collectEntries { [(it.name): it] } + assertEquals("bigint", infoFields.metric.type.name) + assertEquals("metric doc", infoFields.metric.comment) + assertEquals("new note", infoFields.note.comment) + assertTrue(infoFields.clear_me.comment == null || infoFields.clear_me.comment == "") + assertEquals("payload doc", infoFields.payload.comment) + def payloadFields = infoFields.payload.type.fields.collectEntries { [(it.name): it] } + assertTrue(payloadFields.name.comment == null || payloadFields.name.comment == "") + assertEquals("bigint", payloadFields.count.type.name) + assertEquals("count doc", payloadFields.count.comment) + sql """ CREATE TABLE ${dorisTable} ( id INT NOT NULL, From 22996d476c86e810c2f8fc1d24a11c00ec14e592 Mon Sep 17 00:00:00 2001 From: daidai Date: Mon, 20 Jul 2026 18:03:29 +0800 Subject: [PATCH 28/32] [fix](iceberg) Fix schema evolution regression checks ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: The external Iceberg regression used Spark DESCRIBE TABLE AS JSON to verify nested field comments, but Spark rejects that command for Iceberg V2 tables. A second regression expectation still assumed that MODIFY COLUMN without COMMENT cleared the existing comment, which contradicts the preserved-comment behavior. Read the authoritative current schema from the Iceberg REST loadTable response and update the legacy expected result to preserve the comment. ### Release note None ### Check List (For Author) - Test: Regression test - Targeted regression dry run passed - Iceberg REST loadTable response and metadata shape verified against the local fixture - Full targeted regression was blocked by a stale local FE; the FE rebuild was stopped at the requested 5-minute limit - Behavior changed: No, test-only correction - Does this need documentation: No --- .../iceberg/iceberg_schema_change_ddl.out | 3 +- .../iceberg/iceberg_schema_change_ddl.groovy | 2 +- ...chema_evolution_spark_doris_interop.groovy | 33 ++++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out b/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out index dab31ada31b925..4565871aa8aa71 100644 --- a/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out +++ b/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out @@ -149,7 +149,7 @@ phone text Yes true \N User phone number grade double Yes true \N email text Yes true \N address struct Yes true \N -col1 double Yes true \N +col1 double Yes true \N Updated column1 type col2 text Yes true \N User defined column2 -- !after_no_comment -- @@ -329,4 +329,3 @@ department_id bigint Yes true \N 2 Bob 30 9223372036854775806 3 Charlie 22 100 3 Charlie 22 9223372036854775805 - diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy index 8bd7e240c4d83e..7d7672a4b3e6cc 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy @@ -193,7 +193,7 @@ suite("iceberg_schema_change_ddl", "p0,external") { sql """ ALTER TABLE ${table_name} MODIFY COLUMN non_col bigint""" exception "Column non_col does not exist" } - // not comment, the comment will be removed + // Omitted COMMENT preserves the existing comment. qt_before_no_comment "desc ${table_name}" sql """ ALTER TABLE ${table_name} MODIFY COLUMN col1 DOUBLE""" qt_after_no_comment "desc ${table_name}" diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy index bdcb12f734b144..70b938eab02f39 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy @@ -78,24 +78,25 @@ suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,i STRUCT """ - spark_iceberg """REFRESH TABLE demo.${dbName}.${commentTable}""" - def commentMetadataRows = spark_iceberg """ - DESCRIBE TABLE EXTENDED demo.${dbName}.${commentTable} AS JSON - """ - assertEquals(1, commentMetadataRows.size()) - def tableMetadata = new JsonSlurper().parseText(commentMetadataRows[0][0].toString()) - def infoColumn = tableMetadata.columns.find { it.name == "info" } - assertNotNull(infoColumn, "info column should exist in Spark's Iceberg schema") + String loadTableUrl = "http://${externalEnvIp}:${restPort}/v1/namespaces/${dbName}/tables/${commentTable}" + def loadTableResponse = new JsonSlurper().parseText(new URL(loadTableUrl).getText("UTF-8")) + def tableMetadata = loadTableResponse.metadata + def currentSchema = tableMetadata.schemas.find { + it["schema-id"] == tableMetadata["current-schema-id"] + } + assertNotNull(currentSchema, "current schema should exist in Iceberg metadata") + def infoColumn = currentSchema.fields.find { it.name == "info" } + assertNotNull(infoColumn, "info column should exist in Iceberg metadata") def infoFields = infoColumn.type.fields.collectEntries { [(it.name): it] } - assertEquals("bigint", infoFields.metric.type.name) - assertEquals("metric doc", infoFields.metric.comment) - assertEquals("new note", infoFields.note.comment) - assertTrue(infoFields.clear_me.comment == null || infoFields.clear_me.comment == "") - assertEquals("payload doc", infoFields.payload.comment) + assertEquals("long", infoFields.metric.type) + assertEquals("metric doc", infoFields.metric.doc) + assertEquals("new note", infoFields.note.doc) + assertTrue(infoFields.clear_me.doc == null || infoFields.clear_me.doc == "") + assertEquals("payload doc", infoFields.payload.doc) def payloadFields = infoFields.payload.type.fields.collectEntries { [(it.name): it] } - assertTrue(payloadFields.name.comment == null || payloadFields.name.comment == "") - assertEquals("bigint", payloadFields.count.type.name) - assertEquals("count doc", payloadFields.count.comment) + assertTrue(payloadFields.name.doc == null || payloadFields.name.doc == "") + assertEquals("long", payloadFields.count.type) + assertEquals("count doc", payloadFields.count.doc) sql """ CREATE TABLE ${dorisTable} ( From 6a88d6cb20aa652270e14c4353bd2ea39cc759b5 Mon Sep 17 00:00:00 2001 From: daidai Date: Tue, 21 Jul 2026 15:01:12 +0800 Subject: [PATCH 29/32] [test](iceberg) Add nested schema evolution coverage ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Add end-to-end coverage for moving existing nested fields, required-to-optional changes, float and decimal promotions, and deeply nested schema evolution. Add a real Iceberg catalog conflict test to verify that a stale schema commit fails atomically without partially applying nested type changes. ### Release note None ### Check List (For Author) - Test: Unit Test and Regression Test - IcebergMetadataOpsValidationTest - test_iceberg_nested_schema_evolution_spark_doris_interop - Behavior changed: No - Does this need documentation: No --- .../IcebergMetadataOpsValidationTest.java | 80 +++++++ ...d_schema_evolution_spark_doris_interop.out | 8 + ...chema_evolution_spark_doris_interop.groovy | 196 ++++++++++++++++++ 3 files changed, 284 insertions(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index e4b388a8796993..80c6feb1c9458b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -31,16 +31,24 @@ import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalTable; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.UpdateSchema; import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.hadoop.HadoopCatalog; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -48,7 +56,10 @@ import java.lang.reflect.Method; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; public class IcebergMetadataOpsValidationTest { @@ -57,6 +68,9 @@ public class IcebergMetadataOpsValidationTest { private Method validateForModifyColumnMethod; private Method validateForModifyComplexColumnMethod; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before public void setUp() throws Exception { dorisCatalog = Mockito.mock(ExternalCatalog.class); @@ -689,6 +703,72 @@ public void testExplicitNullableModifyMakesRequiredFieldsOptional() throws Throw Mockito.verify(updateSchema, Mockito.times(2)).commit(); } + @Test + public void testSchemaCommitConflictDoesNotPartiallyApplyComplexModify() throws Exception { + String dbName = "db"; + String tableName = "conflict_table"; + TableIdentifier tableIdentifier = TableIdentifier.of(dbName, tableName); + Map properties = new HashMap<>(); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, + temporaryFolder.newFolder("iceberg_warehouse").toURI().toString()); + + HadoopCatalog icebergCatalog = new HadoopCatalog(); + icebergCatalog.setConf(new Configuration()); + icebergCatalog.initialize("conflict_catalog", properties); + icebergCatalog.createNamespace(Namespace.of(dbName)); + Table staleTable = icebergCatalog.createTable(tableIdentifier, new Schema( + Types.NestedField.optional(1, "info", Types.StructType.of( + Types.NestedField.optional(2, "a", Types.IntegerType.get()), + Types.NestedField.optional(3, "b", Types.IntegerType.get()))))); + Table concurrentTable = icebergCatalog.loadTable(tableIdentifier); + + ExternalCatalog conflictDorisCatalog = Mockito.mock(ExternalCatalog.class); + AtomicBoolean conflictInjected = new AtomicBoolean(false); + Mockito.when(conflictDorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public void execute(Runnable task) { + Assert.assertTrue("schema commit should execute only once", conflictInjected.compareAndSet(false, true)); + concurrentTable.updateSchema().renameColumn("info.a", "concurrent_a").commit(); + task.run(); + } + }); + Mockito.when(conflictDorisCatalog.getProperties()).thenReturn(Collections.emptyMap()); + Mockito.doReturn(Optional.empty()).when(conflictDorisCatalog).getDbForReplay(Mockito.anyString()); + IcebergMetadataOps conflictOps = new IcebergMetadataOps(conflictDorisCatalog, icebergCatalog); + + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn(dbName); + Mockito.when(dorisTable.getRemoteName()).thenReturn(tableName); + StructType promotedInfo = new StructType( + new StructField("a", Type.BIGINT), + new StructField("b", Type.BIGINT)); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(staleTable); + + try { + conflictOps.modifyColumn(dorisTable, ColumnPath.of("info"), + new Column("info", promotedInfo, true), null, 1L); + Assert.fail("expected schema commit conflict"); + } catch (UserException e) { + Assert.assertTrue(e.getMessage().contains("Failed to modify column: info")); + Assert.assertTrue("expected Iceberg commit conflict but got " + e.getCause(), + e.getCause() instanceof CommitFailedException); + } + } + + Schema committedSchema = icebergCatalog.loadTable(tableIdentifier).schema(); + Assert.assertNull(committedSchema.findField("info.a")); + Assert.assertEquals(Types.IntegerType.get(), committedSchema.findType("info.concurrent_a")); + Assert.assertEquals(Types.IntegerType.get(), committedSchema.findType("info.b")); + Mockito.verify(conflictDorisCatalog, Mockito.never()).getDbForReplay(Mockito.anyString()); + + icebergCatalog.dropTable(tableIdentifier); + icebergCatalog.dropNamespace(Namespace.of(dbName)); + icebergCatalog.close(); + } + @Test public void testNestedColumnOperationsRejectDefaultMetadata() { Schema schema = new Schema(Types.NestedField.optional(1, "s", Types.StructType.of( diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out index 6aef898cfc7950..54a02e79582045 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out @@ -17,3 +17,11 @@ attrs map> 1 \N 10 \N spark_before \N \N 100 \N \N \N 1000 \N \N 2 spark_first_field 20 spark_after_metric_field spark_after spark_new_field 202 200 203 201 2002 2000 2003 2001 3 doris_first_field 30 doris_after_metric_field doris_after_spark doris_can_write_spark_field 302 300 303 301 3002 3000 3003 3001 + +-- !required_nested_rows -- +1 10 old-label +2 20 \N + +-- !deep_nested_rows -- +1 last-old middle-old first-old old-note 1.5 12.34 +2 last-new middle-new first-new new-note 2.5 56.78 diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy index 70b938eab02f39..16dfbbb2a9d60b 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy @@ -30,6 +30,8 @@ suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,i String commentTable = "doris_nested_comment_semantics" String sparkTable = "spark_nested_evolution_to_doris" String mixedCaseTable = "spark_mixed_case_nested_collision" + String requiredTable = "spark_required_nested_evolution" + String advancedTable = "spark_deep_nested_evolution" String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") String minioPort = context.config.otherConfigs.get("iceberg_minio_port") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") @@ -353,4 +355,198 @@ suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,i order_qt_spark_driven_rows_after_write sparkDrivenDorisQuery def sparkDrivenDorisRows = sql sparkDrivenDorisQuery assertSparkDorisResultEquals(sparkDrivenSparkRows, sparkDrivenDorisRows) + + spark_iceberg_multi """ + DROP TABLE IF EXISTS demo.${dbName}.${requiredTable}; + CREATE TABLE demo.${dbName}.${requiredTable} ( + id INT, + info STRUCT + ) USING iceberg; + INSERT INTO demo.${dbName}.${requiredTable} VALUES ( + 1, + NAMED_STRUCT('required_metric', 10, 'required_label', 'old-label') + ); + """ + + sql """refresh catalog ${catalogName}""" + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + sql """ALTER TABLE ${requiredTable} MODIFY COLUMN info.required_metric BIGINT""" + sql """ALTER TABLE ${requiredTable} MODIFY COLUMN info.required_label STRING NULL""" + test { + sql """ALTER TABLE ${requiredTable} MODIFY COLUMN info.required_label STRING NOT NULL""" + exception "Can not change nullable column info.required_label to not null" + } + + String requiredTableUrl = + "http://${externalEnvIp}:${restPort}/v1/namespaces/${dbName}/tables/${requiredTable}" + def requiredTableMetadata = new JsonSlurper().parseText( + new URL(requiredTableUrl).getText("UTF-8")).metadata + def requiredSchema = requiredTableMetadata.schemas.find { + it["schema-id"] == requiredTableMetadata["current-schema-id"] + } + def requiredInfo = requiredSchema.fields.find { it.name == "info" } + def requiredInfoFields = requiredInfo.type.fields.collectEntries { [(it.name): it] } + assertEquals("long", requiredInfoFields.required_metric.type) + assertTrue(requiredInfoFields.required_metric.required) + assertFalse(requiredInfoFields.required_label.required) + + sql """ + INSERT INTO ${requiredTable} VALUES ( + 2, + STRUCT(20, NULL) + ) + """ + + String requiredDorisQuery = """ + SELECT id, + element_at(info, 'required_metric'), + element_at(info, 'required_label') + FROM ${requiredTable} + ORDER BY id + """ + order_qt_required_nested_rows requiredDorisQuery + spark_iceberg """REFRESH TABLE demo.${dbName}.${requiredTable}""" + def requiredSparkRows = spark_iceberg """ + SELECT id, info.required_metric, info.required_label + FROM demo.${dbName}.${requiredTable} + ORDER BY id + """ + def requiredDorisRows = sql requiredDorisQuery + assertSparkDorisResultEquals(requiredSparkRows, requiredDorisRows) + + spark_iceberg_multi """ + DROP TABLE IF EXISTS demo.${dbName}.${advancedTable}; + CREATE TABLE demo.${dbName}.${advancedTable} ( + id INT, + root STRUCT< + first_field: STRING, + middle_field: STRING, + last_field: STRING, + items: ARRAY>>> + > + ) USING iceberg; + INSERT INTO demo.${dbName}.${advancedTable} VALUES ( + 1, + NAMED_STRUCT( + 'first_field', 'first-old', + 'middle_field', 'middle-old', + 'last_field', 'last-old', + 'items', ARRAY(NAMED_STRUCT( + 'attrs', MAP('k', NAMED_STRUCT( + 'score', CAST(1.5 AS FLOAT), + 'amount', CAST(12.34 AS DECIMAL(9, 2)), + 'note', 'old-note' + )) + )) + ) + ); + """ + + sql """refresh catalog ${catalogName}""" + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + sql """ALTER TABLE ${advancedTable} MODIFY COLUMN root.last_field STRING FIRST""" + sql """ALTER TABLE ${advancedTable} MODIFY COLUMN root.first_field STRING AFTER middle_field""" + sql """ALTER TABLE ${advancedTable} MODIFY COLUMN root.items.element.attrs.value.score DOUBLE""" + sql """ALTER TABLE ${advancedTable} MODIFY COLUMN root.items.element.attrs.value.amount DECIMAL(18, 2)""" + test { + sql """ALTER TABLE ${advancedTable} + MODIFY COLUMN root.items.element.attrs.value.amount DECIMAL(18, 3)""" + exception "Cannot change column type" + } + test { + sql """ALTER TABLE ${advancedTable} + MODIFY COLUMN root.items.element.attrs.value.score FLOAT""" + exception "Cannot change column type" + } + sql """ALTER TABLE ${advancedTable} + ADD COLUMN root.items.element.attrs.value.added STRING NULL AFTER amount""" + sql """ALTER TABLE ${advancedTable} + RENAME COLUMN root.items.element.attrs.value.added TO renamed""" + sql """ALTER TABLE ${advancedTable} + MODIFY COLUMN root.items.element.attrs.value.renamed COMMENT 'deep field'""" + sql """ALTER TABLE ${advancedTable} + MODIFY COLUMN root.items.element.attrs.value.note STRING FIRST""" + + String advancedTableUrl = + "http://${externalEnvIp}:${restPort}/v1/namespaces/${dbName}/tables/${advancedTable}" + def advancedTableMetadata = new JsonSlurper().parseText( + new URL(advancedTableUrl).getText("UTF-8")).metadata + def advancedSchema = advancedTableMetadata.schemas.find { + it["schema-id"] == advancedTableMetadata["current-schema-id"] + } + def rootField = advancedSchema.fields.find { it.name == "root" } + assertEquals(["last_field", "middle_field", "first_field", "items"], + rootField.type.fields.collect { it.name }) + def itemsField = rootField.type.fields.find { it.name == "items" } + def attrsField = itemsField.type.element.fields.find { it.name == "attrs" } + def deepFields = attrsField.type.value.fields + assertEquals(["note", "score", "amount", "renamed"], deepFields.collect { it.name }) + assertEquals("double", deepFields.find { it.name == "score" }.type) + assertEquals("decimal(18, 2)", deepFields.find { it.name == "amount" }.type) + assertEquals("deep field", deepFields.find { it.name == "renamed" }.doc) + + sql """ALTER TABLE ${advancedTable} + DROP COLUMN root.items.element.attrs.value.renamed""" + + def droppedFieldMetadata = new JsonSlurper().parseText( + new URL(advancedTableUrl).getText("UTF-8")).metadata + def droppedFieldSchema = droppedFieldMetadata.schemas.find { + it["schema-id"] == droppedFieldMetadata["current-schema-id"] + } + def droppedRootField = droppedFieldSchema.fields.find { it.name == "root" } + def droppedItemsField = droppedRootField.type.fields.find { it.name == "items" } + def droppedAttrsField = droppedItemsField.type.element.fields.find { it.name == "attrs" } + assertEquals(["note", "score", "amount"], + droppedAttrsField.type.value.fields.collect { it.name }) + + sql """ + INSERT INTO ${advancedTable} VALUES ( + 2, + STRUCT( + 'last-new', + 'middle-new', + 'first-new', + ARRAY(STRUCT(MAP('k', STRUCT( + 'new-note', + CAST(2.5 AS DOUBLE), + CAST(56.78 AS DECIMAL(18, 2)) + )))) + ) + ) + """ + + String advancedDorisQuery = """ + SELECT id, + element_at(root, 'last_field'), + element_at(root, 'middle_field'), + element_at(root, 'first_field'), + element_at(element_at(element_at(root, 'items')[1], 'attrs')['k'], 'note'), + element_at(element_at(element_at(root, 'items')[1], 'attrs')['k'], 'score'), + element_at(element_at(element_at(root, 'items')[1], 'attrs')['k'], 'amount') + FROM ${advancedTable} + ORDER BY id + """ + order_qt_deep_nested_rows advancedDorisQuery + spark_iceberg """REFRESH TABLE demo.${dbName}.${advancedTable}""" + def advancedSparkRows = spark_iceberg """ + SELECT id, + root.last_field, + root.middle_field, + root.first_field, + root.items[0].attrs['k'].note, + root.items[0].attrs['k'].score, + root.items[0].attrs['k'].amount + FROM demo.${dbName}.${advancedTable} + ORDER BY id + """ + def advancedDorisRows = sql advancedDorisQuery + assertSparkDorisResultEquals(advancedSparkRows, advancedDorisRows) } From fe5bf1123277cb4da7b14151f89f2172b42a5acf Mon Sep 17 00:00:00 2001 From: daidai Date: Wed, 22 Jul 2026 11:55:00 +0800 Subject: [PATCH 30/32] [fix](iceberg) Address nested schema review feedback ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Remove the unreachable legacy recursive nullability path while preserving the old top-level API bridge. Keep ordinary struct field names unquoted while quoting reserved or special identifiers for replayable schema-change SQL. Document omitted nullability and comment rendering, and add round-trip coverage for persisted ADD/MODIFY SQL. ### Release note Iceberg whole-complex MODIFY no longer infers recursive required-to-optional changes; use an explicit NULL clause on the target nested field. ### Check List (For Author) - Test: Unit Test - IcebergMetadataOpsValidationTest - IcebergNestedSchemaEvolutionParserTest - PruneNestedColumnTest - Behavior changed: Yes. Omitted nested nullability preserves required fields and ordinary struct field names keep their existing display format. - Does this need documentation: Yes. Follow-up documentation is required for the nested schema-change syntax and behavior. --- .../iceberg/IcebergMetadataOps.java | 72 ++++++++----------- .../doris/nereids/parser/NereidsParser.java | 5 +- .../plans/commands/info/ColumnDefinition.java | 3 +- .../doris/nereids/types/StructField.java | 4 +- .../IcebergMetadataOpsValidationTest.java | 10 +-- ...cebergNestedSchemaEvolutionParserTest.java | 33 +++++++++ .../rules/rewrite/PruneNestedColumnTest.java | 24 +++---- 7 files changed, 89 insertions(+), 62 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 123912da1e5f47..2e6751e404af4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -1042,11 +1042,17 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String @Override public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { - modifyTopLevelColumn(dorisTable, ColumnPath.of(column.getName()), column, position, updateTime, true); + // This overload predates nullableSpecified/commentSpecified. Keep its top-level values + // explicit while delegating to the path-aware implementation. + Column explicitColumn = new Column(column); + explicitColumn.setIsKey(false); + explicitColumn.setNullableSpecified(true); + explicitColumn.setCommentSpecified(true); + modifyColumn(dorisTable, ColumnPath.of(column.getName()), explicitColumn, position, updateTime); } private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, - ColumnPosition position, long updateTime, boolean legacyMode) throws UserException { + ColumnPosition position, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify"); NestedField currentCol = icebergTable.schema().asStruct() @@ -1057,7 +1063,7 @@ private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPat ResolvedColumnPath resolvedPath = new ResolvedColumnPath(ColumnPath.of(currentCol.name()), currentCol.type(), currentCol); - validateModifyColumnMetadata(column, resolvedPath.getFullPath(), !legacyMode); + validateModifyColumnMetadata(column, resolvedPath.getFullPath(), true); org.apache.iceberg.types.Type targetType; if (column.getType().isComplexType()) { validateForModifyComplexColumn(column, currentCol); @@ -1069,10 +1075,10 @@ private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPat } UpdateSchema updateSchema = icebergTable.updateSchema(); - String targetComment = resolveTargetComment(currentCol, column, legacyMode); + String targetComment = resolveTargetComment(currentCol, column); if (column.getType().isComplexType()) { applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), - column.getType(), legacyMode); + column.getType()); if (!Objects.equals(currentCol.doc(), targetComment)) { updateSchema.updateColumnDoc(resolvedPath.getFullPath(), targetComment); } @@ -1080,7 +1086,7 @@ private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPat applyPrimitiveColumnChange(updateSchema, resolvedPath.getFullPath(), currentCol, targetType.asPrimitiveType(), targetComment); } - applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column, legacyMode); + applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column); if (position != null) { applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); @@ -1098,7 +1104,7 @@ private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPat public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, long updateTime) throws UserException { if (!columnPath.isNested()) { - modifyTopLevelColumn(dorisTable, columnPath, column, position, updateTime, false); + modifyTopLevelColumn(dorisTable, columnPath, column, position, updateTime); return; } @@ -1123,10 +1129,10 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column } UpdateSchema updateSchema = icebergTable.updateSchema(); - String targetComment = resolveTargetComment(currentCol, column, false); + String targetComment = resolveTargetComment(currentCol, column); if (column.getType().isComplexType()) { applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), - column.getType(), false); + column.getType()); if (!Objects.equals(currentCol.doc(), targetComment)) { updateSchema.updateColumnDoc(resolvedPath.getFullPath(), targetComment); } @@ -1134,7 +1140,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column applyPrimitiveColumnChange(updateSchema, resolvedPath.getFullPath(), currentCol, targetType.asPrimitiveType(), targetComment); } - applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column, false); + applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column); if (position != null) { applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); @@ -1185,15 +1191,14 @@ private void validateCollectionPseudoFieldComment(Schema schema, ResolvedColumnP } } - private void applyExplicitNullableChange(UpdateSchema updateSchema, String columnPath, Column column, - boolean legacyNullableExplicit) { - if ((legacyNullableExplicit || column.isNullableSpecified()) && column.isAllowNull()) { + private void applyExplicitNullableChange(UpdateSchema updateSchema, String columnPath, Column column) { + if (column.isNullableSpecified() && column.isAllowNull()) { updateSchema.makeColumnOptional(columnPath); } } - private String resolveTargetComment(NestedField currentCol, Column column, boolean legacyMode) { - return legacyMode || column.isCommentSpecified() ? column.getComment() : currentCol.doc(); + private String resolveTargetComment(NestedField currentCol, Column column) { + return column.isCommentSpecified() ? column.getComment() : currentCol.doc(); } private org.apache.iceberg.types.Type.PrimitiveType resolvePrimitiveTypeForModify( @@ -1498,19 +1503,16 @@ private void validateExistingTypeChange(org.apache.iceberg.types.Type oldIceberg private void applyComplexTypeChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldIcebergType, - org.apache.doris.catalog.Type newDorisType, boolean legacyMode) throws UserException { + org.apache.doris.catalog.Type newDorisType) throws UserException { switch (oldIcebergType.typeId()) { case STRUCT: - applyStructChange(updateSchema, path, oldIcebergType.asStructType(), (StructType) newDorisType, - legacyMode); + applyStructChange(updateSchema, path, oldIcebergType.asStructType(), (StructType) newDorisType); break; case LIST: - applyListChange(updateSchema, path, (Types.ListType) oldIcebergType, (ArrayType) newDorisType, - legacyMode); + applyListChange(updateSchema, path, (Types.ListType) oldIcebergType, (ArrayType) newDorisType); break; case MAP: - applyMapChange(updateSchema, path, (Types.MapType) oldIcebergType, (MapType) newDorisType, - legacyMode); + applyMapChange(updateSchema, path, (Types.MapType) oldIcebergType, (MapType) newDorisType); break; default: throw new UserException("Unsupported complex type for modify: " + oldIcebergType); @@ -1518,7 +1520,7 @@ private void applyComplexTypeChange(UpdateSchema updateSchema, String path, } private void applyStructChange(UpdateSchema updateSchema, String path, - Types.StructType oldStructType, StructType newStructType, boolean legacyMode) + Types.StructType oldStructType, StructType newStructType) throws UserException { List oldFields = oldStructType.fields(); List newFields = newStructType.getFields(); @@ -1534,7 +1536,7 @@ private void applyStructChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldFieldType = oldField.type(); org.apache.doris.catalog.Type newFieldType = newField.getType(); - String targetComment = legacyMode || newField.isCommentSpecified() + String targetComment = newField.isCommentSpecified() ? newField.getComment() : oldField.doc(); if (oldFieldType.isPrimitiveType()) { org.apache.doris.catalog.Type oldDorisFieldType = mappedDorisType(oldFieldType); @@ -1549,15 +1551,11 @@ private void applyStructChange(UpdateSchema updateSchema, String path, updateSchema.updateColumnDoc(fieldPath, targetComment); } } else { - applyComplexTypeChange(updateSchema, fieldPath, oldFieldType, newFieldType, - legacyMode); + applyComplexTypeChange(updateSchema, fieldPath, oldFieldType, newFieldType); if (!Objects.equals(oldField.doc(), targetComment)) { updateSchema.updateColumnDoc(fieldPath, targetComment); } } - if (legacyMode && !oldField.isOptional() && newField.getContainsNull()) { - updateSchema.makeColumnOptional(fieldPath); - } } for (int i = oldFields.size(); i < newFields.size(); i++) { @@ -1572,7 +1570,7 @@ private void applyStructChange(UpdateSchema updateSchema, String path, } private void applyListChange(UpdateSchema updateSchema, String path, - Types.ListType oldListType, ArrayType newArrayType, boolean legacyMode) + Types.ListType oldListType, ArrayType newArrayType) throws UserException { String elementPath = path + "." + oldListType.field(oldListType.elementId()).name(); if (oldListType.isElementOptional() && !newArrayType.getContainsNull()) { @@ -1588,16 +1586,12 @@ private void applyListChange(UpdateSchema updateSchema, String path, updateSchema.updateColumn(elementPath, newIcebergElementType.asPrimitiveType(), null); } } else { - applyComplexTypeChange(updateSchema, elementPath, oldElementType, newElementType, - legacyMode); - } - if (legacyMode && !oldListType.isElementOptional() && newArrayType.getContainsNull()) { - updateSchema.makeColumnOptional(elementPath); + applyComplexTypeChange(updateSchema, elementPath, oldElementType, newElementType); } } private void applyMapChange(UpdateSchema updateSchema, String path, - Types.MapType oldMapType, MapType newMapType, boolean legacyMode) + Types.MapType oldMapType, MapType newMapType) throws UserException { org.apache.iceberg.types.Type oldKeyType = oldMapType.keyType(); org.apache.doris.catalog.Type newKeyType = newMapType.getKeyType(); @@ -1621,11 +1615,7 @@ private void applyMapChange(UpdateSchema updateSchema, String path, updateSchema.updateColumn(valuePath, newIcebergValueType.asPrimitiveType(), null); } } else { - applyComplexTypeChange(updateSchema, valuePath, oldValueType, newValueType, - legacyMode); - } - if (legacyMode && !oldMapType.isValueOptional() && newMapType.getIsValueContainsNull()) { - updateSchema.makeColumnOptional(valuePath); + applyComplexTypeChange(updateSchema, valuePath, oldValueType, newValueType); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 83f978fff948b9..c997045e7d35a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -300,13 +300,14 @@ public List> parseMultiple(String sql, } public Expression parseExpression(String expression) { - if (isSimpleIdentifier(expression)) { + if (isValidUnquotedIdentifier(expression)) { return new UnboundSlot(expression); } return parse(expression, DorisParser::expressionWithEof); } - private static boolean isSimpleIdentifier(String expression) { + /** Return whether the text can be emitted as an unquoted Nereids identifier. */ + public static boolean isValidUnquotedIdentifier(String expression) { if (expression == null || expression.isEmpty()) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 3513e40cb4ff39..6c386489139492 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -243,7 +243,8 @@ public String toSql() { } /** - * Convert this column definition to SQL with a caller-provided column name. + * Convert this column definition to schema-change SQL with a caller-provided column name. + * Unlike {@link #toSql()}, this overload emits COMMENT only when it was explicitly specified. */ public String toSql(String columnNameSql) { return toSql(columnNameSql, commentSpecified); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java index c008886b4677c4..aefcf20f227ff1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.types; import org.apache.doris.common.util.SqlUtils; +import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.util.SqlLiteralUtils; import org.apache.doris.nereids.util.Utils; @@ -96,7 +97,8 @@ public org.apache.doris.catalog.StructField toCatalogDataType() { } public String toSql() { - return SqlUtils.getIdentSql(name) + ":" + dataType.toSql() + String nameSql = NereidsParser.isValidUnquotedIdentifier(name) ? name : SqlUtils.getIdentSql(name); + return nameSql + ":" + dataType.toSql() + (nullable ? "" : " NOT NULL") + (!commentSpecified ? "" : " COMMENT " + SqlLiteralUtils.quoteStringLiteral(comment)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 80c6feb1c9458b..fd9ac33e955f29 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -641,7 +641,7 @@ public void testLegacyModifyColumnTreatsNullabilityAsExplicit() throws Throwable } @Test - public void testLegacyComplexModifyRestoresRecursiveNullableSemantics() throws Throwable { + public void testLegacyComplexModifyDoesNotInferRecursiveNullableChanges() throws Throwable { Schema schema = requiredNestedSchema(); ExternalTable dorisTable = Mockito.mock(ExternalTable.class); Table icebergTable = Mockito.mock(Table.class); @@ -664,10 +664,10 @@ public void testLegacyComplexModifyRestoresRecursiveNullableSemantics() throws T } Mockito.verify(updateSchema).makeColumnOptional("info"); - Mockito.verify(updateSchema).makeColumnOptional("info.metric"); - Mockito.verify(updateSchema).makeColumnOptional("info.child.value"); - Mockito.verify(updateSchema).makeColumnOptional("info.events.element"); - Mockito.verify(updateSchema).makeColumnOptional("info.attrs.value"); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional("info.metric"); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional("info.child.value"); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional("info.events.element"); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional("info.attrs.value"); Mockito.verify(updateSchema).commit(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index 29caecc49101d5..f6efa1ab7c5ff4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -105,6 +105,33 @@ public void testTopLevelColumnKeepsExistingDefaultGrammar() { Assertions.assertTrue(add.getColumnDef().hasDefaultValue()); } + @Test + public void testTopLevelColumnRoundTripPreservesOmittedIntent() { + AddColumnOp add = assertSingleClausePath( + "ALTER TABLE t ADD COLUMN added INT", AddColumnOp.class, "added"); + String renderedAdd = add.toSql(); + Assertions.assertFalse(renderedAdd.contains(" NULL")); + Assertions.assertFalse(renderedAdd.contains(" COMMENT ")); + AddColumnOp reparsedAdd = assertSingleClausePath( + "ALTER TABLE t " + renderedAdd, AddColumnOp.class, "added"); + Assertions.assertFalse(reparsedAdd.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + Assertions.assertFalse(reparsedAdd.getColumnDef() + .translateToCatalogStyleForSchemaChange().isCommentSpecified()); + + ModifyColumnOp modify = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN existing BIGINT", ModifyColumnOp.class, "existing"); + String renderedModify = modify.toSql(); + Assertions.assertFalse(renderedModify.contains(" NULL")); + Assertions.assertFalse(renderedModify.contains(" COMMENT ")); + ModifyColumnOp reparsedModify = assertSingleClausePath( + "ALTER TABLE t " + renderedModify, ModifyColumnOp.class, "existing"); + Assertions.assertFalse(reparsedModify.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + Assertions.assertFalse(reparsedModify.getColumnDef() + .translateToCatalogStyleForSchemaChange().isCommentSpecified()); + } + @Test public void testLegacyStringConstructorsKeepDottedTopLevelNames() { DropColumnOp drop = new DropColumnOp("top.level", null, Collections.emptyMap()); @@ -157,6 +184,7 @@ public void testStructMemberIdentifiersRoundTrip() { + "STRUCT<`key`:INT,`Metric``Name`:STRING> NULL", AddColumnOp.class, "info.payload"); assertStructMemberNames((StructType) add.getColumnDef().getType()); + Assertions.assertTrue(add.toSql().contains("STRUCT<`key`:INT,`metric``name`:TEXT>")); AddColumnOp reparsedAdd = assertSingleClausePath( "ALTER TABLE t " + add.toSql(), AddColumnOp.class, "info.payload"); assertStructMemberNames((StructType) reparsedAdd.getColumnDef().getType()); @@ -169,6 +197,11 @@ public void testStructMemberIdentifiersRoundTrip() { ModifyColumnOp reparsedModify = assertSingleClausePath( "ALTER TABLE t " + modify.toSql(), ModifyColumnOp.class, "info.payload"); assertStructMemberNames((StructType) reparsedModify.getColumnDef().getType()); + + ModifyColumnOp ordinary = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.payload STRUCT", + ModifyColumnOp.class, "info.payload"); + Assertions.assertTrue(ordinary.toSql().contains("STRUCT")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index 4309493abb3be3..3c92606e4e3fcc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -1015,38 +1015,38 @@ public void testDataTypeAccessTree() { DataTypeAccessTree tree = trees.get(0).second; Assertions.assertEquals(NullType.INSTANCE, tree.getType()); Assertions.assertEquals(1, tree.getChildren().size()); - Assertions.assertEquals("STRUCT<`city`:TEXT>", tree.getChildren().get("s").getType().toSql()); + Assertions.assertEquals("STRUCT", tree.getChildren().get("s").getType().toSql()); SlotReference slot = trees.get(0).first; Type columnType = slot.getOriginalColumn().get().getType(); Assertions.assertEquals("struct>>>", columnType.toSql()); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "city"), "STRUCT<`city`:TEXT>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "city"), "STRUCT"); setAccessPathAndAssertType(slot, ImmutableList.of("s", "data"), - "STRUCT<`data`:ARRAY>>>"); + "STRUCT>>>"); setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*"), - "STRUCT<`data`:ARRAY>>>"); + "STRUCT>>>"); setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "KEYS"), - "STRUCT<`data`:ARRAY>>>"); + "STRUCT>>>"); setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES"), - "STRUCT<`data`:ARRAY>>>"); + "STRUCT>>>"); setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES", "a"), - "STRUCT<`data`:ARRAY>>>"); + "STRUCT>>>"); setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES", "b"), - "STRUCT<`data`:ARRAY>>>"); + "STRUCT>>>"); setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*"), - "STRUCT<`data`:ARRAY>>>"); + "STRUCT>>>"); setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*", "a"), - "STRUCT<`data`:ARRAY>>>"); + "STRUCT>>>"); setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*", "b"), - "STRUCT<`data`:ARRAY>>>"); + "STRUCT>>>"); setAccessPathsAndAssertType(slot, ImmutableList.of( ImmutableList.of("s", "data", "*", "*", "b"), ImmutableList.of("s", "city") ), - "STRUCT<`city`:TEXT,`data`:ARRAY>>>" + "STRUCT>>>" ); } From c49d90fd8d148572c9da93d0b334eb0c8a971714 Mon Sep 17 00:00:00 2001 From: daidai Date: Wed, 22 Jul 2026 17:19:39 +0800 Subject: [PATCH 31/32] [fix](iceberg) Reject non-atomic compound column changes ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Iceberg external ALTER dispatches clauses independently and commits each schema update separately. A later failing column clause could therefore leave earlier clauses persisted even though the statement returned an error. Reject compound ALTER statements containing column operations before execution, keep grouped multi-column ADD as one atomic operation, and restore STRUCT error baselines to match conditional identifier quoting. ### Release note Compound Iceberg ALTER statements containing column operations are rejected before execution. Use separate statements, or grouped ADD COLUMN for atomic multi-column additions. ### Check List (For Author) - Test: Static analysis and targeted unit-test build - Maven Checkstyle passed; scoped git diff check passed; targeted FE unit tests reached fe-core compilation but did not execute before the 300-second timeout. - Behavior changed: Yes. Non-atomic compound Iceberg column ALTER statements are rejected. - Does this need documentation: No --- .../trees/plans/commands/AlterTableCommand.java | 9 +++++---- .../trees/plans/commands/AlterTableCommandTest.java | 12 ++++++++---- .../basic-elements/data-types/struct-md.groovy | 12 ++++++------ .../sql-function/test_array_function.groovy | 4 ++-- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java index 49fca1b1571ccd..91fb9bf9739019 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java @@ -188,10 +188,9 @@ private static void checkIcebergCompoundColumnOperations(List alte return; } for (AlterTableOp alterTableOp : alterTableOps) { - if (getNestedColumnPath(alterTableOp) != null - || alterTableOp instanceof ModifyColumnCommentOp) { - throw new AnalysisException("Multiple Iceberg column operations are not supported when a statement " - + "contains a nested column path or MODIFY COLUMN COMMENT"); + if (isIcebergColumnSchemaOperation(alterTableOp)) { + throw new AnalysisException("Multiple Iceberg ALTER clauses are not supported when a statement " + + "contains a column operation"); } } } @@ -226,7 +225,9 @@ private static boolean isIcebergColumnSchemaOperation(AlterTableOp alterTableOp) return alterTableOp instanceof AddColumnOp || alterTableOp instanceof AddColumnsOp || alterTableOp instanceof DropColumnOp + || alterTableOp instanceof RenameColumnOp || alterTableOp instanceof ModifyColumnOp + || alterTableOp instanceof ModifyColumnCommentOp || alterTableOp instanceof ReorderColumnsOp; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index 234389a2b2ccb2..f248e4382a0436 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -308,19 +308,23 @@ void testRejectUnsupportedDefaultChangesForIcebergTable() throws AnalysisExcepti } @Test - void testRejectCompoundNestedIcebergColumnOperations() throws AnalysisException { + void testRejectCompoundIcebergColumnOperations() throws AnalysisException { IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); for (String sql : Arrays.asList( "ALTER TABLE t ADD COLUMN s.good INT NULL, DROP COLUMN m.value.x", - "ALTER TABLE t MODIFY COLUMN c COMMENT 'new comment', ADD COLUMN d INT NULL")) { + "ALTER TABLE t MODIFY COLUMN c COMMENT 'new comment', ADD COLUMN d INT NULL", + "ALTER TABLE t ADD COLUMN c INT NULL, DROP COLUMN d", + "ALTER TABLE t RENAME COLUMN c TO c2, RENAME COLUMN d TO d2", + "ALTER TABLE t MODIFY COLUMN c COMMENT 'c', MODIFY COLUMN d COMMENT 'd'", + "ALTER TABLE t ORDER BY (c, d), ORDER BY (d, c)")) { AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); Assertions.assertTrue(exception.getMessage() - .contains("Multiple Iceberg column operations are not supported")); + .contains("Multiple Iceberg ALTER clauses are not supported")); } AlterTableCommand.checkColumnOperationsSupported(table, - parseAlter("ALTER TABLE t ADD COLUMN c INT NULL, DROP COLUMN d").getOps()); + parseAlter("ALTER TABLE t ADD COLUMN (c1 INT NULL, c2 BIGINT NULL)").getOps()); } @Test diff --git a/regression-test/suites/doc/sql-manual/basic-elements/data-types/struct-md.groovy b/regression-test/suites/doc/sql-manual/basic-elements/data-types/struct-md.groovy index 1ab47fd80dfdc7..ec088e300e8b31 100644 --- a/regression-test/suites/doc/sql-manual/basic-elements/data-types/struct-md.groovy +++ b/regression-test/suites/doc/sql-manual/basic-elements/data-types/struct-md.groovy @@ -184,7 +184,7 @@ suite("struct-md", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ - exception "Aggregate type SUM is not compatible with primitive type STRUCT<`id`:INT,`name`:TEXT>" + exception "Aggregate type SUM is not compatible with primitive type STRUCT" } test{ @@ -200,7 +200,7 @@ suite("struct-md", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ - exception "Aggregate type MIN is not compatible with primitive type STRUCT<`id`:INT,`name`:TEXT>" + exception "Aggregate type MIN is not compatible with primitive type STRUCT" } test { @@ -216,7 +216,7 @@ suite("struct-md", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ - exception "Aggregate type MAX is not compatible with primitive type STRUCT<`id`:INT,`name`:TEXT>" + exception "Aggregate type MAX is not compatible with primitive type STRUCT" } test { @@ -232,7 +232,7 @@ suite("struct-md", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ - exception "Aggregate type HLL_UNION is not compatible with primitive type STRUCT<`id`:INT,`name`:TEXT>" + exception "Aggregate type HLL_UNION is not compatible with primitive type STRUCT" } test { @@ -248,7 +248,7 @@ suite("struct-md", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ - exception "Aggregate type BITMAP_UNION is not compatible with primitive type STRUCT<`id`:INT,`name`:TEXT>" + exception "Aggregate type BITMAP_UNION is not compatible with primitive type STRUCT" } test { @@ -264,7 +264,7 @@ suite("struct-md", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ - exception "Aggregate type QUANTILE_UNION is not compatible with primitive type STRUCT<`id`:INT,`name`:TEXT>" + exception "Aggregate type QUANTILE_UNION is not compatible with primitive type STRUCT" } test { diff --git a/regression-test/suites/doc/sql-manual/sql-function/test_array_function.groovy b/regression-test/suites/doc/sql-manual/sql-function/test_array_function.groovy index c6073b28c6763b..f20bfb60f38aa9 100644 --- a/regression-test/suites/doc/sql-manual/sql-function/test_array_function.groovy +++ b/regression-test/suites/doc/sql-manual/sql-function/test_array_function.groovy @@ -126,7 +126,7 @@ suite("test_array_function_doc", "p0") { test { sql """ SELECT ARRAYS_OVERLAP(array_struct, array_struct) from ${tableName}; """ - exception "arrays_overlap does not support types: ARRAY>" + exception "arrays_overlap does not support types: ARRAY>" } test { @@ -400,7 +400,7 @@ suite("test_array_function_doc", "p0") { test { sql """ SELECT ARRAY_UNION(array_struct, array_struct) from ${tableName}; """ - exception "array_union does not support types: ARRAY>" + exception "array_union does not support types: ARRAY>" } test { From 7ebdcf9145a308b85ec0a10c6cae25fb30821a25 Mon Sep 17 00:00:00 2001 From: daidai Date: Wed, 22 Jul 2026 19:21:43 +0800 Subject: [PATCH 32/32] [fix](iceberg) Preserve identifier metadata during nested renames ### What problem does this PR solve? Issue Number: None Related PR: #65329 Problem Summary: Whole-PR review found that Iceberg 1.10.1 loses or leaves stale identifier-field paths when a nested identifier field or one of its ancestors is renamed, causing an otherwise valid schema update to fail. Rewrite only the affected identifier paths by field ID in the same UpdateSchema transaction, preserving identifier metadata and avoiding false matches for dotted sibling names. Also schedule the main Spark/Doris interoperability suite in the Iceberg Docker CI group and make struct-field identifier rendering independent of the JVM default locale. ### Release note Nested Iceberg identifier fields and their ancestors can be renamed without losing identifier metadata. Struct type SQL rendering is also stable across JVM locales. ### Check List (For Author) - Test: Unit Test and static analysis - IcebergMetadataOpsValidationTest: 55 tests passed; final identifier rename cases: 2/2 passed. - Turkish-locale struct field round-trip: 1/1 passed with targeted compilation and JUnit execution. - Maven Checkstyle: 0 violations. - BE/Cloud clang-format v16 check and scoped git diff checks passed. - Behavior changed: Yes. Nested identifier-field renames now preserve identifier metadata, and the interoperability suite is included in Iceberg Docker CI. - Does this need documentation: No --- .../iceberg/IcebergMetadataOps.java | 35 ++++++- .../doris/nereids/parser/NereidsParser.java | 3 +- .../IcebergMetadataOpsValidationTest.java | 91 +++++++++++++++++++ ...cebergNestedSchemaEvolutionParserTest.java | 16 ++++ ...chema_evolution_spark_doris_interop.groovy | 3 +- 5 files changed, 144 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 2e6751e404af4b..c6363c61cecd61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -1005,7 +1005,7 @@ public void renameColumn(ExternalTable dorisTable, String oldName, String newNam validateNoCaseInsensitiveSiblingCollision( schema.asStruct(), "", newName, oldPath.getField(), "rename"); UpdateSchema updateSchema = icebergTable.updateSchema(); - updateSchema.renameColumn(oldPath.getFullPath(), newName); + applyRenameColumn(schema, updateSchema, oldPath, newName); try { executionAuthenticator.execute(() -> updateSchema.commit()); } catch (Exception e) { @@ -1029,7 +1029,7 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String parentPath.getColumnPath(), newName, resolvedPath.getField(), "rename"); UpdateSchema updateSchema = icebergTable.updateSchema(); - updateSchema.renameColumn(resolvedPath.getFullPath(), newName); + applyRenameColumn(icebergTable.schema(), updateSchema, resolvedPath, newName); try { executionAuthenticator.execute(() -> updateSchema.commit()); } catch (Exception e) { @@ -1039,6 +1039,37 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String refreshTable(dorisTable, updateTime); } + private void applyRenameColumn(Schema schema, UpdateSchema updateSchema, + ResolvedColumnPath oldPath, String newName) { + String oldFullPath = oldPath.getFullPath(); + ColumnPath renamedPath = oldPath.getColumnPath().isNested() + ? childPath(oldPath.getColumnPath().getParentPath(), newName) + : ColumnPath.of(newName); + String renamedFullPath = renamedPath.getFullPath(); + boolean identifierFieldRenamed = false; + Set renamedIdentifierFields = new TreeSet<>(); + int renamedFieldId = oldPath.getField().fieldId(); + // Iceberg 1.10.1 does not preserve full identifier paths when an identifier field or one + // of its ancestors is renamed. Use field identity so dotted sibling names are not mistaken for descendants. + for (int identifierFieldId : schema.identifierFieldIds()) { + String identifierField = schema.findColumnName(identifierFieldId); + boolean isRenamedField = identifierFieldId == renamedFieldId; + boolean isDescendant = TypeUtil.ancestorFields(schema, identifierFieldId).stream() + .anyMatch(field -> field.fieldId() == renamedFieldId); + if (isRenamedField || isDescendant) { + renamedIdentifierFields.add(renamedFullPath + identifierField.substring(oldFullPath.length())); + identifierFieldRenamed = true; + } else { + renamedIdentifierFields.add(identifierField); + } + } + + updateSchema.renameColumn(oldFullPath, newName); + if (identifierFieldRenamed) { + updateSchema.setIdentifierFields(renamedIdentifierFields); + } + } + @Override public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index c997045e7d35a4..c7a7bed074e9f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -59,6 +59,7 @@ import java.util.BitSet; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -324,7 +325,7 @@ public static boolean isValidUnquotedIdentifier(String expression) { if (!hasLetter) { return false; } - String upperCase = expression.toUpperCase(); + String upperCase = expression.toUpperCase(Locale.ROOT); return (NON_RESERVED_KEYWORDS.contains(upperCase) || !LITERAL_TOKENS.containsKey(upperCase)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index fd9ac33e955f29..f78c186d0d1b5c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -769,6 +769,97 @@ public void execute(Runnable task) { icebergCatalog.close(); } + @Test + public void testRenamePreservesNestedIdentifierFieldPaths() throws Exception { + String dbName = "db"; + String tableName = "identifier_table"; + TableIdentifier tableIdentifier = TableIdentifier.of(dbName, tableName); + Map properties = new HashMap<>(); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, + temporaryFolder.newFolder("identifier_warehouse").toURI().toString()); + + HadoopCatalog icebergCatalog = new HadoopCatalog(); + icebergCatalog.setConf(new Configuration()); + icebergCatalog.initialize("identifier_catalog", properties); + icebergCatalog.createNamespace(Namespace.of(dbName)); + Schema schema = new Schema(Arrays.asList( + Types.NestedField.required(1, "root", Types.StructType.of( + Types.NestedField.required(2, "child", Types.StructType.of( + Types.NestedField.required(3, "id", Types.IntegerType.get()), + Types.NestedField.optional(4, "value", Types.StringType.get())))))), + Collections.singleton(3)); + Table icebergTable = icebergCatalog.createTable(tableIdentifier, schema); + + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn(dbName); + Mockito.when(dorisTable.getRemoteName()).thenReturn(tableName); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.renameColumn(dorisTable, ColumnPath.fromDotName("root.child.id"), "renamed_id", 1L); + icebergTable.refresh(); + Assert.assertEquals(Collections.singleton("root.child.renamed_id"), + icebergTable.schema().identifierFieldNames()); + + ops.renameColumn(dorisTable, ColumnPath.fromDotName("root.child"), "renamed_child", 1L); + icebergTable.refresh(); + Assert.assertEquals(Collections.singleton("root.renamed_child.renamed_id"), + icebergTable.schema().identifierFieldNames()); + + ops.renameColumn(dorisTable, "root", "renamed_root", 1L); + icebergTable.refresh(); + Assert.assertEquals(Collections.singleton("renamed_root.renamed_child.renamed_id"), + icebergTable.schema().identifierFieldNames()); + Assert.assertEquals(3, icebergTable.schema().findField( + "renamed_root.renamed_child.renamed_id").fieldId()); + } + + icebergCatalog.dropTable(tableIdentifier); + icebergCatalog.dropNamespace(Namespace.of(dbName)); + icebergCatalog.close(); + } + + @Test + public void testRenameDoesNotRewriteDottedIdentifierSibling() throws Exception { + String dbName = "db"; + String tableName = "dotted_identifier_table"; + TableIdentifier tableIdentifier = TableIdentifier.of(dbName, tableName); + Map properties = new HashMap<>(); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, + temporaryFolder.newFolder("dotted_identifier_warehouse").toURI().toString()); + + HadoopCatalog icebergCatalog = new HadoopCatalog(); + icebergCatalog.setConf(new Configuration()); + icebergCatalog.initialize("dotted_identifier_catalog", properties); + icebergCatalog.createNamespace(Namespace.of(dbName)); + Schema schema = new Schema(Arrays.asList( + Types.NestedField.required(1, "a", Types.IntegerType.get()), + Types.NestedField.required(2, "a.b", Types.IntegerType.get())), + Collections.singleton(2)); + Table icebergTable = icebergCatalog.createTable(tableIdentifier, schema); + + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn(dbName); + Mockito.when(dorisTable.getRemoteName()).thenReturn(tableName); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.renameColumn(dorisTable, "a", "renamed", 1L); + icebergTable.refresh(); + Assert.assertNotNull(icebergTable.schema().findField("renamed")); + Assert.assertEquals(Collections.singleton("a.b"), icebergTable.schema().identifierFieldNames()); + Assert.assertEquals(2, icebergTable.schema().findField("a.b").fieldId()); + } + + icebergCatalog.dropTable(tableIdentifier); + icebergCatalog.dropNamespace(Namespace.of(dbName)); + icebergCatalog.close(); + } + @Test public void testNestedColumnOperationsRejectDefaultMetadata() { Schema schema = new Schema(Types.NestedField.optional(1, "s", Types.StructType.of( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java index f6efa1ab7c5ff4..3b3355c06079e1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -38,6 +38,7 @@ import java.util.Collections; import java.util.List; +import java.util.Locale; public class IcebergNestedSchemaEvolutionParserTest { @@ -204,6 +205,21 @@ public void testStructMemberIdentifiersRoundTrip() { Assertions.assertTrue(ordinary.toSql().contains("STRUCT")); } + @Test + public void testStructFieldSqlRoundTripIsLocaleIndependent() { + Locale originalLocale = Locale.getDefault(); + try { + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + StructType structType = (StructType) parser.parseDataType("STRUCT<`in`:INT>"); + String renderedType = structType.toSql(); + + Assertions.assertTrue(renderedType.contains("`in`:INT")); + Assertions.assertDoesNotThrow(() -> parser.parseDataType(renderedType)); + } finally { + Locale.setDefault(originalLocale); + } + } + @Test public void testEmptyQuotedIdentifiersAreRejectedAsParseErrors() { for (String sql : List.of( diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy index 16dfbbb2a9d60b..96725c2aedd884 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy @@ -17,7 +17,8 @@ import groovy.json.JsonSlurper -suite("test_iceberg_nested_schema_evolution_spark_doris_interop", "p0,external,iceberg") { +suite("test_iceberg_nested_schema_evolution_spark_doris_interop", + "p0,external,iceberg,external_docker,external_docker_iceberg") { String enabled = context.config.otherConfigs.get("enableIcebergTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { logger.info("disable iceberg test.")