diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DatasourcePrintableMap.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DatasourcePrintableMap.java index 09ba42359c2e6b..2bf2a525f445f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DatasourcePrintableMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DatasourcePrintableMap.java @@ -18,6 +18,7 @@ package org.apache.doris.common.util; import org.apache.doris.common.maxcompute.MCProperties; +import org.apache.doris.datasource.property.constants.AIProperties; import org.apache.doris.datasource.property.metastore.AWSGlueMetaStoreBaseProperties; import org.apache.doris.datasource.property.metastore.AliyunDLFBaseProperties; import org.apache.doris.datasource.property.metastore.IcebergRestProperties; @@ -55,6 +56,7 @@ public class DatasourcePrintableMap extends BasicPrintableMap { SENSITIVE_KEY.add("bos_secret_accesskey"); SENSITIVE_KEY.add("jdbc.password"); SENSITIVE_KEY.add("elasticsearch.password"); + SENSITIVE_KEY.add(AIProperties.API_KEY); SENSITIVE_KEY.addAll(Arrays.asList( MCProperties.SECRET_KEY)); SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(S3Properties.class)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java index 03752cad308ee2..1c43dd62d8cdf2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java @@ -113,6 +113,30 @@ public LogicalPlan visitCreateCatalog(DorisParser.CreateCatalogContext ctx) { return super.visitCreateCatalog(ctx); } + // create resource clause + @Override + public LogicalPlan visitCreateResource(DorisParser.CreateResourceContext ctx) { + if (ctx.properties != null && ctx.properties.fileProperties != null) { + DorisParser.PropertyClauseContext propertyClauseContext = ctx.properties; + encryptProperty(visitPropertyClause(propertyClauseContext), + propertyClauseContext.fileProperties.start.getStartIndex(), + propertyClauseContext.fileProperties.stop.getStopIndex()); + } + return super.visitCreateResource(ctx); + } + + // alter resource clause + @Override + public LogicalPlan visitAlterResource(DorisParser.AlterResourceContext ctx) { + if (ctx.propertyClause() != null && ctx.propertyClause().fileProperties != null) { + DorisParser.PropertyClauseContext propertyClauseContext = ctx.propertyClause(); + encryptProperty(visitPropertyClause(propertyClauseContext), + propertyClauseContext.fileProperties.start.getStartIndex(), + propertyClauseContext.fileProperties.stop.getStopIndex()); + } + return super.visitAlterResource(ctx); + } + // create repository clause (CREATE [READ ONLY] REPOSITORY ... WITH ... PROPERTIES(...)) @Override public LogicalPlan visitCreateRepository(DorisParser.CreateRepositoryContext ctx) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index dac8b50ac05297..d63e467e9d548d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -100,6 +100,7 @@ import org.apache.doris.nereids.trees.plans.commands.EmptyCommand; import org.apache.doris.nereids.trees.plans.commands.Forward; import org.apache.doris.nereids.trees.plans.commands.LoadCommand; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.commands.PrepareCommand; import org.apache.doris.nereids.trees.plans.commands.Redirect; import org.apache.doris.nereids.trees.plans.commands.SupportProfile; @@ -584,11 +585,27 @@ public void execute() throws Exception { TUniqueId queryId = UniqueIdUtils.fastUniqueId(); if (Config.enable_print_request_before_execution) { LOG.info("begin to execute query {} {}", - DebugUtil.printId(queryId), originStmt == null ? "null" : originStmt.originStmt); + DebugUtil.printId(queryId), getStmtForExecutionLog(parsedStmt, originStmt)); } queryRetry(queryId); } + static String getStmtForExecutionLog(StatementBase parsedStmt, OriginStatement originStmt) { + String stmt = originStmt == null ? "null" : originStmt.originStmt; + if (stmt == null) { + return null; + } + if (parsedStmt instanceof LogicalPlanAdapter) { + LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + if (logicalPlan instanceof NeedAuditEncryption) { + return ((NeedAuditEncryption) logicalPlan).geneEncryptionSQL(stmt); + } + } else if (parsedStmt != null && parsedStmt.needAuditEncryption()) { + return ""; + } + return stmt; + } + public void queryRetry(TUniqueId queryId) throws Exception { TUniqueId firstQueryId = queryId; int retryTime = Config.max_query_retry_time; diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java index dc2d3d2ce9e596..9208dda772d5fa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java @@ -43,6 +43,7 @@ public void testSensitiveKeysContainAliyunDLFProperties() { Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("bos_secret_accesskey")); Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("jdbc.password")); Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("elasticsearch.password")); + Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("ai.api_key")); Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("iceberg.rest.oauth2.credential")); Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("iceberg.rest.oauth2.token")); @@ -160,6 +161,7 @@ public void testHidePasswordWithSensitiveKeys() { testMap.put("kerberos_keytab_content", "kerberos_content"); testMap.put("iceberg.rest.oauth2.credential", "iceberg_rest_credential"); testMap.put("iceberg.rest.oauth2.token", "iceberg_rest_token"); + testMap.put("AI.API_KEY", "ai_api_key"); DatasourcePrintableMap printableMap = new DatasourcePrintableMap<>(testMap, "=", false, false, true); String result = printableMap.toString(); @@ -173,6 +175,7 @@ public void testHidePasswordWithSensitiveKeys() { + DatasourcePrintableMap.PASSWORD_MASK)); Assertions.assertTrue(result.contains("iceberg.rest.oauth2.token = " + DatasourcePrintableMap.PASSWORD_MASK)); + Assertions.assertTrue(result.contains("AI.API_KEY = " + DatasourcePrintableMap.PASSWORD_MASK)); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ResourceAuditEncryptionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ResourceAuditEncryptionTest.java new file mode 100644 index 00000000000000..e66d0dfb6c6c5b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ResourceAuditEncryptionTest.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ResourceAuditEncryptionTest { + + private String encrypt(String sql) { + LogicalPlan plan = new NereidsParser().parseSingle(sql); + Assertions.assertTrue(plan instanceof NeedAuditEncryption, + "command should be NeedAuditEncryption: " + plan.getClass().getName()); + NeedAuditEncryption cmd = (NeedAuditEncryption) plan; + Assertions.assertTrue(cmd.needAuditEncryption()); + return cmd.geneEncryptionSQL(sql); + } + + @Test + public void testCreateAiResourceMasksApiKey() { + String sql = "CREATE RESOURCE IF NOT EXISTS `ai_resource` PROPERTIES (" + + "'type' = 'ai', " + + "'ai.provider_type' = 'deepseek', " + + "'ai.endpoint' = 'https://api.deepseek.com/chat/completions', " + + "'ai.model_name' = 'deepseek-chat', " + + "'ai.api_key' = 'sk-secret')"; + String masked = encrypt(sql); + Assertions.assertFalse(masked.contains("sk-secret"), "api key must be masked: " + masked); + Assertions.assertTrue(masked.contains("*XXX"), "expected mask token: " + masked); + Assertions.assertTrue(masked.contains("https://api.deepseek.com/chat/completions"), masked); + Assertions.assertTrue(masked.contains("deepseek-chat"), masked); + } + + @Test + public void testAlterAiResourceMasksApiKey() { + String sql = "ALTER RESOURCE `ai_resource` PROPERTIES (" + + "'ai.api_key' = 'new-secret', " + + "'ai.endpoint' = 'https://api.deepseek.com/chat/completions')"; + String masked = encrypt(sql); + Assertions.assertFalse(masked.contains("new-secret"), "api key must be masked: " + masked); + Assertions.assertTrue(masked.contains("*XXX"), "expected mask token: " + masked); + Assertions.assertTrue(masked.contains("https://api.deepseek.com/chat/completions"), masked); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index b7b12bc91c3deb..5f4e14f127367c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -24,6 +24,10 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlSerializer; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.Planner; import org.apache.doris.planner.ResultFileSink; @@ -99,6 +103,24 @@ public void testSet() throws Exception { Assert.assertEquals(QueryState.MysqlStateType.OK, connectContext.getState().getStateType()); } + @Test + public void testExecutionLogStmtMasksNeedAuditEncryptionSql() { + String sql = "CREATE RESOURCE IF NOT EXISTS `ai_resource` PROPERTIES (" + + "'type' = 'ai', " + + "'ai.provider_type' = 'deepseek', " + + "'ai.endpoint' = 'https://api.deepseek.com/chat/completions', " + + "'ai.model_name' = 'deepseek-chat', " + + "'ai.api_key' = 'sk-secret')"; + LogicalPlan plan = new NereidsParser().parseSingle(sql); + LogicalPlanAdapter adapter = new LogicalPlanAdapter(plan, new StatementContext()); + + String stmtForLog = StmtExecutor.getStmtForExecutionLog(adapter, new OriginStatement(sql, 0)); + + Assertions.assertFalse(stmtForLog.contains("sk-secret"), stmtForLog); + Assertions.assertTrue(stmtForLog.contains("*XXX"), stmtForLog); + Assertions.assertTrue(stmtForLog.contains("https://api.deepseek.com/chat/completions"), stmtForLog); + } + @Test public void testDdlFail() throws Exception { StmtExecutor executor = new StmtExecutor(connectContext, "CREATE FILE \\\"ca.pem\\\"\\n\"\n" diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index ec01b743813df3..a2c1198667b881 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -24,18 +24,18 @@ defaultDb = "regression_test" // init cmd like: select @@session.tx_read_only // at each time we connect. // add allowLoadLocalInfile so that the jdbc can execute mysql load data from client. -jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" -targetJdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" +jdbcUrl = "jdbc:mysql://127.0.0.1:9330/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" +targetJdbcUrl = "jdbc:mysql://127.0.0.1:9330/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" jdbcUser = "root" jdbcPassword = "" -feSourceThriftAddress = "127.0.0.1:9020" -feTargetThriftAddress = "127.0.0.1:9020" +feSourceThriftAddress = "127.0.0.1:9220" +feTargetThriftAddress = "127.0.0.1:9220" syncerAddress = "127.0.0.1:9190" feSyncerUser = "root" feSyncerPassword = "" -feHttpAddress = "127.0.0.1:8030" +feHttpAddress = "127.0.0.1:8330" feHttpUser = "root" feHttpPassword = "" @@ -254,7 +254,7 @@ polaris_minio_port=20001 // If the failure suite num exceeds this config // all following suite will be skipped to fast quit the run. // <=0 means no limit. -max_failure_num=0 +max_failure_num=1 // used for exporting test s3ExportBucketName = "" @@ -302,7 +302,7 @@ hudiEmrCatalog = "" icebergS3TablesCatalog="" icebergS3TablesCatalogGlueRest="" -// The path of the cert configuration file for the testing framework +// The path of the cert configuration file for the testing framework // is consistent with the path of the cert file for the cluster enableTLS=false tlsVerifyMode="strict" @@ -332,7 +332,7 @@ hudiHmsPort=19083 hudiMinioPort=19100 hudiMinioAccessKey="minio" hudiMinioSecretKey="minio123" + icebergDlfRestCatalog="'type' = 'iceberg', 'warehouse' = 'new_dlf_iceberg_catalog', 'iceberg.catalog.type' = 'rest', 'iceberg.rest.uri' = 'http://cn-beijing-vpc.dlf.aliyuncs.com/iceberg', 'iceberg.rest.sigv4-enabled' = 'true', 'iceberg.rest.signing-name' = 'DlfNext', 'iceberg.rest.access-key-id' = 'ak', 'iceberg.rest.secret-access-key' = 'sk', 'iceberg.rest.signing-region' = 'cn-beijing', 'iceberg.rest.vended-credentials-enabled' = 'true', 'io-impl' = 'org.apache.iceberg.rest.DlfFileIO', 'fs.oss.support' = 'true'" -// For python UDF test, set the runtime version of python, default: 3.8.10 -// pythonUdfRuntimeVersion = "" \ No newline at end of file +pythonUdfRuntimeVersion = "3.12.11" \ No newline at end of file