[Fix](log) mask AI resource api key in logs

fe/fe-core/src/main/java/org/apache/doris/common/util/DatasourcePrintableMap.java

@@ -18,6 +18,7 @@
 package org.apache.doris.common.util;
 
 import org.apache.doris.common.maxcompute.MCProperties;
+import org.apache.doris.datasource.property.constants.AIProperties;
 import org.apache.doris.datasource.property.metastore.AWSGlueMetaStoreBaseProperties;
 import org.apache.doris.datasource.property.metastore.AliyunDLFBaseProperties;
 import org.apache.doris.datasource.property.metastore.IcebergRestProperties;
@@ -55,6 +56,7 @@ public class DatasourcePrintableMap<K, V> extends BasicPrintableMap<K, V> {
         SENSITIVE_KEY.add("bos_secret_accesskey");
         SENSITIVE_KEY.add("jdbc.password");
         SENSITIVE_KEY.add("elasticsearch.password");
+        SENSITIVE_KEY.add(AIProperties.API_KEY);
         SENSITIVE_KEY.addAll(Arrays.asList(
                 MCProperties.SECRET_KEY));
         SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(S3Properties.class));

fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java

@@ -113,6 +113,30 @@ public LogicalPlan visitCreateCatalog(DorisParser.CreateCatalogContext ctx) {
         return super.visitCreateCatalog(ctx);
     }
 
+    // create resource clause
+    @Override
+    public LogicalPlan visitCreateResource(DorisParser.CreateResourceContext ctx) {
+        if (ctx.properties != null && ctx.properties.fileProperties != null) {
+            DorisParser.PropertyClauseContext propertyClauseContext = ctx.properties;
+            encryptProperty(visitPropertyClause(propertyClauseContext),
+                    propertyClauseContext.fileProperties.start.getStartIndex(),
+                    propertyClauseContext.fileProperties.stop.getStopIndex());
+        }
+        return super.visitCreateResource(ctx);
+    }
+
+    // alter resource clause
+    @Override
+    public LogicalPlan visitAlterResource(DorisParser.AlterResourceContext ctx) {
+        if (ctx.propertyClause() != null && ctx.propertyClause().fileProperties != null) {
+            DorisParser.PropertyClauseContext propertyClauseContext = ctx.propertyClause();
+            encryptProperty(visitPropertyClause(propertyClauseContext),
+                    propertyClauseContext.fileProperties.start.getStartIndex(),
+                    propertyClauseContext.fileProperties.stop.getStopIndex());
+        }
+        return super.visitAlterResource(ctx);
+    }
+
     // create repository clause (CREATE [READ ONLY] REPOSITORY ... WITH <backend> ... PROPERTIES(...))
     @Override
     public LogicalPlan visitCreateRepository(DorisParser.CreateRepositoryContext ctx) {

fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java

@@ -100,6 +100,7 @@
 import org.apache.doris.nereids.trees.plans.commands.EmptyCommand;
 import org.apache.doris.nereids.trees.plans.commands.Forward;
 import org.apache.doris.nereids.trees.plans.commands.LoadCommand;
+import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption;
 import org.apache.doris.nereids.trees.plans.commands.PrepareCommand;
 import org.apache.doris.nereids.trees.plans.commands.Redirect;
 import org.apache.doris.nereids.trees.plans.commands.SupportProfile;
@@ -584,11 +585,27 @@ public void execute() throws Exception {
         TUniqueId queryId = UniqueIdUtils.fastUniqueId();
         if (Config.enable_print_request_before_execution) {
             LOG.info("begin to execute query {} {}",
-                    DebugUtil.printId(queryId), originStmt == null ? "null" : originStmt.originStmt);
+                    DebugUtil.printId(queryId), getStmtForExecutionLog(parsedStmt, originStmt));
         }
         queryRetry(queryId);
     }
 
+    static String getStmtForExecutionLog(StatementBase parsedStmt, OriginStatement originStmt) {
+        String stmt = originStmt == null ? "null" : originStmt.originStmt;
+        if (stmt == null) {
+            return null;
+        }
+        if (parsedStmt instanceof LogicalPlanAdapter) {
+            LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan();
+            if (logicalPlan instanceof NeedAuditEncryption) {
+                return ((NeedAuditEncryption) logicalPlan).geneEncryptionSQL(stmt);
+            }
+        } else if (parsedStmt != null && parsedStmt.needAuditEncryption()) {
+            return "";
+        }
+        return stmt;
+    }
+
     public void queryRetry(TUniqueId queryId) throws Exception {
         TUniqueId firstQueryId = queryId;
         int retryTime = Config.max_query_retry_time;

fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java

@@ -43,6 +43,7 @@ public void testSensitiveKeysContainAliyunDLFProperties() {
         Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("bos_secret_accesskey"));
         Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("jdbc.password"));
         Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("elasticsearch.password"));
+        Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("ai.api_key"));
         Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("iceberg.rest.oauth2.credential"));
         Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("iceberg.rest.oauth2.token"));
 
@@ -160,6 +161,7 @@ public void testHidePasswordWithSensitiveKeys() {
         testMap.put("kerberos_keytab_content", "kerberos_content");
         testMap.put("iceberg.rest.oauth2.credential", "iceberg_rest_credential");
         testMap.put("iceberg.rest.oauth2.token", "iceberg_rest_token");
+        testMap.put("AI.API_KEY", "ai_api_key");
 
         DatasourcePrintableMap<String, String> printableMap = new DatasourcePrintableMap<>(testMap, "=", false, false, true);
         String result = printableMap.toString();
@@ -173,6 +175,7 @@ public void testHidePasswordWithSensitiveKeys() {
                 + DatasourcePrintableMap.PASSWORD_MASK));
         Assertions.assertTrue(result.contains("iceberg.rest.oauth2.token = "
                 + DatasourcePrintableMap.PASSWORD_MASK));
+        Assertions.assertTrue(result.contains("AI.API_KEY = " + DatasourcePrintableMap.PASSWORD_MASK));
     }
 
     @Test

fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ResourceAuditEncryptionTest.java

@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.parser;
+
+import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class ResourceAuditEncryptionTest {
+
+    private String encrypt(String sql) {
+        LogicalPlan plan = new NereidsParser().parseSingle(sql);
+        Assertions.assertTrue(plan instanceof NeedAuditEncryption,
+                "command should be NeedAuditEncryption: " + plan.getClass().getName());
+        NeedAuditEncryption cmd = (NeedAuditEncryption) plan;
+        Assertions.assertTrue(cmd.needAuditEncryption());
+        return cmd.geneEncryptionSQL(sql);
+    }
+
+    @Test
+    public void testCreateAiResourceMasksApiKey() {
+        String sql = "CREATE RESOURCE IF NOT EXISTS `ai_resource` PROPERTIES ("
+                + "'type' = 'ai', "
+                + "'ai.provider_type' = 'deepseek', "
+                + "'ai.endpoint' = 'https://api.deepseek.com/chat/completions', "
+                + "'ai.model_name' = 'deepseek-chat', "
+                + "'ai.api_key' = 'sk-secret')";
+        String masked = encrypt(sql);
+        Assertions.assertFalse(masked.contains("sk-secret"), "api key must be masked: " + masked);
+        Assertions.assertTrue(masked.contains("*XXX"), "expected mask token: " + masked);
+        Assertions.assertTrue(masked.contains("https://api.deepseek.com/chat/completions"), masked);
+        Assertions.assertTrue(masked.contains("deepseek-chat"), masked);
+    }
+
+    @Test
+    public void testAlterAiResourceMasksApiKey() {
+        String sql = "ALTER RESOURCE `ai_resource` PROPERTIES ("
+                + "'ai.api_key' = 'new-secret', "
+                + "'ai.endpoint' = 'https://api.deepseek.com/chat/completions')";
+        String masked = encrypt(sql);
+        Assertions.assertFalse(masked.contains("new-secret"), "api key must be masked: " + masked);
+        Assertions.assertTrue(masked.contains("*XXX"), "expected mask token: " + masked);
+        Assertions.assertTrue(masked.contains("https://api.deepseek.com/chat/completions"), masked);
+    }
+}

fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java

@@ -24,6 +24,10 @@
 import org.apache.doris.common.FeConstants;
 import org.apache.doris.mysql.MysqlChannel;
 import org.apache.doris.mysql.MysqlSerializer;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.planner.PlanFragment;
 import org.apache.doris.planner.Planner;
 import org.apache.doris.planner.ResultFileSink;
@@ -99,6 +103,24 @@ public void testSet() throws Exception {
         Assert.assertEquals(QueryState.MysqlStateType.OK, connectContext.getState().getStateType());
     }
 
+    @Test
+    public void testExecutionLogStmtMasksNeedAuditEncryptionSql() {
+        String sql = "CREATE RESOURCE IF NOT EXISTS `ai_resource` PROPERTIES ("
+                + "'type' = 'ai', "
+                + "'ai.provider_type' = 'deepseek', "
+                + "'ai.endpoint' = 'https://api.deepseek.com/chat/completions', "
+                + "'ai.model_name' = 'deepseek-chat', "
+                + "'ai.api_key' = 'sk-secret')";
+        LogicalPlan plan = new NereidsParser().parseSingle(sql);
+        LogicalPlanAdapter adapter = new LogicalPlanAdapter(plan, new StatementContext());
+
+        String stmtForLog = StmtExecutor.getStmtForExecutionLog(adapter, new OriginStatement(sql, 0));
+
+        Assertions.assertFalse(stmtForLog.contains("sk-secret"), stmtForLog);
+        Assertions.assertTrue(stmtForLog.contains("*XXX"), stmtForLog);
+        Assertions.assertTrue(stmtForLog.contains("https://api.deepseek.com/chat/completions"), stmtForLog);
+    }
+
     @Test
     public void testDdlFail() throws Exception {
         StmtExecutor executor = new StmtExecutor(connectContext, "CREATE FILE \\\"ca.pem\\\"\\n\"\n"

regression-test/conf/regression-conf.groovy

@@ -24,18 +24,18 @@ defaultDb = "regression_test"
 // init cmd like: select @@session.tx_read_only
 // at each time we connect.
 // add allowLoadLocalInfile so that the jdbc can execute mysql load data from client.
-jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round"
-targetJdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round"
+jdbcUrl = "jdbc:mysql://127.0.0.1:9330/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round"
+targetJdbcUrl = "jdbc:mysql://127.0.0.1:9330/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round"
 jdbcUser = "root"
 jdbcPassword = ""
 
-feSourceThriftAddress = "127.0.0.1:9020"
-feTargetThriftAddress = "127.0.0.1:9020"
+feSourceThriftAddress = "127.0.0.1:9220"
+feTargetThriftAddress = "127.0.0.1:9220"
 syncerAddress = "127.0.0.1:9190"
 feSyncerUser = "root"
 feSyncerPassword = ""
 
-feHttpAddress = "127.0.0.1:8030"
+feHttpAddress = "127.0.0.1:8330"
 feHttpUser = "root"
 feHttpPassword = ""
 
@@ -254,7 +254,7 @@ polaris_minio_port=20001
 // If the failure suite num exceeds this config
 // all following suite will be skipped to fast quit the run.
 // <=0 means no limit.
-max_failure_num=0
+max_failure_num=1
 
 // used for exporting test
 s3ExportBucketName = ""
@@ -302,7 +302,7 @@ hudiEmrCatalog = ""
 icebergS3TablesCatalog=""
 icebergS3TablesCatalogGlueRest=""
 
-// The path of the cert configuration file for the testing framework
+// The path of the cert configuration file for the testing framework 
 // is consistent with the path of the cert file for the cluster
 enableTLS=false
 tlsVerifyMode="strict"
@@ -332,7 +332,7 @@ hudiHmsPort=19083
 hudiMinioPort=19100
 hudiMinioAccessKey="minio"
 hudiMinioSecretKey="minio123"
+
 icebergDlfRestCatalog="'type' = 'iceberg', 'warehouse' = 'new_dlf_iceberg_catalog', 'iceberg.catalog.type' = 'rest', 'iceberg.rest.uri' = 'http://cn-beijing-vpc.dlf.aliyuncs.com/iceberg', 'iceberg.rest.sigv4-enabled' = 'true', 'iceberg.rest.signing-name' = 'DlfNext', 'iceberg.rest.access-key-id' = 'ak', 'iceberg.rest.secret-access-key' = 'sk', 'iceberg.rest.signing-region' = 'cn-beijing', 'iceberg.rest.vended-credentials-enabled' = 'true', 'io-impl' = 'org.apache.iceberg.rest.DlfFileIO', 'fs.oss.support' = 'true'"
 
-// For python UDF test, set the runtime version of python, default: 3.8.10
-// pythonUdfRuntimeVersion = ""
+pythonUdfRuntimeVersion = "3.12.11"