diff --git a/be/src/information_schema/schema_tso_status_scanner.cpp b/be/src/information_schema/schema_tso_status_scanner.cpp index ef9a0584fef7b0..9f453a32c264e6 100644 --- a/be/src/information_schema/schema_tso_status_scanner.cpp +++ b/be/src/information_schema/schema_tso_status_scanner.cpp @@ -34,6 +34,11 @@ std::vector SchemaTsoStatusScanner::_s_tso_status_col {"CURRENT_TSO", TYPE_BIGINT, sizeof(int64_t), true}, {"CURRENT_TSO_PHYSICAL_TIME", TYPE_BIGINT, sizeof(int64_t), true}, {"CURRENT_TSO_LOGICAL_COUNTER", TYPE_BIGINT, sizeof(int64_t), true}, + {.name = "COMMITTED_TSO", .type = TYPE_BIGINT, .size = sizeof(int64_t), .is_null = true}, + {.name = "COMMITTED_TSO_PHYSICAL_TIME", + .type = TYPE_BIGINT, + .size = sizeof(int64_t), + .is_null = true}, }; SchemaTsoStatusScanner::SchemaTsoStatusScanner() @@ -51,6 +56,11 @@ Status SchemaTsoStatusScanner::_get_tso_status_block_from_fe() { TNetworkAddress master_addr = ExecEnv::GetInstance()->cluster_info()->master_fe_addr; TSchemaTableRequestParams schema_table_request_params; + std::vector columns; + for (const auto& column : _s_tso_status_columns) { + columns.emplace_back(column.name); + } + schema_table_request_params.__set_columns_name(columns); TFetchSchemaTableDataRequest request; request.__set_schema_table_name(TSchemaTableName::TSO_STATUS); request.__set_schema_table_params(schema_table_request_params); diff --git a/be/test/exec/schema_scanner/schema_tso_status_scanner_test.cpp b/be/test/exec/schema_scanner/schema_tso_status_scanner_test.cpp index 60a95e3bffcb13..6cf5b09117d025 100644 --- a/be/test/exec/schema_scanner/schema_tso_status_scanner_test.cpp +++ b/be/test/exec/schema_scanner/schema_tso_status_scanner_test.cpp @@ -37,7 +37,7 @@ namespace doris { namespace { -TRow create_tso_status_row(const std::array& values) { +TRow create_tso_status_row(const std::array& values) { std::vector cells; cells.reserve(values.size()); for (int64_t value : values) { @@ -65,7 +65,7 @@ std::unique_ptr create_output_block(SchemaTsoStatusScanner* scanner) { } void expect_tso_status_row(const Block& block, size_t row_idx, - const std::array& expected) { + const std::array& expected) { ASSERT_EQ(expected.size(), block.columns()); for (size_t column_idx = 0; column_idx < expected.size(); ++column_idx) { const auto& column = block.get_by_position(column_idx).column; @@ -80,11 +80,13 @@ TEST(SchemaTsoStatusScannerTest, test_create_tso_status_scanner) { auto scanner = SchemaScanner::create(TSchemaTableType::SCH_TSO_STATUS); ASSERT_NE(nullptr, scanner); EXPECT_EQ(TSchemaTableType::SCH_TSO_STATUS, scanner->type()); - ASSERT_EQ(4, scanner->get_column_desc().size()); + ASSERT_EQ(6, scanner->get_column_desc().size()); EXPECT_STREQ("WINDOW_END_PHYSICAL_TIME", scanner->get_column_desc()[0].name); EXPECT_STREQ("CURRENT_TSO", scanner->get_column_desc()[1].name); EXPECT_STREQ("CURRENT_TSO_PHYSICAL_TIME", scanner->get_column_desc()[2].name); EXPECT_STREQ("CURRENT_TSO_LOGICAL_COUNTER", scanner->get_column_desc()[3].name); + EXPECT_STREQ("COMMITTED_TSO", scanner->get_column_desc()[4].name); + EXPECT_STREQ("COMMITTED_TSO_PHYSICAL_TIME", scanner->get_column_desc()[5].name); for (const auto& column : scanner->get_column_desc()) { EXPECT_EQ(TYPE_BIGINT, column.type); EXPECT_TRUE(column.is_null); @@ -140,8 +142,8 @@ TEST(SchemaTsoStatusScannerTest, test_process_tso_status_result_error) { } TEST(SchemaTsoStatusScannerTest, test_process_tso_status_result) { - const std::array first_row = {1000, 2000, 3000, 4000}; - const std::array second_row = {1001, 2001, 3001, 4001}; + const std::array first_row = {1000, 2000, 3000, 4000}; + const std::array second_row = {1001, 2001, 3001, 4001}; auto result = create_tso_status_result( {create_tso_status_row(first_row), create_tso_status_row(second_row)}); @@ -149,7 +151,7 @@ TEST(SchemaTsoStatusScannerTest, test_process_tso_status_result) { ASSERT_TRUE(scanner._process_tso_status_result(result).ok()); ASSERT_NE(nullptr, scanner._tso_status_block); - EXPECT_EQ(4, scanner._tso_status_block->columns()); + EXPECT_EQ(6, scanner._tso_status_block->columns()); EXPECT_EQ(2, scanner._tso_status_block->rows()); EXPECT_EQ(2, scanner._total_rows); expect_tso_status_row(*scanner._tso_status_block, 0, first_row); @@ -158,7 +160,8 @@ TEST(SchemaTsoStatusScannerTest, test_process_tso_status_result) { TEST(SchemaTsoStatusScannerTest, test_process_tso_status_result_schema_mismatch) { TRow invalid_row = create_tso_status_row({1000, 2000, 3000, 4000}); - invalid_row.column_value.pop_back(); + invalid_row.column_value.resize( + 4); // Response from an older FE cannot provide a committed prefix. auto result = create_tso_status_result({invalid_row}); SchemaTsoStatusScanner scanner; @@ -170,6 +173,17 @@ TEST(SchemaTsoStatusScannerTest, test_process_tso_status_result_schema_mismatch) EXPECT_EQ(0, scanner._total_rows); } +TEST(SchemaTsoStatusScannerTest, test_unknown_committed_tso_is_null) { + TRow row = create_tso_status_row({1000, 2000, 3000, 4000, 0, 0}); + row.column_value[4].__set_isNull(true); + row.column_value[5].__set_isNull(true); + SchemaTsoStatusScanner scanner; + ASSERT_TRUE(scanner._process_tso_status_result(create_tso_status_result({row})).ok()); + EXPECT_FALSE(scanner._tso_status_block->get_by_position(3).column->is_null_at(0)); + EXPECT_TRUE(scanner._tso_status_block->get_by_position(4).column->is_null_at(0)); + EXPECT_TRUE(scanner._tso_status_block->get_by_position(5).column->is_null_at(0)); +} + TEST(SchemaTsoStatusScannerTest, test_get_next_block_empty_result) { MockRuntimeState state; SchemaScannerParam param; @@ -188,9 +202,9 @@ TEST(SchemaTsoStatusScannerTest, test_get_next_block_empty_result) { } TEST(SchemaTsoStatusScannerTest, test_get_next_block_in_batches) { - const std::array first_row = {1000, 2000, 3000, 4000}; - const std::array second_row = {1001, 2001, 3001, 4001}; - const std::array third_row = {1002, 2002, 3002, 4002}; + const std::array first_row = {1000, 2000, 3000, 4000}; + const std::array second_row = {1001, 2001, 3001, 4001}; + const std::array third_row = {1002, 2002, 3002, 4002}; MockRuntimeState state; state._batch_size = 2; diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 18da953b1c1e97..487ada73d3dde0 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -43,6 +43,7 @@ BvarLatencyRecorderWithTag g_bvar_ms_create_meta_sync_point("ms", "create_meta_s BvarLatencyRecorderWithTag g_bvar_ms_begin_sub_txn("ms", "begin_sub_txn"); BvarLatencyRecorderWithTag g_bvar_ms_abort_sub_txn("ms", "abort_sub_txn"); BvarLatencyRecorderWithTag g_bvar_ms_check_txn_conflict("ms", "check_txn_conflict"); +BvarLatencyRecorderWithTag g_bvar_ms_advance_tso_fence("ms", "advance_tso_fence"); BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator("ms", "abort_txn_with_coordinator"); BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator("ms", "get_prepare_txn_by_coordinator"); BvarLatencyRecorderWithTag g_bvar_ms_clean_txn_label("ms", "clean_txn_label"); @@ -500,6 +501,8 @@ mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter("rpc_kv_abo mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_counter("rpc_kv_get_prepare_txn_by_coordinator_get_counter",{"instance_id"}); // check_txn_conflict mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter("rpc_kv_check_txn_conflict_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_counter("rpc_kv_advance_tso_fence_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_counter("rpc_kv_advance_tso_fence_put_counter",{"instance_id"}); // clean_txn_label mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter("rpc_kv_clean_txn_label_get_counter",{"instance_id"}); mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter("rpc_kv_clean_txn_label_put_counter",{"instance_id"}); @@ -710,6 +713,8 @@ mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes("rpc_kv_abort mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_bytes("rpc_kv_get_prepare_txn_by_coordinator_get_bytes",{"instance_id"}); // check_txn_conflict mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes("rpc_kv_check_txn_conflict_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_bytes("rpc_kv_advance_tso_fence_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_bytes("rpc_kv_advance_tso_fence_put_bytes",{"instance_id"}); // clean_txn_label mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_bytes("rpc_kv_clean_txn_label_get_bytes",{"instance_id"}); mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_bytes("rpc_kv_clean_txn_label_put_bytes",{"instance_id"}); diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index 8c40c1316665a5..6eb1da13a70af6 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -553,6 +553,7 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_get_txn; extern BvarLatencyRecorderWithTag g_bvar_ms_get_current_max_txn_id; extern BvarLatencyRecorderWithTag g_bvar_ms_create_meta_sync_point; extern BvarLatencyRecorderWithTag g_bvar_ms_check_txn_conflict; +extern BvarLatencyRecorderWithTag g_bvar_ms_advance_tso_fence; extern BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator; extern BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator; extern BvarLatencyRecorderWithTag g_bvar_ms_begin_sub_txn; @@ -907,6 +908,8 @@ extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_counter; extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_counter; @@ -1050,6 +1053,8 @@ extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_bytes; diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index 26b265f1d05ab1..7023ccf56f6286 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -145,6 +145,10 @@ class MetaServiceImpl : public cloud::MetaService { CheckTxnConflictResponse* response, ::google::protobuf::Closure* done) override; + void advance_tso_fence(::google::protobuf::RpcController* controller, + const AdvanceTsoFenceRequest* request, AdvanceTsoFenceResponse* response, + ::google::protobuf::Closure* done) override; + void abort_txn_with_coordinator(::google::protobuf::RpcController* controller, const AbortTxnWithCoordinatorRequest* request, AbortTxnWithCoordinatorResponse* response, @@ -619,6 +623,12 @@ class MetaServiceProxy final : public MetaService { call_impl(&cloud::MetaService::check_txn_conflict, controller, request, response, done); } + void advance_tso_fence(::google::protobuf::RpcController* controller, + const AdvanceTsoFenceRequest* request, AdvanceTsoFenceResponse* response, + ::google::protobuf::Closure* done) override { + call_impl(&cloud::MetaService::advance_tso_fence, controller, request, response, done); + } + void abort_txn_with_coordinator(::google::protobuf::RpcController* controller, const AbortTxnWithCoordinatorRequest* request, AbortTxnWithCoordinatorResponse* response, diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 2dd1a000e3a5a4..c012aad0bd971e 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -61,6 +61,11 @@ inline std::pair resolve_response_code_and_msg(Met "[TXN_ALREADY_COMMITED will be converted to code=UNDEFINED_ERR for old version " "clients]"; return {MetaServiceCode::UNDEFINED_ERR, std::move(msg)}; + case MetaServiceCode::TXN_COMMIT_TSO_FENCED: + msg += std::string((msg.empty() ? "" : ", ")) + + "[TXN_COMMIT_TSO_FENCED will be converted to code=UNDEFINED_ERR for old " + "version clients]"; + return {MetaServiceCode::UNDEFINED_ERR, std::move(msg)}; default: return {code, std::move(msg)}; } diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 76963cb0627e8f..abd1ea4600f7fb 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -127,6 +127,39 @@ static void append_table_stream_commit_size_error(TxnErrorCode err, std::string& } } +static bool check_txn_commit_tso_fence(Transaction* txn, const std::string& instance_id, + int64_t commit_tso, CommitTxnResponse* response, + MetaServiceCode& code, std::string& msg) { + if (commit_tso <= 0) { + return true; + } + + std::string value; + TxnErrorCode err = txn->get(txn_tso_fence_key({instance_id}), &value); + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + return true; + } + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = fmt::format("failed to read TSO fence, err={}", err); + return false; + } + + TxnTsoFencePB fence; + if (!fence.ParseFromString(value) || !fence.has_fence_tso() || fence.fence_tso() <= 0) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + msg = "failed to parse TSO fence"; + return false; + } + if (commit_tso <= fence.fence_tso()) { + response->set_tso_fence(fence.fence_tso()); + code = MetaServiceCode::TXN_COMMIT_TSO_FENCED; + msg = fmt::format("commit TSO {} is fenced by {}", commit_tso, fence.fence_tso()); + return false; + } + return true; +} + class TableStreamUpdateTxnContext { public: TableStreamUpdateTxnContext(Transaction* txn, const std::string& instance_id, @@ -1911,6 +1944,11 @@ void MetaServiceImpl::commit_txn_immediately( return; } + if (txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED && + !check_txn_commit_tso_fence(txn.get(), instance_id, commit_tso, response, code, msg)) { + return; + } + MultiVersionStatus table_stream_multi_version_status = MultiVersionStatus::MULTI_VERSION_DISABLED; if (!request->table_stream_updates().empty()) { @@ -2743,6 +2781,11 @@ void MetaServiceImpl::commit_txn_eventually( return; } + if (txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED && + !check_txn_commit_tso_fence(txn.get(), instance_id, commit_tso, response, code, msg)) { + return; + } + auto now_time = system_clock::now(); uint64_t commit_time = duration_cast(now_time.time_since_epoch()).count(); if ((txn_info.prepare_time() + txn_info.timeout_ms()) < commit_time) { @@ -3143,6 +3186,11 @@ void MetaServiceImpl::commit_txn_with_sub_txn(const CommitTxnRequest* request, return; } + if (txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED && + !check_txn_commit_tso_fence(txn.get(), instance_id, commit_tso, response, code, msg)) { + return; + } + LOG(INFO) << "txn_id=" << txn_id << " txn_info=" << txn_info.ShortDebugString(); AnnotateTag txn_tag("txn_id", txn_id); @@ -4724,6 +4772,69 @@ std::string get_txn_info_key_from_txn_running_key(std::string_view txn_running_k return conflict_txn_info_key; } +void MetaServiceImpl::advance_tso_fence(::google::protobuf::RpcController* controller, + const AdvanceTsoFenceRequest* request, + AdvanceTsoFenceResponse* response, + ::google::protobuf::Closure* done) { + RPC_PREPROCESS(advance_tso_fence, get, put); + if (!request->has_proposed_fence_tso() || request->proposed_fence_tso() <= 0) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "invalid proposed TSO fence"; + return; + } + instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); + if (instance_id.empty()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "cannot find instance_id for TSO fence"; + return; + } + RPC_RATE_LIMIT(advance_tso_fence) + + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = "failed to create TSO fence transaction"; + return; + } + + const std::string key = txn_tso_fence_key({instance_id}); + std::string value; + err = txn->get(key, &value); + int64_t current_fence_tso = 0; + if (err == TxnErrorCode::TXN_OK) { + TxnTsoFencePB fence; + if (!fence.ParseFromString(value) || !fence.has_fence_tso() || fence.fence_tso() <= 0) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + msg = "failed to parse TSO fence"; + return; + } + current_fence_tso = fence.fence_tso(); + } else if (err != TxnErrorCode::TXN_KEY_NOT_FOUND) { + code = cast_as(err); + msg = "failed to read TSO fence"; + return; + } + + const int64_t effective_fence_tso = std::max(current_fence_tso, request->proposed_fence_tso()); + if (effective_fence_tso > current_fence_tso) { + TxnTsoFencePB fence; + fence.set_fence_tso(effective_fence_tso); + if (!fence.SerializeToString(&value)) { + code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; + msg = "failed to serialize TSO fence"; + return; + } + txn->put(key, value); + err = txn->commit(); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = "failed to commit TSO fence"; + return; + } + } + response->set_tso_fence(effective_fence_tso); +} + void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* controller, const CheckTxnConflictRequest* request, CheckTxnConflictResponse* response, diff --git a/cloud/src/meta-store/keys.cpp b/cloud/src/meta-store/keys.cpp index 3c4fb91c4f5658..4e3ac10753c86e 100644 --- a/cloud/src/meta-store/keys.cpp +++ b/cloud/src/meta-store/keys.cpp @@ -42,6 +42,7 @@ static const char* TXN_KEY_INFIX_LABEL = "txn_label"; static const char* TXN_KEY_INFIX_INFO = "txn_info"; static const char* TXN_KEY_INFIX_INDEX = "txn_index"; static const char* TXN_KEY_INFIX_RUNNING = "txn_running"; +static const char* TXN_KEY_INFIX_TSO_FENCE = "tso_fence"; static const char* PARTITION_VERSION_KEY_INFIX = "partition"; static const char* TABLE_VERSION_KEY_INFIX = "table"; @@ -145,7 +146,7 @@ static void encode_prefix(const T& t, std::string* key) { // Input type T must be one of the following, add if needed static_assert(check_types_v || std::is_same_v || std::is_same_v - || std::is_same_v) { + || std::is_same_v + || std::is_same_v) { encode_bytes(TXN_KEY_PREFIX, key); } else if constexpr (std::is_same_v || std::is_same_v @@ -259,6 +261,11 @@ void txn_running_key(const TxnRunningKeyInfo& in, std::string* out) { encode_int64(std::get<2>(in), out); // txn_id } +void txn_tso_fence_key(const TxnTsoFenceKeyInfo& in, std::string* out) { + encode_prefix(in, out); // 0x01 "txn" ${instance_id} + encode_bytes(TXN_KEY_INFIX_TSO_FENCE, out); // "tso_fence" +} + //============================================================================== // Version keys //============================================================================== diff --git a/cloud/src/meta-store/keys.h b/cloud/src/meta-store/keys.h index c1142f62a08500..2662eeed4df345 100644 --- a/cloud/src/meta-store/keys.h +++ b/cloud/src/meta-store/keys.h @@ -165,6 +165,9 @@ using TxnIndexKeyInfo = BasicKeyInfo<__LINE__ , std::tuple>; +// 0:instance_id +using TxnTsoFenceKeyInfo = BasicKeyInfo<__LINE__ , std::tuple>; + // 0:instance_id 1:db_id 2:tbl_id 3:partition_id using PartitionVersionKeyInfo = BasicKeyInfo<__LINE__ , std::tuple>; @@ -361,10 +364,12 @@ void txn_label_key(const TxnLabelKeyInfo& in, std::string* out); void txn_info_key(const TxnInfoKeyInfo& in, std::string* out); void txn_index_key(const TxnIndexKeyInfo& in, std::string* out); void txn_running_key(const TxnRunningKeyInfo& in, std::string* out); +void txn_tso_fence_key(const TxnTsoFenceKeyInfo& in, std::string* out); static inline std::string txn_label_key(const TxnLabelKeyInfo& in) { std::string s; txn_label_key(in, &s); return s; } static inline std::string txn_info_key(const TxnInfoKeyInfo& in) { std::string s; txn_info_key(in, &s); return s; } static inline std::string txn_index_key(const TxnIndexKeyInfo& in) { std::string s; txn_index_key(in, &s); return s; } static inline std::string txn_running_key(const TxnRunningKeyInfo& in) { std::string s; txn_running_key(in, &s); return s; } +static inline std::string txn_tso_fence_key(const TxnTsoFenceKeyInfo& in) { std::string s; txn_tso_fence_key(in, &s); return s; } std::string version_key_prefix(std::string_view instance_id); void partition_version_key(const PartitionVersionKeyInfo& in, std::string* out); diff --git a/cloud/test/keys_test.cpp b/cloud/test/keys_test.cpp index 9e9da03bf8d632..ad51182a68cdc1 100644 --- a/cloud/test/keys_test.cpp +++ b/cloud/test/keys_test.cpp @@ -545,6 +545,23 @@ TEST(KeysTest, TxnKeysTest) { ASSERT_GT(encoded_txn_running_key1, encoded_txn_running_key0); } + + // 0x01 "txn" ${instance_id} "tso_fence" -> TxnTsoFencePB + { + std::string encoded_key = txn_tso_fence_key({instance_id}); + std::string_view key_sv(encoded_key); + std::string decoded_prefix; + std::string decoded_instance_id; + std::string decoded_infix; + remove_user_space_prefix(&key_sv); + ASSERT_EQ(decode_bytes(&key_sv, &decoded_prefix), 0); + ASSERT_EQ(decode_bytes(&key_sv, &decoded_instance_id), 0); + ASSERT_EQ(decode_bytes(&key_sv, &decoded_infix), 0); + ASSERT_TRUE(key_sv.empty()); + EXPECT_EQ("txn", decoded_prefix); + EXPECT_EQ(instance_id, decoded_instance_id); + EXPECT_EQ("tso_fence", decoded_infix); + } } TEST(KeysTest, RecycleKeysTest) { diff --git a/cloud/test/meta_service_helper_test.cpp b/cloud/test/meta_service_helper_test.cpp index 73ccf12f42dc2c..e741df470dd9c8 100644 --- a/cloud/test/meta_service_helper_test.cpp +++ b/cloud/test/meta_service_helper_test.cpp @@ -413,6 +413,8 @@ TEST(MetaServiceHelperTest, ResponseStatusCoversEveryMetaServiceCode) { expect_legacy_fallback_response_status(covered_codes, MetaServiceCode::TXN_ALREADY_COMMITED, LegacyFallbackCode::UNDEFINED_ERR); + expect_legacy_fallback_response_status(covered_codes, MetaServiceCode::TXN_COMMIT_TSO_FENCED, + LegacyFallbackCode::UNDEFINED_ERR); EXPECT_EQ(covered_codes.size(), static_cast(MetaServiceCode_descriptor()->value_count())) << "A new MetaServiceCode was added. Map it to a LegacyFallbackCode in " diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index 91c28d8e7ac7eb..cce90421fb50b7 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -2782,6 +2782,61 @@ TEST(MetaServiceTest, GetCurrentMaxTxnIdTest) { ASSERT_GE(max_txn_id_res.current_max_txn_id(), begin_txn_res.txn_id()); } +TEST(MetaServiceTest, TsoFenceIsMonotonicAndRejectsStaleCommit) { + auto meta_service = get_meta_service(); + brpc::Controller cntl; + AdvanceTsoFenceRequest fence_request; + fence_request.set_cloud_unique_id("test_cloud_unique_id"); + fence_request.set_proposed_fence_tso(100); + AdvanceTsoFenceResponse fence_response; + meta_service->advance_tso_fence(&cntl, &fence_request, &fence_response, nullptr); + ASSERT_EQ(fence_response.status().code(), MetaServiceCode::OK); + ASSERT_EQ(fence_response.tso_fence(), 100); + + fence_request.set_proposed_fence_tso(90); + fence_response.Clear(); + meta_service->advance_tso_fence(&cntl, &fence_request, &fence_response, nullptr); + ASSERT_EQ(fence_response.status().code(), MetaServiceCode::OK); + ASSERT_EQ(fence_response.tso_fence(), 100); + + int64_t txn_id = -1; + begin_txn(meta_service.get(), 666, "tso_fence_commit", 1234, txn_id); + CommitTxnRequest commit_request; + commit_request.set_db_id(666); + commit_request.set_txn_id(txn_id); + commit_request.set_commit_tso(100); + CommitTxnResponse commit_response; + meta_service->commit_txn(&cntl, &commit_request, &commit_response, nullptr); + ASSERT_EQ(commit_response.status().actual_code(), MetaServiceCode::TXN_COMMIT_TSO_FENCED); + ASSERT_EQ(commit_response.tso_fence(), 100); + + commit_request.set_commit_tso(101); + commit_response.Clear(); + meta_service->commit_txn(&cntl, &commit_request, &commit_response, nullptr); + ASSERT_EQ(commit_response.status().code(), MetaServiceCode::OK); + + // A response retry after the transaction became visible remains idempotent. + commit_request.set_commit_tso(100); + commit_response.Clear(); + meta_service->commit_txn(&cntl, &commit_request, &commit_response, nullptr); + ASSERT_EQ(commit_response.status().code(), MetaServiceCode::OK); + + // Transactions without a commit TSO do not participate in binlog fencing. + int64_t non_tso_txn_id = -1; + begin_txn(meta_service.get(), 666, "tso_fence_non_tso_commit", 1234, non_tso_txn_id); + CommitTxnRequest non_tso_commit_request; + non_tso_commit_request.set_db_id(666); + non_tso_commit_request.set_txn_id(non_tso_txn_id); + CommitTxnResponse non_tso_commit_response; + meta_service->commit_txn(&cntl, &non_tso_commit_request, &non_tso_commit_response, nullptr); + ASSERT_EQ(non_tso_commit_response.status().code(), MetaServiceCode::OK); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->remove(txn_tso_fence_key({mock_instance})); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); +} + TEST(MetaServiceTest, CreateMetaSyncPointTest) { auto meta_service = get_meta_service(); const std::string cloud_unique_id = "test_cloud_unique_id"; diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp index 41374e67a75428..9cd952851a1a05 100644 --- a/cloud/test/txn_lazy_commit_test.cpp +++ b/cloud/test/txn_lazy_commit_test.cpp @@ -1270,6 +1270,7 @@ TEST(TxnLazyCommitTest, CommitTxnEventuallyWithFailedLazyCommitTaskTest) { commit_req.set_db_id(db_id); commit_req.set_txn_id(txn_id); commit_req.set_is_2pc(false); + commit_req.set_commit_tso(12345); commit_req.set_enable_txn_lazy_commit(true); CommitTxnResponse commit_res; meta_service->commit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 1b67e2ed8d75bc..aec18954d27a75 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3601,8 +3601,8 @@ public void handle(Field field, String value) throws Exception { public static int tso_max_get_retry_count = 10; @ConfField(mutable = true, masterOnly = true, description = "TSO service time window in milliseconds. Default is " - + "5000, which means the TSO service will apply for a " + "TSO time window of 5000ms from BDBJE once.") - public static int tso_service_window_duration_ms = 5000; + + "1000. Persist the readable committed TSO together with the reserved allocation window.") + public static int tso_service_window_duration_ms = 1000; @ConfField(mutable = true, masterOnly = true, description = "Max tolerated clock backward threshold during TSO " + "calibration in milliseconds. Exceeding this " + "threshold will fail enabling TSO. Default is 30 " diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java index 8f5fe32bb302b2..3ed4fe7c19c15f 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java @@ -1234,6 +1234,12 @@ public enum ErrorCode { ERR_NO_CLUSTER_ERROR(5099, new byte[]{'4', '2', '0', '0', '0'}, "No compute group (cloud cluster) selected"), + ERR_INCR_WINDOW_NOT_READY(5100, new byte[]{'H', 'Y', '0', '0', '0'}, + "The requested incremental read window is not yet visible; retry the same window."), + + ERR_INCR_VISIBLE_WAIT_TIMEOUT(5101, new byte[]{'H', 'Y', '0', '0', '0'}, + "Timed out waiting for incremental read transactions to become visible; retry the same window."), + ERR_NOT_CLOUD_MODE(6000, new byte[]{'4', '2', '0', '0', '0'}, "Command only support in cloud mode."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/arrowflight/DorisFlightSqlProducer.java index 1bf4bdfa2945d7..bd12f1c61eaef2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/arrowflight/DorisFlightSqlProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/arrowflight/DorisFlightSqlProducer.java @@ -24,10 +24,13 @@ import org.apache.doris.arrowflight.results.FlightSqlEndpointsLocation; import org.apache.doris.arrowflight.results.FlightSqlResultCacheEntry; import org.apache.doris.arrowflight.sessions.FlightSessionsManager; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.Util; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QueryState; import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.thrift.TUniqueId; @@ -40,6 +43,7 @@ import org.apache.arrow.flight.CloseSessionRequest; import org.apache.arrow.flight.CloseSessionResult; import org.apache.arrow.flight.Criteria; +import org.apache.arrow.flight.ErrorFlightMetadata; import org.apache.arrow.flight.FlightDescriptor; import org.apache.arrow.flight.FlightEndpoint; import org.apache.arrow.flight.FlightInfo; @@ -302,12 +306,24 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con + ", error code: " + connectContext.getState().getErrorCode() + ", error msg: " + connectContext.getState().getErrorMessage(); LOG.error(errMsg, e); - throw CallStatus.INTERNAL.withDescription(errMsg).withCause(e).toRuntimeException(); + throw queryFailure(connectContext.getState(), errMsg, e); } finally { connectContext.setCommand(MysqlCommand.COM_SLEEP); } } + static FlightRuntimeException queryFailure(QueryState state, String message, Throwable cause) { + if (IncrWindowNotReadyException.isWindowError(state.getErrorCode())) { + ErrorFlightMetadata metadata = new ErrorFlightMetadata(); + metadata.insert("doris-error-code", Integer.toString(state.getErrorCode().getCode())); + metadata.insert("doris-error-name", state.getErrorCode().name()); + // The description preserves the requested end, committed prefix and retry delay from QueryState. + return CallStatus.UNAVAILABLE.withDescription(message).withCause(cause) + .withMetadata(metadata).toRuntimeException(); + } + return CallStatus.INTERNAL.withDescription(message).withCause(cause).toRuntimeException(); + } + @Override public FlightInfo getFlightInfoStatement(final CommandStatementQuery request, final CallContext context, final FlightDescriptor descriptor) { @@ -321,6 +337,18 @@ public FlightInfo getFlightInfoStatement(final CommandStatementQuery request, fi // command lock; wrapping it as INTERNAL would hide that. throw e; } catch (Throwable e) { + if (e instanceof FlightRuntimeException) { + FlightRuntimeException flightError = (FlightRuntimeException) e; + ErrorFlightMetadata metadata = flightError.status().metadata(); + // Only the two incremental-window errors bypass the original INTERNAL wrapper. + if (metadata.containsKey("doris-error-code")) { + String code = metadata.get("doris-error-code"); + if (Integer.toString(ErrorCode.ERR_INCR_WINDOW_NOT_READY.getCode()).equals(code) + || Integer.toString(ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT.getCode()).equals(code)) { + throw flightError; + } + } + } String errMsg = "get flight info statement failed, " + e.getMessage(); LOG.error(errMsg, e); throw CallStatus.INTERNAL.withDescription(errMsg).withCause(e).toRuntimeException(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index d81a3177fed138..4cad0aa0c1701d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -913,6 +913,8 @@ public class SchemaTable extends Table { ScalarType.createType(PrimitiveType.BIGINT)) .column("CURRENT_TSO_LOGICAL_COUNTER", ScalarType.createType(PrimitiveType.BIGINT)) + .column("COMMITTED_TSO", ScalarType.createType(PrimitiveType.BIGINT)) + .column("COMMITTED_TSO_PHYSICAL_TIME", ScalarType.createType(PrimitiveType.BIGINT)) .build())) .put("be_compaction_tasks", new SchemaTable(SystemIdGenerator.getNextId(), "be_compaction_tasks", TableType.SCHEMA, diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java index ebefdb44aa9bcf..d3d243e9d40405 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java @@ -371,6 +371,11 @@ public Cloud.CheckTxnConflictResponse checkTxnConflict(Cloud.CheckTxnConflictReq .checkTxnConflict(request); } + public Cloud.AdvanceTsoFenceResponse advanceTsoFence(Cloud.AdvanceTsoFenceRequest request) { + return blockingStub.withDeadlineAfter(Config.meta_service_brpc_timeout_ms, TimeUnit.MILLISECONDS) + .advanceTsoFence(request); + } + public Cloud.CleanTxnLabelResponse cleanTxnLabel(Cloud.CleanTxnLabelRequest request) { return blockingStub.withDeadlineAfter(Config.meta_service_brpc_timeout_ms, TimeUnit.MILLISECONDS) .cleanTxnLabel(request); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java index e23bf6d5ddf721..7014c1ba9963c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java @@ -243,6 +243,12 @@ public MetaServiceClientWrapper(MetaServiceProxy proxy) { public Response executeRequest(String methodName, Function function, Function statusExtractor) throws RpcException { + return executeRequest(methodName, function, statusExtractor, true); + } + + public Response executeRequest(String methodName, Function function, + Function statusExtractor, boolean retryRpcFailure) + throws RpcException { long maxRetries = Config.meta_service_rpc_retry_cnt; for (long tried = 1; tried <= maxRetries; tried++) { MetaServiceClient client = null; @@ -280,7 +286,7 @@ public Response executeRequest(String methodName, Function= maxRetries) { + if (!retryRpcFailure || !shouldRetry || tried >= maxRetries) { throw new RpcException("", sre.getMessage(), sre); } } catch (RpcException e) { @@ -288,7 +294,7 @@ public Response executeRequest(String methodName, Function= maxRetries) { + if (!retryRpcFailure || tried >= maxRetries) { throw new RpcException("", e.getMessage(), e); } } finally { @@ -318,6 +324,12 @@ public Response executeRequest(String methodName, Function Response executeWithMetrics(String methodName, Function function, Function statusExtractor) throws RpcException { + return executeWithMetrics(methodName, function, statusExtractor, true); + } + + private Response executeWithMetrics(String methodName, Function function, + Function statusExtractor, boolean retryRpcFailure) + throws RpcException { long startTime = System.currentTimeMillis(); if (MetricRepo.isInit && Config.isCloudMode()) { CloudMetrics.META_SERVICE_RPC_ALL_TOTAL.increase(1L); @@ -325,7 +337,7 @@ private Response executeWithMetrics(String methodName, Function client.commitTxn(request), - Cloud.CommitTxnResponse::getStatus); + Cloud.CommitTxnResponse::getStatus, !request.hasCommitTso() || request.getCommitTso() <= 0); } public Cloud.AbortTxnResponse abortTxn(Cloud.AbortTxnRequest request) @@ -470,6 +482,12 @@ public Cloud.CheckTxnConflictResponse checkTxnConflict(Cloud.CheckTxnConflictReq Cloud.CheckTxnConflictResponse::getStatus); } + public Cloud.AdvanceTsoFenceResponse advanceTsoFence(Cloud.AdvanceTsoFenceRequest request) + throws RpcException { + return executeWithMetrics("advanceTsoFence", (client) -> client.advanceTsoFence(request), + Cloud.AdvanceTsoFenceResponse::getStatus); + } + public Cloud.CleanTxnLabelResponse cleanTxnLabel(Cloud.CleanTxnLabelRequest request) throws RpcException { return executeWithMetrics("cleanTxnLabel", (client) -> client.cleanTxnLabel(request), diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index 52b7a84bea2dbc..236b43e241c604 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -36,6 +36,8 @@ import org.apache.doris.cloud.proto.Cloud.AbortSubTxnResponse; import org.apache.doris.cloud.proto.Cloud.AbortTxnRequest; import org.apache.doris.cloud.proto.Cloud.AbortTxnResponse; +import org.apache.doris.cloud.proto.Cloud.AdvanceTsoFenceRequest; +import org.apache.doris.cloud.proto.Cloud.AdvanceTsoFenceResponse; import org.apache.doris.cloud.proto.Cloud.BeginSubTxnRequest; import org.apache.doris.cloud.proto.Cloud.BeginSubTxnResponse; import org.apache.doris.cloud.proto.Cloud.BeginTxnRequest; @@ -45,6 +47,7 @@ import org.apache.doris.cloud.proto.Cloud.CleanTxnLabelRequest; import org.apache.doris.cloud.proto.Cloud.CleanTxnLabelResponse; import org.apache.doris.cloud.proto.Cloud.CommitTxnRequest; +import org.apache.doris.cloud.proto.Cloud.CommitTxnRequestOrBuilder; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; import org.apache.doris.cloud.proto.Cloud.GetCurrentMaxTxnRequest; import org.apache.doris.cloud.proto.Cloud.GetCurrentMaxTxnResponse; @@ -446,9 +449,6 @@ private void commitTransactionWithoutLock(long dbId, List tableList, long LOG.info("try to commit transaction, transactionId: {}, tableIds: {}", transactionId, tableList.stream().map(Table::getId).collect(Collectors.toList())); Map> backendToPartitionInfos = null; - Database database = Env.getCurrentInternalCatalog().getDbOrMetaException(dbId); - long commitTSO = TransactionUtil.getCommitTSO(transactionId, database, - tableList.stream().map(Table::getId).collect(Collectors.toSet())); if (!mowTableList.isEmpty()) { if (!checkTransactionStateBeforeCommit(dbId, transactionId)) { return; @@ -462,7 +462,7 @@ private void commitTransactionWithoutLock(long dbId, List
tableList, long backendToPartitionInfos = getCalcDeleteBitmapInfo(lockContext, null); } commitTransactionWithoutLock(dbId, tableList, transactionId, tabletCommitInfos, txnCommitAttachment, false, - mowTableList, backendToPartitionInfos, commitTSO, streamUpdateInfos); + mowTableList, backendToPartitionInfos, streamUpdateInfos); // clear signature after commit succeeds clearTxnLastSignature(dbId, transactionId); } catch (Exception e) { @@ -700,17 +700,16 @@ private Set getBaseTabletsFromTables(List
tableList, List tableList, long transactionId, List tabletCommitInfos, TxnCommitAttachment txnCommitAttachment, boolean is2PC, - List mowTableList, Map> backendToPartitionInfos, - long commitTSO) + List mowTableList, Map> backendToPartitionInfos) throws UserException { commitTransactionWithoutLock(dbId, tableList, transactionId, tabletCommitInfos, txnCommitAttachment, - is2PC, mowTableList, backendToPartitionInfos, commitTSO, Collections.emptyList()); + is2PC, mowTableList, backendToPartitionInfos, Collections.emptyList()); } private void commitTransactionWithoutLock(long dbId, List
tableList, long transactionId, List tabletCommitInfos, TxnCommitAttachment txnCommitAttachment, boolean is2PC, List mowTableList, Map> backendToPartitionInfos, - long commitTSO, List streamUpdateInfos) + List streamUpdateInfos) throws UserException { if (Config.disable_load_job) { throw new TransactionCommitFailedException( @@ -730,7 +729,6 @@ private void commitTransactionWithoutLock(long dbId, List
tableList, long .setTxnId(transactionId) .setIs2Pc(is2PC) .setCloudUniqueId(Config.cloud_unique_id) - .setCommitTso(commitTSO) .addAllBaseTabletIds(getBaseTabletsFromTables(tableList, tabletCommitInfos)) .setEnableTxnLazyCommit(Config.enable_cloud_txn_lazy_commit); for (OlapTable olapTable : mowTableList) { @@ -784,13 +782,13 @@ private void commitTransactionWithoutLock(long dbId, List
tableList, long } } - final CommitTxnRequest commitTxnRequest = builder.build(); - executeCommitTxnRequest(commitTxnRequest, transactionId, is2PC, txnCommitAttachment, tabletCommitInfos, + executeCommitTxnRequest(builder, tableList, transactionId, is2PC, txnCommitAttachment, tabletCommitInfos, tabletCommitInfos == null ? Collections.emptyList() : tabletCommitInfos.stream().map(t -> t.getTabletId()).collect(Collectors.toList())); } - private void executeCommitTxnRequest(CommitTxnRequest commitTxnRequest, long transactionId, boolean is2PC, + private void executeCommitTxnRequest(CommitTxnRequest.Builder builder, List
tableList, + long transactionId, boolean is2PC, TxnCommitAttachment txnCommitAttachment, List tabletCommitInfos, List tabletIds) throws UserException { if (DebugPointUtil.isEnable("FE.mow.commit.exception")) { @@ -815,7 +813,7 @@ private void executeCommitTxnRequest(CommitTxnRequest commitTxnRequest, long tra StopWatch stopWatch = new StopWatch(); stopWatch.start(); try { - txnState = commitTxn(commitTxnRequest, transactionId, is2PC, tabletCommitInfos, tabletIds); + txnState = commitTxn(builder, tableList, transactionId, is2PC, tabletCommitInfos, tabletIds); txnOperated = true; if (DebugPointUtil.isEnable("CloudGlobalTransactionMgr.commitTransaction.timeout")) { throw new UserException(InternalErrorCode.DELETE_BITMAP_LOCK_ERR, @@ -854,9 +852,31 @@ private void executeCommitTxnRequest(CommitTxnRequest commitTxnRequest, long tra } } - private TransactionState commitTxn(CommitTxnRequest commitTxnRequest, long transactionId, boolean is2PC, - List tabletCommitInfos, List tabletIds) throws UserException { - checkCommitInfo(commitTxnRequest); + private TransactionState commitTxn(CommitTxnRequest.Builder builder, List
tableList, + long transactionId, boolean is2PC, List tabletCommitInfos, List tabletIds) + throws UserException { + checkCommitInfo(builder); + // Bitmap work, attachments and metadata validation do not need a commit TSO. Allocate only + // when ready to send, while retaining the existing table locks and callback cleanup scope. + Database database = Env.getCurrentInternalCatalog().getDbOrMetaException(builder.getDbId()); + Set commitTsoTableIds = tableList.stream().map(Table::getId).collect(Collectors.toSet()); + long commitTso = TransactionUtil.getCommitTSO(transactionId, database, commitTsoTableIds); + if (commitTso > 0) { + builder.setCommitTso(commitTso); + } + CommitTxnRequest commitTxnRequest = builder.build(); + try { + while (DebugPointUtil.isEnable("CloudGlobalTransactionMgr.commitTxn.blockAfterTso")) { + Thread.sleep(100); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + if (commitTxnRequest.hasCommitTso()) { + Env.getCurrentEnv().getTSOService().abandonCommitTso( + commitTxnRequest.getDbId(), transactionId, commitTxnRequest.getCommitTso()); + } + throw new UserException("Interrupted before sending commit transaction", e); + } CommitTxnResponse commitTxnResponse = null; TransactionState txnState = null; @@ -871,6 +891,20 @@ private TransactionState commitTxn(CommitTxnRequest commitTxnRequest, long trans if (LOG.isDebugEnabled()) { LOG.debug("retryTime:{}, commitTxnResponse:{}", retryTime, commitTxnResponse); } + if (commitTxnResponse.getStatus().getCode() == MetaServiceCode.TXN_COMMIT_TSO_FENCED) { + if (!commitTxnResponse.hasTsoFence() || commitTxnRequest.getCommitTso() <= 0) { + throw new UserException("MetaService returned an invalid TSO fence response"); + } + long replacementTso = Env.getCurrentEnv().getTSOService().getCommitTSOAfterFence( + builder.getDbId(), transactionId, commitTsoTableIds, + commitTxnRequest.getCommitTso(), commitTxnResponse.getTsoFence()); + LOG.info("commitTxn replaces fenced TSO, transactionId:{}, oldTso:{}, newTso:{}, fenceTso:{}", + transactionId, commitTxnRequest.getCommitTso(), replacementTso, + commitTxnResponse.getTsoFence()); + commitTxnRequest = commitTxnRequest.toBuilder().setCommitTso(replacementTso).build(); + retryTime++; + continue; + } if (commitTxnResponse.getStatus().getCode() != MetaServiceCode.KV_TXN_CONFLICT) { break; } @@ -884,7 +918,30 @@ private TransactionState commitTxn(CommitTxnRequest commitTxnRequest, long trans Preconditions.checkNotNull(commitTxnResponse.getStatus()); } catch (Exception e) { LOG.warn("commitTxn failed, transactionId:{}, exception:", transactionId, e); - throw new UserException("commitTxn() failed, errMsg:" + e.getMessage()); + if (commitTxnRequest.hasCommitTso()) { + try { + Env.getCurrentEnv().getTSOService().fenceAndAbandonCommitTso( + commitTxnRequest.getDbId(), transactionId, commitTxnRequest.getCommitTso()); + } catch (UserException fenceException) { + fenceException.addSuppressed(e); + throw fenceException; + } + } + throw new UserException("commitTxn() failed, errMsg:" + e.getMessage(), e); + } + + MetaServiceCode code = commitTxnResponse.getStatus().getCode(); + if (commitTxnRequest.hasCommitTso()) { + if (code == MetaServiceCode.KV_TXN_MAYBE_COMMITTED) { + Env.getCurrentEnv().getTSOService().fenceAndAbandonCommitTso( + commitTxnRequest.getDbId(), transactionId, commitTxnRequest.getCommitTso()); + } else if (code == MetaServiceCode.OK || code == MetaServiceCode.TXN_ALREADY_VISIBLE + || code == MetaServiceCode.TXN_ALREADY_ABORTED) { + Env.getCurrentEnv().getTSOService().transactionFinished(commitTxnRequest.getDbId(), transactionId); + } else { + Env.getCurrentEnv().getTSOService().abandonCommitTso( + commitTxnRequest.getDbId(), transactionId, commitTxnRequest.getCommitTso()); + } } if (is2PC && (commitTxnResponse.getStatus().getCode() == MetaServiceCode.TXN_ALREADY_VISIBLE @@ -919,7 +976,7 @@ private TransactionState commitTxn(CommitTxnRequest commitTxnRequest, long trans return txnState; } - private void checkCommitInfo(CommitTxnRequest commitTxnRequest) throws UserException { + private void checkCommitInfo(CommitTxnRequestOrBuilder commitTxnRequest) throws UserException { List commitTabletIds = Lists.newArrayList(); List commitIndexIds = Lists.newArrayList(); commitTabletIds.addAll(commitTxnRequest.getBaseTabletIdsList()); @@ -1627,8 +1684,6 @@ public boolean commitAndPublishTransaction(DatabaseIf db, long transactionId, List mowTableList = getMowTableList(tableList, tabletCommitInfos); try { Map> backendToPartitionInfos = null; - long commitTSO = TransactionUtil.getCommitTSO(transactionId, (Database) db, - tableList.stream().map(Table::getId).collect(Collectors.toSet())); if (!mowTableList.isEmpty()) { if (!checkTransactionStateBeforeCommit(db.getId(), transactionId)) { return true; @@ -1646,7 +1701,7 @@ public boolean commitAndPublishTransaction(DatabaseIf db, long transactionId, lockContext, partitionToSubTxnIds); } commitTransactionWithSubTxns(db.getId(), tableList, transactionId, subTransactionStates, mowTableList, - backendToPartitionInfos, commitTSO); + backendToPartitionInfos); // clear signature after commit succeeds clearTxnLastSignature(db.getId(), transactionId); } catch (Exception e) { @@ -1684,7 +1739,7 @@ public boolean commitAndPublishTransaction(DatabaseIf db, long transactionId, private void commitTransactionWithSubTxns(long dbId, List
tableList, long transactionId, List subTransactionStates, List mowTableList, - Map> backendToPartitionInfos, long commitTSO) + Map> backendToPartitionInfos) throws UserException { if (!mowTableList.isEmpty()) { List mowTableIds = mowTableList.stream().map(Table::getId).collect(Collectors.toList()); @@ -1700,7 +1755,6 @@ private void commitTransactionWithSubTxns(long dbId, List
tableList, long .setIs2Pc(false) .setCloudUniqueId(Config.cloud_unique_id) .setIsTxnLoad(true) - .setCommitTso(commitTSO) .setEnableTxnLazyCommit(Config.enable_cloud_txn_lazy_commit); for (OlapTable olapTable : mowTableList) { builder.addMowTableIds(olapTable.getId()); @@ -1721,8 +1775,7 @@ private void commitTransactionWithSubTxns(long dbId, List
tableList, long } } - final CommitTxnRequest commitTxnRequest = builder.build(); - executeCommitTxnRequest(commitTxnRequest, transactionId, false, null, null, new ArrayList<>(tabletIds)); + executeCommitTxnRequest(builder, tableList, transactionId, false, null, null, new ArrayList<>(tabletIds)); } private List
getTablesNeedCommitLock(List
tableList) { @@ -1907,10 +1960,8 @@ public void commitTransaction2PC(Database db, List
tableList, long transa return; } } - long commitTSO = TransactionUtil.getCommitTSO(transactionId, db, - tableList.stream().map(Table::getId).collect(Collectors.toSet())); commitTransactionWithoutLock(db.getId(), tableList, transactionId, null, null, true, - mowTableList, null, commitTSO); + mowTableList, null); } finally { afterCommitTransaction(tableList, transactionId); } @@ -2055,6 +2106,13 @@ private void handleAfterAbort(AbortTxnResponse abortTxnResponse, TxnCommitAttach private void afterAbortTxnResp(AbortTxnResponse abortTxnResponse, String txnIdOrLabel, TxnCommitAttachment txnCommitAttachment) throws UserException { + // MS fills txn_info only after committing the abort's KV transaction. + if (abortTxnResponse.hasTxnInfo() + && (abortTxnResponse.getTxnInfo().getStatus() == TxnStatusPB.TXN_STATUS_ABORTED + || abortTxnResponse.getTxnInfo().getStatus() == TxnStatusPB.TXN_STATUS_VISIBLE)) { + Env.getCurrentEnv().getTSOService().transactionFinished( + abortTxnResponse.getTxnInfo().getDbId(), abortTxnResponse.getTxnInfo().getTxnId()); + } if (abortTxnResponse.getStatus().getCode() != MetaServiceCode.OK) { LOG.warn("abortTxn failed, transaction:{}, response:{}", txnIdOrLabel, abortTxnResponse); switch (abortTxnResponse.getStatus().getCode()) { @@ -2187,6 +2245,25 @@ public List getUnFinishedPreviousLoad(long endTransactionId, l return conflictTxns; } + @Override + public long advanceTsoFence(long proposedFenceTso) throws UserException { + AdvanceTsoFenceRequest request = AdvanceTsoFenceRequest.newBuilder() + .setCloudUniqueId(Config.cloud_unique_id) + .setRequestIp(FrontendOptions.getLocalHostAddressCached()) + .setProposedFenceTso(proposedFenceTso) + .build(); + AdvanceTsoFenceResponse response; + try { + response = MetaServiceProxy.getInstance().advanceTsoFence(request); + } catch (RpcException e) { + throw new UserException("Failed to advance TSO fence", e); + } + if (response.getStatus().getCode() != MetaServiceCode.OK || !response.hasTsoFence()) { + throw new UserException("Failed to advance TSO fence: " + response.getStatus().getMsg()); + } + return response.getTsoFence(); + } + @Override public boolean isPreviousTransactionsFinished(long endTransactionId, long dbId, List tableIdList) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/IncrWindowNotReadyException.java b/fe/fe-core/src/main/java/org/apache/doris/common/IncrWindowNotReadyException.java new file mode 100644 index 00000000000000..070ce257b48bc4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/IncrWindowNotReadyException.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common; + +import org.apache.doris.tso.TSOTimestamp; + +/** A bounded incremental read must retry the same window after its upper bound becomes readable. */ +public class IncrWindowNotReadyException extends UserException { + private final long requestedEndTimestampMs; + private final long committedTso; + private final long retryAfterMs; + private final long currentTso; + private final long timeoutMs; + private final String reason; + + public IncrWindowNotReadyException(long requestedEndTimestampMs, long committedTso, long retryAfterMs) { + this(ErrorCode.ERR_INCR_WINDOW_NOT_READY, "WINDOW_NOT_READY", requestedEndTimestampMs, + 0, committedTso, retryAfterMs, 0); + } + + public IncrWindowNotReadyException(ErrorCode errorCode, String reason, long requestedEndTimestampMs, + long currentTso, long committedTso, long retryAfterMs, long timeoutMs) { + super(String.format("%s: reason=%s, requestedEndTimestampMs=%d, currentTSO=%d, " + + "currentTSOPhysicalTimeMs=%d, committedTSO=%d, committedTSOPhysicalTimeMs=%d, " + + "retryAfterMs=%d, timeoutMs=%d", + errorCode.name(), reason, requestedEndTimestampMs, currentTso, + TSOTimestamp.extractPhysicalTime(currentTso), committedTso, + TSOTimestamp.extractPhysicalTime(committedTso), retryAfterMs, timeoutMs)); + setMysqlErrorCode(errorCode); + this.requestedEndTimestampMs = requestedEndTimestampMs; + this.committedTso = committedTso; + this.retryAfterMs = retryAfterMs; + this.currentTso = currentTso; + this.timeoutMs = timeoutMs; + this.reason = reason; + } + + public static boolean isWindowError(ErrorCode errorCode) { + return errorCode == ErrorCode.ERR_INCR_WINDOW_NOT_READY + || errorCode == ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT; + } + + public long getCurrentTso() { + return currentTso; + } + + public long getTimeoutMs() { + return timeoutMs; + } + + public String getReason() { + return reason; + } + + public long getRequestedEndTimestampMs() { + return requestedEndTimestampMs; + } + + public long getCommittedTso() { + return committedTso; + } + + public long getRetryAfterMs() { + return retryAfterMs; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 76f57fe443a01a..2151f11b1dfa15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -147,7 +147,7 @@ import org.apache.doris.system.Backend; import org.apache.doris.system.Frontend; import org.apache.doris.transaction.TransactionState; -import org.apache.doris.tso.TSOTimestamp; +import org.apache.doris.tso.TSOServiceState; import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; @@ -1035,7 +1035,7 @@ public void readFields(DataInput in) throws IOException { break; } case OperationType.OP_TSO_TIMESTAMP_WINDOW_END: { - data = TSOTimestamp.read(in); + data = TSOServiceState.read(in); isRead = true; break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java index 3c1aeb6a0373d3..7c7d5a54e9e2e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java @@ -314,6 +314,9 @@ public final class MetricRepo { public static LongCounterMetric COUNTER_TSO_CLOCK_UPDATED; public static LongCounterMetric COUNTER_TSO_CLOCK_UPDATE_FAILED; public static LongCounterMetric COUNTER_TSO_CLOCK_GET_SUCCESS; + public static LongCounterMetric COUNTER_TSO_STATE_PERSISTED; + public static LongCounterMetric COUNTER_TSO_STATE_PERSIST_FAILED; + public static Histogram HISTO_TSO_STATE_PERSIST_LATENCY; private static Map, Long> loadJobNum = Maps.newHashMap(); @@ -1173,6 +1176,14 @@ public Integer getValue() { COUNTER_TSO_CLOCK_GET_SUCCESS = new LongCounterMetric("tso_clock_get_success", MetricUnit.NOUNIT, "counter of tso clock get success"); DORIS_METRIC_REGISTER.addMetrics(COUNTER_TSO_CLOCK_GET_SUCCESS); + COUNTER_TSO_STATE_PERSISTED = new LongCounterMetric("tso_state_persisted", MetricUnit.NOUNIT, + "successful combined committed TSO and window journal writes"); + DORIS_METRIC_REGISTER.addMetrics(COUNTER_TSO_STATE_PERSISTED); + COUNTER_TSO_STATE_PERSIST_FAILED = new LongCounterMetric("tso_state_persist_failed", MetricUnit.NOUNIT, + "failed TSO state journal writes"); + DORIS_METRIC_REGISTER.addMetrics(COUNTER_TSO_STATE_PERSIST_FAILED); + HISTO_TSO_STATE_PERSIST_LATENCY = METRIC_REGISTER.histogram("tso_state_persist_latency_ms"); + Env.getCurrentEnv().getTSOService().registerMetrics(); // init system metrics initSystemMetrics(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index a29ef2a6326876..b3c6466e878df9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -109,7 +109,7 @@ import org.apache.doris.system.Frontend; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; -import org.apache.doris.tso.TSOTimestamp; +import org.apache.doris.tso.TSOServiceState; import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; @@ -1493,7 +1493,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { break; } case OperationType.OP_TSO_TIMESTAMP_WINDOW_END: { - env.getTSOService().replayWindowEndTSO((TSOTimestamp) journal.getData()); + env.getTSOService().replayWindowEndTSO((TSOServiceState) journal.getData()); break; } default: { @@ -1947,7 +1947,7 @@ public void logTimestamp(Timestamp stamp) { logEdit(OperationType.OP_TIMESTAMP, stamp); } - public void logTSOTimestampWindowEnd(TSOTimestamp windowEnd) { + public void logTSOTimestampWindowEnd(TSOServiceState windowEnd) { logEdit(OperationType.OP_TSO_TIMESTAMP_WINDOW_END, windowEnd); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index 98d9056e1af08c..ead7960a5a106e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -690,7 +690,7 @@ public static void setVisibleVersionForOlapScanNodes(List scanNodes) t // A time-based change read may have just waited for an old transaction to finish. // Bypass the FE cache so the scan uses the version made visible by that transaction. versions = hasIncrementalRead - ? CloudPartition.getSnapshotVisibleVersionFromMs(partitions, false) + ? CloudPartition.getSnapshotVisibleVersionFromMs(partitions, true) : CloudPartition.getSnapshotVisibleVersion(partitions); } catch (RpcException e) { throw new UserException("get visible version for OlapScanNode failed", e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 9dd0c5f637e2b6..869577ae5aecde 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -45,6 +45,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.NereidsException; import org.apache.doris.common.QueryTimeoutException; import org.apache.doris.common.Status; @@ -713,7 +714,16 @@ public void execute(TUniqueId queryId) throws Exception { throw new UserException(e.getMessage()); } LOG.warn("Analyze failed. {}", context.getQueryIdentifier(), e); - context.getState().setError(e.getMessage()); + // Planning wraps the window rejection in NereidsException/AnalysisException. + // NereidsException(Exception) keeps its wrapped exception outside Throwable.cause. + Throwable cause = e instanceof NereidsException + ? Util.getRootCause(((NereidsException) e).getException()) : e; + if (cause instanceof IncrWindowNotReadyException) { + context.getState().setError(((IncrWindowNotReadyException) cause).getMysqlErrorCode(), + e.getMessage()); + } else { + context.getState().setError(e.getMessage()); + } return; } catch (Exception e) { LOG.warn("Nereids execute failed. {}", context.getQueryIdentifier(), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiter.java b/fe/fe-core/src/main/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiter.java index 8faeb7f51ee490..278a155d598ec3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiter.java @@ -26,6 +26,8 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ClientPool; import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.UserException; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.trees.plans.Plan; @@ -34,6 +36,7 @@ import org.apache.doris.thrift.FrontendService; import org.apache.doris.thrift.TAcquireTimeBasedChangeReadFenceRequest; import org.apache.doris.thrift.TAcquireTimeBasedChangeReadFenceResult; +import org.apache.doris.thrift.TIncrWindowNotReady; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.transaction.GlobalTransactionMgrIface; @@ -55,7 +58,9 @@ /** * Establishes a closed upper fence before planning a time-based incremental read. * - *

The master FE first captures its TSO, validates an explicit end timestamp, then captures a + *

Bounded, strongly consistent cloud reads use the master's durable committed TSO or wait for + * registered transactions involving the target tables before the requested end. Other reads + * retain the transaction drain: the master validates its TSO, then captures a * transaction ID watermark and drains earlier transactions involving the target tables. In classic * mode, it also synchronizes with transaction publishers through the target table locks and returns * a journal watermark for a follower FE to replay. In cloud mode, partition versions are refreshed @@ -68,12 +73,22 @@ public class TimeBasedChangeVisibleWaiter { public static final class ChangeReadFence { private final long currentTso; private final long maxJournalId; + private final long committedTso; public ChangeReadFence(long currentTso, long maxJournalId) { + this(currentTso, maxJournalId, 0); + } + + public ChangeReadFence(long currentTso, long maxJournalId, long committedTso) { + this.committedTso = committedTso; this.currentTso = currentTso; this.maxJournalId = maxJournalId; } + public long getCommittedTso() { + return committedTso; + } + public long getCurrentTso() { return currentTso; } @@ -87,12 +102,18 @@ public long getMaxJournalId() { static final class ChangeReadInfo { private final Map> dbToTableIds; private final Long maxEndTimestampMs; + private final boolean allEndsExplicit; - private ChangeReadInfo(Map> dbToTableIds, Long maxEndTimestampMs) { + private ChangeReadInfo(Map> dbToTableIds, Long maxEndTimestampMs, boolean allEndsExplicit) { + this.allEndsExplicit = allEndsExplicit; this.dbToTableIds = dbToTableIds; this.maxEndTimestampMs = maxEndTimestampMs; } + boolean isAllEndsExplicit() { + return allEndsExplicit; + } + Map> getDbToTableIds() { return dbToTableIds; } @@ -124,7 +145,8 @@ public static void waitForVisible(ConnectContext context, Plan plan, Map> dbToTableIds, Long maxEndTimestampMs, long timeoutMs, boolean waitForTransactions) throws UserException { + return acquireFenceOnMaster(dbToTableIds, maxEndTimestampMs, timeoutMs, waitForTransactions, false); + } + + public static ChangeReadFence acquireFenceOnMaster(Map> dbToTableIds, + Long maxEndTimestampMs, long timeoutMs, boolean waitForTransactions, boolean allEndsExplicit) + throws UserException { Env env = Env.getCurrentEnv(); if (!env.isMaster()) { throw new UserException("time-based change read fence must be acquired on the master FE"); } + if (Config.isCloudMode() && waitForTransactions && allEndsExplicit) { + Preconditions.checkArgument(maxEndTimestampMs != null, "bounded read requires an end timestamp"); + TSOService.TSOStatusSnapshot ready = env.getTSOService().waitForReadableWindow( + dbToTableIds, maxEndTimestampMs, timeoutMs); + // Partition versions are still refreshed from MS during planning. + return new ChangeReadFence(ready.getCurrentTso(), env.getMaxJournalId(), ready.getCommittedTso()); + } TSOService.TSOStatusSnapshot tsoSnapshot = env.getTSOService().getStatusSnapshot(); if (!tsoSnapshot.isInitialized()) { throw new UserException("TSO timestamp is not calibrated, please check"); @@ -182,6 +217,7 @@ static ChangeReadInfo collectChangeReadInfo(ConnectContext context, Plan plan, Map, TableIf> tables) { Map> dbToTableIdSets = new TreeMap<>(); long[] maxEndTimestampMs = {-1L}; + boolean[] allEndsExplicit = {true}; plan.foreach(node -> { if (!(node instanceof UnboundRelation)) { return; @@ -203,13 +239,18 @@ static ChangeReadInfo collectChangeReadInfo(ConnectContext context, Plan plan, scanParams.getMapParams().get(OlapScanNode.OLAP_END_TIMESTAMP)); if (endTimestampMs > 0) { maxEndTimestampMs[0] = Math.max(maxEndTimestampMs[0], endTimestampMs); + } else { + allEndsExplicit[0] = false; } + } else { + allEndsExplicit[0] = false; } }); Map> dbToTableIds = new TreeMap<>(); dbToTableIdSets.forEach((dbId, tableIds) -> dbToTableIds.put(dbId, new ArrayList<>(tableIds))); - return new ChangeReadInfo(dbToTableIds, maxEndTimestampMs[0] < 0 ? null : maxEndTimestampMs[0]); + return new ChangeReadInfo(dbToTableIds, maxEndTimestampMs[0] < 0 ? null : maxEndTimestampMs[0], + allEndsExplicit[0]); } private static void validateEndTimestamp(Long maxEndTimestampMs, long currentTso) throws UserException { @@ -297,6 +338,7 @@ private static ChangeReadFence acquireFenceFromMaster(ConnectContext context, Ch request.setDbToTableIds(changeReadInfo.getDbToTableIds()); request.setTimeoutMs(timeoutMs); request.setWaitForTransactions(waitForTransactions); + request.setAllEndsExplicit(changeReadInfo.isAllEndsExplicit()); if (changeReadInfo.getMaxEndTimestampMs() != null) { request.setEndTimestampMs(changeReadInfo.getMaxEndTimestampMs()); } @@ -304,6 +346,10 @@ private static ChangeReadFence acquireFenceFromMaster(ConnectContext context, Ch TNetworkAddress masterAddress = new TNetworkAddress( context.getEnv().getMasterHost(), context.getEnv().getMasterRpcPort()); int thriftTimeoutMs = (int) Math.min(Integer.MAX_VALUE, Math.max(1L, timeoutMs)); + if (Config.isCloudMode() && waitForTransactions && changeReadInfo.isAllEndsExplicit()) { + // Let the master's typed wait-timeout response arrive before the socket times out. + thriftTimeoutMs = (int) Math.min(Integer.MAX_VALUE, (long) thriftTimeoutMs + 1000); + } FrontendService.Client client; try { client = ClientPool.frontendPool.borrowObject(masterAddress, thriftTimeoutMs); @@ -315,6 +361,19 @@ private static ChangeReadFence acquireFenceFromMaster(ConnectContext context, Ch try { TAcquireTimeBasedChangeReadFenceResult result = client.acquireTimeBasedChangeReadFence(request); returnToPool = true; + if (result.isSetWindowNotReady()) { + TIncrWindowNotReady error = result.getWindowNotReady(); + int code = error.isSetErrorCode() + ? error.getErrorCode() : ErrorCode.ERR_INCR_WINDOW_NOT_READY.getCode(); + Preconditions.checkState(code == ErrorCode.ERR_INCR_WINDOW_NOT_READY.getCode() + || code == ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT.getCode(), + "unsupported incremental window error code: %s", code); + throw new IncrWindowNotReadyException(code == ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT.getCode() + ? ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT : ErrorCode.ERR_INCR_WINDOW_NOT_READY, + error.isSetReason() ? error.getReason() : "WINDOW_NOT_READY", + error.getRequestedEndTimestampMs(), + error.getCurrentTso(), error.getCommittedTso(), error.getRetryAfterMs(), error.getTimeoutMs()); + } if (result.getStatus().getStatusCode() != TStatusCode.OK) { String error = result.getStatus().isSetErrorMsgs() ? String.join(". ", result.getStatus().getErrorMsgs()) @@ -323,7 +382,7 @@ private static ChangeReadFence acquireFenceFromMaster(ConnectContext context, Ch } Preconditions.checkState(result.isSetCurrentTso(), "master FE did not return current_tso"); Preconditions.checkState(result.isSetMaxJournalId(), "master FE did not return max_journal_id"); - return new ChangeReadFence(result.getCurrentTso(), result.getMaxJournalId()); + return new ChangeReadFence(result.getCurrentTso(), result.getMaxJournalId(), result.getCommittedTso()); } catch (UserException e) { throw e; } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 447128e66b584d..2f98052fd9a321 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -69,6 +69,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.DuplicatedRequestException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.InternalErrorCode; import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.LoadException; @@ -232,6 +233,7 @@ import org.apache.doris.thrift.TGetTabletReplicaInfosRequest; import org.apache.doris.thrift.TGetTabletReplicaInfosResult; import org.apache.doris.thrift.TGroupCommitInfo; +import org.apache.doris.thrift.TIncrWindowNotReady; import org.apache.doris.thrift.TInitExternalCtlMetaRequest; import org.apache.doris.thrift.TInitExternalCtlMetaResult; import org.apache.doris.thrift.TInsertOverwriteRecordRequest; @@ -3402,9 +3404,17 @@ public TAcquireTimeBasedChangeReadFenceResult acquireTimeBasedChangeReadFence( TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( request.getDbToTableIds(), request.isSetEndTimestampMs() ? request.getEndTimestampMs() : null, - request.getTimeoutMs(), request.isWaitForTransactions()); + request.getTimeoutMs(), request.isWaitForTransactions(), request.isAllEndsExplicit()); result.setCurrentTso(fence.getCurrentTso()); result.setMaxJournalId(fence.getMaxJournalId()); + result.setCommittedTso(fence.getCommittedTso()); + } catch (IncrWindowNotReadyException e) { + status.setStatusCode(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs(e.getDetailMessage()); + result.setWindowNotReady(new TIncrWindowNotReady( + e.getRequestedEndTimestampMs(), e.getCommittedTso(), e.getRetryAfterMs()) + .setCurrentTso(e.getCurrentTso()).setErrorCode(e.getMysqlErrorCode().getCode()) + .setTimeoutMs(e.getTimeoutMs()).setReason(e.getReason())); } catch (UserException e) { status.setStatusCode(TStatusCode.ANALYSIS_ERROR); status.addToErrorMsgs(e.getDetailMessage()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 8bdc9386654e11..3db535c8bceeea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -367,7 +367,7 @@ public static TFetchSchemaTableDataResult getSchemaTableData(TFetchSchemaTableDa columnIndex = TABLE_STREAM_CONSUMPTION_COLUMN_TO_INDEX; break; case TSO_STATUS: - result = tsoStatusMetadataResult(); + result = tsoStatusMetadataResult(schemaTableParams.isSetColumnsName()); columnIndex = TSO_STATUS_COLUMN_TO_INDEX; break; case STATISTICS: @@ -2339,7 +2339,7 @@ private static TFetchSchemaTableDataResult streamConsumptionMetadataResult(TSche return result; } - private static TFetchSchemaTableDataResult tsoStatusMetadataResult() { + private static TFetchSchemaTableDataResult tsoStatusMetadataResult(boolean includeCommittedTso) { if (!Config.enable_feature_binlog) { return errorResult("TSO feature is disabled, please check enable_feature_binlog"); } @@ -2355,6 +2355,14 @@ private static TFetchSchemaTableDataResult tsoStatusMetadataResult() { row.addToColumnValue(new TCell().setLongVal(currentTso)); row.addToColumnValue(new TCell().setLongVal(TSOTimestamp.extractPhysicalTime(currentTso))); row.addToColumnValue(new TCell().setLongVal(TSOTimestamp.extractLogicalCounter(currentTso))); + // Older BE scanners omit columns_name and require exactly the original four columns. + if (includeCommittedTso) { + long committedTso = statusSnapshot.getCommittedTso(); + row.addToColumnValue(committedTso == 0 ? new TCell().setIsNull(true) + : new TCell().setLongVal(committedTso)); + row.addToColumnValue(committedTso == 0 ? new TCell().setIsNull(true) + : new TCell().setLongVal(TSOTimestamp.extractPhysicalTime(committedTso))); + } TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); result.setDataBatch(Lists.newArrayList(row)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java index 0838d12e0f0b61..e241ff37982fdf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java @@ -145,6 +145,10 @@ public void abortTransaction(Long dbId, Long txnId, String reason, public void finishTransaction(long dbId, long transactionId, Map partitionVisibleVersions, Map> backendPartitions) throws UserException; + default long advanceTsoFence(long proposedFenceTso) throws UserException { + throw new UserException("TSO fence is only supported in cloud mode"); + } + public boolean isPreviousTransactionsFinished(long endTransactionId, long dbId, List tableIdList) throws AnalysisException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java index aea3478bf51f7b..7219a1b27b1aab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java @@ -103,7 +103,9 @@ public static long getCommitTSO(long transactionId, Database db, Set table throw new TransactionCommitFailedException("failed to get TSO for txn " + transactionId + ": TSO service is unavailable"); } - long fetched = env.getTSOService().getTSO(); + long fetched = Config.isCloudMode() + ? env.getTSOService().getCommitTSO(db.getId(), transactionId, tableIds) + : env.getTSOService().getTSO(); if (fetched <= 0) { throw new TransactionCommitFailedException("failed to get TSO for txn " + transactionId + ", fetched=" + fetched); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java index bb4d3e0e94956d..5f8afc6e6f6601 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java @@ -19,21 +19,33 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; import org.apache.doris.common.io.CountingDataOutputStream; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.journal.local.LocalJournal; +import org.apache.doris.metric.GaugeMetric; +import org.apache.doris.metric.Metric.MetricUnit; import org.apache.doris.metric.MetricRepo; import org.apache.doris.persist.EditLog; +import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.DataInputStream; import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.LongSupplier; public class TSOService extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(TSOService.class); @@ -47,7 +59,11 @@ public class TSOService extends MasterDaemon { private final AtomicBoolean isInitialized = new AtomicBoolean(false); private final AtomicBoolean fatalClockBackwardReported = new AtomicBoolean(false); - private final AtomicLong windowEndTSO = new AtomicLong(0); + private volatile TSOServiceState durableState = new TSOServiceState(0, 0); + private final TSOTransactionTracker transactionTracker = new TSOTransactionTracker(lock); + private long pendingCalibrationPhysicalTime; + private long pendingCalibrationWindowEnd; + private long lastPersistNanos; /** * Immutable snapshot of the current TSO service status. @@ -56,11 +72,21 @@ public static final class TSOStatusSnapshot { private final boolean initialized; private final long currentTso; private final long windowEndPhysicalTime; + private final long committedTso; public TSOStatusSnapshot(boolean initialized, long currentTso, long windowEndPhysicalTime) { + this(initialized, currentTso, windowEndPhysicalTime, 0); + } + + public TSOStatusSnapshot(boolean initialized, long currentTso, long windowEndPhysicalTime, long committedTso) { this.initialized = initialized; this.currentTso = currentTso; this.windowEndPhysicalTime = windowEndPhysicalTime; + this.committedTso = committedTso; + } + + public long getCommittedTso() { + return committedTso; } public boolean isInitialized() { @@ -89,12 +115,21 @@ public TSOService() { super("TSO-service", Config.tso_service_update_interval_ms); } - /** - * Start the TSO service. - */ - @Override - public synchronized void start() { - super.start(); + public void registerMetrics() { + Map gauges = new LinkedHashMap<>(); + gauges.put("tso_committed", () -> durableState.getCommittedTso()); + gauges.put("tso_window_end_physical_time", () -> durableState.getPhysicalTimestamp()); + gauges.put("tso_pending_transactions", transactionTracker::getPendingCount); + gauges.put("tso_oldest_pending_tso", transactionTracker::getOldestPendingTso); + gauges.put("tso_oldest_pending_txn_id", transactionTracker::getOldestPendingTxnId); + gauges.put("tso_oldest_pending_age_ms", transactionTracker::getOldestPendingAgeMs); + gauges.forEach((name, value) -> MetricRepo.DORIS_METRIC_REGISTER.addMetrics( + new GaugeMetric(name, MetricUnit.NOUNIT, name) { + @Override + public Long getValue() { + return value.getAsLong(); + } + })); } /** @@ -103,13 +138,9 @@ public synchronized void start() { */ @Override protected void runAfterCatalogReady() { - if (!isTsoEnabled()) { - lock.lock(); - try { - isInitialized.set(false); - } finally { - lock.unlock(); - } + Env env = Env.getCurrentEnv(); + if (!isTsoEnabled() || env == null || !env.isReady() || !env.isMaster()) { + deactivate(); return; } int maxUpdateRetryCount = Math.max(1, Config.tso_max_update_retry_count); @@ -187,6 +218,43 @@ protected void runAfterCatalogReady() { * @throws RuntimeException if TSO is not calibrated or other errors occur */ public long getTSO() { + return getTSO(null, Collections.emptySet()); + } + + public long getCommitTSO(long dbId, long txnId, Set tableIds) { + return getTSO(Pair.of(dbId, txnId), tableIds); + } + + public long getCommitTSOAfterFence(long dbId, long txnId, Set tableIds, + long rejectedTso, long fenceTso) { + return getTSO(Pair.of(dbId, txnId), tableIds, rejectedTso, fenceTso); + } + + public void transactionFinished(long dbId, long txnId) { + transactionTracker.transactionFinished(dbId, txnId); + } + + public void abandonCommitTso(long dbId, long txnId, long tso) { + transactionTracker.abandonCommitTso(dbId, txnId, tso); + } + + public void fenceAndAbandonCommitTso(long dbId, long txnId, long tso) throws UserException { + try { + long effectiveFenceTso = Env.getCurrentEnv().getGlobalTransactionMgr().advanceTsoFence(tso); + Preconditions.checkState(effectiveFenceTso >= tso, "MetaService TSO fence must not regress"); + transactionTracker.abandonCommitTso(dbId, txnId, tso); + } catch (Exception e) { + deactivate(); + throw new UserException("Failed to fence an uncertain commit TSO", e); + } + } + + private long getTSO(Pair transactionIdentity, Set tableIds) { + return getTSO(transactionIdentity, tableIds, -1, -1); + } + + private long getTSO(Pair transactionIdentity, Set tableIds, + long rejectedTso, long fenceTso) { if (!isTsoEnabled()) { throw new RuntimeException("TSO feature is disabled, please check enable_feature_binlog"); } @@ -210,6 +278,10 @@ public long getTSO() { continue; } else if (!env.isMaster()) { LOG.warn("TSO service only run on master FE"); + if (fenceTso > 0) { + throw new RuntimeException( + "TXN_COMMIT_TSO_FENCED: retry the commit through the current master FE"); + } lastFailure = new RuntimeException("Current FE is not master"); try { sleep(200); @@ -220,7 +292,7 @@ public long getTSO() { continue; } - Pair pair = generateTSO(); + Pair pair = generateTSO(transactionIdentity, tableIds, rejectedTso, fenceTso); long physical = pair.first; long logical = pair.second; @@ -270,13 +342,64 @@ public long getCurrentTSO() { public TSOStatusSnapshot getStatusSnapshot() { lock.lock(); try { - return new TSOStatusSnapshot( - isInitialized.get(), globalTimestamp.composeTimestamp(), windowEndTSO.get()); + TSOServiceState state = durableState; + return new TSOStatusSnapshot(isInitialized.get(), globalTimestamp.composeTimestamp(), + state.getPhysicalTimestamp(), state.getCommittedTso()); + } finally { + lock.unlock(); + } + } + + /** Establish a bounded cloud read on the master without draining unrelated or later transactions. */ + public TSOStatusSnapshot waitForReadableWindow(Map> dbToTableIds, + long endTimestampMs, long timeoutMs) throws UserException { + long startNanos = System.nanoTime(); + lock.lock(); + try { + TSOStatusSnapshot snapshot = getStatusSnapshot(); + if (!snapshot.isInitialized()) { + throw new UserException("TSO timestamp is not calibrated, please check"); + } + if (endTimestampMs > TSOTimestamp.extractPhysicalTime(snapshot.getCurrentTso())) { + throw windowError(ErrorCode.ERR_INCR_WINDOW_NOT_READY, "END_AFTER_CURRENT_TSO", + endTimestampMs, snapshot, timeoutMs); + } + if (snapshot.getCommittedTso() > 0 + && endTimestampMs <= TSOTimestamp.extractPhysicalTime(snapshot.getCommittedTso())) { + return snapshot; + } + // Logical zero matches the BE upper boundary for the physical interval [start, end). + TSOTransactionTracker.WaitResult result = transactionTracker.awaitTransactions(dbToTableIds, + TSOTimestamp.composePhysicalTimestamp(endTimestampMs), + TimeUnit.MILLISECONDS.toNanos(timeoutMs) - (System.nanoTime() - startNanos)); + snapshot = getStatusSnapshot(); + if (result == TSOTransactionTracker.WaitResult.RESET) { + throw windowError(ErrorCode.ERR_INCR_WINDOW_NOT_READY, "TSO_MASTER_CHANGED", + endTimestampMs, snapshot, timeoutMs); + } + if (!Env.getCurrentEnv().isMaster()) { + throw windowError(ErrorCode.ERR_INCR_WINDOW_NOT_READY, "TSO_MASTER_CHANGED", + endTimestampMs, snapshot, timeoutMs); + } + if (result == TSOTransactionTracker.WaitResult.TIMED_OUT) { + throw windowError(ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT, "VISIBLE_WAIT_TIMEOUT", + endTimestampMs, snapshot, timeoutMs); + } + return snapshot; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UserException("interrupted while waiting for incremental read transactions", e); } finally { lock.unlock(); } } + private IncrWindowNotReadyException windowError(ErrorCode code, String reason, long endTimestampMs, + TSOStatusSnapshot snapshot, long timeoutMs) { + return new IncrWindowNotReadyException(code, reason, endTimestampMs, snapshot.getCurrentTso(), + snapshot.getCommittedTso(), Config.tso_service_window_duration_ms, timeoutMs); + } + /** * Calibrate the TSO timestamp when service starts * This ensures the timestamp is consistent with the last persisted value @@ -285,7 +408,7 @@ public TSOStatusSnapshot getStatusSnapshot() { * - If Tnow - Tlast < 1ms, then Tnext = Tlast + 1 * - Otherwise Tnext = Tnow */ - private void calibrateTimestamp() { + private void calibrateTimestamp() throws UserException { if (isInitialized.get()) { return; } @@ -296,40 +419,87 @@ private void calibrateTimestamp() { return; } - long timeLast = windowEndTSO.get(); // Last timestamp from image/editlog replay + long timeLast = durableState.getPhysicalTimestamp(); // Last timestamp from image/editlog replay long timeNow = System.currentTimeMillis() + Config.tso_time_offset_debug_mode; - long backwardMs = timeLast - timeNow; - if (backwardMs > Config.tso_clock_backward_startup_threshold_ms) { - throw new TSOClockBackwardException("TSO clock backward too much during calibration, backwardMs=" - + backwardMs + ", thresholdMs=" + Config.tso_clock_backward_startup_threshold_ms - + ", lastWindowEndTSO=" + timeLast + ", currentMillis=" + timeNow); - } - - // Calculate next physical time to ensure monotonicity long nextPhysicalTime; - if (timeNow - timeLast < 1) { - nextPhysicalTime = timeLast + 1; + long timeWindowEnd; + lock.lock(); + try { + nextPhysicalTime = pendingCalibrationPhysicalTime; + timeWindowEnd = pendingCalibrationWindowEnd; + } finally { + lock.unlock(); + } + if (nextPhysicalTime == 0) { + long backwardMs = timeLast - timeNow; + if (backwardMs > Config.tso_clock_backward_startup_threshold_ms) { + throw new TSOClockBackwardException("TSO clock backward too much during calibration, backwardMs=" + + backwardMs + ", thresholdMs=" + Config.tso_clock_backward_startup_threshold_ms + + ", lastWindowEndTSO=" + timeLast + ", currentMillis=" + timeNow); + } + // Calculate next physical time to ensure monotonicity. + nextPhysicalTime = timeNow - timeLast < 1 ? timeLast + 1 : timeNow; + timeWindowEnd = persistCalibrationWindow(nextPhysicalTime); } else { - nextPhysicalTime = timeNow; + Preconditions.checkState(timeWindowEnd > nextPhysicalTime, + "pending calibration window must cover its TSO"); } - // Construct new timestamp (physical time with reset logical counter) - setTSOPhysical(nextPhysicalTime, true); - - // Write the right boundary of time window to BDBJE for persistence - long timeWindowEnd = nextPhysicalTime + Config.tso_service_window_duration_ms; - writeTimestampToBDBJE(timeWindowEnd); - windowEndTSO.set(timeWindowEnd); + long proposedFenceTso; + while (true) { + proposedFenceTso = TSOTimestamp.composePhysicalTimestamp(nextPhysicalTime); + if (!Config.isCloudMode()) { + break; + } + long effectiveFenceTso = env.getGlobalTransactionMgr().advanceTsoFence(proposedFenceTso); + Preconditions.checkState(effectiveFenceTso >= proposedFenceTso, + "MetaService TSO fence must not regress"); + if (effectiveFenceTso == proposedFenceTso) { + break; + } + nextPhysicalTime = TSOTimestamp.extractPhysicalTime(effectiveFenceTso) + 1; + timeWindowEnd = persistCalibrationWindow(nextPhysicalTime); + } + lock.lock(); + try { + pendingCalibrationPhysicalTime = 0; + pendingCalibrationWindowEnd = 0; + } finally { + lock.unlock(); + } isInitialized.set(true); fatalClockBackwardReported.set(false); - LOG.info("TSO timestamp calibrated: lastTimestamp={}, currentMillis={}, nextPhysicalTime={}, timeWindowEnd={}", - timeLast, timeNow, nextPhysicalTime, timeWindowEnd); + LOG.info("TSO timestamp calibrated: lastTimestamp={}, currentMillis={}, nextPhysicalTime={}, " + + "timeWindowEnd={}, fenceTso={}", + timeLast, timeNow, nextPhysicalTime, timeWindowEnd, proposedFenceTso); if (MetricRepo.isInit) { MetricRepo.COUNTER_TSO_CLOCK_CALCULATED.increase(1L); } } + private long persistCalibrationWindow(long physicalTime) { + lock.lock(); + try { + transactionTracker.reset(); + } finally { + lock.unlock(); + } + + // Persist the allocation window before publishing its inclusive commit fence. + long windowEnd = physicalTime + Config.tso_service_window_duration_ms; + writeTimestampToBDBJE(windowEnd); + setTSOPhysical(physicalTime, true); + lock.lock(); + try { + pendingCalibrationPhysicalTime = physicalTime; + pendingCalibrationWindowEnd = windowEnd; + } finally { + lock.unlock(); + } + return windowEnd; + } + /** * Update timestamp periodically to maintain time window * This method handles various time-related issues: @@ -400,11 +570,12 @@ private void updateTimestamp() { } // 4. Check if time window right boundary needs renewal - if ((windowEndTSO.get() - nextPhysicalTime) <= UPDATE_TIME_WINDOW_GUARD) { - // Time window right boundary needs renewal - long nextWindowEnd = nextPhysicalTime + Config.tso_service_window_duration_ms; + if ((durableState.getPhysicalTimestamp() - nextPhysicalTime) <= UPDATE_TIME_WINDOW_GUARD + || System.nanoTime() - lastPersistNanos + >= TimeUnit.MILLISECONDS.toNanos(Config.tso_service_window_duration_ms)) { + long nextWindowEnd = Math.max(durableState.getPhysicalTimestamp(), + nextPhysicalTime + Config.tso_service_window_duration_ms); writeTimestampToBDBJE(nextWindowEnd); - windowEndTSO.set(nextWindowEnd); } // 5. Update global timestamp @@ -445,7 +616,16 @@ private void writeTimestampToBDBJE(long timestamp) { + "timestamp=" + timestamp); } - TSOTimestamp tsoTimestamp = new TSOTimestamp(timestamp, 0); + TSOServiceState nextState; + lock.lock(); + try { + long committedTso = Config.isCloudMode() + ? transactionTracker.candidateCommittedTso(globalTimestamp.composeTimestamp(), + durableState.getCommittedTso()) : 0; + nextState = new TSOServiceState(timestamp, committedTso); + } finally { + lock.unlock(); + } // Check if EditLog is available EditLog editLog = env.getEditLog(); @@ -467,10 +647,25 @@ private void writeTimestampToBDBJE(long timestamp) { } } + long persistStartNanos = System.nanoTime(); try { - editLog.logTSOTimestampWindowEnd(tsoTimestamp); + editLog.logTSOTimestampWindowEnd(nextState); + // Readers must never observe a candidate that has not survived a journal write. + durableState = nextState; + lastPersistNanos = System.nanoTime(); + if (MetricRepo.isInit) { + MetricRepo.COUNTER_TSO_STATE_PERSISTED.increase(1L); + } } catch (Exception e) { + if (MetricRepo.isInit) { + MetricRepo.COUNTER_TSO_STATE_PERSIST_FAILED.increase(1L); + } throw new RuntimeException("Failed to write TSO timestamp to BDBJE, timestamp=" + timestamp, e); + } finally { + if (MetricRepo.isInit) { + MetricRepo.HISTO_TSO_STATE_PERSIST_LATENCY.update( + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - persistStartNanos)); + } } } @@ -480,6 +675,15 @@ private void writeTimestampToBDBJE(long timestamp) { * @return Pair of (physicalTime, updatedLogicalCounter) for the base timestamp */ private Pair generateTSO() { + return generateTSO(null, Collections.emptySet()); + } + + private Pair generateTSO(Pair transactionIdentity, Set tableIds) { + return generateTSO(transactionIdentity, tableIds, -1, -1); + } + + private Pair generateTSO(Pair transactionIdentity, Set tableIds, + long rejectedTso, long fenceTso) { lock.lock(); try { if (!isTsoEnabled() || !isInitialized.get()) { @@ -489,18 +693,42 @@ private Pair generateTSO() { if (physicalTime == 0) { return Pair.of(0L, 0L); } + if (fenceTso > 0 && globalTimestamp.composeTimestamp() < fenceTso) { + throw new RuntimeException("TXN_COMMIT_TSO_FENCED: local TSO " + + globalTimestamp.composeTimestamp() + " is behind MetaService fence " + fenceTso); + } long logicalCounter = globalTimestamp.getLogicalCounter(); if (logicalCounter >= TSOTimestamp.MAX_LOGICAL_COUNTER) { return Pair.of(physicalTime, logicalCounter + 1); } long nextLogical = logicalCounter + 1; globalTimestamp.setLogicalCounter(nextLogical); + if (transactionIdentity != null) { + long tso = TSOTimestamp.composeRealTso(physicalTime, nextLogical); + if (fenceTso > 0) { + transactionTracker.replaceFenced(transactionIdentity, rejectedTso, fenceTso, + tso, System.nanoTime(), tableIds); + } else { + transactionTracker.register(transactionIdentity, tso, System.nanoTime(), tableIds); + } + } return Pair.of(physicalTime, nextLogical); } finally { lock.unlock(); } } + private void deactivate() { + lock.lock(); + try { + if (isInitialized.getAndSet(false)) { + transactionTracker.reset(); + } + } finally { + lock.unlock(); + } + } + /** * Set the physical time component of the global timestamp * @@ -530,34 +758,34 @@ private void setTSOPhysical(long next, boolean force) { * * @param windowEnd New window end physical time */ - public void replayWindowEndTSO(TSOTimestamp windowEnd) { - windowEndTSO.set(windowEnd.getPhysicalTimestamp()); + public void replayWindowEndTSO(TSOServiceState state) { + durableState = state; } public long getWindowEndTSO() { - return windowEndTSO.get(); + return durableState.getPhysicalTimestamp(); } public long saveTSO(CountingDataOutputStream dos, long checksum) throws IOException { if (!isTsoEnabled()) { return checksum; } - long currentWindowEnd = windowEndTSO.get(); + TSOServiceState state = durableState; + long currentWindowEnd = state.getPhysicalTimestamp(); if (currentWindowEnd <= 0) { return checksum; } - TSOTimestamp tsoTimestamp = new TSOTimestamp(currentWindowEnd, 0); - tsoTimestamp.write(dos); - checksum ^= tsoTimestamp.getPhysicalTimestamp(); - LOG.info("Save TSO windowEndTSO {} to image", tsoTimestamp); + state.write(dos); + checksum ^= currentWindowEnd; + LOG.info("Save TSO window end {} and committed TSO {} to image", currentWindowEnd, state.getCommittedTso()); return checksum; } public long loadTSO(DataInputStream dis, long checksum) throws IOException { - TSOTimestamp tsoTimestamp = TSOTimestamp.read(dis); - windowEndTSO.set(tsoTimestamp.getPhysicalTimestamp()); - long newChecksum = checksum ^ tsoTimestamp.getPhysicalTimestamp(); - LOG.info("Finished replay TSO windowEndTSO {} from image", windowEndTSO.get()); + TSOServiceState state = TSOServiceState.read(dis); + durableState = state; + long newChecksum = checksum ^ state.getPhysicalTimestamp(); + LOG.info("Finished replay TSO windowEndTSO {} from image", durableState.getPhysicalTimestamp()); return newChecksum; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOServiceState.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOServiceState.java new file mode 100644 index 00000000000000..cd658da6e06654 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOServiceState.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tso; + +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** The durable allocation window and readable transaction prefix, published as one snapshot. */ +public final class TSOServiceState implements Writable { + // Keep the old TSOTimestamp JSON fields and image checksum for journal/image compatibility. + @SerializedName("physicalTimestamp") + private final long windowEndPhysicalTime; + @SerializedName("logicalCounter") + private final long logicalCounter = 0; + @SerializedName("committedTso") + private final long committedTso; + + public TSOServiceState(long windowEndPhysicalTime, long committedTso) { + this.windowEndPhysicalTime = windowEndPhysicalTime; + this.committedTso = committedTso; + } + + public long getPhysicalTimestamp() { + return windowEndPhysicalTime; + } + + /** Zero means no readable prefix has been established, including records written by older FEs. */ + public long getCommittedTso() { + return committedTso; + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + + public static TSOServiceState read(DataInput in) throws IOException { + TSOServiceState state = GsonUtils.GSON.fromJson(Text.readString(in), TSOServiceState.class); + if (state == null) { + throw new IOException("failed to deserialize TSO service state from journal/image"); + } + return state; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java new file mode 100644 index 00000000000000..d2985844e5318e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tso; + +import org.apache.doris.common.Pair; + +import com.google.common.base.Preconditions; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; + +/** In-memory commit registrations. Uses the allocator's lock so no allocated TSO can be missed. */ +final class TSOTransactionTracker { + private final ReentrantLock lock; + private final Condition transactionsChanged; + private final Map, PendingTransaction> pendingByTxn = new HashMap<>(); + private final TreeMap pendingByTso = new TreeMap<>(); + private long generation; + + enum WaitResult { + FINISHED, TIMED_OUT, RESET + } + + private static final class PendingTransaction { + private final Pair identity; + private final long tso; + private final long registeredAtNanos; + private final Set tableIds; + + private PendingTransaction(Pair identity, long tso, long nowNanos, Collection tableIds) { + this.identity = identity; + this.tso = tso; + this.registeredAtNanos = nowNanos; + this.tableIds = new HashSet<>(tableIds); + } + } + + TSOTransactionTracker(ReentrantLock lock) { + this.lock = lock; + this.transactionsChanged = lock.newCondition(); + } + + void reset() { + Preconditions.checkState(lock.isHeldByCurrentThread()); + generation++; + pendingByTxn.clear(); + pendingByTso.clear(); + transactionsChanged.signalAll(); + } + + void register(Pair identity, long tso, long nowNanos, Set tableIds) { + Preconditions.checkState(lock.isHeldByCurrentThread()); + Preconditions.checkArgument(!tableIds.isEmpty(), "commit registration requires table IDs"); + PendingTransaction existing = pendingByTxn.get(identity); + if (existing != null) { + existing.tableIds.addAll(tableIds); + return; + } + PendingTransaction pending = new PendingTransaction(identity, tso, nowNanos, tableIds); + pendingByTxn.put(identity, pending); + Preconditions.checkState(pendingByTso.put(tso, pending) == null); + } + + void replaceFenced(Pair identity, long rejectedTso, long fenceTso, long newTso, + long nowNanos, Set tableIds) { + Preconditions.checkState(lock.isHeldByCurrentThread()); + Preconditions.checkArgument(!tableIds.isEmpty(), "commit registration requires table IDs"); + Preconditions.checkArgument(rejectedTso <= fenceTso, "rejected TSO must be fenced"); + Preconditions.checkArgument(newTso > fenceTso, "replacement TSO must be above the fence"); + PendingTransaction existing = pendingByTxn.get(identity); + Set mergedTableIds = new HashSet<>(); + if (existing != null) { + Preconditions.checkState(existing.tso <= fenceTso, "registered transaction TSO must be fenced"); + Preconditions.checkState(pendingByTso.remove(existing.tso) == existing); + mergedTableIds.addAll(existing.tableIds); + } + mergedTableIds.addAll(tableIds); + PendingTransaction replacement = new PendingTransaction(identity, newTso, nowNanos, mergedTableIds); + pendingByTxn.put(identity, replacement); + Preconditions.checkState(pendingByTso.put(newTso, replacement) == null); + transactionsChanged.signalAll(); + } + + /** Called with the allocator lock after validating endTso against its current clock. */ + WaitResult awaitTransactions(Map> dbToTableIds, long endTso, long remainingNanos) + throws InterruptedException { + Preconditions.checkState(lock.isHeldByCurrentThread()); + long waitStartNanos = System.nanoTime(); + long waitGeneration = generation; + List remaining = new ArrayList<>(); + for (PendingTransaction pending : pendingByTso.headMap(endTso, true).values()) { + List tables = dbToTableIds.get(pending.identity.first); + if (tables != null && !Collections.disjoint(tables, pending.tableIds)) { + remaining.add(pending); + } + } + while (true) { + if (generation != waitGeneration) { + return WaitResult.RESET; + } + remaining.removeIf(pending -> pendingByTxn.get(pending.identity) != pending); + if (remaining.isEmpty()) { + return WaitResult.FINISHED; + } + long nanosLeft = remainingNanos - (System.nanoTime() - waitStartNanos); + if (nanosLeft <= 0) { + return WaitResult.TIMED_OUT; + } + transactionsChanged.awaitNanos(nanosLeft); + } + } + + long candidateCommittedTso(long currentTso, long durableCommittedTso) { + Preconditions.checkState(lock.isHeldByCurrentThread()); + if (currentTso < durableCommittedTso) { + return durableCommittedTso; + } + long candidate = pendingByTso.isEmpty() ? currentTso + : Math.min(currentTso, pendingByTso.firstKey() - 1); + Preconditions.checkState(candidate >= durableCommittedTso, "committed TSO must not regress"); + return candidate; + } + + void transactionFinished(long dbId, long txnId) { + lock.lock(); + try { + remove(Pair.of(dbId, txnId)); + } finally { + lock.unlock(); + } + } + + void abandonCommitTso(long dbId, long txnId, long tso) { + lock.lock(); + try { + Pair identity = Pair.of(dbId, txnId); + PendingTransaction pending = pendingByTxn.get(identity); + if (pending != null && pending.tso == tso) { + remove(identity); + } + } finally { + lock.unlock(); + } + } + + private void remove(Pair identity) { + Preconditions.checkState(lock.isHeldByCurrentThread()); + PendingTransaction pending = pendingByTxn.remove(identity); + if (pending != null) { + Preconditions.checkState(pendingByTso.remove(pending.tso) == pending); + transactionsChanged.signalAll(); + } + } + + long getPendingCount() { + lock.lock(); + try { + return pendingByTxn.size(); + } finally { + lock.unlock(); + } + } + + long getOldestPendingTso() { + lock.lock(); + try { + return pendingByTso.isEmpty() ? 0 : pendingByTso.firstKey(); + } finally { + lock.unlock(); + } + } + + long getOldestPendingTxnId() { + lock.lock(); + try { + return pendingByTso.isEmpty() ? 0 : pendingByTso.firstEntry().getValue().identity.second; + } finally { + lock.unlock(); + } + } + + long getOldestPendingAgeMs() { + lock.lock(); + try { + return pendingByTso.isEmpty() ? 0 : TimeUnit.NANOSECONDS.toMillis( + System.nanoTime() - pendingByTso.firstEntry().getValue().registeredAtNanos); + } finally { + lock.unlock(); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/arrowflight/DorisFlightSqlProducerTest.java b/fe/fe-core/src/test/java/org/apache/doris/arrowflight/DorisFlightSqlProducerTest.java index f72918e4aa26d9..5052633cea7f23 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/arrowflight/DorisFlightSqlProducerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/arrowflight/DorisFlightSqlProducerTest.java @@ -20,13 +20,20 @@ import org.apache.doris.arrowflight.protocol.FlightProtocolAdapter; import org.apache.doris.arrowflight.results.FlightSqlChannel; import org.apache.doris.arrowflight.sessions.FlightSessionsManager; +import org.apache.doris.common.ErrorCode; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QueryState; import org.apache.doris.qe.StmtExecutor; +import org.apache.arrow.flight.CallStatus; +import org.apache.arrow.flight.ErrorFlightMetadata; import org.apache.arrow.flight.FlightDescriptor; import org.apache.arrow.flight.FlightProducer.CallContext; import org.apache.arrow.flight.FlightProducer.StreamListener; +import org.apache.arrow.flight.FlightRuntimeException; +import org.apache.arrow.flight.FlightStatusCode; import org.apache.arrow.flight.Location; import org.apache.arrow.flight.Result; import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementRequest; @@ -46,6 +53,88 @@ public class DorisFlightSqlProducerTest { private boolean prevRunningUnitTest; + @Test + public void testWindowNotReadyHasRetryableFlightStatusAndStableBusinessCode() { + QueryState state = new QueryState(); + IncrWindowNotReadyException error = new IncrWindowNotReadyException(2000, 100, 1000); + state.setError(error.getMysqlErrorCode(), error.getDetailMessage()); + FlightRuntimeException result = DorisFlightSqlProducer.queryFailure(state, state.getErrorMessage(), error); + Assertions.assertEquals(FlightStatusCode.UNAVAILABLE, result.status().code()); + Assertions.assertEquals(Integer.toString(ErrorCode.ERR_INCR_WINDOW_NOT_READY.getCode()), + result.status().metadata().get("doris-error-code")); + Assertions.assertTrue(result.status().description().contains("requestedEndTimestampMs=2000")); + Assertions.assertTrue(result.status().description().contains("retryAfterMs=1000")); + state.setError(ErrorCode.ERR_UNKNOWN_ERROR, "other failure"); + Assertions.assertEquals(FlightStatusCode.INTERNAL, + DorisFlightSqlProducer.queryFailure(state, "other failure", error).status().code()); + } + + @Test + public void testGetFlightInfoPreservesBothWindowErrors() throws Exception { + for (ErrorCode code : new ErrorCode[] {ErrorCode.ERR_INCR_WINDOW_NOT_READY, + ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT}) { + QueryState state = new QueryState(); + IncrWindowNotReadyException error = new IncrWindowNotReadyException(code, "test reason", + 2000, 3000, 100, 1000, 5000); + state.setError(error.getMysqlErrorCode(), error.getDetailMessage()); + FlightRuntimeException failure = DorisFlightSqlProducer.queryFailure(state, state.getErrorMessage(), error); + Assertions.assertEquals(FlightStatusCode.UNAVAILABLE, failure.status().code()); + Assertions.assertEquals(Integer.toString(code.getCode()), failure.status().metadata().get("doris-error-code")); + Assertions.assertEquals(code.name(), failure.status().metadata().get("doris-error-name")); + Assertions.assertTrue(failure.status().description().contains("currentTSO=3000")); + Assertions.assertTrue(failure.status().description().contains("committedTSO=100")); + Assertions.assertTrue(failure.status().description().contains("timeoutMs=5000")); + Assertions.assertSame(failure, getFlightInfoFailure(failure)); + } + } + + @Test + public void testGetFlightInfoWrapsOtherFlightErrors() throws Exception { + for (CallStatus status : new CallStatus[] {CallStatus.INTERNAL, CallStatus.UNAVAILABLE, + CallStatus.INVALID_ARGUMENT, CallStatus.UNAUTHENTICATED}) { + FlightRuntimeException failure = status.withDescription("other flight failure").toRuntimeException(); + assertLegacyFlightWrapper(failure, getFlightInfoFailure(failure)); + } + } + + @Test + public void testGetFlightInfoWrapsOtherBusinessErrors() throws Exception { + ErrorFlightMetadata metadata = new ErrorFlightMetadata(); + metadata.insert("doris-error-code", Integer.toString(ErrorCode.ERR_UNKNOWN_ERROR.getCode())); + FlightRuntimeException failure = CallStatus.UNAVAILABLE.withDescription("other business failure") + .withMetadata(metadata).toRuntimeException(); + + assertLegacyFlightWrapper(failure, getFlightInfoFailure(failure)); + } + + @Test + public void testGetFlightInfoWrapsNonFlightErrors() throws Exception { + RuntimeException failure = new RuntimeException("session lookup failed"); + assertLegacyFlightWrapper(failure, getFlightInfoFailure(failure)); + } + + private FlightRuntimeException getFlightInfoFailure(RuntimeException failure) throws Exception { + FlightSessionsManager sessionsManager = Mockito.mock(FlightSessionsManager.class); + Mockito.when(sessionsManager.getConnectContext("token")).thenThrow(failure); + CallContext callContext = Mockito.mock(CallContext.class); + Mockito.when(callContext.peerIdentity()).thenReturn("token"); + try (DorisFlightSqlProducer producer = new DorisFlightSqlProducer( + Location.forGrpcInsecure("127.0.0.1", 9090), sessionsManager)) { + CommandStatementQuery request = CommandStatementQuery.newBuilder().setQuery("select 1").build(); + return Assertions.assertThrows(FlightRuntimeException.class, + () -> producer.getFlightInfoStatement(request, callContext, FlightDescriptor.command(new byte[0]))); + } + } + + private void assertLegacyFlightWrapper(RuntimeException failure, FlightRuntimeException result) { + Assertions.assertEquals(FlightStatusCode.INTERNAL, result.status().code()); + Assertions.assertNotSame(failure, result); + Assertions.assertSame(failure, result.getCause()); + Assertions.assertEquals("get flight info statement failed, " + failure.getMessage(), + result.status().description()); + Assertions.assertFalse(result.status().metadata().containsKey("doris-error-code")); + } + @BeforeEach public void setUp() { // ConnectContext.init() only reaches Env when this is false; keep it true so the diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/SchemaTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/SchemaTableTest.java index b1f63ab862cbcb..b0c0a4c683aac5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/SchemaTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/SchemaTableTest.java @@ -120,10 +120,12 @@ public void testShouldFetchAllFe() throws AnalysisException, IOException { SchemaTable tsoStatus = (SchemaTable) SchemaTable.TABLE_MAP.get("tso_status"); Assertions.assertFalse(tsoStatus.shouldFetchAllFe()); Assertions.assertFalse(tsoStatus.shouldAddAgg()); - Assertions.assertEquals(4, tsoStatus.getFullSchema().size()); + Assertions.assertEquals(6, tsoStatus.getFullSchema().size()); Assertions.assertEquals("WINDOW_END_PHYSICAL_TIME", tsoStatus.getFullSchema().get(0).getName()); Assertions.assertEquals("CURRENT_TSO", tsoStatus.getFullSchema().get(1).getName()); Assertions.assertEquals("CURRENT_TSO_PHYSICAL_TIME", tsoStatus.getFullSchema().get(2).getName()); Assertions.assertEquals("CURRENT_TSO_LOGICAL_COUNTER", tsoStatus.getFullSchema().get(3).getName()); + Assertions.assertEquals("COMMITTED_TSO", tsoStatus.getFullSchema().get(4).getName()); + Assertions.assertEquals("COMMITTED_TSO_PHYSICAL_TIME", tsoStatus.getFullSchema().get(5).getName()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceProxyTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceProxyTest.java index 5b80b6ab77a441..bb22d682231fcf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceProxyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceProxyTest.java @@ -139,6 +139,25 @@ public void testExecuteRequestShutdownOnFailure() { Mockito.verify(client).shutdown(true); } + @Test + public void testExecuteRequestCanDisableRpcFailureRetry() { + Config.meta_service_rpc_retry_cnt = 3; + MetaServiceProxy proxy = new MetaServiceProxy(); + MetaServiceClient client = mockNormalClient(); + Map serviceMap = Deencapsulation.getField(proxy, "serviceMap"); + serviceMap.put(Config.meta_service_endpoint, client); + MetaServiceProxy.MetaServiceClientWrapper wrapper = Deencapsulation.getField(proxy, "w"); + AtomicInteger callCount = new AtomicInteger(); + + Assertions.assertThrows(RpcException.class, () -> wrapper.executeRequest("commitTxn", ignored -> { + callCount.incrementAndGet(); + throw new RuntimeException("rpc failed"); + }, Cloud.CommitTxnResponse::getStatus, false)); + + Assertions.assertEquals(1, callCount.get()); + Mockito.verify(client).shutdown(true); + } + @Test public void testGetVisibleVersionAsyncShutdownOnFailure() throws RpcException { MetaServiceProxy proxy = new MetaServiceProxy(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java new file mode 100644 index 00000000000000..ef8315f3c3b1cb --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.transaction; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TabletInvertedIndex; +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; +import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; +import org.apache.doris.cloud.rpc.MetaServiceProxy; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.rpc.RpcException; +import org.apache.doris.transaction.TransactionUtil; +import org.apache.doris.tso.TSOService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class CloudCommittedTsoTest { + @Test + public void testOrdinarySubTransactionAndTwoPhaseCommitReuseTsoAcrossRpcRetries() throws Exception { + Env env = Mockito.mock(Env.class); + TSOService tsoService = Mockito.mock(TSOService.class); + Mockito.when(env.getTSOService()).thenReturn(tsoService); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database db = Mockito.mock(Database.class); + Mockito.when(db.getId()).thenReturn(1L); + Mockito.when(db.getTablesOnIdOrderOrThrowException(Mockito.anyList())).thenReturn(Collections.emptyList()); + Mockito.when(catalog.getDbOrMetaException(1L)).thenReturn(db); + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class); + MockedStatic mockedProxy = Mockito.mockStatic(MetaServiceProxy.class); + MockedStatic allocation = Mockito.mockStatic(TransactionUtil.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + mockedEnv.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + mockedProxy.when(MetaServiceProxy::getInstance).thenReturn(proxy); + allocation.when(() -> TransactionUtil.getCommitTSO(10L, db, Collections.emptySet())).thenReturn(500L); + for (int path = 0; path < 3; path++) { + List requests = new ArrayList<>(); + Mockito.when(proxy.commitTxn(Mockito.any())).thenAnswer(invocation -> { + requests.add(invocation.getArgument(0)); + return CommitTxnResponse.newBuilder().setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(requests.size() == 1 ? MetaServiceCode.KV_TXN_CONFLICT : MetaServiceCode.UNDEFINED_ERR) + .setMsg("injected commit failure")).build(); + }); + CloudGlobalTransactionMgr mgr = new CloudGlobalTransactionMgr(); + int selectedPath = path; + Assertions.assertThrows(UserException.class, () -> { + if (selectedPath == 0) { + mgr.commitTransactionWithoutLock(1, Collections.emptyList(), 10, null, null); + } else if (selectedPath == 1) { + mgr.commitAndPublishTransaction(db, 10, Collections.emptyList(), 1000); + } else { + mgr.commitTransaction2PC(db, Collections.emptyList(), 10, 1000); + } + }); + Assertions.assertEquals(2, requests.size()); + Assertions.assertSame(requests.get(0), requests.get(1)); + Assertions.assertEquals(500L, requests.get(0).getCommitTso()); + Assertions.assertEquals(path == 1, requests.get(0).getIsTxnLoad()); + Assertions.assertEquals(path == 2, requests.get(0).getIs2Pc()); + Mockito.verify(tsoService).abandonCommitTso(1, 10, 500); + allocation.verify(() -> TransactionUtil.getCommitTSO(10L, db, Collections.emptySet())); + allocation.clearInvocations(); + Mockito.clearInvocations(tsoService); + } + } + } + + @Test + public void testUncertainCommitResultsAreFencedBeforeRelease() throws Exception { + Env env = Mockito.mock(Env.class); + TSOService tsoService = Mockito.mock(TSOService.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database db = Mockito.mock(Database.class); + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + Mockito.when(env.getTSOService()).thenReturn(tsoService); + Mockito.when(catalog.getDbOrMetaException(1L)).thenReturn(db); + Mockito.when(proxy.commitTxn(Mockito.any())).thenThrow(new RpcException("ms", "timeout")); + Method commit = CloudGlobalTransactionMgr.class.getDeclaredMethod("commitTxn", + Cloud.CommitTxnRequest.Builder.class, List.class, long.class, boolean.class, List.class, List.class); + commit.setAccessible(true); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class); + MockedStatic mockedProxy = Mockito.mockStatic(MetaServiceProxy.class); + MockedStatic allocation = Mockito.mockStatic(TransactionUtil.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + mockedEnv.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + mockedProxy.when(MetaServiceProxy::getInstance).thenReturn(proxy); + allocation.when(() -> TransactionUtil.getCommitTSO(10L, db, Collections.emptySet())).thenReturn(500L); + + Assertions.assertThrows(InvocationTargetException.class, + () -> commit.invoke(new CloudGlobalTransactionMgr(), + Cloud.CommitTxnRequest.newBuilder().setDbId(1).setTxnId(10), + Collections.emptyList(), 10L, false, Collections.emptyList(), Collections.emptyList())); + Mockito.doReturn(CommitTxnResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.KV_TXN_MAYBE_COMMITTED)) + .build()).when(proxy).commitTxn(Mockito.any()); + Assertions.assertThrows(InvocationTargetException.class, + () -> commit.invoke(new CloudGlobalTransactionMgr(), + Cloud.CommitTxnRequest.newBuilder().setDbId(1).setTxnId(10), + Collections.emptyList(), 10L, false, Collections.emptyList(), Collections.emptyList())); + Mockito.verify(tsoService, Mockito.times(2)).fenceAndAbandonCommitTso(1, 10, 500); + Mockito.verify(tsoService, Mockito.never()).abandonCommitTso(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong()); + } + } + + @Test + public void testValidationFailureDoesNotAllocateTsoOrSendCommit() throws Exception { + Env env = Mockito.mock(Env.class); + TabletInvertedIndex index = Mockito.mock(TabletInvertedIndex.class); + Mockito.when(env.getTabletInvertedIndex()).thenReturn(index); + Mockito.when(index.getTabletMetaList(Mockito.anyList())).thenThrow(new IllegalStateException("validation failed")); + Method commit = CloudGlobalTransactionMgr.class.getDeclaredMethod("commitTxn", Cloud.CommitTxnRequest.Builder.class, + List.class, long.class, boolean.class, List.class, List.class); + commit.setAccessible(true); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class); + MockedStatic allocation = Mockito.mockStatic(TransactionUtil.class); + MockedStatic proxy = Mockito.mockStatic(MetaServiceProxy.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Cloud.CommitTxnRequest.Builder builder = Cloud.CommitTxnRequest.newBuilder() + .setDbId(1).setTxnId(10).addBaseTabletIds(20); + InvocationTargetException error = Assertions.assertThrows(InvocationTargetException.class, + () -> commit.invoke(new CloudGlobalTransactionMgr(), builder, Collections.emptyList(), + 10L, false, Collections.emptyList(), Collections.emptyList())); + Assertions.assertEquals("validation failed", error.getCause().getMessage()); + allocation.verifyNoInteractions(); + proxy.verifyNoInteractions(); + Assertions.assertFalse(builder.hasCommitTso()); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java index 3d200ce80d3ec0..2c8d1a3a08fd8d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java @@ -17,15 +17,18 @@ package org.apache.doris.cloud.transaction; +import org.apache.doris.catalog.BinlogConfig; import org.apache.doris.catalog.CatalogTestUtil; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FakeEditLog; import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.stream.CloudOlapTableStreamUpdate; import org.apache.doris.catalog.stream.TableStreamUpdateInfo; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.proto.Cloud.AbortTxnResponse; +import org.apache.doris.cloud.proto.Cloud.AdvanceTsoFenceResponse; import org.apache.doris.cloud.proto.Cloud.BeginTxnResponse; import org.apache.doris.cloud.proto.Cloud.CheckTxnConflictResponse; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; @@ -44,6 +47,7 @@ import org.apache.doris.transaction.TabletCommitInfo; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TxnStateChangeCallback; +import org.apache.doris.tso.TSOService; import com.google.common.collect.Lists; import org.junit.jupiter.api.AfterEach; @@ -54,6 +58,7 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.List; import java.util.Map; @@ -92,6 +97,22 @@ public void tearDown() { } } + @Test + public void testAdvanceTsoFence() throws Exception { + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + try (MockedStatic mocked = Mockito.mockStatic(MetaServiceProxy.class)) { + mocked.when(MetaServiceProxy::getInstance).thenReturn(proxy); + Mockito.when(proxy.advanceTsoFence(Mockito.any())).thenReturn(AdvanceTsoFenceResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK)) + .setTsoFence(101).build()); + Assertions.assertEquals(101, masterTransMgr.advanceTsoFence(100)); + ArgumentCaptor capture = + ArgumentCaptor.forClass(Cloud.AdvanceTsoFenceRequest.class); + Mockito.verify(proxy).advanceTsoFence(capture.capture()); + Assertions.assertEquals(100, capture.getValue().getProposedFenceTso()); + } + } + @Test public void testBeginTransaction() throws Exception { AtomicLong id = new AtomicLong(1000); @@ -210,6 +231,65 @@ public void testCommitTransaction() throws Exception { } } + @Test + public void testCommitTransactionRetriesWithTsoAboveFence() throws Exception { + boolean originalEnableFeatureBinlog = Config.enable_feature_binlog; + OlapTable table = (OlapTable) masterEnv.getInternalCatalog() + .getDbOrMetaException(CatalogTestUtil.testDbId1) + .getTableOrMetaException(CatalogTestUtil.testTableId1); + BinlogConfig originalBinlogConfig = new BinlogConfig(table.getBinlogConfig()); + try { + Config.enable_feature_binlog = true; + FakeEnv.setEnv(masterEnv); + BinlogConfig binlogConfig = new BinlogConfig(originalBinlogConfig); + binlogConfig.setEnable(true); + binlogConfig.setBinlogFormat(BinlogConfig.BinlogFormat.ROW); + table.setBinlogConfig(binlogConfig); + + TSOService tsoService = Mockito.mock(TSOService.class); + Mockito.when(tsoService.getCommitTSO(Mockito.eq(CatalogTestUtil.testDbId1), + Mockito.eq(123533L), Mockito.anySet())).thenReturn(100L); + Mockito.when(tsoService.getCommitTSOAfterFence(Mockito.eq(CatalogTestUtil.testDbId1), + Mockito.eq(123533L), Mockito.anySet(), Mockito.eq(100L), Mockito.eq(200L))) + .thenReturn(201L); + setEnvTSOService(masterEnv, tsoService); + + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + TxnInfoPB txnInfo = TxnInfoPB.newBuilder() + .setDbId(CatalogTestUtil.testDbId1) + .addTableIds(CatalogTestUtil.testTableId1) + .setTxnId(123533L) + .setListenerId(-1) + .build(); + Mockito.when(proxy.commitTxn(Mockito.any())) + .thenReturn(CommitTxnResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.TXN_COMMIT_TSO_FENCED)) + .setTsoFence(200L) + .build(), + CommitTxnResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK)) + .setTxnInfo(txnInfo) + .build()); + try (MockedStatic mocked = Mockito.mockStatic(MetaServiceProxy.class)) { + mocked.when(MetaServiceProxy::getInstance).thenReturn(proxy); + masterTransMgr.commitTransactionWithoutLock(CatalogTestUtil.testDbId1, + Lists.newArrayList(table), 123533L, null, null); + } + + ArgumentCaptor requests = + ArgumentCaptor.forClass(Cloud.CommitTxnRequest.class); + Mockito.verify(proxy, Mockito.times(2)).commitTxn(requests.capture()); + Assertions.assertEquals(100L, requests.getAllValues().get(0).getCommitTso()); + Assertions.assertEquals(201L, requests.getAllValues().get(1).getCommitTso()); + Mockito.verify(tsoService).transactionFinished(CatalogTestUtil.testDbId1, 123533L); + } finally { + table.setBinlogConfig(originalBinlogConfig); + Config.enable_feature_binlog = originalEnableFeatureBinlog; + } + } + @Test public void testCommitTransactionCarriesTableStreamUpdates() throws Exception { MetaServiceProxy mockProxy = Mockito.mock(MetaServiceProxy.class); @@ -270,7 +350,6 @@ public void testCommitTransactionCarriesTableStreamUpdates() throws Exception { ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(Cloud.CommitTxnRequest.class); Mockito.verify(mockProxy).commitTxn(requestCaptor.capture()); - Assertions.assertTrue(requestCaptor.getValue().hasCommitTso()); Assertions.assertEquals(2, requestCaptor.getValue().getTableStreamUpdatesCount()); Assertions.assertEquals(identity, requestCaptor.getValue().getTableStreamUpdates(0).getIdentity()); Assertions.assertEquals(partitionUpdate, @@ -353,6 +432,12 @@ public void sendMakeCloudTmpRsVisibleTasks(long txnId, } } + private static void setEnvTSOService(Env env, TSOService service) throws Exception { + Field field = Env.class.getDeclaredField("tsoService"); + field.setAccessible(true); + field.set(env, service); + } + @Test public void testCommitTransactionAlreadyVisible() throws Exception { MetaServiceProxy mockProxy = Mockito.mock(MetaServiceProxy.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java index 0c3b524cd12f50..44bf4d8215a0a9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java @@ -342,12 +342,12 @@ public void testIncrementalReadGetsVisibleVersionFromMetaService() throws Except MockedStatic mockedPartition = Mockito.mockStatic(CloudPartition.class)) { mockedConfig.when(Config::isNotCloudMode).thenReturn(false); mockedPartition.when(() -> CloudPartition.getSnapshotVisibleVersionFromMs( - Mockito.anyList(), Mockito.eq(false))).thenReturn(Lists.newArrayList(visibleVersion)); + Mockito.anyList(), Mockito.eq(true))).thenReturn(Lists.newArrayList(visibleVersion)); ScanNode.setVisibleVersionForOlapScanNodes(Lists.newArrayList(scanNode)); mockedPartition.verify(() -> CloudPartition.getSnapshotVisibleVersionFromMs( - Mockito.anyList(), Mockito.eq(false))); + Mockito.anyList(), Mockito.eq(true))); mockedPartition.verify(() -> CloudPartition.getSnapshotVisibleVersion(Mockito.anyList()), Mockito.never()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index 85001fbd0bdfe3..7bbd9b04c090c8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -17,13 +17,18 @@ package org.apache.doris.qe; +import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.InternalSchemaInitializer; import org.apache.doris.catalog.ResourceMgr; import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.IncrWindowNotReadyException; +import org.apache.doris.common.NereidsException; import org.apache.doris.common.Status; import org.apache.doris.mysql.authenticate.TestLogAppender; +import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.Planner; import org.apache.doris.planner.ResultFileSink; @@ -34,6 +39,7 @@ import com.google.common.collect.Lists; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; import org.mockito.Mockito; import java.lang.reflect.Field; @@ -53,6 +59,28 @@ protected void runBeforeAll() throws Exception { createDatabase("testDb"); } + @Test + public void testCommittedTsoErrorSurvivesPlannerWrapping() throws Exception { + for (ErrorCode code : new ErrorCode[] {ErrorCode.ERR_INCR_WINDOW_NOT_READY, + ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT}) { + connectContext.getState().reset(); + IncrWindowNotReadyException rejected = new IncrWindowNotReadyException(code, "test reason", + 2000, 3000, 1000, 1000, 5000); + StmtExecutor executor = new StmtExecutor(connectContext, "select 1"); + try (MockedConstruction planners = Mockito.mockConstruction(NereidsPlanner.class, + (planner, construction) -> Mockito.doThrow(new NereidsException(rejected.getMessage(), rejected)) + .when(planner).plan(Mockito.any(StatementBase.class), Mockito.any(TQueryOptions.class)))) { + executor.execute(); + Assertions.assertEquals(1, planners.constructed().size()); + } + Assertions.assertEquals(QueryState.MysqlStateType.ERR, connectContext.getState().getStateType()); + Assertions.assertEquals(code, connectContext.getState().getErrorCode()); + Assertions.assertTrue(connectContext.getState().getErrorMessage().contains("requestedEndTimestampMs=2000")); + Assertions.assertTrue(connectContext.getState().getErrorMessage().contains("retryAfterMs=1000")); + Assertions.assertTrue(connectContext.getState().getErrorMessage().contains("timeoutMs=5000")); + } + } + @Test public void testShow() throws Exception { StmtExecutor stmtExecutor = new StmtExecutor(connectContext, ""); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java index 0abf75f443c8ee..a3bd353da1b676 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java @@ -23,7 +23,11 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ClientPool; import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.GenericPool; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.UserException; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.nereids.analyzer.UnboundRelation; @@ -32,12 +36,21 @@ import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.planner.OlapScanNode; +import org.apache.doris.service.FrontendServiceImpl; +import org.apache.doris.thrift.FrontendService; +import org.apache.doris.thrift.TAcquireTimeBasedChangeReadFenceRequest; +import org.apache.doris.thrift.TAcquireTimeBasedChangeReadFenceResult; +import org.apache.doris.thrift.TIncrWindowNotReady; +import org.apache.doris.thrift.TStatus; +import org.apache.doris.thrift.TStatusCode; import org.apache.doris.transaction.GlobalTransactionMgrIface; import org.apache.doris.tso.TSOService; import org.apache.doris.tso.TSOTimestamp; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TSerializer; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.InOrder; @@ -66,6 +79,7 @@ public void testCollectChangeReadInfoWithoutEndTimestamp() { Assertions.assertEquals(ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID)), result.getDbToTableIds()); Assertions.assertNull(result.getMaxEndTimestampMs()); + Assertions.assertFalse(result.isAllEndsExplicit()); } @Test @@ -84,6 +98,7 @@ public void testCollectChangeReadInfoMergesMaximumEndTimestamp() { Assertions.assertEquals(ImmutableList.of(TABLE_ID), result.getDbToTableIds().get(DB_ID)); Assertions.assertEquals(OlapScanNode.parseChangeTimestamp(endTimestamp2), result.getMaxEndTimestampMs()); + Assertions.assertTrue(result.isAllEndsExplicit()); } @Test @@ -213,6 +228,143 @@ public void testFenceFailsWhenConflictCheckFails() throws Exception { } } + @Test + public void testHistoricalWindowDoesNotWaitForTransactionsOutsideWindow() throws Exception { + Env env = mockMasterEnv(); + TSOService service = mockTsoService(env, CURRENT_TSO); + long committedTso = TSOTimestamp.composeTimestamp(CURRENT_PHYSICAL_TIME_MS - 1000, 17); + Mockito.when(service.getStatusSnapshot()).thenReturn( + new TSOService.TSOStatusSnapshot(true, CURRENT_TSO, CURRENT_PHYSICAL_TIME_MS + 1000, committedTso)); + GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(301L); + // A same-table write started after the historical end and remains running. + Mockito.when(txnMgr.isPreviousTransactionsFinished(301L, DB_ID, ImmutableList.of(TABLE_ID))) + .thenReturn(false); + try (MockedStatic config = Mockito.mockStatic(Config.class); + MockedStatic mocked = Mockito.mockStatic(Env.class)) { + config.when(Config::isCloudMode).thenReturn(true); + mocked.when(Env::getCurrentEnv).thenReturn(env); + mocked.when(Env::getCurrentGlobalTransactionMgr).thenReturn(txnMgr); + TimeBasedChangeVisibleWaiter.ChangeReadFence fence = TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( + ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID)), CURRENT_PHYSICAL_TIME_MS - 1000, 0, true, true); + Assertions.assertEquals(committedTso, fence.getCommittedTso()); + Mockito.verifyNoInteractions(txnMgr); + // Negative control: the legacy table-wide drain cannot distinguish this later write. + Assertions.assertThrows(UserException.class, () -> TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( + ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID)), CURRENT_PHYSICAL_TIME_MS - 1000, 0, true)); + } + } + + @Test + public void testBoundedReadDelegatesRegisteredTransactionWait() throws Exception { + Env env = mockMasterEnv(); + TSOService service = mockTsoService(env, CURRENT_TSO); + GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); + Map> tables = ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID)); + try (MockedStatic config = Mockito.mockStatic(Config.class); + MockedStatic mocked = Mockito.mockStatic(Env.class)) { + config.when(Config::isCloudMode).thenReturn(true); + mocked.when(Env::getCurrentEnv).thenReturn(env); + mocked.when(Env::getCurrentGlobalTransactionMgr).thenReturn(txnMgr); + for (long committed : new long[] {0, TSOTimestamp.composeTimestamp(CURRENT_PHYSICAL_TIME_MS - 1, 17)}) { + Mockito.when(service.getStatusSnapshot()).thenReturn( + new TSOService.TSOStatusSnapshot(true, CURRENT_TSO, CURRENT_PHYSICAL_TIME_MS + 1000, committed)); + TimeBasedChangeVisibleWaiter.ChangeReadFence result = TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( + tables, CURRENT_PHYSICAL_TIME_MS, 1000, true, true); + Assertions.assertEquals(committed, result.getCommittedTso()); + } + Mockito.verify(service, Mockito.times(2)).waitForReadableWindow(tables, CURRENT_PHYSICAL_TIME_MS, 1000); + Mockito.verifyNoInteractions(txnMgr); + } + } + + @Test + public void testMixedBoundedAndUnboundedRelationsRetainDrain() { + Plan plan = new LogicalJoin<>(JoinType.INNER_JOIN, + newChangeRelation(1, ImmutableMap.of(OlapScanNode.OLAP_END_TIMESTAMP, "2024-01-01 00:00:00")), + newChangeRelation(2, ImmutableMap.of()), null); + TimeBasedChangeVisibleWaiter.ChangeReadInfo info = TimeBasedChangeVisibleWaiter.collectChangeReadInfo( + mockContext(), plan, ImmutableMap.of(TABLE_QUALIFIER, mockOlapTable(DB_ID, TABLE_ID))); + Assertions.assertNotNull(info.getMaxEndTimestampMs()); + Assertions.assertFalse(info.isAllEndsExplicit()); + } + + @Test + public void testMasterRpcAndFollowerPreserveBothWindowErrors() throws Exception { + for (ErrorCode code : new ErrorCode[] {ErrorCode.ERR_INCR_WINDOW_NOT_READY, + ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT}) { + Env master = mockMasterEnv(); + TSOService service = mockTsoService(master, CURRENT_TSO); + String reason = code == ErrorCode.ERR_INCR_WINDOW_NOT_READY ? "TSO_MASTER_CHANGED" : "VISIBLE_WAIT_TIMEOUT"; + IncrWindowNotReadyException failure = new IncrWindowNotReadyException(code, reason, + CURRENT_PHYSICAL_TIME_MS, CURRENT_TSO, 0, 1000, 1000); + Mockito.doThrow(failure).when(service) + .waitForReadableWindow(Mockito.anyMap(), Mockito.anyLong(), Mockito.anyLong()); + TAcquireTimeBasedChangeReadFenceRequest request = new TAcquireTimeBasedChangeReadFenceRequest(); + request.setDbToTableIds(ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID))); + request.setEndTimestampMs(CURRENT_PHYSICAL_TIME_MS); + request.setTimeoutMs(1000); + request.setWaitForTransactions(true); + request.setAllEndsExplicit(true); + TAcquireTimeBasedChangeReadFenceResult response; + try (MockedStatic mocked = Mockito.mockStatic(Env.class); + MockedStatic config = Mockito.mockStatic(Config.class)) { + mocked.when(Env::getCurrentEnv).thenReturn(master); + config.when(Config::isCloudMode).thenReturn(true); + response = new FrontendServiceImpl(null).acquireTimeBasedChangeReadFence(request); + } + Assertions.assertEquals(TStatusCode.ANALYSIS_ERROR, response.getStatus().getStatusCode()); + Assertions.assertEquals(code.getCode(), response.getWindowNotReady().getErrorCode()); + // Exercise the optional fields over the actual Thrift encoding before the follower receives them. + TAcquireTimeBasedChangeReadFenceResult decoded = new TAcquireTimeBasedChangeReadFenceResult(); + new TDeserializer().deserialize(decoded, new TSerializer().serialize(response)); + IncrWindowNotReadyException propagated = receiveWindowErrorOnFollower(decoded); + Assertions.assertEquals(code, propagated.getMysqlErrorCode()); + Assertions.assertEquals(CURRENT_TSO, propagated.getCurrentTso()); + Assertions.assertEquals(1000, propagated.getTimeoutMs()); + Assertions.assertEquals(reason, propagated.getReason()); + } + } + + @Test + public void testFollowerAcceptsWindowErrorFromOlderMaster() throws Exception { + TAcquireTimeBasedChangeReadFenceResult response = new TAcquireTimeBasedChangeReadFenceResult(); + response.setStatus(new TStatus(TStatusCode.ANALYSIS_ERROR)); + response.setWindowNotReady(new TIncrWindowNotReady(CURRENT_PHYSICAL_TIME_MS, 0, 1000)); + IncrWindowNotReadyException propagated = receiveWindowErrorOnFollower(response); + Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, propagated.getMysqlErrorCode()); + Assertions.assertEquals("WINDOW_NOT_READY", propagated.getReason()); + } + + private IncrWindowNotReadyException receiveWindowErrorOnFollower(TAcquireTimeBasedChangeReadFenceResult response) + throws Exception { + Env follower = Mockito.mock(Env.class); + Mockito.when(follower.getMasterHost()).thenReturn("127.0.0.1"); + Mockito.when(follower.getMasterRpcPort()).thenReturn(9020); + ConnectContext context = mockContext(); + Mockito.when(context.getEnv()).thenReturn(follower); + FrontendService.Client client = Mockito.mock(FrontendService.Client.class); + Mockito.when(client.acquireTimeBasedChangeReadFence(Mockito.any())).thenReturn(response); + GenericPool originalPool = ClientPool.frontendPool; + GenericPool pool = Mockito.mock(GenericPool.class); + Mockito.when(pool.borrowObject(Mockito.any(), Mockito.anyInt())).thenReturn(client); + ClientPool.frontendPool = pool; + try (MockedStatic config = Mockito.mockStatic(Config.class)) { + config.when(Config::isCloudMode).thenReturn(true); + IncrWindowNotReadyException error = Assertions.assertThrows(IncrWindowNotReadyException.class, + () -> TimeBasedChangeVisibleWaiter.waitForVisible(context, + newChangeRelation(1, ImmutableMap.of(OlapScanNode.OLAP_END_TIMESTAMP, "2024-01-01 00:00:00")), + ImmutableMap.of(TABLE_QUALIFIER, mockOlapTable(DB_ID, TABLE_ID)))); + Assertions.assertEquals(CURRENT_PHYSICAL_TIME_MS, error.getRequestedEndTimestampMs()); + Assertions.assertEquals(0, error.getCommittedTso()); + Mockito.verify(pool).borrowObject(Mockito.any(), Mockito.eq(2000)); + Mockito.verify(pool).returnObject(Mockito.any(), Mockito.eq(client)); + return error; + } finally { + ClientPool.frontendPool = originalPool; + } + } + private ConnectContext mockContext() { ConnectContext context = Mockito.mock(ConnectContext.class); SessionVariable sessionVariable = new SessionVariable(); @@ -228,10 +380,12 @@ private Env mockMasterEnv() { return env; } - private TSOService mockTsoService(Env env, long currentTso) { + private TSOService mockTsoService(Env env, long currentTso) throws UserException { TSOService tsoService = Mockito.mock(TSOService.class); Mockito.when(tsoService.getStatusSnapshot()).thenReturn( new TSOService.TSOStatusSnapshot(true, currentTso, CURRENT_PHYSICAL_TIME_MS + 1000)); + Mockito.when(tsoService.waitForReadableWindow(Mockito.anyMap(), Mockito.anyLong(), Mockito.anyLong())) + .thenAnswer(invocation -> tsoService.getStatusSnapshot()); Mockito.when(env.getTSOService()).thenReturn(tsoService); return tsoService; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/TsoStatusMetadataGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/TsoStatusMetadataGeneratorTest.java index dc8761c5ac3a5f..753bbe920c5be0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/TsoStatusMetadataGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/TsoStatusMetadataGeneratorTest.java @@ -74,6 +74,7 @@ public void testTsoStatusResult() throws Exception { Assertions.assertEquals(TStatusCode.OK, result.getStatus().getStatusCode()); Assertions.assertEquals(1, result.getDataBatchSize()); TRow row = result.getDataBatch().get(0); + Assertions.assertEquals(4, row.getColumnValueSize()); // Legacy BE request. Assertions.assertEquals(windowEndPhysicalTime, row.getColumnValue().get(0).getLongVal()); Assertions.assertEquals(currentTso, row.getColumnValue().get(1).getLongVal()); Assertions.assertEquals(physicalTime, row.getColumnValue().get(2).getLongVal()); @@ -123,6 +124,25 @@ public void testNotCalibrated() throws Exception { Mockito.verify(tsoService).getStatusSnapshot(); } + @Test + public void testCommittedTsoProjectionAndUnknownPrefix() throws Exception { + long committed = TSOTimestamp.composeTimestamp(1700000000000L, 17); + TFetchSchemaTableDataRequest request = newRequest(); + request.getSchemaTableParams().setColumnsName( + ImmutableList.of("committed_tso_physical_time", "committed_tso")); + Mockito.when(tsoService.getStatusSnapshot()).thenReturn( + new TSOService.TSOStatusSnapshot(true, committed + 100, 1700000001000L, committed), + new TSOService.TSOStatusSnapshot(true, committed + 100, 1700000001000L, 0)); + TRow row = MetadataGenerator.getSchemaTableData(request).getDataBatch().get(0); + Assertions.assertEquals(2, row.getColumnValueSize()); + Assertions.assertEquals(1700000000000L, row.getColumnValue().get(0).getLongVal()); + Assertions.assertEquals(committed, row.getColumnValue().get(1).getLongVal()); + TRow unknown = MetadataGenerator.getSchemaTableData(request).getDataBatch().get(0); + Assertions.assertTrue(unknown.getColumnValue().get(0).isIsNull()); + Assertions.assertTrue(unknown.getColumnValue().get(1).isIsNull()); + Mockito.verify(tsoService, Mockito.never()).getTSO(); + } + private TFetchSchemaTableDataRequest newRequest() { TFetchSchemaTableDataRequest request = new TFetchSchemaTableDataRequest(); request.setSchemaTableName(TSchemaTableName.TSO_STATUS); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java index a591bc9f3592dc..0c95b98d475ac2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java @@ -19,12 +19,19 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; import org.apache.doris.common.io.CountingDataOutputStream; import org.apache.doris.journal.Journal; +import org.apache.doris.journal.JournalEntity; import org.apache.doris.metric.LongCounterMetric; import org.apache.doris.metric.MetricRepo; import org.apache.doris.persist.EditLog; +import org.apache.doris.persist.OperationType; +import org.apache.doris.qe.TimeBasedChangeVisibleWaiter; +import org.apache.doris.transaction.GlobalTransactionMgrIface; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -36,10 +43,22 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; /** * Unit tests for TSOService class. @@ -48,6 +67,7 @@ public class TSOServiceTest { private TSOService tsoService; private Env env; + private GlobalTransactionMgrIface globalTxnMgr; private MockedStatic mockedEnv; private int originalMaxGetTSORetryCount; @@ -57,7 +77,7 @@ public class TSOServiceTest { private long originalClockBackwardThresholdMs; @BeforeEach - public void setUp() { + public void setUp() throws Exception { mockedEnv = Mockito.mockStatic(Env.class); originalMaxGetTSORetryCount = Config.tso_max_get_retry_count; @@ -73,7 +93,11 @@ public void setUp() { Config.tso_clock_backward_startup_threshold_ms = 30L * 60 * 1000; env = Mockito.mock(Env.class); + globalTxnMgr = Mockito.mock(GlobalTransactionMgrIface.class); mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getGlobalTransactionMgr()).thenReturn(globalTxnMgr); + Mockito.when(globalTxnMgr.advanceTsoFence(Mockito.anyLong())) + .thenAnswer(invocation -> invocation.getArgument(0)); tsoService = new TSOService(); } @@ -239,7 +263,8 @@ public void testRunAfterCatalogReadyUpdateFailureDoesNotTouchMetricWhenNotInit() MetricRepo.isInit = false; MetricRepo.COUNTER_TSO_CLOCK_UPDATE_FAILED = null; Mockito.when(env.isReady()).thenReturn(true); - Mockito.when(env.isMaster()).thenThrow(new RuntimeException("injected update failure")); + Mockito.when(env.isMaster()).thenReturn(true) + .thenThrow(new RuntimeException("injected update failure")); tsoService.runAfterCatalogReady(); } finally { Config.enable_feature_binlog = originalEnableFeatureBinlog; @@ -251,7 +276,7 @@ public void testRunAfterCatalogReadyUpdateFailureDoesNotTouchMetricWhenNotInit() @Test public void testReplayWindowEndTSOUpdatesServiceState() { long windowEnd = 12345L; - tsoService.replayWindowEndTSO(new TSOTimestamp(windowEnd, 0L)); + tsoService.replayWindowEndTSO(new TSOServiceState(windowEnd, 0L)); Assertions.assertEquals(windowEnd, tsoService.getWindowEndTSO()); } @@ -261,7 +286,7 @@ public void testSaveTSOPersistsWindowEndWhenBinlogEnabled() throws IOException { try { Config.enable_feature_binlog = true; long windowEnd = 12345L; - tsoService.replayWindowEndTSO(new TSOTimestamp(windowEnd, 0L)); + tsoService.replayWindowEndTSO(new TSOServiceState(windowEnd, 0L)); byte[] bytes = saveTSOBytes(tsoService); Assertions.assertTrue(bytes.length > 0); @@ -320,7 +345,7 @@ public void testWriteTimestampToBdbJeWritesWhenEnabledAndJournalReady() throws E Mockito.when(editLog.getJournal()).thenReturn(journal); invokeWriteTimestampToBdbJe(tsoService, 123L); - Mockito.verify(editLog).logTSOTimestampWindowEnd(Mockito.any(TSOTimestamp.class)); + Mockito.verify(editLog).logTSOTimestampWindowEnd(Mockito.any(TSOServiceState.class)); } @Test @@ -352,7 +377,7 @@ public void testCalibrateTimestampThrowsWhenPersistWriteFailsAndKeepNotInitializ TSOService.TSOStatusSnapshot statusSnapshot = tsoService.getStatusSnapshot(); Assertions.assertFalse(statusSnapshot.isInitialized()); - Assertions.assertTrue(statusSnapshot.getCurrentTso() > 0L); + Assertions.assertEquals(0L, statusSnapshot.getCurrentTso()); Assertions.assertEquals(0L, statusSnapshot.getWindowEndPhysicalTime()); try { @@ -371,7 +396,7 @@ public void testCalibrateTimestampThrowsWhenClockBackwardExceedsThreshold() thro Mockito.when(env.isReady()).thenReturn(true); Mockito.when(env.isMaster()).thenReturn(true); long now = System.currentTimeMillis() + Config.tso_time_offset_debug_mode; - tsoService.replayWindowEndTSO(new TSOTimestamp( + tsoService.replayWindowEndTSO(new TSOServiceState( now + Config.tso_clock_backward_startup_threshold_ms + 60_000, 0L)); try { invokeCalibrateTimestamp(tsoService); @@ -393,6 +418,174 @@ public void testCalibrateTimestampResetsFatalClockBackwardReportedOnSuccess() th Assertions.assertFalse(getFatalClockBackwardReportedFlag(tsoService)); } + @Test + public void testCloudCalibrationPersistsWindowBeforePublishingFenceAndCatchesUp() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + mockPersistReady(); + AtomicBoolean firstRequest = new AtomicBoolean(true); + Mockito.when(globalTxnMgr.advanceTsoFence(Mockito.anyLong())).thenAnswer(invocation -> { + long proposed = invocation.getArgument(0); + Assertions.assertFalse(tsoService.getStatusSnapshot().isInitialized()); + Assertions.assertTrue(tsoService.getWindowEndTSO() + >= TSOTimestamp.extractPhysicalTime(proposed) + Config.tso_service_window_duration_ms); + if (firstRequest.getAndSet(false)) { + return TSOTimestamp.composePhysicalTimestamp(TSOTimestamp.extractPhysicalTime(proposed) + 10); + } + return proposed; + }); + + try (MockedStatic config = Mockito.mockStatic(Config.class)) { + config.when(Config::isCloudMode).thenReturn(true); + invokeCalibrateTimestamp(tsoService); + } + + Mockito.verify(globalTxnMgr, Mockito.times(2)).advanceTsoFence(Mockito.anyLong()); + Assertions.assertTrue(tsoService.getStatusSnapshot().isInitialized()); + } + + @Test + public void testCloudCalibrationRetriesThePersistedFenceAfterRpcFailure() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + mockPersistReady(); + AtomicBoolean failFirstRequest = new AtomicBoolean(true); + long[] proposedFence = {-1}; + Mockito.when(globalTxnMgr.advanceTsoFence(Mockito.anyLong())).thenAnswer(invocation -> { + long proposed = invocation.getArgument(0); + if (failFirstRequest.getAndSet(false)) { + proposedFence[0] = proposed; + throw new UserException("injected fence RPC failure"); + } + Assertions.assertEquals(proposedFence[0], proposed); + return proposed; + }); + + try (MockedStatic config = Mockito.mockStatic(Config.class)) { + config.when(Config::isCloudMode).thenReturn(true); + Assertions.assertThrows(InvocationTargetException.class, + () -> invokeCalibrateTimestamp(tsoService)); + invokeCalibrateTimestamp(tsoService); + } + + Mockito.verify(env.getEditLog(), Mockito.times(1)) + .logTSOTimestampWindowEnd(Mockito.any()); + Assertions.assertTrue(tsoService.getStatusSnapshot().isInitialized()); + } + + @Test + public void testFencedCommitRetryReplacesRegisteredTso() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + long rejectedTso = tsoService.getCommitTSO(1, 10, Collections.singleton(100L)); + long fenceTso = TSOTimestamp.composePhysicalTimestamp(101); + setGlobalTimestamp(tsoService, 101, 0); + + long replacementTso = tsoService.getCommitTSOAfterFence( + 1, 10, Set.of(100L, 200L), rejectedTso, fenceTso); + + Assertions.assertTrue(replacementTso > fenceTso); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); + Assertions.assertEquals(replacementTso, tracker.getOldestPendingTso()); + Assertions.assertEquals(1, tracker.getPendingCount()); + } + + @Test + public void testFencedCommitRetryRegistersTransactionOnNewMaster() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + setInitializedFlag(tsoService, true); + long fenceTso = TSOTimestamp.composePhysicalTimestamp(101); + setGlobalTimestamp(tsoService, 101, 0); + + long replacementTso = tsoService.getCommitTSOAfterFence( + 1, 10, Set.of(100L), TSOTimestamp.composePhysicalTimestamp(100), fenceTso); + + Assertions.assertTrue(replacementTso > fenceTso); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); + Assertions.assertEquals(replacementTso, tracker.getOldestPendingTso()); + Assertions.assertEquals(1, tracker.getPendingCount()); + } + + @Test + public void testFencedCommitRetryDoesNotAdvanceStaleMasterClock() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + long fenceTso = TSOTimestamp.composePhysicalTimestamp(101); + + RuntimeException failure = Assertions.assertThrows(RuntimeException.class, + () -> tsoService.getCommitTSOAfterFence(1, 10, Set.of(100L), + TSOTimestamp.composePhysicalTimestamp(100), fenceTso)); + + Assertions.assertTrue(failure.getMessage().contains("TXN_COMMIT_TSO_FENCED")); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + Assertions.assertEquals(0, + ((TSOTransactionTracker) trackerField.get(tsoService)).getPendingCount()); + } + + @Test + public void testUncertainCommitIsReleasedAfterFence() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + long commitTso = tsoService.getCommitTSO(1, 10, Collections.singleton(100L)); + + tsoService.fenceAndAbandonCommitTso(1, 10, commitTso); + + Mockito.verify(globalTxnMgr).advanceTsoFence(commitTso); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + Assertions.assertEquals(0, + ((TSOTransactionTracker) trackerField.get(tsoService)).getPendingCount()); + } + + @Test + public void testFenceFailureDeactivatesTsoService() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + long commitTso = tsoService.getCommitTSO(1, 10, Collections.singleton(100L)); + Mockito.when(globalTxnMgr.advanceTsoFence(commitTso)).thenThrow(new UserException("injected failure")); + + Assertions.assertThrows(UserException.class, + () -> tsoService.fenceAndAbandonCommitTso(1, 10, commitTso)); + + Assertions.assertFalse(tsoService.getStatusSnapshot().isInitialized()); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + Assertions.assertEquals(0, + ((TSOTransactionTracker) trackerField.get(tsoService)).getPendingCount()); + } + + @Test + public void testDemotionInvalidatesAllocationState() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + tsoService.getCommitTSO(1, 10, Collections.singleton(100L)); + Mockito.when(env.isMaster()).thenReturn(false); + + tsoService.runAfterCatalogReady(); + + Assertions.assertFalse(tsoService.getStatusSnapshot().isInitialized()); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + Assertions.assertEquals(0, + ((TSOTransactionTracker) trackerField.get(tsoService)).getPendingCount()); + } + @Test public void testRunAfterCatalogReadySkipsWhenBinlogDisabled() throws Exception { Config.enable_feature_binlog = false; @@ -406,7 +599,7 @@ public void testUpdateTimestampReturnsEarlyWhenNotCalibrated() throws Exception Mockito.when(env.isReady()).thenReturn(true); Mockito.when(env.isMaster()).thenReturn(true); long initialWindowEnd = 12345L; - tsoService.replayWindowEndTSO(new TSOTimestamp(initialWindowEnd, 0L)); + tsoService.replayWindowEndTSO(new TSOServiceState(initialWindowEnd, 0L)); invokeUpdateTimestamp(tsoService); @@ -436,6 +629,214 @@ public void testGenerateTSOReturnsZeroWhenDisabledOrNotInitialized() throws Exce } } + @Test + public void testReadableWindowDoesNotRequireAnotherCommittedTsoFlush() throws Exception { + Mockito.when(env.isMaster()).thenReturn(true); + Mockito.when(env.getTSOService()).thenReturn(tsoService); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 1000, 17); + long committed = TSOTimestamp.composeTimestamp(900, 1); + tsoService.replayWindowEndTSO(new TSOServiceState(2000, committed)); + try (MockedStatic config = Mockito.mockStatic(Config.class)) { + config.when(Config::isCloudMode).thenReturn(true); + TimeBasedChangeVisibleWaiter.ChangeReadFence fence = TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( + Collections.singletonMap(1L, Collections.singletonList(2L)), 950L, 0, true, true); + Assertions.assertEquals(committed, fence.getCommittedTso()); + } + } + + private void prepareWindowRead() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + Mockito.when(env.getTSOService()).thenReturn(tsoService); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + tsoService.replayWindowEndTSO(new TSOServiceState(2000, TSOTimestamp.composeTimestamp(80, 1))); + } + + @Test + public void testSlowTableDoesNotBlockAnotherTableOrAnEarlierEnd() throws Exception { + prepareWindowRead(); + long pending = tsoService.getCommitTSO(1, 10, Set.of(100L, 101L)); + setGlobalTimestamp(tsoService, 150, 17); + TSOService.TSOStatusSnapshot unrelated = tsoService.waitForReadableWindow( + Map.of(1L, Collections.singletonList(200L)), 120, 0); + Assertions.assertTrue(unrelated.getCommittedTso() < pending); + tsoService.waitForReadableWindow(Map.of(1L, Collections.singletonList(100L)), 100, 0); + for (long table : new long[] {100, 101}) { + IncrWindowNotReadyException error = Assertions.assertThrows(IncrWindowNotReadyException.class, + () -> tsoService.waitForReadableWindow(Map.of(1L, Collections.singletonList(table)), 120, 0)); + Assertions.assertEquals(ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT, error.getMysqlErrorCode()); + Assertions.assertEquals("VISIBLE_WAIT_TIMEOUT", error.getReason()); + Assertions.assertEquals(120, error.getRequestedEndTimestampMs()); + Assertions.assertEquals(tsoService.getCurrentTSO(), error.getCurrentTso()); + Assertions.assertEquals(unrelated.getCommittedTso(), error.getCommittedTso()); + } + tsoService.transactionFinished(1, 10); + TSOService.TSOStatusSnapshot finished = tsoService.waitForReadableWindow( + Map.of(1L, Collections.singletonList(100L)), 120, 0); + // A table-specific successful read does not advance the global durable prefix. + Assertions.assertEquals(unrelated.getCommittedTso(), finished.getCommittedTso()); + } + + @Test + public void testFutureWindowHasDifferentReasonFromWaitTimeout() throws Exception { + prepareWindowRead(); + Map> tables = Map.of(1L, Collections.singletonList(100L)); + IncrWindowNotReadyException future = Assertions.assertThrows(IncrWindowNotReadyException.class, + () -> tsoService.waitForReadableWindow(tables, 101, 0)); + Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, future.getMysqlErrorCode()); + Assertions.assertEquals("END_AFTER_CURRENT_TSO", future.getReason()); + tsoService.waitForReadableWindow(tables, 90, 0); + tsoService.waitForReadableWindow(tables, 80, 0); + } + + @Test + public void testInterruptedReadWaitRestoresInterruptFlag() throws Exception { + prepareWindowRead(); + tsoService.getCommitTSO(1, 10, Collections.singleton(100L)); + setGlobalTimestamp(tsoService, 150, 17); + try { + Thread.currentThread().interrupt(); + UserException error = Assertions.assertThrows(UserException.class, () -> tsoService.waitForReadableWindow( + Map.of(1L, Collections.singletonList(100L)), 120, 1000)); + Assertions.assertTrue(error.getDetailMessage().contains("interrupted")); + Assertions.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + + @Test + public void testCommittedPrefixIsPublishedOnlyAfterJournalSuccess() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(editLog.getJournal()).thenReturn(Mockito.mock(Journal.class)); + Mockito.when(env.getEditLog()).thenReturn(editLog); + tsoService.replayWindowEndTSO(new TSOServiceState(200, 80)); + setGlobalTimestamp(tsoService, 100, 10); + try (MockedStatic config = Mockito.mockStatic(Config.class)) { + config.when(Config::isCloudMode).thenReturn(true); + Mockito.doAnswer(invocation -> { + Assertions.assertEquals(80, tsoService.getStatusSnapshot().getCommittedTso()); + Assertions.assertEquals(200, tsoService.getStatusSnapshot().getWindowEndPhysicalTime()); + throw new RuntimeException("injected journal failure"); + }).when(editLog).logTSOTimestampWindowEnd(Mockito.any()); + Assertions.assertThrows(RuntimeException.class, () -> invokeWriteTimestampToBdbJe(tsoService, 300)); + Assertions.assertEquals(80, tsoService.getStatusSnapshot().getCommittedTso()); + Assertions.assertEquals(200, tsoService.getStatusSnapshot().getWindowEndPhysicalTime()); + Mockito.doNothing().when(editLog).logTSOTimestampWindowEnd(Mockito.any()); + invokeWriteTimestampToBdbJe(tsoService, 300); + Assertions.assertEquals(tsoService.getCurrentTSO(), tsoService.getStatusSnapshot().getCommittedTso()); + Assertions.assertEquals(300, tsoService.getStatusSnapshot().getWindowEndPhysicalTime()); + } + } + + @Test + public void testCalibrationKeepsPrefixAndPeriodicFlushWorksWithUnchangedWindow() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + mockPersistReady(); + long oldWindow = System.currentTimeMillis() + 60_000; + long oldCommitted = TSOTimestamp.composeTimestamp(oldWindow - 1000, 7); + tsoService.replayWindowEndTSO(new TSOServiceState(oldWindow, oldCommitted)); + try (MockedStatic config = Mockito.mockStatic(Config.class)) { + config.when(Config::isCloudMode).thenReturn(true); + invokeCalibrateTimestamp(tsoService); + Assertions.assertEquals(oldCommitted, tsoService.getStatusSnapshot().getCommittedTso()); + long pendingTso = tsoService.getCommitTSO(1, 10, Collections.singleton(2L)); + long reservedWindow = tsoService.getWindowEndTSO(); + Field lastPersist = TSOService.class.getDeclaredField("lastPersistNanos"); + lastPersist.setAccessible(true); + lastPersist.setLong(tsoService, System.nanoTime() + - TimeUnit.MILLISECONDS.toNanos(Config.tso_service_window_duration_ms + 1L)); + invokeUpdateTimestamp(tsoService); + Assertions.assertEquals(reservedWindow, tsoService.getWindowEndTSO()); + Assertions.assertEquals(pendingTso - 1, tsoService.getStatusSnapshot().getCommittedTso()); + tsoService.transactionFinished(1, 10); + lastPersist.setLong(tsoService, System.nanoTime() + - TimeUnit.MILLISECONDS.toNanos(Config.tso_service_window_duration_ms + 1L)); + invokeUpdateTimestamp(tsoService); + Assertions.assertEquals(reservedWindow, tsoService.getWindowEndTSO()); + Assertions.assertEquals(pendingTso, tsoService.getStatusSnapshot().getCommittedTso()); + } + } + + @Test + public void testStateImageJournalAndOldTimestampCompatibility() throws Exception { + long committed = TSOTimestamp.composeTimestamp(100, 17); + tsoService.replayWindowEndTSO(new TSOServiceState(200, committed)); + TSOService restored = new TSOService(); + Assertions.assertEquals(200, restored.loadTSO( + new DataInputStream(new ByteArrayInputStream(saveTSOBytes(tsoService))), 0)); + Assertions.assertEquals(committed, restored.getStatusSnapshot().getCommittedTso()); + Assertions.assertFalse(restored.getStatusSnapshot().isInitialized()); + for (boolean legacy : new boolean[] {true, false}) { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(bytes); + out.writeShort(OperationType.OP_TSO_TIMESTAMP_WINDOW_END); + if (legacy) { + new TSOTimestamp(200, 0).write(out); + } else { + new TSOServiceState(200, committed).write(out); + } + JournalEntity entity = new JournalEntity(); + entity.readFields(new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))); + TSOServiceState state = (TSOServiceState) entity.getData(); + Assertions.assertEquals(200, state.getPhysicalTimestamp()); + Assertions.assertEquals(legacy ? 0 : committed, state.getCommittedTso()); + } + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + new TSOServiceState(200, committed).write(new DataOutputStream(bytes)); + Assertions.assertEquals(200, TSOTimestamp.read( + new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))).getPhysicalTimestamp()); + } + + @Test + public void testAllocationAndRegistrationShareSnapshotLock() throws Exception { + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + Field lockField = TSOService.class.getDeclaredField("lock"); + lockField.setAccessible(true); + ReentrantLock lock = (ReentrantLock) lockField.get(tsoService); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + TSOTransactionTracker tracker = Mockito.spy((TSOTransactionTracker) trackerField.get(tsoService)); + trackerField.set(tsoService, tracker); + CountDownLatch enteredRegistration = new CountDownLatch(1); + CountDownLatch releaseRegistration = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + enteredRegistration.countDown(); + Assertions.assertTrue(releaseRegistration.await(30, TimeUnit.SECONDS)); + return invocation.callRealMethod(); + }).when(tracker).register(Mockito.any(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anySet()); + Method generate = TSOService.class.getDeclaredMethod("generateTSO", Pair.class, Set.class); + generate.setAccessible(true); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future allocation = executor.submit(() -> { + try { + generate.invoke(tsoService, Pair.of(1L, 10L), Collections.singleton(2L)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + Assertions.assertTrue(enteredRegistration.await(30, TimeUnit.SECONDS)); + boolean snapshotLockAcquired = lock.tryLock(); + if (snapshotLockAcquired) { + lock.unlock(); + } + Assertions.assertFalse(snapshotLockAcquired, "snapshot must not pass an unregistered allocation"); + releaseRegistration.countDown(); + allocation.get(30, TimeUnit.SECONDS); + Assertions.assertEquals(TSOTimestamp.composeTimestamp(100, 1), tracker.getOldestPendingTso()); + } finally { + releaseRegistration.countDown(); + executor.shutdownNow(); + } + } + private static void invokeWriteTimestampToBdbJe(TSOService service, long timestamp) throws Exception { Method m = TSOService.class.getDeclaredMethod("writeTimestampToBDBJE", long.class); m.setAccessible(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java new file mode 100644 index 00000000000000..f2c7d850a0cc70 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tso; + +import org.apache.doris.common.Pair; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +public class TSOTransactionTrackerTest { + private ReentrantLock lock; + private TSOTransactionTracker tracker; + private ExecutorService executor; + + @BeforeEach + public void setUp() { + lock = new ReentrantLock(); + tracker = new TSOTransactionTracker(lock); + executor = Executors.newSingleThreadExecutor(); + lock.lock(); + try { + tracker.reset(); + } finally { + lock.unlock(); + } + } + + @AfterEach + public void tearDown() { + executor.shutdownNow(); + } + + private void register(long dbId, long txnId, long tso, long... tableIds) { + Set tables = new java.util.HashSet<>(); + for (long tableId : tableIds) { + tables.add(tableId); + } + lock.lock(); + try { + tracker.register(Pair.of(dbId, txnId), tso, System.nanoTime(), tables); + } finally { + lock.unlock(); + } + } + + private TSOTransactionTracker.WaitResult await(long dbId, long tableId, long endTso, long timeoutMs) + throws InterruptedException { + lock.lock(); + try { + return tracker.awaitTransactions(Map.of(dbId, List.of(tableId)), endTso, + TimeUnit.MILLISECONDS.toNanos(timeoutMs)); + } finally { + lock.unlock(); + } + } + + private long candidate(long currentTso, long durableTso) { + lock.lock(); + try { + return tracker.candidateCommittedTso(currentTso, durableTso); + } finally { + lock.unlock(); + } + } + + @Test + public void testWaitFiltersDatabaseTableAndEndTso() throws Exception { + register(1, 10, 100, 1000); + register(1, 20, 120, 2000); + register(2, 30, 130, 1000); + + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, await(1, 2000, 110, 0)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, await(2, 1000, 120, 0)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, await(1, 1000, 100, 0)); + Assertions.assertEquals(99, candidate(150, 90)); + + tracker.transactionFinished(1, 10); + Assertions.assertEquals(119, candidate(150, 90)); + tracker.transactionFinished(1, 20); + tracker.transactionFinished(2, 30); + Assertions.assertEquals(150, candidate(150, 90)); + } + + @Test + public void testWaitUsesFixedRegistrationSnapshot() throws Exception { + register(1, 10, 100, 1000); + Future waiting = executor.submit(() -> await(1, 1000, 150, 5000)); + Thread.sleep(100); + register(1, 20, 120, 1000); + tracker.transactionFinished(1, 10); + + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, waiting.get(5, TimeUnit.SECONDS)); + Assertions.assertEquals(119, candidate(150, 90)); + } + + @Test + public void testResetInvalidatesWait() throws Exception { + register(1, 10, 100, 1000); + Future waiting = executor.submit(() -> await(1, 1000, 150, 5000)); + Thread.sleep(100); + lock.lock(); + try { + tracker.reset(); + } finally { + lock.unlock(); + } + + Assertions.assertEquals(TSOTransactionTracker.WaitResult.RESET, waiting.get(5, TimeUnit.SECONDS)); + } + + @Test + public void testFencedReplacementAndAttemptScopedAbandon() { + register(1, 10, 100, 1000); + lock.lock(); + try { + tracker.replaceFenced(Pair.of(1L, 10L), 100, 110, 120, + System.nanoTime(), Collections.singleton(2000L)); + } finally { + lock.unlock(); + } + + Assertions.assertEquals(119, candidate(150, 90)); + tracker.abandonCommitTso(1, 10, 100); + Assertions.assertEquals(1, tracker.getPendingCount()); + tracker.abandonCommitTso(1, 10, 120); + Assertions.assertEquals(0, tracker.getPendingCount()); + } + + @Test + public void testRepeatedRegistrationRetainsEarliestTso() { + register(1, 10, 100, 1000); + register(1, 10, 120, 2000); + + Assertions.assertEquals(1, tracker.getPendingCount()); + Assertions.assertEquals(100, tracker.getOldestPendingTso()); + Assertions.assertEquals(10, tracker.getOldestPendingTxnId()); + Assertions.assertTrue(tracker.getOldestPendingAgeMs() >= 0); + tracker.abandonCommitTso(1, 10, 120); + Assertions.assertEquals(1, tracker.getPendingCount()); + tracker.transactionFinished(1, 10); + Assertions.assertEquals(0, tracker.getPendingCount()); + } +} diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index c1657361a951a8..8b99c3de402666 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -514,6 +514,11 @@ message TxnIndexPB { optional int64 parent_txn_id = 2; } +// Current-state commit TSO fence. This key is intentionally not versioned. +message TxnTsoFencePB { + optional int64 fence_tso = 1; +} + message TxnInfoPB { optional int64 db_id = 1; repeated int64 table_ids = 2; @@ -1115,6 +1120,8 @@ message CommitTxnResponse { // The lazy commit has only completed its first phase. FE must not notify BE to make // temporary rowsets visible until BE observes the final metadata from meta-service. optional bool is_lazy_commit_incomplete = 9; + // The current master fence when commit_tso is rejected as stale. + optional int64 tso_fence = 10; } message AbortTxnRequest { @@ -1256,6 +1263,18 @@ message CheckTxnConflictResponse { repeated TxnInfoPB conflict_txns = 3; } +message AdvanceTsoFenceRequest { + optional string cloud_unique_id = 1; // For auth + // Inclusive upper bound rejected by transaction commit. + optional int64 proposed_fence_tso = 2; + optional string request_ip = 3; +} + +message AdvanceTsoFenceResponse { + optional MetaServiceResponseStatus status = 1; + optional int64 tso_fence = 2; +} + message CleanTxnLabelRequest { optional string cloud_unique_id = 1; // For auth optional int64 db_id = 2; @@ -1949,6 +1968,8 @@ enum MetaServiceCode { STALE_TABLET_CACHE = 2012; STALE_PREPARE_ROWSET = 2013; TXN_ALREADY_COMMITED = 2014; + // The transaction commit TSO is no longer valid. + TXN_COMMIT_TSO_FENCED = 2015; CLUSTER_NOT_FOUND = 3001; ALREADY_EXISTED = 3002; @@ -2427,6 +2448,7 @@ service MetaService { rpc get_current_max_txn_id(GetCurrentMaxTxnRequest) returns (GetCurrentMaxTxnResponse); rpc create_meta_sync_point(CreateMetaSyncPointRequest) returns (CreateMetaSyncPointResponse); rpc check_txn_conflict(CheckTxnConflictRequest) returns (CheckTxnConflictResponse); + rpc advance_tso_fence(AdvanceTsoFenceRequest) returns (AdvanceTsoFenceResponse); rpc clean_txn_label(CleanTxnLabelRequest) returns (CleanTxnLabelResponse); rpc get_txn_id(GetTxnIdRequest) returns (GetTxnIdResponse); rpc begin_sub_txn(BeginSubTxnRequest) returns (BeginSubTxnResponse); diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 1829181a875f63..fb3f4c3860b8ba 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1994,12 +1994,27 @@ struct TAcquireTimeBasedChangeReadFenceRequest { 2: optional i64 end_timestamp_ms 3: required i64 timeout_ms 4: required bool wait_for_transactions + // False/absent retains the legacy drain for old callers and unbounded relations. + 5: optional bool all_ends_explicit +} + +struct TIncrWindowNotReady { + 1: required i64 requested_end_timestamp_ms + 2: required i64 committed_tso + 3: required i64 retry_after_ms + 4: optional i64 current_tso + // Absent on older masters: ERR_INCR_WINDOW_NOT_READY (5100). + 5: optional i32 error_code + 6: optional i64 timeout_ms + 7: optional string reason } struct TAcquireTimeBasedChangeReadFenceResult { 1: required Status.TStatus status 2: optional i64 current_tso 3: optional i64 max_journal_id + 4: optional i64 committed_tso + 5: optional TIncrWindowNotReady window_not_ready } service FrontendService { diff --git a/regression-test/data/tso_p0/test_committed_tso.out b/regression-test/data/tso_p0/test_committed_tso.out new file mode 100644 index 00000000000000..fe1a30cdf41211 --- /dev/null +++ b/regression-test/data/tso_p0/test_committed_tso.out @@ -0,0 +1,23 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !readable_prefix -- +true true true + +-- !historical_window -- +1 +2 + +-- !pending_constrains_prefix -- +true + +-- !history_with_later_write -- +1 +2 + +-- !unrelated_table_above_prefix -- +10 +20 + +-- !retry_original_window -- +1 +2 +3 diff --git a/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy b/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy index 2bcd20b42d4549..70a8c538aefd1d 100644 --- a/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy +++ b/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy @@ -28,6 +28,18 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { def mowPartialTable = "changes_mow_partial" def mowBitmapTable = "changes_mow_bitmap" def incrTimeFormat = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + def fullWindowEnd = { + long current = (sql "SELECT CURRENT_TSO_PHYSICAL_TIME FROM information_schema.tso_status")[0][0] as long + long minimumEnd = (Math.floorDiv(current, 1000L) + 1L) * 1000L + String column = isCloudMode() ? "COMMITTED_TSO_PHYSICAL_TIME" : "CURRENT_TSO_PHYSICAL_TIME" + long readable = 0L + awaitUntil(60, 0.1) { + def value = (sql "SELECT ${column} FROM information_schema.tso_status")[0][0] + readable = value == null ? 0L : value as long + readable >= minimumEnd + } + incrTimeFormat.format(new Date(readable)) + } try { sql "DROP TABLE IF EXISTS ${dupTable}" @@ -80,7 +92,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { sql "INSERT INTO ${dupTable} VALUES (3, 31, 'c2', NULL)" sql "sync" sleep(1200) - def dupT1 = incrTimeFormat.format(new Date()) + def dupT1 = fullWindowEnd() sleep(1200) sql "INSERT INTO ${dupTable} VALUES (5, 50, 'd', 'w')" sql "sync" @@ -161,12 +173,8 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { ORDER BY __DORIS_BINLOG_TSO__, __DORIS_BINLOG_LSN__ """ - // - CURRENT_TSO_PHYSICAL_TIME converted to an INCR timestamp is accepted. - sleep(1200) - def currentTsoPhysicalTime = Long.parseLong(sql(""" - SELECT CURRENT_TSO_PHYSICAL_TIME FROM information_schema.tso_status - """)[0][0].toString()) - def currentTsoEnd = incrTimeFormat.format(new Date(currentTsoPhysicalTime)) + // - The readable TSO physical time converted to an INCR timestamp is accepted. + def currentTsoEnd = fullWindowEnd() order_qt_dup_full_cover """ SELECT id, v1, __DORIS_BINLOG_OP__ FROM ${dupTable}@incr('startTimestamp' = '1971-01-01 00:00:00', @@ -183,7 +191,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { "endTimestamp" = "2999-01-01 00:00:00", "incrementType" = "DETAIL") """ - exception "CURRENT_TSO_PHYSICAL_TIME=" + exception (isCloudMode() ? "ERR_INCR_WINDOW_NOT_READY" : "CURRENT_TSO_PHYSICAL_TIME=") } // 1.8 Cross-check against binlog() TVF — DETAIL with full window must @@ -263,7 +271,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { sql "sync" sleep(1200) - def mowT1 = incrTimeFormat.format(new Date()) + def mowT1 = fullWindowEnd() sleep(1200) sql "INSERT INTO ${mowTable} VALUES (6, 60, 'f')" sql "sync" @@ -410,13 +418,9 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { "incrementType" = "MIN_DELTA") """ - // 2.9 CURRENT_TSO_PHYSICAL_TIME converted to an INCR timestamp covers + // 2.9 The readable TSO physical time converted to an INCR timestamp covers // all changes that were visible before it was captured. - sleep(1200) - def mowCurrentTsoPhysicalTime = Long.parseLong(sql(""" - SELECT CURRENT_TSO_PHYSICAL_TIME FROM information_schema.tso_status - """)[0][0].toString()) - def mowCurrentTsoEnd = incrTimeFormat.format(new Date(mowCurrentTsoPhysicalTime)) + def mowCurrentTsoEnd = fullWindowEnd() order_qt_mow_full_cover """ SELECT id, v1, __DORIS_BINLOG_OP__ FROM ${mowTable}@incr('startTimestamp' = '1971-01-01 00:00:00', @@ -433,7 +437,8 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { "endTimestamp" = "2999-01-01 00:00:00", "incrementType" = "DETAIL") """ - exception "endTimestamp exceeds the maximum supported time for an INCR read" + exception (isCloudMode() ? "ERR_INCR_WINDOW_NOT_READY" + : "endTimestamp exceeds the maximum supported time for an INCR read") } // ============================================================ @@ -482,7 +487,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { """ sql "sync" sleep(1200) - def seqT1 = incrTimeFormat.format(new Date()) + def seqT1 = fullWindowEnd() sleep(1200) // 3.1 DETAIL only captures physically-applied writes. The out-of-order @@ -543,7 +548,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { sql "SET enable_unique_key_partial_update = false" sql "sync" sleep(1200) - def partT1 = incrTimeFormat.format(new Date()) + def partT1 = fullWindowEnd() // 4.1 DETAIL: 3 partial updates -> 6 raw binlog rows. order_qt_part_detail """ @@ -608,7 +613,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { sql "INSERT INTO ${mowBitmapTable} VALUES (1, BITMAP_FROM_STRING('3,4'))" sql "sync" sleep(1200) - def bitmapT1 = incrTimeFormat.format(new Date()) + def bitmapT1 = fullWindowEnd() order_qt_bitmap_min_delta """ SELECT id, BITMAP_TO_STRING(b), __DORIS_BINLOG_OP__ @@ -627,7 +632,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { sql "INSERT INTO ${mowBitmapTable} VALUES (1, BITMAP_FROM_STRING('3,4'))" sql "sync" sleep(1200) - def bitmapEqualT1 = incrTimeFormat.format(new Date()) + def bitmapEqualT1 = fullWindowEnd() order_qt_bitmap_equal_min_delta """ SELECT id, BITMAP_TO_STRING(b), __DORIS_BINLOG_OP__ diff --git a/regression-test/suites/tso_p0/test_committed_tso.groovy b/regression-test/suites/tso_p0/test_committed_tso.groovy new file mode 100644 index 00000000000000..ead5f222c663da --- /dev/null +++ b/regression-test/suites/tso_p0/test_committed_tso.groovy @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.Http + +import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter +import java.util.concurrent.atomic.AtomicReference + +suite("test_committed_tso", "nonConcurrent") { + if (!isCloudMode()) { + return + } + sql "DROP TABLE IF EXISTS test_committed_tso" + sql """ + CREATE TABLE test_committed_tso (id INT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1", "binlog.enable" = "true", "binlog.format" = "ROW") + """ + sql "INSERT INTO test_committed_tso VALUES (1), (2)" + sql "DROP TABLE IF EXISTS test_committed_tso_unrelated" + sql """ + CREATE TABLE test_committed_tso_unrelated (id INT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1", "binlog.enable" = "true", "binlog.format" = "ROW") + """ + sql "INSERT INTO test_committed_tso_unrelated VALUES (10), (20)" + sql "SET change_visible_timeout_ms = 1000" + def currentPhysicalTime = { + (sql "SELECT CURRENT_TSO_PHYSICAL_TIME FROM information_schema.tso_status")[0][0] as long + } + def committedPhysicalTime = { + def value = (sql "SELECT COMMITTED_TSO_PHYSICAL_TIME FROM information_schema.tso_status")[0][0] + value == null ? 0L : value as long + } + def formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.systemDefault()) + def timestamp = { long millis -> formatter.format(Instant.ofEpochMilli(millis)) } + long historicalEnd = (Math.floorDiv(currentPhysicalTime(), 1000L) + 1L) * 1000L + awaitUntil(60, 0.1) { committedPhysicalTime() >= historicalEnd } + order_qt_readable_prefix """ + SELECT COMMITTED_TSO > 0, + bit_shift_right(COMMITTED_TSO, 18) = COMMITTED_TSO_PHYSICAL_TIME, + COMMITTED_TSO <= CURRENT_TSO + FROM information_schema.tso_status + """ + order_qt_historical_window """ + SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(historicalEnd)}') ORDER BY id + """ + + long futureEnd = currentPhysicalTime() + 60_000L + test { + sql "SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(futureEnd)}')" + exception "ERR_INCR_WINDOW_NOT_READY" + } + + def masterHttpAddress = "${getMasterIp()}:${getMasterPort()}" + def pendingMetric = { String name -> + String metrics = Http.GET("http://${masterHttpAddress}/metrics", false, false) + def matcher = metrics =~ /(?m)^doris_fe_${name}(?:\{[^\n]*\})?\s+(\d+)$/ + if (!matcher.find()) { + throw new IllegalStateException("Missing TSO metric: ${name}") + } + Long.parseLong(matcher.group(1)) + } + AtomicReference loadError = new AtomicReference<>() + Thread loadThread + try { + GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.commitTxn.blockAfterTso") + loadThread = Thread.start { + try { + sql "INSERT INTO test_committed_tso VALUES (3)" + } catch (Throwable t) { + loadError.set(t) + } + } + // Observe registration, rather than guessing when the submit thread reaches the debug point. + awaitUntil(60, 0.1) { pendingMetric("tso_pending_transactions") > 0 } + long pendingTso = pendingMetric("tso_oldest_pending_tso") + long blockedEnd = (Math.floorDiv(pendingTso >> 18, 1000L) + 1L) * 1000L + awaitUntil(60, 0.1) { currentPhysicalTime() >= blockedEnd } + order_qt_pending_constrains_prefix """ + SELECT COMMITTED_TSO < ${pendingTso} FROM information_schema.tso_status + """ + order_qt_history_with_later_write """ + SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(historicalEnd)}') ORDER BY id + """ + // The global prefix is held by another table. This closed window must remain readable. + order_qt_unrelated_table_above_prefix """ + SELECT id FROM test_committed_tso_unrelated@incr('endTimestamp' = '${timestamp(blockedEnd)}') ORDER BY id + """ + test { + sql "SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(blockedEnd)}')" + exception "ERR_INCR_VISIBLE_WAIT_TIMEOUT" + } + GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.commitTxn.blockAfterTso") + loadThread.join(60000) + if (loadThread.isAlive()) { + throw new IllegalStateException("Commit did not complete after releasing the debug point") + } + if (loadError.get() != null) { + throw loadError.get() + } + awaitUntil(60, 0.1) { committedPhysicalTime() >= blockedEnd } + // Retry exactly the same window that was refused; it must include the pending write. + order_qt_retry_original_window """ + SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(blockedEnd)}') ORDER BY id + """ + } finally { + GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.commitTxn.blockAfterTso") + if (loadThread != null) { + loadThread.join(60000) + } + } +}