From 752921347a6a0c3ff2defaf312d9ada3909ad034 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Thu, 10 Sep 2026 20:14:31 +0800 Subject: [PATCH 01/13] [fix](cloud) Gate bounded incremental reads with committed TSO ### What problem does this PR solve? Issue Number: None Related PR: #67181, #67594 Problem Summary: Strongly consistent cloud incremental reads drain earlier transactions even when those transactions fall outside the requested historical window. A delayed write can therefore block an otherwise readable window. Track commit TSO allocations and unfinished transactions under the allocator lock, and publish a readable prefix together with the reserved TSO window only after their journal write succeeds. Accepted bounded windows skip transaction watermark/conflict polling and retain the MetaService visible-version refresh. Allocate commit TSO after bitmap preparation, callbacks and metadata validation. Preserve the earliest registration across submission retries and release it only after real VISIBLE/ABORTED, including lazy-commit reconciliation. On master recovery, retain the durable prefix until a fixed, instance-wide transaction bound passes a strict MetaService check after the configured recovery delay. The fixed wait retains the agreed old-master fencing limitation. ### Release note In cloud mode, strongly consistent incremental queries with an explicit end on every incremental relation use committed TSO. Unready windows return MySQL error 5100 (ERR_INCR_WINDOW_NOT_READY), or Flight UNAVAILABLE with business metadata; clients should retry the same window. information_schema.tso_status exposes COMMITTED_TSO and COMMITTED_TSO_PHYSICAL_TIME. The TSO persistence window defaults to one second. Upgrade MetaService before enabling prefix recovery on new FEs. ### Check List (For Author) - Test: Unit Test / Regression test / Manual test - 79 distinct focused FE tests, 9 BE scanner tests and 3 MetaService tests passed. - test_committed_tso and test_binlog_changes_syntax passed; new output generated by the standard regression runner. - Real master/follower MySQL and Flight statement/prepared error contracts passed. - Three-FE failover with an allocated, PREPARED transaction preserved the old prefix and historical reads; confirmed abort released recovery and advanced the prefix. - FE build and Checkstyle, ASAN BE/Cloud builds, clang-format 16 and build hygiene passed. clang-tidy reported no changed-line diagnostics with a matching-toolchain wrapper and an analysis-only overlay for an existing unmatched suppression comment; five unchanged scanner diagnostics remain. - No throughput or latency benchmark was run. - Behavior changed: Yes (bounded cloud read admission, visible-prefix system columns, later commit TSO allocation and one-second persistence) - Does this need documentation: Yes (included docs/committed-tso.md) --- .../schema_tso_status_scanner.cpp | 10 + .../schema_tso_status_scanner_test.cpp | 34 ++- cloud/src/meta-service/meta_service_txn.cpp | 33 ++- cloud/test/meta_service_test.cpp | 75 ++++++ cloud/test/txn_lazy_commit_test.cpp | 10 + docs/committed-tso.md | 34 +++ .../java/org/apache/doris/common/Config.java | 4 +- .../org/apache/doris/common/ErrorCode.java | 3 + .../org/apache/doris/catalog/SchemaTable.java | 2 + .../CloudGlobalTransactionMgr.java | 104 ++++++-- .../common/IncrWindowNotReadyException.java | 49 ++++ .../apache/doris/journal/JournalEntity.java | 4 +- .../org/apache/doris/metric/MetricRepo.java | 17 ++ .../org/apache/doris/persist/EditLog.java | 6 +- .../org/apache/doris/qe/StmtExecutor.java | 11 +- .../qe/TimeBasedChangeVisibleWaiter.java | 57 +++- .../doris/service/FrontendServiceImpl.java | 10 +- .../arrowflight/DorisFlightSqlProducer.java | 20 +- .../tablefunction/MetadataGenerator.java | 12 +- .../GlobalTransactionMgrIface.java | 5 + .../doris/transaction/TransactionUtil.java | 3 +- .../java/org/apache/doris/tso/TSOService.java | 167 ++++++++++-- .../org/apache/doris/tso/TSOServiceState.java | 66 +++++ .../doris/tso/TSOTransactionTracker.java | 246 ++++++++++++++++++ .../transaction/CloudCommittedTsoTest.java | 137 ++++++++++ .../CloudGlobalTransactionMgrTest.java | 24 ++ .../org/apache/doris/qe/StmtExecutorTest.java | 23 ++ .../qe/TimeBasedChangeVisibleWaiterTest.java | 119 +++++++++ .../DorisFlightSqlProducerTest.java | 21 ++ .../TsoStatusMetadataGeneratorTest.java | 20 ++ .../org/apache/doris/tso/TSOServiceTest.java | 166 +++++++++++- .../doris/tso/TSOTransactionTrackerTest.java | 182 +++++++++++++ gensrc/proto/cloud.proto | 3 + gensrc/thrift/FrontendService.thrift | 10 + .../data/tso_p0/test_committed_tso.out | 20 ++ .../test_binlog_changes_syntax.groovy | 45 ++-- .../suites/tso_p0/test_committed_tso.groovy | 111 ++++++++ 37 files changed, 1757 insertions(+), 106 deletions(-) create mode 100644 docs/committed-tso.md create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/IncrWindowNotReadyException.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/tso/TSOServiceState.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java create mode 100644 regression-test/data/tso_p0/test_committed_tso.out create mode 100644 regression-test/suites/tso_p0/test_committed_tso.groovy diff --git a/be/src/information_schema/schema_tso_status_scanner.cpp b/be/src/information_schema/schema_tso_status_scanner.cpp index ef9a0584fef7b0..9f453a32c264e6 100644 --- a/be/src/information_schema/schema_tso_status_scanner.cpp +++ b/be/src/information_schema/schema_tso_status_scanner.cpp @@ -34,6 +34,11 @@ std::vector SchemaTsoStatusScanner::_s_tso_status_col {"CURRENT_TSO", TYPE_BIGINT, sizeof(int64_t), true}, {"CURRENT_TSO_PHYSICAL_TIME", TYPE_BIGINT, sizeof(int64_t), true}, {"CURRENT_TSO_LOGICAL_COUNTER", TYPE_BIGINT, sizeof(int64_t), true}, + {.name = "COMMITTED_TSO", .type = TYPE_BIGINT, .size = sizeof(int64_t), .is_null = true}, + {.name = "COMMITTED_TSO_PHYSICAL_TIME", + .type = TYPE_BIGINT, + .size = sizeof(int64_t), + .is_null = true}, }; SchemaTsoStatusScanner::SchemaTsoStatusScanner() @@ -51,6 +56,11 @@ Status SchemaTsoStatusScanner::_get_tso_status_block_from_fe() { TNetworkAddress master_addr = ExecEnv::GetInstance()->cluster_info()->master_fe_addr; TSchemaTableRequestParams schema_table_request_params; + std::vector columns; + for (const auto& column : _s_tso_status_columns) { + columns.emplace_back(column.name); + } + schema_table_request_params.__set_columns_name(columns); TFetchSchemaTableDataRequest request; request.__set_schema_table_name(TSchemaTableName::TSO_STATUS); request.__set_schema_table_params(schema_table_request_params); diff --git a/be/test/exec/schema_scanner/schema_tso_status_scanner_test.cpp b/be/test/exec/schema_scanner/schema_tso_status_scanner_test.cpp index 60a95e3bffcb13..6cf5b09117d025 100644 --- a/be/test/exec/schema_scanner/schema_tso_status_scanner_test.cpp +++ b/be/test/exec/schema_scanner/schema_tso_status_scanner_test.cpp @@ -37,7 +37,7 @@ namespace doris { namespace { -TRow create_tso_status_row(const std::array& values) { +TRow create_tso_status_row(const std::array& values) { std::vector cells; cells.reserve(values.size()); for (int64_t value : values) { @@ -65,7 +65,7 @@ std::unique_ptr create_output_block(SchemaTsoStatusScanner* scanner) { } void expect_tso_status_row(const Block& block, size_t row_idx, - const std::array& expected) { + const std::array& expected) { ASSERT_EQ(expected.size(), block.columns()); for (size_t column_idx = 0; column_idx < expected.size(); ++column_idx) { const auto& column = block.get_by_position(column_idx).column; @@ -80,11 +80,13 @@ TEST(SchemaTsoStatusScannerTest, test_create_tso_status_scanner) { auto scanner = SchemaScanner::create(TSchemaTableType::SCH_TSO_STATUS); ASSERT_NE(nullptr, scanner); EXPECT_EQ(TSchemaTableType::SCH_TSO_STATUS, scanner->type()); - ASSERT_EQ(4, scanner->get_column_desc().size()); + ASSERT_EQ(6, scanner->get_column_desc().size()); EXPECT_STREQ("WINDOW_END_PHYSICAL_TIME", scanner->get_column_desc()[0].name); EXPECT_STREQ("CURRENT_TSO", scanner->get_column_desc()[1].name); EXPECT_STREQ("CURRENT_TSO_PHYSICAL_TIME", scanner->get_column_desc()[2].name); EXPECT_STREQ("CURRENT_TSO_LOGICAL_COUNTER", scanner->get_column_desc()[3].name); + EXPECT_STREQ("COMMITTED_TSO", scanner->get_column_desc()[4].name); + EXPECT_STREQ("COMMITTED_TSO_PHYSICAL_TIME", scanner->get_column_desc()[5].name); for (const auto& column : scanner->get_column_desc()) { EXPECT_EQ(TYPE_BIGINT, column.type); EXPECT_TRUE(column.is_null); @@ -140,8 +142,8 @@ TEST(SchemaTsoStatusScannerTest, test_process_tso_status_result_error) { } TEST(SchemaTsoStatusScannerTest, test_process_tso_status_result) { - const std::array first_row = {1000, 2000, 3000, 4000}; - const std::array second_row = {1001, 2001, 3001, 4001}; + const std::array first_row = {1000, 2000, 3000, 4000}; + const std::array second_row = {1001, 2001, 3001, 4001}; auto result = create_tso_status_result( {create_tso_status_row(first_row), create_tso_status_row(second_row)}); @@ -149,7 +151,7 @@ TEST(SchemaTsoStatusScannerTest, test_process_tso_status_result) { ASSERT_TRUE(scanner._process_tso_status_result(result).ok()); ASSERT_NE(nullptr, scanner._tso_status_block); - EXPECT_EQ(4, scanner._tso_status_block->columns()); + EXPECT_EQ(6, scanner._tso_status_block->columns()); EXPECT_EQ(2, scanner._tso_status_block->rows()); EXPECT_EQ(2, scanner._total_rows); expect_tso_status_row(*scanner._tso_status_block, 0, first_row); @@ -158,7 +160,8 @@ TEST(SchemaTsoStatusScannerTest, test_process_tso_status_result) { TEST(SchemaTsoStatusScannerTest, test_process_tso_status_result_schema_mismatch) { TRow invalid_row = create_tso_status_row({1000, 2000, 3000, 4000}); - invalid_row.column_value.pop_back(); + invalid_row.column_value.resize( + 4); // Response from an older FE cannot provide a committed prefix. auto result = create_tso_status_result({invalid_row}); SchemaTsoStatusScanner scanner; @@ -170,6 +173,17 @@ TEST(SchemaTsoStatusScannerTest, test_process_tso_status_result_schema_mismatch) EXPECT_EQ(0, scanner._total_rows); } +TEST(SchemaTsoStatusScannerTest, test_unknown_committed_tso_is_null) { + TRow row = create_tso_status_row({1000, 2000, 3000, 4000, 0, 0}); + row.column_value[4].__set_isNull(true); + row.column_value[5].__set_isNull(true); + SchemaTsoStatusScanner scanner; + ASSERT_TRUE(scanner._process_tso_status_result(create_tso_status_result({row})).ok()); + EXPECT_FALSE(scanner._tso_status_block->get_by_position(3).column->is_null_at(0)); + EXPECT_TRUE(scanner._tso_status_block->get_by_position(4).column->is_null_at(0)); + EXPECT_TRUE(scanner._tso_status_block->get_by_position(5).column->is_null_at(0)); +} + TEST(SchemaTsoStatusScannerTest, test_get_next_block_empty_result) { MockRuntimeState state; SchemaScannerParam param; @@ -188,9 +202,9 @@ TEST(SchemaTsoStatusScannerTest, test_get_next_block_empty_result) { } TEST(SchemaTsoStatusScannerTest, test_get_next_block_in_batches) { - const std::array first_row = {1000, 2000, 3000, 4000}; - const std::array second_row = {1001, 2001, 3001, 4001}; - const std::array third_row = {1002, 2002, 3002, 4002}; + const std::array first_row = {1000, 2000, 3000, 4000}; + const std::array second_row = {1001, 2001, 3001, 4001}; + const std::array third_row = {1002, 2002, 3002, 4002}; MockRuntimeState state; state._batch_size = 2; diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 76963cb0627e8f..06328ac6c76407 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -4729,7 +4729,9 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont CheckTxnConflictResponse* response, ::google::protobuf::Closure* done) { RPC_PREPROCESS(check_txn_conflict, get); - if (!request->has_db_id() || !request->has_end_txn_id() || (request->table_ids_size() <= 0)) { + const bool strict_recovery = request->strict_recovery_check(); + if (!request->has_end_txn_id() || (strict_recovery && request->end_txn_id() <= 0) || + (!strict_recovery && (!request->has_db_id() || request->table_ids_size() <= 0))) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "invalid db id, end txn id or table_ids."; return; @@ -4749,6 +4751,14 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont std::string begin_running_key = txn_running_key({instance_id, db_id, 0}); std::string end_running_key = txn_running_key({instance_id, db_id, request->end_txn_id()}); + if (strict_recovery) { + // Database and transaction IDs are non-negative. Include the entire instance and apply + // the exclusive transaction bound after decoding each key (keys sort by database first). + begin_running_key = txn_running_key({instance_id, 0, 0}); + end_running_key = txn_running_key({instance_id, INT64_MAX, INT64_MAX}); + end_running_key.push_back('\x00'); + response->set_strict_recovery_check_applied(true); + } LOG(INFO) << "begin_running_key:" << hex(begin_running_key) << " end_running_key:" << hex(end_running_key); @@ -4785,6 +4795,27 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont while (it->has_next()) { total_iteration_cnt++; auto [k, v] = it->next(); + if (strict_recovery) { + std::string_view encoded_key = k; + encoded_key.remove_prefix(1); + std::vector, int, int>> fields; + if (decode_key(&encoded_key, &fields) != 0 || fields.size() != 5 || + !std::holds_alternative(std::get<0>(fields[4]))) { + code = MetaServiceCode::UNDEFINED_ERR; + msg = "failed to decode running transaction key during TSO recovery"; + return; + } + if (std::get(std::get<0>(fields[4])) < request->end_txn_id()) { + // A running key is removed atomically with real VISIBLE/ABORTED. In + // particular an expired COMMITTED lazy transaction must still block. + response->set_finished(false); + return; + } + if (!it->has_next()) { + begin_running_key = k; + } + continue; + } LOG(INFO) << "check watermark conflict range_get txn_run_key=" << hex(k); TxnRunningPB running_pb; if (!running_pb.ParseFromArray(v.data(), v.size())) { diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index 924eb3897a6e13..85ad80f14ee554 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -2781,6 +2781,81 @@ TEST(MetaServiceTest, GetCurrentMaxTxnIdTest) { ASSERT_GE(max_txn_id_res.current_max_txn_id(), begin_txn_res.txn_id()); } +TEST(MetaServiceTest, StrictTsoRecoveryChecksAllDatabasesAndExpiredLazyTransactions) { + auto meta_service = get_meta_service(); + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + TxnRunningPB running; + running.set_timeout_time(1); // Expired is not terminal for a persisted COMMITTED lazy txn. + running.add_table_ids(777); + txn->put(txn_running_key({mock_instance, 1, 100}), running.SerializeAsString()); + txn->put(txn_running_key({mock_instance, 2, 150}), running.SerializeAsString()); + txn->put(txn_running_key({"another_instance", 1, 1}), running.SerializeAsString()); + const auto blocking_key = txn_running_key({mock_instance, 999, 50}); + txn->put(blocking_key, running.SerializeAsString()); + TxnInfoPB info; + info.set_db_id(999); + info.set_txn_id(50); + info.set_status(TxnStatusPB::TXN_STATUS_COMMITTED); + const auto info_key = txn_info_key({mock_instance, 999, 50}); + txn->put(info_key, info.SerializeAsString()); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + brpc::Controller cntl; + CheckTxnConflictRequest request; + request.set_cloud_unique_id("test_cloud_unique_id"); + request.set_end_txn_id(100); + request.set_strict_recovery_check(true); + CheckTxnConflictResponse response; + meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + ASSERT_EQ(response.status().code(), MetaServiceCode::OK); + ASSERT_TRUE(response.strict_recovery_check_applied()); + ASSERT_FALSE(response.finished()); + + // The legacy table-scoped check still skips expired transactions; its result cannot recover TSO. + CheckTxnConflictRequest legacy = request; + legacy.clear_strict_recovery_check(); + legacy.set_db_id(999); + legacy.add_table_ids(777); + response.Clear(); + meta_service->check_txn_conflict(&cntl, &legacy, &response, nullptr); + ASSERT_EQ(response.status().code(), MetaServiceCode::OK); + ASSERT_TRUE(response.finished()); + ASSERT_FALSE(response.has_strict_recovery_check_applied()); + + // Real publication removes the running key in the same KV transaction as the terminal state. + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + info.set_status(TxnStatusPB::TXN_STATUS_VISIBLE); + txn->put(info_key, info.SerializeAsString()); + txn->remove(blocking_key); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + response.Clear(); + meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + ASSERT_EQ(response.status().code(), MetaServiceCode::OK); + ASSERT_TRUE(response.strict_recovery_check_applied()); + ASSERT_TRUE( + response.finished()); // IDs equal to or above the fixed exclusive bound are ignored. +} + +TEST(MetaServiceTest, StrictTsoRecoveryRejectsMalformedRunningKeys) { + auto meta_service = get_meta_service(); + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string key = txn_running_key({mock_instance, 1, 1}); + key.push_back('\xff'); + txn->put(key, ""); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + brpc::Controller cntl; + CheckTxnConflictRequest request; + request.set_cloud_unique_id("test_cloud_unique_id"); + request.set_end_txn_id(100); + request.set_strict_recovery_check(true); + CheckTxnConflictResponse response; + meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + ASSERT_NE(response.status().code(), MetaServiceCode::OK); + ASSERT_FALSE(response.finished()); +} + TEST(MetaServiceTest, CreateMetaSyncPointTest) { auto meta_service = get_meta_service(); const std::string cloud_unique_id = "test_cloud_unique_id"; diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp index 41374e67a75428..365a80ee8c88af 100644 --- a/cloud/test/txn_lazy_commit_test.cpp +++ b/cloud/test/txn_lazy_commit_test.cpp @@ -1281,6 +1281,16 @@ TEST(TxnLazyCommitTest, CommitTxnEventuallyWithFailedLazyCommitTaskTest) { ASSERT_TRUE(commit_res.has_is_lazy_commit_incomplete()); ASSERT_TRUE(commit_res.is_lazy_commit_incomplete()); + CheckTxnConflictRequest recovery_req; + recovery_req.set_cloud_unique_id("test_cloud_unique_id"); + recovery_req.set_strict_recovery_check(true); + recovery_req.set_end_txn_id(txn_id + 1); + CheckTxnConflictResponse recovery_res; + meta_service->check_txn_conflict(&cntl, &recovery_req, &recovery_res, nullptr); + ASSERT_EQ(recovery_res.status().code(), MetaServiceCode::OK); + ASSERT_TRUE(recovery_res.strict_recovery_check_applied()); + ASSERT_FALSE(recovery_res.finished()); + std::unique_ptr txn; ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); check_txn_committed(txn, db_id, txn_id, label); diff --git a/docs/committed-tso.md b/docs/committed-tso.md new file mode 100644 index 00000000000000..b38c8e8adcef90 --- /dev/null +++ b/docs/committed-tso.md @@ -0,0 +1,34 @@ +# Committed TSO for bounded incremental reads + +In cloud mode, a strongly consistent `@incr` query with an explicit `endTimestamp` on every incremental relation uses the master FE's durable committed TSO. Transactions with commit TSO at or below this prefix are truly visible or aborted. The name does not refer to the intermediate `COMMITTED` transaction state. + +```sql +SELECT COMMITTED_TSO, COMMITTED_TSO_PHYSICAL_TIME +FROM information_schema.tso_status; +``` + +Both columns are nullable BIGINTs. `COMMITTED_TSO_PHYSICAL_TIME` is Unix epoch milliseconds and gives the maximum allowed end timestamp; the full encoded `COMMITTED_TSO` must not be passed as an `endTimestamp` string. Convert milliseconds to the timestamp format and time zone accepted by `@incr`; clients producing whole-second windows must round down. The interval remains `[start, end)`. Existing binlog retention and table requirements still apply. + +The system table reads the master's persisted prefix without allocating a TSO. While a new master recovers, it keeps exposing the previous persisted prefix. A first startup, an old image without a prefix, or classic mode returns NULL for the new columns. Existing disabled/uninitialized TSO errors remain unchanged. + +If the requested end exceeds the prefix, the query immediately returns MySQL error 5100, `ERR_INCR_WINDOW_NOT_READY`. Its message includes `requestedEndTimestampMs`, `committedTSO`, `committedTSOPhysicalTimeMs`, and `retryAfterMs`. Master-to-follower RPC preserves this classification. Arrow Flight SQL returns UNAVAILABLE with metadata `doris-error-code=5100` and `doris-error-name=ERR_INCR_WINDOW_NOT_READY`; the description preserves the window and retry details. Clients should retry the same split/window/offset after a cancellable delay, and advance offsets only after that window completes. Shortening a refused window or treating it as an empty success can lose data. This Doris change does not implement a Connector's retry loop. + +For accepted windows, planning skips the transaction watermark and conflict polling. Cloud partition visible versions are still refreshed from MetaService. Classic reads, eventual consistency, unbounded reads, and queries mixing bounded and unbounded incremental relations retain the existing behavior. + +## Allocation and persistence + +The allocator registers transaction identity and its first commit TSO under the same lock that advances its clock. Retries retain the earliest registration until a real terminal result is known. Bitmap preparation, callbacks, and commit metadata validation precede allocation; one request reuses the same TSO for RPC retries. A lazy commit response marked incomplete cannot release a registration even if its returned transaction status says VISIBLE. A separate worker reconciles at most 64 old registrations per cycle; missing/error responses do not release them. + +After recovery, the next candidate prefix is the current allocated TSO when no registrations remain, otherwise the smaller of that TSO and the oldest pending TSO minus one. The candidate is published only after its journal write succeeds. The reservation window and committed prefix share one journal record and one immutable persisted snapshot. `tso_service_window_duration_ms` defaults to 1000 ms. A monotonic timer also persists the prefix when the reservation window does not move. This is approximately one combined journal write per second, compared with the previous five-second window renewal; it does not reduce total journal frequency relative to the old implementation. + +## Recovery and upgrades + +A new master calibrates beyond the previous reserved window, registers new allocations, and waits `tso_service_window_duration_ms + 1000` milliseconds before taking a fixed exclusive transaction-ID bound from MetaService. An instance-wide strict check must then find no running transaction below that bound. The check covers every database/table and never skips expired running transactions. An expired lazy transaction can still await real publication. Normal pending registrations continue to constrain the prefix after the recovery check succeeds. + +Upgrade MetaService before using the new FE's committed prefix: recovery requires an explicit acknowledgement of the strict-check option. Old journal/image records remain readable and imply an unknown prefix. Old BE requests without column names retain the original four-column system-table response; new BE requests explicitly name all six columns. A new BE cannot obtain the new columns from an old FE. + +The fixed recovery wait is a temporary operational assumption, not a fencing protocol. It cannot prevent an old master that continues allocating after the wait from assigning an old TSO outside the captured transaction bound. Clock skew, an old longer reservation window, or long process pauses can violate that assumption. This change deliberately retains that accepted limitation. Recovery can also wait for long-running old transactions, and any oldest pending transaction can delay the global prefix across unrelated tables. + +## Diagnosis + +FE metrics expose the committed prefix, reserved window, pending count, oldest pending TSO/transaction/age, recovery readiness and transaction bound. TSO persistence and reconciliation have counters and latency histograms. Reconciliation failures preserve the watermark and produce a rate-limited warning. Unknown transaction status must be investigated; registrations are not discarded by TTL. diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 54a16d46685cae..4e41d3c2686fe4 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3593,8 +3593,8 @@ public void handle(Field field, String value) throws Exception { public static int tso_max_get_retry_count = 10; @ConfField(mutable = true, masterOnly = true, description = "TSO service time window in milliseconds. Default is " - + "5000, which means the TSO service will apply for a " + "TSO time window of 5000ms from BDBJE once.") - public static int tso_service_window_duration_ms = 5000; + + "1000. Persist the readable committed TSO together with the reserved allocation window.") + public static int tso_service_window_duration_ms = 1000; @ConfField(mutable = true, masterOnly = true, description = "Max tolerated clock backward threshold during TSO " + "calibration in milliseconds. Exceeding this " + "threshold will fail enabling TSO. Default is 30 " diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java index 8f5fe32bb302b2..887f6b073d8923 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java @@ -1234,6 +1234,9 @@ public enum ErrorCode { ERR_NO_CLUSTER_ERROR(5099, new byte[]{'4', '2', '0', '0', '0'}, "No compute group (cloud cluster) selected"), + ERR_INCR_WINDOW_NOT_READY(5100, new byte[]{'H', 'Y', '0', '0', '0'}, + "The requested incremental read window is not yet visible; retry the same window."), + ERR_NOT_CLOUD_MODE(6000, new byte[]{'4', '2', '0', '0', '0'}, "Command only support in cloud mode."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index 91dd81e5af5952..1ddf2e1f41426e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -908,6 +908,8 @@ public class SchemaTable extends Table { ScalarType.createType(PrimitiveType.BIGINT)) .column("CURRENT_TSO_LOGICAL_COUNTER", ScalarType.createType(PrimitiveType.BIGINT)) + .column("COMMITTED_TSO", ScalarType.createType(PrimitiveType.BIGINT)) + .column("COMMITTED_TSO_PHYSICAL_TIME", ScalarType.createType(PrimitiveType.BIGINT)) .build())) .put("be_compaction_tasks", new SchemaTable(SystemIdGenerator.getNextId(), "be_compaction_tasks", TableType.SCHEMA, diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index 52b7a84bea2dbc..0e35325eaaccc8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -45,6 +45,7 @@ import org.apache.doris.cloud.proto.Cloud.CleanTxnLabelRequest; import org.apache.doris.cloud.proto.Cloud.CleanTxnLabelResponse; import org.apache.doris.cloud.proto.Cloud.CommitTxnRequest; +import org.apache.doris.cloud.proto.Cloud.CommitTxnRequestOrBuilder; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; import org.apache.doris.cloud.proto.Cloud.GetCurrentMaxTxnRequest; import org.apache.doris.cloud.proto.Cloud.GetCurrentMaxTxnResponse; @@ -446,9 +447,6 @@ private void commitTransactionWithoutLock(long dbId, List tableList, long LOG.info("try to commit transaction, transactionId: {}, tableIds: {}", transactionId, tableList.stream().map(Table::getId).collect(Collectors.toList())); Map> backendToPartitionInfos = null; - Database database = Env.getCurrentInternalCatalog().getDbOrMetaException(dbId); - long commitTSO = TransactionUtil.getCommitTSO(transactionId, database, - tableList.stream().map(Table::getId).collect(Collectors.toSet())); if (!mowTableList.isEmpty()) { if (!checkTransactionStateBeforeCommit(dbId, transactionId)) { return; @@ -462,7 +460,7 @@ private void commitTransactionWithoutLock(long dbId, List
tableList, long backendToPartitionInfos = getCalcDeleteBitmapInfo(lockContext, null); } commitTransactionWithoutLock(dbId, tableList, transactionId, tabletCommitInfos, txnCommitAttachment, false, - mowTableList, backendToPartitionInfos, commitTSO, streamUpdateInfos); + mowTableList, backendToPartitionInfos, streamUpdateInfos); // clear signature after commit succeeds clearTxnLastSignature(dbId, transactionId); } catch (Exception e) { @@ -700,17 +698,16 @@ private Set getBaseTabletsFromTables(List
tableList, List tableList, long transactionId, List tabletCommitInfos, TxnCommitAttachment txnCommitAttachment, boolean is2PC, - List mowTableList, Map> backendToPartitionInfos, - long commitTSO) + List mowTableList, Map> backendToPartitionInfos) throws UserException { commitTransactionWithoutLock(dbId, tableList, transactionId, tabletCommitInfos, txnCommitAttachment, - is2PC, mowTableList, backendToPartitionInfos, commitTSO, Collections.emptyList()); + is2PC, mowTableList, backendToPartitionInfos, Collections.emptyList()); } private void commitTransactionWithoutLock(long dbId, List
tableList, long transactionId, List tabletCommitInfos, TxnCommitAttachment txnCommitAttachment, boolean is2PC, List mowTableList, Map> backendToPartitionInfos, - long commitTSO, List streamUpdateInfos) + List streamUpdateInfos) throws UserException { if (Config.disable_load_job) { throw new TransactionCommitFailedException( @@ -730,7 +727,6 @@ private void commitTransactionWithoutLock(long dbId, List
tableList, long .setTxnId(transactionId) .setIs2Pc(is2PC) .setCloudUniqueId(Config.cloud_unique_id) - .setCommitTso(commitTSO) .addAllBaseTabletIds(getBaseTabletsFromTables(tableList, tabletCommitInfos)) .setEnableTxnLazyCommit(Config.enable_cloud_txn_lazy_commit); for (OlapTable olapTable : mowTableList) { @@ -784,13 +780,13 @@ private void commitTransactionWithoutLock(long dbId, List
tableList, long } } - final CommitTxnRequest commitTxnRequest = builder.build(); - executeCommitTxnRequest(commitTxnRequest, transactionId, is2PC, txnCommitAttachment, tabletCommitInfos, + executeCommitTxnRequest(builder, tableList, transactionId, is2PC, txnCommitAttachment, tabletCommitInfos, tabletCommitInfos == null ? Collections.emptyList() : tabletCommitInfos.stream().map(t -> t.getTabletId()).collect(Collectors.toList())); } - private void executeCommitTxnRequest(CommitTxnRequest commitTxnRequest, long transactionId, boolean is2PC, + private void executeCommitTxnRequest(CommitTxnRequest.Builder builder, List
tableList, + long transactionId, boolean is2PC, TxnCommitAttachment txnCommitAttachment, List tabletCommitInfos, List tabletIds) throws UserException { if (DebugPointUtil.isEnable("FE.mow.commit.exception")) { @@ -815,7 +811,7 @@ private void executeCommitTxnRequest(CommitTxnRequest commitTxnRequest, long tra StopWatch stopWatch = new StopWatch(); stopWatch.start(); try { - txnState = commitTxn(commitTxnRequest, transactionId, is2PC, tabletCommitInfos, tabletIds); + txnState = commitTxn(builder, tableList, transactionId, is2PC, tabletCommitInfos, tabletIds); txnOperated = true; if (DebugPointUtil.isEnable("CloudGlobalTransactionMgr.commitTransaction.timeout")) { throw new UserException(InternalErrorCode.DELETE_BITMAP_LOCK_ERR, @@ -854,9 +850,24 @@ private void executeCommitTxnRequest(CommitTxnRequest commitTxnRequest, long tra } } - private TransactionState commitTxn(CommitTxnRequest commitTxnRequest, long transactionId, boolean is2PC, - List tabletCommitInfos, List tabletIds) throws UserException { - checkCommitInfo(commitTxnRequest); + private TransactionState commitTxn(CommitTxnRequest.Builder builder, List
tableList, + long transactionId, boolean is2PC, List tabletCommitInfos, List tabletIds) + throws UserException { + checkCommitInfo(builder); + // Bitmap work, attachments and metadata validation do not need a commit TSO. Allocate only + // when ready to send, while retaining the existing table locks and callback cleanup scope. + Database database = Env.getCurrentInternalCatalog().getDbOrMetaException(builder.getDbId()); + builder.setCommitTso(TransactionUtil.getCommitTSO(transactionId, database, + tableList.stream().map(Table::getId).collect(Collectors.toSet()))); + final CommitTxnRequest commitTxnRequest = builder.build(); + try { + while (DebugPointUtil.isEnable("CloudGlobalTransactionMgr.commitTxn.blockAfterTso")) { + Thread.sleep(100); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UserException("Interrupted before sending commit transaction", e); + } CommitTxnResponse commitTxnResponse = null; TransactionState txnState = null; @@ -887,6 +898,8 @@ private TransactionState commitTxn(CommitTxnRequest commitTxnRequest, long trans throw new UserException("commitTxn() failed, errMsg:" + e.getMessage()); } + releaseFinishedTso(commitTxnRequest.getDbId(), transactionId, commitTxnResponse); + if (is2PC && (commitTxnResponse.getStatus().getCode() == MetaServiceCode.TXN_ALREADY_VISIBLE || commitTxnResponse.getStatus().getCode() == MetaServiceCode.TXN_ALREADY_ABORTED)) { throw new UserException(commitTxnResponse.getStatus().getMsg()); @@ -919,7 +932,21 @@ private TransactionState commitTxn(CommitTxnRequest commitTxnRequest, long trans return txnState; } - private void checkCommitInfo(CommitTxnRequest commitTxnRequest) throws UserException { + // A lazy commit response can report VISIBLE before the persistent transaction is visible. + static void releaseFinishedTso(long dbId, long txnId, CommitTxnResponse response) { + if (response.getIsLazyCommitIncomplete()) { + return; + } + MetaServiceCode code = response.getStatus().getCode(); + if (code == MetaServiceCode.TXN_ALREADY_VISIBLE || code == MetaServiceCode.TXN_ALREADY_ABORTED + || (code == MetaServiceCode.OK && response.hasTxnInfo() + && (response.getTxnInfo().getStatus() == TxnStatusPB.TXN_STATUS_VISIBLE + || response.getTxnInfo().getStatus() == TxnStatusPB.TXN_STATUS_ABORTED))) { + Env.getCurrentEnv().getTSOService().transactionFinished(dbId, txnId); + } + } + + private void checkCommitInfo(CommitTxnRequestOrBuilder commitTxnRequest) throws UserException { List commitTabletIds = Lists.newArrayList(); List commitIndexIds = Lists.newArrayList(); commitTabletIds.addAll(commitTxnRequest.getBaseTabletIdsList()); @@ -1627,8 +1654,6 @@ public boolean commitAndPublishTransaction(DatabaseIf db, long transactionId, List mowTableList = getMowTableList(tableList, tabletCommitInfos); try { Map> backendToPartitionInfos = null; - long commitTSO = TransactionUtil.getCommitTSO(transactionId, (Database) db, - tableList.stream().map(Table::getId).collect(Collectors.toSet())); if (!mowTableList.isEmpty()) { if (!checkTransactionStateBeforeCommit(db.getId(), transactionId)) { return true; @@ -1646,7 +1671,7 @@ public boolean commitAndPublishTransaction(DatabaseIf db, long transactionId, lockContext, partitionToSubTxnIds); } commitTransactionWithSubTxns(db.getId(), tableList, transactionId, subTransactionStates, mowTableList, - backendToPartitionInfos, commitTSO); + backendToPartitionInfos); // clear signature after commit succeeds clearTxnLastSignature(db.getId(), transactionId); } catch (Exception e) { @@ -1684,7 +1709,7 @@ public boolean commitAndPublishTransaction(DatabaseIf db, long transactionId, private void commitTransactionWithSubTxns(long dbId, List
tableList, long transactionId, List subTransactionStates, List mowTableList, - Map> backendToPartitionInfos, long commitTSO) + Map> backendToPartitionInfos) throws UserException { if (!mowTableList.isEmpty()) { List mowTableIds = mowTableList.stream().map(Table::getId).collect(Collectors.toList()); @@ -1700,7 +1725,6 @@ private void commitTransactionWithSubTxns(long dbId, List
tableList, long .setIs2Pc(false) .setCloudUniqueId(Config.cloud_unique_id) .setIsTxnLoad(true) - .setCommitTso(commitTSO) .setEnableTxnLazyCommit(Config.enable_cloud_txn_lazy_commit); for (OlapTable olapTable : mowTableList) { builder.addMowTableIds(olapTable.getId()); @@ -1721,8 +1745,7 @@ private void commitTransactionWithSubTxns(long dbId, List
tableList, long } } - final CommitTxnRequest commitTxnRequest = builder.build(); - executeCommitTxnRequest(commitTxnRequest, transactionId, false, null, null, new ArrayList<>(tabletIds)); + executeCommitTxnRequest(builder, tableList, transactionId, false, null, null, new ArrayList<>(tabletIds)); } private List
getTablesNeedCommitLock(List
tableList) { @@ -1907,10 +1930,8 @@ public void commitTransaction2PC(Database db, List
tableList, long transa return; } } - long commitTSO = TransactionUtil.getCommitTSO(transactionId, db, - tableList.stream().map(Table::getId).collect(Collectors.toSet())); commitTransactionWithoutLock(db.getId(), tableList, transactionId, null, null, true, - mowTableList, null, commitTSO); + mowTableList, null); } finally { afterCommitTransaction(tableList, transactionId); } @@ -2055,6 +2076,13 @@ private void handleAfterAbort(AbortTxnResponse abortTxnResponse, TxnCommitAttach private void afterAbortTxnResp(AbortTxnResponse abortTxnResponse, String txnIdOrLabel, TxnCommitAttachment txnCommitAttachment) throws UserException { + // MS fills txn_info only after committing the abort's KV transaction. + if (abortTxnResponse.hasTxnInfo() + && (abortTxnResponse.getTxnInfo().getStatus() == TxnStatusPB.TXN_STATUS_ABORTED + || abortTxnResponse.getTxnInfo().getStatus() == TxnStatusPB.TXN_STATUS_VISIBLE)) { + Env.getCurrentEnv().getTSOService().transactionFinished( + abortTxnResponse.getTxnInfo().getDbId(), abortTxnResponse.getTxnInfo().getTxnId()); + } if (abortTxnResponse.getStatus().getCode() != MetaServiceCode.OK) { LOG.warn("abortTxn failed, transaction:{}, response:{}", txnIdOrLabel, abortTxnResponse); switch (abortTxnResponse.getStatus().getCode()) { @@ -2187,6 +2215,28 @@ public List getUnFinishedPreviousLoad(long endTransactionId, l return conflictTxns; } + @Override + public boolean isPreviousTransactionsFinishedForTsoRecovery(long endTransactionId) throws UserException { + CheckTxnConflictRequest request = CheckTxnConflictRequest.newBuilder() + .setCloudUniqueId(Config.cloud_unique_id) + .setRequestIp(FrontendOptions.getLocalHostAddressCached()) + .setEndTxnId(endTransactionId) + .setStrictRecoveryCheck(true).build(); + CheckTxnConflictResponse response; + try { + response = MetaServiceProxy.getInstance().checkTxnConflict(request); + } catch (RpcException e) { + throw new UserException("Strict TSO recovery check failed", e); + } + if (response.getStatus().getCode() != MetaServiceCode.OK) { + throw new UserException(response.getStatus().getMsg()); + } + if (!response.getStrictRecoveryCheckApplied() || !response.hasFinished()) { + throw new UserException("MetaService does not support strict TSO recovery; upgrade MetaService first"); + } + return response.getFinished(); + } + @Override public boolean isPreviousTransactionsFinished(long endTransactionId, long dbId, List tableIdList) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/IncrWindowNotReadyException.java b/fe/fe-core/src/main/java/org/apache/doris/common/IncrWindowNotReadyException.java new file mode 100644 index 00000000000000..51046aae79f7e1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/IncrWindowNotReadyException.java @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common; + +import org.apache.doris.tso.TSOTimestamp; + +/** A bounded incremental read must retry the same window after its upper bound becomes readable. */ +public class IncrWindowNotReadyException extends UserException { + private final long requestedEndTimestampMs; + private final long committedTso; + private final long retryAfterMs; + + public IncrWindowNotReadyException(long requestedEndTimestampMs, long committedTso, long retryAfterMs) { + super(String.format("ERR_INCR_WINDOW_NOT_READY: requestedEndTimestampMs=%d, committedTSO=%d, " + + "committedTSOPhysicalTimeMs=%d, retryAfterMs=%d", + requestedEndTimestampMs, committedTso, TSOTimestamp.extractPhysicalTime(committedTso), retryAfterMs)); + setMysqlErrorCode(ErrorCode.ERR_INCR_WINDOW_NOT_READY); + this.requestedEndTimestampMs = requestedEndTimestampMs; + this.committedTso = committedTso; + this.retryAfterMs = retryAfterMs; + } + + public long getRequestedEndTimestampMs() { + return requestedEndTimestampMs; + } + + public long getCommittedTso() { + return committedTso; + } + + public long getRetryAfterMs() { + return retryAfterMs; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 76f57fe443a01a..2151f11b1dfa15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -147,7 +147,7 @@ import org.apache.doris.system.Backend; import org.apache.doris.system.Frontend; import org.apache.doris.transaction.TransactionState; -import org.apache.doris.tso.TSOTimestamp; +import org.apache.doris.tso.TSOServiceState; import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; @@ -1035,7 +1035,7 @@ public void readFields(DataInput in) throws IOException { break; } case OperationType.OP_TSO_TIMESTAMP_WINDOW_END: { - data = TSOTimestamp.read(in); + data = TSOServiceState.read(in); isRead = true; break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java index 3c1aeb6a0373d3..00bd318071913d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java @@ -314,6 +314,11 @@ public final class MetricRepo { public static LongCounterMetric COUNTER_TSO_CLOCK_UPDATED; public static LongCounterMetric COUNTER_TSO_CLOCK_UPDATE_FAILED; public static LongCounterMetric COUNTER_TSO_CLOCK_GET_SUCCESS; + public static LongCounterMetric COUNTER_TSO_STATE_PERSISTED; + public static LongCounterMetric COUNTER_TSO_STATE_PERSIST_FAILED; + public static LongCounterMetric COUNTER_TSO_RECONCILE_FAILED; + public static Histogram HISTO_TSO_STATE_PERSIST_LATENCY; + public static Histogram HISTO_TSO_RECONCILE_LATENCY; private static Map, Long> loadJobNum = Maps.newHashMap(); @@ -1173,6 +1178,18 @@ public Integer getValue() { COUNTER_TSO_CLOCK_GET_SUCCESS = new LongCounterMetric("tso_clock_get_success", MetricUnit.NOUNIT, "counter of tso clock get success"); DORIS_METRIC_REGISTER.addMetrics(COUNTER_TSO_CLOCK_GET_SUCCESS); + COUNTER_TSO_STATE_PERSISTED = new LongCounterMetric("tso_state_persisted", MetricUnit.NOUNIT, + "successful combined committed TSO and window journal writes"); + DORIS_METRIC_REGISTER.addMetrics(COUNTER_TSO_STATE_PERSISTED); + COUNTER_TSO_STATE_PERSIST_FAILED = new LongCounterMetric("tso_state_persist_failed", MetricUnit.NOUNIT, + "failed TSO state journal writes"); + DORIS_METRIC_REGISTER.addMetrics(COUNTER_TSO_STATE_PERSIST_FAILED); + COUNTER_TSO_RECONCILE_FAILED = new LongCounterMetric("tso_reconcile_failed", MetricUnit.NOUNIT, + "failed TSO transaction reconciliation cycles"); + DORIS_METRIC_REGISTER.addMetrics(COUNTER_TSO_RECONCILE_FAILED); + HISTO_TSO_STATE_PERSIST_LATENCY = METRIC_REGISTER.histogram("tso_state_persist_latency_ms"); + HISTO_TSO_RECONCILE_LATENCY = METRIC_REGISTER.histogram("tso_reconcile_latency_ms"); + Env.getCurrentEnv().getTSOService().registerMetrics(); // init system metrics initSystemMetrics(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index a29ef2a6326876..b3c6466e878df9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -109,7 +109,7 @@ import org.apache.doris.system.Frontend; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; -import org.apache.doris.tso.TSOTimestamp; +import org.apache.doris.tso.TSOServiceState; import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; @@ -1493,7 +1493,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { break; } case OperationType.OP_TSO_TIMESTAMP_WINDOW_END: { - env.getTSOService().replayWindowEndTSO((TSOTimestamp) journal.getData()); + env.getTSOService().replayWindowEndTSO((TSOServiceState) journal.getData()); break; } default: { @@ -1947,7 +1947,7 @@ public void logTimestamp(Timestamp stamp) { logEdit(OperationType.OP_TIMESTAMP, stamp); } - public void logTSOTimestampWindowEnd(TSOTimestamp windowEnd) { + public void logTSOTimestampWindowEnd(TSOServiceState windowEnd) { logEdit(OperationType.OP_TSO_TIMESTAMP_WINDOW_END, windowEnd); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index e5f726628fd074..b37440c6d52506 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -48,6 +48,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.NereidsException; import org.apache.doris.common.QueryTimeoutException; import org.apache.doris.common.Status; @@ -729,7 +730,15 @@ public void execute(TUniqueId queryId) throws Exception { throw new UserException(e.getMessage()); } LOG.warn("Analyze failed. {}", context.getQueryIdentifier(), e); - context.getState().setError(e.getMessage()); + // Planning wraps the window rejection in NereidsException/AnalysisException. + // NereidsException(Exception) keeps its wrapped exception outside Throwable.cause. + Throwable cause = e instanceof NereidsException + ? Util.getRootCause(((NereidsException) e).getException()) : e; + if (cause instanceof IncrWindowNotReadyException) { + context.getState().setError(ErrorCode.ERR_INCR_WINDOW_NOT_READY, e.getMessage()); + } else { + context.getState().setError(e.getMessage()); + } return; } catch (Exception e) { LOG.warn("Nereids execute failed. {}", context.getQueryIdentifier(), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiter.java b/fe/fe-core/src/main/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiter.java index 8faeb7f51ee490..e750ce48caa007 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiter.java @@ -26,6 +26,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ClientPool; import org.apache.doris.common.Config; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.UserException; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.trees.plans.Plan; @@ -55,7 +56,8 @@ /** * Establishes a closed upper fence before planning a time-based incremental read. * - *

The master FE first captures its TSO, validates an explicit end timestamp, then captures a + *

Bounded, strongly consistent cloud reads use the master's durable committed TSO. Other reads + * retain the transaction drain: the master validates its TSO, then captures a * transaction ID watermark and drains earlier transactions involving the target tables. In classic * mode, it also synchronizes with transaction publishers through the target table locks and returns * a journal watermark for a follower FE to replay. In cloud mode, partition versions are refreshed @@ -68,12 +70,22 @@ public class TimeBasedChangeVisibleWaiter { public static final class ChangeReadFence { private final long currentTso; private final long maxJournalId; + private final long committedTso; public ChangeReadFence(long currentTso, long maxJournalId) { + this(currentTso, maxJournalId, 0); + } + + public ChangeReadFence(long currentTso, long maxJournalId, long committedTso) { + this.committedTso = committedTso; this.currentTso = currentTso; this.maxJournalId = maxJournalId; } + public long getCommittedTso() { + return committedTso; + } + public long getCurrentTso() { return currentTso; } @@ -87,12 +99,18 @@ public long getMaxJournalId() { static final class ChangeReadInfo { private final Map> dbToTableIds; private final Long maxEndTimestampMs; + private final boolean allEndsExplicit; - private ChangeReadInfo(Map> dbToTableIds, Long maxEndTimestampMs) { + private ChangeReadInfo(Map> dbToTableIds, Long maxEndTimestampMs, boolean allEndsExplicit) { + this.allEndsExplicit = allEndsExplicit; this.dbToTableIds = dbToTableIds; this.maxEndTimestampMs = maxEndTimestampMs; } + boolean isAllEndsExplicit() { + return allEndsExplicit; + } + Map> getDbToTableIds() { return dbToTableIds; } @@ -124,7 +142,8 @@ public static void waitForVisible(ConnectContext context, Plan plan, Map> dbToTableIds, Long maxEndTimestampMs, long timeoutMs, boolean waitForTransactions) throws UserException { + return acquireFenceOnMaster(dbToTableIds, maxEndTimestampMs, timeoutMs, waitForTransactions, false); + } + + public static ChangeReadFence acquireFenceOnMaster(Map> dbToTableIds, + Long maxEndTimestampMs, long timeoutMs, boolean waitForTransactions, boolean allEndsExplicit) + throws UserException { Env env = Env.getCurrentEnv(); if (!env.isMaster()) { throw new UserException("time-based change read fence must be acquired on the master FE"); @@ -158,6 +183,17 @@ public static ChangeReadFence acquireFenceOnMaster(Map> dbToTab throw new UserException("TSO timestamp is not calibrated, please check"); } long currentTso = tsoSnapshot.getCurrentTso(); + if (Config.isCloudMode() && waitForTransactions && allEndsExplicit) { + Preconditions.checkArgument(maxEndTimestampMs != null, "bounded read requires an end timestamp"); + long committedTso = tsoSnapshot.getCommittedTso(); + if (committedTso == 0 || maxEndTimestampMs > TSOTimestamp.extractPhysicalTime(committedTso)) { + throw new IncrWindowNotReadyException(maxEndTimestampMs, committedTso, + Config.tso_service_window_duration_ms); + } + // Partition versions are still refreshed from MS during planning. No transaction + // watermark/RPC is needed for a window already covered by the durable prefix. + return new ChangeReadFence(currentTso, env.getMaxJournalId(), committedTso); + } validateEndTimestamp(maxEndTimestampMs, currentTso); if (waitForTransactions) { @@ -182,6 +218,7 @@ static ChangeReadInfo collectChangeReadInfo(ConnectContext context, Plan plan, Map, TableIf> tables) { Map> dbToTableIdSets = new TreeMap<>(); long[] maxEndTimestampMs = {-1L}; + boolean[] allEndsExplicit = {true}; plan.foreach(node -> { if (!(node instanceof UnboundRelation)) { return; @@ -203,13 +240,18 @@ static ChangeReadInfo collectChangeReadInfo(ConnectContext context, Plan plan, scanParams.getMapParams().get(OlapScanNode.OLAP_END_TIMESTAMP)); if (endTimestampMs > 0) { maxEndTimestampMs[0] = Math.max(maxEndTimestampMs[0], endTimestampMs); + } else { + allEndsExplicit[0] = false; } + } else { + allEndsExplicit[0] = false; } }); Map> dbToTableIds = new TreeMap<>(); dbToTableIdSets.forEach((dbId, tableIds) -> dbToTableIds.put(dbId, new ArrayList<>(tableIds))); - return new ChangeReadInfo(dbToTableIds, maxEndTimestampMs[0] < 0 ? null : maxEndTimestampMs[0]); + return new ChangeReadInfo(dbToTableIds, maxEndTimestampMs[0] < 0 ? null : maxEndTimestampMs[0], + allEndsExplicit[0]); } private static void validateEndTimestamp(Long maxEndTimestampMs, long currentTso) throws UserException { @@ -297,6 +339,7 @@ private static ChangeReadFence acquireFenceFromMaster(ConnectContext context, Ch request.setDbToTableIds(changeReadInfo.getDbToTableIds()); request.setTimeoutMs(timeoutMs); request.setWaitForTransactions(waitForTransactions); + request.setAllEndsExplicit(changeReadInfo.isAllEndsExplicit()); if (changeReadInfo.getMaxEndTimestampMs() != null) { request.setEndTimestampMs(changeReadInfo.getMaxEndTimestampMs()); } @@ -315,6 +358,10 @@ private static ChangeReadFence acquireFenceFromMaster(ConnectContext context, Ch try { TAcquireTimeBasedChangeReadFenceResult result = client.acquireTimeBasedChangeReadFence(request); returnToPool = true; + if (result.isSetWindowNotReady()) { + throw new IncrWindowNotReadyException(result.getWindowNotReady().getRequestedEndTimestampMs(), + result.getWindowNotReady().getCommittedTso(), result.getWindowNotReady().getRetryAfterMs()); + } if (result.getStatus().getStatusCode() != TStatusCode.OK) { String error = result.getStatus().isSetErrorMsgs() ? String.join(". ", result.getStatus().getErrorMsgs()) @@ -323,7 +370,7 @@ private static ChangeReadFence acquireFenceFromMaster(ConnectContext context, Ch } Preconditions.checkState(result.isSetCurrentTso(), "master FE did not return current_tso"); Preconditions.checkState(result.isSetMaxJournalId(), "master FE did not return max_journal_id"); - return new ChangeReadFence(result.getCurrentTso(), result.getMaxJournalId()); + return new ChangeReadFence(result.getCurrentTso(), result.getMaxJournalId(), result.getCommittedTso()); } catch (UserException e) { throw e; } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 3bf85463ce5ec4..19ee038ecaae7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -69,6 +69,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.DuplicatedRequestException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.InternalErrorCode; import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.LoadException; @@ -232,6 +233,7 @@ import org.apache.doris.thrift.TGetTabletReplicaInfosRequest; import org.apache.doris.thrift.TGetTabletReplicaInfosResult; import org.apache.doris.thrift.TGroupCommitInfo; +import org.apache.doris.thrift.TIncrWindowNotReady; import org.apache.doris.thrift.TInitExternalCtlMetaRequest; import org.apache.doris.thrift.TInitExternalCtlMetaResult; import org.apache.doris.thrift.TInsertOverwriteRecordRequest; @@ -3402,9 +3404,15 @@ public TAcquireTimeBasedChangeReadFenceResult acquireTimeBasedChangeReadFence( TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( request.getDbToTableIds(), request.isSetEndTimestampMs() ? request.getEndTimestampMs() : null, - request.getTimeoutMs(), request.isWaitForTransactions()); + request.getTimeoutMs(), request.isWaitForTransactions(), request.isAllEndsExplicit()); result.setCurrentTso(fence.getCurrentTso()); result.setMaxJournalId(fence.getMaxJournalId()); + result.setCommittedTso(fence.getCommittedTso()); + } catch (IncrWindowNotReadyException e) { + status.setStatusCode(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs(e.getDetailMessage()); + result.setWindowNotReady(new TIncrWindowNotReady( + e.getRequestedEndTimestampMs(), e.getCommittedTso(), e.getRetryAfterMs())); } catch (UserException e) { status.setStatusCode(TStatusCode.ANALYSIS_ERROR); status.addToErrorMsgs(e.getDetailMessage()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java index e64690a54cb10f..6dec245e25888c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java @@ -20,10 +20,12 @@ package org.apache.doris.service.arrowflight; +import org.apache.doris.common.ErrorCode; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.Util; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QueryState; import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.service.arrowflight.results.FlightSqlEndpointsLocation; import org.apache.doris.service.arrowflight.results.FlightSqlResultCacheEntry; @@ -39,9 +41,11 @@ import org.apache.arrow.flight.CloseSessionRequest; import org.apache.arrow.flight.CloseSessionResult; import org.apache.arrow.flight.Criteria; +import org.apache.arrow.flight.ErrorFlightMetadata; import org.apache.arrow.flight.FlightDescriptor; import org.apache.arrow.flight.FlightEndpoint; import org.apache.arrow.flight.FlightInfo; +import org.apache.arrow.flight.FlightRuntimeException; import org.apache.arrow.flight.FlightStream; import org.apache.arrow.flight.Location; import org.apache.arrow.flight.PutResult; @@ -296,18 +300,32 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con + ", error code: " + connectContext.getState().getErrorCode() + ", error msg: " + connectContext.getState().getErrorMessage(); LOG.error(errMsg, e); - throw CallStatus.INTERNAL.withDescription(errMsg).withCause(e).toRuntimeException(); + throw queryFailure(connectContext.getState(), errMsg, e); } finally { connectContext.setCommand(MysqlCommand.COM_SLEEP); } } + static FlightRuntimeException queryFailure(QueryState state, String message, Throwable cause) { + if (state.getErrorCode() == ErrorCode.ERR_INCR_WINDOW_NOT_READY) { + ErrorFlightMetadata metadata = new ErrorFlightMetadata(); + metadata.insert("doris-error-code", Integer.toString(ErrorCode.ERR_INCR_WINDOW_NOT_READY.getCode())); + metadata.insert("doris-error-name", "ERR_INCR_WINDOW_NOT_READY"); + // The description preserves the requested end, committed prefix and retry delay from QueryState. + return CallStatus.UNAVAILABLE.withDescription(message).withCause(cause) + .withMetadata(metadata).toRuntimeException(); + } + return CallStatus.INTERNAL.withDescription(message).withCause(cause).toRuntimeException(); + } + @Override public FlightInfo getFlightInfoStatement(final CommandStatementQuery request, final CallContext context, final FlightDescriptor descriptor) { try { ConnectContext connectContext = flightSessionsManager.getConnectContext(context.peerIdentity()); return executeQueryStatement(context.peerIdentity(), connectContext, request.getQuery(), descriptor); + } catch (FlightRuntimeException e) { + throw e; } catch (Throwable e) { String errMsg = "get flight info statement failed, " + e.getMessage(); LOG.error(errMsg, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 6313f3bdf6fd7c..ebb7352639240b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -367,7 +367,7 @@ public static TFetchSchemaTableDataResult getSchemaTableData(TFetchSchemaTableDa columnIndex = TABLE_STREAM_CONSUMPTION_COLUMN_TO_INDEX; break; case TSO_STATUS: - result = tsoStatusMetadataResult(); + result = tsoStatusMetadataResult(schemaTableParams.isSetColumnsName()); columnIndex = TSO_STATUS_COLUMN_TO_INDEX; break; case STATISTICS: @@ -2335,7 +2335,7 @@ private static TFetchSchemaTableDataResult streamConsumptionMetadataResult(TSche return result; } - private static TFetchSchemaTableDataResult tsoStatusMetadataResult() { + private static TFetchSchemaTableDataResult tsoStatusMetadataResult(boolean includeCommittedTso) { if (!Config.enable_feature_binlog) { return errorResult("TSO feature is disabled, please check enable_feature_binlog"); } @@ -2351,6 +2351,14 @@ private static TFetchSchemaTableDataResult tsoStatusMetadataResult() { row.addToColumnValue(new TCell().setLongVal(currentTso)); row.addToColumnValue(new TCell().setLongVal(TSOTimestamp.extractPhysicalTime(currentTso))); row.addToColumnValue(new TCell().setLongVal(TSOTimestamp.extractLogicalCounter(currentTso))); + // Older BE scanners omit columns_name and require exactly the original four columns. + if (includeCommittedTso) { + long committedTso = statusSnapshot.getCommittedTso(); + row.addToColumnValue(committedTso == 0 ? new TCell().setIsNull(true) + : new TCell().setLongVal(committedTso)); + row.addToColumnValue(committedTso == 0 ? new TCell().setIsNull(true) + : new TCell().setLongVal(TSOTimestamp.extractPhysicalTime(committedTso))); + } TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); result.setDataBatch(Lists.newArrayList(row)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java index 0838d12e0f0b61..ef7d42cfa3638a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java @@ -145,6 +145,11 @@ public void abortTransaction(Long dbId, Long txnId, String reason, public void finishTransaction(long dbId, long transactionId, Map partitionVisibleVersions, Map> backendPartitions) throws UserException; + /** Instance-wide, timeout-independent check of the exclusive recovery transaction bound. */ + default boolean isPreviousTransactionsFinishedForTsoRecovery(long endTransactionId) throws UserException { + throw new UserException("Strict TSO recovery is only supported in cloud mode"); + } + public boolean isPreviousTransactionsFinished(long endTransactionId, long dbId, List tableIdList) throws AnalysisException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java index aea3478bf51f7b..19d5638f218eac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java @@ -103,7 +103,8 @@ public static long getCommitTSO(long transactionId, Database db, Set table throw new TransactionCommitFailedException("failed to get TSO for txn " + transactionId + ": TSO service is unavailable"); } - long fetched = env.getTSOService().getTSO(); + long fetched = Config.isCloudMode() + ? env.getTSOService().getCommitTSO(db.getId(), transactionId) : env.getTSOService().getTSO(); if (fetched <= 0) { throw new TransactionCommitFailedException("failed to get TSO for txn " + transactionId + ", fetched=" + fetched); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java index 2acfff15564ad4..193c82d07be64f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java @@ -23,6 +23,8 @@ import org.apache.doris.common.io.CountingDataOutputStream; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.journal.local.LocalJournal; +import org.apache.doris.metric.GaugeMetric; +import org.apache.doris.metric.Metric.MetricUnit; import org.apache.doris.metric.MetricRepo; import org.apache.doris.persist.EditLog; @@ -31,9 +33,12 @@ import java.io.DataInputStream; import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.LongSupplier; public class TSOService extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(TSOService.class); @@ -47,7 +52,37 @@ public class TSOService extends MasterDaemon { private final AtomicBoolean isInitialized = new AtomicBoolean(false); private final AtomicBoolean fatalClockBackwardReported = new AtomicBoolean(false); - private final AtomicLong windowEndTSO = new AtomicLong(0); + private volatile TSOServiceState durableState = new TSOServiceState(0, 0); + private final TSOTransactionTracker transactionTracker = new TSOTransactionTracker(lock); + private long lastPersistNanos; + private final MasterDaemon transactionChecker = new MasterDaemon("TSO-transaction-checker", 1000) { + private long lastFailureLogNanos; + + @Override + protected void runAfterCatalogReady() { + if (!Config.isCloudMode() || !isTsoEnabled() || !isInitialized.get() + || !Env.getCurrentEnv().isMaster()) { + return; + } + long startNanos = System.nanoTime(); + try { + transactionTracker.checkTransactions(Env.getCurrentGlobalTransactionMgr(), startNanos); + } catch (Exception e) { + if (lastFailureLogNanos == 0 || startNanos - lastFailureLogNanos >= TimeUnit.MINUTES.toNanos(1)) { + LOG.warn("Failed to reconcile TSO transactions; retaining the committed TSO", e); + lastFailureLogNanos = startNanos; + } + if (MetricRepo.isInit) { + MetricRepo.COUNTER_TSO_RECONCILE_FAILED.increase(1L); + } + } finally { + if (MetricRepo.isInit) { + MetricRepo.HISTO_TSO_RECONCILE_LATENCY.update( + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos)); + } + } + } + }; /** * Immutable snapshot of the current TSO service status. @@ -56,11 +91,21 @@ public static final class TSOStatusSnapshot { private final boolean initialized; private final long currentTso; private final long windowEndPhysicalTime; + private final long committedTso; public TSOStatusSnapshot(boolean initialized, long currentTso, long windowEndPhysicalTime) { + this(initialized, currentTso, windowEndPhysicalTime, 0); + } + + public TSOStatusSnapshot(boolean initialized, long currentTso, long windowEndPhysicalTime, long committedTso) { this.initialized = initialized; this.currentTso = currentTso; this.windowEndPhysicalTime = windowEndPhysicalTime; + this.committedTso = committedTso; + } + + public long getCommittedTso() { + return committedTso; } public boolean isInitialized() { @@ -89,12 +134,32 @@ public TSOService() { super("TSO-service", Config.tso_service_update_interval_ms); } + public void registerMetrics() { + Map gauges = new LinkedHashMap<>(); + gauges.put("tso_committed", () -> durableState.getCommittedTso()); + gauges.put("tso_window_end_physical_time", () -> durableState.getPhysicalTimestamp()); + gauges.put("tso_pending_transactions", transactionTracker::getPendingCount); + gauges.put("tso_oldest_pending_tso", transactionTracker::getOldestPendingTso); + gauges.put("tso_oldest_pending_txn_id", transactionTracker::getOldestPendingTxnId); + gauges.put("tso_oldest_pending_age_ms", transactionTracker::getOldestPendingAgeMs); + gauges.put("tso_recovery_ready", () -> transactionTracker.isRecoveryReady() ? 1 : 0); + gauges.put("tso_recovery_watermark", transactionTracker::getRecoveryWatermark); + gauges.forEach((name, value) -> MetricRepo.DORIS_METRIC_REGISTER.addMetrics( + new GaugeMetric(name, MetricUnit.NOUNIT, name) { + @Override + public Long getValue() { + return value.getAsLong(); + } + })); + } + /** * Start the TSO service. */ @Override public synchronized void start() { super.start(); + transactionChecker.start(); } /** @@ -187,6 +252,18 @@ protected void runAfterCatalogReady() { * @throws RuntimeException if TSO is not calibrated or other errors occur */ public long getTSO() { + return getTSO(null); + } + + public long getCommitTSO(long dbId, long txnId) { + return getTSO(Pair.of(dbId, txnId)); + } + + public void transactionFinished(long dbId, long txnId) { + transactionTracker.transactionFinished(dbId, txnId); + } + + private long getTSO(Pair transactionIdentity) { if (!isTsoEnabled()) { throw new RuntimeException("TSO feature is disabled, please check enable_feature_binlog"); } @@ -220,7 +297,7 @@ public long getTSO() { continue; } - Pair pair = generateTSO(); + Pair pair = generateTSO(transactionIdentity); long physical = pair.first; long logical = pair.second; @@ -270,8 +347,9 @@ public long getCurrentTSO() { public TSOStatusSnapshot getStatusSnapshot() { lock.lock(); try { - return new TSOStatusSnapshot( - isInitialized.get(), globalTimestamp.composeTimestamp(), windowEndTSO.get()); + TSOServiceState state = durableState; + return new TSOStatusSnapshot(isInitialized.get(), globalTimestamp.composeTimestamp(), + state.getPhysicalTimestamp(), state.getCommittedTso()); } finally { lock.unlock(); } @@ -296,7 +374,7 @@ private void calibrateTimestamp() { return; } - long timeLast = windowEndTSO.get(); // Last timestamp from image/editlog replay + long timeLast = durableState.getPhysicalTimestamp(); // Last timestamp from image/editlog replay long timeNow = System.currentTimeMillis() + Config.tso_time_offset_debug_mode; long backwardMs = timeLast - timeNow; if (backwardMs > Config.tso_clock_backward_startup_threshold_ms) { @@ -313,13 +391,19 @@ private void calibrateTimestamp() { nextPhysicalTime = timeNow; } + lock.lock(); + try { + transactionTracker.reset(System.nanoTime(), Config.tso_service_window_duration_ms + 1000L); + } finally { + lock.unlock(); + } + // Construct new timestamp (physical time with reset logical counter) setTSOPhysical(nextPhysicalTime, true); // Write the right boundary of time window to BDBJE for persistence long timeWindowEnd = nextPhysicalTime + Config.tso_service_window_duration_ms; writeTimestampToBDBJE(timeWindowEnd); - windowEndTSO.set(timeWindowEnd); isInitialized.set(true); fatalClockBackwardReported.set(false); @@ -400,11 +484,12 @@ private void updateTimestamp() { } // 4. Check if time window right boundary needs renewal - if ((windowEndTSO.get() - nextPhysicalTime) <= UPDATE_TIME_WINDOW_GUARD) { - // Time window right boundary needs renewal - long nextWindowEnd = nextPhysicalTime + Config.tso_service_window_duration_ms; + if ((durableState.getPhysicalTimestamp() - nextPhysicalTime) <= UPDATE_TIME_WINDOW_GUARD + || System.nanoTime() - lastPersistNanos + >= TimeUnit.MILLISECONDS.toNanos(Config.tso_service_window_duration_ms)) { + long nextWindowEnd = Math.max(durableState.getPhysicalTimestamp(), + nextPhysicalTime + Config.tso_service_window_duration_ms); writeTimestampToBDBJE(nextWindowEnd); - windowEndTSO.set(nextWindowEnd); } // 5. Update global timestamp @@ -445,7 +530,16 @@ private void writeTimestampToBDBJE(long timestamp) { + "timestamp=" + timestamp); } - TSOTimestamp tsoTimestamp = new TSOTimestamp(timestamp, 0); + TSOServiceState nextState; + lock.lock(); + try { + long committedTso = Config.isCloudMode() + ? transactionTracker.candidateCommittedTso(globalTimestamp.composeTimestamp(), + durableState.getCommittedTso()) : 0; + nextState = new TSOServiceState(timestamp, committedTso); + } finally { + lock.unlock(); + } // Check if EditLog is available EditLog editLog = env.getEditLog(); @@ -467,10 +561,25 @@ private void writeTimestampToBDBJE(long timestamp) { } } + long persistStartNanos = System.nanoTime(); try { - editLog.logTSOTimestampWindowEnd(tsoTimestamp); + editLog.logTSOTimestampWindowEnd(nextState); + // Readers must never observe a candidate that has not survived a journal write. + durableState = nextState; + lastPersistNanos = System.nanoTime(); + if (MetricRepo.isInit) { + MetricRepo.COUNTER_TSO_STATE_PERSISTED.increase(1L); + } } catch (Exception e) { + if (MetricRepo.isInit) { + MetricRepo.COUNTER_TSO_STATE_PERSIST_FAILED.increase(1L); + } throw new RuntimeException("Failed to write TSO timestamp to BDBJE, timestamp=" + timestamp, e); + } finally { + if (MetricRepo.isInit) { + MetricRepo.HISTO_TSO_STATE_PERSIST_LATENCY.update( + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - persistStartNanos)); + } } } @@ -480,6 +589,10 @@ private void writeTimestampToBDBJE(long timestamp) { * @return Pair of (physicalTime, updatedLogicalCounter) for the base timestamp */ private Pair generateTSO() { + return generateTSO(null); + } + + private Pair generateTSO(Pair transactionIdentity) { lock.lock(); try { if (!isTsoEnabled() || !isInitialized.get()) { @@ -495,6 +608,10 @@ private Pair generateTSO() { } long nextLogical = logicalCounter + 1; globalTimestamp.setLogicalCounter(nextLogical); + if (transactionIdentity != null) { + transactionTracker.register(transactionIdentity, + TSOTimestamp.composeTimestamp(physicalTime, nextLogical), System.nanoTime()); + } return Pair.of(physicalTime, nextLogical); } finally { lock.unlock(); @@ -530,34 +647,34 @@ private void setTSOPhysical(long next, boolean force) { * * @param windowEnd New window end physical time */ - public void replayWindowEndTSO(TSOTimestamp windowEnd) { - windowEndTSO.set(windowEnd.getPhysicalTimestamp()); + public void replayWindowEndTSO(TSOServiceState state) { + durableState = state; } public long getWindowEndTSO() { - return windowEndTSO.get(); + return durableState.getPhysicalTimestamp(); } public long saveTSO(CountingDataOutputStream dos, long checksum) throws IOException { if (!isTsoEnabled()) { return checksum; } - long currentWindowEnd = windowEndTSO.get(); + TSOServiceState state = durableState; + long currentWindowEnd = state.getPhysicalTimestamp(); if (currentWindowEnd <= 0) { return checksum; } - TSOTimestamp tsoTimestamp = new TSOTimestamp(currentWindowEnd, 0); - tsoTimestamp.write(dos); - checksum ^= tsoTimestamp.getPhysicalTimestamp(); - LOG.info("Save TSO windowEndTSO {} to image", tsoTimestamp); + state.write(dos); + checksum ^= currentWindowEnd; + LOG.info("Save TSO window end {} and committed TSO {} to image", currentWindowEnd, state.getCommittedTso()); return checksum; } public long loadTSO(DataInputStream dis, long checksum) throws IOException { - TSOTimestamp tsoTimestamp = TSOTimestamp.read(dis); - windowEndTSO.set(tsoTimestamp.getPhysicalTimestamp()); - long newChecksum = checksum ^ tsoTimestamp.getPhysicalTimestamp(); - LOG.info("Finished replay TSO windowEndTSO {} from image", windowEndTSO.get()); + TSOServiceState state = TSOServiceState.read(dis); + durableState = state; + long newChecksum = checksum ^ state.getPhysicalTimestamp(); + LOG.info("Finished replay TSO windowEndTSO {} from image", durableState.getPhysicalTimestamp()); return newChecksum; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOServiceState.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOServiceState.java new file mode 100644 index 00000000000000..cd658da6e06654 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOServiceState.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tso; + +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** The durable allocation window and readable transaction prefix, published as one snapshot. */ +public final class TSOServiceState implements Writable { + // Keep the old TSOTimestamp JSON fields and image checksum for journal/image compatibility. + @SerializedName("physicalTimestamp") + private final long windowEndPhysicalTime; + @SerializedName("logicalCounter") + private final long logicalCounter = 0; + @SerializedName("committedTso") + private final long committedTso; + + public TSOServiceState(long windowEndPhysicalTime, long committedTso) { + this.windowEndPhysicalTime = windowEndPhysicalTime; + this.committedTso = committedTso; + } + + public long getPhysicalTimestamp() { + return windowEndPhysicalTime; + } + + /** Zero means no readable prefix has been established, including records written by older FEs. */ + public long getCommittedTso() { + return committedTso; + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + + public static TSOServiceState read(DataInput in) throws IOException { + TSOServiceState state = GsonUtils.GSON.fromJson(Text.readString(in), TSOServiceState.class); + if (state == null) { + throw new IOException("failed to deserialize TSO service state from journal/image"); + } + return state; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java new file mode 100644 index 00000000000000..07cffa789898d4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java @@ -0,0 +1,246 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tso; + +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; +import org.apache.doris.transaction.GlobalTransactionMgrIface; +import org.apache.doris.transaction.TransactionState; +import org.apache.doris.transaction.TransactionStatus; + +import com.google.common.base.Preconditions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +/** In-memory commit registrations. Uses the allocator's lock so no allocated TSO can be missed. */ +final class TSOTransactionTracker { + private static final Logger LOG = LogManager.getLogger(TSOTransactionTracker.class); + private static final int CHECK_BATCH_SIZE = 64; + private static final long CHECK_AGE_NANOS = TimeUnit.SECONDS.toNanos(1); + private final ReentrantLock lock; + private final Map, PendingTransaction> pendingByTxn = new HashMap<>(); + private final TreeMap pendingByTso = new TreeMap<>(); + private long generation; + private long recoveryDeadlineNanos; + private long recoveryWatermark; + private boolean recoveryReady; + private long pollCursor; + + private static final class PendingTransaction { + private final Pair identity; + private final long tso; + private final long registeredAtNanos; + + private PendingTransaction(Pair identity, long tso, long nowNanos) { + this.identity = identity; + this.tso = tso; + this.registeredAtNanos = nowNanos; + } + } + + TSOTransactionTracker(ReentrantLock lock) { + this.lock = lock; + } + + void reset(long nowNanos, long recoveryDelayMs) { + Preconditions.checkState(lock.isHeldByCurrentThread()); + generation++; + pendingByTxn.clear(); + pendingByTso.clear(); + recoveryDeadlineNanos = nowNanos + TimeUnit.MILLISECONDS.toNanos(recoveryDelayMs); + recoveryWatermark = 0; + recoveryReady = false; + pollCursor = 0; + } + + void register(Pair identity, long tso, long nowNanos) { + Preconditions.checkState(lock.isHeldByCurrentThread()); + if (pendingByTxn.containsKey(identity)) { + // A timed-out request can still commit using the earlier TSO. + return; + } + PendingTransaction pending = new PendingTransaction(identity, tso, nowNanos); + pendingByTxn.put(identity, pending); + Preconditions.checkState(pendingByTso.put(tso, pending) == null); + } + + long candidateCommittedTso(long currentTso, long durableCommittedTso) { + Preconditions.checkState(lock.isHeldByCurrentThread()); + if (!recoveryReady) { + return durableCommittedTso; + } + long candidate = pendingByTso.isEmpty() ? currentTso + : Math.min(currentTso, pendingByTso.firstKey() - 1); + Preconditions.checkState(candidate >= durableCommittedTso, "committed TSO must not regress"); + return candidate; + } + + void transactionFinished(long dbId, long txnId) { + lock.lock(); + try { + PendingTransaction pending = pendingByTxn.remove(Pair.of(dbId, txnId)); + if (pending != null) { + pendingByTso.remove(pending.tso); + } + } finally { + lock.unlock(); + } + } + + /** Runs on a separate daemon. Neither recovery nor transaction RPCs hold the allocator lock. */ + void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws UserException { + long checkGeneration; + long watermark; + boolean checkRecovery; + List batch = new ArrayList<>(); + lock.lock(); + try { + checkGeneration = generation; + watermark = recoveryWatermark; + checkRecovery = !recoveryReady && nowNanos - recoveryDeadlineNanos >= 0; + if (!pendingByTso.isEmpty()) { + // Always check the transaction blocking the prefix, then rotate through the rest. + PendingTransaction oldest = pendingByTso.firstEntry().getValue(); + if (nowNanos - oldest.registeredAtNanos >= CHECK_AGE_NANOS) { + batch.add(oldest); + } + for (int i = 0; i < Math.min(CHECK_BATCH_SIZE - 1, pendingByTso.size()); i++) { + Map.Entry next = pendingByTso.higherEntry(pollCursor); + if (next == null) { + next = pendingByTso.firstEntry(); + } + pollCursor = next.getKey(); + PendingTransaction pending = next.getValue(); + if (pending != oldest && nowNanos - pending.registeredAtNanos >= CHECK_AGE_NANOS) { + batch.add(pending); + } + } + } + } finally { + lock.unlock(); + } + + // Reconcile registrations even while the recovery scan is failing or waiting on old transactions. + for (PendingTransaction pending : batch) { + TransactionState state = txnMgr.getTransactionState(pending.identity.first, pending.identity.second); + // null includes RPC errors and NOT_FOUND; neither proves that a transaction is finished. + if (state == null || (state.getTransactionStatus() != TransactionStatus.VISIBLE + && state.getTransactionStatus() != TransactionStatus.ABORTED)) { + continue; + } + lock.lock(); + try { + if (generation == checkGeneration && pendingByTxn.get(pending.identity) == pending) { + pendingByTxn.remove(pending.identity); + pendingByTso.remove(pending.tso); + } + } finally { + lock.unlock(); + } + } + + if (checkRecovery) { + if (watermark == 0) { + watermark = txnMgr.getTransactionIdWatermark(); + Preconditions.checkState(watermark > 0, "invalid recovery transaction watermark"); + lock.lock(); + try { + if (generation != checkGeneration) { + return; + } + recoveryWatermark = watermark; + } finally { + lock.unlock(); + } + } + boolean finished = txnMgr.isPreviousTransactionsFinishedForTsoRecovery(watermark); + lock.lock(); + try { + if (generation == checkGeneration && finished) { + recoveryReady = true; + LOG.info("TSO recovery completed, transaction watermark={}", watermark); + } + } finally { + lock.unlock(); + } + } + } + + long getPendingCount() { + lock.lock(); + try { + return pendingByTxn.size(); + } finally { + lock.unlock(); + } + } + + long getOldestPendingTso() { + lock.lock(); + try { + return pendingByTso.isEmpty() ? 0 : pendingByTso.firstKey(); + } finally { + lock.unlock(); + } + } + + long getOldestPendingTxnId() { + lock.lock(); + try { + return pendingByTso.isEmpty() ? 0 : pendingByTso.firstEntry().getValue().identity.second; + } finally { + lock.unlock(); + } + } + + long getOldestPendingAgeMs() { + lock.lock(); + try { + return pendingByTso.isEmpty() ? 0 : TimeUnit.NANOSECONDS.toMillis( + System.nanoTime() - pendingByTso.firstEntry().getValue().registeredAtNanos); + } finally { + lock.unlock(); + } + } + + long getRecoveryWatermark() { + lock.lock(); + try { + return recoveryWatermark; + } finally { + lock.unlock(); + } + } + + boolean isRecoveryReady() { + lock.lock(); + try { + return recoveryReady; + } finally { + lock.unlock(); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java new file mode 100644 index 00000000000000..c7ec27e130b8f4 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.transaction; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TabletInvertedIndex; +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; +import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; +import org.apache.doris.cloud.proto.Cloud.TxnInfoPB; +import org.apache.doris.cloud.rpc.MetaServiceProxy; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.transaction.TransactionUtil; +import org.apache.doris.tso.TSOService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class CloudCommittedTsoTest { + @Test + public void testLazyCommitResponseDoesNotReleaseTsoUntilReallyVisible() { + Env env = Mockito.mock(Env.class); + TSOService tsoService = Mockito.mock(TSOService.class); + Mockito.when(env.getTSOService()).thenReturn(tsoService); + CommitTxnResponse.Builder response = CommitTxnResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK)) + .setTxnInfo(TxnInfoPB.newBuilder().setStatus(Cloud.TxnStatusPB.TXN_STATUS_VISIBLE)) + .setIsLazyCommit(true).setIsLazyCommitIncomplete(true); + try (MockedStatic mocked = Mockito.mockStatic(Env.class)) { + mocked.when(Env::getCurrentEnv).thenReturn(env); + CloudGlobalTransactionMgr.releaseFinishedTso(1, 10, response.build()); + Mockito.verifyNoInteractions(tsoService); + response.setIsLazyCommitIncomplete(false); + CloudGlobalTransactionMgr.releaseFinishedTso(1, 10, response.build()); + Mockito.verify(tsoService).transactionFinished(1, 10); + response.setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.TXN_ALREADY_ABORTED)); + CloudGlobalTransactionMgr.releaseFinishedTso(2, 20, response.build()); + Mockito.verify(tsoService).transactionFinished(2, 20); + } + } + + @Test + public void testOrdinarySubTransactionAndTwoPhaseCommitReuseTsoAcrossRpcRetries() throws Exception { + Env env = Mockito.mock(Env.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database db = Mockito.mock(Database.class); + Mockito.when(db.getId()).thenReturn(1L); + Mockito.when(db.getTablesOnIdOrderOrThrowException(Mockito.anyList())).thenReturn(Collections.emptyList()); + Mockito.when(catalog.getDbOrMetaException(1L)).thenReturn(db); + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class); + MockedStatic mockedProxy = Mockito.mockStatic(MetaServiceProxy.class); + MockedStatic allocation = Mockito.mockStatic(TransactionUtil.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + mockedEnv.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + mockedProxy.when(MetaServiceProxy::getInstance).thenReturn(proxy); + allocation.when(() -> TransactionUtil.getCommitTSO(10L, db, Collections.emptySet())).thenReturn(500L); + for (int path = 0; path < 3; path++) { + List requests = new ArrayList<>(); + Mockito.when(proxy.commitTxn(Mockito.any())).thenAnswer(invocation -> { + requests.add(invocation.getArgument(0)); + return CommitTxnResponse.newBuilder().setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(requests.size() == 1 ? MetaServiceCode.KV_TXN_CONFLICT : MetaServiceCode.UNDEFINED_ERR) + .setMsg("injected commit failure")).build(); + }); + CloudGlobalTransactionMgr mgr = new CloudGlobalTransactionMgr(); + int selectedPath = path; + Assertions.assertThrows(UserException.class, () -> { + if (selectedPath == 0) { + mgr.commitTransactionWithoutLock(1, Collections.emptyList(), 10, null, null); + } else if (selectedPath == 1) { + mgr.commitAndPublishTransaction(db, 10, Collections.emptyList(), 1000); + } else { + mgr.commitTransaction2PC(db, Collections.emptyList(), 10, 1000); + } + }); + Assertions.assertEquals(2, requests.size()); + Assertions.assertSame(requests.get(0), requests.get(1)); + Assertions.assertEquals(500L, requests.get(0).getCommitTso()); + Assertions.assertEquals(path == 1, requests.get(0).getIsTxnLoad()); + Assertions.assertEquals(path == 2, requests.get(0).getIs2Pc()); + allocation.verify(() -> TransactionUtil.getCommitTSO(10L, db, Collections.emptySet())); + allocation.clearInvocations(); + } + } + } + + @Test + public void testValidationFailureDoesNotAllocateTsoOrSendCommit() throws Exception { + Env env = Mockito.mock(Env.class); + TabletInvertedIndex index = Mockito.mock(TabletInvertedIndex.class); + Mockito.when(env.getTabletInvertedIndex()).thenReturn(index); + Mockito.when(index.getTabletMetaList(Mockito.anyList())).thenThrow(new IllegalStateException("validation failed")); + Method commit = CloudGlobalTransactionMgr.class.getDeclaredMethod("commitTxn", Cloud.CommitTxnRequest.Builder.class, + List.class, long.class, boolean.class, List.class, List.class); + commit.setAccessible(true); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class); + MockedStatic allocation = Mockito.mockStatic(TransactionUtil.class); + MockedStatic proxy = Mockito.mockStatic(MetaServiceProxy.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Cloud.CommitTxnRequest.Builder builder = Cloud.CommitTxnRequest.newBuilder() + .setDbId(1).setTxnId(10).addBaseTabletIds(20); + InvocationTargetException error = Assertions.assertThrows(InvocationTargetException.class, + () -> commit.invoke(new CloudGlobalTransactionMgr(), builder, Collections.emptyList(), + 10L, false, Collections.emptyList(), Collections.emptyList())); + Assertions.assertEquals("validation failed", error.getCause().getMessage()); + allocation.verifyNoInteractions(); + proxy.verifyNoInteractions(); + Assertions.assertFalse(builder.hasCommitTso()); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java index 3d200ce80d3ec0..64a9f37d267e8f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java @@ -92,6 +92,30 @@ public void tearDown() { } } + @Test + public void testStrictRecoveryRequiresExplicitMsCapability() throws Exception { + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + try (MockedStatic mocked = Mockito.mockStatic(MetaServiceProxy.class)) { + mocked.when(MetaServiceProxy::getInstance).thenReturn(proxy); + CheckTxnConflictResponse.Builder response = CheckTxnConflictResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK)) + .setFinished(true); + Mockito.when(proxy.checkTxnConflict(Mockito.any())).thenReturn(response.build()); + Assertions.assertThrows(UserException.class, + () -> masterTransMgr.isPreviousTransactionsFinishedForTsoRecovery(1000)); + Mockito.when(proxy.checkTxnConflict(Mockito.any())).thenReturn( + response.setStrictRecoveryCheckApplied(true).build()); + Assertions.assertTrue(masterTransMgr.isPreviousTransactionsFinishedForTsoRecovery(1000)); + ArgumentCaptor capture = + ArgumentCaptor.forClass(Cloud.CheckTxnConflictRequest.class); + Mockito.verify(proxy, Mockito.times(2)).checkTxnConflict(capture.capture()); + Assertions.assertTrue(capture.getValue().getStrictRecoveryCheck()); + Assertions.assertEquals(1000, capture.getValue().getEndTxnId()); + Assertions.assertFalse(capture.getValue().hasDbId()); + Assertions.assertEquals(0, capture.getValue().getTableIdsCount()); + } + } + @Test public void testBeginTransaction() throws Exception { AtomicLong id = new AtomicLong(1000); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index fafcc0d8bef65a..fdec02c8654e92 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -17,6 +17,7 @@ package org.apache.doris.qe; +import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.InternalSchemaInitializer; @@ -24,11 +25,15 @@ import org.apache.doris.catalog.ResourceMgr; import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.IncrWindowNotReadyException; +import org.apache.doris.common.NereidsException; import org.apache.doris.common.Status; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlSerializer; import org.apache.doris.mysql.authenticate.TestLogAppender; +import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.Planner; import org.apache.doris.planner.ResultFileSink; @@ -42,6 +47,7 @@ import com.google.common.collect.Lists; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -70,6 +76,23 @@ protected void runBeforeAll() throws Exception { createDatabase("testDb"); } + @Test + public void testCommittedTsoErrorSurvivesPlannerWrapping() throws Exception { + connectContext.getState().reset(); + IncrWindowNotReadyException rejected = new IncrWindowNotReadyException(2000, 1000, 1000); + StmtExecutor executor = new StmtExecutor(connectContext, "select 1"); + try (MockedConstruction planners = Mockito.mockConstruction(NereidsPlanner.class, + (planner, construction) -> Mockito.doThrow(new NereidsException(rejected.getMessage(), rejected)) + .when(planner).plan(Mockito.any(StatementBase.class), Mockito.any(TQueryOptions.class)))) { + executor.execute(); + Assertions.assertEquals(1, planners.constructed().size()); + } + Assertions.assertEquals(QueryState.MysqlStateType.ERR, connectContext.getState().getStateType()); + Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, connectContext.getState().getErrorCode()); + Assertions.assertTrue(connectContext.getState().getErrorMessage().contains("requestedEndTimestampMs=2000")); + Assertions.assertTrue(connectContext.getState().getErrorMessage().contains("retryAfterMs=1000")); + } + @Test public void testShow() throws Exception { StmtExecutor stmtExecutor = new StmtExecutor(connectContext, ""); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java index 0abf75f443c8ee..6997b380e7985a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java @@ -23,7 +23,11 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ClientPool; import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.GenericPool; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.UserException; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.nereids.analyzer.UnboundRelation; @@ -32,6 +36,11 @@ import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.planner.OlapScanNode; +import org.apache.doris.service.FrontendServiceImpl; +import org.apache.doris.thrift.FrontendService; +import org.apache.doris.thrift.TAcquireTimeBasedChangeReadFenceRequest; +import org.apache.doris.thrift.TAcquireTimeBasedChangeReadFenceResult; +import org.apache.doris.thrift.TStatusCode; import org.apache.doris.transaction.GlobalTransactionMgrIface; import org.apache.doris.tso.TSOService; import org.apache.doris.tso.TSOTimestamp; @@ -66,6 +75,7 @@ public void testCollectChangeReadInfoWithoutEndTimestamp() { Assertions.assertEquals(ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID)), result.getDbToTableIds()); Assertions.assertNull(result.getMaxEndTimestampMs()); + Assertions.assertFalse(result.isAllEndsExplicit()); } @Test @@ -84,6 +94,7 @@ public void testCollectChangeReadInfoMergesMaximumEndTimestamp() { Assertions.assertEquals(ImmutableList.of(TABLE_ID), result.getDbToTableIds().get(DB_ID)); Assertions.assertEquals(OlapScanNode.parseChangeTimestamp(endTimestamp2), result.getMaxEndTimestampMs()); + Assertions.assertTrue(result.isAllEndsExplicit()); } @Test @@ -213,6 +224,114 @@ public void testFenceFailsWhenConflictCheckFails() throws Exception { } } + @Test + public void testHistoricalWindowDoesNotWaitForTransactionsOutsideWindow() throws Exception { + Env env = mockMasterEnv(); + TSOService service = mockTsoService(env, CURRENT_TSO); + long committedTso = TSOTimestamp.composeTimestamp(CURRENT_PHYSICAL_TIME_MS - 1000, 17); + Mockito.when(service.getStatusSnapshot()).thenReturn( + new TSOService.TSOStatusSnapshot(true, CURRENT_TSO, CURRENT_PHYSICAL_TIME_MS + 1000, committedTso)); + GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(301L); + // A same-table write started after the historical end and remains running. + Mockito.when(txnMgr.isPreviousTransactionsFinished(301L, DB_ID, ImmutableList.of(TABLE_ID))) + .thenReturn(false); + try (MockedStatic config = Mockito.mockStatic(Config.class); + MockedStatic mocked = Mockito.mockStatic(Env.class)) { + config.when(Config::isCloudMode).thenReturn(true); + mocked.when(Env::getCurrentEnv).thenReturn(env); + mocked.when(Env::getCurrentGlobalTransactionMgr).thenReturn(txnMgr); + TimeBasedChangeVisibleWaiter.ChangeReadFence fence = TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( + ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID)), CURRENT_PHYSICAL_TIME_MS - 1000, 0, true, true); + Assertions.assertEquals(committedTso, fence.getCommittedTso()); + Mockito.verifyNoInteractions(txnMgr); + // Negative control: the legacy table-wide drain cannot distinguish this later write. + Assertions.assertThrows(UserException.class, () -> TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( + ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID)), CURRENT_PHYSICAL_TIME_MS - 1000, 0, true)); + } + } + + @Test + public void testBoundedReadRejectsUnknownOrInsufficientCommittedTso() throws Exception { + Env env = mockMasterEnv(); + TSOService service = mockTsoService(env, CURRENT_TSO); + GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); + try (MockedStatic config = Mockito.mockStatic(Config.class); + MockedStatic mocked = Mockito.mockStatic(Env.class)) { + config.when(Config::isCloudMode).thenReturn(true); + mocked.when(Env::getCurrentEnv).thenReturn(env); + mocked.when(Env::getCurrentGlobalTransactionMgr).thenReturn(txnMgr); + for (long committed : new long[] {0, TSOTimestamp.composeTimestamp(CURRENT_PHYSICAL_TIME_MS - 1, 17)}) { + Mockito.when(service.getStatusSnapshot()).thenReturn( + new TSOService.TSOStatusSnapshot(true, CURRENT_TSO, CURRENT_PHYSICAL_TIME_MS + 1000, committed)); + IncrWindowNotReadyException error = Assertions.assertThrows(IncrWindowNotReadyException.class, + () -> TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( + ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID)), + CURRENT_PHYSICAL_TIME_MS, 1000, true, true)); + Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, error.getMysqlErrorCode()); + Assertions.assertEquals(committed, error.getCommittedTso()); + Assertions.assertEquals(CURRENT_PHYSICAL_TIME_MS, error.getRequestedEndTimestampMs()); + } + Mockito.verifyNoInteractions(txnMgr); + } + } + + @Test + public void testMixedBoundedAndUnboundedRelationsRetainDrain() { + Plan plan = new LogicalJoin<>(JoinType.INNER_JOIN, + newChangeRelation(1, ImmutableMap.of(OlapScanNode.OLAP_END_TIMESTAMP, "2024-01-01 00:00:00")), + newChangeRelation(2, ImmutableMap.of()), null); + TimeBasedChangeVisibleWaiter.ChangeReadInfo info = TimeBasedChangeVisibleWaiter.collectChangeReadInfo( + mockContext(), plan, ImmutableMap.of(TABLE_QUALIFIER, mockOlapTable(DB_ID, TABLE_ID))); + Assertions.assertNotNull(info.getMaxEndTimestampMs()); + Assertions.assertFalse(info.isAllEndsExplicit()); + } + + @Test + public void testMasterRpcAndFollowerPreserveWindowNotReady() throws Exception { + Env master = mockMasterEnv(); + mockTsoService(master, CURRENT_TSO); // Unknown committed prefix. + TAcquireTimeBasedChangeReadFenceRequest request = new TAcquireTimeBasedChangeReadFenceRequest(); + request.setDbToTableIds(ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID))); + request.setEndTimestampMs(CURRENT_PHYSICAL_TIME_MS); + request.setTimeoutMs(1000); + request.setWaitForTransactions(true); + request.setAllEndsExplicit(true); + TAcquireTimeBasedChangeReadFenceResult response; + try (MockedStatic mocked = Mockito.mockStatic(Env.class); + MockedStatic config = Mockito.mockStatic(Config.class)) { + mocked.when(Env::getCurrentEnv).thenReturn(master); + config.when(Config::isCloudMode).thenReturn(true); + response = new FrontendServiceImpl(null).acquireTimeBasedChangeReadFence(request); + } + Assertions.assertEquals(TStatusCode.ANALYSIS_ERROR, response.getStatus().getStatusCode()); + Assertions.assertTrue(response.isSetWindowNotReady()); + Assertions.assertEquals(CURRENT_PHYSICAL_TIME_MS, response.getWindowNotReady().getRequestedEndTimestampMs()); + Env follower = Mockito.mock(Env.class); + Mockito.when(follower.getMasterHost()).thenReturn("127.0.0.1"); + Mockito.when(follower.getMasterRpcPort()).thenReturn(9020); + ConnectContext context = mockContext(); + Mockito.when(context.getEnv()).thenReturn(follower); + FrontendService.Client client = Mockito.mock(FrontendService.Client.class); + Mockito.when(client.acquireTimeBasedChangeReadFence(Mockito.any())).thenReturn(response); + GenericPool originalPool = ClientPool.frontendPool; + GenericPool pool = Mockito.mock(GenericPool.class); + Mockito.when(pool.borrowObject(Mockito.any(), Mockito.anyInt())).thenReturn(client); + ClientPool.frontendPool = pool; + try { + IncrWindowNotReadyException error = Assertions.assertThrows(IncrWindowNotReadyException.class, + () -> TimeBasedChangeVisibleWaiter.waitForVisible(context, + newChangeRelation(1, ImmutableMap.of(OlapScanNode.OLAP_END_TIMESTAMP, "2024-01-01 00:00:00")), + ImmutableMap.of(TABLE_QUALIFIER, mockOlapTable(DB_ID, TABLE_ID)))); + Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, error.getMysqlErrorCode()); + Assertions.assertEquals(CURRENT_PHYSICAL_TIME_MS, error.getRequestedEndTimestampMs()); + Assertions.assertEquals(0, error.getCommittedTso()); + Mockito.verify(pool).returnObject(Mockito.any(), Mockito.eq(client)); + } finally { + ClientPool.frontendPool = originalPool; + } + } + private ConnectContext mockContext() { ConnectContext context = Mockito.mock(ConnectContext.class); SessionVariable sessionVariable = new SessionVariable(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java index 1879766fecd47a..234294d1c19864 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java @@ -17,8 +17,11 @@ package org.apache.doris.service.arrowflight; +import org.apache.doris.common.ErrorCode; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QueryState; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.service.arrowflight.results.FlightSqlChannel; import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager; @@ -27,6 +30,8 @@ import org.apache.arrow.flight.FlightDescriptor; import org.apache.arrow.flight.FlightProducer.CallContext; import org.apache.arrow.flight.FlightProducer.StreamListener; +import org.apache.arrow.flight.FlightRuntimeException; +import org.apache.arrow.flight.FlightStatusCode; import org.apache.arrow.flight.Location; import org.apache.arrow.flight.Result; import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementRequest; @@ -46,6 +51,22 @@ public class DorisFlightSqlProducerTest { private boolean prevRunningUnitTest; + @Test + public void testWindowNotReadyHasRetryableFlightStatusAndStableBusinessCode() { + QueryState state = new QueryState(); + IncrWindowNotReadyException error = new IncrWindowNotReadyException(2000, 100, 1000); + state.setError(error.getMysqlErrorCode(), error.getDetailMessage()); + FlightRuntimeException result = DorisFlightSqlProducer.queryFailure(state, state.getErrorMessage(), error); + Assertions.assertEquals(FlightStatusCode.UNAVAILABLE, result.status().code()); + Assertions.assertEquals(Integer.toString(ErrorCode.ERR_INCR_WINDOW_NOT_READY.getCode()), + result.status().metadata().get("doris-error-code")); + Assertions.assertTrue(result.status().description().contains("requestedEndTimestampMs=2000")); + Assertions.assertTrue(result.status().description().contains("retryAfterMs=1000")); + state.setError(ErrorCode.ERR_UNKNOWN_ERROR, "other failure"); + Assertions.assertEquals(FlightStatusCode.INTERNAL, + DorisFlightSqlProducer.queryFailure(state, "other failure", error).status().code()); + } + @BeforeEach public void setUp() { // FlightSqlConnectContext.init() only reaches Env when this is false; keep it true so the diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/TsoStatusMetadataGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/TsoStatusMetadataGeneratorTest.java index dc8761c5ac3a5f..753bbe920c5be0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/TsoStatusMetadataGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/TsoStatusMetadataGeneratorTest.java @@ -74,6 +74,7 @@ public void testTsoStatusResult() throws Exception { Assertions.assertEquals(TStatusCode.OK, result.getStatus().getStatusCode()); Assertions.assertEquals(1, result.getDataBatchSize()); TRow row = result.getDataBatch().get(0); + Assertions.assertEquals(4, row.getColumnValueSize()); // Legacy BE request. Assertions.assertEquals(windowEndPhysicalTime, row.getColumnValue().get(0).getLongVal()); Assertions.assertEquals(currentTso, row.getColumnValue().get(1).getLongVal()); Assertions.assertEquals(physicalTime, row.getColumnValue().get(2).getLongVal()); @@ -123,6 +124,25 @@ public void testNotCalibrated() throws Exception { Mockito.verify(tsoService).getStatusSnapshot(); } + @Test + public void testCommittedTsoProjectionAndUnknownPrefix() throws Exception { + long committed = TSOTimestamp.composeTimestamp(1700000000000L, 17); + TFetchSchemaTableDataRequest request = newRequest(); + request.getSchemaTableParams().setColumnsName( + ImmutableList.of("committed_tso_physical_time", "committed_tso")); + Mockito.when(tsoService.getStatusSnapshot()).thenReturn( + new TSOService.TSOStatusSnapshot(true, committed + 100, 1700000001000L, committed), + new TSOService.TSOStatusSnapshot(true, committed + 100, 1700000001000L, 0)); + TRow row = MetadataGenerator.getSchemaTableData(request).getDataBatch().get(0); + Assertions.assertEquals(2, row.getColumnValueSize()); + Assertions.assertEquals(1700000000000L, row.getColumnValue().get(0).getLongVal()); + Assertions.assertEquals(committed, row.getColumnValue().get(1).getLongVal()); + TRow unknown = MetadataGenerator.getSchemaTableData(request).getDataBatch().get(0); + Assertions.assertTrue(unknown.getColumnValue().get(0).isIsNull()); + Assertions.assertTrue(unknown.getColumnValue().get(1).isIsNull()); + Mockito.verify(tsoService, Mockito.never()).getTSO(); + } + private TFetchSchemaTableDataRequest newRequest() { TFetchSchemaTableDataRequest request = new TFetchSchemaTableDataRequest(); request.setSchemaTableName(TSchemaTableName.TSO_STATUS); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java index a591bc9f3592dc..9aaa51d66aa281 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java @@ -22,9 +22,12 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.io.CountingDataOutputStream; import org.apache.doris.journal.Journal; +import org.apache.doris.journal.JournalEntity; import org.apache.doris.metric.LongCounterMetric; import org.apache.doris.metric.MetricRepo; import org.apache.doris.persist.EditLog; +import org.apache.doris.persist.OperationType; +import org.apache.doris.transaction.GlobalTransactionMgrIface; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -36,10 +39,17 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; /** * Unit tests for TSOService class. @@ -251,7 +261,7 @@ public void testRunAfterCatalogReadyUpdateFailureDoesNotTouchMetricWhenNotInit() @Test public void testReplayWindowEndTSOUpdatesServiceState() { long windowEnd = 12345L; - tsoService.replayWindowEndTSO(new TSOTimestamp(windowEnd, 0L)); + tsoService.replayWindowEndTSO(new TSOServiceState(windowEnd, 0L)); Assertions.assertEquals(windowEnd, tsoService.getWindowEndTSO()); } @@ -261,7 +271,7 @@ public void testSaveTSOPersistsWindowEndWhenBinlogEnabled() throws IOException { try { Config.enable_feature_binlog = true; long windowEnd = 12345L; - tsoService.replayWindowEndTSO(new TSOTimestamp(windowEnd, 0L)); + tsoService.replayWindowEndTSO(new TSOServiceState(windowEnd, 0L)); byte[] bytes = saveTSOBytes(tsoService); Assertions.assertTrue(bytes.length > 0); @@ -320,7 +330,7 @@ public void testWriteTimestampToBdbJeWritesWhenEnabledAndJournalReady() throws E Mockito.when(editLog.getJournal()).thenReturn(journal); invokeWriteTimestampToBdbJe(tsoService, 123L); - Mockito.verify(editLog).logTSOTimestampWindowEnd(Mockito.any(TSOTimestamp.class)); + Mockito.verify(editLog).logTSOTimestampWindowEnd(Mockito.any(TSOServiceState.class)); } @Test @@ -371,7 +381,7 @@ public void testCalibrateTimestampThrowsWhenClockBackwardExceedsThreshold() thro Mockito.when(env.isReady()).thenReturn(true); Mockito.when(env.isMaster()).thenReturn(true); long now = System.currentTimeMillis() + Config.tso_time_offset_debug_mode; - tsoService.replayWindowEndTSO(new TSOTimestamp( + tsoService.replayWindowEndTSO(new TSOServiceState( now + Config.tso_clock_backward_startup_threshold_ms + 60_000, 0L)); try { invokeCalibrateTimestamp(tsoService); @@ -406,7 +416,7 @@ public void testUpdateTimestampReturnsEarlyWhenNotCalibrated() throws Exception Mockito.when(env.isReady()).thenReturn(true); Mockito.when(env.isMaster()).thenReturn(true); long initialWindowEnd = 12345L; - tsoService.replayWindowEndTSO(new TSOTimestamp(initialWindowEnd, 0L)); + tsoService.replayWindowEndTSO(new TSOServiceState(initialWindowEnd, 0L)); invokeUpdateTimestamp(tsoService); @@ -436,6 +446,152 @@ public void testGenerateTSOReturnsZeroWhenDisabledOrNotInitialized() throws Exce } } + @Test + public void testCommittedPrefixIsPublishedOnlyAfterJournalSuccess() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(editLog.getJournal()).thenReturn(Mockito.mock(Journal.class)); + Mockito.when(env.getEditLog()).thenReturn(editLog); + tsoService.replayWindowEndTSO(new TSOServiceState(200, 80)); + setGlobalTimestamp(tsoService, 100, 10); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); + GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)).thenReturn(true); + tracker.checkTransactions(txnMgr, Long.MAX_VALUE); + try (MockedStatic config = Mockito.mockStatic(Config.class)) { + config.when(Config::isCloudMode).thenReturn(true); + Mockito.doAnswer(invocation -> { + Assertions.assertEquals(80, tsoService.getStatusSnapshot().getCommittedTso()); + Assertions.assertEquals(200, tsoService.getStatusSnapshot().getWindowEndPhysicalTime()); + throw new RuntimeException("injected journal failure"); + }).when(editLog).logTSOTimestampWindowEnd(Mockito.any()); + Assertions.assertThrows(RuntimeException.class, () -> invokeWriteTimestampToBdbJe(tsoService, 300)); + Assertions.assertEquals(80, tsoService.getStatusSnapshot().getCommittedTso()); + Assertions.assertEquals(200, tsoService.getStatusSnapshot().getWindowEndPhysicalTime()); + Mockito.doNothing().when(editLog).logTSOTimestampWindowEnd(Mockito.any()); + invokeWriteTimestampToBdbJe(tsoService, 300); + Assertions.assertEquals(tsoService.getCurrentTSO(), tsoService.getStatusSnapshot().getCommittedTso()); + Assertions.assertEquals(300, tsoService.getStatusSnapshot().getWindowEndPhysicalTime()); + } + } + + @Test + public void testRecoveryFreezesPrefixAndPeriodicFlushWorksWithUnchangedWindow() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + mockPersistReady(); + long oldWindow = System.currentTimeMillis() + 60_000; + long oldCommitted = TSOTimestamp.composeTimestamp(oldWindow - 1000, 7); + tsoService.replayWindowEndTSO(new TSOServiceState(oldWindow, oldCommitted)); + try (MockedStatic config = Mockito.mockStatic(Config.class)) { + config.when(Config::isCloudMode).thenReturn(true); + invokeCalibrateTimestamp(tsoService); + Assertions.assertEquals(oldCommitted, tsoService.getStatusSnapshot().getCommittedTso()); + long pendingTso = tsoService.getCommitTSO(1, 10); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); + GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)).thenReturn(true); + long afterRecoveryDelay = System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos(Config.tso_service_window_duration_ms + 1001L); + tracker.checkTransactions(txnMgr, afterRecoveryDelay); + long reservedWindow = tsoService.getWindowEndTSO(); + Field lastPersist = TSOService.class.getDeclaredField("lastPersistNanos"); + lastPersist.setAccessible(true); + lastPersist.setLong(tsoService, System.nanoTime() + - TimeUnit.MILLISECONDS.toNanos(Config.tso_service_window_duration_ms + 1L)); + invokeUpdateTimestamp(tsoService); + Assertions.assertEquals(reservedWindow, tsoService.getWindowEndTSO()); + Assertions.assertEquals(pendingTso - 1, tsoService.getStatusSnapshot().getCommittedTso()); + tsoService.transactionFinished(1, 10); + lastPersist.setLong(tsoService, System.nanoTime() + - TimeUnit.MILLISECONDS.toNanos(Config.tso_service_window_duration_ms + 1L)); + invokeUpdateTimestamp(tsoService); + Assertions.assertEquals(reservedWindow, tsoService.getWindowEndTSO()); + Assertions.assertEquals(pendingTso, tsoService.getStatusSnapshot().getCommittedTso()); + } + } + + @Test + public void testStateImageJournalAndOldTimestampCompatibility() throws Exception { + long committed = TSOTimestamp.composeTimestamp(100, 17); + tsoService.replayWindowEndTSO(new TSOServiceState(200, committed)); + TSOService restored = new TSOService(); + Assertions.assertEquals(200, restored.loadTSO( + new DataInputStream(new ByteArrayInputStream(saveTSOBytes(tsoService))), 0)); + Assertions.assertEquals(committed, restored.getStatusSnapshot().getCommittedTso()); + Assertions.assertFalse(restored.getStatusSnapshot().isInitialized()); + for (boolean legacy : new boolean[] {true, false}) { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(bytes); + out.writeShort(OperationType.OP_TSO_TIMESTAMP_WINDOW_END); + if (legacy) { + new TSOTimestamp(200, 0).write(out); + } else { + new TSOServiceState(200, committed).write(out); + } + JournalEntity entity = new JournalEntity(); + entity.readFields(new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))); + TSOServiceState state = (TSOServiceState) entity.getData(); + Assertions.assertEquals(200, state.getPhysicalTimestamp()); + Assertions.assertEquals(legacy ? 0 : committed, state.getCommittedTso()); + } + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + new TSOServiceState(200, committed).write(new DataOutputStream(bytes)); + Assertions.assertEquals(200, TSOTimestamp.read( + new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))).getPhysicalTimestamp()); + } + + @Test + public void testAllocationAndRegistrationShareSnapshotLock() throws Exception { + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + Field lockField = TSOService.class.getDeclaredField("lock"); + lockField.setAccessible(true); + ReentrantLock lock = (ReentrantLock) lockField.get(tsoService); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + TSOTransactionTracker tracker = Mockito.spy((TSOTransactionTracker) trackerField.get(tsoService)); + trackerField.set(tsoService, tracker); + CountDownLatch enteredRegistration = new CountDownLatch(1); + CountDownLatch releaseRegistration = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + enteredRegistration.countDown(); + Assertions.assertTrue(releaseRegistration.await(30, TimeUnit.SECONDS)); + return invocation.callRealMethod(); + }).when(tracker).register(Mockito.any(), Mockito.anyLong(), Mockito.anyLong()); + Method generate = TSOService.class.getDeclaredMethod("generateTSO", Pair.class); + generate.setAccessible(true); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future allocation = executor.submit(() -> { + try { + generate.invoke(tsoService, Pair.of(1L, 10L)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + Assertions.assertTrue(enteredRegistration.await(30, TimeUnit.SECONDS)); + boolean snapshotLockAcquired = lock.tryLock(); + if (snapshotLockAcquired) { + lock.unlock(); + } + Assertions.assertFalse(snapshotLockAcquired, "snapshot must not pass an unregistered allocation"); + releaseRegistration.countDown(); + allocation.get(30, TimeUnit.SECONDS); + Assertions.assertEquals(TSOTimestamp.composeTimestamp(100, 1), tracker.getOldestPendingTso()); + } finally { + releaseRegistration.countDown(); + executor.shutdownNow(); + } + } + private static void invokeWriteTimestampToBdbJe(TSOService service, long timestamp) throws Exception { Method m = TSOService.class.getDeclaredMethod("writeTimestampToBDBJE", long.class); m.setAccessible(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java new file mode 100644 index 00000000000000..89d38308367059 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tso; + +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; +import org.apache.doris.transaction.GlobalTransactionMgrIface; +import org.apache.doris.transaction.TransactionState; +import org.apache.doris.transaction.TransactionStatus; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +public class TSOTransactionTrackerTest { + private final ReentrantLock lock = new ReentrantLock(); + private final TSOTransactionTracker tracker = new TSOTransactionTracker(lock); + private final GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); + + private void reset(long delayMs) { + lock.lock(); + try { + tracker.reset(0, delayMs); + } finally { + lock.unlock(); + } + } + + private void register(long dbId, long txnId, long tso) { + lock.lock(); + try { + tracker.register(Pair.of(dbId, txnId), tso, 0); + } finally { + lock.unlock(); + } + } + + private long candidate(long currentTso, long durableTso) { + lock.lock(); + try { + return tracker.candidateCommittedTso(currentTso, durableTso); + } finally { + lock.unlock(); + } + } + + private void finishRecovery() throws Exception { + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.doReturn(true).when(txnMgr).isPreviousTransactionsFinishedForTsoRecovery(1000L); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); + } + + @Test + public void testOutOfOrderVisibilityAndRetryRetainEarliestTso() throws Exception { + reset(2000); + finishRecovery(); + register(1, 10, 100); + register(1, 20, 120); + tracker.transactionFinished(1, 20); + Assertions.assertEquals(99, candidate(150, 80)); + register(1, 10, 200); // A retry must not hide a delayed request carrying TSO 100. + Assertions.assertEquals(99, candidate(250, 99)); + tracker.transactionFinished(1, 10); + tracker.transactionFinished(1, 10); // Duplicate terminal notification is harmless. + Assertions.assertEquals(250, candidate(250, 99)); + } + + @Test + public void testRecoveryCapturesFixedWatermarkAfterDelayAndPreservesNewPending() throws Exception { + reset(2000); + register(2, 2001, 200); + tracker.checkTransactions(txnMgr, TimeUnit.MILLISECONDS.toNanos(1999)); + Mockito.verify(txnMgr, Mockito.never()).getTransactionIdWatermark(); + Assertions.assertEquals(80, candidate(250, 80)); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L, 2000L); + Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)).thenReturn(false, true); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(2)); + Assertions.assertEquals(80, candidate(250, 80)); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); + Assertions.assertEquals(199, candidate(250, 80)); + Mockito.verify(txnMgr).getTransactionIdWatermark(); + Mockito.verify(txnMgr, Mockito.times(2)).isPreviousTransactionsFinishedForTsoRecovery(1000L); + } + + @Test + public void testOnlyRealTerminalStatesReleasePending() throws Exception { + reset(2000); + finishRecovery(); + for (TransactionStatus status : TransactionStatus.values()) { + register(1, 10, 100); + TransactionState state = Mockito.mock(TransactionState.class); + Mockito.when(state.getTransactionStatus()).thenReturn(status); + Mockito.when(txnMgr.getTransactionState(1, 10)).thenReturn(state); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); + boolean terminal = status == TransactionStatus.VISIBLE || status == TransactionStatus.ABORTED; + Assertions.assertEquals(terminal ? 150 : 99, candidate(150, 80), status.toString()); + } + } + + @Test + public void testMissingTransactionAndFailedRecoveryNeverAdvance() throws Exception { + reset(0); + register(1, 10, 100); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)) + .thenThrow(new UserException("old MS has no strict check capability")); + Assertions.assertThrows(UserException.class, + () -> tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3))); + Assertions.assertEquals(80, candidate(150, 80)); + Assertions.assertEquals(1, tracker.getPendingCount()); + finishRecovery(); + Assertions.assertEquals(99, candidate(150, 80)); + } + + @Test + public void testRpcDoesNotHoldAllocatorLockAndOldResultCannotRemoveNewRegistration() throws Exception { + reset(2000); + finishRecovery(); + register(1, 10, 100); + TransactionState visible = Mockito.mock(TransactionState.class); + Mockito.when(visible.getTransactionStatus()).thenReturn(TransactionStatus.VISIBLE); + Mockito.when(txnMgr.getTransactionState(1, 10)).thenAnswer(invocation -> { + Assertions.assertFalse(lock.isHeldByCurrentThread()); + reset(2000); // Simulate reinitialization while an old reconciliation request is in flight. + register(1, 10, 200); + return visible; + }); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); + Assertions.assertEquals(1, tracker.getPendingCount()); + Assertions.assertEquals(200, tracker.getOldestPendingTso()); + Assertions.assertFalse(tracker.isRecoveryReady()); + } + + @Test + public void testOldRecoveryResultCannotOpenNewRecovery() throws Exception { + reset(0); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)).thenAnswer(invocation -> { + Assertions.assertFalse(lock.isHeldByCurrentThread()); + reset(2000); + return true; + }); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); + Assertions.assertFalse(tracker.isRecoveryReady()); + Assertions.assertEquals(80, candidate(150, 80)); + } + + @Test + public void testReconciliationBatchIsBoundedAndRotatesPastOldest() throws Exception { + reset(0); + finishRecovery(); + for (int i = 1; i <= 150; i++) { + register(1, i, 1000 + i); + } + Mockito.clearInvocations(txnMgr); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); + Mockito.verify(txnMgr, Mockito.atMost(64)).getTransactionState(Mockito.anyLong(), Mockito.anyLong()); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(5)); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(6)); + Mockito.verify(txnMgr, Mockito.times(3)).getTransactionState(1, 1); + Mockito.verify(txnMgr).getTransactionState(1, 150); + Assertions.assertEquals(150, tracker.getPendingCount()); + } +} diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index a63ac2d518195a..8133e611b0d749 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -1248,12 +1248,15 @@ message CheckTxnConflictRequest { repeated int64 table_ids = 4; optional bool ignore_timeout_txn = 5; optional string request_ip = 6; + // Check every database/table in the instance, including expired running transactions. + optional bool strict_recovery_check = 7 [default = false]; } message CheckTxnConflictResponse { optional MetaServiceResponseStatus status = 1; optional bool finished = 2; repeated TxnInfoPB conflict_txns = 3; + optional bool strict_recovery_check_applied = 4; } message CleanTxnLabelRequest { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 1829181a875f63..f797c2e7c9bcd9 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1994,12 +1994,22 @@ struct TAcquireTimeBasedChangeReadFenceRequest { 2: optional i64 end_timestamp_ms 3: required i64 timeout_ms 4: required bool wait_for_transactions + // False/absent retains the legacy drain for old callers and unbounded relations. + 5: optional bool all_ends_explicit +} + +struct TIncrWindowNotReady { + 1: required i64 requested_end_timestamp_ms + 2: required i64 committed_tso + 3: required i64 retry_after_ms } struct TAcquireTimeBasedChangeReadFenceResult { 1: required Status.TStatus status 2: optional i64 current_tso 3: optional i64 max_journal_id + 4: optional i64 committed_tso + 5: optional TIncrWindowNotReady window_not_ready } service FrontendService { diff --git a/regression-test/data/tso_p0/test_committed_tso.out b/regression-test/data/tso_p0/test_committed_tso.out new file mode 100644 index 00000000000000..a9a8961e82a93d --- /dev/null +++ b/regression-test/data/tso_p0/test_committed_tso.out @@ -0,0 +1,20 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !readable_prefix -- +true true true + +-- !historical_window -- +1 +2 + +-- !pending_constrains_prefix -- +true + +-- !history_with_later_write -- +1 +2 + +-- !retry_original_window -- +1 +2 +3 + diff --git a/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy b/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy index 2bcd20b42d4549..70a8c538aefd1d 100644 --- a/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy +++ b/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy @@ -28,6 +28,18 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { def mowPartialTable = "changes_mow_partial" def mowBitmapTable = "changes_mow_bitmap" def incrTimeFormat = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + def fullWindowEnd = { + long current = (sql "SELECT CURRENT_TSO_PHYSICAL_TIME FROM information_schema.tso_status")[0][0] as long + long minimumEnd = (Math.floorDiv(current, 1000L) + 1L) * 1000L + String column = isCloudMode() ? "COMMITTED_TSO_PHYSICAL_TIME" : "CURRENT_TSO_PHYSICAL_TIME" + long readable = 0L + awaitUntil(60, 0.1) { + def value = (sql "SELECT ${column} FROM information_schema.tso_status")[0][0] + readable = value == null ? 0L : value as long + readable >= minimumEnd + } + incrTimeFormat.format(new Date(readable)) + } try { sql "DROP TABLE IF EXISTS ${dupTable}" @@ -80,7 +92,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { sql "INSERT INTO ${dupTable} VALUES (3, 31, 'c2', NULL)" sql "sync" sleep(1200) - def dupT1 = incrTimeFormat.format(new Date()) + def dupT1 = fullWindowEnd() sleep(1200) sql "INSERT INTO ${dupTable} VALUES (5, 50, 'd', 'w')" sql "sync" @@ -161,12 +173,8 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { ORDER BY __DORIS_BINLOG_TSO__, __DORIS_BINLOG_LSN__ """ - // - CURRENT_TSO_PHYSICAL_TIME converted to an INCR timestamp is accepted. - sleep(1200) - def currentTsoPhysicalTime = Long.parseLong(sql(""" - SELECT CURRENT_TSO_PHYSICAL_TIME FROM information_schema.tso_status - """)[0][0].toString()) - def currentTsoEnd = incrTimeFormat.format(new Date(currentTsoPhysicalTime)) + // - The readable TSO physical time converted to an INCR timestamp is accepted. + def currentTsoEnd = fullWindowEnd() order_qt_dup_full_cover """ SELECT id, v1, __DORIS_BINLOG_OP__ FROM ${dupTable}@incr('startTimestamp' = '1971-01-01 00:00:00', @@ -183,7 +191,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { "endTimestamp" = "2999-01-01 00:00:00", "incrementType" = "DETAIL") """ - exception "CURRENT_TSO_PHYSICAL_TIME=" + exception (isCloudMode() ? "ERR_INCR_WINDOW_NOT_READY" : "CURRENT_TSO_PHYSICAL_TIME=") } // 1.8 Cross-check against binlog() TVF — DETAIL with full window must @@ -263,7 +271,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { sql "sync" sleep(1200) - def mowT1 = incrTimeFormat.format(new Date()) + def mowT1 = fullWindowEnd() sleep(1200) sql "INSERT INTO ${mowTable} VALUES (6, 60, 'f')" sql "sync" @@ -410,13 +418,9 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { "incrementType" = "MIN_DELTA") """ - // 2.9 CURRENT_TSO_PHYSICAL_TIME converted to an INCR timestamp covers + // 2.9 The readable TSO physical time converted to an INCR timestamp covers // all changes that were visible before it was captured. - sleep(1200) - def mowCurrentTsoPhysicalTime = Long.parseLong(sql(""" - SELECT CURRENT_TSO_PHYSICAL_TIME FROM information_schema.tso_status - """)[0][0].toString()) - def mowCurrentTsoEnd = incrTimeFormat.format(new Date(mowCurrentTsoPhysicalTime)) + def mowCurrentTsoEnd = fullWindowEnd() order_qt_mow_full_cover """ SELECT id, v1, __DORIS_BINLOG_OP__ FROM ${mowTable}@incr('startTimestamp' = '1971-01-01 00:00:00', @@ -433,7 +437,8 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { "endTimestamp" = "2999-01-01 00:00:00", "incrementType" = "DETAIL") """ - exception "endTimestamp exceeds the maximum supported time for an INCR read" + exception (isCloudMode() ? "ERR_INCR_WINDOW_NOT_READY" + : "endTimestamp exceeds the maximum supported time for an INCR read") } // ============================================================ @@ -482,7 +487,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { """ sql "sync" sleep(1200) - def seqT1 = incrTimeFormat.format(new Date()) + def seqT1 = fullWindowEnd() sleep(1200) // 3.1 DETAIL only captures physically-applied writes. The out-of-order @@ -543,7 +548,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { sql "SET enable_unique_key_partial_update = false" sql "sync" sleep(1200) - def partT1 = incrTimeFormat.format(new Date()) + def partT1 = fullWindowEnd() // 4.1 DETAIL: 3 partial updates -> 6 raw binlog rows. order_qt_part_detail """ @@ -608,7 +613,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { sql "INSERT INTO ${mowBitmapTable} VALUES (1, BITMAP_FROM_STRING('3,4'))" sql "sync" sleep(1200) - def bitmapT1 = incrTimeFormat.format(new Date()) + def bitmapT1 = fullWindowEnd() order_qt_bitmap_min_delta """ SELECT id, BITMAP_TO_STRING(b), __DORIS_BINLOG_OP__ @@ -627,7 +632,7 @@ suite("test_binlog_changes_syntax", "nonConcurrent") { sql "INSERT INTO ${mowBitmapTable} VALUES (1, BITMAP_FROM_STRING('3,4'))" sql "sync" sleep(1200) - def bitmapEqualT1 = incrTimeFormat.format(new Date()) + def bitmapEqualT1 = fullWindowEnd() order_qt_bitmap_equal_min_delta """ SELECT id, BITMAP_TO_STRING(b), __DORIS_BINLOG_OP__ diff --git a/regression-test/suites/tso_p0/test_committed_tso.groovy b/regression-test/suites/tso_p0/test_committed_tso.groovy new file mode 100644 index 00000000000000..39bdd2119d8b85 --- /dev/null +++ b/regression-test/suites/tso_p0/test_committed_tso.groovy @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.Http + +import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter +import java.util.concurrent.atomic.AtomicReference + +suite("test_committed_tso", "nonConcurrent") { + if (!isCloudMode()) { + return + } + sql "DROP TABLE IF EXISTS test_committed_tso" + sql """ + CREATE TABLE test_committed_tso (id INT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1", "binlog.enable" = "true", "binlog.format" = "ROW") + """ + sql "INSERT INTO test_committed_tso VALUES (1), (2)" + def currentPhysicalTime = { + (sql "SELECT CURRENT_TSO_PHYSICAL_TIME FROM information_schema.tso_status")[0][0] as long + } + def committedPhysicalTime = { + def value = (sql "SELECT COMMITTED_TSO_PHYSICAL_TIME FROM information_schema.tso_status")[0][0] + value == null ? 0L : value as long + } + def formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.systemDefault()) + def timestamp = { long millis -> formatter.format(Instant.ofEpochMilli(millis)) } + long historicalEnd = (Math.floorDiv(currentPhysicalTime(), 1000L) + 1L) * 1000L + awaitUntil(60, 0.1) { committedPhysicalTime() >= historicalEnd } + order_qt_readable_prefix """ + SELECT COMMITTED_TSO > 0, + bit_shift_right(COMMITTED_TSO, 18) = COMMITTED_TSO_PHYSICAL_TIME, + COMMITTED_TSO <= CURRENT_TSO + FROM information_schema.tso_status + """ + order_qt_historical_window """ + SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(historicalEnd)}') ORDER BY id + """ + + def masterHttpAddress = "${getMasterIp()}:${getMasterPort()}" + def pendingMetric = { String name -> + String metrics = Http.GET("http://${masterHttpAddress}/metrics", false, false) + def matcher = metrics =~ /(?m)^doris_fe_${name}(?:\{[^\n]*\})?\s+(\d+)$/ + if (!matcher.find()) { + throw new IllegalStateException("Missing TSO metric: ${name}") + } + Long.parseLong(matcher.group(1)) + } + AtomicReference loadError = new AtomicReference<>() + Thread loadThread + try { + GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.commitTxn.blockAfterTso") + loadThread = Thread.start { + try { + sql "INSERT INTO test_committed_tso VALUES (3)" + } catch (Throwable t) { + loadError.set(t) + } + } + // Observe registration, rather than guessing when the submit thread reaches the debug point. + awaitUntil(60, 0.1) { pendingMetric("tso_pending_transactions") > 0 } + long pendingTso = pendingMetric("tso_oldest_pending_tso") + long blockedEnd = (Math.floorDiv(pendingTso >> 18, 1000L) + 1L) * 1000L + awaitUntil(60, 0.1) { currentPhysicalTime() >= blockedEnd } + order_qt_pending_constrains_prefix """ + SELECT COMMITTED_TSO < ${pendingTso} FROM information_schema.tso_status + """ + order_qt_history_with_later_write """ + SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(historicalEnd)}') ORDER BY id + """ + test { + sql "SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(blockedEnd)}')" + exception "ERR_INCR_WINDOW_NOT_READY" + } + GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.commitTxn.blockAfterTso") + loadThread.join(60000) + if (loadThread.isAlive()) { + throw new IllegalStateException("Commit did not complete after releasing the debug point") + } + if (loadError.get() != null) { + throw loadError.get() + } + awaitUntil(60, 0.1) { committedPhysicalTime() >= blockedEnd } + // Retry exactly the same window that was refused; it must include the pending write. + order_qt_retry_original_window """ + SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(blockedEnd)}') ORDER BY id + """ + } finally { + GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.commitTxn.blockAfterTso") + if (loadThread != null) { + loadThread.join(60000) + } + } +} From e0fec9c9240142c4356ed6ef3e03896b00fe482b Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Thu, 10 Sep 2026 21:12:49 +0800 Subject: [PATCH 02/13] [fix](fe) Limit Flight error passthrough to unready incremental windows ### What problem does this PR solve? Issue Number: None Related PR: #67181, #67594 Problem Summary: The committed-TSO window check made getFlightInfoStatement pass through every FlightRuntimeException. Other Flight failures therefore lost the original INTERNAL wrapper, message prefix and cause chain, and unrelated status codes could reach clients unchanged. Only pass through an exception carrying the ERR_INCR_WINDOW_NOT_READY business code; retain the original wrapping for all other exceptions. ### Release note Preserve the existing Arrow Flight SQL error wrapping for failures other than ERR_INCR_WINDOW_NOT_READY. Window-not-ready errors still expose the retryable status and committed TSO details. ### Check List (For Author) - Test: Unit Test (all 7 DorisFlightSqlProducerTest tests passed via run-fe-ut.sh; the new cases reproduce the previous wrapping failures); FE Checkstyle passed with 0 violations - Behavior changed: Yes (restore the original INTERNAL wrapper for other Flight errors) - Does this need documentation: No (restore existing error handling) --- .../arrowflight/DorisFlightSqlProducer.java | 12 +++- .../DorisFlightSqlProducerTest.java | 60 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java index 6dec245e25888c..c743df4fa76d4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java @@ -324,9 +324,17 @@ public FlightInfo getFlightInfoStatement(final CommandStatementQuery request, fi try { ConnectContext connectContext = flightSessionsManager.getConnectContext(context.peerIdentity()); return executeQueryStatement(context.peerIdentity(), connectContext, request.getQuery(), descriptor); - } catch (FlightRuntimeException e) { - throw e; } catch (Throwable e) { + if (e instanceof FlightRuntimeException) { + FlightRuntimeException flightError = (FlightRuntimeException) e; + ErrorFlightMetadata metadata = flightError.status().metadata(); + // Only the incremental-window error bypasses the original INTERNAL wrapper. + if (metadata.containsKey("doris-error-code") + && Integer.toString(ErrorCode.ERR_INCR_WINDOW_NOT_READY.getCode()) + .equals(metadata.get("doris-error-code"))) { + throw flightError; + } + } String errMsg = "get flight info statement failed, " + e.getMessage(); LOG.error(errMsg, e); throw CallStatus.INTERNAL.withDescription(errMsg).withCause(e).toRuntimeException(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java index 234294d1c19864..5ad9a88bec59ce 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java @@ -27,6 +27,8 @@ import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager; import org.apache.doris.service.arrowflight.sessions.FlightSqlConnectContext; +import org.apache.arrow.flight.CallStatus; +import org.apache.arrow.flight.ErrorFlightMetadata; import org.apache.arrow.flight.FlightDescriptor; import org.apache.arrow.flight.FlightProducer.CallContext; import org.apache.arrow.flight.FlightProducer.StreamListener; @@ -67,6 +69,64 @@ public void testWindowNotReadyHasRetryableFlightStatusAndStableBusinessCode() { DorisFlightSqlProducer.queryFailure(state, "other failure", error).status().code()); } + @Test + public void testGetFlightInfoPreservesWindowNotReadyError() throws Exception { + QueryState state = new QueryState(); + IncrWindowNotReadyException error = new IncrWindowNotReadyException(2000, 100, 1000); + state.setError(error.getMysqlErrorCode(), error.getDetailMessage()); + FlightRuntimeException failure = DorisFlightSqlProducer.queryFailure(state, state.getErrorMessage(), error); + + // Preserve the retryable status, business metadata and committed TSO details without wrapping. + Assertions.assertSame(failure, getFlightInfoFailure(failure)); + } + + @Test + public void testGetFlightInfoWrapsOtherFlightErrors() throws Exception { + for (CallStatus status : new CallStatus[] {CallStatus.INTERNAL, CallStatus.UNAVAILABLE, + CallStatus.INVALID_ARGUMENT, CallStatus.UNAUTHENTICATED}) { + FlightRuntimeException failure = status.withDescription("other flight failure").toRuntimeException(); + assertLegacyFlightWrapper(failure, getFlightInfoFailure(failure)); + } + } + + @Test + public void testGetFlightInfoWrapsOtherBusinessErrors() throws Exception { + ErrorFlightMetadata metadata = new ErrorFlightMetadata(); + metadata.insert("doris-error-code", Integer.toString(ErrorCode.ERR_UNKNOWN_ERROR.getCode())); + FlightRuntimeException failure = CallStatus.UNAVAILABLE.withDescription("other business failure") + .withMetadata(metadata).toRuntimeException(); + + assertLegacyFlightWrapper(failure, getFlightInfoFailure(failure)); + } + + @Test + public void testGetFlightInfoWrapsNonFlightErrors() throws Exception { + RuntimeException failure = new RuntimeException("session lookup failed"); + assertLegacyFlightWrapper(failure, getFlightInfoFailure(failure)); + } + + private FlightRuntimeException getFlightInfoFailure(RuntimeException failure) throws Exception { + FlightSessionsManager sessionsManager = Mockito.mock(FlightSessionsManager.class); + Mockito.when(sessionsManager.getConnectContext("token")).thenThrow(failure); + CallContext callContext = Mockito.mock(CallContext.class); + Mockito.when(callContext.peerIdentity()).thenReturn("token"); + try (DorisFlightSqlProducer producer = new DorisFlightSqlProducer( + Location.forGrpcInsecure("127.0.0.1", 9090), sessionsManager)) { + CommandStatementQuery request = CommandStatementQuery.newBuilder().setQuery("select 1").build(); + return Assertions.assertThrows(FlightRuntimeException.class, + () -> producer.getFlightInfoStatement(request, callContext, FlightDescriptor.command(new byte[0]))); + } + } + + private void assertLegacyFlightWrapper(RuntimeException failure, FlightRuntimeException result) { + Assertions.assertEquals(FlightStatusCode.INTERNAL, result.status().code()); + Assertions.assertNotSame(failure, result); + Assertions.assertSame(failure, result.getCause()); + Assertions.assertEquals("get flight info statement failed, " + failure.getMessage(), + result.status().description()); + Assertions.assertFalse(result.status().metadata().containsKey("doris-error-code")); + } + @BeforeEach public void setUp() { // FlightSqlConnectContext.init() only reaches Env when this is false; keep it true so the From 92c170f307cde369de3d6c4e70580f2077318168 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Thu, 10 Sep 2026 22:05:41 +0800 Subject: [PATCH 03/13] [fix](cloud) Wait only for related transactions in bounded incremental reads ### What problem does this PR solve? Issue Number: None Related PR: #67181, #67594 Problem Summary: A slow commit on one table holds the global committed TSO and rejects otherwise complete incremental windows on unrelated tables. Keep the durable-prefix fast path, reject ends after the current TSO immediately, and let intermediate windows wait for a fixed snapshot of registered transactions involving their tables. Capture the snapshot under the allocator lock and release that lock during the wait; real terminal notifications and reconciliation wake readers without another journal flush. Preserve the recovery guard and distinguish future/recovering windows from visibility wait timeouts through follower RPC, MySQL and Arrow Flight SQL. ### Release note Bounded strongly consistent cloud incremental reads can proceed above the durable committed TSO when their relevant transactions are finished. Visibility wait timeouts return error 5101 (ERR_INCR_VISIBLE_WAIT_TIMEOUT); future or recovering windows retain error 5100 (ERR_INCR_WINDOW_NOT_READY). Both include the current and committed TSO and retry details. ### Check List (For Author) - Test: 65 distinct focused FE unit tests; full FE build and Checkstyle; test_committed_tso SQL regression generated and verified; live MySQL and Flight statement/prepared checks on one master and two followers, including transaction visibility wakeup. - Behavior changed: Yes (table-scoped waiting above the durable prefix and distinct visibility-timeout error). - Does this need documentation: Yes (docs/committed-tso.md updated). --- docs/committed-tso.md | 26 ++-- .../org/apache/doris/common/ErrorCode.java | 3 + .../common/IncrWindowNotReadyException.java | 40 +++++- .../org/apache/doris/qe/StmtExecutor.java | 3 +- .../qe/TimeBasedChangeVisibleWaiter.java | 40 ++++-- .../doris/service/FrontendServiceImpl.java | 4 +- .../arrowflight/DorisFlightSqlProducer.java | 19 +-- .../doris/transaction/TransactionUtil.java | 3 +- .../java/org/apache/doris/tso/TSOService.java | 72 +++++++++-- .../doris/tso/TSOTransactionTracker.java | 60 ++++++++- .../org/apache/doris/qe/StmtExecutorTest.java | 29 +++-- .../qe/TimeBasedChangeVisibleWaiterTest.java | 93 +++++++++----- .../DorisFlightSqlProducerTest.java | 24 ++-- .../org/apache/doris/tso/TSOServiceTest.java | 112 +++++++++++++++- .../doris/tso/TSOTransactionTrackerTest.java | 121 +++++++++++++++++- gensrc/thrift/FrontendService.thrift | 5 + .../data/tso_p0/test_committed_tso.out | 4 + .../suites/tso_p0/test_committed_tso.groovy | 20 ++- 18 files changed, 573 insertions(+), 105 deletions(-) diff --git a/docs/committed-tso.md b/docs/committed-tso.md index b38c8e8adcef90..a7de997c29a3a7 100644 --- a/docs/committed-tso.md +++ b/docs/committed-tso.md @@ -1,33 +1,41 @@ # Committed TSO for bounded incremental reads -In cloud mode, a strongly consistent `@incr` query with an explicit `endTimestamp` on every incremental relation uses the master FE's durable committed TSO. Transactions with commit TSO at or below this prefix are truly visible or aborted. The name does not refer to the intermediate `COMMITTED` transaction state. +In cloud mode, a strongly consistent `@incr` query with an explicit `endTimestamp` on every incremental relation uses the master FE's durable committed TSO as a boundary for reads that require no transaction wait. Transactions with commit TSO at or below this prefix are truly visible or aborted. The name does not refer to the intermediate `COMMITTED` transaction state. ```sql -SELECT COMMITTED_TSO, COMMITTED_TSO_PHYSICAL_TIME +SELECT CURRENT_TSO_PHYSICAL_TIME, COMMITTED_TSO, COMMITTED_TSO_PHYSICAL_TIME FROM information_schema.tso_status; ``` -Both columns are nullable BIGINTs. `COMMITTED_TSO_PHYSICAL_TIME` is Unix epoch milliseconds and gives the maximum allowed end timestamp; the full encoded `COMMITTED_TSO` must not be passed as an `endTimestamp` string. Convert milliseconds to the timestamp format and time zone accepted by `@incr`; clients producing whole-second windows must round down. The interval remains `[start, end)`. Existing binlog retention and table requirements still apply. +The committed columns are nullable BIGINTs. `COMMITTED_TSO_PHYSICAL_TIME` is Unix epoch milliseconds and gives an end timestamp guaranteed to require no transaction wait across all tables. Larger ends can also be readable after checking only the relevant tables, up to `CURRENT_TSO_PHYSICAL_TIME`; the full encoded `COMMITTED_TSO` must not be passed as an `endTimestamp` string. Convert milliseconds to the timestamp format and time zone accepted by `@incr`; clients producing whole-second windows must round down. The interval remains `[start, end)`. Existing binlog retention and table requirements still apply. The system table reads the master's persisted prefix without allocating a TSO. While a new master recovers, it keeps exposing the previous persisted prefix. A first startup, an old image without a prefix, or classic mode returns NULL for the new columns. Existing disabled/uninitialized TSO errors remain unchanged. -If the requested end exceeds the prefix, the query immediately returns MySQL error 5100, `ERR_INCR_WINDOW_NOT_READY`. Its message includes `requestedEndTimestampMs`, `committedTSO`, `committedTSOPhysicalTimeMs`, and `retryAfterMs`. Master-to-follower RPC preserves this classification. Arrow Flight SQL returns UNAVAILABLE with metadata `doris-error-code=5100` and `doris-error-name=ERR_INCR_WINDOW_NOT_READY`; the description preserves the window and retry details. Clients should retry the same split/window/offset after a cancellable delay, and advance offsets only after that window completes. Shortening a refused window or treating it as an empty success can lose data. This Doris change does not implement a Connector's retry loop. +The master handles a bounded window in three ways: -For accepted windows, planning skips the transaction watermark and conflict polling. Cloud partition visible versions are still refreshed from MetaService. Classic reads, eventual consistency, unbounded reads, and queries mixing bounded and unbounded incremental relations retain the existing behavior. +| Requested end | Behavior | +| --- | --- | +| After the current TSO's physical time | Immediately return MySQL error 5100, `ERR_INCR_WINDOW_NOT_READY`, with reason `END_AFTER_CURRENT_TSO`. Later allocations could still fall in this window. | +| At or before the durable committed TSO's physical time | Proceed without waiting for transactions. | +| Between those boundaries | After recovery, wait only for registered transactions involving the queried tables whose earliest possible commit TSO falls before the physical end. An empty matching set proceeds immediately. Exceeding `change_visible_timeout_ms` returns error 5101, `ERR_INCR_VISIBLE_WAIT_TIMEOUT`. | + +The error message includes `reason`, `requestedEndTimestampMs`, `currentTSO`, `currentTSOPhysicalTimeMs`, `committedTSO`, `committedTSOPhysicalTimeMs`, `retryAfterMs`, and `timeoutMs`. Master-to-follower RPC preserves both classifications and allows one additional second for the typed wait-timeout response to arrive. Arrow Flight SQL returns UNAVAILABLE for both errors, with their respective `doris-error-code` and `doris-error-name` metadata. Other exceptions keep their existing wrapping. Clients should retry the same split/window/offset after a cancellable delay, and advance offsets only after that window completes. Shortening a refused window or treating it as an empty success can lose data. This Doris change does not implement a Connector's retry loop. + +Bounded reads skip the transaction-ID watermark and table-wide conflict polling. The allocator lock protects the clock and the fixed set of matching registrations; waiting releases this lock and terminal notifications wake the readers. A successful table-specific wait does not advance the global durable prefix or require its next journal write. Queries with different ends on multiple incremental relations use their maximum end for the matching table set, which can conservatively wait longer. Cloud partition visible versions are still refreshed from MetaService. Classic reads, eventual consistency, unbounded reads, and queries mixing bounded and unbounded incremental relations retain the existing behavior. ## Allocation and persistence -The allocator registers transaction identity and its first commit TSO under the same lock that advances its clock. Retries retain the earliest registration until a real terminal result is known. Bitmap preparation, callbacks, and commit metadata validation precede allocation; one request reuses the same TSO for RPC retries. A lazy commit response marked incomplete cannot release a registration even if its returned transaction status says VISIBLE. A separate worker reconciles at most 64 old registrations per cycle; missing/error responses do not release them. +The allocator registers transaction identity, the involved table IDs, and its first commit TSO under the same lock that advances its clock. Retries retain the earliest registration and the union of involved table IDs until a real terminal result is known. Bitmap preparation, callbacks, and commit metadata validation precede allocation; one request reuses the same TSO for RPC retries. A lazy commit response marked incomplete cannot release a registration even if its returned transaction status says VISIBLE. A separate worker reconciles at most 64 old registrations per cycle; missing/error responses do not release them. After recovery, the next candidate prefix is the current allocated TSO when no registrations remain, otherwise the smaller of that TSO and the oldest pending TSO minus one. The candidate is published only after its journal write succeeds. The reservation window and committed prefix share one journal record and one immutable persisted snapshot. `tso_service_window_duration_ms` defaults to 1000 ms. A monotonic timer also persists the prefix when the reservation window does not move. This is approximately one combined journal write per second, compared with the previous five-second window renewal; it does not reduce total journal frequency relative to the old implementation. ## Recovery and upgrades -A new master calibrates beyond the previous reserved window, registers new allocations, and waits `tso_service_window_duration_ms + 1000` milliseconds before taking a fixed exclusive transaction-ID bound from MetaService. An instance-wide strict check must then find no running transaction below that bound. The check covers every database/table and never skips expired running transactions. An expired lazy transaction can still await real publication. Normal pending registrations continue to constrain the prefix after the recovery check succeeds. +A new master calibrates beyond the previous reserved window, registers new allocations, and waits `tso_service_window_duration_ms + 1000` milliseconds before taking a fixed exclusive transaction-ID bound from MetaService. An instance-wide strict check must then find no running transaction below that bound. The check covers every database/table and never skips expired running transactions. An expired lazy transaction can still await real publication. Normal pending registrations continue to constrain the prefix after the recovery check succeeds. Before recovery completes, reads above the durable prefix return 5100 with reason `TSO_RECOVERING`; an empty new-master registration set does not prove old transactions are visible. Reinitialization invalidates any existing read wait. -Upgrade MetaService before using the new FE's committed prefix: recovery requires an explicit acknowledgement of the strict-check option. Old journal/image records remain readable and imply an unknown prefix. Old BE requests without column names retain the original four-column system-table response; new BE requests explicitly name all six columns. A new BE cannot obtain the new columns from an old FE. +Upgrade MetaService before using the new FE's committed prefix: recovery requires an explicit acknowledgement of the strict-check option. Old journal/image records remain readable and imply an unknown prefix. The new window-error fields in the FE RPC are optional: new followers treat an absent business code as 5100; older followers classify a new master's 5101 as the original retryable 5100 until upgraded. Old BE requests without column names retain the original four-column system-table response; new BE requests explicitly name all six columns. A new BE cannot obtain the new columns from an old FE. -The fixed recovery wait is a temporary operational assumption, not a fencing protocol. It cannot prevent an old master that continues allocating after the wait from assigning an old TSO outside the captured transaction bound. Clock skew, an old longer reservation window, or long process pauses can violate that assumption. This change deliberately retains that accepted limitation. Recovery can also wait for long-running old transactions, and any oldest pending transaction can delay the global prefix across unrelated tables. +The fixed recovery wait is a temporary operational assumption, not a fencing protocol. It cannot prevent an old master that continues allocating after the wait from assigning an old TSO outside the captured transaction bound. Clock skew, an old longer reservation window, or long process pauses can violate that assumption. This change deliberately retains that accepted limitation. Recovery can also wait for long-running old transactions, and the oldest pending transaction can delay the global prefix across unrelated tables. After recovery, that delay no longer blocks reads of unrelated tables whose requested end is at or before the current TSO physical time. ## Diagnosis diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java index 887f6b073d8923..3ed4fe7c19c15f 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java @@ -1237,6 +1237,9 @@ public enum ErrorCode { ERR_INCR_WINDOW_NOT_READY(5100, new byte[]{'H', 'Y', '0', '0', '0'}, "The requested incremental read window is not yet visible; retry the same window."), + ERR_INCR_VISIBLE_WAIT_TIMEOUT(5101, new byte[]{'H', 'Y', '0', '0', '0'}, + "Timed out waiting for incremental read transactions to become visible; retry the same window."), + ERR_NOT_CLOUD_MODE(6000, new byte[]{'4', '2', '0', '0', '0'}, "Command only support in cloud mode."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/IncrWindowNotReadyException.java b/fe/fe-core/src/main/java/org/apache/doris/common/IncrWindowNotReadyException.java index 51046aae79f7e1..070ce257b48bc4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/IncrWindowNotReadyException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/IncrWindowNotReadyException.java @@ -24,15 +24,47 @@ public class IncrWindowNotReadyException extends UserException { private final long requestedEndTimestampMs; private final long committedTso; private final long retryAfterMs; + private final long currentTso; + private final long timeoutMs; + private final String reason; public IncrWindowNotReadyException(long requestedEndTimestampMs, long committedTso, long retryAfterMs) { - super(String.format("ERR_INCR_WINDOW_NOT_READY: requestedEndTimestampMs=%d, committedTSO=%d, " - + "committedTSOPhysicalTimeMs=%d, retryAfterMs=%d", - requestedEndTimestampMs, committedTso, TSOTimestamp.extractPhysicalTime(committedTso), retryAfterMs)); - setMysqlErrorCode(ErrorCode.ERR_INCR_WINDOW_NOT_READY); + this(ErrorCode.ERR_INCR_WINDOW_NOT_READY, "WINDOW_NOT_READY", requestedEndTimestampMs, + 0, committedTso, retryAfterMs, 0); + } + + public IncrWindowNotReadyException(ErrorCode errorCode, String reason, long requestedEndTimestampMs, + long currentTso, long committedTso, long retryAfterMs, long timeoutMs) { + super(String.format("%s: reason=%s, requestedEndTimestampMs=%d, currentTSO=%d, " + + "currentTSOPhysicalTimeMs=%d, committedTSO=%d, committedTSOPhysicalTimeMs=%d, " + + "retryAfterMs=%d, timeoutMs=%d", + errorCode.name(), reason, requestedEndTimestampMs, currentTso, + TSOTimestamp.extractPhysicalTime(currentTso), committedTso, + TSOTimestamp.extractPhysicalTime(committedTso), retryAfterMs, timeoutMs)); + setMysqlErrorCode(errorCode); this.requestedEndTimestampMs = requestedEndTimestampMs; this.committedTso = committedTso; this.retryAfterMs = retryAfterMs; + this.currentTso = currentTso; + this.timeoutMs = timeoutMs; + this.reason = reason; + } + + public static boolean isWindowError(ErrorCode errorCode) { + return errorCode == ErrorCode.ERR_INCR_WINDOW_NOT_READY + || errorCode == ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT; + } + + public long getCurrentTso() { + return currentTso; + } + + public long getTimeoutMs() { + return timeoutMs; + } + + public String getReason() { + return reason; } public long getRequestedEndTimestampMs() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index b37440c6d52506..8ce8d1879afa1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -735,7 +735,8 @@ public void execute(TUniqueId queryId) throws Exception { Throwable cause = e instanceof NereidsException ? Util.getRootCause(((NereidsException) e).getException()) : e; if (cause instanceof IncrWindowNotReadyException) { - context.getState().setError(ErrorCode.ERR_INCR_WINDOW_NOT_READY, e.getMessage()); + context.getState().setError(((IncrWindowNotReadyException) cause).getMysqlErrorCode(), + e.getMessage()); } else { context.getState().setError(e.getMessage()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiter.java b/fe/fe-core/src/main/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiter.java index e750ce48caa007..278a155d598ec3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiter.java @@ -26,6 +26,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ClientPool; import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.UserException; import org.apache.doris.nereids.analyzer.UnboundRelation; @@ -35,6 +36,7 @@ import org.apache.doris.thrift.FrontendService; import org.apache.doris.thrift.TAcquireTimeBasedChangeReadFenceRequest; import org.apache.doris.thrift.TAcquireTimeBasedChangeReadFenceResult; +import org.apache.doris.thrift.TIncrWindowNotReady; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.transaction.GlobalTransactionMgrIface; @@ -56,7 +58,8 @@ /** * Establishes a closed upper fence before planning a time-based incremental read. * - *

Bounded, strongly consistent cloud reads use the master's durable committed TSO. Other reads + *

Bounded, strongly consistent cloud reads use the master's durable committed TSO or wait for + * registered transactions involving the target tables before the requested end. Other reads * retain the transaction drain: the master validates its TSO, then captures a * transaction ID watermark and drains earlier transactions involving the target tables. In classic * mode, it also synchronizes with transaction publishers through the target table locks and returns @@ -178,22 +181,18 @@ public static ChangeReadFence acquireFenceOnMaster(Map> dbToTab throw new UserException("time-based change read fence must be acquired on the master FE"); } + if (Config.isCloudMode() && waitForTransactions && allEndsExplicit) { + Preconditions.checkArgument(maxEndTimestampMs != null, "bounded read requires an end timestamp"); + TSOService.TSOStatusSnapshot ready = env.getTSOService().waitForReadableWindow( + dbToTableIds, maxEndTimestampMs, timeoutMs); + // Partition versions are still refreshed from MS during planning. + return new ChangeReadFence(ready.getCurrentTso(), env.getMaxJournalId(), ready.getCommittedTso()); + } TSOService.TSOStatusSnapshot tsoSnapshot = env.getTSOService().getStatusSnapshot(); if (!tsoSnapshot.isInitialized()) { throw new UserException("TSO timestamp is not calibrated, please check"); } long currentTso = tsoSnapshot.getCurrentTso(); - if (Config.isCloudMode() && waitForTransactions && allEndsExplicit) { - Preconditions.checkArgument(maxEndTimestampMs != null, "bounded read requires an end timestamp"); - long committedTso = tsoSnapshot.getCommittedTso(); - if (committedTso == 0 || maxEndTimestampMs > TSOTimestamp.extractPhysicalTime(committedTso)) { - throw new IncrWindowNotReadyException(maxEndTimestampMs, committedTso, - Config.tso_service_window_duration_ms); - } - // Partition versions are still refreshed from MS during planning. No transaction - // watermark/RPC is needed for a window already covered by the durable prefix. - return new ChangeReadFence(currentTso, env.getMaxJournalId(), committedTso); - } validateEndTimestamp(maxEndTimestampMs, currentTso); if (waitForTransactions) { @@ -347,6 +346,10 @@ private static ChangeReadFence acquireFenceFromMaster(ConnectContext context, Ch TNetworkAddress masterAddress = new TNetworkAddress( context.getEnv().getMasterHost(), context.getEnv().getMasterRpcPort()); int thriftTimeoutMs = (int) Math.min(Integer.MAX_VALUE, Math.max(1L, timeoutMs)); + if (Config.isCloudMode() && waitForTransactions && changeReadInfo.isAllEndsExplicit()) { + // Let the master's typed wait-timeout response arrive before the socket times out. + thriftTimeoutMs = (int) Math.min(Integer.MAX_VALUE, (long) thriftTimeoutMs + 1000); + } FrontendService.Client client; try { client = ClientPool.frontendPool.borrowObject(masterAddress, thriftTimeoutMs); @@ -359,8 +362,17 @@ private static ChangeReadFence acquireFenceFromMaster(ConnectContext context, Ch TAcquireTimeBasedChangeReadFenceResult result = client.acquireTimeBasedChangeReadFence(request); returnToPool = true; if (result.isSetWindowNotReady()) { - throw new IncrWindowNotReadyException(result.getWindowNotReady().getRequestedEndTimestampMs(), - result.getWindowNotReady().getCommittedTso(), result.getWindowNotReady().getRetryAfterMs()); + TIncrWindowNotReady error = result.getWindowNotReady(); + int code = error.isSetErrorCode() + ? error.getErrorCode() : ErrorCode.ERR_INCR_WINDOW_NOT_READY.getCode(); + Preconditions.checkState(code == ErrorCode.ERR_INCR_WINDOW_NOT_READY.getCode() + || code == ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT.getCode(), + "unsupported incremental window error code: %s", code); + throw new IncrWindowNotReadyException(code == ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT.getCode() + ? ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT : ErrorCode.ERR_INCR_WINDOW_NOT_READY, + error.isSetReason() ? error.getReason() : "WINDOW_NOT_READY", + error.getRequestedEndTimestampMs(), + error.getCurrentTso(), error.getCommittedTso(), error.getRetryAfterMs(), error.getTimeoutMs()); } if (result.getStatus().getStatusCode() != TStatusCode.OK) { String error = result.getStatus().isSetErrorMsgs() diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 19ee038ecaae7e..b4fd5ab1264f3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -3412,7 +3412,9 @@ public TAcquireTimeBasedChangeReadFenceResult acquireTimeBasedChangeReadFence( status.setStatusCode(TStatusCode.ANALYSIS_ERROR); status.addToErrorMsgs(e.getDetailMessage()); result.setWindowNotReady(new TIncrWindowNotReady( - e.getRequestedEndTimestampMs(), e.getCommittedTso(), e.getRetryAfterMs())); + e.getRequestedEndTimestampMs(), e.getCommittedTso(), e.getRetryAfterMs()) + .setCurrentTso(e.getCurrentTso()).setErrorCode(e.getMysqlErrorCode().getCode()) + .setTimeoutMs(e.getTimeoutMs()).setReason(e.getReason())); } catch (UserException e) { status.setStatusCode(TStatusCode.ANALYSIS_ERROR); status.addToErrorMsgs(e.getDetailMessage()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java index c743df4fa76d4a..2b40f64163d3e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java @@ -21,6 +21,7 @@ package org.apache.doris.service.arrowflight; import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.Util; import org.apache.doris.mysql.MysqlCommand; @@ -307,10 +308,10 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con } static FlightRuntimeException queryFailure(QueryState state, String message, Throwable cause) { - if (state.getErrorCode() == ErrorCode.ERR_INCR_WINDOW_NOT_READY) { + if (IncrWindowNotReadyException.isWindowError(state.getErrorCode())) { ErrorFlightMetadata metadata = new ErrorFlightMetadata(); - metadata.insert("doris-error-code", Integer.toString(ErrorCode.ERR_INCR_WINDOW_NOT_READY.getCode())); - metadata.insert("doris-error-name", "ERR_INCR_WINDOW_NOT_READY"); + metadata.insert("doris-error-code", Integer.toString(state.getErrorCode().getCode())); + metadata.insert("doris-error-name", state.getErrorCode().name()); // The description preserves the requested end, committed prefix and retry delay from QueryState. return CallStatus.UNAVAILABLE.withDescription(message).withCause(cause) .withMetadata(metadata).toRuntimeException(); @@ -328,11 +329,13 @@ public FlightInfo getFlightInfoStatement(final CommandStatementQuery request, fi if (e instanceof FlightRuntimeException) { FlightRuntimeException flightError = (FlightRuntimeException) e; ErrorFlightMetadata metadata = flightError.status().metadata(); - // Only the incremental-window error bypasses the original INTERNAL wrapper. - if (metadata.containsKey("doris-error-code") - && Integer.toString(ErrorCode.ERR_INCR_WINDOW_NOT_READY.getCode()) - .equals(metadata.get("doris-error-code"))) { - throw flightError; + // Only the two incremental-window errors bypass the original INTERNAL wrapper. + if (metadata.containsKey("doris-error-code")) { + String code = metadata.get("doris-error-code"); + if (Integer.toString(ErrorCode.ERR_INCR_WINDOW_NOT_READY.getCode()).equals(code) + || Integer.toString(ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT.getCode()).equals(code)) { + throw flightError; + } } } String errMsg = "get flight info statement failed, " + e.getMessage(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java index 19d5638f218eac..7219a1b27b1aab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java @@ -104,7 +104,8 @@ public static long getCommitTSO(long transactionId, Database db, Set table + transactionId + ": TSO service is unavailable"); } long fetched = Config.isCloudMode() - ? env.getTSOService().getCommitTSO(db.getId(), transactionId) : env.getTSOService().getTSO(); + ? env.getTSOService().getCommitTSO(db.getId(), transactionId, tableIds) + : env.getTSOService().getTSO(); if (fetched <= 0) { throw new TransactionCommitFailedException("failed to get TSO for txn " + transactionId + ", fetched=" + fetched); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java index 193c82d07be64f..c1f195ada40674 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java @@ -19,7 +19,10 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; import org.apache.doris.common.io.CountingDataOutputStream; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.journal.local.LocalJournal; @@ -33,8 +36,11 @@ import java.io.DataInputStream; import java.io.IOException; +import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; @@ -252,18 +258,18 @@ protected void runAfterCatalogReady() { * @throws RuntimeException if TSO is not calibrated or other errors occur */ public long getTSO() { - return getTSO(null); + return getTSO(null, Collections.emptySet()); } - public long getCommitTSO(long dbId, long txnId) { - return getTSO(Pair.of(dbId, txnId)); + public long getCommitTSO(long dbId, long txnId, Set tableIds) { + return getTSO(Pair.of(dbId, txnId), tableIds); } public void transactionFinished(long dbId, long txnId) { transactionTracker.transactionFinished(dbId, txnId); } - private long getTSO(Pair transactionIdentity) { + private long getTSO(Pair transactionIdentity, Set tableIds) { if (!isTsoEnabled()) { throw new RuntimeException("TSO feature is disabled, please check enable_feature_binlog"); } @@ -297,7 +303,7 @@ private long getTSO(Pair transactionIdentity) { continue; } - Pair pair = generateTSO(transactionIdentity); + Pair pair = generateTSO(transactionIdentity, tableIds); long physical = pair.first; long logical = pair.second; @@ -355,6 +361,56 @@ public TSOStatusSnapshot getStatusSnapshot() { } } + /** Establish a bounded cloud read on the master without draining unrelated or later transactions. */ + public TSOStatusSnapshot waitForReadableWindow(Map> dbToTableIds, + long endTimestampMs, long timeoutMs) throws UserException { + long startNanos = System.nanoTime(); + lock.lock(); + try { + TSOStatusSnapshot snapshot = getStatusSnapshot(); + if (!snapshot.isInitialized()) { + throw new UserException("TSO timestamp is not calibrated, please check"); + } + if (endTimestampMs > TSOTimestamp.extractPhysicalTime(snapshot.getCurrentTso())) { + throw windowError(ErrorCode.ERR_INCR_WINDOW_NOT_READY, "END_AFTER_CURRENT_TSO", + endTimestampMs, snapshot, timeoutMs); + } + if (snapshot.getCommittedTso() > 0 + && endTimestampMs <= TSOTimestamp.extractPhysicalTime(snapshot.getCommittedTso())) { + return snapshot; + } + // Logical zero matches the BE upper boundary for the physical interval [start, end). + TSOTransactionTracker.WaitResult result = transactionTracker.awaitTransactions(dbToTableIds, + TSOTimestamp.composePhysicalTimestamp(endTimestampMs), + TimeUnit.MILLISECONDS.toNanos(timeoutMs) - (System.nanoTime() - startNanos)); + snapshot = getStatusSnapshot(); + if (result == TSOTransactionTracker.WaitResult.RECOVERING) { + throw windowError(ErrorCode.ERR_INCR_WINDOW_NOT_READY, "TSO_RECOVERING", + endTimestampMs, snapshot, timeoutMs); + } + if (!Env.getCurrentEnv().isMaster()) { + throw windowError(ErrorCode.ERR_INCR_WINDOW_NOT_READY, "TSO_MASTER_CHANGED", + endTimestampMs, snapshot, timeoutMs); + } + if (result == TSOTransactionTracker.WaitResult.TIMED_OUT) { + throw windowError(ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT, "VISIBLE_WAIT_TIMEOUT", + endTimestampMs, snapshot, timeoutMs); + } + return snapshot; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UserException("interrupted while waiting for incremental read transactions", e); + } finally { + lock.unlock(); + } + } + + private IncrWindowNotReadyException windowError(ErrorCode code, String reason, long endTimestampMs, + TSOStatusSnapshot snapshot, long timeoutMs) { + return new IncrWindowNotReadyException(code, reason, endTimestampMs, snapshot.getCurrentTso(), + snapshot.getCommittedTso(), Config.tso_service_window_duration_ms, timeoutMs); + } + /** * Calibrate the TSO timestamp when service starts * This ensures the timestamp is consistent with the last persisted value @@ -589,10 +645,10 @@ private void writeTimestampToBDBJE(long timestamp) { * @return Pair of (physicalTime, updatedLogicalCounter) for the base timestamp */ private Pair generateTSO() { - return generateTSO(null); + return generateTSO(null, Collections.emptySet()); } - private Pair generateTSO(Pair transactionIdentity) { + private Pair generateTSO(Pair transactionIdentity, Set tableIds) { lock.lock(); try { if (!isTsoEnabled() || !isInitialized.get()) { @@ -610,7 +666,7 @@ private Pair generateTSO(Pair transactionIdentity) { globalTimestamp.setLogicalCounter(nextLogical); if (transactionIdentity != null) { transactionTracker.register(transactionIdentity, - TSOTimestamp.composeTimestamp(physicalTime, nextLogical), System.nanoTime()); + TSOTimestamp.composeTimestamp(physicalTime, nextLogical), System.nanoTime(), tableIds); } return Pair.of(physicalTime, nextLogical); } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java index 07cffa789898d4..738515ffd62263 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java @@ -28,11 +28,15 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; /** In-memory commit registrations. Uses the allocator's lock so no allocated TSO can be missed. */ @@ -41,6 +45,7 @@ final class TSOTransactionTracker { private static final int CHECK_BATCH_SIZE = 64; private static final long CHECK_AGE_NANOS = TimeUnit.SECONDS.toNanos(1); private final ReentrantLock lock; + private final Condition transactionsChanged; private final Map, PendingTransaction> pendingByTxn = new HashMap<>(); private final TreeMap pendingByTso = new TreeMap<>(); private long generation; @@ -49,20 +54,27 @@ final class TSOTransactionTracker { private boolean recoveryReady; private long pollCursor; + enum WaitResult { + FINISHED, TIMED_OUT, RECOVERING + } + private static final class PendingTransaction { private final Pair identity; private final long tso; private final long registeredAtNanos; + private final Set tableIds; - private PendingTransaction(Pair identity, long tso, long nowNanos) { + private PendingTransaction(Pair identity, long tso, long nowNanos, Set tableIds) { this.identity = identity; this.tso = tso; this.registeredAtNanos = nowNanos; + this.tableIds = new HashSet<>(tableIds); } } TSOTransactionTracker(ReentrantLock lock) { this.lock = lock; + this.transactionsChanged = lock.newCondition(); } void reset(long nowNanos, long recoveryDelayMs) { @@ -74,19 +86,57 @@ void reset(long nowNanos, long recoveryDelayMs) { recoveryWatermark = 0; recoveryReady = false; pollCursor = 0; + transactionsChanged.signalAll(); } - void register(Pair identity, long tso, long nowNanos) { + void register(Pair identity, long tso, long nowNanos, Set tableIds) { Preconditions.checkState(lock.isHeldByCurrentThread()); - if (pendingByTxn.containsKey(identity)) { + Preconditions.checkArgument(!tableIds.isEmpty(), "commit registration requires table IDs"); + PendingTransaction existing = pendingByTxn.get(identity); + if (existing != null) { // A timed-out request can still commit using the earlier TSO. + existing.tableIds.addAll(tableIds); return; } - PendingTransaction pending = new PendingTransaction(identity, tso, nowNanos); + PendingTransaction pending = new PendingTransaction(identity, tso, nowNanos, tableIds); pendingByTxn.put(identity, pending); Preconditions.checkState(pendingByTso.put(tso, pending) == null); } + /** Called with the allocator lock after validating endTso against its current clock. */ + WaitResult awaitTransactions(Map> dbToTableIds, long endTso, long remainingNanos) + throws InterruptedException { + Preconditions.checkState(lock.isHeldByCurrentThread()); + if (!recoveryReady) { + return WaitResult.RECOVERING; + } + long waitStartNanos = System.nanoTime(); + long waitGeneration = generation; + List remaining = new ArrayList<>(); + for (PendingTransaction pending : pendingByTso.headMap(endTso, true).values()) { + List tables = dbToTableIds.get(pending.identity.first); + if (tables != null && !Collections.disjoint(tables, pending.tableIds)) { + remaining.add(pending); + } + } + // Allocation/registration and this snapshot share the lock. Later allocations are outside + // the validated window; only this fixed set can affect the read. awaitNanos releases the lock. + while (true) { + if (generation != waitGeneration) { + return WaitResult.RECOVERING; + } + remaining.removeIf(pending -> pendingByTxn.get(pending.identity) != pending); + if (remaining.isEmpty()) { + return WaitResult.FINISHED; + } + long nanosLeft = remainingNanos - (System.nanoTime() - waitStartNanos); + if (nanosLeft <= 0) { + return WaitResult.TIMED_OUT; + } + transactionsChanged.awaitNanos(nanosLeft); + } + } + long candidateCommittedTso(long currentTso, long durableCommittedTso) { Preconditions.checkState(lock.isHeldByCurrentThread()); if (!recoveryReady) { @@ -104,6 +154,7 @@ void transactionFinished(long dbId, long txnId) { PendingTransaction pending = pendingByTxn.remove(Pair.of(dbId, txnId)); if (pending != null) { pendingByTso.remove(pending.tso); + transactionsChanged.signalAll(); } } finally { lock.unlock(); @@ -156,6 +207,7 @@ void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws U if (generation == checkGeneration && pendingByTxn.get(pending.identity) == pending) { pendingByTxn.remove(pending.identity); pendingByTso.remove(pending.tso); + transactionsChanged.signalAll(); } } finally { lock.unlock(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index fdec02c8654e92..fa66af66fefcdb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -78,19 +78,24 @@ protected void runBeforeAll() throws Exception { @Test public void testCommittedTsoErrorSurvivesPlannerWrapping() throws Exception { - connectContext.getState().reset(); - IncrWindowNotReadyException rejected = new IncrWindowNotReadyException(2000, 1000, 1000); - StmtExecutor executor = new StmtExecutor(connectContext, "select 1"); - try (MockedConstruction planners = Mockito.mockConstruction(NereidsPlanner.class, - (planner, construction) -> Mockito.doThrow(new NereidsException(rejected.getMessage(), rejected)) - .when(planner).plan(Mockito.any(StatementBase.class), Mockito.any(TQueryOptions.class)))) { - executor.execute(); - Assertions.assertEquals(1, planners.constructed().size()); + for (ErrorCode code : new ErrorCode[] {ErrorCode.ERR_INCR_WINDOW_NOT_READY, + ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT}) { + connectContext.getState().reset(); + IncrWindowNotReadyException rejected = new IncrWindowNotReadyException(code, "test reason", + 2000, 3000, 1000, 1000, 5000); + StmtExecutor executor = new StmtExecutor(connectContext, "select 1"); + try (MockedConstruction planners = Mockito.mockConstruction(NereidsPlanner.class, + (planner, construction) -> Mockito.doThrow(new NereidsException(rejected.getMessage(), rejected)) + .when(planner).plan(Mockito.any(StatementBase.class), Mockito.any(TQueryOptions.class)))) { + executor.execute(); + Assertions.assertEquals(1, planners.constructed().size()); + } + Assertions.assertEquals(QueryState.MysqlStateType.ERR, connectContext.getState().getStateType()); + Assertions.assertEquals(code, connectContext.getState().getErrorCode()); + Assertions.assertTrue(connectContext.getState().getErrorMessage().contains("requestedEndTimestampMs=2000")); + Assertions.assertTrue(connectContext.getState().getErrorMessage().contains("retryAfterMs=1000")); + Assertions.assertTrue(connectContext.getState().getErrorMessage().contains("timeoutMs=5000")); } - Assertions.assertEquals(QueryState.MysqlStateType.ERR, connectContext.getState().getStateType()); - Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, connectContext.getState().getErrorCode()); - Assertions.assertTrue(connectContext.getState().getErrorMessage().contains("requestedEndTimestampMs=2000")); - Assertions.assertTrue(connectContext.getState().getErrorMessage().contains("retryAfterMs=1000")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java index 6997b380e7985a..c97ffdb07bfe78 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java @@ -40,6 +40,8 @@ import org.apache.doris.thrift.FrontendService; import org.apache.doris.thrift.TAcquireTimeBasedChangeReadFenceRequest; import org.apache.doris.thrift.TAcquireTimeBasedChangeReadFenceResult; +import org.apache.doris.thrift.TIncrWindowNotReady; +import org.apache.doris.thrift.TStatus; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.transaction.GlobalTransactionMgrIface; import org.apache.doris.tso.TSOService; @@ -47,6 +49,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TSerializer; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.InOrder; @@ -252,10 +256,11 @@ public void testHistoricalWindowDoesNotWaitForTransactionsOutsideWindow() throws } @Test - public void testBoundedReadRejectsUnknownOrInsufficientCommittedTso() throws Exception { + public void testBoundedReadDelegatesRegisteredTransactionWait() throws Exception { Env env = mockMasterEnv(); TSOService service = mockTsoService(env, CURRENT_TSO); GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); + Map> tables = ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID)); try (MockedStatic config = Mockito.mockStatic(Config.class); MockedStatic mocked = Mockito.mockStatic(Env.class)) { config.when(Config::isCloudMode).thenReturn(true); @@ -264,14 +269,11 @@ public void testBoundedReadRejectsUnknownOrInsufficientCommittedTso() throws Exc for (long committed : new long[] {0, TSOTimestamp.composeTimestamp(CURRENT_PHYSICAL_TIME_MS - 1, 17)}) { Mockito.when(service.getStatusSnapshot()).thenReturn( new TSOService.TSOStatusSnapshot(true, CURRENT_TSO, CURRENT_PHYSICAL_TIME_MS + 1000, committed)); - IncrWindowNotReadyException error = Assertions.assertThrows(IncrWindowNotReadyException.class, - () -> TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( - ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID)), - CURRENT_PHYSICAL_TIME_MS, 1000, true, true)); - Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, error.getMysqlErrorCode()); - Assertions.assertEquals(committed, error.getCommittedTso()); - Assertions.assertEquals(CURRENT_PHYSICAL_TIME_MS, error.getRequestedEndTimestampMs()); + TimeBasedChangeVisibleWaiter.ChangeReadFence result = TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( + tables, CURRENT_PHYSICAL_TIME_MS, 1000, true, true); + Assertions.assertEquals(committed, result.getCommittedTso()); } + Mockito.verify(service, Mockito.times(2)).waitForReadableWindow(tables, CURRENT_PHYSICAL_TIME_MS, 1000); Mockito.verifyNoInteractions(txnMgr); } } @@ -288,25 +290,54 @@ public void testMixedBoundedAndUnboundedRelationsRetainDrain() { } @Test - public void testMasterRpcAndFollowerPreserveWindowNotReady() throws Exception { - Env master = mockMasterEnv(); - mockTsoService(master, CURRENT_TSO); // Unknown committed prefix. - TAcquireTimeBasedChangeReadFenceRequest request = new TAcquireTimeBasedChangeReadFenceRequest(); - request.setDbToTableIds(ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID))); - request.setEndTimestampMs(CURRENT_PHYSICAL_TIME_MS); - request.setTimeoutMs(1000); - request.setWaitForTransactions(true); - request.setAllEndsExplicit(true); - TAcquireTimeBasedChangeReadFenceResult response; - try (MockedStatic mocked = Mockito.mockStatic(Env.class); - MockedStatic config = Mockito.mockStatic(Config.class)) { - mocked.when(Env::getCurrentEnv).thenReturn(master); - config.when(Config::isCloudMode).thenReturn(true); - response = new FrontendServiceImpl(null).acquireTimeBasedChangeReadFence(request); + public void testMasterRpcAndFollowerPreserveBothWindowErrors() throws Exception { + for (ErrorCode code : new ErrorCode[] {ErrorCode.ERR_INCR_WINDOW_NOT_READY, + ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT}) { + Env master = mockMasterEnv(); + TSOService service = mockTsoService(master, CURRENT_TSO); + String reason = code == ErrorCode.ERR_INCR_WINDOW_NOT_READY ? "TSO_RECOVERING" : "VISIBLE_WAIT_TIMEOUT"; + IncrWindowNotReadyException failure = new IncrWindowNotReadyException(code, reason, + CURRENT_PHYSICAL_TIME_MS, CURRENT_TSO, 0, 1000, 1000); + Mockito.doThrow(failure).when(service) + .waitForReadableWindow(Mockito.anyMap(), Mockito.anyLong(), Mockito.anyLong()); + TAcquireTimeBasedChangeReadFenceRequest request = new TAcquireTimeBasedChangeReadFenceRequest(); + request.setDbToTableIds(ImmutableMap.of(DB_ID, ImmutableList.of(TABLE_ID))); + request.setEndTimestampMs(CURRENT_PHYSICAL_TIME_MS); + request.setTimeoutMs(1000); + request.setWaitForTransactions(true); + request.setAllEndsExplicit(true); + TAcquireTimeBasedChangeReadFenceResult response; + try (MockedStatic mocked = Mockito.mockStatic(Env.class); + MockedStatic config = Mockito.mockStatic(Config.class)) { + mocked.when(Env::getCurrentEnv).thenReturn(master); + config.when(Config::isCloudMode).thenReturn(true); + response = new FrontendServiceImpl(null).acquireTimeBasedChangeReadFence(request); + } + Assertions.assertEquals(TStatusCode.ANALYSIS_ERROR, response.getStatus().getStatusCode()); + Assertions.assertEquals(code.getCode(), response.getWindowNotReady().getErrorCode()); + // Exercise the optional fields over the actual Thrift encoding before the follower receives them. + TAcquireTimeBasedChangeReadFenceResult decoded = new TAcquireTimeBasedChangeReadFenceResult(); + new TDeserializer().deserialize(decoded, new TSerializer().serialize(response)); + IncrWindowNotReadyException propagated = receiveWindowErrorOnFollower(decoded); + Assertions.assertEquals(code, propagated.getMysqlErrorCode()); + Assertions.assertEquals(CURRENT_TSO, propagated.getCurrentTso()); + Assertions.assertEquals(1000, propagated.getTimeoutMs()); + Assertions.assertEquals(reason, propagated.getReason()); } - Assertions.assertEquals(TStatusCode.ANALYSIS_ERROR, response.getStatus().getStatusCode()); - Assertions.assertTrue(response.isSetWindowNotReady()); - Assertions.assertEquals(CURRENT_PHYSICAL_TIME_MS, response.getWindowNotReady().getRequestedEndTimestampMs()); + } + + @Test + public void testFollowerAcceptsWindowErrorFromOlderMaster() throws Exception { + TAcquireTimeBasedChangeReadFenceResult response = new TAcquireTimeBasedChangeReadFenceResult(); + response.setStatus(new TStatus(TStatusCode.ANALYSIS_ERROR)); + response.setWindowNotReady(new TIncrWindowNotReady(CURRENT_PHYSICAL_TIME_MS, 0, 1000)); + IncrWindowNotReadyException propagated = receiveWindowErrorOnFollower(response); + Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, propagated.getMysqlErrorCode()); + Assertions.assertEquals("WINDOW_NOT_READY", propagated.getReason()); + } + + private IncrWindowNotReadyException receiveWindowErrorOnFollower(TAcquireTimeBasedChangeReadFenceResult response) + throws Exception { Env follower = Mockito.mock(Env.class); Mockito.when(follower.getMasterHost()).thenReturn("127.0.0.1"); Mockito.when(follower.getMasterRpcPort()).thenReturn(9020); @@ -318,15 +349,17 @@ public void testMasterRpcAndFollowerPreserveWindowNotReady() throws Exception { GenericPool pool = Mockito.mock(GenericPool.class); Mockito.when(pool.borrowObject(Mockito.any(), Mockito.anyInt())).thenReturn(client); ClientPool.frontendPool = pool; - try { + try (MockedStatic config = Mockito.mockStatic(Config.class)) { + config.when(Config::isCloudMode).thenReturn(true); IncrWindowNotReadyException error = Assertions.assertThrows(IncrWindowNotReadyException.class, () -> TimeBasedChangeVisibleWaiter.waitForVisible(context, newChangeRelation(1, ImmutableMap.of(OlapScanNode.OLAP_END_TIMESTAMP, "2024-01-01 00:00:00")), ImmutableMap.of(TABLE_QUALIFIER, mockOlapTable(DB_ID, TABLE_ID)))); - Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, error.getMysqlErrorCode()); Assertions.assertEquals(CURRENT_PHYSICAL_TIME_MS, error.getRequestedEndTimestampMs()); Assertions.assertEquals(0, error.getCommittedTso()); + Mockito.verify(pool).borrowObject(Mockito.any(), Mockito.eq(2000)); Mockito.verify(pool).returnObject(Mockito.any(), Mockito.eq(client)); + return error; } finally { ClientPool.frontendPool = originalPool; } @@ -347,10 +380,12 @@ private Env mockMasterEnv() { return env; } - private TSOService mockTsoService(Env env, long currentTso) { + private TSOService mockTsoService(Env env, long currentTso) throws UserException { TSOService tsoService = Mockito.mock(TSOService.class); Mockito.when(tsoService.getStatusSnapshot()).thenReturn( new TSOService.TSOStatusSnapshot(true, currentTso, CURRENT_PHYSICAL_TIME_MS + 1000)); + Mockito.when(tsoService.waitForReadableWindow(Mockito.anyMap(), Mockito.anyLong(), Mockito.anyLong())) + .thenAnswer(invocation -> tsoService.getStatusSnapshot()); Mockito.when(env.getTSOService()).thenReturn(tsoService); return tsoService; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java index 5ad9a88bec59ce..db009abc8cf51e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java @@ -70,14 +70,22 @@ public void testWindowNotReadyHasRetryableFlightStatusAndStableBusinessCode() { } @Test - public void testGetFlightInfoPreservesWindowNotReadyError() throws Exception { - QueryState state = new QueryState(); - IncrWindowNotReadyException error = new IncrWindowNotReadyException(2000, 100, 1000); - state.setError(error.getMysqlErrorCode(), error.getDetailMessage()); - FlightRuntimeException failure = DorisFlightSqlProducer.queryFailure(state, state.getErrorMessage(), error); - - // Preserve the retryable status, business metadata and committed TSO details without wrapping. - Assertions.assertSame(failure, getFlightInfoFailure(failure)); + public void testGetFlightInfoPreservesBothWindowErrors() throws Exception { + for (ErrorCode code : new ErrorCode[] {ErrorCode.ERR_INCR_WINDOW_NOT_READY, + ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT}) { + QueryState state = new QueryState(); + IncrWindowNotReadyException error = new IncrWindowNotReadyException(code, "test reason", + 2000, 3000, 100, 1000, 5000); + state.setError(error.getMysqlErrorCode(), error.getDetailMessage()); + FlightRuntimeException failure = DorisFlightSqlProducer.queryFailure(state, state.getErrorMessage(), error); + Assertions.assertEquals(FlightStatusCode.UNAVAILABLE, failure.status().code()); + Assertions.assertEquals(Integer.toString(code.getCode()), failure.status().metadata().get("doris-error-code")); + Assertions.assertEquals(code.name(), failure.status().metadata().get("doris-error-name")); + Assertions.assertTrue(failure.status().description().contains("currentTSO=3000")); + Assertions.assertTrue(failure.status().description().contains("committedTSO=100")); + Assertions.assertTrue(failure.status().description().contains("timeoutMs=5000")); + Assertions.assertSame(failure, getFlightInfoFailure(failure)); + } } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java index 9aaa51d66aa281..d2bd438364210a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java @@ -19,7 +19,10 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.IncrWindowNotReadyException; import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; import org.apache.doris.common.io.CountingDataOutputStream; import org.apache.doris.journal.Journal; import org.apache.doris.journal.JournalEntity; @@ -27,6 +30,7 @@ import org.apache.doris.metric.MetricRepo; import org.apache.doris.persist.EditLog; import org.apache.doris.persist.OperationType; +import org.apache.doris.qe.TimeBasedChangeVisibleWaiter; import org.apache.doris.transaction.GlobalTransactionMgrIface; import org.junit.jupiter.api.AfterEach; @@ -44,6 +48,10 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -446,6 +454,102 @@ public void testGenerateTSOReturnsZeroWhenDisabledOrNotInitialized() throws Exce } } + @Test + public void testReadableWindowDoesNotRequireAnotherCommittedTsoFlush() throws Exception { + Mockito.when(env.isMaster()).thenReturn(true); + Mockito.when(env.getTSOService()).thenReturn(tsoService); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 1000, 17); + long committed = TSOTimestamp.composeTimestamp(900, 1); + tsoService.replayWindowEndTSO(new TSOServiceState(2000, committed)); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); + GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)).thenReturn(true); + tracker.checkTransactions(txnMgr, Long.MAX_VALUE); + try (MockedStatic config = Mockito.mockStatic(Config.class)) { + config.when(Config::isCloudMode).thenReturn(true); + TimeBasedChangeVisibleWaiter.ChangeReadFence fence = TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( + Collections.singletonMap(1L, Collections.singletonList(2L)), 950L, 0, true, true); + Assertions.assertEquals(committed, fence.getCommittedTso()); + } + } + + private void prepareWindowRead(boolean finishRecovery) throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + Mockito.when(env.getTSOService()).thenReturn(tsoService); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + tsoService.replayWindowEndTSO(new TSOServiceState(2000, TSOTimestamp.composeTimestamp(80, 1))); + if (finishRecovery) { + Field field = TSOService.class.getDeclaredField("transactionTracker"); + field.setAccessible(true); + GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)).thenReturn(true); + ((TSOTransactionTracker) field.get(tsoService)).checkTransactions(txnMgr, Long.MAX_VALUE); + } + } + + @Test + public void testSlowTableDoesNotBlockAnotherTableOrAnEarlierEnd() throws Exception { + prepareWindowRead(true); + long pending = tsoService.getCommitTSO(1, 10, Set.of(100L, 101L)); + setGlobalTimestamp(tsoService, 150, 17); + TSOService.TSOStatusSnapshot unrelated = tsoService.waitForReadableWindow( + Map.of(1L, Collections.singletonList(200L)), 120, 0); + Assertions.assertTrue(unrelated.getCommittedTso() < pending); + tsoService.waitForReadableWindow(Map.of(1L, Collections.singletonList(100L)), 100, 0); + for (long table : new long[] {100, 101}) { + IncrWindowNotReadyException error = Assertions.assertThrows(IncrWindowNotReadyException.class, + () -> tsoService.waitForReadableWindow(Map.of(1L, Collections.singletonList(table)), 120, 0)); + Assertions.assertEquals(ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT, error.getMysqlErrorCode()); + Assertions.assertEquals("VISIBLE_WAIT_TIMEOUT", error.getReason()); + Assertions.assertEquals(120, error.getRequestedEndTimestampMs()); + Assertions.assertEquals(tsoService.getCurrentTSO(), error.getCurrentTso()); + Assertions.assertEquals(unrelated.getCommittedTso(), error.getCommittedTso()); + } + tsoService.transactionFinished(1, 10); + TSOService.TSOStatusSnapshot finished = tsoService.waitForReadableWindow( + Map.of(1L, Collections.singletonList(100L)), 120, 0); + // A table-specific successful read does not advance the global durable prefix. + Assertions.assertEquals(unrelated.getCommittedTso(), finished.getCommittedTso()); + } + + @Test + public void testFutureWindowAndRecoveryHaveDifferentReasonsFromWaitTimeout() throws Exception { + prepareWindowRead(false); + Map> tables = Map.of(1L, Collections.singletonList(100L)); + IncrWindowNotReadyException future = Assertions.assertThrows(IncrWindowNotReadyException.class, + () -> tsoService.waitForReadableWindow(tables, 101, 0)); + Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, future.getMysqlErrorCode()); + Assertions.assertEquals("END_AFTER_CURRENT_TSO", future.getReason()); + IncrWindowNotReadyException recovering = Assertions.assertThrows(IncrWindowNotReadyException.class, + () -> tsoService.waitForReadableWindow(tables, 90, 0)); + Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, recovering.getMysqlErrorCode()); + Assertions.assertEquals("TSO_RECOVERING", recovering.getReason()); + tsoService.waitForReadableWindow(tables, 80, 0); + } + + @Test + public void testInterruptedReadWaitRestoresInterruptFlag() throws Exception { + prepareWindowRead(true); + tsoService.getCommitTSO(1, 10, Collections.singleton(100L)); + setGlobalTimestamp(tsoService, 150, 17); + try { + Thread.currentThread().interrupt(); + UserException error = Assertions.assertThrows(UserException.class, () -> tsoService.waitForReadableWindow( + Map.of(1L, Collections.singletonList(100L)), 120, 1000)); + Assertions.assertTrue(error.getDetailMessage().contains("interrupted")); + Assertions.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + @Test public void testCommittedPrefixIsPublishedOnlyAfterJournalSuccess() throws Exception { Mockito.when(env.isReady()).thenReturn(true); @@ -491,7 +595,7 @@ public void testRecoveryFreezesPrefixAndPeriodicFlushWorksWithUnchangedWindow() config.when(Config::isCloudMode).thenReturn(true); invokeCalibrateTimestamp(tsoService); Assertions.assertEquals(oldCommitted, tsoService.getStatusSnapshot().getCommittedTso()); - long pendingTso = tsoService.getCommitTSO(1, 10); + long pendingTso = tsoService.getCommitTSO(1, 10, Collections.singleton(2L)); Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); trackerField.setAccessible(true); TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); @@ -565,14 +669,14 @@ public void testAllocationAndRegistrationShareSnapshotLock() throws Exception { enteredRegistration.countDown(); Assertions.assertTrue(releaseRegistration.await(30, TimeUnit.SECONDS)); return invocation.callRealMethod(); - }).when(tracker).register(Mockito.any(), Mockito.anyLong(), Mockito.anyLong()); - Method generate = TSOService.class.getDeclaredMethod("generateTSO", Pair.class); + }).when(tracker).register(Mockito.any(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anySet()); + Method generate = TSOService.class.getDeclaredMethod("generateTSO", Pair.class, Set.class); generate.setAccessible(true); ExecutorService executor = Executors.newSingleThreadExecutor(); try { Future allocation = executor.submit(() -> { try { - generate.invoke(tsoService, Pair.of(1L, 10L)); + generate.invoke(tsoService, Pair.of(1L, 10L), Collections.singleton(2L)); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java index 89d38308367059..cc3580425eb65e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java @@ -27,6 +27,13 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; @@ -47,7 +54,7 @@ private void reset(long delayMs) { private void register(long dbId, long txnId, long tso) { lock.lock(); try { - tracker.register(Pair.of(dbId, txnId), tso, 0); + tracker.register(Pair.of(dbId, txnId), tso, 0, Collections.singleton(100L)); } finally { lock.unlock(); } @@ -68,6 +75,118 @@ private void finishRecovery() throws Exception { tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); } + private TSOTransactionTracker.WaitResult awaitTable(long dbId, long tableId, long endTso) throws Exception { + lock.lock(); + try { + return tracker.awaitTransactions(Collections.singletonMap(dbId, Collections.singletonList(tableId)), + endTso, 0); + } finally { + lock.unlock(); + } + } + + @Test + public void testReadWaitFiltersDatabaseTableAndExclusivePhysicalEnd() throws Exception { + reset(0); + finishRecovery(); + register(1, 10, TSOTimestamp.composeTimestamp(100, 1)); + long end = TSOTimestamp.composePhysicalTimestamp(101); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(2, 100, end)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 200, end)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, + awaitTable(1, 100, TSOTimestamp.composePhysicalTimestamp(100))); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, end)); + // Retrying later must not hide the original in-window request or lose any involved table. + lock.lock(); + try { + tracker.register(Pair.of(1L, 10L), TSOTimestamp.composeTimestamp(200, 1), 0, Set.of(100L, 200L)); + } finally { + lock.unlock(); + } + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, end)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 200, end)); + tracker.transactionFinished(1, 10); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 200, end)); + } + + @Test + public void testEmptyRegistrationSetCannotBypassRecovery() throws Exception { + reset(2000); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.RECOVERING, awaitTable(1, 100, 200)); + } + + private Future submitReadWait(ExecutorService executor, CountDownLatch started) { + return executor.submit(() -> { + lock.lock(); + try { + started.countDown(); + return tracker.awaitTransactions(Map.of(1L, Collections.singletonList(100L)), + 200, TimeUnit.SECONDS.toNanos(30)); + } finally { + lock.unlock(); + } + }); + } + + @Test + public void testReconciliationWakesReadWithoutAdvancingTheGlobalPrefix() throws Exception { + reset(0); + finishRecovery(); + register(1, 10, 100); + register(2, 20, 90); // An unrelated database continues to hold the global prefix. + CountDownLatch started = new CountDownLatch(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future waiting = submitReadWait(executor, started); + Assertions.assertTrue(started.await(30, TimeUnit.SECONDS)); + TransactionState aborted = Mockito.mock(TransactionState.class); + Mockito.when(aborted.getTransactionStatus()).thenReturn(TransactionStatus.ABORTED); + Mockito.when(txnMgr.getTransactionState(1, 10)).thenReturn(aborted); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, waiting.get(30, TimeUnit.SECONDS)); + Assertions.assertEquals(89, candidate(300, 80)); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testWaitReleasesAllocatorLockAndDoesNotFollowLaterWrites() throws Exception { + reset(0); + finishRecovery(); + register(1, 10, 100); + CountDownLatch started = new CountDownLatch(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future waiting = submitReadWait(executor, started); + Assertions.assertTrue(started.await(30, TimeUnit.SECONDS)); + // Acquiring the lock proves the waiter released it; allocation can continue while it waits. + register(1, 20, 300); + tracker.transactionFinished(1, 10); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, waiting.get(30, TimeUnit.SECONDS)); + Assertions.assertEquals(1, tracker.getPendingCount()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testResetInvalidatesAnInFlightReadWait() throws Exception { + reset(0); + finishRecovery(); + register(1, 10, 100); + CountDownLatch started = new CountDownLatch(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future waiting = submitReadWait(executor, started); + Assertions.assertTrue(started.await(30, TimeUnit.SECONDS)); + reset(2000); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.RECOVERING, waiting.get(30, TimeUnit.SECONDS)); + } finally { + executor.shutdownNow(); + } + } + @Test public void testOutOfOrderVisibilityAndRetryRetainEarliestTso() throws Exception { reset(2000); diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index f797c2e7c9bcd9..fb3f4c3860b8ba 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -2002,6 +2002,11 @@ struct TIncrWindowNotReady { 1: required i64 requested_end_timestamp_ms 2: required i64 committed_tso 3: required i64 retry_after_ms + 4: optional i64 current_tso + // Absent on older masters: ERR_INCR_WINDOW_NOT_READY (5100). + 5: optional i32 error_code + 6: optional i64 timeout_ms + 7: optional string reason } struct TAcquireTimeBasedChangeReadFenceResult { diff --git a/regression-test/data/tso_p0/test_committed_tso.out b/regression-test/data/tso_p0/test_committed_tso.out index a9a8961e82a93d..8f49ac168c2865 100644 --- a/regression-test/data/tso_p0/test_committed_tso.out +++ b/regression-test/data/tso_p0/test_committed_tso.out @@ -13,6 +13,10 @@ true 1 2 +-- !unrelated_table_above_prefix -- +10 +20 + -- !retry_original_window -- 1 2 diff --git a/regression-test/suites/tso_p0/test_committed_tso.groovy b/regression-test/suites/tso_p0/test_committed_tso.groovy index 39bdd2119d8b85..ead5f222c663da 100644 --- a/regression-test/suites/tso_p0/test_committed_tso.groovy +++ b/regression-test/suites/tso_p0/test_committed_tso.groovy @@ -33,6 +33,14 @@ suite("test_committed_tso", "nonConcurrent") { PROPERTIES ("replication_num" = "1", "binlog.enable" = "true", "binlog.format" = "ROW") """ sql "INSERT INTO test_committed_tso VALUES (1), (2)" + sql "DROP TABLE IF EXISTS test_committed_tso_unrelated" + sql """ + CREATE TABLE test_committed_tso_unrelated (id INT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1", "binlog.enable" = "true", "binlog.format" = "ROW") + """ + sql "INSERT INTO test_committed_tso_unrelated VALUES (10), (20)" + sql "SET change_visible_timeout_ms = 1000" def currentPhysicalTime = { (sql "SELECT CURRENT_TSO_PHYSICAL_TIME FROM information_schema.tso_status")[0][0] as long } @@ -54,6 +62,12 @@ suite("test_committed_tso", "nonConcurrent") { SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(historicalEnd)}') ORDER BY id """ + long futureEnd = currentPhysicalTime() + 60_000L + test { + sql "SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(futureEnd)}')" + exception "ERR_INCR_WINDOW_NOT_READY" + } + def masterHttpAddress = "${getMasterIp()}:${getMasterPort()}" def pendingMetric = { String name -> String metrics = Http.GET("http://${masterHttpAddress}/metrics", false, false) @@ -85,9 +99,13 @@ suite("test_committed_tso", "nonConcurrent") { order_qt_history_with_later_write """ SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(historicalEnd)}') ORDER BY id """ + // The global prefix is held by another table. This closed window must remain readable. + order_qt_unrelated_table_above_prefix """ + SELECT id FROM test_committed_tso_unrelated@incr('endTimestamp' = '${timestamp(blockedEnd)}') ORDER BY id + """ test { sql "SELECT id FROM test_committed_tso@incr('endTimestamp' = '${timestamp(blockedEnd)}')" - exception "ERR_INCR_WINDOW_NOT_READY" + exception "ERR_INCR_VISIBLE_WAIT_TIMEOUT" } GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.commitTxn.blockAfterTso") loadThread.join(60000) From f2d1739ff0130d0a83dcf94034a642beaa5fd61a Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Thu, 10 Sep 2026 22:19:44 +0800 Subject: [PATCH 04/13] [doc](fe) Remove the committed TSO design document ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Remove docs/committed-tso.md from the change as requested. The implementation and tests are unchanged. ### Release note None ### Check List (For Author) - Test: No need to test (documentation deletion only); git diff --check passed. - Behavior changed: No - Does this need documentation: No --- docs/committed-tso.md | 42 ------------------------------------------ 1 file changed, 42 deletions(-) delete mode 100644 docs/committed-tso.md diff --git a/docs/committed-tso.md b/docs/committed-tso.md deleted file mode 100644 index a7de997c29a3a7..00000000000000 --- a/docs/committed-tso.md +++ /dev/null @@ -1,42 +0,0 @@ -# Committed TSO for bounded incremental reads - -In cloud mode, a strongly consistent `@incr` query with an explicit `endTimestamp` on every incremental relation uses the master FE's durable committed TSO as a boundary for reads that require no transaction wait. Transactions with commit TSO at or below this prefix are truly visible or aborted. The name does not refer to the intermediate `COMMITTED` transaction state. - -```sql -SELECT CURRENT_TSO_PHYSICAL_TIME, COMMITTED_TSO, COMMITTED_TSO_PHYSICAL_TIME -FROM information_schema.tso_status; -``` - -The committed columns are nullable BIGINTs. `COMMITTED_TSO_PHYSICAL_TIME` is Unix epoch milliseconds and gives an end timestamp guaranteed to require no transaction wait across all tables. Larger ends can also be readable after checking only the relevant tables, up to `CURRENT_TSO_PHYSICAL_TIME`; the full encoded `COMMITTED_TSO` must not be passed as an `endTimestamp` string. Convert milliseconds to the timestamp format and time zone accepted by `@incr`; clients producing whole-second windows must round down. The interval remains `[start, end)`. Existing binlog retention and table requirements still apply. - -The system table reads the master's persisted prefix without allocating a TSO. While a new master recovers, it keeps exposing the previous persisted prefix. A first startup, an old image without a prefix, or classic mode returns NULL for the new columns. Existing disabled/uninitialized TSO errors remain unchanged. - -The master handles a bounded window in three ways: - -| Requested end | Behavior | -| --- | --- | -| After the current TSO's physical time | Immediately return MySQL error 5100, `ERR_INCR_WINDOW_NOT_READY`, with reason `END_AFTER_CURRENT_TSO`. Later allocations could still fall in this window. | -| At or before the durable committed TSO's physical time | Proceed without waiting for transactions. | -| Between those boundaries | After recovery, wait only for registered transactions involving the queried tables whose earliest possible commit TSO falls before the physical end. An empty matching set proceeds immediately. Exceeding `change_visible_timeout_ms` returns error 5101, `ERR_INCR_VISIBLE_WAIT_TIMEOUT`. | - -The error message includes `reason`, `requestedEndTimestampMs`, `currentTSO`, `currentTSOPhysicalTimeMs`, `committedTSO`, `committedTSOPhysicalTimeMs`, `retryAfterMs`, and `timeoutMs`. Master-to-follower RPC preserves both classifications and allows one additional second for the typed wait-timeout response to arrive. Arrow Flight SQL returns UNAVAILABLE for both errors, with their respective `doris-error-code` and `doris-error-name` metadata. Other exceptions keep their existing wrapping. Clients should retry the same split/window/offset after a cancellable delay, and advance offsets only after that window completes. Shortening a refused window or treating it as an empty success can lose data. This Doris change does not implement a Connector's retry loop. - -Bounded reads skip the transaction-ID watermark and table-wide conflict polling. The allocator lock protects the clock and the fixed set of matching registrations; waiting releases this lock and terminal notifications wake the readers. A successful table-specific wait does not advance the global durable prefix or require its next journal write. Queries with different ends on multiple incremental relations use their maximum end for the matching table set, which can conservatively wait longer. Cloud partition visible versions are still refreshed from MetaService. Classic reads, eventual consistency, unbounded reads, and queries mixing bounded and unbounded incremental relations retain the existing behavior. - -## Allocation and persistence - -The allocator registers transaction identity, the involved table IDs, and its first commit TSO under the same lock that advances its clock. Retries retain the earliest registration and the union of involved table IDs until a real terminal result is known. Bitmap preparation, callbacks, and commit metadata validation precede allocation; one request reuses the same TSO for RPC retries. A lazy commit response marked incomplete cannot release a registration even if its returned transaction status says VISIBLE. A separate worker reconciles at most 64 old registrations per cycle; missing/error responses do not release them. - -After recovery, the next candidate prefix is the current allocated TSO when no registrations remain, otherwise the smaller of that TSO and the oldest pending TSO minus one. The candidate is published only after its journal write succeeds. The reservation window and committed prefix share one journal record and one immutable persisted snapshot. `tso_service_window_duration_ms` defaults to 1000 ms. A monotonic timer also persists the prefix when the reservation window does not move. This is approximately one combined journal write per second, compared with the previous five-second window renewal; it does not reduce total journal frequency relative to the old implementation. - -## Recovery and upgrades - -A new master calibrates beyond the previous reserved window, registers new allocations, and waits `tso_service_window_duration_ms + 1000` milliseconds before taking a fixed exclusive transaction-ID bound from MetaService. An instance-wide strict check must then find no running transaction below that bound. The check covers every database/table and never skips expired running transactions. An expired lazy transaction can still await real publication. Normal pending registrations continue to constrain the prefix after the recovery check succeeds. Before recovery completes, reads above the durable prefix return 5100 with reason `TSO_RECOVERING`; an empty new-master registration set does not prove old transactions are visible. Reinitialization invalidates any existing read wait. - -Upgrade MetaService before using the new FE's committed prefix: recovery requires an explicit acknowledgement of the strict-check option. Old journal/image records remain readable and imply an unknown prefix. The new window-error fields in the FE RPC are optional: new followers treat an absent business code as 5100; older followers classify a new master's 5101 as the original retryable 5100 until upgraded. Old BE requests without column names retain the original four-column system-table response; new BE requests explicitly name all six columns. A new BE cannot obtain the new columns from an old FE. - -The fixed recovery wait is a temporary operational assumption, not a fencing protocol. It cannot prevent an old master that continues allocating after the wait from assigning an old TSO outside the captured transaction bound. Clock skew, an old longer reservation window, or long process pauses can violate that assumption. This change deliberately retains that accepted limitation. Recovery can also wait for long-running old transactions, and the oldest pending transaction can delay the global prefix across unrelated tables. After recovery, that delay no longer blocks reads of unrelated tables whose requested end is at or before the current TSO physical time. - -## Diagnosis - -FE metrics expose the committed prefix, reserved window, pending count, oldest pending TSO/transaction/age, recovery readiness and transaction bound. TSO persistence and reconciliation have counters and latency histograms. Reconciliation failures preserve the watermark and produce a rate-limited warning. Unknown transaction status must be investigated; registrations are not discarded by TTL. From efab9dd943fcf3952713349fea7e1c37ec772ef6 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Fri, 11 Sep 2026 01:09:49 +0800 Subject: [PATCH 05/13] [test](fe) Update TSO status schema expectations ### What problem does this PR solve? Issue Number: None Related PR: #67820 Problem Summary: The committed TSO change adds two columns to information_schema.tso_status, but SchemaTableTest still expects four columns and fails FE unit CI. Expect all six columns and verify the names and positions of the two new columns while retaining the original column checks. ### Release note None ### Check List (For Author) - Test: Unit Test; reproduced the original SchemaTableTest failure, then passed all 7 SchemaTableTest and TsoStatusMetadataGeneratorTest cases with run-fe-ut.sh. FE Checkstyle and git diff --check passed. - Behavior changed: No - Does this need documentation: No --- .../test/java/org/apache/doris/catalog/SchemaTableTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/SchemaTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/SchemaTableTest.java index b1f63ab862cbcb..b0c0a4c683aac5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/SchemaTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/SchemaTableTest.java @@ -120,10 +120,12 @@ public void testShouldFetchAllFe() throws AnalysisException, IOException { SchemaTable tsoStatus = (SchemaTable) SchemaTable.TABLE_MAP.get("tso_status"); Assertions.assertFalse(tsoStatus.shouldFetchAllFe()); Assertions.assertFalse(tsoStatus.shouldAddAgg()); - Assertions.assertEquals(4, tsoStatus.getFullSchema().size()); + Assertions.assertEquals(6, tsoStatus.getFullSchema().size()); Assertions.assertEquals("WINDOW_END_PHYSICAL_TIME", tsoStatus.getFullSchema().get(0).getName()); Assertions.assertEquals("CURRENT_TSO", tsoStatus.getFullSchema().get(1).getName()); Assertions.assertEquals("CURRENT_TSO_PHYSICAL_TIME", tsoStatus.getFullSchema().get(2).getName()); Assertions.assertEquals("CURRENT_TSO_LOGICAL_COUNTER", tsoStatus.getFullSchema().get(3).getName()); + Assertions.assertEquals("COMMITTED_TSO", tsoStatus.getFullSchema().get(4).getName()); + Assertions.assertEquals("COMMITTED_TSO_PHYSICAL_TIME", tsoStatus.getFullSchema().get(5).getName()); } } From 8eaf6b388dc3f347e4db7be6fa7952635f9c7e16 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Fri, 11 Sep 2026 11:43:37 +0800 Subject: [PATCH 06/13] [improvement](cloud) Restore table-scoped TSO waits after FE failover ### What problem does this PR solve? Issue Number: None Related PR: #67820 Problem Summary: After FE failover, one unfinished old transaction blocks every bounded incremental read above the durable committed TSO, including reads of unrelated tables. Reuse check_txn_conflict to fetch old running transactions in bounded batches after capturing the fixed recovery transaction-ID bound. Once the complete list is loaded, wait only for related tables and known TSO boundaries. Retain unknown-TSO transactions for their tables and keep the global committed TSO frozen until all recovered transactions are VISIBLE or ABORTED. Preserve concurrent registrations and resume failed scans from the last successful batch without opening an incomplete recovery. ### Release note After cloud FE failover, unrelated tables can read bounded incremental windows once the old transaction list is loaded, while relevant transactions still wait with the existing visibility timeout. Upgrade MetaService before FE to support fetching recovery transactions in batches. ### Check List (For Author) - Test: 86 focused FE unit tests; 6 ASAN MetaService/recovery/lazy-commit tests; test_committed_tso regression; three-FE failover and original-window retry; FE/MS product builds, FE Checkstyle, clang-format 16 and Cloud clang-tidy. - Behavior changed: Yes, enable table-scoped waits while old recovery transactions remain pending. - Does this need documentation: No --- cloud/src/meta-service/meta_service_txn.cpp | 65 ++++++- cloud/test/meta_service_test.cpp | 135 +++++++++++++ cloud/test/txn_lazy_commit_test.cpp | 11 ++ .../CloudGlobalTransactionMgr.java | 13 +- .../GlobalTransactionMgrIface.java | 8 +- .../doris/tso/TSOTransactionTracker.java | 105 ++++++++-- .../CloudGlobalTransactionMgrTest.java | 13 +- .../org/apache/doris/tso/TSOServiceTest.java | 13 +- .../doris/tso/TSOTransactionTrackerTest.java | 180 +++++++++++++++++- gensrc/proto/cloud.proto | 6 + 10 files changed, 508 insertions(+), 41 deletions(-) diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 06328ac6c76407..3c44dcf7ab19ea 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -4730,8 +4730,11 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont ::google::protobuf::Closure* done) { RPC_PREPROCESS(check_txn_conflict, get); const bool strict_recovery = request->strict_recovery_check(); + const bool recovery_batch = request->has_recovery_batch_size(); if (!request->has_end_txn_id() || (strict_recovery && request->end_txn_id() <= 0) || - (!strict_recovery && (!request->has_db_id() || request->table_ids_size() <= 0))) { + (!strict_recovery && (!request->has_db_id() || request->table_ids_size() <= 0)) || + (recovery_batch && (!strict_recovery || request->recovery_batch_size() <= 0 || + request->recovery_batch_size() > 1000))) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "invalid db id, end txn id or table_ids."; return; @@ -4759,6 +4762,18 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont end_running_key.push_back('\x00'); response->set_strict_recovery_check_applied(true); } + if (recovery_batch) { + if (!request->recovery_start_key().empty()) { + if (request->recovery_start_key() < begin_running_key || + request->recovery_start_key() >= end_running_key) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "invalid TSO recovery start key"; + return; + } + begin_running_key = request->recovery_start_key(); + } + response->set_recovery_batch_applied(true); + } LOG(INFO) << "begin_running_key:" << hex(begin_running_key) << " end_running_key:" << hex(end_running_key); @@ -4777,7 +4792,8 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont int total_iteration_cnt = 0; bool finished = true; while (it == nullptr /* may be not init */ || it->more()) { - err = txn->get(begin_running_key, end_running_key, &it, true); + err = txn->get(begin_running_key, end_running_key, &it, true, + recovery_batch ? request->recovery_batch_size() : 10000); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); ss << "failed to get txn running info. err=" << err; @@ -4800,6 +4816,7 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont encoded_key.remove_prefix(1); std::vector, int, int>> fields; if (decode_key(&encoded_key, &fields) != 0 || fields.size() != 5 || + !std::holds_alternative(std::get<0>(fields[3])) || !std::holds_alternative(std::get<0>(fields[4]))) { code = MetaServiceCode::UNDEFINED_ERR; msg = "failed to decode running transaction key during TSO recovery"; @@ -4808,8 +4825,44 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont if (std::get(std::get<0>(fields[4])) < request->end_txn_id()) { // A running key is removed atomically with real VISIBLE/ABORTED. In // particular an expired COMMITTED lazy transaction must still block. - response->set_finished(false); - return; + if (!recovery_batch) { + response->set_finished(false); + return; + } + const auto running_db_id = std::get(std::get<0>(fields[3])); + const auto running_txn_id = std::get(std::get<0>(fields[4])); + std::string info_val; + err = txn->get(txn_info_key({instance_id, running_db_id, running_txn_id}), + &info_val, true); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = fmt::format( + "failed to read TSO recovery transaction, db_id={}, txn_id={}", + running_db_id, running_txn_id); + return; + } + TxnInfoPB info; + if (!info.ParseFromString(info_val)) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + msg = "failed to parse TSO recovery transaction"; + return; + } + // Subtransactions update txn_info.table_ids; the running record can be older. + if (info.db_id() != running_db_id || info.txn_id() != running_txn_id || + info.table_ids().empty()) { + code = MetaServiceCode::UNDEFINED_ERR; + msg = "invalid TSO recovery transaction identity or tables"; + return; + } + // Recovery needs identities and visibility boundaries, not commit attachments. + auto* recovered = response->add_conflict_txns(); + recovered->set_db_id(running_db_id); + recovered->set_txn_id(running_txn_id); + recovered->mutable_table_ids()->CopyFrom(info.table_ids()); + recovered->set_status(info.status()); + if (info.has_commit_tso()) { + recovered->set_commit_tso(info.commit_tso()); + } } if (!it->has_next()) { begin_running_key = k; @@ -4875,6 +4928,10 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont } } begin_running_key.push_back('\x00'); // Update to next smallest key for iteration + if (recovery_batch) { + response->set_next_recovery_key(it->more() ? begin_running_key : ""); + return; + } } LOG(INFO) << "skip timeout txn count: " << skip_timeout_txn_cnt << " conflict txn count: " << response->conflict_txns_size() diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index 85ad80f14ee554..d2b55ee65d1dc4 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -2856,6 +2856,141 @@ TEST(MetaServiceTest, StrictTsoRecoveryRejectsMalformedRunningKeys) { ASSERT_FALSE(response.finished()); } +TEST(MetaServiceTest, TsoRecoveryBatchesUseCurrentTablesAndKeepExpiredTransactions) { + auto meta_service = get_meta_service(); + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + TxnRunningPB running; + running.set_timeout_time(1); + running.add_table_ids(777); + auto put_transaction = [&](const std::string& instance, int64_t db, int64_t id, int64_t table) { + txn->put(txn_running_key({instance, db, id}), running.SerializeAsString()); + TxnInfoPB info; + info.set_db_id(db); + info.set_txn_id(id); + info.add_table_ids(table); + info.set_status(TxnStatusPB::TXN_STATUS_PREPARED); + if (id == 99) { + info.set_status(TxnStatusPB::TXN_STATUS_COMMITTED); + info.set_commit_tso(12345); + } + txn->put(txn_info_key({instance, db, id}), info.SerializeAsString()); + }; + put_transaction(mock_instance, 1, 99, 100); + put_transaction(mock_instance, 1, 100, + 200); // Equal to the exclusive bound, still consumes a scan slot. + put_transaction(mock_instance, 2, 10, 300); // txn_info includes tables added after begin_txn. + put_transaction("another_instance", 1, 1, 400); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + brpc::Controller cntl; + CheckTxnConflictRequest request; + request.set_cloud_unique_id("test_cloud_unique_id"); + request.set_end_txn_id(100); + request.set_strict_recovery_check(true); + request.set_recovery_batch_size(2); + CheckTxnConflictResponse response; + meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + ASSERT_EQ(response.status().code(), MetaServiceCode::OK); + ASSERT_TRUE(response.strict_recovery_check_applied()); + ASSERT_TRUE(response.recovery_batch_applied()); + ASSERT_FALSE(response.next_recovery_key().empty()); + ASSERT_EQ(response.conflict_txns_size(), 1); + EXPECT_EQ(response.conflict_txns(0).txn_id(), 99); + EXPECT_EQ(response.conflict_txns(0).commit_tso(), 12345); + EXPECT_EQ(response.conflict_txns(0).table_ids(0), 100); + + // Deleting already scanned keys does not invalidate the next batch position. + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->remove(txn_running_key({mock_instance, 1, 99})); + txn->remove(txn_running_key({mock_instance, 1, 100})); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + request.set_recovery_start_key(response.next_recovery_key()); + response.Clear(); + meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + ASSERT_EQ(response.status().code(), MetaServiceCode::OK); + ASSERT_TRUE(response.has_next_recovery_key()); + ASSERT_TRUE(response.next_recovery_key().empty()); + ASSERT_EQ(response.conflict_txns_size(), 1); + EXPECT_EQ(response.conflict_txns(0).db_id(), 2); + EXPECT_EQ(response.conflict_txns(0).txn_id(), 10); + EXPECT_EQ(response.conflict_txns(0).table_ids(0), 300); + EXPECT_FALSE(response.conflict_txns(0).has_commit_tso()); +} + +TEST(MetaServiceTest, TsoRecoveryBatchCanBeEmptyBeforeTheScanCompletes) { + auto meta_service = get_meta_service(); + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + TxnRunningPB running; + // All IDs in the first database exceed the bound. A later database still contains an old txn. + txn->put(txn_running_key({mock_instance, 1, 200}), running.SerializeAsString()); + txn->put(txn_running_key({mock_instance, 2, 10}), running.SerializeAsString()); + TxnInfoPB info; + info.set_db_id(2); + info.set_txn_id(10); + info.add_table_ids(300); + info.set_status(TxnStatusPB::TXN_STATUS_PREPARED); + txn->put(txn_info_key({mock_instance, 2, 10}), info.SerializeAsString()); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + brpc::Controller cntl; + CheckTxnConflictRequest request; + request.set_cloud_unique_id("test_cloud_unique_id"); + request.set_end_txn_id(100); + request.set_strict_recovery_check(true); + request.set_recovery_batch_size(1); + CheckTxnConflictResponse response; + meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + ASSERT_EQ(response.status().code(), MetaServiceCode::OK); + ASSERT_EQ(response.conflict_txns_size(), 0); + ASSERT_FALSE(response.next_recovery_key().empty()); + request.set_recovery_start_key(response.next_recovery_key()); + response.Clear(); + meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + ASSERT_EQ(response.status().code(), MetaServiceCode::OK); + ASSERT_EQ(response.conflict_txns_size(), 1); + EXPECT_EQ(response.conflict_txns(0).txn_id(), 10); + // Hitting the KV batch limit can require one final empty batch to establish completion. + ASSERT_FALSE(response.next_recovery_key().empty()); + request.set_recovery_start_key(response.next_recovery_key()); + response.Clear(); + meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + ASSERT_EQ(response.status().code(), MetaServiceCode::OK); + EXPECT_EQ(response.conflict_txns_size(), 0); + EXPECT_TRUE(response.has_next_recovery_key()); + EXPECT_TRUE(response.next_recovery_key().empty()); +} + +TEST(MetaServiceTest, TsoRecoveryBatchRejectsInvalidArgumentsAndMissingDetails) { + auto meta_service = get_meta_service(); + brpc::Controller cntl; + CheckTxnConflictRequest request; + request.set_cloud_unique_id("test_cloud_unique_id"); + request.set_end_txn_id(100); + request.set_strict_recovery_check(true); + CheckTxnConflictResponse response; + for (int size : {0, 1001}) { + request.set_recovery_batch_size(size); + response.Clear(); + meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + EXPECT_EQ(response.status().code(), MetaServiceCode::INVALID_ARGUMENT); + } + request.set_recovery_batch_size(1); + request.set_recovery_start_key(txn_running_key({"another_instance", 1, 1})); + response.Clear(); + meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + EXPECT_EQ(response.status().code(), MetaServiceCode::INVALID_ARGUMENT); + request.clear_recovery_start_key(); + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(txn_running_key({mock_instance, 1, 10}), TxnRunningPB().SerializeAsString()); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + response.Clear(); + meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + EXPECT_NE(response.status().code(), MetaServiceCode::OK); + EXPECT_FALSE(response.has_next_recovery_key()); +} + TEST(MetaServiceTest, CreateMetaSyncPointTest) { auto meta_service = get_meta_service(); const std::string cloud_unique_id = "test_cloud_unique_id"; diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp index 365a80ee8c88af..af14c414430def 100644 --- a/cloud/test/txn_lazy_commit_test.cpp +++ b/cloud/test/txn_lazy_commit_test.cpp @@ -1270,6 +1270,7 @@ TEST(TxnLazyCommitTest, CommitTxnEventuallyWithFailedLazyCommitTaskTest) { commit_req.set_db_id(db_id); commit_req.set_txn_id(txn_id); commit_req.set_is_2pc(false); + commit_req.set_commit_tso(12345); commit_req.set_enable_txn_lazy_commit(true); CommitTxnResponse commit_res; meta_service->commit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), @@ -1290,6 +1291,16 @@ TEST(TxnLazyCommitTest, CommitTxnEventuallyWithFailedLazyCommitTaskTest) { ASSERT_EQ(recovery_res.status().code(), MetaServiceCode::OK); ASSERT_TRUE(recovery_res.strict_recovery_check_applied()); ASSERT_FALSE(recovery_res.finished()); + recovery_req.set_recovery_batch_size(256); + recovery_res.Clear(); + meta_service->check_txn_conflict(&cntl, &recovery_req, &recovery_res, nullptr); + ASSERT_EQ(recovery_res.status().code(), MetaServiceCode::OK); + ASSERT_TRUE(recovery_res.recovery_batch_applied()); + ASSERT_EQ(recovery_res.conflict_txns_size(), 1); + EXPECT_EQ(recovery_res.conflict_txns(0).txn_id(), txn_id); + EXPECT_EQ(recovery_res.conflict_txns(0).status(), TxnStatusPB::TXN_STATUS_COMMITTED); + EXPECT_EQ(recovery_res.conflict_txns(0).commit_tso(), 12345); + EXPECT_EQ(recovery_res.conflict_txns(0).table_ids(0), table_id); std::unique_ptr txn; ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index 0e35325eaaccc8..b396488160434d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -138,6 +138,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.protobuf.ByteString; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; import org.apache.logging.log4j.LogManager; @@ -2216,12 +2217,13 @@ public List getUnFinishedPreviousLoad(long endTransactionId, l } @Override - public boolean isPreviousTransactionsFinishedForTsoRecovery(long endTransactionId) throws UserException { + public CheckTxnConflictResponse getTsoRecoveryTransactions(long endTransactionId, + ByteString startKey) throws UserException { CheckTxnConflictRequest request = CheckTxnConflictRequest.newBuilder() .setCloudUniqueId(Config.cloud_unique_id) .setRequestIp(FrontendOptions.getLocalHostAddressCached()) .setEndTxnId(endTransactionId) - .setStrictRecoveryCheck(true).build(); + .setStrictRecoveryCheck(true).setRecoveryBatchSize(256).setRecoveryStartKey(startKey).build(); CheckTxnConflictResponse response; try { response = MetaServiceProxy.getInstance().checkTxnConflict(request); @@ -2231,10 +2233,11 @@ public boolean isPreviousTransactionsFinishedForTsoRecovery(long endTransactionI if (response.getStatus().getCode() != MetaServiceCode.OK) { throw new UserException(response.getStatus().getMsg()); } - if (!response.getStrictRecoveryCheckApplied() || !response.hasFinished()) { - throw new UserException("MetaService does not support strict TSO recovery; upgrade MetaService first"); + if (!response.getStrictRecoveryCheckApplied() || !response.getRecoveryBatchApplied() + || !response.hasNextRecoveryKey()) { + throw new UserException("MetaService does not support TSO recovery batches; upgrade MetaService first"); } - return response.getFinished(); + return response; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java index ef7d42cfa3638a..f7fd624a881a81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.stream.TableStreamUpdateInfo; +import org.apache.doris.cloud.proto.Cloud.CheckTxnConflictResponse; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DuplicatedRequestException; @@ -38,6 +39,8 @@ import org.apache.doris.transaction.TransactionState.LoadJobSourceType; import org.apache.doris.transaction.TransactionState.TxnCoordinator; +import com.google.protobuf.ByteString; + import java.io.DataInput; import java.io.IOException; import java.util.List; @@ -145,8 +148,9 @@ public void abortTransaction(Long dbId, Long txnId, String reason, public void finishTransaction(long dbId, long transactionId, Map partitionVisibleVersions, Map> backendPartitions) throws UserException; - /** Instance-wide, timeout-independent check of the exclusive recovery transaction bound. */ - default boolean isPreviousTransactionsFinishedForTsoRecovery(long endTransactionId) throws UserException { + /** Fetch a batch of running transactions below the exclusive recovery transaction bound. */ + default CheckTxnConflictResponse getTsoRecoveryTransactions(long endTransactionId, ByteString startKey) + throws UserException { throw new UserException("Strict TSO recovery is only supported in cloud mode"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java index 738515ffd62263..c4e13361de4ed0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java @@ -17,6 +17,9 @@ package org.apache.doris.tso; +import org.apache.doris.cloud.proto.Cloud.CheckTxnConflictResponse; +import org.apache.doris.cloud.proto.Cloud.TxnInfoPB; +import org.apache.doris.cloud.proto.Cloud.TxnStatusPB; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.transaction.GlobalTransactionMgrIface; @@ -24,10 +27,12 @@ import org.apache.doris.transaction.TransactionStatus; import com.google.common.base.Preconditions; +import com.google.protobuf.ByteString; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -48,10 +53,15 @@ final class TSOTransactionTracker { private final Condition transactionsChanged; private final Map, PendingTransaction> pendingByTxn = new HashMap<>(); private final TreeMap pendingByTso = new TreeMap<>(); + // Recovered transactions may have no persisted TSO yet. Keep them out of the TSO index. + private final TreeMap recoveryByTxn = new TreeMap<>(); private long generation; private long recoveryDeadlineNanos; private long recoveryWatermark; private boolean recoveryReady; + private boolean recoveryLoaded; + private ByteString recoveryStartKey = ByteString.EMPTY; + private long recoveryPollCursor; private long pollCursor; enum WaitResult { @@ -64,7 +74,7 @@ private static final class PendingTransaction { private final long registeredAtNanos; private final Set tableIds; - private PendingTransaction(Pair identity, long tso, long nowNanos, Set tableIds) { + private PendingTransaction(Pair identity, long tso, long nowNanos, Collection tableIds) { this.identity = identity; this.tso = tso; this.registeredAtNanos = nowNanos; @@ -82,9 +92,13 @@ void reset(long nowNanos, long recoveryDelayMs) { generation++; pendingByTxn.clear(); pendingByTso.clear(); + recoveryByTxn.clear(); recoveryDeadlineNanos = nowNanos + TimeUnit.MILLISECONDS.toNanos(recoveryDelayMs); recoveryWatermark = 0; recoveryReady = false; + recoveryLoaded = false; + recoveryStartKey = ByteString.EMPTY; + recoveryPollCursor = 0; pollCursor = 0; transactionsChanged.signalAll(); } @@ -92,6 +106,10 @@ void reset(long nowNanos, long recoveryDelayMs) { void register(Pair identity, long tso, long nowNanos, Set tableIds) { Preconditions.checkState(lock.isHeldByCurrentThread()); Preconditions.checkArgument(!tableIds.isEmpty(), "commit registration requires table IDs"); + PendingTransaction recovered = recoveryByTxn.get(identity.second); + if (recovered != null) { + recovered.tableIds.addAll(tableIds); + } PendingTransaction existing = pendingByTxn.get(identity); if (existing != null) { // A timed-out request can still commit using the earlier TSO. @@ -107,12 +125,19 @@ void register(Pair identity, long tso, long nowNanos, Set tabl WaitResult awaitTransactions(Map> dbToTableIds, long endTso, long remainingNanos) throws InterruptedException { Preconditions.checkState(lock.isHeldByCurrentThread()); - if (!recoveryReady) { + if (!recoveryLoaded) { return WaitResult.RECOVERING; } long waitStartNanos = System.nanoTime(); long waitGeneration = generation; List remaining = new ArrayList<>(); + for (PendingTransaction pending : recoveryByTxn.values()) { + List tables = dbToTableIds.get(pending.identity.first); + if ((pending.tso <= 0 || pending.tso <= endTso) + && tables != null && !Collections.disjoint(tables, pending.tableIds)) { + remaining.add(pending); + } + } for (PendingTransaction pending : pendingByTso.headMap(endTso, true).values()) { List tables = dbToTableIds.get(pending.identity.first); if (tables != null && !Collections.disjoint(tables, pending.tableIds)) { @@ -125,7 +150,8 @@ WaitResult awaitTransactions(Map> dbToTableIds, long endTso, lo if (generation != waitGeneration) { return WaitResult.RECOVERING; } - remaining.removeIf(pending -> pendingByTxn.get(pending.identity) != pending); + remaining.removeIf(pending -> pendingByTxn.get(pending.identity) != pending + && recoveryByTxn.get(pending.identity.second) != pending); if (remaining.isEmpty()) { return WaitResult.FINISHED; } @@ -154,8 +180,10 @@ void transactionFinished(long dbId, long txnId) { PendingTransaction pending = pendingByTxn.remove(Pair.of(dbId, txnId)); if (pending != null) { pendingByTso.remove(pending.tso); - transactionsChanged.signalAll(); } + recoveryByTxn.remove(txnId); + updateRecoveryReady(); + transactionsChanged.signalAll(); } finally { lock.unlock(); } @@ -166,12 +194,14 @@ void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws U long checkGeneration; long watermark; boolean checkRecovery; + ByteString startKey; List batch = new ArrayList<>(); lock.lock(); try { checkGeneration = generation; watermark = recoveryWatermark; - checkRecovery = !recoveryReady && nowNanos - recoveryDeadlineNanos >= 0; + checkRecovery = !recoveryLoaded && nowNanos - recoveryDeadlineNanos >= 0; + startKey = recoveryStartKey; if (!pendingByTso.isEmpty()) { // Always check the transaction blocking the prefix, then rotate through the rest. PendingTransaction oldest = pendingByTso.firstEntry().getValue(); @@ -190,6 +220,17 @@ void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws U } } } + for (int i = 0; i < Math.min(CHECK_BATCH_SIZE, recoveryByTxn.size()); i++) { + Map.Entry next = recoveryByTxn.higherEntry(recoveryPollCursor); + if (next == null) { + next = recoveryByTxn.firstEntry(); + } + recoveryPollCursor = next.getKey(); + // A local registration already supplies the reconciliation RPC for this transaction. + if (!pendingByTxn.containsKey(next.getValue().identity)) { + batch.add(next.getValue()); + } + } } finally { lock.unlock(); } @@ -204,10 +245,9 @@ void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws U } lock.lock(); try { - if (generation == checkGeneration && pendingByTxn.get(pending.identity) == pending) { - pendingByTxn.remove(pending.identity); - pendingByTso.remove(pending.tso); - transactionsChanged.signalAll(); + if (generation == checkGeneration && (pendingByTxn.get(pending.identity) == pending + || recoveryByTxn.get(pending.identity.second) == pending)) { + transactionFinished(pending.identity.first, pending.identity.second); } } finally { lock.unlock(); @@ -228,19 +268,50 @@ void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws U lock.unlock(); } } - boolean finished = txnMgr.isPreviousTransactionsFinishedForTsoRecovery(watermark); - lock.lock(); - try { - if (generation == checkGeneration && finished) { - recoveryReady = true; - LOG.info("TSO recovery completed, transaction watermark={}", watermark); + while (true) { + CheckTxnConflictResponse response = txnMgr.getTsoRecoveryTransactions(watermark, startKey); + lock.lock(); + try { + if (generation != checkGeneration) { + return; + } + for (TxnInfoPB info : response.getConflictTxnsList()) { + Preconditions.checkState(info.getStatus() != TxnStatusPB.TXN_STATUS_VISIBLE + && info.getStatus() != TxnStatusPB.TXN_STATUS_ABORTED, + "TSO recovery batch contains a terminal transaction: %s", info.getTxnId()); + Pair identity = Pair.of(info.getDbId(), info.getTxnId()); + PendingTransaction recovered = new PendingTransaction(identity, + info.hasCommitTso() ? info.getCommitTso() : 0, + nowNanos, info.getTableIdsList()); + PendingTransaction local = pendingByTxn.get(identity); + if (local != null) { + recovered.tableIds.addAll(local.tableIds); + } + recoveryByTxn.put(info.getTxnId(), recovered); + } + startKey = response.getNextRecoveryKey(); + recoveryStartKey = startKey; + if (startKey.isEmpty()) { + recoveryLoaded = true; + updateRecoveryReady(); + LOG.info("Loaded TSO recovery transactions, watermark={}, pending={}", + watermark, recoveryByTxn.size()); + return; + } + } finally { + lock.unlock(); } - } finally { - lock.unlock(); } } } + private void updateRecoveryReady() { + if (!recoveryReady && recoveryLoaded && recoveryByTxn.isEmpty()) { + recoveryReady = true; + LOG.info("TSO recovery completed, transaction watermark={}", recoveryWatermark); + } + } + long getPendingCount() { lock.lock(); try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java index 64a9f37d267e8f..9c5d96da283b45 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java @@ -46,6 +46,7 @@ import org.apache.doris.transaction.TxnStateChangeCallback; import com.google.common.collect.Lists; +import com.google.protobuf.ByteString; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -102,13 +103,19 @@ public void testStrictRecoveryRequiresExplicitMsCapability() throws Exception { .setFinished(true); Mockito.when(proxy.checkTxnConflict(Mockito.any())).thenReturn(response.build()); Assertions.assertThrows(UserException.class, - () -> masterTransMgr.isPreviousTransactionsFinishedForTsoRecovery(1000)); + () -> masterTransMgr.getTsoRecoveryTransactions(1000, ByteString.EMPTY)); Mockito.when(proxy.checkTxnConflict(Mockito.any())).thenReturn( response.setStrictRecoveryCheckApplied(true).build()); - Assertions.assertTrue(masterTransMgr.isPreviousTransactionsFinishedForTsoRecovery(1000)); + Assertions.assertThrows(UserException.class, + () -> masterTransMgr.getTsoRecoveryTransactions(1000, ByteString.EMPTY)); + Mockito.when(proxy.checkTxnConflict(Mockito.any())).thenReturn( + response.setRecoveryBatchApplied(true).setNextRecoveryKey(ByteString.EMPTY).build()); + Assertions.assertTrue(masterTransMgr.getTsoRecoveryTransactions(1000, ByteString.EMPTY) + .getNextRecoveryKey().isEmpty()); ArgumentCaptor capture = ArgumentCaptor.forClass(Cloud.CheckTxnConflictRequest.class); - Mockito.verify(proxy, Mockito.times(2)).checkTxnConflict(capture.capture()); + Mockito.verify(proxy, Mockito.times(3)).checkTxnConflict(capture.capture()); + Assertions.assertEquals(256, capture.getValue().getRecoveryBatchSize()); Assertions.assertTrue(capture.getValue().getStrictRecoveryCheck()); Assertions.assertEquals(1000, capture.getValue().getEndTxnId()); Assertions.assertFalse(capture.getValue().hasDbId()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java index d2bd438364210a..62aa89c5589154 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java @@ -33,6 +33,7 @@ import org.apache.doris.qe.TimeBasedChangeVisibleWaiter; import org.apache.doris.transaction.GlobalTransactionMgrIface; +import com.google.protobuf.ByteString; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -467,7 +468,8 @@ public void testReadableWindowDoesNotRequireAnotherCommittedTsoFlush() throws Ex TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)).thenReturn(true); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + .thenReturn(TSOTransactionTrackerTest.recoveryBatch(ByteString.EMPTY)); tracker.checkTransactions(txnMgr, Long.MAX_VALUE); try (MockedStatic config = Mockito.mockStatic(Config.class)) { config.when(Config::isCloudMode).thenReturn(true); @@ -489,7 +491,8 @@ private void prepareWindowRead(boolean finishRecovery) throws Exception { field.setAccessible(true); GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)).thenReturn(true); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + .thenReturn(TSOTransactionTrackerTest.recoveryBatch(ByteString.EMPTY)); ((TSOTransactionTracker) field.get(tsoService)).checkTransactions(txnMgr, Long.MAX_VALUE); } } @@ -564,7 +567,8 @@ public void testCommittedPrefixIsPublishedOnlyAfterJournalSuccess() throws Excep TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)).thenReturn(true); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + .thenReturn(TSOTransactionTrackerTest.recoveryBatch(ByteString.EMPTY)); tracker.checkTransactions(txnMgr, Long.MAX_VALUE); try (MockedStatic config = Mockito.mockStatic(Config.class)) { config.when(Config::isCloudMode).thenReturn(true); @@ -601,7 +605,8 @@ public void testRecoveryFreezesPrefixAndPeriodicFlushWorksWithUnchangedWindow() TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)).thenReturn(true); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + .thenReturn(TSOTransactionTrackerTest.recoveryBatch(ByteString.EMPTY)); long afterRecoveryDelay = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(Config.tso_service_window_duration_ms + 1001L); tracker.checkTransactions(txnMgr, afterRecoveryDelay); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java index cc3580425eb65e..22a3a3fbbc475b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java @@ -17,16 +17,21 @@ package org.apache.doris.tso; +import org.apache.doris.cloud.proto.Cloud.CheckTxnConflictResponse; +import org.apache.doris.cloud.proto.Cloud.TxnInfoPB; +import org.apache.doris.cloud.proto.Cloud.TxnStatusPB; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.transaction.GlobalTransactionMgrIface; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; +import com.google.protobuf.ByteString; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.util.Arrays; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -69,9 +74,28 @@ private long candidate(long currentTso, long durableTso) { } } + static CheckTxnConflictResponse recoveryBatch(ByteString nextKey, TxnInfoPB... transactions) { + return CheckTxnConflictResponse.newBuilder().setStrictRecoveryCheckApplied(true) + .setRecoveryBatchApplied(true).setNextRecoveryKey(nextKey) + .addAllConflictTxns(Arrays.asList(transactions)).build(); + } + + private static TxnInfoPB recoveryTxn(long dbId, long txnId, long tso, long... tables) { + TxnInfoPB.Builder info = TxnInfoPB.newBuilder().setDbId(dbId).setTxnId(txnId) + .setStatus(TxnStatusPB.TXN_STATUS_PREPARED); + for (long table : tables) { + info.addTableIds(table); + } + if (tso > 0) { + info.setCommitTso(tso).setStatus(TxnStatusPB.TXN_STATUS_COMMITTED); + } + return info.build(); + } + private void finishRecovery() throws Exception { Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.doReturn(true).when(txnMgr).isPreviousTransactionsFinishedForTsoRecovery(1000L); + Mockito.doReturn(recoveryBatch(ByteString.EMPTY)).when(txnMgr) + .getTsoRecoveryTransactions(1000L, ByteString.EMPTY); tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); } @@ -115,6 +139,146 @@ public void testEmptyRegistrationSetCannotBypassRecovery() throws Exception { Assertions.assertEquals(TSOTransactionTracker.WaitResult.RECOVERING, awaitTable(1, 100, 200)); } + @Test + public void testLoadedRecoveryWaitsOnlyRelatedTablesAndKeepsPrefixFrozen() throws Exception { + reset(2000); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + .thenReturn(recoveryBatch(ByteString.EMPTY, + recoveryTxn(1, 10, 100, 100), recoveryTxn(1, 20, 0, 200))); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(2)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 300, 200)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(2, 100, 200)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 100, 90)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, 200)); + // No persisted TSO is not proof that an old in-flight commit lies outside this window. + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 200, 90)); + Assertions.assertEquals(80, candidate(250, 80)); + tracker.transactionFinished(1, 10); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 100, 200)); + Assertions.assertEquals(80, candidate(250, 80)); + tracker.transactionFinished(1, 20); + Assertions.assertTrue(tracker.isRecoveryReady()); + Assertions.assertEquals(250, candidate(250, 80)); + } + + @Test + public void testFailedBatchResumesWithoutOpeningAnIncompleteRecovery() throws Exception { + reset(0); + ByteString nextKey = ByteString.copyFromUtf8("next batch"); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + .thenReturn(recoveryBatch(nextKey, recoveryTxn(1, 10, 100, 100))); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, nextKey)) + .thenThrow(new UserException("batch RPC failed")) + .thenReturn(recoveryBatch(ByteString.EMPTY, recoveryTxn(2, 20, 0, 200))); + Assertions.assertThrows(UserException.class, + () -> tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3))); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.RECOVERING, awaitTable(1, 300, 200)); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 300, 200)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, 200)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(2, 200, 200)); + Assertions.assertEquals(80, candidate(250, 80)); + Mockito.verify(txnMgr).getTransactionIdWatermark(); + Mockito.verify(txnMgr).getTsoRecoveryTransactions(1000L, ByteString.EMPTY); + } + + @Test + public void testRecoveryBatchesMergeConcurrentRegistrationsAndFinishNotifications() throws Exception { + reset(0); + ByteString nextKey = ByteString.copyFromUtf8("next batch"); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)).thenAnswer(invocation -> { + Assertions.assertFalse(lock.isHeldByCurrentThread()); + register(1, 10, 200); // Same old transaction is retried through the new master. + return recoveryBatch(nextKey, recoveryTxn(1, 10, 0, 200)); + }); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, nextKey)).thenAnswer(invocation -> { + Assertions.assertFalse(lock.isHeldByCurrentThread()); + tracker.transactionFinished(1, 10); + register(2, 2000, 150); // New transactions must survive importing the old list. + return recoveryBatch(ByteString.EMPTY, recoveryTxn(1, 20, 0, 300)); + }); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 100, 300)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 200, 300)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(2, 100, 300)); + Assertions.assertEquals(80, candidate(300, 80)); + tracker.transactionFinished(1, 20); + Assertions.assertEquals(149, candidate(300, 80)); + } + + @Test + public void testRecoveredUnknownTsoRetainsTablesAcrossLocalRetry() throws Exception { + reset(0); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + .thenReturn(recoveryBatch(ByteString.EMPTY, recoveryTxn(1, 10, 0, 200))); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); + register(1, 10, 300); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, 200)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 200, 200)); + Assertions.assertEquals(80, candidate(400, 80)); + tracker.transactionFinished(1, 10); + Assertions.assertEquals(400, candidate(400, 80)); + } + + @Test + public void testRecoveredTransactionsAreReconciledInBoundedRotatingBatches() throws Exception { + reset(0); + TxnInfoPB[] transactions = new TxnInfoPB[150]; + for (int i = 0; i < transactions.length; i++) { + transactions[i] = recoveryTxn(1, i + 1, 0, 100); + } + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + .thenReturn(recoveryBatch(ByteString.EMPTY, transactions)); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); + Mockito.verify(txnMgr, Mockito.atMost(64)).getTransactionState(Mockito.anyLong(), Mockito.anyLong()); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(5)); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(6)); + Mockito.verify(txnMgr).getTransactionState(1, 150); + Assertions.assertEquals(80, candidate(200, 80)); // Missing state must retain recovery entries. + TransactionState state = Mockito.mock(TransactionState.class); + Mockito.when(state.getTransactionStatus()).thenReturn(TransactionStatus.COMMITTED); + Mockito.when(txnMgr.getTransactionState(Mockito.anyLong(), Mockito.anyLong())).thenReturn(state); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(7)); + Assertions.assertEquals(80, candidate(200, 80)); + Mockito.when(state.getTransactionStatus()).thenReturn(TransactionStatus.ABORTED); + for (int i = 0; i < 3; i++) { + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(8 + i)); + } + Assertions.assertTrue(tracker.isRecoveryReady()); + Assertions.assertEquals(200, candidate(200, 80)); + } + + @Test + public void testRecoveredTransactionCompletionWakesOnlyItsWaiters() throws Exception { + reset(0); + Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + .thenReturn(recoveryBatch(ByteString.EMPTY, + recoveryTxn(1, 10, 0, 100), recoveryTxn(2, 20, 0, 100))); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); + CountDownLatch started = new CountDownLatch(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future waiting = submitReadWait(executor, started); + Assertions.assertTrue(started.await(30, TimeUnit.SECONDS)); + TransactionState aborted = Mockito.mock(TransactionState.class); + Mockito.when(aborted.getTransactionStatus()).thenReturn(TransactionStatus.ABORTED); + Mockito.when(txnMgr.getTransactionState(1, 10)).thenReturn(aborted); + tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, waiting.get(30, TimeUnit.SECONDS)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(2, 100, 200)); + Assertions.assertEquals(80, candidate(300, 80)); + } finally { + executor.shutdownNow(); + } + } + private Future submitReadWait(ExecutorService executor, CountDownLatch started) { return executor.submit(() -> { lock.lock(); @@ -210,13 +374,17 @@ public void testRecoveryCapturesFixedWatermarkAfterDelayAndPreservesNewPending() Mockito.verify(txnMgr, Mockito.never()).getTransactionIdWatermark(); Assertions.assertEquals(80, candidate(250, 80)); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L, 2000L); - Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)).thenReturn(false, true); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + .thenReturn(recoveryBatch(ByteString.EMPTY, recoveryTxn(1, 10, 0, 100))); tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(2)); Assertions.assertEquals(80, candidate(250, 80)); + TransactionState aborted = Mockito.mock(TransactionState.class); + Mockito.when(aborted.getTransactionStatus()).thenReturn(TransactionStatus.ABORTED); + Mockito.when(txnMgr.getTransactionState(1, 10)).thenReturn(aborted); tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); Assertions.assertEquals(199, candidate(250, 80)); Mockito.verify(txnMgr).getTransactionIdWatermark(); - Mockito.verify(txnMgr, Mockito.times(2)).isPreviousTransactionsFinishedForTsoRecovery(1000L); + Mockito.verify(txnMgr).getTsoRecoveryTransactions(1000L, ByteString.EMPTY); } @Test @@ -239,7 +407,7 @@ public void testMissingTransactionAndFailedRecoveryNeverAdvance() throws Excepti reset(0); register(1, 10, 100); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)) + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) .thenThrow(new UserException("old MS has no strict check capability")); Assertions.assertThrows(UserException.class, () -> tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3))); @@ -272,10 +440,10 @@ public void testRpcDoesNotHoldAllocatorLockAndOldResultCannotRemoveNewRegistrati public void testOldRecoveryResultCannotOpenNewRecovery() throws Exception { reset(0); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.isPreviousTransactionsFinishedForTsoRecovery(1000L)).thenAnswer(invocation -> { + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)).thenAnswer(invocation -> { Assertions.assertFalse(lock.isHeldByCurrentThread()); reset(2000); - return true; + return recoveryBatch(ByteString.EMPTY); }); tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); Assertions.assertFalse(tracker.isRecoveryReady()); diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 8133e611b0d749..9a93c76428845d 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -1250,6 +1250,9 @@ message CheckTxnConflictRequest { optional string request_ip = 6; // Check every database/table in the instance, including expired running transactions. optional bool strict_recovery_check = 7 [default = false]; + // Fetch running transaction details in bounded batches for FE TSO recovery. + optional int32 recovery_batch_size = 8; + optional bytes recovery_start_key = 9; } message CheckTxnConflictResponse { @@ -1257,6 +1260,9 @@ message CheckTxnConflictResponse { optional bool finished = 2; repeated TxnInfoPB conflict_txns = 3; optional bool strict_recovery_check_applied = 4; + // Present for batch responses; empty means the full instance has been scanned. + optional bytes next_recovery_key = 5; + optional bool recovery_batch_applied = 6; } message CleanTxnLabelRequest { From 17272039558890753e79534b68515d85b00c7b41 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Fri, 11 Sep 2026 13:03:00 +0800 Subject: [PATCH 07/13] [refactor](cloud) Add a dedicated RPC for TSO recovery ### What problem does this PR solve? Issue Number: None Related PR: #67820 Problem Summary: TSO recovery had added strict-check and batch modes to the existing transaction-conflict RPC, mixing recovery scans with table-scoped conflict checks. Add get_tso_recovery_transactions with its own request, response, client wrappers, handler and metrics. Restore check_txn_conflict and its messages to their original definitions. Preserve fixed-bound batch scanning, table-scoped waits and committed-TSO advancement. An unavailable new RPC or incomplete batch keeps recovery closed without falling back to the old conflict check. ### Release note Upgrade MetaService before FE to provide the dedicated TSO recovery RPC. Incremental-query waiting behavior and error codes remain unchanged. ### Check List (For Author) - Test: 108 FE unit tests; 9 ASAN MetaService/recovery/legacy-conflict/lazy-commit tests; test_committed_tso regression; live old/new RPC and three-FE failover verification; FE/MS product builds, FE Checkstyle, clang-format 16 and Cloud clang-tidy. - Behavior changed: Yes, the internal recovery RPC changes; SQL behavior is unchanged. - Does this need documentation: No --- cloud/src/common/bvars.cpp | 3 + cloud/src/common/bvars.h | 3 + cloud/src/meta-service/meta_service.h | 13 ++ cloud/src/meta-service/meta_service_txn.cpp | 191 +++++++++--------- cloud/test/meta_service_test.cpp | 145 +++++++------ cloud/test/txn_lazy_commit_test.cpp | 26 +-- .../doris/cloud/rpc/MetaServiceClient.java | 6 + .../doris/cloud/rpc/MetaServiceProxy.java | 6 + .../CloudGlobalTransactionMgr.java | 19 +- .../GlobalTransactionMgrIface.java | 6 +- .../doris/tso/TSOTransactionTracker.java | 8 +- .../CloudGlobalTransactionMgrTest.java | 48 +++-- .../doris/tso/TSOTransactionTrackerTest.java | 11 +- gensrc/proto/cloud.proto | 28 ++- 14 files changed, 284 insertions(+), 229 deletions(-) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 18da953b1c1e97..3d81a38283637a 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -43,6 +43,7 @@ BvarLatencyRecorderWithTag g_bvar_ms_create_meta_sync_point("ms", "create_meta_s BvarLatencyRecorderWithTag g_bvar_ms_begin_sub_txn("ms", "begin_sub_txn"); BvarLatencyRecorderWithTag g_bvar_ms_abort_sub_txn("ms", "abort_sub_txn"); BvarLatencyRecorderWithTag g_bvar_ms_check_txn_conflict("ms", "check_txn_conflict"); +BvarLatencyRecorderWithTag g_bvar_ms_get_tso_recovery_transactions("ms", "get_tso_recovery_transactions"); BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator("ms", "abort_txn_with_coordinator"); BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator("ms", "get_prepare_txn_by_coordinator"); BvarLatencyRecorderWithTag g_bvar_ms_clean_txn_label("ms", "clean_txn_label"); @@ -500,6 +501,7 @@ mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter("rpc_kv_abo mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_counter("rpc_kv_get_prepare_txn_by_coordinator_get_counter",{"instance_id"}); // check_txn_conflict mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter("rpc_kv_check_txn_conflict_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_counter("rpc_kv_get_tso_recovery_transactions_get_counter",{"instance_id"}); // clean_txn_label mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter("rpc_kv_clean_txn_label_get_counter",{"instance_id"}); mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter("rpc_kv_clean_txn_label_put_counter",{"instance_id"}); @@ -710,6 +712,7 @@ mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes("rpc_kv_abort mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_bytes("rpc_kv_get_prepare_txn_by_coordinator_get_bytes",{"instance_id"}); // check_txn_conflict mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes("rpc_kv_check_txn_conflict_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_bytes("rpc_kv_get_tso_recovery_transactions_get_bytes",{"instance_id"}); // clean_txn_label mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_bytes("rpc_kv_clean_txn_label_get_bytes",{"instance_id"}); mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_bytes("rpc_kv_clean_txn_label_put_bytes",{"instance_id"}); diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index 8c40c1316665a5..a12265ec633850 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -553,6 +553,7 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_get_txn; extern BvarLatencyRecorderWithTag g_bvar_ms_get_current_max_txn_id; extern BvarLatencyRecorderWithTag g_bvar_ms_create_meta_sync_point; extern BvarLatencyRecorderWithTag g_bvar_ms_check_txn_conflict; +extern BvarLatencyRecorderWithTag g_bvar_ms_get_tso_recovery_transactions; extern BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator; extern BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator; extern BvarLatencyRecorderWithTag g_bvar_ms_begin_sub_txn; @@ -907,6 +908,7 @@ extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_counter; extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_counter; @@ -1050,6 +1052,7 @@ extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_bytes; diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index 26b265f1d05ab1..90782d4ee81b90 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -145,6 +145,11 @@ class MetaServiceImpl : public cloud::MetaService { CheckTxnConflictResponse* response, ::google::protobuf::Closure* done) override; + void get_tso_recovery_transactions(::google::protobuf::RpcController* controller, + const GetTsoRecoveryTransactionsRequest* request, + GetTsoRecoveryTransactionsResponse* response, + ::google::protobuf::Closure* done) override; + void abort_txn_with_coordinator(::google::protobuf::RpcController* controller, const AbortTxnWithCoordinatorRequest* request, AbortTxnWithCoordinatorResponse* response, @@ -619,6 +624,14 @@ class MetaServiceProxy final : public MetaService { call_impl(&cloud::MetaService::check_txn_conflict, controller, request, response, done); } + void get_tso_recovery_transactions(::google::protobuf::RpcController* controller, + const GetTsoRecoveryTransactionsRequest* request, + GetTsoRecoveryTransactionsResponse* response, + ::google::protobuf::Closure* done) override { + call_impl(&cloud::MetaService::get_tso_recovery_transactions, controller, request, response, + done); + } + void abort_txn_with_coordinator(::google::protobuf::RpcController* controller, const AbortTxnWithCoordinatorRequest* request, AbortTxnWithCoordinatorResponse* response, diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 3c44dcf7ab19ea..1986fcfe7e181e 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -4724,17 +4724,111 @@ std::string get_txn_info_key_from_txn_running_key(std::string_view txn_running_k return conflict_txn_info_key; } +void MetaServiceImpl::get_tso_recovery_transactions( + ::google::protobuf::RpcController* controller, + const GetTsoRecoveryTransactionsRequest* request, + GetTsoRecoveryTransactionsResponse* response, ::google::protobuf::Closure* done) { + RPC_PREPROCESS(get_tso_recovery_transactions, get); + if (request->end_txn_id() <= 0 || request->batch_size() <= 0 || request->batch_size() > 1000) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "invalid TSO recovery transaction bound or batch size"; + return; + } + instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); + if (instance_id.empty()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "cannot find instance_id for TSO recovery"; + return; + } + RPC_RATE_LIMIT(get_tso_recovery_transactions) + // Keys sort by database first. Scan the instance and apply the fixed exclusive ID bound + // to each key; a batch containing only newer transactions does not finish the scan. + std::string begin_key = txn_running_key({instance_id, 0, 0}); + std::string end_key = txn_running_key({instance_id, INT64_MAX, INT64_MAX}); + end_key.push_back('\x00'); + if (!request->start_key().empty()) { + if (request->start_key() < begin_key || request->start_key() >= end_key) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "invalid TSO recovery start key"; + return; + } + begin_key = request->start_key(); + } + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = "failed to create TSO recovery read transaction"; + return; + } + std::unique_ptr it; + err = txn->get(begin_key, end_key, &it, true, request->batch_size()); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = "failed to get running transactions during TSO recovery"; + return; + } + while (it->has_next()) { + auto [key, value] = it->next(); + if (!it->has_next()) { + begin_key = key; + } + std::string_view encoded_key = key; + encoded_key.remove_prefix(1); + std::vector, int, int>> fields; + if (decode_key(&encoded_key, &fields) != 0 || fields.size() != 5 || + !std::holds_alternative(std::get<0>(fields[3])) || + !std::holds_alternative(std::get<0>(fields[4]))) { + code = MetaServiceCode::UNDEFINED_ERR; + msg = "failed to decode running transaction key during TSO recovery"; + return; + } + const auto db_id = std::get(std::get<0>(fields[3])); + const auto txn_id = std::get(std::get<0>(fields[4])); + if (txn_id >= request->end_txn_id()) { + continue; + } + // Running keys are removed atomically with real VISIBLE/ABORTED. Expired COMMITTED + // lazy transactions still block. Read the details in the same KV snapshot. + std::string info_val; + err = txn->get(txn_info_key({instance_id, db_id, txn_id}), &info_val, true); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = fmt::format("failed to read TSO recovery transaction, db_id={}, txn_id={}", db_id, + txn_id); + return; + } + TxnInfoPB info; + if (!info.ParseFromString(info_val)) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + msg = "failed to parse TSO recovery transaction"; + return; + } + // Subtransactions update txn_info.table_ids; the running record can be older. + if (info.db_id() != db_id || info.txn_id() != txn_id || info.table_ids().empty()) { + code = MetaServiceCode::UNDEFINED_ERR; + msg = "invalid TSO recovery transaction identity or tables"; + return; + } + // Recovery needs identities and visibility boundaries, not commit attachments. + auto* recovered = response->add_txn_infos(); + recovered->set_db_id(db_id); + recovered->set_txn_id(txn_id); + recovered->mutable_table_ids()->CopyFrom(info.table_ids()); + recovered->set_status(info.status()); + if (info.has_commit_tso()) { + recovered->set_commit_tso(info.commit_tso()); + } + } + begin_key.push_back('\x00'); + response->set_next_start_key(it->more() ? begin_key : ""); +} + void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* controller, const CheckTxnConflictRequest* request, CheckTxnConflictResponse* response, ::google::protobuf::Closure* done) { RPC_PREPROCESS(check_txn_conflict, get); - const bool strict_recovery = request->strict_recovery_check(); - const bool recovery_batch = request->has_recovery_batch_size(); - if (!request->has_end_txn_id() || (strict_recovery && request->end_txn_id() <= 0) || - (!strict_recovery && (!request->has_db_id() || request->table_ids_size() <= 0)) || - (recovery_batch && (!strict_recovery || request->recovery_batch_size() <= 0 || - request->recovery_batch_size() > 1000))) { + if (!request->has_db_id() || !request->has_end_txn_id() || (request->table_ids_size() <= 0)) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "invalid db id, end txn id or table_ids."; return; @@ -4754,26 +4848,6 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont std::string begin_running_key = txn_running_key({instance_id, db_id, 0}); std::string end_running_key = txn_running_key({instance_id, db_id, request->end_txn_id()}); - if (strict_recovery) { - // Database and transaction IDs are non-negative. Include the entire instance and apply - // the exclusive transaction bound after decoding each key (keys sort by database first). - begin_running_key = txn_running_key({instance_id, 0, 0}); - end_running_key = txn_running_key({instance_id, INT64_MAX, INT64_MAX}); - end_running_key.push_back('\x00'); - response->set_strict_recovery_check_applied(true); - } - if (recovery_batch) { - if (!request->recovery_start_key().empty()) { - if (request->recovery_start_key() < begin_running_key || - request->recovery_start_key() >= end_running_key) { - code = MetaServiceCode::INVALID_ARGUMENT; - msg = "invalid TSO recovery start key"; - return; - } - begin_running_key = request->recovery_start_key(); - } - response->set_recovery_batch_applied(true); - } LOG(INFO) << "begin_running_key:" << hex(begin_running_key) << " end_running_key:" << hex(end_running_key); @@ -4792,8 +4866,7 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont int total_iteration_cnt = 0; bool finished = true; while (it == nullptr /* may be not init */ || it->more()) { - err = txn->get(begin_running_key, end_running_key, &it, true, - recovery_batch ? request->recovery_batch_size() : 10000); + err = txn->get(begin_running_key, end_running_key, &it, true); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); ss << "failed to get txn running info. err=" << err; @@ -4811,64 +4884,6 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont while (it->has_next()) { total_iteration_cnt++; auto [k, v] = it->next(); - if (strict_recovery) { - std::string_view encoded_key = k; - encoded_key.remove_prefix(1); - std::vector, int, int>> fields; - if (decode_key(&encoded_key, &fields) != 0 || fields.size() != 5 || - !std::holds_alternative(std::get<0>(fields[3])) || - !std::holds_alternative(std::get<0>(fields[4]))) { - code = MetaServiceCode::UNDEFINED_ERR; - msg = "failed to decode running transaction key during TSO recovery"; - return; - } - if (std::get(std::get<0>(fields[4])) < request->end_txn_id()) { - // A running key is removed atomically with real VISIBLE/ABORTED. In - // particular an expired COMMITTED lazy transaction must still block. - if (!recovery_batch) { - response->set_finished(false); - return; - } - const auto running_db_id = std::get(std::get<0>(fields[3])); - const auto running_txn_id = std::get(std::get<0>(fields[4])); - std::string info_val; - err = txn->get(txn_info_key({instance_id, running_db_id, running_txn_id}), - &info_val, true); - if (err != TxnErrorCode::TXN_OK) { - code = cast_as(err); - msg = fmt::format( - "failed to read TSO recovery transaction, db_id={}, txn_id={}", - running_db_id, running_txn_id); - return; - } - TxnInfoPB info; - if (!info.ParseFromString(info_val)) { - code = MetaServiceCode::PROTOBUF_PARSE_ERR; - msg = "failed to parse TSO recovery transaction"; - return; - } - // Subtransactions update txn_info.table_ids; the running record can be older. - if (info.db_id() != running_db_id || info.txn_id() != running_txn_id || - info.table_ids().empty()) { - code = MetaServiceCode::UNDEFINED_ERR; - msg = "invalid TSO recovery transaction identity or tables"; - return; - } - // Recovery needs identities and visibility boundaries, not commit attachments. - auto* recovered = response->add_conflict_txns(); - recovered->set_db_id(running_db_id); - recovered->set_txn_id(running_txn_id); - recovered->mutable_table_ids()->CopyFrom(info.table_ids()); - recovered->set_status(info.status()); - if (info.has_commit_tso()) { - recovered->set_commit_tso(info.commit_tso()); - } - } - if (!it->has_next()) { - begin_running_key = k; - } - continue; - } LOG(INFO) << "check watermark conflict range_get txn_run_key=" << hex(k); TxnRunningPB running_pb; if (!running_pb.ParseFromArray(v.data(), v.size())) { @@ -4928,10 +4943,6 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont } } begin_running_key.push_back('\x00'); // Update to next smallest key for iteration - if (recovery_batch) { - response->set_next_recovery_key(it->more() ? begin_running_key : ""); - return; - } } LOG(INFO) << "skip timeout txn count: " << skip_timeout_txn_cnt << " conflict txn count: " << response->conflict_txns_size() diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index d2b55ee65d1dc4..da7e772df7f0f2 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -2781,7 +2781,7 @@ TEST(MetaServiceTest, GetCurrentMaxTxnIdTest) { ASSERT_GE(max_txn_id_res.current_max_txn_id(), begin_txn_res.txn_id()); } -TEST(MetaServiceTest, StrictTsoRecoveryChecksAllDatabasesAndExpiredLazyTransactions) { +TEST(MetaServiceTest, TsoRecoveryChecksAllDatabasesAndExpiredLazyTransactions) { auto meta_service = get_meta_service(); std::unique_ptr txn; ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); @@ -2796,32 +2796,32 @@ TEST(MetaServiceTest, StrictTsoRecoveryChecksAllDatabasesAndExpiredLazyTransacti TxnInfoPB info; info.set_db_id(999); info.set_txn_id(50); + info.add_table_ids(777); info.set_status(TxnStatusPB::TXN_STATUS_COMMITTED); const auto info_key = txn_info_key({mock_instance, 999, 50}); txn->put(info_key, info.SerializeAsString()); ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); brpc::Controller cntl; - CheckTxnConflictRequest request; + GetTsoRecoveryTransactionsRequest request; request.set_cloud_unique_id("test_cloud_unique_id"); request.set_end_txn_id(100); - request.set_strict_recovery_check(true); - CheckTxnConflictResponse response; - meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + request.set_batch_size(256); + GetTsoRecoveryTransactionsResponse response; + meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - ASSERT_TRUE(response.strict_recovery_check_applied()); - ASSERT_FALSE(response.finished()); + ASSERT_EQ(response.txn_infos_size(), 1); // The legacy table-scoped check still skips expired transactions; its result cannot recover TSO. - CheckTxnConflictRequest legacy = request; - legacy.clear_strict_recovery_check(); + CheckTxnConflictRequest legacy; + legacy.set_cloud_unique_id("test_cloud_unique_id"); + legacy.set_end_txn_id(100); legacy.set_db_id(999); legacy.add_table_ids(777); - response.Clear(); - meta_service->check_txn_conflict(&cntl, &legacy, &response, nullptr); - ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - ASSERT_TRUE(response.finished()); - ASSERT_FALSE(response.has_strict_recovery_check_applied()); + CheckTxnConflictResponse legacy_response; + meta_service->check_txn_conflict(&cntl, &legacy, &legacy_response, nullptr); + ASSERT_EQ(legacy_response.status().code(), MetaServiceCode::OK); + ASSERT_TRUE(legacy_response.finished()); // Real publication removes the running key in the same KV transaction as the terminal state. ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); @@ -2830,14 +2830,15 @@ TEST(MetaServiceTest, StrictTsoRecoveryChecksAllDatabasesAndExpiredLazyTransacti txn->remove(blocking_key); ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); response.Clear(); - meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - ASSERT_TRUE(response.strict_recovery_check_applied()); - ASSERT_TRUE( - response.finished()); // IDs equal to or above the fixed exclusive bound are ignored. + // IDs equal to or above the fixed exclusive bound are ignored. + ASSERT_EQ(response.txn_infos_size(), 0); + ASSERT_TRUE(response.has_next_start_key()); + ASSERT_TRUE(response.next_start_key().empty()); } -TEST(MetaServiceTest, StrictTsoRecoveryRejectsMalformedRunningKeys) { +TEST(MetaServiceTest, TsoRecoveryRejectsMalformedRunningKeys) { auto meta_service = get_meta_service(); std::unique_ptr txn; ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); @@ -2846,14 +2847,14 @@ TEST(MetaServiceTest, StrictTsoRecoveryRejectsMalformedRunningKeys) { txn->put(key, ""); ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); brpc::Controller cntl; - CheckTxnConflictRequest request; + GetTsoRecoveryTransactionsRequest request; request.set_cloud_unique_id("test_cloud_unique_id"); request.set_end_txn_id(100); - request.set_strict_recovery_check(true); - CheckTxnConflictResponse response; - meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + request.set_batch_size(256); + GetTsoRecoveryTransactionsResponse response; + meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); ASSERT_NE(response.status().code(), MetaServiceCode::OK); - ASSERT_FALSE(response.finished()); + ASSERT_FALSE(response.has_next_start_key()); } TEST(MetaServiceTest, TsoRecoveryBatchesUseCurrentTablesAndKeepExpiredTransactions) { @@ -2884,38 +2885,35 @@ TEST(MetaServiceTest, TsoRecoveryBatchesUseCurrentTablesAndKeepExpiredTransactio ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); brpc::Controller cntl; - CheckTxnConflictRequest request; + GetTsoRecoveryTransactionsRequest request; request.set_cloud_unique_id("test_cloud_unique_id"); request.set_end_txn_id(100); - request.set_strict_recovery_check(true); - request.set_recovery_batch_size(2); - CheckTxnConflictResponse response; - meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + request.set_batch_size(2); + GetTsoRecoveryTransactionsResponse response; + meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - ASSERT_TRUE(response.strict_recovery_check_applied()); - ASSERT_TRUE(response.recovery_batch_applied()); - ASSERT_FALSE(response.next_recovery_key().empty()); - ASSERT_EQ(response.conflict_txns_size(), 1); - EXPECT_EQ(response.conflict_txns(0).txn_id(), 99); - EXPECT_EQ(response.conflict_txns(0).commit_tso(), 12345); - EXPECT_EQ(response.conflict_txns(0).table_ids(0), 100); + ASSERT_FALSE(response.next_start_key().empty()); + ASSERT_EQ(response.txn_infos_size(), 1); + EXPECT_EQ(response.txn_infos(0).txn_id(), 99); + EXPECT_EQ(response.txn_infos(0).commit_tso(), 12345); + EXPECT_EQ(response.txn_infos(0).table_ids(0), 100); // Deleting already scanned keys does not invalidate the next batch position. ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); txn->remove(txn_running_key({mock_instance, 1, 99})); txn->remove(txn_running_key({mock_instance, 1, 100})); ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); - request.set_recovery_start_key(response.next_recovery_key()); + request.set_start_key(response.next_start_key()); response.Clear(); - meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - ASSERT_TRUE(response.has_next_recovery_key()); - ASSERT_TRUE(response.next_recovery_key().empty()); - ASSERT_EQ(response.conflict_txns_size(), 1); - EXPECT_EQ(response.conflict_txns(0).db_id(), 2); - EXPECT_EQ(response.conflict_txns(0).txn_id(), 10); - EXPECT_EQ(response.conflict_txns(0).table_ids(0), 300); - EXPECT_FALSE(response.conflict_txns(0).has_commit_tso()); + ASSERT_TRUE(response.has_next_start_key()); + ASSERT_TRUE(response.next_start_key().empty()); + ASSERT_EQ(response.txn_infos_size(), 1); + EXPECT_EQ(response.txn_infos(0).db_id(), 2); + EXPECT_EQ(response.txn_infos(0).txn_id(), 10); + EXPECT_EQ(response.txn_infos(0).table_ids(0), 300); + EXPECT_FALSE(response.txn_infos(0).has_commit_tso()); } TEST(MetaServiceTest, TsoRecoveryBatchCanBeEmptyBeforeTheScanCompletes) { @@ -2934,61 +2932,62 @@ TEST(MetaServiceTest, TsoRecoveryBatchCanBeEmptyBeforeTheScanCompletes) { txn->put(txn_info_key({mock_instance, 2, 10}), info.SerializeAsString()); ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); brpc::Controller cntl; - CheckTxnConflictRequest request; + GetTsoRecoveryTransactionsRequest request; request.set_cloud_unique_id("test_cloud_unique_id"); request.set_end_txn_id(100); - request.set_strict_recovery_check(true); - request.set_recovery_batch_size(1); - CheckTxnConflictResponse response; - meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + request.set_batch_size(1); + GetTsoRecoveryTransactionsResponse response; + meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - ASSERT_EQ(response.conflict_txns_size(), 0); - ASSERT_FALSE(response.next_recovery_key().empty()); - request.set_recovery_start_key(response.next_recovery_key()); + ASSERT_EQ(response.txn_infos_size(), 0); + ASSERT_FALSE(response.next_start_key().empty()); + request.set_start_key(response.next_start_key()); response.Clear(); - meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - ASSERT_EQ(response.conflict_txns_size(), 1); - EXPECT_EQ(response.conflict_txns(0).txn_id(), 10); + ASSERT_EQ(response.txn_infos_size(), 1); + EXPECT_EQ(response.txn_infos(0).txn_id(), 10); // Hitting the KV batch limit can require one final empty batch to establish completion. - ASSERT_FALSE(response.next_recovery_key().empty()); - request.set_recovery_start_key(response.next_recovery_key()); + ASSERT_FALSE(response.next_start_key().empty()); + request.set_start_key(response.next_start_key()); response.Clear(); - meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - EXPECT_EQ(response.conflict_txns_size(), 0); - EXPECT_TRUE(response.has_next_recovery_key()); - EXPECT_TRUE(response.next_recovery_key().empty()); + EXPECT_EQ(response.txn_infos_size(), 0); + EXPECT_TRUE(response.has_next_start_key()); + EXPECT_TRUE(response.next_start_key().empty()); } TEST(MetaServiceTest, TsoRecoveryBatchRejectsInvalidArgumentsAndMissingDetails) { auto meta_service = get_meta_service(); brpc::Controller cntl; - CheckTxnConflictRequest request; + GetTsoRecoveryTransactionsRequest request; request.set_cloud_unique_id("test_cloud_unique_id"); + request.set_batch_size(256); + GetTsoRecoveryTransactionsResponse response; + meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); + EXPECT_EQ(response.status().code(), MetaServiceCode::INVALID_ARGUMENT); request.set_end_txn_id(100); - request.set_strict_recovery_check(true); - CheckTxnConflictResponse response; for (int size : {0, 1001}) { - request.set_recovery_batch_size(size); + request.set_batch_size(size); response.Clear(); - meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); EXPECT_EQ(response.status().code(), MetaServiceCode::INVALID_ARGUMENT); } - request.set_recovery_batch_size(1); - request.set_recovery_start_key(txn_running_key({"another_instance", 1, 1})); + request.set_batch_size(1); + request.set_start_key(txn_running_key({"another_instance", 1, 1})); response.Clear(); - meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); EXPECT_EQ(response.status().code(), MetaServiceCode::INVALID_ARGUMENT); - request.clear_recovery_start_key(); + request.clear_start_key(); std::unique_ptr txn; ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); txn->put(txn_running_key({mock_instance, 1, 10}), TxnRunningPB().SerializeAsString()); ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); response.Clear(); - meta_service->check_txn_conflict(&cntl, &request, &response, nullptr); + meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); EXPECT_NE(response.status().code(), MetaServiceCode::OK); - EXPECT_FALSE(response.has_next_recovery_key()); + EXPECT_FALSE(response.has_next_start_key()); } TEST(MetaServiceTest, CreateMetaSyncPointTest) { diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp index af14c414430def..e413b63828b1b0 100644 --- a/cloud/test/txn_lazy_commit_test.cpp +++ b/cloud/test/txn_lazy_commit_test.cpp @@ -1282,25 +1282,19 @@ TEST(TxnLazyCommitTest, CommitTxnEventuallyWithFailedLazyCommitTaskTest) { ASSERT_TRUE(commit_res.has_is_lazy_commit_incomplete()); ASSERT_TRUE(commit_res.is_lazy_commit_incomplete()); - CheckTxnConflictRequest recovery_req; + GetTsoRecoveryTransactionsRequest recovery_req; recovery_req.set_cloud_unique_id("test_cloud_unique_id"); - recovery_req.set_strict_recovery_check(true); recovery_req.set_end_txn_id(txn_id + 1); - CheckTxnConflictResponse recovery_res; - meta_service->check_txn_conflict(&cntl, &recovery_req, &recovery_res, nullptr); + recovery_req.set_batch_size(256); + GetTsoRecoveryTransactionsResponse recovery_res; + meta_service->get_tso_recovery_transactions(&cntl, &recovery_req, &recovery_res, nullptr); ASSERT_EQ(recovery_res.status().code(), MetaServiceCode::OK); - ASSERT_TRUE(recovery_res.strict_recovery_check_applied()); - ASSERT_FALSE(recovery_res.finished()); - recovery_req.set_recovery_batch_size(256); - recovery_res.Clear(); - meta_service->check_txn_conflict(&cntl, &recovery_req, &recovery_res, nullptr); - ASSERT_EQ(recovery_res.status().code(), MetaServiceCode::OK); - ASSERT_TRUE(recovery_res.recovery_batch_applied()); - ASSERT_EQ(recovery_res.conflict_txns_size(), 1); - EXPECT_EQ(recovery_res.conflict_txns(0).txn_id(), txn_id); - EXPECT_EQ(recovery_res.conflict_txns(0).status(), TxnStatusPB::TXN_STATUS_COMMITTED); - EXPECT_EQ(recovery_res.conflict_txns(0).commit_tso(), 12345); - EXPECT_EQ(recovery_res.conflict_txns(0).table_ids(0), table_id); + ASSERT_TRUE(recovery_res.has_next_start_key()); + ASSERT_EQ(recovery_res.txn_infos_size(), 1); + EXPECT_EQ(recovery_res.txn_infos(0).txn_id(), txn_id); + EXPECT_EQ(recovery_res.txn_infos(0).status(), TxnStatusPB::TXN_STATUS_COMMITTED); + EXPECT_EQ(recovery_res.txn_infos(0).commit_tso(), 12345); + EXPECT_EQ(recovery_res.txn_infos(0).table_ids(0), table_id); std::unique_ptr txn; ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java index ebefdb44aa9bcf..649536e8be29e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java @@ -371,6 +371,12 @@ public Cloud.CheckTxnConflictResponse checkTxnConflict(Cloud.CheckTxnConflictReq .checkTxnConflict(request); } + public Cloud.GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions( + Cloud.GetTsoRecoveryTransactionsRequest request) { + return blockingStub.withDeadlineAfter(Config.meta_service_brpc_timeout_ms, TimeUnit.MILLISECONDS) + .getTsoRecoveryTransactions(request); + } + public Cloud.CleanTxnLabelResponse cleanTxnLabel(Cloud.CleanTxnLabelRequest request) { return blockingStub.withDeadlineAfter(Config.meta_service_brpc_timeout_ms, TimeUnit.MILLISECONDS) .cleanTxnLabel(request); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java index e23bf6d5ddf721..5f28367e38ec3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java @@ -470,6 +470,12 @@ public Cloud.CheckTxnConflictResponse checkTxnConflict(Cloud.CheckTxnConflictReq Cloud.CheckTxnConflictResponse::getStatus); } + public Cloud.GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions( + Cloud.GetTsoRecoveryTransactionsRequest request) throws RpcException { + return executeWithMetrics("getTsoRecoveryTransactions", (client) -> client.getTsoRecoveryTransactions(request), + Cloud.GetTsoRecoveryTransactionsResponse::getStatus); + } + public Cloud.CleanTxnLabelResponse cleanTxnLabel(Cloud.CleanTxnLabelRequest request) throws RpcException { return executeWithMetrics("cleanTxnLabel", (client) -> client.cleanTxnLabel(request), diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index b396488160434d..96874e30870e4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -53,6 +53,8 @@ import org.apache.doris.cloud.proto.Cloud.GetDeleteBitmapUpdateLockResponse; import org.apache.doris.cloud.proto.Cloud.GetPrepareTxnByCoordinatorRequest; import org.apache.doris.cloud.proto.Cloud.GetPrepareTxnByCoordinatorResponse; +import org.apache.doris.cloud.proto.Cloud.GetTsoRecoveryTransactionsRequest; +import org.apache.doris.cloud.proto.Cloud.GetTsoRecoveryTransactionsResponse; import org.apache.doris.cloud.proto.Cloud.GetTxnIdRequest; import org.apache.doris.cloud.proto.Cloud.GetTxnIdResponse; import org.apache.doris.cloud.proto.Cloud.GetTxnRequest; @@ -2217,25 +2219,24 @@ public List getUnFinishedPreviousLoad(long endTransactionId, l } @Override - public CheckTxnConflictResponse getTsoRecoveryTransactions(long endTransactionId, + public GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions(long endTransactionId, ByteString startKey) throws UserException { - CheckTxnConflictRequest request = CheckTxnConflictRequest.newBuilder() + GetTsoRecoveryTransactionsRequest request = GetTsoRecoveryTransactionsRequest.newBuilder() .setCloudUniqueId(Config.cloud_unique_id) .setRequestIp(FrontendOptions.getLocalHostAddressCached()) .setEndTxnId(endTransactionId) - .setStrictRecoveryCheck(true).setRecoveryBatchSize(256).setRecoveryStartKey(startKey).build(); - CheckTxnConflictResponse response; + .setBatchSize(256).setStartKey(startKey).build(); + GetTsoRecoveryTransactionsResponse response; try { - response = MetaServiceProxy.getInstance().checkTxnConflict(request); + response = MetaServiceProxy.getInstance().getTsoRecoveryTransactions(request); } catch (RpcException e) { - throw new UserException("Strict TSO recovery check failed", e); + throw new UserException("Failed to fetch TSO recovery transactions", e); } if (response.getStatus().getCode() != MetaServiceCode.OK) { throw new UserException(response.getStatus().getMsg()); } - if (!response.getStrictRecoveryCheckApplied() || !response.getRecoveryBatchApplied() - || !response.hasNextRecoveryKey()) { - throw new UserException("MetaService does not support TSO recovery batches; upgrade MetaService first"); + if (!response.hasNextStartKey()) { + throw new UserException("MetaService returned an incomplete TSO recovery batch"); } return response; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java index f7fd624a881a81..d5272657116648 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java @@ -21,8 +21,8 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.stream.TableStreamUpdateInfo; -import org.apache.doris.cloud.proto.Cloud.CheckTxnConflictResponse; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; +import org.apache.doris.cloud.proto.Cloud.GetTsoRecoveryTransactionsResponse; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DuplicatedRequestException; import org.apache.doris.common.LabelAlreadyUsedException; @@ -149,9 +149,9 @@ public void finishTransaction(long dbId, long transactionId, Map par Map> backendPartitions) throws UserException; /** Fetch a batch of running transactions below the exclusive recovery transaction bound. */ - default CheckTxnConflictResponse getTsoRecoveryTransactions(long endTransactionId, ByteString startKey) + default GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions(long endTransactionId, ByteString startKey) throws UserException { - throw new UserException("Strict TSO recovery is only supported in cloud mode"); + throw new UserException("TSO recovery is only supported in cloud mode"); } public boolean isPreviousTransactionsFinished(long endTransactionId, long dbId, List tableIdList) diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java index c4e13361de4ed0..777adcdb7ad738 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java @@ -17,7 +17,7 @@ package org.apache.doris.tso; -import org.apache.doris.cloud.proto.Cloud.CheckTxnConflictResponse; +import org.apache.doris.cloud.proto.Cloud.GetTsoRecoveryTransactionsResponse; import org.apache.doris.cloud.proto.Cloud.TxnInfoPB; import org.apache.doris.cloud.proto.Cloud.TxnStatusPB; import org.apache.doris.common.Pair; @@ -269,13 +269,13 @@ void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws U } } while (true) { - CheckTxnConflictResponse response = txnMgr.getTsoRecoveryTransactions(watermark, startKey); + GetTsoRecoveryTransactionsResponse response = txnMgr.getTsoRecoveryTransactions(watermark, startKey); lock.lock(); try { if (generation != checkGeneration) { return; } - for (TxnInfoPB info : response.getConflictTxnsList()) { + for (TxnInfoPB info : response.getTxnInfosList()) { Preconditions.checkState(info.getStatus() != TxnStatusPB.TXN_STATUS_VISIBLE && info.getStatus() != TxnStatusPB.TXN_STATUS_ABORTED, "TSO recovery batch contains a terminal transaction: %s", info.getTxnId()); @@ -289,7 +289,7 @@ void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws U } recoveryByTxn.put(info.getTxnId(), recovered); } - startKey = response.getNextRecoveryKey(); + startKey = response.getNextStartKey(); recoveryStartKey = startKey; if (startKey.isEmpty()) { recoveryLoaded = true; diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java index 9c5d96da283b45..717e7dd42febe6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java @@ -30,6 +30,7 @@ import org.apache.doris.cloud.proto.Cloud.CheckTxnConflictResponse; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; import org.apache.doris.cloud.proto.Cloud.GetCurrentMaxTxnResponse; +import org.apache.doris.cloud.proto.Cloud.GetTsoRecoveryTransactionsResponse; import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; import org.apache.doris.cloud.proto.Cloud.TxnInfoPB; import org.apache.doris.cloud.rpc.MetaServiceProxy; @@ -39,6 +40,7 @@ import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.UserException; import org.apache.doris.load.routineload.RLTaskTxnCommitAttachment; +import org.apache.doris.rpc.RpcException; import org.apache.doris.thrift.TTabletCommitInfo; import org.apache.doris.transaction.GlobalTransactionMgrIface; import org.apache.doris.transaction.TabletCommitInfo; @@ -94,32 +96,40 @@ public void tearDown() { } @Test - public void testStrictRecoveryRequiresExplicitMsCapability() throws Exception { + public void testTsoRecoveryRequiresCompleteBatchResponse() throws Exception { MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); try (MockedStatic mocked = Mockito.mockStatic(MetaServiceProxy.class)) { mocked.when(MetaServiceProxy::getInstance).thenReturn(proxy); - CheckTxnConflictResponse.Builder response = CheckTxnConflictResponse.newBuilder() - .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK)) - .setFinished(true); - Mockito.when(proxy.checkTxnConflict(Mockito.any())).thenReturn(response.build()); + GetTsoRecoveryTransactionsResponse.Builder response = GetTsoRecoveryTransactionsResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK)); + Mockito.when(proxy.getTsoRecoveryTransactions(Mockito.any())).thenReturn(response.build()); Assertions.assertThrows(UserException.class, () -> masterTransMgr.getTsoRecoveryTransactions(1000, ByteString.EMPTY)); - Mockito.when(proxy.checkTxnConflict(Mockito.any())).thenReturn( - response.setStrictRecoveryCheckApplied(true).build()); + Mockito.when(proxy.getTsoRecoveryTransactions(Mockito.any())).thenReturn( + response.setNextStartKey(ByteString.EMPTY).build()); + ByteString startKey = ByteString.copyFromUtf8("next batch"); + Assertions.assertTrue(masterTransMgr.getTsoRecoveryTransactions(1000, startKey) + .getNextStartKey().isEmpty()); + ArgumentCaptor capture = + ArgumentCaptor.forClass(Cloud.GetTsoRecoveryTransactionsRequest.class); + Mockito.verify(proxy, Mockito.times(2)).getTsoRecoveryTransactions(capture.capture()); + Assertions.assertEquals(256, capture.getValue().getBatchSize()); + Assertions.assertEquals(1000, capture.getValue().getEndTxnId()); + Assertions.assertEquals(startKey, capture.getValue().getStartKey()); + Mockito.verify(proxy, Mockito.never()).checkTxnConflict(Mockito.any()); + } + } + + @Test + public void testTsoRecoveryRpcFailureDoesNotFallBackToConflictCheck() throws Exception { + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + try (MockedStatic mocked = Mockito.mockStatic(MetaServiceProxy.class)) { + mocked.when(MetaServiceProxy::getInstance).thenReturn(proxy); + Mockito.when(proxy.getTsoRecoveryTransactions(Mockito.any())) + .thenThrow(new RpcException("ms", "unknown method")); Assertions.assertThrows(UserException.class, () -> masterTransMgr.getTsoRecoveryTransactions(1000, ByteString.EMPTY)); - Mockito.when(proxy.checkTxnConflict(Mockito.any())).thenReturn( - response.setRecoveryBatchApplied(true).setNextRecoveryKey(ByteString.EMPTY).build()); - Assertions.assertTrue(masterTransMgr.getTsoRecoveryTransactions(1000, ByteString.EMPTY) - .getNextRecoveryKey().isEmpty()); - ArgumentCaptor capture = - ArgumentCaptor.forClass(Cloud.CheckTxnConflictRequest.class); - Mockito.verify(proxy, Mockito.times(3)).checkTxnConflict(capture.capture()); - Assertions.assertEquals(256, capture.getValue().getRecoveryBatchSize()); - Assertions.assertTrue(capture.getValue().getStrictRecoveryCheck()); - Assertions.assertEquals(1000, capture.getValue().getEndTxnId()); - Assertions.assertFalse(capture.getValue().hasDbId()); - Assertions.assertEquals(0, capture.getValue().getTableIdsCount()); + Mockito.verify(proxy, Mockito.never()).checkTxnConflict(Mockito.any()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java index 22a3a3fbbc475b..9fff60d699375d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java @@ -17,7 +17,7 @@ package org.apache.doris.tso; -import org.apache.doris.cloud.proto.Cloud.CheckTxnConflictResponse; +import org.apache.doris.cloud.proto.Cloud.GetTsoRecoveryTransactionsResponse; import org.apache.doris.cloud.proto.Cloud.TxnInfoPB; import org.apache.doris.cloud.proto.Cloud.TxnStatusPB; import org.apache.doris.common.Pair; @@ -74,10 +74,9 @@ private long candidate(long currentTso, long durableTso) { } } - static CheckTxnConflictResponse recoveryBatch(ByteString nextKey, TxnInfoPB... transactions) { - return CheckTxnConflictResponse.newBuilder().setStrictRecoveryCheckApplied(true) - .setRecoveryBatchApplied(true).setNextRecoveryKey(nextKey) - .addAllConflictTxns(Arrays.asList(transactions)).build(); + static GetTsoRecoveryTransactionsResponse recoveryBatch(ByteString nextKey, TxnInfoPB... transactions) { + return GetTsoRecoveryTransactionsResponse.newBuilder().setNextStartKey(nextKey) + .addAllTxnInfos(Arrays.asList(transactions)).build(); } private static TxnInfoPB recoveryTxn(long dbId, long txnId, long tso, long... tables) { @@ -408,7 +407,7 @@ public void testMissingTransactionAndFailedRecoveryNeverAdvance() throws Excepti register(1, 10, 100); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) - .thenThrow(new UserException("old MS has no strict check capability")); + .thenThrow(new UserException("old MS has no recovery RPC")); Assertions.assertThrows(UserException.class, () -> tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3))); Assertions.assertEquals(80, candidate(150, 80)); diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 9a93c76428845d..62e94a9b75a198 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -1248,21 +1248,30 @@ message CheckTxnConflictRequest { repeated int64 table_ids = 4; optional bool ignore_timeout_txn = 5; optional string request_ip = 6; - // Check every database/table in the instance, including expired running transactions. - optional bool strict_recovery_check = 7 [default = false]; - // Fetch running transaction details in bounded batches for FE TSO recovery. - optional int32 recovery_batch_size = 8; - optional bytes recovery_start_key = 9; } message CheckTxnConflictResponse { optional MetaServiceResponseStatus status = 1; optional bool finished = 2; repeated TxnInfoPB conflict_txns = 3; - optional bool strict_recovery_check_applied = 4; - // Present for batch responses; empty means the full instance has been scanned. - optional bytes next_recovery_key = 5; - optional bool recovery_batch_applied = 6; +} + +message GetTsoRecoveryTransactionsRequest { + optional string cloud_unique_id = 1; // For auth + // Exclusive transaction-ID bound, fixed for the complete recovery scan. + optional int64 end_txn_id = 2; + // Maximum running records scanned per batch, including IDs above the bound. + optional int32 batch_size = 3; + optional bytes start_key = 4; + optional string request_ip = 5; +} + +message GetTsoRecoveryTransactionsResponse { + optional MetaServiceResponseStatus status = 1; + // Includes expired running transactions from every database/table in the instance. + repeated TxnInfoPB txn_infos = 2; + // Always present on success; empty means the full instance has been scanned. + optional bytes next_start_key = 3; } message CleanTxnLabelRequest { @@ -2417,6 +2426,7 @@ service MetaService { rpc get_current_max_txn_id(GetCurrentMaxTxnRequest) returns (GetCurrentMaxTxnResponse); rpc create_meta_sync_point(CreateMetaSyncPointRequest) returns (CreateMetaSyncPointResponse); rpc check_txn_conflict(CheckTxnConflictRequest) returns (CheckTxnConflictResponse); + rpc get_tso_recovery_transactions(GetTsoRecoveryTransactionsRequest) returns (GetTsoRecoveryTransactionsResponse); rpc clean_txn_label(CleanTxnLabelRequest) returns (CleanTxnLabelResponse); rpc get_txn_id(GetTxnIdRequest) returns (GetTxnIdResponse); rpc begin_sub_txn(BeginSubTxnRequest) returns (BeginSubTxnResponse); From d16e67197b2f53e1c9a0c140a31c8669bd8e200b Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Fri, 11 Sep 2026 21:08:32 +0800 Subject: [PATCH 08/13] [fix](binlog) Fence stale commit TSOs after FE failover ### What problem does this PR solve? Issue Number: None Related PR: #67181, #67594 Problem Summary: A new FE master previously waited for all pre-failover transactions, including ordinary PREPARED transactions that could remain open until timeout. Publish a monotonic per-instance TSO fence after persisting the new allocation window, reject stale binlog commits in Meta Service, retry them with the new master's TSO, and recover only pre-fence COMMITTED TSO transactions. ### Release note Fence stale binlog transaction commit TSOs across FE master failover and remove the fixed recovery delay and PREPARED-transaction wait. ### Check List (For Author) - Test: Unit Test - FE TSO service/tracker and cloud transaction manager tests - Cloud Meta Service TSO fence/recovery, key, compatibility, and lazy commit tests - Behavior changed: Yes. Binlog commits with a TSO from an older FE master are retried with a fresh TSO; recovery no longer waits for PREPARED or non-TSO transactions. - Does this need documentation: No --- cloud/src/common/bvars.cpp | 5 + cloud/src/common/bvars.h | 5 + cloud/src/meta-service/meta_service.h | 10 ++ cloud/src/meta-service/meta_service_helper.h | 5 + cloud/src/meta-service/meta_service_txn.cpp | 122 +++++++++++++- cloud/src/meta-store/keys.cpp | 11 +- cloud/src/meta-store/keys.h | 5 + cloud/test/keys_test.cpp | 17 ++ cloud/test/meta_service_helper_test.cpp | 2 + cloud/test/meta_service_test.cpp | 89 +++++++++- cloud/test/txn_lazy_commit_test.cpp | 1 + .../doris/cloud/rpc/MetaServiceClient.java | 5 + .../doris/cloud/rpc/MetaServiceProxy.java | 6 + .../CloudGlobalTransactionMgr.java | 44 ++++- .../GlobalTransactionMgrIface.java | 10 +- .../java/org/apache/doris/tso/TSOService.java | 144 ++++++++++++---- .../doris/tso/TSOTransactionTracker.java | 69 ++++++-- .../CloudGlobalTransactionMgrTest.java | 92 +++++++++- .../org/apache/doris/tso/TSOServiceTest.java | 157 ++++++++++++++++-- .../doris/tso/TSOTransactionTrackerTest.java | 136 +++++++++------ gensrc/proto/cloud.proto | 24 +++ 21 files changed, 825 insertions(+), 134 deletions(-) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 3d81a38283637a..d80f935777055e 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -44,6 +44,7 @@ BvarLatencyRecorderWithTag g_bvar_ms_begin_sub_txn("ms", "begin_sub_txn"); BvarLatencyRecorderWithTag g_bvar_ms_abort_sub_txn("ms", "abort_sub_txn"); BvarLatencyRecorderWithTag g_bvar_ms_check_txn_conflict("ms", "check_txn_conflict"); BvarLatencyRecorderWithTag g_bvar_ms_get_tso_recovery_transactions("ms", "get_tso_recovery_transactions"); +BvarLatencyRecorderWithTag g_bvar_ms_advance_tso_fence("ms", "advance_tso_fence"); BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator("ms", "abort_txn_with_coordinator"); BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator("ms", "get_prepare_txn_by_coordinator"); BvarLatencyRecorderWithTag g_bvar_ms_clean_txn_label("ms", "clean_txn_label"); @@ -502,6 +503,8 @@ mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_counter("rpc_kv // check_txn_conflict mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter("rpc_kv_check_txn_conflict_get_counter",{"instance_id"}); mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_counter("rpc_kv_get_tso_recovery_transactions_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_counter("rpc_kv_advance_tso_fence_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_counter("rpc_kv_advance_tso_fence_put_counter",{"instance_id"}); // clean_txn_label mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter("rpc_kv_clean_txn_label_get_counter",{"instance_id"}); mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter("rpc_kv_clean_txn_label_put_counter",{"instance_id"}); @@ -713,6 +716,8 @@ mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_bytes("rpc_kv_g // check_txn_conflict mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes("rpc_kv_check_txn_conflict_get_bytes",{"instance_id"}); mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_bytes("rpc_kv_get_tso_recovery_transactions_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_bytes("rpc_kv_advance_tso_fence_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_bytes("rpc_kv_advance_tso_fence_put_bytes",{"instance_id"}); // clean_txn_label mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_bytes("rpc_kv_clean_txn_label_get_bytes",{"instance_id"}); mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_bytes("rpc_kv_clean_txn_label_put_bytes",{"instance_id"}); diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index a12265ec633850..32203c3ed46262 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -554,6 +554,7 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_get_current_max_txn_id; extern BvarLatencyRecorderWithTag g_bvar_ms_create_meta_sync_point; extern BvarLatencyRecorderWithTag g_bvar_ms_check_txn_conflict; extern BvarLatencyRecorderWithTag g_bvar_ms_get_tso_recovery_transactions; +extern BvarLatencyRecorderWithTag g_bvar_ms_advance_tso_fence; extern BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator; extern BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator; extern BvarLatencyRecorderWithTag g_bvar_ms_begin_sub_txn; @@ -909,6 +910,8 @@ extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_counter; @@ -1053,6 +1056,8 @@ extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_bytes; diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index 90782d4ee81b90..c47a5786861968 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -150,6 +150,10 @@ class MetaServiceImpl : public cloud::MetaService { GetTsoRecoveryTransactionsResponse* response, ::google::protobuf::Closure* done) override; + void advance_tso_fence(::google::protobuf::RpcController* controller, + const AdvanceTsoFenceRequest* request, AdvanceTsoFenceResponse* response, + ::google::protobuf::Closure* done) override; + void abort_txn_with_coordinator(::google::protobuf::RpcController* controller, const AbortTxnWithCoordinatorRequest* request, AbortTxnWithCoordinatorResponse* response, @@ -632,6 +636,12 @@ class MetaServiceProxy final : public MetaService { done); } + void advance_tso_fence(::google::protobuf::RpcController* controller, + const AdvanceTsoFenceRequest* request, AdvanceTsoFenceResponse* response, + ::google::protobuf::Closure* done) override { + call_impl(&cloud::MetaService::advance_tso_fence, controller, request, response, done); + } + void abort_txn_with_coordinator(::google::protobuf::RpcController* controller, const AbortTxnWithCoordinatorRequest* request, AbortTxnWithCoordinatorResponse* response, diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 2dd1a000e3a5a4..c012aad0bd971e 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -61,6 +61,11 @@ inline std::pair resolve_response_code_and_msg(Met "[TXN_ALREADY_COMMITED will be converted to code=UNDEFINED_ERR for old version " "clients]"; return {MetaServiceCode::UNDEFINED_ERR, std::move(msg)}; + case MetaServiceCode::TXN_COMMIT_TSO_FENCED: + msg += std::string((msg.empty() ? "" : ", ")) + + "[TXN_COMMIT_TSO_FENCED will be converted to code=UNDEFINED_ERR for old " + "version clients]"; + return {MetaServiceCode::UNDEFINED_ERR, std::move(msg)}; default: return {code, std::move(msg)}; } diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 1986fcfe7e181e..5cd8a3915b3d0d 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -127,6 +127,39 @@ static void append_table_stream_commit_size_error(TxnErrorCode err, std::string& } } +static bool check_txn_commit_tso_fence(Transaction* txn, const std::string& instance_id, + int64_t commit_tso, CommitTxnResponse* response, + MetaServiceCode& code, std::string& msg) { + if (commit_tso <= 0) { + return true; + } + + std::string value; + TxnErrorCode err = txn->get(txn_tso_fence_key({instance_id}), &value); + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + return true; + } + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = fmt::format("failed to read TSO fence, err={}", err); + return false; + } + + TxnTsoFencePB fence; + if (!fence.ParseFromString(value) || !fence.has_fence_tso() || fence.fence_tso() <= 0) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + msg = "failed to parse TSO fence"; + return false; + } + if (commit_tso <= fence.fence_tso()) { + response->set_tso_fence(fence.fence_tso()); + code = MetaServiceCode::TXN_COMMIT_TSO_FENCED; + msg = fmt::format("commit TSO {} is fenced by {}", commit_tso, fence.fence_tso()); + return false; + } + return true; +} + class TableStreamUpdateTxnContext { public: TableStreamUpdateTxnContext(Transaction* txn, const std::string& instance_id, @@ -1911,6 +1944,11 @@ void MetaServiceImpl::commit_txn_immediately( return; } + if (txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED && + !check_txn_commit_tso_fence(txn.get(), instance_id, commit_tso, response, code, msg)) { + return; + } + MultiVersionStatus table_stream_multi_version_status = MultiVersionStatus::MULTI_VERSION_DISABLED; if (!request->table_stream_updates().empty()) { @@ -2743,6 +2781,11 @@ void MetaServiceImpl::commit_txn_eventually( return; } + if (txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED && + !check_txn_commit_tso_fence(txn.get(), instance_id, commit_tso, response, code, msg)) { + return; + } + auto now_time = system_clock::now(); uint64_t commit_time = duration_cast(now_time.time_since_epoch()).count(); if ((txn_info.prepare_time() + txn_info.timeout_ms()) < commit_time) { @@ -3143,6 +3186,11 @@ void MetaServiceImpl::commit_txn_with_sub_txn(const CommitTxnRequest* request, return; } + if (txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED && + !check_txn_commit_tso_fence(txn.get(), instance_id, commit_tso, response, code, msg)) { + return; + } + LOG(INFO) << "txn_id=" << txn_id << " txn_info=" << txn_info.ShortDebugString(); AnnotateTag txn_tag("txn_id", txn_id); @@ -4724,12 +4772,76 @@ std::string get_txn_info_key_from_txn_running_key(std::string_view txn_running_k return conflict_txn_info_key; } +void MetaServiceImpl::advance_tso_fence(::google::protobuf::RpcController* controller, + const AdvanceTsoFenceRequest* request, + AdvanceTsoFenceResponse* response, + ::google::protobuf::Closure* done) { + RPC_PREPROCESS(advance_tso_fence, get, put); + if (!request->has_proposed_fence_tso() || request->proposed_fence_tso() <= 0) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "invalid proposed TSO fence"; + return; + } + instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); + if (instance_id.empty()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "cannot find instance_id for TSO fence"; + return; + } + RPC_RATE_LIMIT(advance_tso_fence) + + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = "failed to create TSO fence transaction"; + return; + } + + const std::string key = txn_tso_fence_key({instance_id}); + std::string value; + err = txn->get(key, &value); + int64_t current_fence_tso = 0; + if (err == TxnErrorCode::TXN_OK) { + TxnTsoFencePB fence; + if (!fence.ParseFromString(value) || !fence.has_fence_tso() || fence.fence_tso() <= 0) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + msg = "failed to parse TSO fence"; + return; + } + current_fence_tso = fence.fence_tso(); + } else if (err != TxnErrorCode::TXN_KEY_NOT_FOUND) { + code = cast_as(err); + msg = "failed to read TSO fence"; + return; + } + + const int64_t effective_fence_tso = std::max(current_fence_tso, request->proposed_fence_tso()); + if (effective_fence_tso > current_fence_tso) { + TxnTsoFencePB fence; + fence.set_fence_tso(effective_fence_tso); + if (!fence.SerializeToString(&value)) { + code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; + msg = "failed to serialize TSO fence"; + return; + } + txn->put(key, value); + err = txn->commit(); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = "failed to commit TSO fence"; + return; + } + } + response->set_tso_fence(effective_fence_tso); +} + void MetaServiceImpl::get_tso_recovery_transactions( ::google::protobuf::RpcController* controller, const GetTsoRecoveryTransactionsRequest* request, GetTsoRecoveryTransactionsResponse* response, ::google::protobuf::Closure* done) { RPC_PREPROCESS(get_tso_recovery_transactions, get); - if (request->end_txn_id() <= 0 || request->batch_size() <= 0 || request->batch_size() > 1000) { + if (request->end_txn_id() <= 0 || request->batch_size() <= 0 || request->batch_size() > 1000 || + request->tso_fence() <= 0) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "invalid TSO recovery transaction bound or batch size"; return; @@ -4809,15 +4921,17 @@ void MetaServiceImpl::get_tso_recovery_transactions( msg = "invalid TSO recovery transaction identity or tables"; return; } + if (info.status() != TxnStatusPB::TXN_STATUS_COMMITTED || !info.has_commit_tso() || + info.commit_tso() <= 0 || info.commit_tso() > request->tso_fence()) { + continue; + } // Recovery needs identities and visibility boundaries, not commit attachments. auto* recovered = response->add_txn_infos(); recovered->set_db_id(db_id); recovered->set_txn_id(txn_id); recovered->mutable_table_ids()->CopyFrom(info.table_ids()); recovered->set_status(info.status()); - if (info.has_commit_tso()) { - recovered->set_commit_tso(info.commit_tso()); - } + recovered->set_commit_tso(info.commit_tso()); } begin_key.push_back('\x00'); response->set_next_start_key(it->more() ? begin_key : ""); diff --git a/cloud/src/meta-store/keys.cpp b/cloud/src/meta-store/keys.cpp index 3c4fb91c4f5658..4e3ac10753c86e 100644 --- a/cloud/src/meta-store/keys.cpp +++ b/cloud/src/meta-store/keys.cpp @@ -42,6 +42,7 @@ static const char* TXN_KEY_INFIX_LABEL = "txn_label"; static const char* TXN_KEY_INFIX_INFO = "txn_info"; static const char* TXN_KEY_INFIX_INDEX = "txn_index"; static const char* TXN_KEY_INFIX_RUNNING = "txn_running"; +static const char* TXN_KEY_INFIX_TSO_FENCE = "tso_fence"; static const char* PARTITION_VERSION_KEY_INFIX = "partition"; static const char* TABLE_VERSION_KEY_INFIX = "table"; @@ -145,7 +146,7 @@ static void encode_prefix(const T& t, std::string* key) { // Input type T must be one of the following, add if needed static_assert(check_types_v || std::is_same_v || std::is_same_v - || std::is_same_v) { + || std::is_same_v + || std::is_same_v) { encode_bytes(TXN_KEY_PREFIX, key); } else if constexpr (std::is_same_v || std::is_same_v @@ -259,6 +261,11 @@ void txn_running_key(const TxnRunningKeyInfo& in, std::string* out) { encode_int64(std::get<2>(in), out); // txn_id } +void txn_tso_fence_key(const TxnTsoFenceKeyInfo& in, std::string* out) { + encode_prefix(in, out); // 0x01 "txn" ${instance_id} + encode_bytes(TXN_KEY_INFIX_TSO_FENCE, out); // "tso_fence" +} + //============================================================================== // Version keys //============================================================================== diff --git a/cloud/src/meta-store/keys.h b/cloud/src/meta-store/keys.h index c1142f62a08500..2662eeed4df345 100644 --- a/cloud/src/meta-store/keys.h +++ b/cloud/src/meta-store/keys.h @@ -165,6 +165,9 @@ using TxnIndexKeyInfo = BasicKeyInfo<__LINE__ , std::tuple>; +// 0:instance_id +using TxnTsoFenceKeyInfo = BasicKeyInfo<__LINE__ , std::tuple>; + // 0:instance_id 1:db_id 2:tbl_id 3:partition_id using PartitionVersionKeyInfo = BasicKeyInfo<__LINE__ , std::tuple>; @@ -361,10 +364,12 @@ void txn_label_key(const TxnLabelKeyInfo& in, std::string* out); void txn_info_key(const TxnInfoKeyInfo& in, std::string* out); void txn_index_key(const TxnIndexKeyInfo& in, std::string* out); void txn_running_key(const TxnRunningKeyInfo& in, std::string* out); +void txn_tso_fence_key(const TxnTsoFenceKeyInfo& in, std::string* out); static inline std::string txn_label_key(const TxnLabelKeyInfo& in) { std::string s; txn_label_key(in, &s); return s; } static inline std::string txn_info_key(const TxnInfoKeyInfo& in) { std::string s; txn_info_key(in, &s); return s; } static inline std::string txn_index_key(const TxnIndexKeyInfo& in) { std::string s; txn_index_key(in, &s); return s; } static inline std::string txn_running_key(const TxnRunningKeyInfo& in) { std::string s; txn_running_key(in, &s); return s; } +static inline std::string txn_tso_fence_key(const TxnTsoFenceKeyInfo& in) { std::string s; txn_tso_fence_key(in, &s); return s; } std::string version_key_prefix(std::string_view instance_id); void partition_version_key(const PartitionVersionKeyInfo& in, std::string* out); diff --git a/cloud/test/keys_test.cpp b/cloud/test/keys_test.cpp index 9e9da03bf8d632..ad51182a68cdc1 100644 --- a/cloud/test/keys_test.cpp +++ b/cloud/test/keys_test.cpp @@ -545,6 +545,23 @@ TEST(KeysTest, TxnKeysTest) { ASSERT_GT(encoded_txn_running_key1, encoded_txn_running_key0); } + + // 0x01 "txn" ${instance_id} "tso_fence" -> TxnTsoFencePB + { + std::string encoded_key = txn_tso_fence_key({instance_id}); + std::string_view key_sv(encoded_key); + std::string decoded_prefix; + std::string decoded_instance_id; + std::string decoded_infix; + remove_user_space_prefix(&key_sv); + ASSERT_EQ(decode_bytes(&key_sv, &decoded_prefix), 0); + ASSERT_EQ(decode_bytes(&key_sv, &decoded_instance_id), 0); + ASSERT_EQ(decode_bytes(&key_sv, &decoded_infix), 0); + ASSERT_TRUE(key_sv.empty()); + EXPECT_EQ("txn", decoded_prefix); + EXPECT_EQ(instance_id, decoded_instance_id); + EXPECT_EQ("tso_fence", decoded_infix); + } } TEST(KeysTest, RecycleKeysTest) { diff --git a/cloud/test/meta_service_helper_test.cpp b/cloud/test/meta_service_helper_test.cpp index 73ccf12f42dc2c..e741df470dd9c8 100644 --- a/cloud/test/meta_service_helper_test.cpp +++ b/cloud/test/meta_service_helper_test.cpp @@ -413,6 +413,8 @@ TEST(MetaServiceHelperTest, ResponseStatusCoversEveryMetaServiceCode) { expect_legacy_fallback_response_status(covered_codes, MetaServiceCode::TXN_ALREADY_COMMITED, LegacyFallbackCode::UNDEFINED_ERR); + expect_legacy_fallback_response_status(covered_codes, MetaServiceCode::TXN_COMMIT_TSO_FENCED, + LegacyFallbackCode::UNDEFINED_ERR); EXPECT_EQ(covered_codes.size(), static_cast(MetaServiceCode_descriptor()->value_count())) << "A new MetaServiceCode was added. Map it to a LegacyFallbackCode in " diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index da7e772df7f0f2..08cd9142d7b1aa 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -2781,6 +2781,61 @@ TEST(MetaServiceTest, GetCurrentMaxTxnIdTest) { ASSERT_GE(max_txn_id_res.current_max_txn_id(), begin_txn_res.txn_id()); } +TEST(MetaServiceTest, TsoFenceIsMonotonicAndRejectsStaleCommit) { + auto meta_service = get_meta_service(); + brpc::Controller cntl; + AdvanceTsoFenceRequest fence_request; + fence_request.set_cloud_unique_id("test_cloud_unique_id"); + fence_request.set_proposed_fence_tso(100); + AdvanceTsoFenceResponse fence_response; + meta_service->advance_tso_fence(&cntl, &fence_request, &fence_response, nullptr); + ASSERT_EQ(fence_response.status().code(), MetaServiceCode::OK); + ASSERT_EQ(fence_response.tso_fence(), 100); + + fence_request.set_proposed_fence_tso(90); + fence_response.Clear(); + meta_service->advance_tso_fence(&cntl, &fence_request, &fence_response, nullptr); + ASSERT_EQ(fence_response.status().code(), MetaServiceCode::OK); + ASSERT_EQ(fence_response.tso_fence(), 100); + + int64_t txn_id = -1; + begin_txn(meta_service.get(), 666, "tso_fence_commit", 1234, txn_id); + CommitTxnRequest commit_request; + commit_request.set_db_id(666); + commit_request.set_txn_id(txn_id); + commit_request.set_commit_tso(100); + CommitTxnResponse commit_response; + meta_service->commit_txn(&cntl, &commit_request, &commit_response, nullptr); + ASSERT_EQ(commit_response.status().actual_code(), MetaServiceCode::TXN_COMMIT_TSO_FENCED); + ASSERT_EQ(commit_response.tso_fence(), 100); + + commit_request.set_commit_tso(101); + commit_response.Clear(); + meta_service->commit_txn(&cntl, &commit_request, &commit_response, nullptr); + ASSERT_EQ(commit_response.status().code(), MetaServiceCode::OK); + + // A response retry after the transaction became visible remains idempotent. + commit_request.set_commit_tso(100); + commit_response.Clear(); + meta_service->commit_txn(&cntl, &commit_request, &commit_response, nullptr); + ASSERT_EQ(commit_response.status().code(), MetaServiceCode::OK); + + // Transactions without a commit TSO do not participate in binlog fencing. + int64_t non_tso_txn_id = -1; + begin_txn(meta_service.get(), 666, "tso_fence_non_tso_commit", 1234, non_tso_txn_id); + CommitTxnRequest non_tso_commit_request; + non_tso_commit_request.set_db_id(666); + non_tso_commit_request.set_txn_id(non_tso_txn_id); + CommitTxnResponse non_tso_commit_response; + meta_service->commit_txn(&cntl, &non_tso_commit_request, &non_tso_commit_response, nullptr); + ASSERT_EQ(non_tso_commit_response.status().code(), MetaServiceCode::OK); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->remove(txn_tso_fence_key({mock_instance})); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); +} + TEST(MetaServiceTest, TsoRecoveryChecksAllDatabasesAndExpiredLazyTransactions) { auto meta_service = get_meta_service(); std::unique_ptr txn; @@ -2798,8 +2853,25 @@ TEST(MetaServiceTest, TsoRecoveryChecksAllDatabasesAndExpiredLazyTransactions) { info.set_txn_id(50); info.add_table_ids(777); info.set_status(TxnStatusPB::TXN_STATUS_COMMITTED); + info.set_commit_tso(12345); const auto info_key = txn_info_key({mock_instance, 999, 50}); txn->put(info_key, info.SerializeAsString()); + for (const auto& [txn_id, status, commit_tso] : + std::vector> { + {30, TxnStatusPB::TXN_STATUS_COMMITTED, 20001}, + {35, TxnStatusPB::TXN_STATUS_COMMITTED, 0}, + {40, TxnStatusPB::TXN_STATUS_PREPARED, 0}}) { + txn->put(txn_running_key({mock_instance, 999, txn_id}), running.SerializeAsString()); + TxnInfoPB additional_info = info; + additional_info.set_txn_id(txn_id); + additional_info.set_status(status); + if (commit_tso > 0) { + additional_info.set_commit_tso(commit_tso); + } else { + additional_info.clear_commit_tso(); + } + txn->put(txn_info_key({mock_instance, 999, txn_id}), additional_info.SerializeAsString()); + } ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); brpc::Controller cntl; @@ -2807,6 +2879,7 @@ TEST(MetaServiceTest, TsoRecoveryChecksAllDatabasesAndExpiredLazyTransactions) { request.set_cloud_unique_id("test_cloud_unique_id"); request.set_end_txn_id(100); request.set_batch_size(256); + request.set_tso_fence(20000); GetTsoRecoveryTransactionsResponse response; meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); ASSERT_EQ(response.status().code(), MetaServiceCode::OK); @@ -2851,6 +2924,7 @@ TEST(MetaServiceTest, TsoRecoveryRejectsMalformedRunningKeys) { request.set_cloud_unique_id("test_cloud_unique_id"); request.set_end_txn_id(100); request.set_batch_size(256); + request.set_tso_fence(20000); GetTsoRecoveryTransactionsResponse response; meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); ASSERT_NE(response.status().code(), MetaServiceCode::OK); @@ -2870,11 +2944,8 @@ TEST(MetaServiceTest, TsoRecoveryBatchesUseCurrentTablesAndKeepExpiredTransactio info.set_db_id(db); info.set_txn_id(id); info.add_table_ids(table); - info.set_status(TxnStatusPB::TXN_STATUS_PREPARED); - if (id == 99) { - info.set_status(TxnStatusPB::TXN_STATUS_COMMITTED); - info.set_commit_tso(12345); - } + info.set_status(TxnStatusPB::TXN_STATUS_COMMITTED); + info.set_commit_tso(12345); txn->put(txn_info_key({instance, db, id}), info.SerializeAsString()); }; put_transaction(mock_instance, 1, 99, 100); @@ -2889,6 +2960,7 @@ TEST(MetaServiceTest, TsoRecoveryBatchesUseCurrentTablesAndKeepExpiredTransactio request.set_cloud_unique_id("test_cloud_unique_id"); request.set_end_txn_id(100); request.set_batch_size(2); + request.set_tso_fence(20000); GetTsoRecoveryTransactionsResponse response; meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); ASSERT_EQ(response.status().code(), MetaServiceCode::OK); @@ -2913,7 +2985,7 @@ TEST(MetaServiceTest, TsoRecoveryBatchesUseCurrentTablesAndKeepExpiredTransactio EXPECT_EQ(response.txn_infos(0).db_id(), 2); EXPECT_EQ(response.txn_infos(0).txn_id(), 10); EXPECT_EQ(response.txn_infos(0).table_ids(0), 300); - EXPECT_FALSE(response.txn_infos(0).has_commit_tso()); + EXPECT_EQ(response.txn_infos(0).commit_tso(), 12345); } TEST(MetaServiceTest, TsoRecoveryBatchCanBeEmptyBeforeTheScanCompletes) { @@ -2928,7 +3000,8 @@ TEST(MetaServiceTest, TsoRecoveryBatchCanBeEmptyBeforeTheScanCompletes) { info.set_db_id(2); info.set_txn_id(10); info.add_table_ids(300); - info.set_status(TxnStatusPB::TXN_STATUS_PREPARED); + info.set_status(TxnStatusPB::TXN_STATUS_COMMITTED); + info.set_commit_tso(12345); txn->put(txn_info_key({mock_instance, 2, 10}), info.SerializeAsString()); ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); brpc::Controller cntl; @@ -2936,6 +3009,7 @@ TEST(MetaServiceTest, TsoRecoveryBatchCanBeEmptyBeforeTheScanCompletes) { request.set_cloud_unique_id("test_cloud_unique_id"); request.set_end_txn_id(100); request.set_batch_size(1); + request.set_tso_fence(20000); GetTsoRecoveryTransactionsResponse response; meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); ASSERT_EQ(response.status().code(), MetaServiceCode::OK); @@ -2968,6 +3042,7 @@ TEST(MetaServiceTest, TsoRecoveryBatchRejectsInvalidArgumentsAndMissingDetails) meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); EXPECT_EQ(response.status().code(), MetaServiceCode::INVALID_ARGUMENT); request.set_end_txn_id(100); + request.set_tso_fence(20000); for (int size : {0, 1001}) { request.set_batch_size(size); response.Clear(); diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp index e413b63828b1b0..b2fabdcfc5aaed 100644 --- a/cloud/test/txn_lazy_commit_test.cpp +++ b/cloud/test/txn_lazy_commit_test.cpp @@ -1286,6 +1286,7 @@ TEST(TxnLazyCommitTest, CommitTxnEventuallyWithFailedLazyCommitTaskTest) { recovery_req.set_cloud_unique_id("test_cloud_unique_id"); recovery_req.set_end_txn_id(txn_id + 1); recovery_req.set_batch_size(256); + recovery_req.set_tso_fence(12345); GetTsoRecoveryTransactionsResponse recovery_res; meta_service->get_tso_recovery_transactions(&cntl, &recovery_req, &recovery_res, nullptr); ASSERT_EQ(recovery_res.status().code(), MetaServiceCode::OK); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java index 649536e8be29e3..d87d537a2a2548 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java @@ -377,6 +377,11 @@ public Cloud.GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions( .getTsoRecoveryTransactions(request); } + public Cloud.AdvanceTsoFenceResponse advanceTsoFence(Cloud.AdvanceTsoFenceRequest request) { + return blockingStub.withDeadlineAfter(Config.meta_service_brpc_timeout_ms, TimeUnit.MILLISECONDS) + .advanceTsoFence(request); + } + public Cloud.CleanTxnLabelResponse cleanTxnLabel(Cloud.CleanTxnLabelRequest request) { return blockingStub.withDeadlineAfter(Config.meta_service_brpc_timeout_ms, TimeUnit.MILLISECONDS) .cleanTxnLabel(request); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java index 5f28367e38ec3f..efa06cebc62907 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java @@ -476,6 +476,12 @@ public Cloud.GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions( Cloud.GetTsoRecoveryTransactionsResponse::getStatus); } + public Cloud.AdvanceTsoFenceResponse advanceTsoFence(Cloud.AdvanceTsoFenceRequest request) + throws RpcException { + return executeWithMetrics("advanceTsoFence", (client) -> client.advanceTsoFence(request), + Cloud.AdvanceTsoFenceResponse::getStatus); + } + public Cloud.CleanTxnLabelResponse cleanTxnLabel(Cloud.CleanTxnLabelRequest request) throws RpcException { return executeWithMetrics("cleanTxnLabel", (client) -> client.cleanTxnLabel(request), diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index 96874e30870e4a..eccd3fcc7c2b56 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -36,6 +36,8 @@ import org.apache.doris.cloud.proto.Cloud.AbortSubTxnResponse; import org.apache.doris.cloud.proto.Cloud.AbortTxnRequest; import org.apache.doris.cloud.proto.Cloud.AbortTxnResponse; +import org.apache.doris.cloud.proto.Cloud.AdvanceTsoFenceRequest; +import org.apache.doris.cloud.proto.Cloud.AdvanceTsoFenceResponse; import org.apache.doris.cloud.proto.Cloud.BeginSubTxnRequest; import org.apache.doris.cloud.proto.Cloud.BeginSubTxnResponse; import org.apache.doris.cloud.proto.Cloud.BeginTxnRequest; @@ -860,9 +862,9 @@ private TransactionState commitTxn(CommitTxnRequest.Builder builder, List

// Bitmap work, attachments and metadata validation do not need a commit TSO. Allocate only // when ready to send, while retaining the existing table locks and callback cleanup scope. Database database = Env.getCurrentInternalCatalog().getDbOrMetaException(builder.getDbId()); - builder.setCommitTso(TransactionUtil.getCommitTSO(transactionId, database, - tableList.stream().map(Table::getId).collect(Collectors.toSet()))); - final CommitTxnRequest commitTxnRequest = builder.build(); + Set commitTsoTableIds = tableList.stream().map(Table::getId).collect(Collectors.toSet()); + builder.setCommitTso(TransactionUtil.getCommitTSO(transactionId, database, commitTsoTableIds)); + CommitTxnRequest commitTxnRequest = builder.build(); try { while (DebugPointUtil.isEnable("CloudGlobalTransactionMgr.commitTxn.blockAfterTso")) { Thread.sleep(100); @@ -885,6 +887,20 @@ private TransactionState commitTxn(CommitTxnRequest.Builder builder, List
if (LOG.isDebugEnabled()) { LOG.debug("retryTime:{}, commitTxnResponse:{}", retryTime, commitTxnResponse); } + if (commitTxnResponse.getStatus().getCode() == MetaServiceCode.TXN_COMMIT_TSO_FENCED) { + if (!commitTxnResponse.hasTsoFence() || commitTxnRequest.getCommitTso() <= 0) { + throw new UserException("MetaService returned an invalid TSO fence response"); + } + long replacementTso = Env.getCurrentEnv().getTSOService().getCommitTSOAfterFence( + builder.getDbId(), transactionId, commitTsoTableIds, + commitTxnRequest.getCommitTso(), commitTxnResponse.getTsoFence()); + LOG.info("commitTxn replaces fenced TSO, transactionId:{}, oldTso:{}, newTso:{}, fenceTso:{}", + transactionId, commitTxnRequest.getCommitTso(), replacementTso, + commitTxnResponse.getTsoFence()); + commitTxnRequest = commitTxnRequest.toBuilder().setCommitTso(replacementTso).build(); + retryTime++; + continue; + } if (commitTxnResponse.getStatus().getCode() != MetaServiceCode.KV_TXN_CONFLICT) { break; } @@ -2220,11 +2236,12 @@ public List getUnFinishedPreviousLoad(long endTransactionId, l @Override public GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions(long endTransactionId, - ByteString startKey) throws UserException { + long tsoFence, ByteString startKey) throws UserException { GetTsoRecoveryTransactionsRequest request = GetTsoRecoveryTransactionsRequest.newBuilder() .setCloudUniqueId(Config.cloud_unique_id) .setRequestIp(FrontendOptions.getLocalHostAddressCached()) .setEndTxnId(endTransactionId) + .setTsoFence(tsoFence) .setBatchSize(256).setStartKey(startKey).build(); GetTsoRecoveryTransactionsResponse response; try { @@ -2241,6 +2258,25 @@ public GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions(long endTra return response; } + @Override + public long advanceTsoFence(long proposedFenceTso) throws UserException { + AdvanceTsoFenceRequest request = AdvanceTsoFenceRequest.newBuilder() + .setCloudUniqueId(Config.cloud_unique_id) + .setRequestIp(FrontendOptions.getLocalHostAddressCached()) + .setProposedFenceTso(proposedFenceTso) + .build(); + AdvanceTsoFenceResponse response; + try { + response = MetaServiceProxy.getInstance().advanceTsoFence(request); + } catch (RpcException e) { + throw new UserException("Failed to advance TSO fence", e); + } + if (response.getStatus().getCode() != MetaServiceCode.OK || !response.hasTsoFence()) { + throw new UserException("Failed to advance TSO fence: " + response.getStatus().getMsg()); + } + return response.getTsoFence(); + } + @Override public boolean isPreviousTransactionsFinished(long endTransactionId, long dbId, List tableIdList) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java index d5272657116648..3fec6d943a2d00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java @@ -148,12 +148,16 @@ public void abortTransaction(Long dbId, Long txnId, String reason, public void finishTransaction(long dbId, long transactionId, Map partitionVisibleVersions, Map> backendPartitions) throws UserException; - /** Fetch a batch of running transactions below the exclusive recovery transaction bound. */ - default GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions(long endTransactionId, ByteString startKey) - throws UserException { + /** Fetch committed TSO transactions below the exclusive recovery transaction bound. */ + default GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions(long endTransactionId, + long tsoFence, ByteString startKey) throws UserException { throw new UserException("TSO recovery is only supported in cloud mode"); } + default long advanceTsoFence(long proposedFenceTso) throws UserException { + throw new UserException("TSO fence is only supported in cloud mode"); + } + public boolean isPreviousTransactionsFinished(long endTransactionId, long dbId, List tableIdList) throws AnalysisException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java index 174b64e447df5e..ffb8552cb5dbc5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java @@ -31,6 +31,7 @@ import org.apache.doris.metric.MetricRepo; import org.apache.doris.persist.EditLog; +import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -60,6 +61,8 @@ public class TSOService extends MasterDaemon { private final AtomicBoolean fatalClockBackwardReported = new AtomicBoolean(false); private volatile TSOServiceState durableState = new TSOServiceState(0, 0); private final TSOTransactionTracker transactionTracker = new TSOTransactionTracker(lock); + private long pendingCalibrationPhysicalTime; + private long pendingCalibrationWindowEnd; private long lastPersistNanos; private final MasterDaemon transactionChecker = new MasterDaemon("TSO-transaction-checker", 1000) { private long lastFailureLogNanos; @@ -174,13 +177,9 @@ public synchronized void start() { */ @Override protected void runAfterCatalogReady() { - if (!isTsoEnabled()) { - lock.lock(); - try { - isInitialized.set(false); - } finally { - lock.unlock(); - } + Env env = Env.getCurrentEnv(); + if (!isTsoEnabled() || env == null || !env.isReady() || !env.isMaster()) { + deactivate(); return; } int maxUpdateRetryCount = Math.max(1, Config.tso_max_update_retry_count); @@ -265,11 +264,21 @@ public long getCommitTSO(long dbId, long txnId, Set tableIds) { return getTSO(Pair.of(dbId, txnId), tableIds); } + public long getCommitTSOAfterFence(long dbId, long txnId, Set tableIds, + long rejectedTso, long fenceTso) { + return getTSO(Pair.of(dbId, txnId), tableIds, rejectedTso, fenceTso); + } + public void transactionFinished(long dbId, long txnId) { transactionTracker.transactionFinished(dbId, txnId); } private long getTSO(Pair transactionIdentity, Set tableIds) { + return getTSO(transactionIdentity, tableIds, -1, -1); + } + + private long getTSO(Pair transactionIdentity, Set tableIds, + long rejectedTso, long fenceTso) { if (!isTsoEnabled()) { throw new RuntimeException("TSO feature is disabled, please check enable_feature_binlog"); } @@ -293,6 +302,10 @@ private long getTSO(Pair transactionIdentity, Set tableIds) { continue; } else if (!env.isMaster()) { LOG.warn("TSO service only run on master FE"); + if (fenceTso > 0) { + throw new RuntimeException( + "TXN_COMMIT_TSO_FENCED: retry the commit through the current master FE"); + } lastFailure = new RuntimeException("Current FE is not master"); try { sleep(200); @@ -303,7 +316,7 @@ private long getTSO(Pair transactionIdentity, Set tableIds) { continue; } - Pair pair = generateTSO(transactionIdentity, tableIds); + Pair pair = generateTSO(transactionIdentity, tableIds, rejectedTso, fenceTso); long physical = pair.first; long logical = pair.second; @@ -419,7 +432,7 @@ private IncrWindowNotReadyException windowError(ErrorCode code, String reason, l * - If Tnow - Tlast < 1ms, then Tnext = Tlast + 1 * - Otherwise Tnext = Tnow */ - private void calibrateTimestamp() { + private void calibrateTimestamp() throws UserException { if (isInitialized.get()) { return; } @@ -432,44 +445,86 @@ private void calibrateTimestamp() { long timeLast = durableState.getPhysicalTimestamp(); // Last timestamp from image/editlog replay long timeNow = System.currentTimeMillis() + Config.tso_time_offset_debug_mode; - long backwardMs = timeLast - timeNow; - if (backwardMs > Config.tso_clock_backward_startup_threshold_ms) { - throw new TSOClockBackwardException("TSO clock backward too much during calibration, backwardMs=" - + backwardMs + ", thresholdMs=" + Config.tso_clock_backward_startup_threshold_ms - + ", lastWindowEndTSO=" + timeLast + ", currentMillis=" + timeNow); - } - - // Calculate next physical time to ensure monotonicity long nextPhysicalTime; - if (timeNow - timeLast < 1) { - nextPhysicalTime = timeLast + 1; + long timeWindowEnd; + lock.lock(); + try { + nextPhysicalTime = pendingCalibrationPhysicalTime; + timeWindowEnd = pendingCalibrationWindowEnd; + } finally { + lock.unlock(); + } + if (nextPhysicalTime == 0) { + long backwardMs = timeLast - timeNow; + if (backwardMs > Config.tso_clock_backward_startup_threshold_ms) { + throw new TSOClockBackwardException("TSO clock backward too much during calibration, backwardMs=" + + backwardMs + ", thresholdMs=" + Config.tso_clock_backward_startup_threshold_ms + + ", lastWindowEndTSO=" + timeLast + ", currentMillis=" + timeNow); + } + // Calculate next physical time to ensure monotonicity. + nextPhysicalTime = timeNow - timeLast < 1 ? timeLast + 1 : timeNow; + timeWindowEnd = persistCalibrationWindow(nextPhysicalTime); } else { - nextPhysicalTime = timeNow; + Preconditions.checkState(timeWindowEnd > nextPhysicalTime, + "pending calibration window must cover its TSO"); } + long proposedFenceTso; + while (true) { + proposedFenceTso = TSOTimestamp.composePhysicalTimestamp(nextPhysicalTime); + if (!Config.isCloudMode()) { + break; + } + long effectiveFenceTso = env.getGlobalTransactionMgr().advanceTsoFence(proposedFenceTso); + Preconditions.checkState(effectiveFenceTso >= proposedFenceTso, + "MetaService TSO fence must not regress"); + if (effectiveFenceTso == proposedFenceTso) { + break; + } + nextPhysicalTime = TSOTimestamp.extractPhysicalTime(effectiveFenceTso) + 1; + timeWindowEnd = persistCalibrationWindow(nextPhysicalTime); + } lock.lock(); try { - transactionTracker.reset(System.nanoTime(), Config.tso_service_window_duration_ms + 1000L); + pendingCalibrationPhysicalTime = 0; + pendingCalibrationWindowEnd = 0; } finally { lock.unlock(); } - - // Construct new timestamp (physical time with reset logical counter) - setTSOPhysical(nextPhysicalTime, true); - - // Write the right boundary of time window to BDBJE for persistence - long timeWindowEnd = nextPhysicalTime + Config.tso_service_window_duration_ms; - writeTimestampToBDBJE(timeWindowEnd); isInitialized.set(true); fatalClockBackwardReported.set(false); - LOG.info("TSO timestamp calibrated: lastTimestamp={}, currentMillis={}, nextPhysicalTime={}, timeWindowEnd={}", - timeLast, timeNow, nextPhysicalTime, timeWindowEnd); + LOG.info("TSO timestamp calibrated: lastTimestamp={}, currentMillis={}, nextPhysicalTime={}, " + + "timeWindowEnd={}, fenceTso={}", + timeLast, timeNow, nextPhysicalTime, timeWindowEnd, proposedFenceTso); if (MetricRepo.isInit) { MetricRepo.COUNTER_TSO_CLOCK_CALCULATED.increase(1L); } } + private long persistCalibrationWindow(long physicalTime) { + long fenceTso = TSOTimestamp.composePhysicalTimestamp(physicalTime); + lock.lock(); + try { + transactionTracker.reset(fenceTso); + } finally { + lock.unlock(); + } + + // Persist the allocation window before publishing its inclusive commit fence. + long windowEnd = physicalTime + Config.tso_service_window_duration_ms; + writeTimestampToBDBJE(windowEnd); + setTSOPhysical(physicalTime, true); + lock.lock(); + try { + pendingCalibrationPhysicalTime = physicalTime; + pendingCalibrationWindowEnd = windowEnd; + } finally { + lock.unlock(); + } + return windowEnd; + } + /** * Update timestamp periodically to maintain time window * This method handles various time-related issues: @@ -649,6 +704,11 @@ private Pair generateTSO() { } private Pair generateTSO(Pair transactionIdentity, Set tableIds) { + return generateTSO(transactionIdentity, tableIds, -1, -1); + } + + private Pair generateTSO(Pair transactionIdentity, Set tableIds, + long rejectedTso, long fenceTso) { lock.lock(); try { if (!isTsoEnabled() || !isInitialized.get()) { @@ -658,6 +718,10 @@ private Pair generateTSO(Pair transactionIdentity, Set 0 && globalTimestamp.composeTimestamp() < fenceTso) { + throw new RuntimeException("TXN_COMMIT_TSO_FENCED: local TSO " + + globalTimestamp.composeTimestamp() + " is behind MetaService fence " + fenceTso); + } long logicalCounter = globalTimestamp.getLogicalCounter(); if (logicalCounter >= TSOTimestamp.MAX_LOGICAL_COUNTER) { return Pair.of(physicalTime, logicalCounter + 1); @@ -665,8 +729,13 @@ private Pair generateTSO(Pair transactionIdentity, Set 0) { + transactionTracker.replaceFenced(transactionIdentity, rejectedTso, fenceTso, + tso, System.nanoTime(), tableIds); + } else { + transactionTracker.register(transactionIdentity, tso, System.nanoTime(), tableIds); + } } return Pair.of(physicalTime, nextLogical); } finally { @@ -674,6 +743,17 @@ private Pair generateTSO(Pair transactionIdentity, Set, PendingTransaction> pendingByTxn = new HashMap<>(); private final TreeMap pendingByTso = new TreeMap<>(); - // Recovered transactions may have no persisted TSO yet. Keep them out of the TSO index. + // Recovery entries stay separate so the durable committed prefix remains frozen until all finish. private final TreeMap recoveryByTxn = new TreeMap<>(); private long generation; - private long recoveryDeadlineNanos; + private long recoveryFenceTso; private long recoveryWatermark; private boolean recoveryReady; private boolean recoveryLoaded; @@ -87,13 +87,24 @@ private PendingTransaction(Pair identity, long tso, long nowNanos, C this.transactionsChanged = lock.newCondition(); } - void reset(long nowNanos, long recoveryDelayMs) { + void reset(long fenceTso) { Preconditions.checkState(lock.isHeldByCurrentThread()); + Preconditions.checkArgument(fenceTso > 0, "recovery fence TSO must be positive"); + clearForNewGeneration(); + recoveryFenceTso = fenceTso; + } + + void invalidate() { + Preconditions.checkState(lock.isHeldByCurrentThread()); + clearForNewGeneration(); + recoveryFenceTso = 0; + } + + private void clearForNewGeneration() { generation++; pendingByTxn.clear(); pendingByTso.clear(); recoveryByTxn.clear(); - recoveryDeadlineNanos = nowNanos + TimeUnit.MILLISECONDS.toNanos(recoveryDelayMs); recoveryWatermark = 0; recoveryReady = false; recoveryLoaded = false; @@ -121,6 +132,31 @@ void register(Pair identity, long tso, long nowNanos, Set tabl Preconditions.checkState(pendingByTso.put(tso, pending) == null); } + void replaceFenced(Pair identity, long rejectedTso, long fenceTso, long newTso, + long nowNanos, Set tableIds) { + Preconditions.checkState(lock.isHeldByCurrentThread()); + Preconditions.checkArgument(!tableIds.isEmpty(), "commit registration requires table IDs"); + Preconditions.checkArgument(rejectedTso <= fenceTso, "rejected TSO must be fenced"); + Preconditions.checkArgument(newTso > fenceTso, "replacement TSO must be above the fence"); + PendingTransaction existing = pendingByTxn.get(identity); + Set mergedTableIds = new HashSet<>(); + if (existing != null) { + Preconditions.checkState(existing.tso <= fenceTso, + "registered transaction TSO must be fenced"); + Preconditions.checkState(pendingByTso.remove(existing.tso) == existing); + mergedTableIds.addAll(existing.tableIds); + } + mergedTableIds.addAll(tableIds); + PendingTransaction replacement = new PendingTransaction(identity, newTso, nowNanos, mergedTableIds); + pendingByTxn.put(identity, replacement); + Preconditions.checkState(pendingByTso.put(newTso, replacement) == null); + PendingTransaction recovered = recoveryByTxn.get(identity.second); + if (recovered != null) { + recovered.tableIds.addAll(tableIds); + } + transactionsChanged.signalAll(); + } + /** Called with the allocator lock after validating endTso against its current clock. */ WaitResult awaitTransactions(Map> dbToTableIds, long endTso, long remainingNanos) throws InterruptedException { @@ -133,8 +169,7 @@ WaitResult awaitTransactions(Map> dbToTableIds, long endTso, lo List remaining = new ArrayList<>(); for (PendingTransaction pending : recoveryByTxn.values()) { List tables = dbToTableIds.get(pending.identity.first); - if ((pending.tso <= 0 || pending.tso <= endTso) - && tables != null && !Collections.disjoint(tables, pending.tableIds)) { + if (pending.tso <= endTso && tables != null && !Collections.disjoint(tables, pending.tableIds)) { remaining.add(pending); } } @@ -193,6 +228,7 @@ void transactionFinished(long dbId, long txnId) { void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws UserException { long checkGeneration; long watermark; + long fenceTso; boolean checkRecovery; ByteString startKey; List batch = new ArrayList<>(); @@ -200,7 +236,8 @@ void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws U try { checkGeneration = generation; watermark = recoveryWatermark; - checkRecovery = !recoveryLoaded && nowNanos - recoveryDeadlineNanos >= 0; + fenceTso = recoveryFenceTso; + checkRecovery = !recoveryLoaded; startKey = recoveryStartKey; if (!pendingByTso.isEmpty()) { // Always check the transaction blocking the prefix, then rotate through the rest. @@ -269,20 +306,22 @@ void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws U } } while (true) { - GetTsoRecoveryTransactionsResponse response = txnMgr.getTsoRecoveryTransactions(watermark, startKey); + GetTsoRecoveryTransactionsResponse response = + txnMgr.getTsoRecoveryTransactions(watermark, fenceTso, startKey); lock.lock(); try { if (generation != checkGeneration) { return; } for (TxnInfoPB info : response.getTxnInfosList()) { - Preconditions.checkState(info.getStatus() != TxnStatusPB.TXN_STATUS_VISIBLE - && info.getStatus() != TxnStatusPB.TXN_STATUS_ABORTED, - "TSO recovery batch contains a terminal transaction: %s", info.getTxnId()); + Preconditions.checkState(info.getStatus() == TxnStatusPB.TXN_STATUS_COMMITTED, + "TSO recovery batch contains a non-committed transaction: %s", info.getTxnId()); + Preconditions.checkState(info.hasCommitTso() && info.getCommitTso() > 0 + && info.getCommitTso() <= fenceTso, + "TSO recovery transaction exceeds fence: %s", info.getTxnId()); Pair identity = Pair.of(info.getDbId(), info.getTxnId()); PendingTransaction recovered = new PendingTransaction(identity, - info.hasCommitTso() ? info.getCommitTso() : 0, - nowNanos, info.getTableIdsList()); + info.getCommitTso(), nowNanos, info.getTableIdsList()); PendingTransaction local = pendingByTxn.get(identity); if (local != null) { recovered.tableIds.addAll(local.tableIds); @@ -294,8 +333,8 @@ void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws U if (startKey.isEmpty()) { recoveryLoaded = true; updateRecoveryReady(); - LOG.info("Loaded TSO recovery transactions, watermark={}, pending={}", - watermark, recoveryByTxn.size()); + LOG.info("Loaded TSO recovery transactions, watermark={}, fenceTso={}, pending={}", + watermark, fenceTso, recoveryByTxn.size()); return; } } finally { diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java index 717e7dd42febe6..37a2f92c531c08 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java @@ -17,15 +17,18 @@ package org.apache.doris.cloud.transaction; +import org.apache.doris.catalog.BinlogConfig; import org.apache.doris.catalog.CatalogTestUtil; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FakeEditLog; import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.stream.CloudOlapTableStreamUpdate; import org.apache.doris.catalog.stream.TableStreamUpdateInfo; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.proto.Cloud.AbortTxnResponse; +import org.apache.doris.cloud.proto.Cloud.AdvanceTsoFenceResponse; import org.apache.doris.cloud.proto.Cloud.BeginTxnResponse; import org.apache.doris.cloud.proto.Cloud.CheckTxnConflictResponse; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; @@ -46,6 +49,7 @@ import org.apache.doris.transaction.TabletCommitInfo; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TxnStateChangeCallback; +import org.apache.doris.tso.TSOService; import com.google.common.collect.Lists; import com.google.protobuf.ByteString; @@ -57,6 +61,7 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.List; import java.util.Map; @@ -104,17 +109,18 @@ public void testTsoRecoveryRequiresCompleteBatchResponse() throws Exception { .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK)); Mockito.when(proxy.getTsoRecoveryTransactions(Mockito.any())).thenReturn(response.build()); Assertions.assertThrows(UserException.class, - () -> masterTransMgr.getTsoRecoveryTransactions(1000, ByteString.EMPTY)); + () -> masterTransMgr.getTsoRecoveryTransactions(1000, 900, ByteString.EMPTY)); Mockito.when(proxy.getTsoRecoveryTransactions(Mockito.any())).thenReturn( response.setNextStartKey(ByteString.EMPTY).build()); ByteString startKey = ByteString.copyFromUtf8("next batch"); - Assertions.assertTrue(masterTransMgr.getTsoRecoveryTransactions(1000, startKey) + Assertions.assertTrue(masterTransMgr.getTsoRecoveryTransactions(1000, 900, startKey) .getNextStartKey().isEmpty()); ArgumentCaptor capture = ArgumentCaptor.forClass(Cloud.GetTsoRecoveryTransactionsRequest.class); Mockito.verify(proxy, Mockito.times(2)).getTsoRecoveryTransactions(capture.capture()); Assertions.assertEquals(256, capture.getValue().getBatchSize()); Assertions.assertEquals(1000, capture.getValue().getEndTxnId()); + Assertions.assertEquals(900, capture.getValue().getTsoFence()); Assertions.assertEquals(startKey, capture.getValue().getStartKey()); Mockito.verify(proxy, Mockito.never()).checkTxnConflict(Mockito.any()); } @@ -128,11 +134,27 @@ public void testTsoRecoveryRpcFailureDoesNotFallBackToConflictCheck() throws Exc Mockito.when(proxy.getTsoRecoveryTransactions(Mockito.any())) .thenThrow(new RpcException("ms", "unknown method")); Assertions.assertThrows(UserException.class, - () -> masterTransMgr.getTsoRecoveryTransactions(1000, ByteString.EMPTY)); + () -> masterTransMgr.getTsoRecoveryTransactions(1000, 900, ByteString.EMPTY)); Mockito.verify(proxy, Mockito.never()).checkTxnConflict(Mockito.any()); } } + @Test + public void testAdvanceTsoFence() throws Exception { + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + try (MockedStatic mocked = Mockito.mockStatic(MetaServiceProxy.class)) { + mocked.when(MetaServiceProxy::getInstance).thenReturn(proxy); + Mockito.when(proxy.advanceTsoFence(Mockito.any())).thenReturn(AdvanceTsoFenceResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK)) + .setTsoFence(101).build()); + Assertions.assertEquals(101, masterTransMgr.advanceTsoFence(100)); + ArgumentCaptor capture = + ArgumentCaptor.forClass(Cloud.AdvanceTsoFenceRequest.class); + Mockito.verify(proxy).advanceTsoFence(capture.capture()); + Assertions.assertEquals(100, capture.getValue().getProposedFenceTso()); + } + } + @Test public void testBeginTransaction() throws Exception { AtomicLong id = new AtomicLong(1000); @@ -251,6 +273,64 @@ public void testCommitTransaction() throws Exception { } } + @Test + public void testCommitTransactionRetriesWithTsoAboveFence() throws Exception { + boolean originalEnableFeatureBinlog = Config.enable_feature_binlog; + OlapTable table = (OlapTable) masterEnv.getInternalCatalog() + .getDbOrMetaException(CatalogTestUtil.testDbId1) + .getTableOrMetaException(CatalogTestUtil.testTableId1); + BinlogConfig originalBinlogConfig = new BinlogConfig(table.getBinlogConfig()); + try { + Config.enable_feature_binlog = true; + FakeEnv.setEnv(masterEnv); + BinlogConfig binlogConfig = new BinlogConfig(originalBinlogConfig); + binlogConfig.setEnable(true); + binlogConfig.setBinlogFormat(BinlogConfig.BinlogFormat.ROW); + table.setBinlogConfig(binlogConfig); + + TSOService tsoService = Mockito.mock(TSOService.class); + Mockito.when(tsoService.getCommitTSO(Mockito.eq(CatalogTestUtil.testDbId1), + Mockito.eq(123533L), Mockito.anySet())).thenReturn(100L); + Mockito.when(tsoService.getCommitTSOAfterFence(Mockito.eq(CatalogTestUtil.testDbId1), + Mockito.eq(123533L), Mockito.anySet(), Mockito.eq(100L), Mockito.eq(200L))) + .thenReturn(201L); + setEnvTSOService(masterEnv, tsoService); + + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + TxnInfoPB txnInfo = TxnInfoPB.newBuilder() + .setDbId(CatalogTestUtil.testDbId1) + .addTableIds(CatalogTestUtil.testTableId1) + .setTxnId(123533L) + .setListenerId(-1) + .build(); + Mockito.when(proxy.commitTxn(Mockito.any())) + .thenReturn(CommitTxnResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.TXN_COMMIT_TSO_FENCED)) + .setTsoFence(200L) + .build(), + CommitTxnResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK)) + .setTxnInfo(txnInfo) + .build()); + try (MockedStatic mocked = Mockito.mockStatic(MetaServiceProxy.class)) { + mocked.when(MetaServiceProxy::getInstance).thenReturn(proxy); + masterTransMgr.commitTransactionWithoutLock(CatalogTestUtil.testDbId1, + Lists.newArrayList(table), 123533L, null, null); + } + + ArgumentCaptor requests = + ArgumentCaptor.forClass(Cloud.CommitTxnRequest.class); + Mockito.verify(proxy, Mockito.times(2)).commitTxn(requests.capture()); + Assertions.assertEquals(100L, requests.getAllValues().get(0).getCommitTso()); + Assertions.assertEquals(201L, requests.getAllValues().get(1).getCommitTso()); + } finally { + table.setBinlogConfig(originalBinlogConfig); + Config.enable_feature_binlog = originalEnableFeatureBinlog; + } + } + @Test public void testCommitTransactionCarriesTableStreamUpdates() throws Exception { MetaServiceProxy mockProxy = Mockito.mock(MetaServiceProxy.class); @@ -394,6 +474,12 @@ public void sendMakeCloudTmpRsVisibleTasks(long txnId, } } + private static void setEnvTSOService(Env env, TSOService service) throws Exception { + Field field = Env.class.getDeclaredField("tsoService"); + field.setAccessible(true); + field.set(env, service); + } + @Test public void testCommitTransactionAlreadyVisible() throws Exception { MetaServiceProxy mockProxy = Mockito.mock(MetaServiceProxy.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java index 62aa89c5589154..87afe145b8aa00 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java @@ -58,6 +58,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; /** @@ -67,6 +68,7 @@ public class TSOServiceTest { private TSOService tsoService; private Env env; + private GlobalTransactionMgrIface globalTxnMgr; private MockedStatic mockedEnv; private int originalMaxGetTSORetryCount; @@ -76,7 +78,7 @@ public class TSOServiceTest { private long originalClockBackwardThresholdMs; @BeforeEach - public void setUp() { + public void setUp() throws Exception { mockedEnv = Mockito.mockStatic(Env.class); originalMaxGetTSORetryCount = Config.tso_max_get_retry_count; @@ -92,7 +94,11 @@ public void setUp() { Config.tso_clock_backward_startup_threshold_ms = 30L * 60 * 1000; env = Mockito.mock(Env.class); + globalTxnMgr = Mockito.mock(GlobalTransactionMgrIface.class); mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getGlobalTransactionMgr()).thenReturn(globalTxnMgr); + Mockito.when(globalTxnMgr.advanceTsoFence(Mockito.anyLong())) + .thenAnswer(invocation -> invocation.getArgument(0)); tsoService = new TSOService(); } @@ -258,7 +264,8 @@ public void testRunAfterCatalogReadyUpdateFailureDoesNotTouchMetricWhenNotInit() MetricRepo.isInit = false; MetricRepo.COUNTER_TSO_CLOCK_UPDATE_FAILED = null; Mockito.when(env.isReady()).thenReturn(true); - Mockito.when(env.isMaster()).thenThrow(new RuntimeException("injected update failure")); + Mockito.when(env.isMaster()).thenReturn(true) + .thenThrow(new RuntimeException("injected update failure")); tsoService.runAfterCatalogReady(); } finally { Config.enable_feature_binlog = originalEnableFeatureBinlog; @@ -371,7 +378,7 @@ public void testCalibrateTimestampThrowsWhenPersistWriteFailsAndKeepNotInitializ TSOService.TSOStatusSnapshot statusSnapshot = tsoService.getStatusSnapshot(); Assertions.assertFalse(statusSnapshot.isInitialized()); - Assertions.assertTrue(statusSnapshot.getCurrentTso() > 0L); + Assertions.assertEquals(0L, statusSnapshot.getCurrentTso()); Assertions.assertEquals(0L, statusSnapshot.getWindowEndPhysicalTime()); try { @@ -412,6 +419,138 @@ public void testCalibrateTimestampResetsFatalClockBackwardReportedOnSuccess() th Assertions.assertFalse(getFatalClockBackwardReportedFlag(tsoService)); } + @Test + public void testCloudCalibrationPersistsWindowBeforePublishingFenceAndCatchesUp() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + mockPersistReady(); + AtomicBoolean firstRequest = new AtomicBoolean(true); + Mockito.when(globalTxnMgr.advanceTsoFence(Mockito.anyLong())).thenAnswer(invocation -> { + long proposed = invocation.getArgument(0); + Assertions.assertFalse(tsoService.getStatusSnapshot().isInitialized()); + Assertions.assertTrue(tsoService.getWindowEndTSO() + >= TSOTimestamp.extractPhysicalTime(proposed) + Config.tso_service_window_duration_ms); + if (firstRequest.getAndSet(false)) { + return TSOTimestamp.composePhysicalTimestamp(TSOTimestamp.extractPhysicalTime(proposed) + 10); + } + return proposed; + }); + + try (MockedStatic config = Mockito.mockStatic(Config.class)) { + config.when(Config::isCloudMode).thenReturn(true); + invokeCalibrateTimestamp(tsoService); + } + + Mockito.verify(globalTxnMgr, Mockito.times(2)).advanceTsoFence(Mockito.anyLong()); + Assertions.assertTrue(tsoService.getStatusSnapshot().isInitialized()); + } + + @Test + public void testCloudCalibrationRetriesThePersistedFenceAfterRpcFailure() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + mockPersistReady(); + AtomicBoolean failFirstRequest = new AtomicBoolean(true); + long[] proposedFence = {-1}; + Mockito.when(globalTxnMgr.advanceTsoFence(Mockito.anyLong())).thenAnswer(invocation -> { + long proposed = invocation.getArgument(0); + if (failFirstRequest.getAndSet(false)) { + proposedFence[0] = proposed; + throw new UserException("injected fence RPC failure"); + } + Assertions.assertEquals(proposedFence[0], proposed); + return proposed; + }); + + try (MockedStatic config = Mockito.mockStatic(Config.class)) { + config.when(Config::isCloudMode).thenReturn(true); + Assertions.assertThrows(InvocationTargetException.class, + () -> invokeCalibrateTimestamp(tsoService)); + invokeCalibrateTimestamp(tsoService); + } + + Mockito.verify(env.getEditLog(), Mockito.times(1)) + .logTSOTimestampWindowEnd(Mockito.any()); + Assertions.assertTrue(tsoService.getStatusSnapshot().isInitialized()); + } + + @Test + public void testFencedCommitRetryReplacesRegisteredTso() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + long rejectedTso = tsoService.getCommitTSO(1, 10, Collections.singleton(100L)); + long fenceTso = TSOTimestamp.composePhysicalTimestamp(101); + setGlobalTimestamp(tsoService, 101, 0); + + long replacementTso = tsoService.getCommitTSOAfterFence( + 1, 10, Set.of(100L, 200L), rejectedTso, fenceTso); + + Assertions.assertTrue(replacementTso > fenceTso); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); + Assertions.assertEquals(replacementTso, tracker.getOldestPendingTso()); + Assertions.assertEquals(1, tracker.getPendingCount()); + } + + @Test + public void testFencedCommitRetryRegistersTransactionOnNewMaster() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + setInitializedFlag(tsoService, true); + long fenceTso = TSOTimestamp.composePhysicalTimestamp(101); + setGlobalTimestamp(tsoService, 101, 0); + + long replacementTso = tsoService.getCommitTSOAfterFence( + 1, 10, Set.of(100L), TSOTimestamp.composePhysicalTimestamp(100), fenceTso); + + Assertions.assertTrue(replacementTso > fenceTso); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); + Assertions.assertEquals(replacementTso, tracker.getOldestPendingTso()); + Assertions.assertEquals(1, tracker.getPendingCount()); + } + + @Test + public void testFencedCommitRetryDoesNotAdvanceStaleMasterClock() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + long fenceTso = TSOTimestamp.composePhysicalTimestamp(101); + + RuntimeException failure = Assertions.assertThrows(RuntimeException.class, + () -> tsoService.getCommitTSOAfterFence(1, 10, Set.of(100L), + TSOTimestamp.composePhysicalTimestamp(100), fenceTso)); + + Assertions.assertTrue(failure.getMessage().contains("TXN_COMMIT_TSO_FENCED")); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + Assertions.assertEquals(0, + ((TSOTransactionTracker) trackerField.get(tsoService)).getPendingCount()); + } + + @Test + public void testDemotionInvalidatesAllocationState() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + tsoService.getCommitTSO(1, 10, Collections.singleton(100L)); + Mockito.when(env.isMaster()).thenReturn(false); + + tsoService.runAfterCatalogReady(); + + Assertions.assertFalse(tsoService.getStatusSnapshot().isInitialized()); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + Assertions.assertEquals(0, + ((TSOTransactionTracker) trackerField.get(tsoService)).getPendingCount()); + } + @Test public void testRunAfterCatalogReadySkipsWhenBinlogDisabled() throws Exception { Config.enable_feature_binlog = false; @@ -468,7 +607,7 @@ public void testReadableWindowDoesNotRequireAnotherCommittedTsoFlush() throws Ex TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + Mockito.when(txnMgr.getTsoRecoveryTransactions(Mockito.eq(1000L), Mockito.anyLong(), Mockito.eq(ByteString.EMPTY))) .thenReturn(TSOTransactionTrackerTest.recoveryBatch(ByteString.EMPTY)); tracker.checkTransactions(txnMgr, Long.MAX_VALUE); try (MockedStatic config = Mockito.mockStatic(Config.class)) { @@ -491,7 +630,7 @@ private void prepareWindowRead(boolean finishRecovery) throws Exception { field.setAccessible(true); GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + Mockito.when(txnMgr.getTsoRecoveryTransactions(Mockito.eq(1000L), Mockito.anyLong(), Mockito.eq(ByteString.EMPTY))) .thenReturn(TSOTransactionTrackerTest.recoveryBatch(ByteString.EMPTY)); ((TSOTransactionTracker) field.get(tsoService)).checkTransactions(txnMgr, Long.MAX_VALUE); } @@ -567,7 +706,7 @@ public void testCommittedPrefixIsPublishedOnlyAfterJournalSuccess() throws Excep TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + Mockito.when(txnMgr.getTsoRecoveryTransactions(Mockito.eq(1000L), Mockito.anyLong(), Mockito.eq(ByteString.EMPTY))) .thenReturn(TSOTransactionTrackerTest.recoveryBatch(ByteString.EMPTY)); tracker.checkTransactions(txnMgr, Long.MAX_VALUE); try (MockedStatic config = Mockito.mockStatic(Config.class)) { @@ -605,11 +744,9 @@ public void testRecoveryFreezesPrefixAndPeriodicFlushWorksWithUnchangedWindow() TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + Mockito.when(txnMgr.getTsoRecoveryTransactions(Mockito.eq(1000L), Mockito.anyLong(), Mockito.eq(ByteString.EMPTY))) .thenReturn(TSOTransactionTrackerTest.recoveryBatch(ByteString.EMPTY)); - long afterRecoveryDelay = System.nanoTime() - + TimeUnit.MILLISECONDS.toNanos(Config.tso_service_window_duration_ms + 1001L); - tracker.checkTransactions(txnMgr, afterRecoveryDelay); + tracker.checkTransactions(txnMgr, System.nanoTime()); long reservedWindow = tsoService.getWindowEndTSO(); Field lastPersist = TSOService.class.getDeclaredField("lastPersistNanos"); lastPersist.setAccessible(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java index 9fff60d699375d..bf6bdf77d9f116 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java @@ -43,14 +43,15 @@ import java.util.concurrent.locks.ReentrantLock; public class TSOTransactionTrackerTest { + static final long RECOVERY_FENCE_TSO = 1000L; private final ReentrantLock lock = new ReentrantLock(); private final TSOTransactionTracker tracker = new TSOTransactionTracker(lock); private final GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); - private void reset(long delayMs) { + private void reset() { lock.lock(); try { - tracker.reset(0, delayMs); + tracker.reset(RECOVERY_FENCE_TSO); } finally { lock.unlock(); } @@ -81,12 +82,12 @@ static GetTsoRecoveryTransactionsResponse recoveryBatch(ByteString nextKey, TxnI private static TxnInfoPB recoveryTxn(long dbId, long txnId, long tso, long... tables) { TxnInfoPB.Builder info = TxnInfoPB.newBuilder().setDbId(dbId).setTxnId(txnId) - .setStatus(TxnStatusPB.TXN_STATUS_PREPARED); + .setStatus(TxnStatusPB.TXN_STATUS_COMMITTED); for (long table : tables) { info.addTableIds(table); } if (tso > 0) { - info.setCommitTso(tso).setStatus(TxnStatusPB.TXN_STATUS_COMMITTED); + info.setCommitTso(tso); } return info.build(); } @@ -94,7 +95,7 @@ private static TxnInfoPB recoveryTxn(long dbId, long txnId, long tso, long... ta private void finishRecovery() throws Exception { Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); Mockito.doReturn(recoveryBatch(ByteString.EMPTY)).when(txnMgr) - .getTsoRecoveryTransactions(1000L, ByteString.EMPTY); + .getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY); tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); } @@ -110,7 +111,7 @@ private TSOTransactionTracker.WaitResult awaitTable(long dbId, long tableId, lon @Test public void testReadWaitFiltersDatabaseTableAndExclusivePhysicalEnd() throws Exception { - reset(0); + reset(); finishRecovery(); register(1, 10, TSOTimestamp.composeTimestamp(100, 1)); long end = TSOTimestamp.composePhysicalTimestamp(101); @@ -134,23 +135,22 @@ public void testReadWaitFiltersDatabaseTableAndExclusivePhysicalEnd() throws Exc @Test public void testEmptyRegistrationSetCannotBypassRecovery() throws Exception { - reset(2000); + reset(); Assertions.assertEquals(TSOTransactionTracker.WaitResult.RECOVERING, awaitTable(1, 100, 200)); } @Test public void testLoadedRecoveryWaitsOnlyRelatedTablesAndKeepsPrefixFrozen() throws Exception { - reset(2000); + reset(); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) .thenReturn(recoveryBatch(ByteString.EMPTY, - recoveryTxn(1, 10, 100, 100), recoveryTxn(1, 20, 0, 200))); + recoveryTxn(1, 10, 100, 100), recoveryTxn(1, 20, 80, 200))); tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(2)); Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 300, 200)); Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(2, 100, 200)); Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 100, 90)); Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, 200)); - // No persisted TSO is not proof that an old in-flight commit lies outside this window. Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 200, 90)); Assertions.assertEquals(80, candidate(250, 80)); tracker.transactionFinished(1, 10); @@ -163,14 +163,14 @@ public void testLoadedRecoveryWaitsOnlyRelatedTablesAndKeepsPrefixFrozen() throw @Test public void testFailedBatchResumesWithoutOpeningAnIncompleteRecovery() throws Exception { - reset(0); + reset(); ByteString nextKey = ByteString.copyFromUtf8("next batch"); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) .thenReturn(recoveryBatch(nextKey, recoveryTxn(1, 10, 100, 100))); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, nextKey)) + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, nextKey)) .thenThrow(new UserException("batch RPC failed")) - .thenReturn(recoveryBatch(ByteString.EMPTY, recoveryTxn(2, 20, 0, 200))); + .thenReturn(recoveryBatch(ByteString.EMPTY, recoveryTxn(2, 20, 80, 200))); Assertions.assertThrows(UserException.class, () -> tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3))); Assertions.assertEquals(TSOTransactionTracker.WaitResult.RECOVERING, awaitTable(1, 300, 200)); @@ -180,24 +180,24 @@ public void testFailedBatchResumesWithoutOpeningAnIncompleteRecovery() throws Ex Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(2, 200, 200)); Assertions.assertEquals(80, candidate(250, 80)); Mockito.verify(txnMgr).getTransactionIdWatermark(); - Mockito.verify(txnMgr).getTsoRecoveryTransactions(1000L, ByteString.EMPTY); + Mockito.verify(txnMgr).getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY); } @Test public void testRecoveryBatchesMergeConcurrentRegistrationsAndFinishNotifications() throws Exception { - reset(0); + reset(); ByteString nextKey = ByteString.copyFromUtf8("next batch"); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)).thenAnswer(invocation -> { + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)).thenAnswer(invocation -> { Assertions.assertFalse(lock.isHeldByCurrentThread()); register(1, 10, 200); // Same old transaction is retried through the new master. - return recoveryBatch(nextKey, recoveryTxn(1, 10, 0, 200)); + return recoveryBatch(nextKey, recoveryTxn(1, 10, 80, 200)); }); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, nextKey)).thenAnswer(invocation -> { + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, nextKey)).thenAnswer(invocation -> { Assertions.assertFalse(lock.isHeldByCurrentThread()); tracker.transactionFinished(1, 10); register(2, 2000, 150); // New transactions must survive importing the old list. - return recoveryBatch(ByteString.EMPTY, recoveryTxn(1, 20, 0, 300)); + return recoveryBatch(ByteString.EMPTY, recoveryTxn(1, 20, 80, 300)); }); tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 100, 300)); @@ -209,11 +209,11 @@ public void testRecoveryBatchesMergeConcurrentRegistrationsAndFinishNotification } @Test - public void testRecoveredUnknownTsoRetainsTablesAcrossLocalRetry() throws Exception { - reset(0); + public void testRecoveredTransactionRetainsTablesAcrossLocalRetry() throws Exception { + reset(); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) - .thenReturn(recoveryBatch(ByteString.EMPTY, recoveryTxn(1, 10, 0, 200))); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) + .thenReturn(recoveryBatch(ByteString.EMPTY, recoveryTxn(1, 10, 80, 200))); tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); register(1, 10, 300); Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, 200)); @@ -225,13 +225,13 @@ public void testRecoveredUnknownTsoRetainsTablesAcrossLocalRetry() throws Except @Test public void testRecoveredTransactionsAreReconciledInBoundedRotatingBatches() throws Exception { - reset(0); + reset(); TxnInfoPB[] transactions = new TxnInfoPB[150]; for (int i = 0; i < transactions.length; i++) { - transactions[i] = recoveryTxn(1, i + 1, 0, 100); + transactions[i] = recoveryTxn(1, i + 1, 80, 100); } Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) .thenReturn(recoveryBatch(ByteString.EMPTY, transactions)); tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); @@ -255,11 +255,11 @@ public void testRecoveredTransactionsAreReconciledInBoundedRotatingBatches() thr @Test public void testRecoveredTransactionCompletionWakesOnlyItsWaiters() throws Exception { - reset(0); + reset(); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) .thenReturn(recoveryBatch(ByteString.EMPTY, - recoveryTxn(1, 10, 0, 100), recoveryTxn(2, 20, 0, 100))); + recoveryTxn(1, 10, 80, 100), recoveryTxn(2, 20, 80, 100))); tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); CountDownLatch started = new CountDownLatch(1); ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -293,7 +293,7 @@ private Future submitReadWait(ExecutorService @Test public void testReconciliationWakesReadWithoutAdvancingTheGlobalPrefix() throws Exception { - reset(0); + reset(); finishRecovery(); register(1, 10, 100); register(2, 20, 90); // An unrelated database continues to hold the global prefix. @@ -315,7 +315,7 @@ public void testReconciliationWakesReadWithoutAdvancingTheGlobalPrefix() throws @Test public void testWaitReleasesAllocatorLockAndDoesNotFollowLaterWrites() throws Exception { - reset(0); + reset(); finishRecovery(); register(1, 10, 100); CountDownLatch started = new CountDownLatch(1); @@ -335,7 +335,7 @@ public void testWaitReleasesAllocatorLockAndDoesNotFollowLaterWrites() throws Ex @Test public void testResetInvalidatesAnInFlightReadWait() throws Exception { - reset(0); + reset(); finishRecovery(); register(1, 10, 100); CountDownLatch started = new CountDownLatch(1); @@ -343,7 +343,7 @@ public void testResetInvalidatesAnInFlightReadWait() throws Exception { try { Future waiting = submitReadWait(executor, started); Assertions.assertTrue(started.await(30, TimeUnit.SECONDS)); - reset(2000); + reset(); Assertions.assertEquals(TSOTransactionTracker.WaitResult.RECOVERING, waiting.get(30, TimeUnit.SECONDS)); } finally { executor.shutdownNow(); @@ -352,7 +352,7 @@ public void testResetInvalidatesAnInFlightReadWait() throws Exception { @Test public void testOutOfOrderVisibilityAndRetryRetainEarliestTso() throws Exception { - reset(2000); + reset(); finishRecovery(); register(1, 10, 100); register(1, 20, 120); @@ -366,16 +366,44 @@ public void testOutOfOrderVisibilityAndRetryRetainEarliestTso() throws Exception } @Test - public void testRecoveryCapturesFixedWatermarkAfterDelayAndPreservesNewPending() throws Exception { - reset(2000); + public void testFencedRetryReplacesRejectedTso() throws Exception { + reset(); + finishRecovery(); + register(1, 10, 100); + Assertions.assertEquals(99, candidate(250, 80)); + lock.lock(); + try { + tracker.replaceFenced(Pair.of(1L, 10L), 100, 200, 201, 0, Set.of(100L, 200L)); + } finally { + lock.unlock(); + } + Assertions.assertEquals(200, candidate(250, 99)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 100, 150)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 200, 250)); + } + + @Test + public void testFencedRetryRegistersTransactionRecoveredOnNewMaster() throws Exception { + reset(); + finishRecovery(); + lock.lock(); + try { + tracker.replaceFenced(Pair.of(1L, 10L), 100, 200, 201, 0, Set.of(100L)); + } finally { + lock.unlock(); + } + Assertions.assertEquals(200, candidate(250, 99)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, 250)); + } + + @Test + public void testRecoveryStartsImmediatelyAndPreservesNewPending() throws Exception { + reset(); register(2, 2001, 200); - tracker.checkTransactions(txnMgr, TimeUnit.MILLISECONDS.toNanos(1999)); - Mockito.verify(txnMgr, Mockito.never()).getTransactionIdWatermark(); - Assertions.assertEquals(80, candidate(250, 80)); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L, 2000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) - .thenReturn(recoveryBatch(ByteString.EMPTY, recoveryTxn(1, 10, 0, 100))); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(2)); + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) + .thenReturn(recoveryBatch(ByteString.EMPTY, recoveryTxn(1, 10, 80, 100))); + tracker.checkTransactions(txnMgr, 0); Assertions.assertEquals(80, candidate(250, 80)); TransactionState aborted = Mockito.mock(TransactionState.class); Mockito.when(aborted.getTransactionStatus()).thenReturn(TransactionStatus.ABORTED); @@ -383,12 +411,12 @@ public void testRecoveryCapturesFixedWatermarkAfterDelayAndPreservesNewPending() tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); Assertions.assertEquals(199, candidate(250, 80)); Mockito.verify(txnMgr).getTransactionIdWatermark(); - Mockito.verify(txnMgr).getTsoRecoveryTransactions(1000L, ByteString.EMPTY); + Mockito.verify(txnMgr).getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY); } @Test public void testOnlyRealTerminalStatesReleasePending() throws Exception { - reset(2000); + reset(); finishRecovery(); for (TransactionStatus status : TransactionStatus.values()) { register(1, 10, 100); @@ -403,10 +431,10 @@ public void testOnlyRealTerminalStatesReleasePending() throws Exception { @Test public void testMissingTransactionAndFailedRecoveryNeverAdvance() throws Exception { - reset(0); + reset(); register(1, 10, 100); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)) + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) .thenThrow(new UserException("old MS has no recovery RPC")); Assertions.assertThrows(UserException.class, () -> tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3))); @@ -418,14 +446,14 @@ public void testMissingTransactionAndFailedRecoveryNeverAdvance() throws Excepti @Test public void testRpcDoesNotHoldAllocatorLockAndOldResultCannotRemoveNewRegistration() throws Exception { - reset(2000); + reset(); finishRecovery(); register(1, 10, 100); TransactionState visible = Mockito.mock(TransactionState.class); Mockito.when(visible.getTransactionStatus()).thenReturn(TransactionStatus.VISIBLE); Mockito.when(txnMgr.getTransactionState(1, 10)).thenAnswer(invocation -> { Assertions.assertFalse(lock.isHeldByCurrentThread()); - reset(2000); // Simulate reinitialization while an old reconciliation request is in flight. + reset(); // Simulate reinitialization while an old reconciliation request is in flight. register(1, 10, 200); return visible; }); @@ -437,11 +465,11 @@ public void testRpcDoesNotHoldAllocatorLockAndOldResultCannotRemoveNewRegistrati @Test public void testOldRecoveryResultCannotOpenNewRecovery() throws Exception { - reset(0); + reset(); Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, ByteString.EMPTY)).thenAnswer(invocation -> { + Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)).thenAnswer(invocation -> { Assertions.assertFalse(lock.isHeldByCurrentThread()); - reset(2000); + reset(); return recoveryBatch(ByteString.EMPTY); }); tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); @@ -451,7 +479,7 @@ public void testOldRecoveryResultCannotOpenNewRecovery() throws Exception { @Test public void testReconciliationBatchIsBoundedAndRotatesPastOldest() throws Exception { - reset(0); + reset(); finishRecovery(); for (int i = 1; i <= 150; i++) { register(1, i, 1000 + i); diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 62e94a9b75a198..cecbd0175099c4 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -514,6 +514,11 @@ message TxnIndexPB { optional int64 parent_txn_id = 2; } +// Current-state FE master epoch fence. This key is intentionally not versioned. +message TxnTsoFencePB { + optional int64 fence_tso = 1; +} + message TxnInfoPB { optional int64 db_id = 1; repeated int64 table_ids = 2; @@ -1115,6 +1120,8 @@ message CommitTxnResponse { // The lazy commit has only completed its first phase. FE must not notify BE to make // temporary rowsets visible until BE observes the final metadata from meta-service. optional bool is_lazy_commit_incomplete = 9; + // The current master fence when commit_tso is rejected as stale. + optional int64 tso_fence = 10; } message AbortTxnRequest { @@ -1264,6 +1271,8 @@ message GetTsoRecoveryTransactionsRequest { optional int32 batch_size = 3; optional bytes start_key = 4; optional string request_ip = 5; + // Only committed TSO transactions at or below this inclusive fence are recovered. + optional int64 tso_fence = 6; } message GetTsoRecoveryTransactionsResponse { @@ -1274,6 +1283,18 @@ message GetTsoRecoveryTransactionsResponse { optional bytes next_start_key = 3; } +message AdvanceTsoFenceRequest { + optional string cloud_unique_id = 1; // For auth + // Inclusive upper bound rejected by transaction commit. + optional int64 proposed_fence_tso = 2; + optional string request_ip = 3; +} + +message AdvanceTsoFenceResponse { + optional MetaServiceResponseStatus status = 1; + optional int64 tso_fence = 2; +} + message CleanTxnLabelRequest { optional string cloud_unique_id = 1; // For auth optional int64 db_id = 2; @@ -1967,6 +1988,8 @@ enum MetaServiceCode { STALE_TABLET_CACHE = 2012; STALE_PREPARE_ROWSET = 2013; TXN_ALREADY_COMMITED = 2014; + // The transaction commit TSO belongs to an earlier FE master epoch. + TXN_COMMIT_TSO_FENCED = 2015; CLUSTER_NOT_FOUND = 3001; ALREADY_EXISTED = 3002; @@ -2427,6 +2450,7 @@ service MetaService { rpc create_meta_sync_point(CreateMetaSyncPointRequest) returns (CreateMetaSyncPointResponse); rpc check_txn_conflict(CheckTxnConflictRequest) returns (CheckTxnConflictResponse); rpc get_tso_recovery_transactions(GetTsoRecoveryTransactionsRequest) returns (GetTsoRecoveryTransactionsResponse); + rpc advance_tso_fence(AdvanceTsoFenceRequest) returns (AdvanceTsoFenceResponse); rpc clean_txn_label(CleanTxnLabelRequest) returns (CleanTxnLabelResponse); rpc get_txn_id(GetTxnIdRequest) returns (GetTxnIdResponse); rpc begin_sub_txn(BeginSubTxnRequest) returns (BeginSubTxnResponse); From e53291d196b273930ddb0af217f016a2e42dde1d Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Mon, 14 Sep 2026 20:45:44 +0800 Subject: [PATCH 09/13] [fix](binlog) Fence uncertain commit TSO attempts ### What problem does this PR solve? Issue Number: None Related PR: #67181, #67594 Problem Summary: Incremental reads track TSO-bearing commit attempts in FE memory. A commit RPC can finish without a conclusive response, leaving the attempt in the tracker indefinitely, while scanning Meta Service transactions during startup and periodic reconciliation adds complexity and can block unrelated reads. Fence an uncertain attempt in Meta Service before removing its exact TSO from the tracker, release definite responses immediately, and let incremental scans wait for pending lazy commits on their selected partitions. Disable transport retries for TSO-bearing commit RPCs so every uncertain delivery is fenced. Startup calibration advances the same per-instance fence, so stale commits from an old master are rejected without a recovery scan. ### Release note Bounded incremental reads now fence uncertain commit TSO attempts and no longer require startup or periodic transaction recovery scans. ### Check List (For Author) - Test: Unit Test - FE unit tests for TSO tracking, commit response handling, Meta Service retry behavior, and incremental scan version waits - Cloud unit tests for TSO fence monotonicity, fence-key encoding, and response-code compatibility - ./build.sh -j32 - ./build.sh --cloud -j32 - Cloud clang-tidy and build hygiene checks - Behavior changed: Yes. Uncertain TSO commit attempts are fenced before release, definite failures are released directly, and incremental scans wait for selected-partition pending lazy commits. - Does this need documentation: Yes. Connector-facing error-code documentation is included in the PR description. --- cloud/src/common/bvars.cpp | 3 - cloud/src/common/bvars.h | 3 - cloud/src/meta-service/meta_service.h | 13 - cloud/src/meta-service/meta_service_txn.cpp | 102 ---- cloud/test/meta_service_test.cpp | 229 --------- cloud/test/txn_lazy_commit_test.cpp | 15 - .../doris/cloud/rpc/MetaServiceClient.java | 6 - .../doris/cloud/rpc/MetaServiceProxy.java | 26 +- .../CloudGlobalTransactionMgr.java | 75 ++- .../org/apache/doris/metric/MetricRepo.java | 6 - .../org/apache/doris/planner/ScanNode.java | 2 +- .../GlobalTransactionMgrIface.java | 9 - .../java/org/apache/doris/tso/TSOService.java | 63 +-- .../doris/tso/TSOTransactionTracker.java | 228 +-------- .../doris/cloud/rpc/MetaServiceProxyTest.java | 19 + .../transaction/CloudCommittedTsoTest.java | 65 ++- .../CloudGlobalTransactionMgrTest.java | 44 +- .../doris/planner/OlapScanNodeTest.java | 4 +- .../qe/TimeBasedChangeVisibleWaiterTest.java | 2 +- .../org/apache/doris/tso/TSOServiceTest.java | 87 ++-- .../doris/tso/TSOTransactionTrackerTest.java | 470 +++--------------- gensrc/proto/cloud.proto | 25 +- 22 files changed, 268 insertions(+), 1228 deletions(-) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index d80f935777055e..487ada73d3dde0 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -43,7 +43,6 @@ BvarLatencyRecorderWithTag g_bvar_ms_create_meta_sync_point("ms", "create_meta_s BvarLatencyRecorderWithTag g_bvar_ms_begin_sub_txn("ms", "begin_sub_txn"); BvarLatencyRecorderWithTag g_bvar_ms_abort_sub_txn("ms", "abort_sub_txn"); BvarLatencyRecorderWithTag g_bvar_ms_check_txn_conflict("ms", "check_txn_conflict"); -BvarLatencyRecorderWithTag g_bvar_ms_get_tso_recovery_transactions("ms", "get_tso_recovery_transactions"); BvarLatencyRecorderWithTag g_bvar_ms_advance_tso_fence("ms", "advance_tso_fence"); BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator("ms", "abort_txn_with_coordinator"); BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator("ms", "get_prepare_txn_by_coordinator"); @@ -502,7 +501,6 @@ mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter("rpc_kv_abo mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_counter("rpc_kv_get_prepare_txn_by_coordinator_get_counter",{"instance_id"}); // check_txn_conflict mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter("rpc_kv_check_txn_conflict_get_counter",{"instance_id"}); -mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_counter("rpc_kv_get_tso_recovery_transactions_get_counter",{"instance_id"}); mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_counter("rpc_kv_advance_tso_fence_get_counter",{"instance_id"}); mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_counter("rpc_kv_advance_tso_fence_put_counter",{"instance_id"}); // clean_txn_label @@ -715,7 +713,6 @@ mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes("rpc_kv_abort mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_bytes("rpc_kv_get_prepare_txn_by_coordinator_get_bytes",{"instance_id"}); // check_txn_conflict mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes("rpc_kv_check_txn_conflict_get_bytes",{"instance_id"}); -mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_bytes("rpc_kv_get_tso_recovery_transactions_get_bytes",{"instance_id"}); mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_bytes("rpc_kv_advance_tso_fence_get_bytes",{"instance_id"}); mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_bytes("rpc_kv_advance_tso_fence_put_bytes",{"instance_id"}); // clean_txn_label diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index 32203c3ed46262..6eb1da13a70af6 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -553,7 +553,6 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_get_txn; extern BvarLatencyRecorderWithTag g_bvar_ms_get_current_max_txn_id; extern BvarLatencyRecorderWithTag g_bvar_ms_create_meta_sync_point; extern BvarLatencyRecorderWithTag g_bvar_ms_check_txn_conflict; -extern BvarLatencyRecorderWithTag g_bvar_ms_get_tso_recovery_transactions; extern BvarLatencyRecorderWithTag g_bvar_ms_advance_tso_fence; extern BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator; extern BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator; @@ -909,7 +908,6 @@ extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_counter; extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter; -extern mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter; @@ -1055,7 +1053,6 @@ extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_get_prepare_txn_by_coordinator_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes; -extern mBvarInt64Adder g_bvar_rpc_kv_get_tso_recovery_transactions_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_get_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_advance_tso_fence_put_bytes; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_bytes; diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index c47a5786861968..7023ccf56f6286 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -145,11 +145,6 @@ class MetaServiceImpl : public cloud::MetaService { CheckTxnConflictResponse* response, ::google::protobuf::Closure* done) override; - void get_tso_recovery_transactions(::google::protobuf::RpcController* controller, - const GetTsoRecoveryTransactionsRequest* request, - GetTsoRecoveryTransactionsResponse* response, - ::google::protobuf::Closure* done) override; - void advance_tso_fence(::google::protobuf::RpcController* controller, const AdvanceTsoFenceRequest* request, AdvanceTsoFenceResponse* response, ::google::protobuf::Closure* done) override; @@ -628,14 +623,6 @@ class MetaServiceProxy final : public MetaService { call_impl(&cloud::MetaService::check_txn_conflict, controller, request, response, done); } - void get_tso_recovery_transactions(::google::protobuf::RpcController* controller, - const GetTsoRecoveryTransactionsRequest* request, - GetTsoRecoveryTransactionsResponse* response, - ::google::protobuf::Closure* done) override { - call_impl(&cloud::MetaService::get_tso_recovery_transactions, controller, request, response, - done); - } - void advance_tso_fence(::google::protobuf::RpcController* controller, const AdvanceTsoFenceRequest* request, AdvanceTsoFenceResponse* response, ::google::protobuf::Closure* done) override { diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 5cd8a3915b3d0d..abd1ea4600f7fb 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -4835,108 +4835,6 @@ void MetaServiceImpl::advance_tso_fence(::google::protobuf::RpcController* contr response->set_tso_fence(effective_fence_tso); } -void MetaServiceImpl::get_tso_recovery_transactions( - ::google::protobuf::RpcController* controller, - const GetTsoRecoveryTransactionsRequest* request, - GetTsoRecoveryTransactionsResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_tso_recovery_transactions, get); - if (request->end_txn_id() <= 0 || request->batch_size() <= 0 || request->batch_size() > 1000 || - request->tso_fence() <= 0) { - code = MetaServiceCode::INVALID_ARGUMENT; - msg = "invalid TSO recovery transaction bound or batch size"; - return; - } - instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); - if (instance_id.empty()) { - code = MetaServiceCode::INVALID_ARGUMENT; - msg = "cannot find instance_id for TSO recovery"; - return; - } - RPC_RATE_LIMIT(get_tso_recovery_transactions) - // Keys sort by database first. Scan the instance and apply the fixed exclusive ID bound - // to each key; a batch containing only newer transactions does not finish the scan. - std::string begin_key = txn_running_key({instance_id, 0, 0}); - std::string end_key = txn_running_key({instance_id, INT64_MAX, INT64_MAX}); - end_key.push_back('\x00'); - if (!request->start_key().empty()) { - if (request->start_key() < begin_key || request->start_key() >= end_key) { - code = MetaServiceCode::INVALID_ARGUMENT; - msg = "invalid TSO recovery start key"; - return; - } - begin_key = request->start_key(); - } - TxnErrorCode err = txn_kv_->create_txn(&txn); - if (err != TxnErrorCode::TXN_OK) { - code = cast_as(err); - msg = "failed to create TSO recovery read transaction"; - return; - } - std::unique_ptr it; - err = txn->get(begin_key, end_key, &it, true, request->batch_size()); - if (err != TxnErrorCode::TXN_OK) { - code = cast_as(err); - msg = "failed to get running transactions during TSO recovery"; - return; - } - while (it->has_next()) { - auto [key, value] = it->next(); - if (!it->has_next()) { - begin_key = key; - } - std::string_view encoded_key = key; - encoded_key.remove_prefix(1); - std::vector, int, int>> fields; - if (decode_key(&encoded_key, &fields) != 0 || fields.size() != 5 || - !std::holds_alternative(std::get<0>(fields[3])) || - !std::holds_alternative(std::get<0>(fields[4]))) { - code = MetaServiceCode::UNDEFINED_ERR; - msg = "failed to decode running transaction key during TSO recovery"; - return; - } - const auto db_id = std::get(std::get<0>(fields[3])); - const auto txn_id = std::get(std::get<0>(fields[4])); - if (txn_id >= request->end_txn_id()) { - continue; - } - // Running keys are removed atomically with real VISIBLE/ABORTED. Expired COMMITTED - // lazy transactions still block. Read the details in the same KV snapshot. - std::string info_val; - err = txn->get(txn_info_key({instance_id, db_id, txn_id}), &info_val, true); - if (err != TxnErrorCode::TXN_OK) { - code = cast_as(err); - msg = fmt::format("failed to read TSO recovery transaction, db_id={}, txn_id={}", db_id, - txn_id); - return; - } - TxnInfoPB info; - if (!info.ParseFromString(info_val)) { - code = MetaServiceCode::PROTOBUF_PARSE_ERR; - msg = "failed to parse TSO recovery transaction"; - return; - } - // Subtransactions update txn_info.table_ids; the running record can be older. - if (info.db_id() != db_id || info.txn_id() != txn_id || info.table_ids().empty()) { - code = MetaServiceCode::UNDEFINED_ERR; - msg = "invalid TSO recovery transaction identity or tables"; - return; - } - if (info.status() != TxnStatusPB::TXN_STATUS_COMMITTED || !info.has_commit_tso() || - info.commit_tso() <= 0 || info.commit_tso() > request->tso_fence()) { - continue; - } - // Recovery needs identities and visibility boundaries, not commit attachments. - auto* recovered = response->add_txn_infos(); - recovered->set_db_id(db_id); - recovered->set_txn_id(txn_id); - recovered->mutable_table_ids()->CopyFrom(info.table_ids()); - recovered->set_status(info.status()); - recovered->set_commit_tso(info.commit_tso()); - } - begin_key.push_back('\x00'); - response->set_next_start_key(it->more() ? begin_key : ""); -} - void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* controller, const CheckTxnConflictRequest* request, CheckTxnConflictResponse* response, diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index 08cd9142d7b1aa..cf54b35ba502fb 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -2836,235 +2836,6 @@ TEST(MetaServiceTest, TsoFenceIsMonotonicAndRejectsStaleCommit) { ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); } -TEST(MetaServiceTest, TsoRecoveryChecksAllDatabasesAndExpiredLazyTransactions) { - auto meta_service = get_meta_service(); - std::unique_ptr txn; - ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); - TxnRunningPB running; - running.set_timeout_time(1); // Expired is not terminal for a persisted COMMITTED lazy txn. - running.add_table_ids(777); - txn->put(txn_running_key({mock_instance, 1, 100}), running.SerializeAsString()); - txn->put(txn_running_key({mock_instance, 2, 150}), running.SerializeAsString()); - txn->put(txn_running_key({"another_instance", 1, 1}), running.SerializeAsString()); - const auto blocking_key = txn_running_key({mock_instance, 999, 50}); - txn->put(blocking_key, running.SerializeAsString()); - TxnInfoPB info; - info.set_db_id(999); - info.set_txn_id(50); - info.add_table_ids(777); - info.set_status(TxnStatusPB::TXN_STATUS_COMMITTED); - info.set_commit_tso(12345); - const auto info_key = txn_info_key({mock_instance, 999, 50}); - txn->put(info_key, info.SerializeAsString()); - for (const auto& [txn_id, status, commit_tso] : - std::vector> { - {30, TxnStatusPB::TXN_STATUS_COMMITTED, 20001}, - {35, TxnStatusPB::TXN_STATUS_COMMITTED, 0}, - {40, TxnStatusPB::TXN_STATUS_PREPARED, 0}}) { - txn->put(txn_running_key({mock_instance, 999, txn_id}), running.SerializeAsString()); - TxnInfoPB additional_info = info; - additional_info.set_txn_id(txn_id); - additional_info.set_status(status); - if (commit_tso > 0) { - additional_info.set_commit_tso(commit_tso); - } else { - additional_info.clear_commit_tso(); - } - txn->put(txn_info_key({mock_instance, 999, txn_id}), additional_info.SerializeAsString()); - } - ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); - - brpc::Controller cntl; - GetTsoRecoveryTransactionsRequest request; - request.set_cloud_unique_id("test_cloud_unique_id"); - request.set_end_txn_id(100); - request.set_batch_size(256); - request.set_tso_fence(20000); - GetTsoRecoveryTransactionsResponse response; - meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); - ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - ASSERT_EQ(response.txn_infos_size(), 1); - - // The legacy table-scoped check still skips expired transactions; its result cannot recover TSO. - CheckTxnConflictRequest legacy; - legacy.set_cloud_unique_id("test_cloud_unique_id"); - legacy.set_end_txn_id(100); - legacy.set_db_id(999); - legacy.add_table_ids(777); - CheckTxnConflictResponse legacy_response; - meta_service->check_txn_conflict(&cntl, &legacy, &legacy_response, nullptr); - ASSERT_EQ(legacy_response.status().code(), MetaServiceCode::OK); - ASSERT_TRUE(legacy_response.finished()); - - // Real publication removes the running key in the same KV transaction as the terminal state. - ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); - info.set_status(TxnStatusPB::TXN_STATUS_VISIBLE); - txn->put(info_key, info.SerializeAsString()); - txn->remove(blocking_key); - ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); - response.Clear(); - meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); - ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - // IDs equal to or above the fixed exclusive bound are ignored. - ASSERT_EQ(response.txn_infos_size(), 0); - ASSERT_TRUE(response.has_next_start_key()); - ASSERT_TRUE(response.next_start_key().empty()); -} - -TEST(MetaServiceTest, TsoRecoveryRejectsMalformedRunningKeys) { - auto meta_service = get_meta_service(); - std::unique_ptr txn; - ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); - std::string key = txn_running_key({mock_instance, 1, 1}); - key.push_back('\xff'); - txn->put(key, ""); - ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); - brpc::Controller cntl; - GetTsoRecoveryTransactionsRequest request; - request.set_cloud_unique_id("test_cloud_unique_id"); - request.set_end_txn_id(100); - request.set_batch_size(256); - request.set_tso_fence(20000); - GetTsoRecoveryTransactionsResponse response; - meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); - ASSERT_NE(response.status().code(), MetaServiceCode::OK); - ASSERT_FALSE(response.has_next_start_key()); -} - -TEST(MetaServiceTest, TsoRecoveryBatchesUseCurrentTablesAndKeepExpiredTransactions) { - auto meta_service = get_meta_service(); - std::unique_ptr txn; - ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); - TxnRunningPB running; - running.set_timeout_time(1); - running.add_table_ids(777); - auto put_transaction = [&](const std::string& instance, int64_t db, int64_t id, int64_t table) { - txn->put(txn_running_key({instance, db, id}), running.SerializeAsString()); - TxnInfoPB info; - info.set_db_id(db); - info.set_txn_id(id); - info.add_table_ids(table); - info.set_status(TxnStatusPB::TXN_STATUS_COMMITTED); - info.set_commit_tso(12345); - txn->put(txn_info_key({instance, db, id}), info.SerializeAsString()); - }; - put_transaction(mock_instance, 1, 99, 100); - put_transaction(mock_instance, 1, 100, - 200); // Equal to the exclusive bound, still consumes a scan slot. - put_transaction(mock_instance, 2, 10, 300); // txn_info includes tables added after begin_txn. - put_transaction("another_instance", 1, 1, 400); - ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); - - brpc::Controller cntl; - GetTsoRecoveryTransactionsRequest request; - request.set_cloud_unique_id("test_cloud_unique_id"); - request.set_end_txn_id(100); - request.set_batch_size(2); - request.set_tso_fence(20000); - GetTsoRecoveryTransactionsResponse response; - meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); - ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - ASSERT_FALSE(response.next_start_key().empty()); - ASSERT_EQ(response.txn_infos_size(), 1); - EXPECT_EQ(response.txn_infos(0).txn_id(), 99); - EXPECT_EQ(response.txn_infos(0).commit_tso(), 12345); - EXPECT_EQ(response.txn_infos(0).table_ids(0), 100); - - // Deleting already scanned keys does not invalidate the next batch position. - ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); - txn->remove(txn_running_key({mock_instance, 1, 99})); - txn->remove(txn_running_key({mock_instance, 1, 100})); - ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); - request.set_start_key(response.next_start_key()); - response.Clear(); - meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); - ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - ASSERT_TRUE(response.has_next_start_key()); - ASSERT_TRUE(response.next_start_key().empty()); - ASSERT_EQ(response.txn_infos_size(), 1); - EXPECT_EQ(response.txn_infos(0).db_id(), 2); - EXPECT_EQ(response.txn_infos(0).txn_id(), 10); - EXPECT_EQ(response.txn_infos(0).table_ids(0), 300); - EXPECT_EQ(response.txn_infos(0).commit_tso(), 12345); -} - -TEST(MetaServiceTest, TsoRecoveryBatchCanBeEmptyBeforeTheScanCompletes) { - auto meta_service = get_meta_service(); - std::unique_ptr txn; - ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); - TxnRunningPB running; - // All IDs in the first database exceed the bound. A later database still contains an old txn. - txn->put(txn_running_key({mock_instance, 1, 200}), running.SerializeAsString()); - txn->put(txn_running_key({mock_instance, 2, 10}), running.SerializeAsString()); - TxnInfoPB info; - info.set_db_id(2); - info.set_txn_id(10); - info.add_table_ids(300); - info.set_status(TxnStatusPB::TXN_STATUS_COMMITTED); - info.set_commit_tso(12345); - txn->put(txn_info_key({mock_instance, 2, 10}), info.SerializeAsString()); - ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); - brpc::Controller cntl; - GetTsoRecoveryTransactionsRequest request; - request.set_cloud_unique_id("test_cloud_unique_id"); - request.set_end_txn_id(100); - request.set_batch_size(1); - request.set_tso_fence(20000); - GetTsoRecoveryTransactionsResponse response; - meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); - ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - ASSERT_EQ(response.txn_infos_size(), 0); - ASSERT_FALSE(response.next_start_key().empty()); - request.set_start_key(response.next_start_key()); - response.Clear(); - meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); - ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - ASSERT_EQ(response.txn_infos_size(), 1); - EXPECT_EQ(response.txn_infos(0).txn_id(), 10); - // Hitting the KV batch limit can require one final empty batch to establish completion. - ASSERT_FALSE(response.next_start_key().empty()); - request.set_start_key(response.next_start_key()); - response.Clear(); - meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); - ASSERT_EQ(response.status().code(), MetaServiceCode::OK); - EXPECT_EQ(response.txn_infos_size(), 0); - EXPECT_TRUE(response.has_next_start_key()); - EXPECT_TRUE(response.next_start_key().empty()); -} - -TEST(MetaServiceTest, TsoRecoveryBatchRejectsInvalidArgumentsAndMissingDetails) { - auto meta_service = get_meta_service(); - brpc::Controller cntl; - GetTsoRecoveryTransactionsRequest request; - request.set_cloud_unique_id("test_cloud_unique_id"); - request.set_batch_size(256); - GetTsoRecoveryTransactionsResponse response; - meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); - EXPECT_EQ(response.status().code(), MetaServiceCode::INVALID_ARGUMENT); - request.set_end_txn_id(100); - request.set_tso_fence(20000); - for (int size : {0, 1001}) { - request.set_batch_size(size); - response.Clear(); - meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); - EXPECT_EQ(response.status().code(), MetaServiceCode::INVALID_ARGUMENT); - } - request.set_batch_size(1); - request.set_start_key(txn_running_key({"another_instance", 1, 1})); - response.Clear(); - meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); - EXPECT_EQ(response.status().code(), MetaServiceCode::INVALID_ARGUMENT); - request.clear_start_key(); - std::unique_ptr txn; - ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); - txn->put(txn_running_key({mock_instance, 1, 10}), TxnRunningPB().SerializeAsString()); - ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); - response.Clear(); - meta_service->get_tso_recovery_transactions(&cntl, &request, &response, nullptr); - EXPECT_NE(response.status().code(), MetaServiceCode::OK); - EXPECT_FALSE(response.has_next_start_key()); -} - TEST(MetaServiceTest, CreateMetaSyncPointTest) { auto meta_service = get_meta_service(); const std::string cloud_unique_id = "test_cloud_unique_id"; diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp index b2fabdcfc5aaed..9cd952851a1a05 100644 --- a/cloud/test/txn_lazy_commit_test.cpp +++ b/cloud/test/txn_lazy_commit_test.cpp @@ -1282,21 +1282,6 @@ TEST(TxnLazyCommitTest, CommitTxnEventuallyWithFailedLazyCommitTaskTest) { ASSERT_TRUE(commit_res.has_is_lazy_commit_incomplete()); ASSERT_TRUE(commit_res.is_lazy_commit_incomplete()); - GetTsoRecoveryTransactionsRequest recovery_req; - recovery_req.set_cloud_unique_id("test_cloud_unique_id"); - recovery_req.set_end_txn_id(txn_id + 1); - recovery_req.set_batch_size(256); - recovery_req.set_tso_fence(12345); - GetTsoRecoveryTransactionsResponse recovery_res; - meta_service->get_tso_recovery_transactions(&cntl, &recovery_req, &recovery_res, nullptr); - ASSERT_EQ(recovery_res.status().code(), MetaServiceCode::OK); - ASSERT_TRUE(recovery_res.has_next_start_key()); - ASSERT_EQ(recovery_res.txn_infos_size(), 1); - EXPECT_EQ(recovery_res.txn_infos(0).txn_id(), txn_id); - EXPECT_EQ(recovery_res.txn_infos(0).status(), TxnStatusPB::TXN_STATUS_COMMITTED); - EXPECT_EQ(recovery_res.txn_infos(0).commit_tso(), 12345); - EXPECT_EQ(recovery_res.txn_infos(0).table_ids(0), table_id); - std::unique_ptr txn; ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); check_txn_committed(txn, db_id, txn_id, label); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java index d87d537a2a2548..d3d243e9d40405 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java @@ -371,12 +371,6 @@ public Cloud.CheckTxnConflictResponse checkTxnConflict(Cloud.CheckTxnConflictReq .checkTxnConflict(request); } - public Cloud.GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions( - Cloud.GetTsoRecoveryTransactionsRequest request) { - return blockingStub.withDeadlineAfter(Config.meta_service_brpc_timeout_ms, TimeUnit.MILLISECONDS) - .getTsoRecoveryTransactions(request); - } - public Cloud.AdvanceTsoFenceResponse advanceTsoFence(Cloud.AdvanceTsoFenceRequest request) { return blockingStub.withDeadlineAfter(Config.meta_service_brpc_timeout_ms, TimeUnit.MILLISECONDS) .advanceTsoFence(request); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java index efa06cebc62907..7014c1ba9963c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java @@ -243,6 +243,12 @@ public MetaServiceClientWrapper(MetaServiceProxy proxy) { public Response executeRequest(String methodName, Function function, Function statusExtractor) throws RpcException { + return executeRequest(methodName, function, statusExtractor, true); + } + + public Response executeRequest(String methodName, Function function, + Function statusExtractor, boolean retryRpcFailure) + throws RpcException { long maxRetries = Config.meta_service_rpc_retry_cnt; for (long tried = 1; tried <= maxRetries; tried++) { MetaServiceClient client = null; @@ -280,7 +286,7 @@ public Response executeRequest(String methodName, Function= maxRetries) { + if (!retryRpcFailure || !shouldRetry || tried >= maxRetries) { throw new RpcException("", sre.getMessage(), sre); } } catch (RpcException e) { @@ -288,7 +294,7 @@ public Response executeRequest(String methodName, Function= maxRetries) { + if (!retryRpcFailure || tried >= maxRetries) { throw new RpcException("", e.getMessage(), e); } } finally { @@ -318,6 +324,12 @@ public Response executeRequest(String methodName, Function Response executeWithMetrics(String methodName, Function function, Function statusExtractor) throws RpcException { + return executeWithMetrics(methodName, function, statusExtractor, true); + } + + private Response executeWithMetrics(String methodName, Function function, + Function statusExtractor, boolean retryRpcFailure) + throws RpcException { long startTime = System.currentTimeMillis(); if (MetricRepo.isInit && Config.isCloudMode()) { CloudMetrics.META_SERVICE_RPC_ALL_TOTAL.increase(1L); @@ -325,7 +337,7 @@ private Response executeWithMetrics(String methodName, Function client.commitTxn(request), - Cloud.CommitTxnResponse::getStatus); + Cloud.CommitTxnResponse::getStatus, !request.hasCommitTso() || request.getCommitTso() <= 0); } public Cloud.AbortTxnResponse abortTxn(Cloud.AbortTxnRequest request) @@ -470,12 +482,6 @@ public Cloud.CheckTxnConflictResponse checkTxnConflict(Cloud.CheckTxnConflictReq Cloud.CheckTxnConflictResponse::getStatus); } - public Cloud.GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions( - Cloud.GetTsoRecoveryTransactionsRequest request) throws RpcException { - return executeWithMetrics("getTsoRecoveryTransactions", (client) -> client.getTsoRecoveryTransactions(request), - Cloud.GetTsoRecoveryTransactionsResponse::getStatus); - } - public Cloud.AdvanceTsoFenceResponse advanceTsoFence(Cloud.AdvanceTsoFenceRequest request) throws RpcException { return executeWithMetrics("advanceTsoFence", (client) -> client.advanceTsoFence(request), diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index eccd3fcc7c2b56..236b43e241c604 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -55,8 +55,6 @@ import org.apache.doris.cloud.proto.Cloud.GetDeleteBitmapUpdateLockResponse; import org.apache.doris.cloud.proto.Cloud.GetPrepareTxnByCoordinatorRequest; import org.apache.doris.cloud.proto.Cloud.GetPrepareTxnByCoordinatorResponse; -import org.apache.doris.cloud.proto.Cloud.GetTsoRecoveryTransactionsRequest; -import org.apache.doris.cloud.proto.Cloud.GetTsoRecoveryTransactionsResponse; import org.apache.doris.cloud.proto.Cloud.GetTxnIdRequest; import org.apache.doris.cloud.proto.Cloud.GetTxnIdResponse; import org.apache.doris.cloud.proto.Cloud.GetTxnRequest; @@ -142,7 +140,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.google.protobuf.ByteString; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; import org.apache.logging.log4j.LogManager; @@ -863,7 +860,10 @@ private TransactionState commitTxn(CommitTxnRequest.Builder builder, List
// when ready to send, while retaining the existing table locks and callback cleanup scope. Database database = Env.getCurrentInternalCatalog().getDbOrMetaException(builder.getDbId()); Set commitTsoTableIds = tableList.stream().map(Table::getId).collect(Collectors.toSet()); - builder.setCommitTso(TransactionUtil.getCommitTSO(transactionId, database, commitTsoTableIds)); + long commitTso = TransactionUtil.getCommitTSO(transactionId, database, commitTsoTableIds); + if (commitTso > 0) { + builder.setCommitTso(commitTso); + } CommitTxnRequest commitTxnRequest = builder.build(); try { while (DebugPointUtil.isEnable("CloudGlobalTransactionMgr.commitTxn.blockAfterTso")) { @@ -871,6 +871,10 @@ private TransactionState commitTxn(CommitTxnRequest.Builder builder, List
} } catch (InterruptedException e) { Thread.currentThread().interrupt(); + if (commitTxnRequest.hasCommitTso()) { + Env.getCurrentEnv().getTSOService().abandonCommitTso( + commitTxnRequest.getDbId(), transactionId, commitTxnRequest.getCommitTso()); + } throw new UserException("Interrupted before sending commit transaction", e); } @@ -914,10 +918,31 @@ private TransactionState commitTxn(CommitTxnRequest.Builder builder, List
Preconditions.checkNotNull(commitTxnResponse.getStatus()); } catch (Exception e) { LOG.warn("commitTxn failed, transactionId:{}, exception:", transactionId, e); - throw new UserException("commitTxn() failed, errMsg:" + e.getMessage()); + if (commitTxnRequest.hasCommitTso()) { + try { + Env.getCurrentEnv().getTSOService().fenceAndAbandonCommitTso( + commitTxnRequest.getDbId(), transactionId, commitTxnRequest.getCommitTso()); + } catch (UserException fenceException) { + fenceException.addSuppressed(e); + throw fenceException; + } + } + throw new UserException("commitTxn() failed, errMsg:" + e.getMessage(), e); } - releaseFinishedTso(commitTxnRequest.getDbId(), transactionId, commitTxnResponse); + MetaServiceCode code = commitTxnResponse.getStatus().getCode(); + if (commitTxnRequest.hasCommitTso()) { + if (code == MetaServiceCode.KV_TXN_MAYBE_COMMITTED) { + Env.getCurrentEnv().getTSOService().fenceAndAbandonCommitTso( + commitTxnRequest.getDbId(), transactionId, commitTxnRequest.getCommitTso()); + } else if (code == MetaServiceCode.OK || code == MetaServiceCode.TXN_ALREADY_VISIBLE + || code == MetaServiceCode.TXN_ALREADY_ABORTED) { + Env.getCurrentEnv().getTSOService().transactionFinished(commitTxnRequest.getDbId(), transactionId); + } else { + Env.getCurrentEnv().getTSOService().abandonCommitTso( + commitTxnRequest.getDbId(), transactionId, commitTxnRequest.getCommitTso()); + } + } if (is2PC && (commitTxnResponse.getStatus().getCode() == MetaServiceCode.TXN_ALREADY_VISIBLE || commitTxnResponse.getStatus().getCode() == MetaServiceCode.TXN_ALREADY_ABORTED)) { @@ -951,20 +976,6 @@ private TransactionState commitTxn(CommitTxnRequest.Builder builder, List
return txnState; } - // A lazy commit response can report VISIBLE before the persistent transaction is visible. - static void releaseFinishedTso(long dbId, long txnId, CommitTxnResponse response) { - if (response.getIsLazyCommitIncomplete()) { - return; - } - MetaServiceCode code = response.getStatus().getCode(); - if (code == MetaServiceCode.TXN_ALREADY_VISIBLE || code == MetaServiceCode.TXN_ALREADY_ABORTED - || (code == MetaServiceCode.OK && response.hasTxnInfo() - && (response.getTxnInfo().getStatus() == TxnStatusPB.TXN_STATUS_VISIBLE - || response.getTxnInfo().getStatus() == TxnStatusPB.TXN_STATUS_ABORTED))) { - Env.getCurrentEnv().getTSOService().transactionFinished(dbId, txnId); - } - } - private void checkCommitInfo(CommitTxnRequestOrBuilder commitTxnRequest) throws UserException { List commitTabletIds = Lists.newArrayList(); List commitIndexIds = Lists.newArrayList(); @@ -2234,30 +2245,6 @@ public List getUnFinishedPreviousLoad(long endTransactionId, l return conflictTxns; } - @Override - public GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions(long endTransactionId, - long tsoFence, ByteString startKey) throws UserException { - GetTsoRecoveryTransactionsRequest request = GetTsoRecoveryTransactionsRequest.newBuilder() - .setCloudUniqueId(Config.cloud_unique_id) - .setRequestIp(FrontendOptions.getLocalHostAddressCached()) - .setEndTxnId(endTransactionId) - .setTsoFence(tsoFence) - .setBatchSize(256).setStartKey(startKey).build(); - GetTsoRecoveryTransactionsResponse response; - try { - response = MetaServiceProxy.getInstance().getTsoRecoveryTransactions(request); - } catch (RpcException e) { - throw new UserException("Failed to fetch TSO recovery transactions", e); - } - if (response.getStatus().getCode() != MetaServiceCode.OK) { - throw new UserException(response.getStatus().getMsg()); - } - if (!response.hasNextStartKey()) { - throw new UserException("MetaService returned an incomplete TSO recovery batch"); - } - return response; - } - @Override public long advanceTsoFence(long proposedFenceTso) throws UserException { AdvanceTsoFenceRequest request = AdvanceTsoFenceRequest.newBuilder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java index 00bd318071913d..7c7d5a54e9e2e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java @@ -316,9 +316,7 @@ public final class MetricRepo { public static LongCounterMetric COUNTER_TSO_CLOCK_GET_SUCCESS; public static LongCounterMetric COUNTER_TSO_STATE_PERSISTED; public static LongCounterMetric COUNTER_TSO_STATE_PERSIST_FAILED; - public static LongCounterMetric COUNTER_TSO_RECONCILE_FAILED; public static Histogram HISTO_TSO_STATE_PERSIST_LATENCY; - public static Histogram HISTO_TSO_RECONCILE_LATENCY; private static Map, Long> loadJobNum = Maps.newHashMap(); @@ -1184,11 +1182,7 @@ public Integer getValue() { COUNTER_TSO_STATE_PERSIST_FAILED = new LongCounterMetric("tso_state_persist_failed", MetricUnit.NOUNIT, "failed TSO state journal writes"); DORIS_METRIC_REGISTER.addMetrics(COUNTER_TSO_STATE_PERSIST_FAILED); - COUNTER_TSO_RECONCILE_FAILED = new LongCounterMetric("tso_reconcile_failed", MetricUnit.NOUNIT, - "failed TSO transaction reconciliation cycles"); - DORIS_METRIC_REGISTER.addMetrics(COUNTER_TSO_RECONCILE_FAILED); HISTO_TSO_STATE_PERSIST_LATENCY = METRIC_REGISTER.histogram("tso_state_persist_latency_ms"); - HISTO_TSO_RECONCILE_LATENCY = METRIC_REGISTER.histogram("tso_reconcile_latency_ms"); Env.getCurrentEnv().getTSOService().registerMetrics(); // init system metrics diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index 98d9056e1af08c..ead7960a5a106e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -690,7 +690,7 @@ public static void setVisibleVersionForOlapScanNodes(List scanNodes) t // A time-based change read may have just waited for an old transaction to finish. // Bypass the FE cache so the scan uses the version made visible by that transaction. versions = hasIncrementalRead - ? CloudPartition.getSnapshotVisibleVersionFromMs(partitions, false) + ? CloudPartition.getSnapshotVisibleVersionFromMs(partitions, true) : CloudPartition.getSnapshotVisibleVersion(partitions); } catch (RpcException e) { throw new UserException("get visible version for OlapScanNode failed", e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java index 3fec6d943a2d00..e241ff37982fdf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java @@ -22,7 +22,6 @@ import org.apache.doris.catalog.Table; import org.apache.doris.catalog.stream.TableStreamUpdateInfo; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; -import org.apache.doris.cloud.proto.Cloud.GetTsoRecoveryTransactionsResponse; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DuplicatedRequestException; import org.apache.doris.common.LabelAlreadyUsedException; @@ -39,8 +38,6 @@ import org.apache.doris.transaction.TransactionState.LoadJobSourceType; import org.apache.doris.transaction.TransactionState.TxnCoordinator; -import com.google.protobuf.ByteString; - import java.io.DataInput; import java.io.IOException; import java.util.List; @@ -148,12 +145,6 @@ public void abortTransaction(Long dbId, Long txnId, String reason, public void finishTransaction(long dbId, long transactionId, Map partitionVisibleVersions, Map> backendPartitions) throws UserException; - /** Fetch committed TSO transactions below the exclusive recovery transaction bound. */ - default GetTsoRecoveryTransactionsResponse getTsoRecoveryTransactions(long endTransactionId, - long tsoFence, ByteString startKey) throws UserException { - throw new UserException("TSO recovery is only supported in cloud mode"); - } - default long advanceTsoFence(long proposedFenceTso) throws UserException { throw new UserException("TSO fence is only supported in cloud mode"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java index ffb8552cb5dbc5..5f8afc6e6f6601 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java @@ -64,34 +64,6 @@ public class TSOService extends MasterDaemon { private long pendingCalibrationPhysicalTime; private long pendingCalibrationWindowEnd; private long lastPersistNanos; - private final MasterDaemon transactionChecker = new MasterDaemon("TSO-transaction-checker", 1000) { - private long lastFailureLogNanos; - - @Override - protected void runAfterCatalogReady() { - if (!Config.isCloudMode() || !isTsoEnabled() || !isInitialized.get() - || !Env.getCurrentEnv().isMaster()) { - return; - } - long startNanos = System.nanoTime(); - try { - transactionTracker.checkTransactions(Env.getCurrentGlobalTransactionMgr(), startNanos); - } catch (Exception e) { - if (lastFailureLogNanos == 0 || startNanos - lastFailureLogNanos >= TimeUnit.MINUTES.toNanos(1)) { - LOG.warn("Failed to reconcile TSO transactions; retaining the committed TSO", e); - lastFailureLogNanos = startNanos; - } - if (MetricRepo.isInit) { - MetricRepo.COUNTER_TSO_RECONCILE_FAILED.increase(1L); - } - } finally { - if (MetricRepo.isInit) { - MetricRepo.HISTO_TSO_RECONCILE_LATENCY.update( - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos)); - } - } - } - }; /** * Immutable snapshot of the current TSO service status. @@ -151,8 +123,6 @@ public void registerMetrics() { gauges.put("tso_oldest_pending_tso", transactionTracker::getOldestPendingTso); gauges.put("tso_oldest_pending_txn_id", transactionTracker::getOldestPendingTxnId); gauges.put("tso_oldest_pending_age_ms", transactionTracker::getOldestPendingAgeMs); - gauges.put("tso_recovery_ready", () -> transactionTracker.isRecoveryReady() ? 1 : 0); - gauges.put("tso_recovery_watermark", transactionTracker::getRecoveryWatermark); gauges.forEach((name, value) -> MetricRepo.DORIS_METRIC_REGISTER.addMetrics( new GaugeMetric(name, MetricUnit.NOUNIT, name) { @Override @@ -162,15 +132,6 @@ public Long getValue() { })); } - /** - * Start the TSO service. - */ - @Override - public synchronized void start() { - super.start(); - transactionChecker.start(); - } - /** * Periodically update timestamp after catalog is ready * This method is called by the MasterDaemon framework @@ -273,6 +234,21 @@ public void transactionFinished(long dbId, long txnId) { transactionTracker.transactionFinished(dbId, txnId); } + public void abandonCommitTso(long dbId, long txnId, long tso) { + transactionTracker.abandonCommitTso(dbId, txnId, tso); + } + + public void fenceAndAbandonCommitTso(long dbId, long txnId, long tso) throws UserException { + try { + long effectiveFenceTso = Env.getCurrentEnv().getGlobalTransactionMgr().advanceTsoFence(tso); + Preconditions.checkState(effectiveFenceTso >= tso, "MetaService TSO fence must not regress"); + transactionTracker.abandonCommitTso(dbId, txnId, tso); + } catch (Exception e) { + deactivate(); + throw new UserException("Failed to fence an uncertain commit TSO", e); + } + } + private long getTSO(Pair transactionIdentity, Set tableIds) { return getTSO(transactionIdentity, tableIds, -1, -1); } @@ -397,8 +373,8 @@ public TSOStatusSnapshot waitForReadableWindow(Map> dbToTableId TSOTimestamp.composePhysicalTimestamp(endTimestampMs), TimeUnit.MILLISECONDS.toNanos(timeoutMs) - (System.nanoTime() - startNanos)); snapshot = getStatusSnapshot(); - if (result == TSOTransactionTracker.WaitResult.RECOVERING) { - throw windowError(ErrorCode.ERR_INCR_WINDOW_NOT_READY, "TSO_RECOVERING", + if (result == TSOTransactionTracker.WaitResult.RESET) { + throw windowError(ErrorCode.ERR_INCR_WINDOW_NOT_READY, "TSO_MASTER_CHANGED", endTimestampMs, snapshot, timeoutMs); } if (!Env.getCurrentEnv().isMaster()) { @@ -503,10 +479,9 @@ private void calibrateTimestamp() throws UserException { } private long persistCalibrationWindow(long physicalTime) { - long fenceTso = TSOTimestamp.composePhysicalTimestamp(physicalTime); lock.lock(); try { - transactionTracker.reset(fenceTso); + transactionTracker.reset(); } finally { lock.unlock(); } @@ -747,7 +722,7 @@ private void deactivate() { lock.lock(); try { if (isInitialized.getAndSet(false)) { - transactionTracker.invalidate(); + transactionTracker.reset(); } } finally { lock.unlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java index cdc869e734d906..d2985844e5318e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java @@ -17,19 +17,9 @@ package org.apache.doris.tso; -import org.apache.doris.cloud.proto.Cloud.GetTsoRecoveryTransactionsResponse; -import org.apache.doris.cloud.proto.Cloud.TxnInfoPB; -import org.apache.doris.cloud.proto.Cloud.TxnStatusPB; import org.apache.doris.common.Pair; -import org.apache.doris.common.UserException; -import org.apache.doris.transaction.GlobalTransactionMgrIface; -import org.apache.doris.transaction.TransactionState; -import org.apache.doris.transaction.TransactionStatus; import com.google.common.base.Preconditions; -import com.google.protobuf.ByteString; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.Collection; @@ -46,26 +36,14 @@ /** In-memory commit registrations. Uses the allocator's lock so no allocated TSO can be missed. */ final class TSOTransactionTracker { - private static final Logger LOG = LogManager.getLogger(TSOTransactionTracker.class); - private static final int CHECK_BATCH_SIZE = 64; - private static final long CHECK_AGE_NANOS = TimeUnit.SECONDS.toNanos(1); private final ReentrantLock lock; private final Condition transactionsChanged; private final Map, PendingTransaction> pendingByTxn = new HashMap<>(); private final TreeMap pendingByTso = new TreeMap<>(); - // Recovery entries stay separate so the durable committed prefix remains frozen until all finish. - private final TreeMap recoveryByTxn = new TreeMap<>(); private long generation; - private long recoveryFenceTso; - private long recoveryWatermark; - private boolean recoveryReady; - private boolean recoveryLoaded; - private ByteString recoveryStartKey = ByteString.EMPTY; - private long recoveryPollCursor; - private long pollCursor; enum WaitResult { - FINISHED, TIMED_OUT, RECOVERING + FINISHED, TIMED_OUT, RESET } private static final class PendingTransaction { @@ -87,43 +65,19 @@ private PendingTransaction(Pair identity, long tso, long nowNanos, C this.transactionsChanged = lock.newCondition(); } - void reset(long fenceTso) { + void reset() { Preconditions.checkState(lock.isHeldByCurrentThread()); - Preconditions.checkArgument(fenceTso > 0, "recovery fence TSO must be positive"); - clearForNewGeneration(); - recoveryFenceTso = fenceTso; - } - - void invalidate() { - Preconditions.checkState(lock.isHeldByCurrentThread()); - clearForNewGeneration(); - recoveryFenceTso = 0; - } - - private void clearForNewGeneration() { generation++; pendingByTxn.clear(); pendingByTso.clear(); - recoveryByTxn.clear(); - recoveryWatermark = 0; - recoveryReady = false; - recoveryLoaded = false; - recoveryStartKey = ByteString.EMPTY; - recoveryPollCursor = 0; - pollCursor = 0; transactionsChanged.signalAll(); } void register(Pair identity, long tso, long nowNanos, Set tableIds) { Preconditions.checkState(lock.isHeldByCurrentThread()); Preconditions.checkArgument(!tableIds.isEmpty(), "commit registration requires table IDs"); - PendingTransaction recovered = recoveryByTxn.get(identity.second); - if (recovered != null) { - recovered.tableIds.addAll(tableIds); - } PendingTransaction existing = pendingByTxn.get(identity); if (existing != null) { - // A timed-out request can still commit using the earlier TSO. existing.tableIds.addAll(tableIds); return; } @@ -141,8 +95,7 @@ void replaceFenced(Pair identity, long rejectedTso, long fenceTso, l PendingTransaction existing = pendingByTxn.get(identity); Set mergedTableIds = new HashSet<>(); if (existing != null) { - Preconditions.checkState(existing.tso <= fenceTso, - "registered transaction TSO must be fenced"); + Preconditions.checkState(existing.tso <= fenceTso, "registered transaction TSO must be fenced"); Preconditions.checkState(pendingByTso.remove(existing.tso) == existing); mergedTableIds.addAll(existing.tableIds); } @@ -150,10 +103,6 @@ void replaceFenced(Pair identity, long rejectedTso, long fenceTso, l PendingTransaction replacement = new PendingTransaction(identity, newTso, nowNanos, mergedTableIds); pendingByTxn.put(identity, replacement); Preconditions.checkState(pendingByTso.put(newTso, replacement) == null); - PendingTransaction recovered = recoveryByTxn.get(identity.second); - if (recovered != null) { - recovered.tableIds.addAll(tableIds); - } transactionsChanged.signalAll(); } @@ -161,32 +110,20 @@ void replaceFenced(Pair identity, long rejectedTso, long fenceTso, l WaitResult awaitTransactions(Map> dbToTableIds, long endTso, long remainingNanos) throws InterruptedException { Preconditions.checkState(lock.isHeldByCurrentThread()); - if (!recoveryLoaded) { - return WaitResult.RECOVERING; - } long waitStartNanos = System.nanoTime(); long waitGeneration = generation; List remaining = new ArrayList<>(); - for (PendingTransaction pending : recoveryByTxn.values()) { - List tables = dbToTableIds.get(pending.identity.first); - if (pending.tso <= endTso && tables != null && !Collections.disjoint(tables, pending.tableIds)) { - remaining.add(pending); - } - } for (PendingTransaction pending : pendingByTso.headMap(endTso, true).values()) { List tables = dbToTableIds.get(pending.identity.first); if (tables != null && !Collections.disjoint(tables, pending.tableIds)) { remaining.add(pending); } } - // Allocation/registration and this snapshot share the lock. Later allocations are outside - // the validated window; only this fixed set can affect the read. awaitNanos releases the lock. while (true) { if (generation != waitGeneration) { - return WaitResult.RECOVERING; + return WaitResult.RESET; } - remaining.removeIf(pending -> pendingByTxn.get(pending.identity) != pending - && recoveryByTxn.get(pending.identity.second) != pending); + remaining.removeIf(pending -> pendingByTxn.get(pending.identity) != pending); if (remaining.isEmpty()) { return WaitResult.FINISHED; } @@ -200,7 +137,7 @@ WaitResult awaitTransactions(Map> dbToTableIds, long endTso, lo long candidateCommittedTso(long currentTso, long durableCommittedTso) { Preconditions.checkState(lock.isHeldByCurrentThread()); - if (!recoveryReady) { + if (currentTso < durableCommittedTso) { return durableCommittedTso; } long candidate = pendingByTso.isEmpty() ? currentTso @@ -212,142 +149,31 @@ long candidateCommittedTso(long currentTso, long durableCommittedTso) { void transactionFinished(long dbId, long txnId) { lock.lock(); try { - PendingTransaction pending = pendingByTxn.remove(Pair.of(dbId, txnId)); - if (pending != null) { - pendingByTso.remove(pending.tso); - } - recoveryByTxn.remove(txnId); - updateRecoveryReady(); - transactionsChanged.signalAll(); + remove(Pair.of(dbId, txnId)); } finally { lock.unlock(); } } - /** Runs on a separate daemon. Neither recovery nor transaction RPCs hold the allocator lock. */ - void checkTransactions(GlobalTransactionMgrIface txnMgr, long nowNanos) throws UserException { - long checkGeneration; - long watermark; - long fenceTso; - boolean checkRecovery; - ByteString startKey; - List batch = new ArrayList<>(); + void abandonCommitTso(long dbId, long txnId, long tso) { lock.lock(); try { - checkGeneration = generation; - watermark = recoveryWatermark; - fenceTso = recoveryFenceTso; - checkRecovery = !recoveryLoaded; - startKey = recoveryStartKey; - if (!pendingByTso.isEmpty()) { - // Always check the transaction blocking the prefix, then rotate through the rest. - PendingTransaction oldest = pendingByTso.firstEntry().getValue(); - if (nowNanos - oldest.registeredAtNanos >= CHECK_AGE_NANOS) { - batch.add(oldest); - } - for (int i = 0; i < Math.min(CHECK_BATCH_SIZE - 1, pendingByTso.size()); i++) { - Map.Entry next = pendingByTso.higherEntry(pollCursor); - if (next == null) { - next = pendingByTso.firstEntry(); - } - pollCursor = next.getKey(); - PendingTransaction pending = next.getValue(); - if (pending != oldest && nowNanos - pending.registeredAtNanos >= CHECK_AGE_NANOS) { - batch.add(pending); - } - } - } - for (int i = 0; i < Math.min(CHECK_BATCH_SIZE, recoveryByTxn.size()); i++) { - Map.Entry next = recoveryByTxn.higherEntry(recoveryPollCursor); - if (next == null) { - next = recoveryByTxn.firstEntry(); - } - recoveryPollCursor = next.getKey(); - // A local registration already supplies the reconciliation RPC for this transaction. - if (!pendingByTxn.containsKey(next.getValue().identity)) { - batch.add(next.getValue()); - } + Pair identity = Pair.of(dbId, txnId); + PendingTransaction pending = pendingByTxn.get(identity); + if (pending != null && pending.tso == tso) { + remove(identity); } } finally { lock.unlock(); } - - // Reconcile registrations even while the recovery scan is failing or waiting on old transactions. - for (PendingTransaction pending : batch) { - TransactionState state = txnMgr.getTransactionState(pending.identity.first, pending.identity.second); - // null includes RPC errors and NOT_FOUND; neither proves that a transaction is finished. - if (state == null || (state.getTransactionStatus() != TransactionStatus.VISIBLE - && state.getTransactionStatus() != TransactionStatus.ABORTED)) { - continue; - } - lock.lock(); - try { - if (generation == checkGeneration && (pendingByTxn.get(pending.identity) == pending - || recoveryByTxn.get(pending.identity.second) == pending)) { - transactionFinished(pending.identity.first, pending.identity.second); - } - } finally { - lock.unlock(); - } - } - - if (checkRecovery) { - if (watermark == 0) { - watermark = txnMgr.getTransactionIdWatermark(); - Preconditions.checkState(watermark > 0, "invalid recovery transaction watermark"); - lock.lock(); - try { - if (generation != checkGeneration) { - return; - } - recoveryWatermark = watermark; - } finally { - lock.unlock(); - } - } - while (true) { - GetTsoRecoveryTransactionsResponse response = - txnMgr.getTsoRecoveryTransactions(watermark, fenceTso, startKey); - lock.lock(); - try { - if (generation != checkGeneration) { - return; - } - for (TxnInfoPB info : response.getTxnInfosList()) { - Preconditions.checkState(info.getStatus() == TxnStatusPB.TXN_STATUS_COMMITTED, - "TSO recovery batch contains a non-committed transaction: %s", info.getTxnId()); - Preconditions.checkState(info.hasCommitTso() && info.getCommitTso() > 0 - && info.getCommitTso() <= fenceTso, - "TSO recovery transaction exceeds fence: %s", info.getTxnId()); - Pair identity = Pair.of(info.getDbId(), info.getTxnId()); - PendingTransaction recovered = new PendingTransaction(identity, - info.getCommitTso(), nowNanos, info.getTableIdsList()); - PendingTransaction local = pendingByTxn.get(identity); - if (local != null) { - recovered.tableIds.addAll(local.tableIds); - } - recoveryByTxn.put(info.getTxnId(), recovered); - } - startKey = response.getNextStartKey(); - recoveryStartKey = startKey; - if (startKey.isEmpty()) { - recoveryLoaded = true; - updateRecoveryReady(); - LOG.info("Loaded TSO recovery transactions, watermark={}, fenceTso={}, pending={}", - watermark, fenceTso, recoveryByTxn.size()); - return; - } - } finally { - lock.unlock(); - } - } - } } - private void updateRecoveryReady() { - if (!recoveryReady && recoveryLoaded && recoveryByTxn.isEmpty()) { - recoveryReady = true; - LOG.info("TSO recovery completed, transaction watermark={}", recoveryWatermark); + private void remove(Pair identity) { + Preconditions.checkState(lock.isHeldByCurrentThread()); + PendingTransaction pending = pendingByTxn.remove(identity); + if (pending != null) { + Preconditions.checkState(pendingByTso.remove(pending.tso) == pending); + transactionsChanged.signalAll(); } } @@ -387,22 +213,4 @@ long getOldestPendingAgeMs() { lock.unlock(); } } - - long getRecoveryWatermark() { - lock.lock(); - try { - return recoveryWatermark; - } finally { - lock.unlock(); - } - } - - boolean isRecoveryReady() { - lock.lock(); - try { - return recoveryReady; - } finally { - lock.unlock(); - } - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceProxyTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceProxyTest.java index 5b80b6ab77a441..bb22d682231fcf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceProxyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceProxyTest.java @@ -139,6 +139,25 @@ public void testExecuteRequestShutdownOnFailure() { Mockito.verify(client).shutdown(true); } + @Test + public void testExecuteRequestCanDisableRpcFailureRetry() { + Config.meta_service_rpc_retry_cnt = 3; + MetaServiceProxy proxy = new MetaServiceProxy(); + MetaServiceClient client = mockNormalClient(); + Map serviceMap = Deencapsulation.getField(proxy, "serviceMap"); + serviceMap.put(Config.meta_service_endpoint, client); + MetaServiceProxy.MetaServiceClientWrapper wrapper = Deencapsulation.getField(proxy, "w"); + AtomicInteger callCount = new AtomicInteger(); + + Assertions.assertThrows(RpcException.class, () -> wrapper.executeRequest("commitTxn", ignored -> { + callCount.incrementAndGet(); + throw new RuntimeException("rpc failed"); + }, Cloud.CommitTxnResponse::getStatus, false)); + + Assertions.assertEquals(1, callCount.get()); + Mockito.verify(client).shutdown(true); + } + @Test public void testGetVisibleVersionAsyncShutdownOnFailure() throws RpcException { MetaServiceProxy proxy = new MetaServiceProxy(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java index c7ec27e130b8f4..ef8315f3c3b1cb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java @@ -23,10 +23,10 @@ import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; -import org.apache.doris.cloud.proto.Cloud.TxnInfoPB; import org.apache.doris.cloud.rpc.MetaServiceProxy; import org.apache.doris.common.UserException; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.rpc.RpcException; import org.apache.doris.transaction.TransactionUtil; import org.apache.doris.tso.TSOService; @@ -43,30 +43,10 @@ public class CloudCommittedTsoTest { @Test - public void testLazyCommitResponseDoesNotReleaseTsoUntilReallyVisible() { + public void testOrdinarySubTransactionAndTwoPhaseCommitReuseTsoAcrossRpcRetries() throws Exception { Env env = Mockito.mock(Env.class); TSOService tsoService = Mockito.mock(TSOService.class); Mockito.when(env.getTSOService()).thenReturn(tsoService); - CommitTxnResponse.Builder response = CommitTxnResponse.newBuilder() - .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK)) - .setTxnInfo(TxnInfoPB.newBuilder().setStatus(Cloud.TxnStatusPB.TXN_STATUS_VISIBLE)) - .setIsLazyCommit(true).setIsLazyCommitIncomplete(true); - try (MockedStatic mocked = Mockito.mockStatic(Env.class)) { - mocked.when(Env::getCurrentEnv).thenReturn(env); - CloudGlobalTransactionMgr.releaseFinishedTso(1, 10, response.build()); - Mockito.verifyNoInteractions(tsoService); - response.setIsLazyCommitIncomplete(false); - CloudGlobalTransactionMgr.releaseFinishedTso(1, 10, response.build()); - Mockito.verify(tsoService).transactionFinished(1, 10); - response.setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.TXN_ALREADY_ABORTED)); - CloudGlobalTransactionMgr.releaseFinishedTso(2, 20, response.build()); - Mockito.verify(tsoService).transactionFinished(2, 20); - } - } - - @Test - public void testOrdinarySubTransactionAndTwoPhaseCommitReuseTsoAcrossRpcRetries() throws Exception { - Env env = Mockito.mock(Env.class); InternalCatalog catalog = Mockito.mock(InternalCatalog.class); Database db = Mockito.mock(Database.class); Mockito.when(db.getId()).thenReturn(1L); @@ -104,12 +84,53 @@ public void testOrdinarySubTransactionAndTwoPhaseCommitReuseTsoAcrossRpcRetries( Assertions.assertEquals(500L, requests.get(0).getCommitTso()); Assertions.assertEquals(path == 1, requests.get(0).getIsTxnLoad()); Assertions.assertEquals(path == 2, requests.get(0).getIs2Pc()); + Mockito.verify(tsoService).abandonCommitTso(1, 10, 500); allocation.verify(() -> TransactionUtil.getCommitTSO(10L, db, Collections.emptySet())); allocation.clearInvocations(); + Mockito.clearInvocations(tsoService); } } } + @Test + public void testUncertainCommitResultsAreFencedBeforeRelease() throws Exception { + Env env = Mockito.mock(Env.class); + TSOService tsoService = Mockito.mock(TSOService.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database db = Mockito.mock(Database.class); + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + Mockito.when(env.getTSOService()).thenReturn(tsoService); + Mockito.when(catalog.getDbOrMetaException(1L)).thenReturn(db); + Mockito.when(proxy.commitTxn(Mockito.any())).thenThrow(new RpcException("ms", "timeout")); + Method commit = CloudGlobalTransactionMgr.class.getDeclaredMethod("commitTxn", + Cloud.CommitTxnRequest.Builder.class, List.class, long.class, boolean.class, List.class, List.class); + commit.setAccessible(true); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class); + MockedStatic mockedProxy = Mockito.mockStatic(MetaServiceProxy.class); + MockedStatic allocation = Mockito.mockStatic(TransactionUtil.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + mockedEnv.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + mockedProxy.when(MetaServiceProxy::getInstance).thenReturn(proxy); + allocation.when(() -> TransactionUtil.getCommitTSO(10L, db, Collections.emptySet())).thenReturn(500L); + + Assertions.assertThrows(InvocationTargetException.class, + () -> commit.invoke(new CloudGlobalTransactionMgr(), + Cloud.CommitTxnRequest.newBuilder().setDbId(1).setTxnId(10), + Collections.emptyList(), 10L, false, Collections.emptyList(), Collections.emptyList())); + Mockito.doReturn(CommitTxnResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.KV_TXN_MAYBE_COMMITTED)) + .build()).when(proxy).commitTxn(Mockito.any()); + Assertions.assertThrows(InvocationTargetException.class, + () -> commit.invoke(new CloudGlobalTransactionMgr(), + Cloud.CommitTxnRequest.newBuilder().setDbId(1).setTxnId(10), + Collections.emptyList(), 10L, false, Collections.emptyList(), Collections.emptyList())); + Mockito.verify(tsoService, Mockito.times(2)).fenceAndAbandonCommitTso(1, 10, 500); + Mockito.verify(tsoService, Mockito.never()).abandonCommitTso(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong()); + } + } + @Test public void testValidationFailureDoesNotAllocateTsoOrSendCommit() throws Exception { Env env = Mockito.mock(Env.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java index 37a2f92c531c08..2c8d1a3a08fd8d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java @@ -33,7 +33,6 @@ import org.apache.doris.cloud.proto.Cloud.CheckTxnConflictResponse; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; import org.apache.doris.cloud.proto.Cloud.GetCurrentMaxTxnResponse; -import org.apache.doris.cloud.proto.Cloud.GetTsoRecoveryTransactionsResponse; import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; import org.apache.doris.cloud.proto.Cloud.TxnInfoPB; import org.apache.doris.cloud.rpc.MetaServiceProxy; @@ -43,7 +42,6 @@ import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.UserException; import org.apache.doris.load.routineload.RLTaskTxnCommitAttachment; -import org.apache.doris.rpc.RpcException; import org.apache.doris.thrift.TTabletCommitInfo; import org.apache.doris.transaction.GlobalTransactionMgrIface; import org.apache.doris.transaction.TabletCommitInfo; @@ -52,7 +50,6 @@ import org.apache.doris.tso.TSOService; import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -100,45 +97,6 @@ public void tearDown() { } } - @Test - public void testTsoRecoveryRequiresCompleteBatchResponse() throws Exception { - MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); - try (MockedStatic mocked = Mockito.mockStatic(MetaServiceProxy.class)) { - mocked.when(MetaServiceProxy::getInstance).thenReturn(proxy); - GetTsoRecoveryTransactionsResponse.Builder response = GetTsoRecoveryTransactionsResponse.newBuilder() - .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK)); - Mockito.when(proxy.getTsoRecoveryTransactions(Mockito.any())).thenReturn(response.build()); - Assertions.assertThrows(UserException.class, - () -> masterTransMgr.getTsoRecoveryTransactions(1000, 900, ByteString.EMPTY)); - Mockito.when(proxy.getTsoRecoveryTransactions(Mockito.any())).thenReturn( - response.setNextStartKey(ByteString.EMPTY).build()); - ByteString startKey = ByteString.copyFromUtf8("next batch"); - Assertions.assertTrue(masterTransMgr.getTsoRecoveryTransactions(1000, 900, startKey) - .getNextStartKey().isEmpty()); - ArgumentCaptor capture = - ArgumentCaptor.forClass(Cloud.GetTsoRecoveryTransactionsRequest.class); - Mockito.verify(proxy, Mockito.times(2)).getTsoRecoveryTransactions(capture.capture()); - Assertions.assertEquals(256, capture.getValue().getBatchSize()); - Assertions.assertEquals(1000, capture.getValue().getEndTxnId()); - Assertions.assertEquals(900, capture.getValue().getTsoFence()); - Assertions.assertEquals(startKey, capture.getValue().getStartKey()); - Mockito.verify(proxy, Mockito.never()).checkTxnConflict(Mockito.any()); - } - } - - @Test - public void testTsoRecoveryRpcFailureDoesNotFallBackToConflictCheck() throws Exception { - MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); - try (MockedStatic mocked = Mockito.mockStatic(MetaServiceProxy.class)) { - mocked.when(MetaServiceProxy::getInstance).thenReturn(proxy); - Mockito.when(proxy.getTsoRecoveryTransactions(Mockito.any())) - .thenThrow(new RpcException("ms", "unknown method")); - Assertions.assertThrows(UserException.class, - () -> masterTransMgr.getTsoRecoveryTransactions(1000, 900, ByteString.EMPTY)); - Mockito.verify(proxy, Mockito.never()).checkTxnConflict(Mockito.any()); - } - } - @Test public void testAdvanceTsoFence() throws Exception { MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); @@ -325,6 +283,7 @@ public void testCommitTransactionRetriesWithTsoAboveFence() throws Exception { Mockito.verify(proxy, Mockito.times(2)).commitTxn(requests.capture()); Assertions.assertEquals(100L, requests.getAllValues().get(0).getCommitTso()); Assertions.assertEquals(201L, requests.getAllValues().get(1).getCommitTso()); + Mockito.verify(tsoService).transactionFinished(CatalogTestUtil.testDbId1, 123533L); } finally { table.setBinlogConfig(originalBinlogConfig); Config.enable_feature_binlog = originalEnableFeatureBinlog; @@ -391,7 +350,6 @@ public void testCommitTransactionCarriesTableStreamUpdates() throws Exception { ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(Cloud.CommitTxnRequest.class); Mockito.verify(mockProxy).commitTxn(requestCaptor.capture()); - Assertions.assertTrue(requestCaptor.getValue().hasCommitTso()); Assertions.assertEquals(2, requestCaptor.getValue().getTableStreamUpdatesCount()); Assertions.assertEquals(identity, requestCaptor.getValue().getTableStreamUpdates(0).getIdentity()); Assertions.assertEquals(partitionUpdate, diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java index 0c3b524cd12f50..44bf4d8215a0a9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java @@ -342,12 +342,12 @@ public void testIncrementalReadGetsVisibleVersionFromMetaService() throws Except MockedStatic mockedPartition = Mockito.mockStatic(CloudPartition.class)) { mockedConfig.when(Config::isNotCloudMode).thenReturn(false); mockedPartition.when(() -> CloudPartition.getSnapshotVisibleVersionFromMs( - Mockito.anyList(), Mockito.eq(false))).thenReturn(Lists.newArrayList(visibleVersion)); + Mockito.anyList(), Mockito.eq(true))).thenReturn(Lists.newArrayList(visibleVersion)); ScanNode.setVisibleVersionForOlapScanNodes(Lists.newArrayList(scanNode)); mockedPartition.verify(() -> CloudPartition.getSnapshotVisibleVersionFromMs( - Mockito.anyList(), Mockito.eq(false))); + Mockito.anyList(), Mockito.eq(true))); mockedPartition.verify(() -> CloudPartition.getSnapshotVisibleVersion(Mockito.anyList()), Mockito.never()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java index c97ffdb07bfe78..a3bd353da1b676 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/TimeBasedChangeVisibleWaiterTest.java @@ -295,7 +295,7 @@ public void testMasterRpcAndFollowerPreserveBothWindowErrors() throws Exception ErrorCode.ERR_INCR_VISIBLE_WAIT_TIMEOUT}) { Env master = mockMasterEnv(); TSOService service = mockTsoService(master, CURRENT_TSO); - String reason = code == ErrorCode.ERR_INCR_WINDOW_NOT_READY ? "TSO_RECOVERING" : "VISIBLE_WAIT_TIMEOUT"; + String reason = code == ErrorCode.ERR_INCR_WINDOW_NOT_READY ? "TSO_MASTER_CHANGED" : "VISIBLE_WAIT_TIMEOUT"; IncrWindowNotReadyException failure = new IncrWindowNotReadyException(code, reason, CURRENT_PHYSICAL_TIME_MS, CURRENT_TSO, 0, 1000, 1000); Mockito.doThrow(failure).when(service) diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java index 87afe145b8aa00..0c95b98d475ac2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java @@ -33,7 +33,6 @@ import org.apache.doris.qe.TimeBasedChangeVisibleWaiter; import org.apache.doris.transaction.GlobalTransactionMgrIface; -import com.google.protobuf.ByteString; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -533,6 +532,42 @@ public void testFencedCommitRetryDoesNotAdvanceStaleMasterClock() throws Excepti ((TSOTransactionTracker) trackerField.get(tsoService)).getPendingCount()); } + @Test + public void testUncertainCommitIsReleasedAfterFence() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + long commitTso = tsoService.getCommitTSO(1, 10, Collections.singleton(100L)); + + tsoService.fenceAndAbandonCommitTso(1, 10, commitTso); + + Mockito.verify(globalTxnMgr).advanceTsoFence(commitTso); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + Assertions.assertEquals(0, + ((TSOTransactionTracker) trackerField.get(tsoService)).getPendingCount()); + } + + @Test + public void testFenceFailureDeactivatesTsoService() throws Exception { + Mockito.when(env.isReady()).thenReturn(true); + Mockito.when(env.isMaster()).thenReturn(true); + setInitializedFlag(tsoService, true); + setGlobalTimestamp(tsoService, 100, 0); + long commitTso = tsoService.getCommitTSO(1, 10, Collections.singleton(100L)); + Mockito.when(globalTxnMgr.advanceTsoFence(commitTso)).thenThrow(new UserException("injected failure")); + + Assertions.assertThrows(UserException.class, + () -> tsoService.fenceAndAbandonCommitTso(1, 10, commitTso)); + + Assertions.assertFalse(tsoService.getStatusSnapshot().isInitialized()); + Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); + trackerField.setAccessible(true); + Assertions.assertEquals(0, + ((TSOTransactionTracker) trackerField.get(tsoService)).getPendingCount()); + } + @Test public void testDemotionInvalidatesAllocationState() throws Exception { Mockito.when(env.isReady()).thenReturn(true); @@ -602,14 +637,6 @@ public void testReadableWindowDoesNotRequireAnotherCommittedTsoFlush() throws Ex setGlobalTimestamp(tsoService, 1000, 17); long committed = TSOTimestamp.composeTimestamp(900, 1); tsoService.replayWindowEndTSO(new TSOServiceState(2000, committed)); - Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); - trackerField.setAccessible(true); - TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); - GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(Mockito.eq(1000L), Mockito.anyLong(), Mockito.eq(ByteString.EMPTY))) - .thenReturn(TSOTransactionTrackerTest.recoveryBatch(ByteString.EMPTY)); - tracker.checkTransactions(txnMgr, Long.MAX_VALUE); try (MockedStatic config = Mockito.mockStatic(Config.class)) { config.when(Config::isCloudMode).thenReturn(true); TimeBasedChangeVisibleWaiter.ChangeReadFence fence = TimeBasedChangeVisibleWaiter.acquireFenceOnMaster( @@ -618,27 +645,18 @@ public void testReadableWindowDoesNotRequireAnotherCommittedTsoFlush() throws Ex } } - private void prepareWindowRead(boolean finishRecovery) throws Exception { + private void prepareWindowRead() throws Exception { Mockito.when(env.isReady()).thenReturn(true); Mockito.when(env.isMaster()).thenReturn(true); Mockito.when(env.getTSOService()).thenReturn(tsoService); setInitializedFlag(tsoService, true); setGlobalTimestamp(tsoService, 100, 0); tsoService.replayWindowEndTSO(new TSOServiceState(2000, TSOTimestamp.composeTimestamp(80, 1))); - if (finishRecovery) { - Field field = TSOService.class.getDeclaredField("transactionTracker"); - field.setAccessible(true); - GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(Mockito.eq(1000L), Mockito.anyLong(), Mockito.eq(ByteString.EMPTY))) - .thenReturn(TSOTransactionTrackerTest.recoveryBatch(ByteString.EMPTY)); - ((TSOTransactionTracker) field.get(tsoService)).checkTransactions(txnMgr, Long.MAX_VALUE); - } } @Test public void testSlowTableDoesNotBlockAnotherTableOrAnEarlierEnd() throws Exception { - prepareWindowRead(true); + prepareWindowRead(); long pending = tsoService.getCommitTSO(1, 10, Set.of(100L, 101L)); setGlobalTimestamp(tsoService, 150, 17); TSOService.TSOStatusSnapshot unrelated = tsoService.waitForReadableWindow( @@ -662,23 +680,20 @@ public void testSlowTableDoesNotBlockAnotherTableOrAnEarlierEnd() throws Excepti } @Test - public void testFutureWindowAndRecoveryHaveDifferentReasonsFromWaitTimeout() throws Exception { - prepareWindowRead(false); + public void testFutureWindowHasDifferentReasonFromWaitTimeout() throws Exception { + prepareWindowRead(); Map> tables = Map.of(1L, Collections.singletonList(100L)); IncrWindowNotReadyException future = Assertions.assertThrows(IncrWindowNotReadyException.class, () -> tsoService.waitForReadableWindow(tables, 101, 0)); Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, future.getMysqlErrorCode()); Assertions.assertEquals("END_AFTER_CURRENT_TSO", future.getReason()); - IncrWindowNotReadyException recovering = Assertions.assertThrows(IncrWindowNotReadyException.class, - () -> tsoService.waitForReadableWindow(tables, 90, 0)); - Assertions.assertEquals(ErrorCode.ERR_INCR_WINDOW_NOT_READY, recovering.getMysqlErrorCode()); - Assertions.assertEquals("TSO_RECOVERING", recovering.getReason()); + tsoService.waitForReadableWindow(tables, 90, 0); tsoService.waitForReadableWindow(tables, 80, 0); } @Test public void testInterruptedReadWaitRestoresInterruptFlag() throws Exception { - prepareWindowRead(true); + prepareWindowRead(); tsoService.getCommitTSO(1, 10, Collections.singleton(100L)); setGlobalTimestamp(tsoService, 150, 17); try { @@ -701,14 +716,6 @@ public void testCommittedPrefixIsPublishedOnlyAfterJournalSuccess() throws Excep Mockito.when(env.getEditLog()).thenReturn(editLog); tsoService.replayWindowEndTSO(new TSOServiceState(200, 80)); setGlobalTimestamp(tsoService, 100, 10); - Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); - trackerField.setAccessible(true); - TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); - GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(Mockito.eq(1000L), Mockito.anyLong(), Mockito.eq(ByteString.EMPTY))) - .thenReturn(TSOTransactionTrackerTest.recoveryBatch(ByteString.EMPTY)); - tracker.checkTransactions(txnMgr, Long.MAX_VALUE); try (MockedStatic config = Mockito.mockStatic(Config.class)) { config.when(Config::isCloudMode).thenReturn(true); Mockito.doAnswer(invocation -> { @@ -727,7 +734,7 @@ public void testCommittedPrefixIsPublishedOnlyAfterJournalSuccess() throws Excep } @Test - public void testRecoveryFreezesPrefixAndPeriodicFlushWorksWithUnchangedWindow() throws Exception { + public void testCalibrationKeepsPrefixAndPeriodicFlushWorksWithUnchangedWindow() throws Exception { Mockito.when(env.isReady()).thenReturn(true); Mockito.when(env.isMaster()).thenReturn(true); mockPersistReady(); @@ -739,14 +746,6 @@ public void testRecoveryFreezesPrefixAndPeriodicFlushWorksWithUnchangedWindow() invokeCalibrateTimestamp(tsoService); Assertions.assertEquals(oldCommitted, tsoService.getStatusSnapshot().getCommittedTso()); long pendingTso = tsoService.getCommitTSO(1, 10, Collections.singleton(2L)); - Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); - trackerField.setAccessible(true); - TSOTransactionTracker tracker = (TSOTransactionTracker) trackerField.get(tsoService); - GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(Mockito.eq(1000L), Mockito.anyLong(), Mockito.eq(ByteString.EMPTY))) - .thenReturn(TSOTransactionTrackerTest.recoveryBatch(ByteString.EMPTY)); - tracker.checkTransactions(txnMgr, System.nanoTime()); long reservedWindow = tsoService.getWindowEndTSO(); Field lastPersist = TSOService.class.getDeclaredField("lastPersistNanos"); lastPersist.setAccessible(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java index bf6bdf77d9f116..f2c7d850a0cc70 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java @@ -17,25 +17,17 @@ package org.apache.doris.tso; -import org.apache.doris.cloud.proto.Cloud.GetTsoRecoveryTransactionsResponse; -import org.apache.doris.cloud.proto.Cloud.TxnInfoPB; -import org.apache.doris.cloud.proto.Cloud.TxnStatusPB; import org.apache.doris.common.Pair; -import org.apache.doris.common.UserException; -import org.apache.doris.transaction.GlobalTransactionMgrIface; -import org.apache.doris.transaction.TransactionState; -import org.apache.doris.transaction.TransactionStatus; -import com.google.protobuf.ByteString; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -43,454 +35,136 @@ import java.util.concurrent.locks.ReentrantLock; public class TSOTransactionTrackerTest { - static final long RECOVERY_FENCE_TSO = 1000L; - private final ReentrantLock lock = new ReentrantLock(); - private final TSOTransactionTracker tracker = new TSOTransactionTracker(lock); - private final GlobalTransactionMgrIface txnMgr = Mockito.mock(GlobalTransactionMgrIface.class); - - private void reset() { + private ReentrantLock lock; + private TSOTransactionTracker tracker; + private ExecutorService executor; + + @BeforeEach + public void setUp() { + lock = new ReentrantLock(); + tracker = new TSOTransactionTracker(lock); + executor = Executors.newSingleThreadExecutor(); lock.lock(); try { - tracker.reset(RECOVERY_FENCE_TSO); + tracker.reset(); } finally { lock.unlock(); } } - private void register(long dbId, long txnId, long tso) { - lock.lock(); - try { - tracker.register(Pair.of(dbId, txnId), tso, 0, Collections.singleton(100L)); - } finally { - lock.unlock(); - } + @AfterEach + public void tearDown() { + executor.shutdownNow(); } - private long candidate(long currentTso, long durableTso) { + private void register(long dbId, long txnId, long tso, long... tableIds) { + Set tables = new java.util.HashSet<>(); + for (long tableId : tableIds) { + tables.add(tableId); + } lock.lock(); try { - return tracker.candidateCommittedTso(currentTso, durableTso); + tracker.register(Pair.of(dbId, txnId), tso, System.nanoTime(), tables); } finally { lock.unlock(); } } - static GetTsoRecoveryTransactionsResponse recoveryBatch(ByteString nextKey, TxnInfoPB... transactions) { - return GetTsoRecoveryTransactionsResponse.newBuilder().setNextStartKey(nextKey) - .addAllTxnInfos(Arrays.asList(transactions)).build(); - } - - private static TxnInfoPB recoveryTxn(long dbId, long txnId, long tso, long... tables) { - TxnInfoPB.Builder info = TxnInfoPB.newBuilder().setDbId(dbId).setTxnId(txnId) - .setStatus(TxnStatusPB.TXN_STATUS_COMMITTED); - for (long table : tables) { - info.addTableIds(table); - } - if (tso > 0) { - info.setCommitTso(tso); - } - return info.build(); - } - - private void finishRecovery() throws Exception { - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.doReturn(recoveryBatch(ByteString.EMPTY)).when(txnMgr) - .getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); - } - - private TSOTransactionTracker.WaitResult awaitTable(long dbId, long tableId, long endTso) throws Exception { + private TSOTransactionTracker.WaitResult await(long dbId, long tableId, long endTso, long timeoutMs) + throws InterruptedException { lock.lock(); try { - return tracker.awaitTransactions(Collections.singletonMap(dbId, Collections.singletonList(tableId)), - endTso, 0); + return tracker.awaitTransactions(Map.of(dbId, List.of(tableId)), endTso, + TimeUnit.MILLISECONDS.toNanos(timeoutMs)); } finally { lock.unlock(); } } - @Test - public void testReadWaitFiltersDatabaseTableAndExclusivePhysicalEnd() throws Exception { - reset(); - finishRecovery(); - register(1, 10, TSOTimestamp.composeTimestamp(100, 1)); - long end = TSOTimestamp.composePhysicalTimestamp(101); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(2, 100, end)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 200, end)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, - awaitTable(1, 100, TSOTimestamp.composePhysicalTimestamp(100))); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, end)); - // Retrying later must not hide the original in-window request or lose any involved table. + private long candidate(long currentTso, long durableTso) { lock.lock(); try { - tracker.register(Pair.of(1L, 10L), TSOTimestamp.composeTimestamp(200, 1), 0, Set.of(100L, 200L)); + return tracker.candidateCommittedTso(currentTso, durableTso); } finally { lock.unlock(); } - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, end)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 200, end)); - tracker.transactionFinished(1, 10); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 200, end)); } @Test - public void testEmptyRegistrationSetCannotBypassRecovery() throws Exception { - reset(); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.RECOVERING, awaitTable(1, 100, 200)); - } + public void testWaitFiltersDatabaseTableAndEndTso() throws Exception { + register(1, 10, 100, 1000); + register(1, 20, 120, 2000); + register(2, 30, 130, 1000); - @Test - public void testLoadedRecoveryWaitsOnlyRelatedTablesAndKeepsPrefixFrozen() throws Exception { - reset(); - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) - .thenReturn(recoveryBatch(ByteString.EMPTY, - recoveryTxn(1, 10, 100, 100), recoveryTxn(1, 20, 80, 200))); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(2)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 300, 200)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(2, 100, 200)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 100, 90)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, 200)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 200, 90)); - Assertions.assertEquals(80, candidate(250, 80)); - tracker.transactionFinished(1, 10); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 100, 200)); - Assertions.assertEquals(80, candidate(250, 80)); - tracker.transactionFinished(1, 20); - Assertions.assertTrue(tracker.isRecoveryReady()); - Assertions.assertEquals(250, candidate(250, 80)); - } + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, await(1, 2000, 110, 0)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, await(2, 1000, 120, 0)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, await(1, 1000, 100, 0)); + Assertions.assertEquals(99, candidate(150, 90)); - @Test - public void testFailedBatchResumesWithoutOpeningAnIncompleteRecovery() throws Exception { - reset(); - ByteString nextKey = ByteString.copyFromUtf8("next batch"); - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) - .thenReturn(recoveryBatch(nextKey, recoveryTxn(1, 10, 100, 100))); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, nextKey)) - .thenThrow(new UserException("batch RPC failed")) - .thenReturn(recoveryBatch(ByteString.EMPTY, recoveryTxn(2, 20, 80, 200))); - Assertions.assertThrows(UserException.class, - () -> tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3))); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.RECOVERING, awaitTable(1, 300, 200)); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 300, 200)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, 200)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(2, 200, 200)); - Assertions.assertEquals(80, candidate(250, 80)); - Mockito.verify(txnMgr).getTransactionIdWatermark(); - Mockito.verify(txnMgr).getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY); - } - - @Test - public void testRecoveryBatchesMergeConcurrentRegistrationsAndFinishNotifications() throws Exception { - reset(); - ByteString nextKey = ByteString.copyFromUtf8("next batch"); - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)).thenAnswer(invocation -> { - Assertions.assertFalse(lock.isHeldByCurrentThread()); - register(1, 10, 200); // Same old transaction is retried through the new master. - return recoveryBatch(nextKey, recoveryTxn(1, 10, 80, 200)); - }); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, nextKey)).thenAnswer(invocation -> { - Assertions.assertFalse(lock.isHeldByCurrentThread()); - tracker.transactionFinished(1, 10); - register(2, 2000, 150); // New transactions must survive importing the old list. - return recoveryBatch(ByteString.EMPTY, recoveryTxn(1, 20, 80, 300)); - }); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 100, 300)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 200, 300)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(2, 100, 300)); - Assertions.assertEquals(80, candidate(300, 80)); + tracker.transactionFinished(1, 10); + Assertions.assertEquals(119, candidate(150, 90)); tracker.transactionFinished(1, 20); - Assertions.assertEquals(149, candidate(300, 80)); + tracker.transactionFinished(2, 30); + Assertions.assertEquals(150, candidate(150, 90)); } @Test - public void testRecoveredTransactionRetainsTablesAcrossLocalRetry() throws Exception { - reset(); - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) - .thenReturn(recoveryBatch(ByteString.EMPTY, recoveryTxn(1, 10, 80, 200))); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); - register(1, 10, 300); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, 200)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 200, 200)); - Assertions.assertEquals(80, candidate(400, 80)); + public void testWaitUsesFixedRegistrationSnapshot() throws Exception { + register(1, 10, 100, 1000); + Future waiting = executor.submit(() -> await(1, 1000, 150, 5000)); + Thread.sleep(100); + register(1, 20, 120, 1000); tracker.transactionFinished(1, 10); - Assertions.assertEquals(400, candidate(400, 80)); - } - - @Test - public void testRecoveredTransactionsAreReconciledInBoundedRotatingBatches() throws Exception { - reset(); - TxnInfoPB[] transactions = new TxnInfoPB[150]; - for (int i = 0; i < transactions.length; i++) { - transactions[i] = recoveryTxn(1, i + 1, 80, 100); - } - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) - .thenReturn(recoveryBatch(ByteString.EMPTY, transactions)); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); - Mockito.verify(txnMgr, Mockito.atMost(64)).getTransactionState(Mockito.anyLong(), Mockito.anyLong()); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(5)); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(6)); - Mockito.verify(txnMgr).getTransactionState(1, 150); - Assertions.assertEquals(80, candidate(200, 80)); // Missing state must retain recovery entries. - TransactionState state = Mockito.mock(TransactionState.class); - Mockito.when(state.getTransactionStatus()).thenReturn(TransactionStatus.COMMITTED); - Mockito.when(txnMgr.getTransactionState(Mockito.anyLong(), Mockito.anyLong())).thenReturn(state); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(7)); - Assertions.assertEquals(80, candidate(200, 80)); - Mockito.when(state.getTransactionStatus()).thenReturn(TransactionStatus.ABORTED); - for (int i = 0; i < 3; i++) { - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(8 + i)); - } - Assertions.assertTrue(tracker.isRecoveryReady()); - Assertions.assertEquals(200, candidate(200, 80)); - } - - @Test - public void testRecoveredTransactionCompletionWakesOnlyItsWaiters() throws Exception { - reset(); - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) - .thenReturn(recoveryBatch(ByteString.EMPTY, - recoveryTxn(1, 10, 80, 100), recoveryTxn(2, 20, 80, 100))); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); - CountDownLatch started = new CountDownLatch(1); - ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - Future waiting = submitReadWait(executor, started); - Assertions.assertTrue(started.await(30, TimeUnit.SECONDS)); - TransactionState aborted = Mockito.mock(TransactionState.class); - Mockito.when(aborted.getTransactionStatus()).thenReturn(TransactionStatus.ABORTED); - Mockito.when(txnMgr.getTransactionState(1, 10)).thenReturn(aborted); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, waiting.get(30, TimeUnit.SECONDS)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(2, 100, 200)); - Assertions.assertEquals(80, candidate(300, 80)); - } finally { - executor.shutdownNow(); - } - } - - private Future submitReadWait(ExecutorService executor, CountDownLatch started) { - return executor.submit(() -> { - lock.lock(); - try { - started.countDown(); - return tracker.awaitTransactions(Map.of(1L, Collections.singletonList(100L)), - 200, TimeUnit.SECONDS.toNanos(30)); - } finally { - lock.unlock(); - } - }); - } - - @Test - public void testReconciliationWakesReadWithoutAdvancingTheGlobalPrefix() throws Exception { - reset(); - finishRecovery(); - register(1, 10, 100); - register(2, 20, 90); // An unrelated database continues to hold the global prefix. - CountDownLatch started = new CountDownLatch(1); - ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - Future waiting = submitReadWait(executor, started); - Assertions.assertTrue(started.await(30, TimeUnit.SECONDS)); - TransactionState aborted = Mockito.mock(TransactionState.class); - Mockito.when(aborted.getTransactionStatus()).thenReturn(TransactionStatus.ABORTED); - Mockito.when(txnMgr.getTransactionState(1, 10)).thenReturn(aborted); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, waiting.get(30, TimeUnit.SECONDS)); - Assertions.assertEquals(89, candidate(300, 80)); - } finally { - executor.shutdownNow(); - } - } - - @Test - public void testWaitReleasesAllocatorLockAndDoesNotFollowLaterWrites() throws Exception { - reset(); - finishRecovery(); - register(1, 10, 100); - CountDownLatch started = new CountDownLatch(1); - ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - Future waiting = submitReadWait(executor, started); - Assertions.assertTrue(started.await(30, TimeUnit.SECONDS)); - // Acquiring the lock proves the waiter released it; allocation can continue while it waits. - register(1, 20, 300); - tracker.transactionFinished(1, 10); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, waiting.get(30, TimeUnit.SECONDS)); - Assertions.assertEquals(1, tracker.getPendingCount()); - } finally { - executor.shutdownNow(); - } - } - - @Test - public void testResetInvalidatesAnInFlightReadWait() throws Exception { - reset(); - finishRecovery(); - register(1, 10, 100); - CountDownLatch started = new CountDownLatch(1); - ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - Future waiting = submitReadWait(executor, started); - Assertions.assertTrue(started.await(30, TimeUnit.SECONDS)); - reset(); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.RECOVERING, waiting.get(30, TimeUnit.SECONDS)); - } finally { - executor.shutdownNow(); - } - } - @Test - public void testOutOfOrderVisibilityAndRetryRetainEarliestTso() throws Exception { - reset(); - finishRecovery(); - register(1, 10, 100); - register(1, 20, 120); - tracker.transactionFinished(1, 20); - Assertions.assertEquals(99, candidate(150, 80)); - register(1, 10, 200); // A retry must not hide a delayed request carrying TSO 100. - Assertions.assertEquals(99, candidate(250, 99)); - tracker.transactionFinished(1, 10); - tracker.transactionFinished(1, 10); // Duplicate terminal notification is harmless. - Assertions.assertEquals(250, candidate(250, 99)); + Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, waiting.get(5, TimeUnit.SECONDS)); + Assertions.assertEquals(119, candidate(150, 90)); } @Test - public void testFencedRetryReplacesRejectedTso() throws Exception { - reset(); - finishRecovery(); - register(1, 10, 100); - Assertions.assertEquals(99, candidate(250, 80)); + public void testResetInvalidatesWait() throws Exception { + register(1, 10, 100, 1000); + Future waiting = executor.submit(() -> await(1, 1000, 150, 5000)); + Thread.sleep(100); lock.lock(); try { - tracker.replaceFenced(Pair.of(1L, 10L), 100, 200, 201, 0, Set.of(100L, 200L)); + tracker.reset(); } finally { lock.unlock(); } - Assertions.assertEquals(200, candidate(250, 99)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, awaitTable(1, 100, 150)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 200, 250)); + + Assertions.assertEquals(TSOTransactionTracker.WaitResult.RESET, waiting.get(5, TimeUnit.SECONDS)); } @Test - public void testFencedRetryRegistersTransactionRecoveredOnNewMaster() throws Exception { - reset(); - finishRecovery(); + public void testFencedReplacementAndAttemptScopedAbandon() { + register(1, 10, 100, 1000); lock.lock(); try { - tracker.replaceFenced(Pair.of(1L, 10L), 100, 200, 201, 0, Set.of(100L)); + tracker.replaceFenced(Pair.of(1L, 10L), 100, 110, 120, + System.nanoTime(), Collections.singleton(2000L)); } finally { lock.unlock(); } - Assertions.assertEquals(200, candidate(250, 99)); - Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, awaitTable(1, 100, 250)); - } - @Test - public void testRecoveryStartsImmediatelyAndPreservesNewPending() throws Exception { - reset(); - register(2, 2001, 200); - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L, 2000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) - .thenReturn(recoveryBatch(ByteString.EMPTY, recoveryTxn(1, 10, 80, 100))); - tracker.checkTransactions(txnMgr, 0); - Assertions.assertEquals(80, candidate(250, 80)); - TransactionState aborted = Mockito.mock(TransactionState.class); - Mockito.when(aborted.getTransactionStatus()).thenReturn(TransactionStatus.ABORTED); - Mockito.when(txnMgr.getTransactionState(1, 10)).thenReturn(aborted); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); - Assertions.assertEquals(199, candidate(250, 80)); - Mockito.verify(txnMgr).getTransactionIdWatermark(); - Mockito.verify(txnMgr).getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY); + Assertions.assertEquals(119, candidate(150, 90)); + tracker.abandonCommitTso(1, 10, 100); + Assertions.assertEquals(1, tracker.getPendingCount()); + tracker.abandonCommitTso(1, 10, 120); + Assertions.assertEquals(0, tracker.getPendingCount()); } @Test - public void testOnlyRealTerminalStatesReleasePending() throws Exception { - reset(); - finishRecovery(); - for (TransactionStatus status : TransactionStatus.values()) { - register(1, 10, 100); - TransactionState state = Mockito.mock(TransactionState.class); - Mockito.when(state.getTransactionStatus()).thenReturn(status); - Mockito.when(txnMgr.getTransactionState(1, 10)).thenReturn(state); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); - boolean terminal = status == TransactionStatus.VISIBLE || status == TransactionStatus.ABORTED; - Assertions.assertEquals(terminal ? 150 : 99, candidate(150, 80), status.toString()); - } - } + public void testRepeatedRegistrationRetainsEarliestTso() { + register(1, 10, 100, 1000); + register(1, 10, 120, 2000); - @Test - public void testMissingTransactionAndFailedRecoveryNeverAdvance() throws Exception { - reset(); - register(1, 10, 100); - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)) - .thenThrow(new UserException("old MS has no recovery RPC")); - Assertions.assertThrows(UserException.class, - () -> tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3))); - Assertions.assertEquals(80, candidate(150, 80)); Assertions.assertEquals(1, tracker.getPendingCount()); - finishRecovery(); - Assertions.assertEquals(99, candidate(150, 80)); - } - - @Test - public void testRpcDoesNotHoldAllocatorLockAndOldResultCannotRemoveNewRegistration() throws Exception { - reset(); - finishRecovery(); - register(1, 10, 100); - TransactionState visible = Mockito.mock(TransactionState.class); - Mockito.when(visible.getTransactionStatus()).thenReturn(TransactionStatus.VISIBLE); - Mockito.when(txnMgr.getTransactionState(1, 10)).thenAnswer(invocation -> { - Assertions.assertFalse(lock.isHeldByCurrentThread()); - reset(); // Simulate reinitialization while an old reconciliation request is in flight. - register(1, 10, 200); - return visible; - }); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); + Assertions.assertEquals(100, tracker.getOldestPendingTso()); + Assertions.assertEquals(10, tracker.getOldestPendingTxnId()); + Assertions.assertTrue(tracker.getOldestPendingAgeMs() >= 0); + tracker.abandonCommitTso(1, 10, 120); Assertions.assertEquals(1, tracker.getPendingCount()); - Assertions.assertEquals(200, tracker.getOldestPendingTso()); - Assertions.assertFalse(tracker.isRecoveryReady()); - } - - @Test - public void testOldRecoveryResultCannotOpenNewRecovery() throws Exception { - reset(); - Mockito.when(txnMgr.getTransactionIdWatermark()).thenReturn(1000L); - Mockito.when(txnMgr.getTsoRecoveryTransactions(1000L, RECOVERY_FENCE_TSO, ByteString.EMPTY)).thenAnswer(invocation -> { - Assertions.assertFalse(lock.isHeldByCurrentThread()); - reset(); - return recoveryBatch(ByteString.EMPTY); - }); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(3)); - Assertions.assertFalse(tracker.isRecoveryReady()); - Assertions.assertEquals(80, candidate(150, 80)); - } - - @Test - public void testReconciliationBatchIsBoundedAndRotatesPastOldest() throws Exception { - reset(); - finishRecovery(); - for (int i = 1; i <= 150; i++) { - register(1, i, 1000 + i); - } - Mockito.clearInvocations(txnMgr); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(4)); - Mockito.verify(txnMgr, Mockito.atMost(64)).getTransactionState(Mockito.anyLong(), Mockito.anyLong()); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(5)); - tracker.checkTransactions(txnMgr, TimeUnit.SECONDS.toNanos(6)); - Mockito.verify(txnMgr, Mockito.times(3)).getTransactionState(1, 1); - Mockito.verify(txnMgr).getTransactionState(1, 150); - Assertions.assertEquals(150, tracker.getPendingCount()); + tracker.transactionFinished(1, 10); + Assertions.assertEquals(0, tracker.getPendingCount()); } } diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index cecbd0175099c4..b790def308d511 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -514,7 +514,7 @@ message TxnIndexPB { optional int64 parent_txn_id = 2; } -// Current-state FE master epoch fence. This key is intentionally not versioned. +// Current-state commit TSO fence. This key is intentionally not versioned. message TxnTsoFencePB { optional int64 fence_tso = 1; } @@ -1263,26 +1263,6 @@ message CheckTxnConflictResponse { repeated TxnInfoPB conflict_txns = 3; } -message GetTsoRecoveryTransactionsRequest { - optional string cloud_unique_id = 1; // For auth - // Exclusive transaction-ID bound, fixed for the complete recovery scan. - optional int64 end_txn_id = 2; - // Maximum running records scanned per batch, including IDs above the bound. - optional int32 batch_size = 3; - optional bytes start_key = 4; - optional string request_ip = 5; - // Only committed TSO transactions at or below this inclusive fence are recovered. - optional int64 tso_fence = 6; -} - -message GetTsoRecoveryTransactionsResponse { - optional MetaServiceResponseStatus status = 1; - // Includes expired running transactions from every database/table in the instance. - repeated TxnInfoPB txn_infos = 2; - // Always present on success; empty means the full instance has been scanned. - optional bytes next_start_key = 3; -} - message AdvanceTsoFenceRequest { optional string cloud_unique_id = 1; // For auth // Inclusive upper bound rejected by transaction commit. @@ -1988,7 +1968,7 @@ enum MetaServiceCode { STALE_TABLET_CACHE = 2012; STALE_PREPARE_ROWSET = 2013; TXN_ALREADY_COMMITED = 2014; - // The transaction commit TSO belongs to an earlier FE master epoch. + // The transaction commit TSO is no longer valid. TXN_COMMIT_TSO_FENCED = 2015; CLUSTER_NOT_FOUND = 3001; @@ -2449,7 +2429,6 @@ service MetaService { rpc get_current_max_txn_id(GetCurrentMaxTxnRequest) returns (GetCurrentMaxTxnResponse); rpc create_meta_sync_point(CreateMetaSyncPointRequest) returns (CreateMetaSyncPointResponse); rpc check_txn_conflict(CheckTxnConflictRequest) returns (CheckTxnConflictResponse); - rpc get_tso_recovery_transactions(GetTsoRecoveryTransactionsRequest) returns (GetTsoRecoveryTransactionsResponse); rpc advance_tso_fence(AdvanceTsoFenceRequest) returns (AdvanceTsoFenceResponse); rpc clean_txn_label(CleanTxnLabelRequest) returns (CleanTxnLabelResponse); rpc get_txn_id(GetTxnIdRequest) returns (GetTxnIdResponse); From 864546d5531d949e59f4e84f577df3d6bd8dd467 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Mon, 14 Sep 2026 21:53:13 +0800 Subject: [PATCH 10/13] [fix](regression) Remove trailing blank line from committed TSO output ### What problem does this PR solve? Issue Number: None Related PR: #67820 Problem Summary: The generated committed-TSO regression output ended with an extra blank line, causing git diff whitespace validation to fail. Remove the trailing blank line without changing any expected query result. ### Release note None ### Check List (For Author) - Test: No need to test (expected values are unchanged); git diff whitespace validation passed - Behavior changed: No - Does this need documentation: No --- regression-test/data/tso_p0/test_committed_tso.out | 1 - 1 file changed, 1 deletion(-) diff --git a/regression-test/data/tso_p0/test_committed_tso.out b/regression-test/data/tso_p0/test_committed_tso.out index 8f49ac168c2865..fe1a30cdf41211 100644 --- a/regression-test/data/tso_p0/test_committed_tso.out +++ b/regression-test/data/tso_p0/test_committed_tso.out @@ -21,4 +21,3 @@ true 1 2 3 - From 8c55eb086e95b736b6812cd00dfe4d50a9f19a9a Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Tue, 15 Sep 2026 13:14:35 +0800 Subject: [PATCH 11/13] [fix](binlog) Address committed TSO review feedback ### What problem does this PR solve? Issue Number: None Related PR: #67181, #67594 Problem Summary: The commit TSO fence check lacked request-level and Meta Service configuration gates, its error name was unclear, the fence key was unavailable through Meta Service HTTP KV tooling, and the persisted window time did not state its unit. Add both fence-check gates to every Cloud commit path, expose the fence key to HTTP encode/get/set, rename the stale TSO error, and clarify the FE tracker and durable-state names. ### Release note Cloud commit TSO fence checks can be controlled by the request and the mutable Meta Service configuration. Rejected stale commit TSOs now use TXN_COMMIT_TSO_EXPIRED. ### Check List (For Author) - Test: Unit Test - Targeted FE unit tests: 108 passed - Cloud MetaService TSO fence test: passed - Cloud HTTP encode/get/set tests: 11 passed - Full ./build.sh -j32: passed - Cloud clang-tidy on changed lines: passed - Behavior changed: Yes. Commit TSO fencing requires both request and Meta Service gates, and the stale TSO error was renamed. - Does this need documentation: No --- cloud/src/common/config.h | 1 + cloud/src/meta-service/http_encode_key.cpp | 1 + cloud/src/meta-service/meta_service_helper.h | 4 +- cloud/src/meta-service/meta_service_txn.cpp | 13 ++++-- cloud/test/http_encode_key_test.cpp | 11 +++++ cloud/test/meta_service_helper_test.cpp | 2 +- cloud/test/meta_service_test.cpp | 36 +++++++++++++++- .../CloudGlobalTransactionMgr.java | 8 ++-- .../apache/doris/httpv2/rest/TSOAction.java | 2 +- .../tablefunction/MetadataGenerator.java | 2 +- .../java/org/apache/doris/tso/TSOService.java | 41 +++++++++--------- .../org/apache/doris/tso/TSOServiceState.java | 17 ++++---- .../doris/tso/TSOTransactionTracker.java | 2 +- .../CloudGlobalTransactionMgrTest.java | 6 ++- .../org/apache/doris/tso/TSOServiceTest.java | 42 +++++++------------ .../doris/tso/TSOTransactionTrackerTest.java | 10 ++--- gensrc/proto/cloud.proto | 4 +- 17 files changed, 123 insertions(+), 79 deletions(-) diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index faae59ad7e33f2..c8f225fb8ec118 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -354,6 +354,7 @@ CONF_Bool(delete_bitmap_enable_retry_txn_conflict, "true"); CONF_mInt64(max_txn_commit_byte, "7340032"); CONF_Bool(enable_cloud_txn_lazy_commit, "true"); +CONF_mBool(enable_check_commit_tso_fence, "true"); CONF_Int32(txn_lazy_commit_rowsets_thresold, "1000"); CONF_Int32(txn_lazy_commit_num_threads, "8"); CONF_mBool(enable_cloud_parallel_txn_lazy_commit, "true"); diff --git a/cloud/src/meta-service/http_encode_key.cpp b/cloud/src/meta-service/http_encode_key.cpp index bab0864db39e60..31f0b7575acac4 100644 --- a/cloud/src/meta-service/http_encode_key.cpp +++ b/cloud/src/meta-service/http_encode_key.cpp @@ -250,6 +250,7 @@ static std::unordered_map{p}.get()); }, parse , parse_json}}, {"TxnIndexKey", {{"instance_id", "txn_id"}, [](param_type& p) { return txn_index_key(KeyInfoSetter{p}.get()); }, parse , parse_json}}, {"TxnRunningKey", {{"instance_id", "db_id", "txn_id"}, [](param_type& p) { return txn_running_key(KeyInfoSetter{p}.get()); }, parse , parse_json}}, + {"TxnTsoFenceKey", {{"instance_id"}, [](param_type& p) { return txn_tso_fence_key(KeyInfoSetter{p}.get()); }, parse , parse_json}}, {"PartitionVersionKey", {{"instance_id", "db_id", "tbl_id", "partition_id"}, [](param_type& p) { return partition_version_key(KeyInfoSetter{p}.get()); }, parse , parse_json}}, {"TableVersionKey", {{"instance_id", "db_id", "tbl_id"}, [](param_type& p) { return table_version_key(KeyInfoSetter{p}.get()); }, parse , parse_json}}, {"MetaRowsetKey", {{"instance_id", "tablet_id", "version"}, [](param_type& p) { return meta_rowset_key(KeyInfoSetter{p}.get()); }, parse , parse_json}}, diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index c012aad0bd971e..30c9cfce1faa45 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -61,9 +61,9 @@ inline std::pair resolve_response_code_and_msg(Met "[TXN_ALREADY_COMMITED will be converted to code=UNDEFINED_ERR for old version " "clients]"; return {MetaServiceCode::UNDEFINED_ERR, std::move(msg)}; - case MetaServiceCode::TXN_COMMIT_TSO_FENCED: + case MetaServiceCode::TXN_COMMIT_TSO_EXPIRED: msg += std::string((msg.empty() ? "" : ", ")) + - "[TXN_COMMIT_TSO_FENCED will be converted to code=UNDEFINED_ERR for old " + "[TXN_COMMIT_TSO_EXPIRED will be converted to code=UNDEFINED_ERR for old " "version clients]"; return {MetaServiceCode::UNDEFINED_ERR, std::move(msg)}; default: diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index abd1ea4600f7fb..aa210b0df39119 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -127,6 +127,8 @@ static void append_table_stream_commit_size_error(TxnErrorCode err, std::string& } } +// Reads the fence through the commit transaction so a concurrent fence update causes a conflict. +// Allows commits without a TSO or a persisted fence, and rejects a commit at or below the fence. static bool check_txn_commit_tso_fence(Transaction* txn, const std::string& instance_id, int64_t commit_tso, CommitTxnResponse* response, MetaServiceCode& code, std::string& msg) { @@ -153,7 +155,7 @@ static bool check_txn_commit_tso_fence(Transaction* txn, const std::string& inst } if (commit_tso <= fence.fence_tso()) { response->set_tso_fence(fence.fence_tso()); - code = MetaServiceCode::TXN_COMMIT_TSO_FENCED; + code = MetaServiceCode::TXN_COMMIT_TSO_EXPIRED; msg = fmt::format("commit TSO {} is fenced by {}", commit_tso, fence.fence_tso()); return false; } @@ -1944,7 +1946,8 @@ void MetaServiceImpl::commit_txn_immediately( return; } - if (txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED && + if (request->enable_check_commit_tso_fence() && config::enable_check_commit_tso_fence && + txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED && !check_txn_commit_tso_fence(txn.get(), instance_id, commit_tso, response, code, msg)) { return; } @@ -2781,7 +2784,8 @@ void MetaServiceImpl::commit_txn_eventually( return; } - if (txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED && + if (request->enable_check_commit_tso_fence() && config::enable_check_commit_tso_fence && + txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED && !check_txn_commit_tso_fence(txn.get(), instance_id, commit_tso, response, code, msg)) { return; } @@ -3186,7 +3190,8 @@ void MetaServiceImpl::commit_txn_with_sub_txn(const CommitTxnRequest* request, return; } - if (txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED && + if (request->enable_check_commit_tso_fence() && config::enable_check_commit_tso_fence && + txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED && !check_txn_commit_tso_fence(txn.get(), instance_id, commit_tso, response, code, msg)) { return; } diff --git a/cloud/test/http_encode_key_test.cpp b/cloud/test/http_encode_key_test.cpp index bb0bcb58f5af2f..f3d53e52b255d2 100644 --- a/cloud/test/http_encode_key_test.cpp +++ b/cloud/test/http_encode_key_test.cpp @@ -190,6 +190,17 @@ txn_id=126419752960)", }, R"({"table_ids":["10001"]})", }, + Input { + "TxnTsoFenceKey", + "instance_id=gavin-instance", + {hex(txn_tso_fence_key({"gavin-instance"}))}, + []() -> std::vector { + TxnTsoFencePB pb; + pb.set_fence_tso(100); + return {pb.SerializeAsString()}; + }, + R"({"fence_tso":"100"})", + }, Input { "PartitionVersionKey", "instance_id=gavin-instance&db_id=10086&tbl_id=10010&partition_id=10000", diff --git a/cloud/test/meta_service_helper_test.cpp b/cloud/test/meta_service_helper_test.cpp index e741df470dd9c8..7e297d00b1289a 100644 --- a/cloud/test/meta_service_helper_test.cpp +++ b/cloud/test/meta_service_helper_test.cpp @@ -413,7 +413,7 @@ TEST(MetaServiceHelperTest, ResponseStatusCoversEveryMetaServiceCode) { expect_legacy_fallback_response_status(covered_codes, MetaServiceCode::TXN_ALREADY_COMMITED, LegacyFallbackCode::UNDEFINED_ERR); - expect_legacy_fallback_response_status(covered_codes, MetaServiceCode::TXN_COMMIT_TSO_FENCED, + expect_legacy_fallback_response_status(covered_codes, MetaServiceCode::TXN_COMMIT_TSO_EXPIRED, LegacyFallbackCode::UNDEFINED_ERR); EXPECT_EQ(covered_codes.size(), static_cast(MetaServiceCode_descriptor()->value_count())) diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index cce90421fb50b7..4495fda5dda4e4 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -2783,6 +2783,11 @@ TEST(MetaServiceTest, GetCurrentMaxTxnIdTest) { } TEST(MetaServiceTest, TsoFenceIsMonotonicAndRejectsStaleCommit) { + bool old_enable_check_commit_tso_fence = config::enable_check_commit_tso_fence; + DORIS_CLOUD_DEFER { + config::enable_check_commit_tso_fence = old_enable_check_commit_tso_fence; + }; + config::enable_check_commit_tso_fence = true; auto meta_service = get_meta_service(); brpc::Controller cntl; AdvanceTsoFenceRequest fence_request; @@ -2805,9 +2810,10 @@ TEST(MetaServiceTest, TsoFenceIsMonotonicAndRejectsStaleCommit) { commit_request.set_db_id(666); commit_request.set_txn_id(txn_id); commit_request.set_commit_tso(100); + commit_request.set_enable_check_commit_tso_fence(true); CommitTxnResponse commit_response; meta_service->commit_txn(&cntl, &commit_request, &commit_response, nullptr); - ASSERT_EQ(commit_response.status().actual_code(), MetaServiceCode::TXN_COMMIT_TSO_FENCED); + ASSERT_EQ(commit_response.status().actual_code(), MetaServiceCode::TXN_COMMIT_TSO_EXPIRED); ASSERT_EQ(commit_response.tso_fence(), 100); commit_request.set_commit_tso(101); @@ -2821,6 +2827,34 @@ TEST(MetaServiceTest, TsoFenceIsMonotonicAndRejectsStaleCommit) { meta_service->commit_txn(&cntl, &commit_request, &commit_response, nullptr); ASSERT_EQ(commit_response.status().code(), MetaServiceCode::OK); + // FE can disable the check for one request. + int64_t request_check_disabled_txn_id = -1; + begin_txn(meta_service.get(), 666, "tso_fence_request_check_disabled", 1234, + request_check_disabled_txn_id); + CommitTxnRequest request_check_disabled; + request_check_disabled.set_db_id(666); + request_check_disabled.set_txn_id(request_check_disabled_txn_id); + request_check_disabled.set_commit_tso(100); + CommitTxnResponse request_check_disabled_response; + meta_service->commit_txn(&cntl, &request_check_disabled, &request_check_disabled_response, + nullptr); + ASSERT_EQ(request_check_disabled_response.status().code(), MetaServiceCode::OK); + + // MS can disable the check globally. + config::enable_check_commit_tso_fence = false; + int64_t server_check_disabled_txn_id = -1; + begin_txn(meta_service.get(), 666, "tso_fence_server_check_disabled", 1234, + server_check_disabled_txn_id); + CommitTxnRequest server_check_disabled; + server_check_disabled.set_db_id(666); + server_check_disabled.set_txn_id(server_check_disabled_txn_id); + server_check_disabled.set_commit_tso(100); + server_check_disabled.set_enable_check_commit_tso_fence(true); + CommitTxnResponse server_check_disabled_response; + meta_service->commit_txn(&cntl, &server_check_disabled, &server_check_disabled_response, + nullptr); + ASSERT_EQ(server_check_disabled_response.status().code(), MetaServiceCode::OK); + // Transactions without a commit TSO do not participate in binlog fencing. int64_t non_tso_txn_id = -1; begin_txn(meta_service.get(), 666, "tso_fence_non_tso_commit", 1234, non_tso_txn_id); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index 236b43e241c604..595b4c6529570a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -862,7 +862,7 @@ private TransactionState commitTxn(CommitTxnRequest.Builder builder, List
Set commitTsoTableIds = tableList.stream().map(Table::getId).collect(Collectors.toSet()); long commitTso = TransactionUtil.getCommitTSO(transactionId, database, commitTsoTableIds); if (commitTso > 0) { - builder.setCommitTso(commitTso); + builder.setCommitTso(commitTso).setEnableCheckCommitTsoFence(true); } CommitTxnRequest commitTxnRequest = builder.build(); try { @@ -891,7 +891,7 @@ private TransactionState commitTxn(CommitTxnRequest.Builder builder, List
if (LOG.isDebugEnabled()) { LOG.debug("retryTime:{}, commitTxnResponse:{}", retryTime, commitTxnResponse); } - if (commitTxnResponse.getStatus().getCode() == MetaServiceCode.TXN_COMMIT_TSO_FENCED) { + if (commitTxnResponse.getStatus().getCode() == MetaServiceCode.TXN_COMMIT_TSO_EXPIRED) { if (!commitTxnResponse.hasTsoFence() || commitTxnRequest.getCommitTso() <= 0) { throw new UserException("MetaService returned an invalid TSO fence response"); } @@ -937,7 +937,7 @@ private TransactionState commitTxn(CommitTxnRequest.Builder builder, List
commitTxnRequest.getDbId(), transactionId, commitTxnRequest.getCommitTso()); } else if (code == MetaServiceCode.OK || code == MetaServiceCode.TXN_ALREADY_VISIBLE || code == MetaServiceCode.TXN_ALREADY_ABORTED) { - Env.getCurrentEnv().getTSOService().transactionFinished(commitTxnRequest.getDbId(), transactionId); + Env.getCurrentEnv().getTSOService().markTxnFinished(commitTxnRequest.getDbId(), transactionId); } else { Env.getCurrentEnv().getTSOService().abandonCommitTso( commitTxnRequest.getDbId(), transactionId, commitTxnRequest.getCommitTso()); @@ -2110,7 +2110,7 @@ private void afterAbortTxnResp(AbortTxnResponse abortTxnResponse, String txnIdOr if (abortTxnResponse.hasTxnInfo() && (abortTxnResponse.getTxnInfo().getStatus() == TxnStatusPB.TXN_STATUS_ABORTED || abortTxnResponse.getTxnInfo().getStatus() == TxnStatusPB.TXN_STATUS_VISIBLE)) { - Env.getCurrentEnv().getTSOService().transactionFinished( + Env.getCurrentEnv().getTSOService().markTxnFinished( abortTxnResponse.getTxnInfo().getDbId(), abortTxnResponse.getTxnInfo().getTxnId()); } if (abortTxnResponse.getStatus().getCode() != MetaServiceCode.OK) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TSOAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TSOAction.java index 1e7b57563d8559..867518964064ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TSOAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TSOAction.java @@ -87,7 +87,7 @@ public Object getTSO(HttpServletRequest request, HttpServletResponse response) { // Prepare response data with detailed TSO information Map result = Maps.newHashMap(); - result.put("window_end_physical_time", statusSnapshot.getWindowEndPhysicalTime()); + result.put("window_end_physical_time", statusSnapshot.getWindowEndPhysicalTimeMs()); result.put("current_tso", currentTso); result.put("current_tso_physical_time", TSOTimestamp.extractPhysicalTime(currentTso)); result.put("current_tso_logical_counter", TSOTimestamp.extractLogicalCounter(currentTso)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 3db535c8bceeea..773b39dea65319 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -2351,7 +2351,7 @@ private static TFetchSchemaTableDataResult tsoStatusMetadataResult(boolean inclu long currentTso = statusSnapshot.getCurrentTso(); TRow row = new TRow(); - row.addToColumnValue(new TCell().setLongVal(statusSnapshot.getWindowEndPhysicalTime())); + row.addToColumnValue(new TCell().setLongVal(statusSnapshot.getWindowEndPhysicalTimeMs())); row.addToColumnValue(new TCell().setLongVal(currentTso)); row.addToColumnValue(new TCell().setLongVal(TSOTimestamp.extractPhysicalTime(currentTso))); row.addToColumnValue(new TCell().setLongVal(TSOTimestamp.extractLogicalCounter(currentTso))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java index 5f8afc6e6f6601..f0da225290152d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOService.java @@ -71,17 +71,18 @@ public class TSOService extends MasterDaemon { public static final class TSOStatusSnapshot { private final boolean initialized; private final long currentTso; - private final long windowEndPhysicalTime; + private final long windowEndPhysicalTimeMs; private final long committedTso; - public TSOStatusSnapshot(boolean initialized, long currentTso, long windowEndPhysicalTime) { - this(initialized, currentTso, windowEndPhysicalTime, 0); + public TSOStatusSnapshot(boolean initialized, long currentTso, long windowEndPhysicalTimeMs) { + this(initialized, currentTso, windowEndPhysicalTimeMs, 0); } - public TSOStatusSnapshot(boolean initialized, long currentTso, long windowEndPhysicalTime, long committedTso) { + public TSOStatusSnapshot(boolean initialized, long currentTso, long windowEndPhysicalTimeMs, + long committedTso) { this.initialized = initialized; this.currentTso = currentTso; - this.windowEndPhysicalTime = windowEndPhysicalTime; + this.windowEndPhysicalTimeMs = windowEndPhysicalTimeMs; this.committedTso = committedTso; } @@ -97,8 +98,8 @@ public long getCurrentTso() { return currentTso; } - public long getWindowEndPhysicalTime() { - return windowEndPhysicalTime; + public long getWindowEndPhysicalTimeMs() { + return windowEndPhysicalTimeMs; } } @@ -118,7 +119,7 @@ public TSOService() { public void registerMetrics() { Map gauges = new LinkedHashMap<>(); gauges.put("tso_committed", () -> durableState.getCommittedTso()); - gauges.put("tso_window_end_physical_time", () -> durableState.getPhysicalTimestamp()); + gauges.put("tso_window_end_physical_time", () -> durableState.getWindowEndPhysicalTimeMs()); gauges.put("tso_pending_transactions", transactionTracker::getPendingCount); gauges.put("tso_oldest_pending_tso", transactionTracker::getOldestPendingTso); gauges.put("tso_oldest_pending_txn_id", transactionTracker::getOldestPendingTxnId); @@ -230,8 +231,8 @@ public long getCommitTSOAfterFence(long dbId, long txnId, Set tableIds, return getTSO(Pair.of(dbId, txnId), tableIds, rejectedTso, fenceTso); } - public void transactionFinished(long dbId, long txnId) { - transactionTracker.transactionFinished(dbId, txnId); + public void markTxnFinished(long dbId, long txnId) { + transactionTracker.markTxnFinished(dbId, txnId); } public void abandonCommitTso(long dbId, long txnId, long tso) { @@ -280,7 +281,7 @@ private long getTSO(Pair transactionIdentity, Set tableIds, LOG.warn("TSO service only run on master FE"); if (fenceTso > 0) { throw new RuntimeException( - "TXN_COMMIT_TSO_FENCED: retry the commit through the current master FE"); + "TXN_COMMIT_TSO_EXPIRED: retry the commit through the current master FE"); } lastFailure = new RuntimeException("Current FE is not master"); try { @@ -344,7 +345,7 @@ public TSOStatusSnapshot getStatusSnapshot() { try { TSOServiceState state = durableState; return new TSOStatusSnapshot(isInitialized.get(), globalTimestamp.composeTimestamp(), - state.getPhysicalTimestamp(), state.getCommittedTso()); + state.getWindowEndPhysicalTimeMs(), state.getCommittedTso()); } finally { lock.unlock(); } @@ -419,7 +420,7 @@ private void calibrateTimestamp() throws UserException { return; } - long timeLast = durableState.getPhysicalTimestamp(); // Last timestamp from image/editlog replay + long timeLast = durableState.getWindowEndPhysicalTimeMs(); // Last timestamp from image/editlog replay long timeNow = System.currentTimeMillis() + Config.tso_time_offset_debug_mode; long nextPhysicalTime; long timeWindowEnd; @@ -570,10 +571,10 @@ private void updateTimestamp() { } // 4. Check if time window right boundary needs renewal - if ((durableState.getPhysicalTimestamp() - nextPhysicalTime) <= UPDATE_TIME_WINDOW_GUARD + if ((durableState.getWindowEndPhysicalTimeMs() - nextPhysicalTime) <= UPDATE_TIME_WINDOW_GUARD || System.nanoTime() - lastPersistNanos >= TimeUnit.MILLISECONDS.toNanos(Config.tso_service_window_duration_ms)) { - long nextWindowEnd = Math.max(durableState.getPhysicalTimestamp(), + long nextWindowEnd = Math.max(durableState.getWindowEndPhysicalTimeMs(), nextPhysicalTime + Config.tso_service_window_duration_ms); writeTimestampToBDBJE(nextWindowEnd); } @@ -694,7 +695,7 @@ private Pair generateTSO(Pair transactionIdentity, Set 0 && globalTimestamp.composeTimestamp() < fenceTso) { - throw new RuntimeException("TXN_COMMIT_TSO_FENCED: local TSO " + throw new RuntimeException("TXN_COMMIT_TSO_EXPIRED: local TSO " + globalTimestamp.composeTimestamp() + " is behind MetaService fence " + fenceTso); } long logicalCounter = globalTimestamp.getLogicalCounter(); @@ -763,7 +764,7 @@ public void replayWindowEndTSO(TSOServiceState state) { } public long getWindowEndTSO() { - return durableState.getPhysicalTimestamp(); + return durableState.getWindowEndPhysicalTimeMs(); } public long saveTSO(CountingDataOutputStream dos, long checksum) throws IOException { @@ -771,7 +772,7 @@ public long saveTSO(CountingDataOutputStream dos, long checksum) throws IOExcept return checksum; } TSOServiceState state = durableState; - long currentWindowEnd = state.getPhysicalTimestamp(); + long currentWindowEnd = state.getWindowEndPhysicalTimeMs(); if (currentWindowEnd <= 0) { return checksum; } @@ -784,8 +785,8 @@ public long saveTSO(CountingDataOutputStream dos, long checksum) throws IOExcept public long loadTSO(DataInputStream dis, long checksum) throws IOException { TSOServiceState state = TSOServiceState.read(dis); durableState = state; - long newChecksum = checksum ^ state.getPhysicalTimestamp(); - LOG.info("Finished replay TSO windowEndTSO {} from image", durableState.getPhysicalTimestamp()); + long newChecksum = checksum ^ state.getWindowEndPhysicalTimeMs(); + LOG.info("Finished replay TSO windowEndTSO {} from image", durableState.getWindowEndPhysicalTimeMs()); return newChecksum; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOServiceState.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOServiceState.java index cd658da6e06654..917175a3391645 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOServiceState.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOServiceState.java @@ -29,24 +29,21 @@ /** The durable allocation window and readable transaction prefix, published as one snapshot. */ public final class TSOServiceState implements Writable { - // Keep the old TSOTimestamp JSON fields and image checksum for journal/image compatibility. - @SerializedName("physicalTimestamp") - private final long windowEndPhysicalTime; - @SerializedName("logicalCounter") - private final long logicalCounter = 0; + @SerializedName("windowEndPhysicalTimeMs") + private final long windowEndPhysicalTimeMs; @SerializedName("committedTso") private final long committedTso; - public TSOServiceState(long windowEndPhysicalTime, long committedTso) { - this.windowEndPhysicalTime = windowEndPhysicalTime; + public TSOServiceState(long windowEndPhysicalTimeMs, long committedTso) { + this.windowEndPhysicalTimeMs = windowEndPhysicalTimeMs; this.committedTso = committedTso; } - public long getPhysicalTimestamp() { - return windowEndPhysicalTime; + public long getWindowEndPhysicalTimeMs() { + return windowEndPhysicalTimeMs; } - /** Zero means no readable prefix has been established, including records written by older FEs. */ + /** Zero means no readable prefix has been established. */ public long getCommittedTso() { return committedTso; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java index d2985844e5318e..a77ee9c61e77a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tso/TSOTransactionTracker.java @@ -146,7 +146,7 @@ long candidateCommittedTso(long currentTso, long durableCommittedTso) { return candidate; } - void transactionFinished(long dbId, long txnId) { + void markTxnFinished(long dbId, long txnId) { lock.lock(); try { remove(Pair.of(dbId, txnId)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java index 2c8d1a3a08fd8d..6e662ba7bfdade 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java @@ -264,7 +264,7 @@ public void testCommitTransactionRetriesWithTsoAboveFence() throws Exception { Mockito.when(proxy.commitTxn(Mockito.any())) .thenReturn(CommitTxnResponse.newBuilder() .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() - .setCode(MetaServiceCode.TXN_COMMIT_TSO_FENCED)) + .setCode(MetaServiceCode.TXN_COMMIT_TSO_EXPIRED)) .setTsoFence(200L) .build(), CommitTxnResponse.newBuilder() @@ -282,8 +282,10 @@ public void testCommitTransactionRetriesWithTsoAboveFence() throws Exception { ArgumentCaptor.forClass(Cloud.CommitTxnRequest.class); Mockito.verify(proxy, Mockito.times(2)).commitTxn(requests.capture()); Assertions.assertEquals(100L, requests.getAllValues().get(0).getCommitTso()); + Assertions.assertTrue(requests.getAllValues().get(0).getEnableCheckCommitTsoFence()); Assertions.assertEquals(201L, requests.getAllValues().get(1).getCommitTso()); - Mockito.verify(tsoService).transactionFinished(CatalogTestUtil.testDbId1, 123533L); + Assertions.assertTrue(requests.getAllValues().get(1).getEnableCheckCommitTsoFence()); + Mockito.verify(tsoService).markTxnFinished(CatalogTestUtil.testDbId1, 123533L); } finally { table.setBinlogConfig(originalBinlogConfig); Config.enable_feature_binlog = originalEnableFeatureBinlog; diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java index 0c95b98d475ac2..1862a7e911c998 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOServiceTest.java @@ -378,7 +378,7 @@ public void testCalibrateTimestampThrowsWhenPersistWriteFailsAndKeepNotInitializ TSOService.TSOStatusSnapshot statusSnapshot = tsoService.getStatusSnapshot(); Assertions.assertFalse(statusSnapshot.isInitialized()); Assertions.assertEquals(0L, statusSnapshot.getCurrentTso()); - Assertions.assertEquals(0L, statusSnapshot.getWindowEndPhysicalTime()); + Assertions.assertEquals(0L, statusSnapshot.getWindowEndPhysicalTimeMs()); try { tsoService.getTSO(); @@ -525,7 +525,7 @@ public void testFencedCommitRetryDoesNotAdvanceStaleMasterClock() throws Excepti () -> tsoService.getCommitTSOAfterFence(1, 10, Set.of(100L), TSOTimestamp.composePhysicalTimestamp(100), fenceTso)); - Assertions.assertTrue(failure.getMessage().contains("TXN_COMMIT_TSO_FENCED")); + Assertions.assertTrue(failure.getMessage().contains("TXN_COMMIT_TSO_EXPIRED")); Field trackerField = TSOService.class.getDeclaredField("transactionTracker"); trackerField.setAccessible(true); Assertions.assertEquals(0, @@ -672,7 +672,7 @@ public void testSlowTableDoesNotBlockAnotherTableOrAnEarlierEnd() throws Excepti Assertions.assertEquals(tsoService.getCurrentTSO(), error.getCurrentTso()); Assertions.assertEquals(unrelated.getCommittedTso(), error.getCommittedTso()); } - tsoService.transactionFinished(1, 10); + tsoService.markTxnFinished(1, 10); TSOService.TSOStatusSnapshot finished = tsoService.waitForReadableWindow( Map.of(1L, Collections.singletonList(100L)), 120, 0); // A table-specific successful read does not advance the global durable prefix. @@ -720,16 +720,16 @@ public void testCommittedPrefixIsPublishedOnlyAfterJournalSuccess() throws Excep config.when(Config::isCloudMode).thenReturn(true); Mockito.doAnswer(invocation -> { Assertions.assertEquals(80, tsoService.getStatusSnapshot().getCommittedTso()); - Assertions.assertEquals(200, tsoService.getStatusSnapshot().getWindowEndPhysicalTime()); + Assertions.assertEquals(200, tsoService.getStatusSnapshot().getWindowEndPhysicalTimeMs()); throw new RuntimeException("injected journal failure"); }).when(editLog).logTSOTimestampWindowEnd(Mockito.any()); Assertions.assertThrows(RuntimeException.class, () -> invokeWriteTimestampToBdbJe(tsoService, 300)); Assertions.assertEquals(80, tsoService.getStatusSnapshot().getCommittedTso()); - Assertions.assertEquals(200, tsoService.getStatusSnapshot().getWindowEndPhysicalTime()); + Assertions.assertEquals(200, tsoService.getStatusSnapshot().getWindowEndPhysicalTimeMs()); Mockito.doNothing().when(editLog).logTSOTimestampWindowEnd(Mockito.any()); invokeWriteTimestampToBdbJe(tsoService, 300); Assertions.assertEquals(tsoService.getCurrentTSO(), tsoService.getStatusSnapshot().getCommittedTso()); - Assertions.assertEquals(300, tsoService.getStatusSnapshot().getWindowEndPhysicalTime()); + Assertions.assertEquals(300, tsoService.getStatusSnapshot().getWindowEndPhysicalTimeMs()); } } @@ -754,7 +754,7 @@ public void testCalibrationKeepsPrefixAndPeriodicFlushWorksWithUnchangedWindow() invokeUpdateTimestamp(tsoService); Assertions.assertEquals(reservedWindow, tsoService.getWindowEndTSO()); Assertions.assertEquals(pendingTso - 1, tsoService.getStatusSnapshot().getCommittedTso()); - tsoService.transactionFinished(1, 10); + tsoService.markTxnFinished(1, 10); lastPersist.setLong(tsoService, System.nanoTime() - TimeUnit.MILLISECONDS.toNanos(Config.tso_service_window_duration_ms + 1L)); invokeUpdateTimestamp(tsoService); @@ -764,7 +764,7 @@ public void testCalibrationKeepsPrefixAndPeriodicFlushWorksWithUnchangedWindow() } @Test - public void testStateImageJournalAndOldTimestampCompatibility() throws Exception { + public void testStateImageAndJournal() throws Exception { long committed = TSOTimestamp.composeTimestamp(100, 17); tsoService.replayWindowEndTSO(new TSOServiceState(200, committed)); TSOService restored = new TSOService(); @@ -772,25 +772,15 @@ public void testStateImageJournalAndOldTimestampCompatibility() throws Exception new DataInputStream(new ByteArrayInputStream(saveTSOBytes(tsoService))), 0)); Assertions.assertEquals(committed, restored.getStatusSnapshot().getCommittedTso()); Assertions.assertFalse(restored.getStatusSnapshot().isInitialized()); - for (boolean legacy : new boolean[] {true, false}) { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - DataOutputStream out = new DataOutputStream(bytes); - out.writeShort(OperationType.OP_TSO_TIMESTAMP_WINDOW_END); - if (legacy) { - new TSOTimestamp(200, 0).write(out); - } else { - new TSOServiceState(200, committed).write(out); - } - JournalEntity entity = new JournalEntity(); - entity.readFields(new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))); - TSOServiceState state = (TSOServiceState) entity.getData(); - Assertions.assertEquals(200, state.getPhysicalTimestamp()); - Assertions.assertEquals(legacy ? 0 : committed, state.getCommittedTso()); - } ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - new TSOServiceState(200, committed).write(new DataOutputStream(bytes)); - Assertions.assertEquals(200, TSOTimestamp.read( - new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))).getPhysicalTimestamp()); + DataOutputStream out = new DataOutputStream(bytes); + out.writeShort(OperationType.OP_TSO_TIMESTAMP_WINDOW_END); + new TSOServiceState(200, committed).write(out); + JournalEntity entity = new JournalEntity(); + entity.readFields(new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))); + TSOServiceState state = (TSOServiceState) entity.getData(); + Assertions.assertEquals(200, state.getWindowEndPhysicalTimeMs()); + Assertions.assertEquals(committed, state.getCommittedTso()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java index f2c7d850a0cc70..7e9dded69f42b7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tso/TSOTransactionTrackerTest.java @@ -101,10 +101,10 @@ public void testWaitFiltersDatabaseTableAndEndTso() throws Exception { Assertions.assertEquals(TSOTransactionTracker.WaitResult.TIMED_OUT, await(1, 1000, 100, 0)); Assertions.assertEquals(99, candidate(150, 90)); - tracker.transactionFinished(1, 10); + tracker.markTxnFinished(1, 10); Assertions.assertEquals(119, candidate(150, 90)); - tracker.transactionFinished(1, 20); - tracker.transactionFinished(2, 30); + tracker.markTxnFinished(1, 20); + tracker.markTxnFinished(2, 30); Assertions.assertEquals(150, candidate(150, 90)); } @@ -114,7 +114,7 @@ public void testWaitUsesFixedRegistrationSnapshot() throws Exception { Future waiting = executor.submit(() -> await(1, 1000, 150, 5000)); Thread.sleep(100); register(1, 20, 120, 1000); - tracker.transactionFinished(1, 10); + tracker.markTxnFinished(1, 10); Assertions.assertEquals(TSOTransactionTracker.WaitResult.FINISHED, waiting.get(5, TimeUnit.SECONDS)); Assertions.assertEquals(119, candidate(150, 90)); @@ -164,7 +164,7 @@ public void testRepeatedRegistrationRetainsEarliestTso() { Assertions.assertTrue(tracker.getOldestPendingAgeMs() >= 0); tracker.abandonCommitTso(1, 10, 120); Assertions.assertEquals(1, tracker.getPendingCount()); - tracker.transactionFinished(1, 10); + tracker.markTxnFinished(1, 10); Assertions.assertEquals(0, tracker.getPendingCount()); } } diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 8b99c3de402666..81cf0df6a93bdc 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -1090,6 +1090,8 @@ message CommitTxnRequest { optional string request_ip = 12; optional int64 commit_tso = 13; repeated TableStreamUpdatePB table_stream_updates = 14; + // Whether Meta Service should reject commit_tso values at or below its current fence. + optional bool enable_check_commit_tso_fence = 15; } message SubTxnInfo { @@ -1969,7 +1971,7 @@ enum MetaServiceCode { STALE_PREPARE_ROWSET = 2013; TXN_ALREADY_COMMITED = 2014; // The transaction commit TSO is no longer valid. - TXN_COMMIT_TSO_FENCED = 2015; + TXN_COMMIT_TSO_EXPIRED = 2015; CLUSTER_NOT_FOUND = 3001; ALREADY_EXISTED = 3002; From 8d264ce294a4e28287aee7721c1acc381346ccb7 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Wed, 16 Sep 2026 12:02:18 +0800 Subject: [PATCH 12/13] [fix](fe) Preserve Arrow Flight error wrapping ### What problem does this PR solve? Issue Number: None Related PR: #67181, #67594 Problem Summary: A master merge added an earlier FlightRuntimeException catch in getFlightInfoStatement, which bypassed the existing error-code filter and exposed every Flight error directly. Remove that catch so only incremental-window errors retain their retryable status while unrelated errors keep the established INTERNAL wrapper. ### Release note None ### Check List (For Author) - Test: Unit Test - DorisFlightSqlProducerTest: 7 passed - Full ./build.sh -j32: passed - Behavior changed: No. This restores the intended Arrow Flight error handling. - Does this need documentation: No --- .../org/apache/doris/arrowflight/DorisFlightSqlProducer.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/arrowflight/DorisFlightSqlProducer.java index bd12f1c61eaef2..727e8b30d1d382 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/arrowflight/DorisFlightSqlProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/arrowflight/DorisFlightSqlProducer.java @@ -332,10 +332,6 @@ public FlightInfo getFlightInfoStatement(final CommandStatementQuery request, fi return FlightProtocolAdapter.of(connectContext).callCommand(connectContext, () -> executeQueryStatement(context.peerIdentity(), connectContext, request.getQuery(), descriptor)); - } catch (FlightRuntimeException e) { - // Already carries the status meant for the client, e.g. UNAVAILABLE from the session's - // command lock; wrapping it as INTERNAL would hide that. - throw e; } catch (Throwable e) { if (e instanceof FlightRuntimeException) { FlightRuntimeException flightError = (FlightRuntimeException) e; From fc4f996920282b095a09313ba2eac18d32ab00a4 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Thu, 17 Sep 2026 16:16:03 +0800 Subject: [PATCH 13/13] [fix](fe) Release maybe-committed TSO without advancing fence ### What problem does this PR solve? Issue Number: None Related PR: #67181, #67594 Problem Summary: A Meta Service KV_TXN_MAYBE_COMMITTED response means the underlying commit attempt is no longer in flight: it either committed or will never commit. Advancing the TSO fence before releasing this TSO adds an unnecessary Meta Service write. Release the tracked TSO directly for this response while retaining fence advancement for FE-to-Meta-Service RPC failures whose requests may still complete later. ### Release note None ### Check List (For Author) - Test: Unit Test - CloudCommittedTsoTest: 3 passed - Full `./build.sh -j32`: passed - Behavior changed: Yes. KV_TXN_MAYBE_COMMITTED releases its tracked TSO without advancing the fence. - Does this need documentation: No --- .../cloud/transaction/CloudGlobalTransactionMgr.java | 2 +- .../cloud/transaction/CloudCommittedTsoTest.java | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index f50a205cf7cf02..53fad5a02ed613 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -944,7 +944,7 @@ private TransactionState commitTxn(CommitTxnRequest.Builder builder, List
MetaServiceCode code = commitTxnResponse.getStatus().getCode(); if (commitTxnRequest.hasCommitTso()) { if (code == MetaServiceCode.KV_TXN_MAYBE_COMMITTED) { - Env.getCurrentEnv().getTSOService().fenceAndAbandonCommitTso( + Env.getCurrentEnv().getTSOService().abandonCommitTso( commitTxnRequest.getDbId(), transactionId, commitTxnRequest.getCommitTso()); } else if (code == MetaServiceCode.OK || code == MetaServiceCode.TXN_ALREADY_VISIBLE || code == MetaServiceCode.TXN_ALREADY_ABORTED) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java index ef8315f3c3b1cb..bcf770122a29ab 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudCommittedTsoTest.java @@ -93,7 +93,7 @@ public void testOrdinarySubTransactionAndTwoPhaseCommitReuseTsoAcrossRpcRetries( } @Test - public void testUncertainCommitResultsAreFencedBeforeRelease() throws Exception { + public void testRpcFailureIsFencedButMaybeCommittedIsReleased() throws Exception { Env env = Mockito.mock(Env.class); TSOService tsoService = Mockito.mock(TSOService.class); InternalCatalog catalog = Mockito.mock(InternalCatalog.class); @@ -117,6 +117,9 @@ public void testUncertainCommitResultsAreFencedBeforeRelease() throws Exception () -> commit.invoke(new CloudGlobalTransactionMgr(), Cloud.CommitTxnRequest.newBuilder().setDbId(1).setTxnId(10), Collections.emptyList(), 10L, false, Collections.emptyList(), Collections.emptyList())); + Mockito.verify(tsoService).fenceAndAbandonCommitTso(1, 10, 500); + Mockito.clearInvocations(tsoService); + Mockito.doReturn(CommitTxnResponse.newBuilder() .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() .setCode(MetaServiceCode.KV_TXN_MAYBE_COMMITTED)) @@ -125,9 +128,9 @@ public void testUncertainCommitResultsAreFencedBeforeRelease() throws Exception () -> commit.invoke(new CloudGlobalTransactionMgr(), Cloud.CommitTxnRequest.newBuilder().setDbId(1).setTxnId(10), Collections.emptyList(), 10L, false, Collections.emptyList(), Collections.emptyList())); - Mockito.verify(tsoService, Mockito.times(2)).fenceAndAbandonCommitTso(1, 10, 500); - Mockito.verify(tsoService, Mockito.never()).abandonCommitTso(Mockito.anyLong(), Mockito.anyLong(), - Mockito.anyLong()); + Mockito.verify(tsoService).abandonCommitTso(1, 10, 500); + Mockito.verify(tsoService, Mockito.never()).fenceAndAbandonCommitTso( + Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); } }