From 9dbcfe720427860c96b4f17e9ddb2cea3f792dd8 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Fri, 22 Aug 2025 11:31:18 +0800 Subject: [PATCH 01/15] add txn sink --- downstreamadapter/sink/sink.go | 29 +- downstreamadapter/sink/txnsink/db_executor.go | 124 +++++ .../sink/txnsink/db_executor_test.go | 372 ++++++++++++++ .../sink/txnsink/event_processor.go | 98 ++++ downstreamadapter/sink/txnsink/sink.go | 359 +++++++++++++ .../sink/txnsink/sql_generator.go | 216 ++++++++ .../sink/txnsink/sql_generator_test.go | 378 ++++++++++++++ downstreamadapter/sink/txnsink/types.go | 483 ++++++++++++++++++ downstreamadapter/sink/txnsink/types_test.go | 416 +++++++++++++++ pkg/common/types.go | 1 + 10 files changed, 2475 insertions(+), 1 deletion(-) create mode 100644 downstreamadapter/sink/txnsink/db_executor.go create mode 100644 downstreamadapter/sink/txnsink/db_executor_test.go create mode 100644 downstreamadapter/sink/txnsink/event_processor.go create mode 100644 downstreamadapter/sink/txnsink/sink.go create mode 100644 downstreamadapter/sink/txnsink/sql_generator.go create mode 100644 downstreamadapter/sink/txnsink/sql_generator_test.go create mode 100644 downstreamadapter/sink/txnsink/types.go create mode 100644 downstreamadapter/sink/txnsink/types_test.go diff --git a/downstreamadapter/sink/sink.go b/downstreamadapter/sink/sink.go index da922477f..fdfc04402 100644 --- a/downstreamadapter/sink/sink.go +++ b/downstreamadapter/sink/sink.go @@ -22,10 +22,12 @@ import ( "github.com/pingcap/ticdc/downstreamadapter/sink/kafka" "github.com/pingcap/ticdc/downstreamadapter/sink/mysql" "github.com/pingcap/ticdc/downstreamadapter/sink/pulsar" + "github.com/pingcap/ticdc/downstreamadapter/sink/txnsink" "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" + mysqlpkg "github.com/pingcap/ticdc/pkg/sink/mysql" "github.com/pingcap/ticdc/pkg/sink/util" ) @@ -50,7 +52,7 @@ func New(ctx context.Context, cfg *config.ChangefeedConfig, changefeedID common. scheme := config.GetScheme(sinkURI) switch scheme { case config.MySQLScheme, config.MySQLSSLScheme, config.TiDBScheme, config.TiDBSSLScheme: - return mysql.New(ctx, changefeedID, cfg, sinkURI) + return newTxnSinkAdapter(ctx, changefeedID, cfg, sinkURI) case config.KafkaScheme, config.KafkaSSLScheme: return kafka.New(ctx, changefeedID, sinkURI, cfg.SinkConfig) case config.PulsarScheme, config.PulsarSSLScheme, config.PulsarHTTPScheme, config.PulsarHTTPSScheme: @@ -63,6 +65,31 @@ func New(ctx context.Context, cfg *config.ChangefeedConfig, changefeedID common. return nil, errors.ErrSinkURIInvalid.GenWithStackByArgs(sinkURI) } +// newTxnSinkAdapter creates a txnSink adapter that uses the same database connection as mysqlSink +func newTxnSinkAdapter( + ctx context.Context, + changefeedID common.ChangeFeedID, + config *config.ChangefeedConfig, + sinkURI *url.URL, +) (Sink, error) { + // Use the same database connection logic as mysqlSink + _, db, err := mysqlpkg.NewMysqlConfigAndDB(ctx, changefeedID, sinkURI, config) + if err != nil { + return nil, err + } + + // Create txnSink configuration + txnConfig := &txnsink.TxnSinkConfig{ + MaxConcurrentTxns: 16, + BatchSize: 16, + FlushInterval: 100, + MaxSQLBatchSize: 1024 * 16, // 1MB + } + + // Create and return txnSink + return txnsink.New(ctx, changefeedID, db, txnConfig), nil +} + func Verify(ctx context.Context, cfg *config.ChangefeedConfig, changefeedID common.ChangeFeedID) error { sinkURI, err := url.Parse(cfg.SinkURI) if err != nil { diff --git a/downstreamadapter/sink/txnsink/db_executor.go b/downstreamadapter/sink/txnsink/db_executor.go new file mode 100644 index 000000000..0096eedb1 --- /dev/null +++ b/downstreamadapter/sink/txnsink/db_executor.go @@ -0,0 +1,124 @@ +package txnsink + +import ( + "context" + "database/sql" + "strings" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/errors" + "go.uber.org/zap" +) + +// DBExecutor handles database execution for transaction SQL +type DBExecutor struct { + db *sql.DB +} + +// NewDBExecutor creates a new database executor +func NewDBExecutor(db *sql.DB) *DBExecutor { + return &DBExecutor{ + db: db, + } +} + +// ExecuteSQLBatch executes a batch of SQL transactions +func (e *DBExecutor) ExecuteSQLBatch(batch []*TxnSQL) error { + if len(batch) == 0 { + return nil + } + + log.Debug("txnSink: executing SQL batch", + zap.Int("batchSize", len(batch))) + + // If batch size is 1, execute directly (SQL already contains BEGIN/COMMIT) + if len(batch) == 1 { + txnSQL := batch[0] + for _, sql := range txnSQL.SQLs { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + _, execErr := e.db.ExecContext(ctx, sql) + cancel() + + if execErr != nil { + log.Error("txnSink: failed to execute single SQL", + zap.String("sql", sql), + zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), + zap.Uint64("startTs", txnSQL.TxnGroup.StartTs), + zap.Error(execErr)) + return errors.Trace(execErr) + } + } + + log.Debug("txnSink: successfully executed single transaction", + zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), + zap.Uint64("startTs", txnSQL.TxnGroup.StartTs)) + return nil + } + + // For multiple transactions, use explicit transaction and combine SQLs + tx, err := e.db.Begin() + if err != nil { + return errors.Trace(err) + } + defer func() { + if err != nil { + tx.Rollback() + } + }() + + // Build combined SQL from all transactions + var combinedSQL strings.Builder + + // Collect all SQL statements from all transactions + for _, txnSQL := range batch { + for _, sql := range txnSQL.SQLs { + // Keep the original SQL with BEGIN/COMMIT + cleanSQL := strings.TrimSpace(sql) + if len(cleanSQL) > 0 { + combinedSQL.WriteString(cleanSQL) + if !strings.HasSuffix(cleanSQL, ";") { + combinedSQL.WriteString(";") + } + } + } + } + + finalSQL := combinedSQL.String() + + log.Debug("txnSink: executing combined SQL batch with explicit transaction", + zap.String("sql", finalSQL), + zap.Int("batchSize", len(batch))) + + // Execute the combined SQL within transaction + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + _, execErr := tx.ExecContext(ctx, finalSQL) + cancel() + + if execErr != nil { + log.Error("txnSink: failed to execute SQL batch", + zap.String("sql", finalSQL), + zap.Int("batchSize", len(batch)), + zap.Error(execErr)) + return errors.Trace(execErr) + } + + // Commit the transaction + if err = tx.Commit(); err != nil { + log.Error("txnSink: failed to commit batch transaction", + zap.Int("batchSize", len(batch)), + zap.Error(err)) + return errors.Trace(err) + } + + log.Debug("txnSink: successfully executed SQL batch", + zap.Int("batchSize", len(batch)), + zap.Int("sqlLength", len(finalSQL))) + + return nil +} + +// Close closes the database connection +func (e *DBExecutor) Close() error { + return e.db.Close() +} diff --git a/downstreamadapter/sink/txnsink/db_executor_test.go b/downstreamadapter/sink/txnsink/db_executor_test.go new file mode 100644 index 000000000..fc8250a3f --- /dev/null +++ b/downstreamadapter/sink/txnsink/db_executor_test.go @@ -0,0 +1,372 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package txnsink + +import ( + "database/sql" + "errors" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/stretchr/testify/require" +) + +func TestDBExecutor_ExecuteTransaction(t *testing.T) { + t.Parallel() + + // Create a mock database + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + executor := NewDBExecutor(db) + + // Create test transaction SQL + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: []*commonEvent.DMLEvent{}, + } + + txnSQL := &TxnSQL{ + TxnGroup: txnGroup, + SQLs: []string{"BEGIN;INSERT INTO test VALUES (1, 'test');UPDATE test SET name = 'updated' WHERE id = 1;COMMIT;"}, + Keys: map[string]struct{}{"key1": {}}, + } + + // Set up mock expectations - expect the full SQL with BEGIN/COMMIT + mock.ExpectExec("BEGIN;INSERT INTO test VALUES").WillReturnResult(sqlmock.NewResult(1, 1)) + + // Execute transaction + err = executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) + require.NoError(t, err) + + // Verify all expectations were met + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDBExecutor_ExecuteTransaction_EmptySQL(t *testing.T) { + t.Parallel() + + // Create a mock database + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + executor := NewDBExecutor(db) + + // Create test transaction SQL with empty SQL list + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: []*commonEvent.DMLEvent{}, + } + + txnSQL := &TxnSQL{ + TxnGroup: txnGroup, + SQLs: []string{}, + Keys: map[string]struct{}{}, + } + + // Execute transaction - should succeed without any database operations + err = executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) + require.NoError(t, err) + + // Verify no database operations were performed + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDBExecutor_ExecuteTransaction_BeginError(t *testing.T) { + t.Parallel() + + // Create a mock database + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + executor := NewDBExecutor(db) + + // Create test transaction SQL + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: []*commonEvent.DMLEvent{}, + } + + txnSQL := &TxnSQL{ + TxnGroup: txnGroup, + SQLs: []string{"BEGIN;INSERT INTO test VALUES (1, 'test');COMMIT;"}, + Keys: map[string]struct{}{"key1": {}}, + } + + // Set up mock to fail on SQL execution + mock.ExpectExec("BEGIN;INSERT INTO test VALUES").WillReturnError(errors.New("connection failed")) + + // Execute transaction + err = executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) + require.Error(t, err) + require.Contains(t, err.Error(), "connection failed") + + // Verify all expectations were met + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDBExecutor_ExecuteTransaction_ExecError(t *testing.T) { + t.Parallel() + + // Create a mock database + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + executor := NewDBExecutor(db) + + // Create test transaction SQL + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: []*commonEvent.DMLEvent{}, + } + + txnSQL := &TxnSQL{ + TxnGroup: txnGroup, + SQLs: []string{"BEGIN;INSERT INTO test VALUES (1, 'test');UPDATE test SET name = 'updated' WHERE id = 1;COMMIT;"}, + Keys: map[string]struct{}{"key1": {}}, + } + + // Set up mock expectations - SQL execution fails + mock.ExpectExec("BEGIN;INSERT INTO test VALUES").WillReturnError(errors.New("update failed")) + + // Execute transaction + err = executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) + require.Error(t, err) + require.Contains(t, err.Error(), "update failed") + + // Verify all expectations were met + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDBExecutor_ExecuteTransaction_CommitError(t *testing.T) { + t.Parallel() + + // Create a mock database + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + executor := NewDBExecutor(db) + + // Create test transaction SQL + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: []*commonEvent.DMLEvent{}, + } + + txnSQL := &TxnSQL{ + TxnGroup: txnGroup, + SQLs: []string{"BEGIN;INSERT INTO test VALUES (1, 'test');COMMIT;"}, + Keys: map[string]struct{}{"key1": {}}, + } + + // Set up mock expectations - SQL execution fails + mock.ExpectExec("BEGIN;INSERT INTO test VALUES").WillReturnError(errors.New("commit failed")) + + // Execute transaction + err = executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) + require.Error(t, err) + require.Contains(t, err.Error(), "commit failed") + + // Verify all expectations were met + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDBExecutor_ExecuteTransaction_Timeout(t *testing.T) { + t.Parallel() + + // Create a mock database + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + executor := NewDBExecutor(db) + + // Create test transaction SQL + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: []*commonEvent.DMLEvent{}, + } + + txnSQL := &TxnSQL{ + TxnGroup: txnGroup, + SQLs: []string{"BEGIN;INSERT INTO test VALUES (1, 'test');COMMIT;"}, + Keys: map[string]struct{}{"key1": {}}, + } + + // Set up mock expectations with delay + mock.ExpectExec("BEGIN;INSERT INTO test VALUES").WillDelayFor(35 * time.Second).WillReturnResult(sqlmock.NewResult(1, 1)) + // Should timeout after 30 seconds + + // Execute transaction + err = executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) + require.Error(t, err) + require.Contains(t, err.Error(), "canceling query due to user request") + + // Verify all expectations were met + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDBExecutor_ExecuteTransaction_MultipleSQL(t *testing.T) { + t.Parallel() + + // Create a mock database + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + executor := NewDBExecutor(db) + + // Create test transaction SQL with multiple statements + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: []*commonEvent.DMLEvent{}, + } + + txnSQL := &TxnSQL{ + TxnGroup: txnGroup, + SQLs: []string{ + "BEGIN;INSERT INTO users VALUES (1, 'alice');INSERT INTO users VALUES (2, 'bob');UPDATE users SET name = 'alice_updated' WHERE id = 1;DELETE FROM users WHERE id = 2;COMMIT;", + }, + Keys: map[string]struct{}{"user1": {}, "user2": {}}, + } + + // Set up mock expectations for the combined SQL + mock.ExpectExec("BEGIN;INSERT INTO users VALUES").WillReturnResult(sqlmock.NewResult(1, 1)) + + // Execute transaction + err = executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) + require.NoError(t, err) + + // Verify all expectations were met + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDBExecutor_Close(t *testing.T) { + t.Parallel() + + // Create a mock database + db, mock, err := sqlmock.New() + require.NoError(t, err) + + executor := NewDBExecutor(db) + + // Set up mock expectation for Close + mock.ExpectClose() + + // Close the executor + err = executor.Close() + require.NoError(t, err) + + // Verify all expectations were met + require.NoError(t, mock.ExpectationsWereMet()) +} + +// Test helper function to create a mock database with specific behavior +func createMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock, func()) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + + cleanup := func() { + db.Close() + } + + return db, mock, cleanup +} + +// Test helper function to create a test TxnSQL +func createTestTxnSQL(sqls []string) *TxnSQL { + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: []*commonEvent.DMLEvent{}, + } + + return &TxnSQL{ + TxnGroup: txnGroup, + SQLs: sqls, + Keys: map[string]struct{}{"test_key": {}}, + } +} + +// Benchmark tests +func BenchmarkDBExecutor_ExecuteTransaction(b *testing.B) { + db, mock, cleanup := createMockDB(&testing.T{}) + defer cleanup() + + executor := NewDBExecutor(db) + + // Create test transaction SQL + txnSQL := createTestTxnSQL([]string{ + "INSERT INTO test VALUES (1, 'test')", + "UPDATE test SET name = 'updated' WHERE id = 1", + }) + + // Set up mock expectations + mock.ExpectBegin() + mock.ExpectExec("INSERT INTO test VALUES").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE test SET").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Reset mock expectations for each iteration + mock.ExpectBegin() + mock.ExpectExec("INSERT INTO test VALUES").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("UPDATE test SET").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + + err := executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) + require.NoError(b, err) + } +} + +func BenchmarkDBExecutor_ExecuteTransaction_SingleSQL(b *testing.B) { + db, mock, cleanup := createMockDB(&testing.T{}) + defer cleanup() + + executor := NewDBExecutor(db) + + // Create test transaction SQL with single statement + txnSQL := createTestTxnSQL([]string{"INSERT INTO test VALUES (1, 'test')"}) + + // Set up mock expectations + mock.ExpectBegin() + mock.ExpectExec("INSERT INTO test VALUES").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Reset mock expectations for each iteration + mock.ExpectBegin() + mock.ExpectExec("INSERT INTO test VALUES").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + + err := executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) + require.NoError(b, err) + } +} diff --git a/downstreamadapter/sink/txnsink/event_processor.go b/downstreamadapter/sink/txnsink/event_processor.go new file mode 100644 index 000000000..cf305c111 --- /dev/null +++ b/downstreamadapter/sink/txnsink/event_processor.go @@ -0,0 +1,98 @@ +package txnsink + +import ( + "context" + + "github.com/pingcap/log" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "go.uber.org/zap" +) + +// EventProcessor handles DML events and checkpoint processing +type EventProcessor struct { + txnStore *TxnStore +} + +// NewEventProcessor creates a new event processor +func NewEventProcessor(txnStore *TxnStore) *EventProcessor { + return &EventProcessor{ + txnStore: txnStore, + } +} + +// ProcessDMLEvents processes DML events from the input channel +func (p *EventProcessor) ProcessDMLEvents(ctx context.Context, dmlEventChan <-chan *commonEvent.DMLEvent) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case event, ok := <-dmlEventChan: + if !ok { + return nil + } + p.processDMLEvent(event) + } + } +} + +// ProcessCheckpoints processes checkpoint timestamps from the input channel +func (p *EventProcessor) ProcessCheckpoints(ctx context.Context, checkpointChan <-chan uint64, txnChan chan<- *TxnGroup) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case checkpointTs, ok := <-checkpointChan: + if !ok { + return nil + } + if err := p.processCheckpoint(checkpointTs, txnChan); err != nil { + return err + } + } + } +} + +// processDMLEvent processes a single DML event +func (p *EventProcessor) processDMLEvent(event *commonEvent.DMLEvent) { + // Add event to the transaction store + p.txnStore.AddEvent(event) + + log.Debug("txnSink: processed DML event", + zap.Uint64("commitTs", event.CommitTs), + zap.Uint64("startTs", event.StartTs), + zap.Int64("tableID", event.GetTableID()), + zap.Int32("rowCount", event.Len())) +} + +// processCheckpoint processes a checkpoint timestamp +func (p *EventProcessor) processCheckpoint(checkpointTs uint64, txnChan chan<- *TxnGroup) error { + // Get all events with commitTs <= checkpointTs + events := p.txnStore.GetEventsByCheckpointTs(checkpointTs) + if len(events) == 0 { + log.Debug("txnSink: no events to process for checkpoint", + zap.Uint64("checkpointTs", checkpointTs)) + return nil + } + + // Events are already grouped by TxnStore.GetEventsByCheckpointTs + txnGroups := events + + // Send transaction groups to the output channel + for _, txnGroup := range txnGroups { + txnChan <- txnGroup + log.Debug("txnSink: sent transaction group", + zap.Uint64("commitTs", txnGroup.CommitTs), + zap.Uint64("startTs", txnGroup.StartTs), + zap.Int("eventCount", len(txnGroup.Events))) + } + + // Remove processed events from the store + p.txnStore.RemoveEventsByCheckpointTs(checkpointTs) + + log.Info("txnSink: processed checkpoint", + zap.Uint64("checkpointTs", checkpointTs), + zap.Int("eventCount", len(events)), + zap.Int("txnGroupCount", len(txnGroups))) + + return nil +} diff --git a/downstreamadapter/sink/txnsink/sink.go b/downstreamadapter/sink/txnsink/sink.go new file mode 100644 index 000000000..06139d046 --- /dev/null +++ b/downstreamadapter/sink/txnsink/sink.go @@ -0,0 +1,359 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package txnsink + +import ( + "context" + "database/sql" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/pkg/sink/util" + "go.uber.org/atomic" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +// Sink implements the txnSink interface for transaction-level SQL output +type Sink struct { + changefeedID common.ChangeFeedID + + // Core components + txnStore *TxnStore + conflictDetector *ConflictDetector + dbExecutor *DBExecutor + sqlGenerator *SQLGenerator + eventProcessor *EventProcessor + + // Configuration + config *TxnSinkConfig + + // Channels for coordination + dmlEventChan chan *commonEvent.DMLEvent + checkpointChan chan uint64 + txnChan chan *TxnGroup + sqlChan chan *TxnSQL + + // State management + isNormal *atomic.Bool + ctx context.Context + + // Statistics and metrics + statistics *metrics.Statistics +} + +// New creates a new txnSink instance +func New(ctx context.Context, changefeedID common.ChangeFeedID, db *sql.DB, config *TxnSinkConfig) *Sink { + if config == nil { + config = &TxnSinkConfig{ + MaxConcurrentTxns: 16, + BatchSize: 16, + FlushInterval: 100, + MaxSQLBatchSize: 1024 * 16, + } + } + + txnStore := NewTxnStore() + conflictDetector := NewConflictDetector(changefeedID) + dbExecutor := NewDBExecutor(db) + sqlGenerator := NewSQLGenerator() + eventProcessor := NewEventProcessor(txnStore) + + return &Sink{ + changefeedID: changefeedID, + txnStore: txnStore, + conflictDetector: conflictDetector, + dbExecutor: dbExecutor, + sqlGenerator: sqlGenerator, + eventProcessor: eventProcessor, + config: config, + dmlEventChan: make(chan *commonEvent.DMLEvent, 10000), + checkpointChan: make(chan uint64, 100), + txnChan: make(chan *TxnGroup, 10000), + sqlChan: make(chan *TxnSQL, 10000), + isNormal: atomic.NewBool(true), + ctx: ctx, + statistics: metrics.NewStatistics(changefeedID, "txnsink"), + } +} + +// SinkType returns the sink type +func (s *Sink) SinkType() common.SinkType { + return common.TxnSinkType +} + +// IsNormal returns whether the sink is in normal state +func (s *Sink) IsNormal() bool { + return s.isNormal.Load() +} + +// AddDMLEvent adds a DML event to the sink +func (s *Sink) AddDMLEvent(event *commonEvent.DMLEvent) { + s.dmlEventChan <- event +} + +// AddCheckpointTs adds a checkpoint timestamp to trigger transaction processing +func (s *Sink) AddCheckpointTs(ts uint64) { + s.checkpointChan <- ts +} + +// WriteBlockEvent writes a block event (not supported in txnSink) +func (s *Sink) WriteBlockEvent(event commonEvent.BlockEvent) error { + return errors.New("txnSink does not support block events") +} + +// SetTableSchemaStore sets the table schema store (not used in txnSink) +func (s *Sink) SetTableSchemaStore(tableSchemaStore *util.TableSchemaStore) { + // Not used in txnSink +} + +// Close closes the sink and releases resources +func (s *Sink) Close(removeChangefeed bool) { + s.isNormal.Store(false) + + // Close conflict detector + s.conflictDetector.Close() + + // Close database executor + s.dbExecutor.Close() + + // Close channels + close(s.dmlEventChan) + close(s.checkpointChan) + close(s.txnChan) + close(s.sqlChan) + + log.Info("txnSink: closed", + zap.String("namespace", s.changefeedID.Namespace()), + zap.String("changefeed", s.changefeedID.Name())) +} + +// Run starts the sink processing +func (s *Sink) Run(ctx context.Context) error { + namespace := s.changefeedID.Namespace() + changefeed := s.changefeedID.Name() + + log.Info("txnSink: starting", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed)) + + // Start conflict detector + s.conflictDetector.Run(ctx) + + // Start multiple transaction workers + eg, ctx := errgroup.WithContext(ctx) + for i := 0; i < s.config.MaxConcurrentTxns; i++ { + workerID := i + eg.Go(func() error { + return s.runTxnWorker(ctx, workerID) + }) + } + + // Start event processor for DML events + eg.Go(func() error { + return s.eventProcessor.ProcessDMLEvents(ctx, s.dmlEventChan) + }) + + // Start event processor for checkpoints + eg.Go(func() error { + return s.eventProcessor.ProcessCheckpoints(ctx, s.checkpointChan, s.txnChan) + }) + + // Start transaction processor + eg.Go(func() error { + return s.processTransactions(ctx) + }) + + // Start SQL batch processor + eg.Go(func() error { + return s.processSQLBatch(ctx) + }) + + err := eg.Wait() + if err != nil { + log.Error("txnSink: stopped with error", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Error(err)) + return err + } + + log.Info("txnSink: stopped normally", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed)) + + return nil +} + +// runTxnWorker runs a transaction worker (similar to mysqlSink's runDMLWriter) +func (s *Sink) runTxnWorker(ctx context.Context, idx int) error { + namespace := s.changefeedID.Namespace() + changefeed := s.changefeedID.Name() + + log.Info("txnSink: starting txn worker", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Int("workerID", idx)) + + inputCh := s.conflictDetector.GetOutChByCacheID(idx) + if inputCh == nil { + return errors.New("failed to get output channel from conflict detector") + } + + buffer := make([]*TxnGroup, 0, s.config.BatchSize) + for { + select { + case <-ctx.Done(): + return errors.Trace(ctx.Err()) + default: + // Get multiple txn groups from the channel + txnGroups, ok := inputCh.GetMultipleNoGroup(buffer) + if !ok { + return errors.Trace(ctx.Err()) + } + + if len(txnGroups) == 0 { + buffer = buffer[:0] + continue + } + + // Process each txn group + for _, txnGroup := range txnGroups { + if err := s.processTxnGroup(txnGroup); err != nil { + log.Error("txnSink: failed to process transaction group", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Int("workerID", idx), + zap.Uint64("commitTs", txnGroup.CommitTs), + zap.Uint64("startTs", txnGroup.StartTs), + zap.Error(err)) + return err + } + } + + buffer = buffer[:0] + } + } +} + +// processTransactions processes transactions from the transaction channel +func (s *Sink) processTransactions(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case txnGroup, ok := <-s.txnChan: + if !ok { + return nil + } + // Add transaction group to conflict detector + s.conflictDetector.AddTxnGroup(txnGroup) + } + } +} + +// processTxnGroup processes a single transaction group +func (s *Sink) processTxnGroup(txnGroup *TxnGroup) error { + // Convert to SQL and send to SQL channel + txnSQL, err := s.sqlGenerator.ConvertTxnGroupToSQL(txnGroup) + if err != nil { + return err + } + + // Send to SQL channel for batch processing + select { + case s.sqlChan <- txnSQL: + return nil + default: + return errors.New("SQL channel is full") + } +} + +// processSQLBatch processes SQL batches from the SQL channel +func (s *Sink) processSQLBatch(ctx context.Context) error { + namespace := s.changefeedID.Namespace() + changefeed := s.changefeedID.Name() + + log.Info("txnSink: starting SQL batch processor", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed)) + + batch := make([]*TxnSQL, 0, s.config.BatchSize) + currentBatchSize := 0 + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case txnSQL, ok := <-s.sqlChan: + if !ok { + // Channel closed, flush remaining batch + if len(batch) > 0 { + if err := s.executeSQLBatch(batch); err != nil { + log.Error("txnSink: failed to execute final SQL batch", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Error(err)) + return err + } + } + return nil + } + + // Calculate SQL size for this transaction + sqlSize := s.calculateSQLSize(txnSQL) + + // Check if adding this SQL would exceed batch size limit + if len(batch) > 0 && (currentBatchSize+sqlSize > s.config.MaxSQLBatchSize || len(batch) >= s.config.BatchSize) { + // Execute current batch before adding new SQL + if err := s.executeSQLBatch(batch); err != nil { + log.Error("txnSink: failed to execute SQL batch", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Error(err)) + return err + } + + // Reset batch + batch = batch[:0] + currentBatchSize = 0 + } + + // Add SQL to batch + batch = append(batch, txnSQL) + currentBatchSize += sqlSize + } + } +} + +// calculateSQLSize calculates the total size of SQL statements in a transaction +func (s *Sink) calculateSQLSize(txnSQL *TxnSQL) int { + totalSize := 0 + for _, sql := range txnSQL.SQLs { + totalSize += len(sql) + } + return totalSize +} + +// executeSQLBatch executes a batch of SQL transactions +func (s *Sink) executeSQLBatch(batch []*TxnSQL) error { + if len(batch) == 0 { + return nil + } + + return s.dbExecutor.ExecuteSQLBatch(batch) +} diff --git a/downstreamadapter/sink/txnsink/sql_generator.go b/downstreamadapter/sink/txnsink/sql_generator.go new file mode 100644 index 000000000..7b9a50f0e --- /dev/null +++ b/downstreamadapter/sink/txnsink/sql_generator.go @@ -0,0 +1,216 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package txnsink + +import ( + "strings" + + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/sink/sqlmodel" + "github.com/pingcap/tidb/pkg/util/chunk" +) + +// SQLGenerator handles SQL generation for transaction groups +type SQLGenerator struct{} + +// NewSQLGenerator creates a new SQL generator +func NewSQLGenerator() *SQLGenerator { + return &SQLGenerator{} +} + +// ConvertTxnGroupToSQL converts a transaction group to SQL statements +func (g *SQLGenerator) ConvertTxnGroupToSQL(txnGroup *TxnGroup) (*TxnSQL, error) { + // Group events by table for batch processing + tableEvents := make(map[int64][]*commonEvent.DMLEvent) + for _, event := range txnGroup.Events { + tableID := event.GetTableID() + tableEvents[tableID] = append(tableEvents[tableID], event) + } + + var allSQLs []string + var allArgs [][]interface{} + + // Process each table's events + for _, events := range tableEvents { + if len(events) == 0 { + continue + } + + // Generate SQL for this table's events + sqls, args, err := g.generateTableSQL(events) + if err != nil { + return nil, err + } + + allSQLs = append(allSQLs, sqls...) + allArgs = append(allArgs, args...) + } + + // Wrap in transaction + if len(allSQLs) > 0 { + transactionSQL := "BEGIN;" + strings.Join(allSQLs, ";") + ";COMMIT;" + transactionArgs := make([]interface{}, 0) + for _, arg := range allArgs { + transactionArgs = append(transactionArgs, arg...) + } + + return &TxnSQL{ + TxnGroup: txnGroup, + SQLs: []string{transactionSQL}, + Keys: txnGroup.ExtractKeys(), + }, nil + } + + return &TxnSQL{ + TxnGroup: txnGroup, + SQLs: []string{}, + Keys: txnGroup.ExtractKeys(), + }, nil +} + +// generateTableSQL generates SQL statements for events of the same table +func (g *SQLGenerator) generateTableSQL(events []*commonEvent.DMLEvent) ([]string, [][]interface{}, error) { + if len(events) == 0 { + return []string{}, [][]interface{}{}, nil + } + + tableInfo := events[0].TableInfo + + // Group rows by type (insert, update, delete) + insertRows, updateRows, deleteRows := g.groupRowsByType(events, tableInfo) + + var sqls []string + var args [][]interface{} + + // Handle delete operations + if len(deleteRows) > 0 { + for _, rows := range deleteRows { + sql, value := g.genDeleteSQL(rows...) + sqls = append(sqls, sql) + args = append(args, value) + } + } + + // Handle update operations - use INSERT ON DUPLICATE KEY UPDATE + if len(updateRows) > 0 { + for _, rows := range updateRows { + sql, value := g.genInsertOnDuplicateUpdateSQL(rows...) + sqls = append(sqls, sql) + args = append(args, value) + } + } + + // Handle insert operations - use INSERT ON DUPLICATE KEY UPDATE + if len(insertRows) > 0 { + for _, rows := range insertRows { + sql, value := g.genInsertOnDuplicateUpdateSQL(rows...) + sqls = append(sqls, sql) + args = append(args, value) + } + } + + return sqls, args, nil +} + +// groupRowsByType groups rows by their type (insert, update, delete) +func (g *SQLGenerator) groupRowsByType(events []*commonEvent.DMLEvent, tableInfo *common.TableInfo) ( + insertRows, updateRows, deleteRows [][]*sqlmodel.RowChange, +) { + insertRow := make([]*sqlmodel.RowChange, 0) + updateRow := make([]*sqlmodel.RowChange, 0) + deleteRow := make([]*sqlmodel.RowChange, 0) + + for _, event := range events { + event.Rewind() + for { + row, ok := event.GetNextRow() + if !ok { + break + } + + switch row.RowType { + case commonEvent.RowTypeInsert: + args := g.getArgsWithGeneratedColumn(&row.Row, tableInfo) + newInsertRow := sqlmodel.NewRowChange( + &tableInfo.TableName, + nil, + nil, + args, + tableInfo, + nil, nil) + insertRow = append(insertRow, newInsertRow) + + case commonEvent.RowTypeUpdate: + args := g.getArgsWithGeneratedColumn(&row.Row, tableInfo) + preArgs := g.getArgsWithGeneratedColumn(&row.PreRow, tableInfo) + newUpdateRow := sqlmodel.NewRowChange( + &tableInfo.TableName, + nil, + preArgs, + args, + tableInfo, + nil, nil) + updateRow = append(updateRow, newUpdateRow) + + case commonEvent.RowTypeDelete: + preArgs := g.getArgsWithGeneratedColumn(&row.PreRow, tableInfo) + newDeleteRow := sqlmodel.NewRowChange( + &tableInfo.TableName, + nil, + preArgs, + nil, + tableInfo, + nil, nil) + deleteRow = append(deleteRow, newDeleteRow) + } + } + } + + // Group rows into batches + if len(insertRow) > 0 { + insertRows = append(insertRows, insertRow) + } + if len(updateRow) > 0 { + updateRows = append(updateRows, updateRow) + } + if len(deleteRow) > 0 { + deleteRows = append(deleteRows, deleteRow) + } + + return +} + +// genDeleteSQL generates DELETE SQL for multiple rows +func (g *SQLGenerator) genDeleteSQL(rows ...*sqlmodel.RowChange) (string, []interface{}) { + return sqlmodel.GenDeleteSQL(rows...) +} + +// genInsertOnDuplicateUpdateSQL generates INSERT ON DUPLICATE KEY UPDATE SQL +func (g *SQLGenerator) genInsertOnDuplicateUpdateSQL(rows ...*sqlmodel.RowChange) (string, []interface{}) { + return sqlmodel.GenInsertSQL(sqlmodel.DMLInsertOnDuplicateUpdate, rows...) +} + +// getArgsWithGeneratedColumn extracts column values including generated columns +func (g *SQLGenerator) getArgsWithGeneratedColumn(row *chunk.Row, tableInfo *common.TableInfo) []interface{} { + args := make([]interface{}, 0, len(tableInfo.GetColumns())) + for i, col := range tableInfo.GetColumns() { + if col == nil { + continue + } + v := common.ExtractColVal(row, col, i) + args = append(args, v) + } + return args +} diff --git a/downstreamadapter/sink/txnsink/sql_generator_test.go b/downstreamadapter/sink/txnsink/sql_generator_test.go new file mode 100644 index 000000000..c60f111ff --- /dev/null +++ b/downstreamadapter/sink/txnsink/sql_generator_test.go @@ -0,0 +1,378 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package txnsink + +import ( + "strings" + "testing" + + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/stretchr/testify/require" +) + +func TestSQLGenerator_ConvertTxnGroupToSQL(t *testing.T) { + t.Parallel() + + generator := NewSQLGenerator() + helper := commonEvent.NewEventTestHelper(t) + defer helper.Close() + + // Setup test table + helper.Tk().MustExec("use test") + createTableSQL := "create table t (id int primary key, name varchar(32));" + job := helper.DDL2Job(createTableSQL) + require.NotNil(t, job) + + // Create test DML events + event1 := helper.DML2Event("test", "t", "insert into t values (1, 'test1')") + event1.CommitTs = 100 + event1.StartTs = 50 + + event2 := helper.DML2Event("test", "t", "insert into t values (2, 'test2')") + event2.CommitTs = 100 + event2.StartTs = 50 + + // Create transaction group + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: []*commonEvent.DMLEvent{event1, event2}, + } + + // Convert to SQL + txnSQL, err := generator.ConvertTxnGroupToSQL(txnGroup) + require.NoError(t, err) + require.NotNil(t, txnSQL) + + // Verify transaction structure + require.Equal(t, txnGroup, txnSQL.TxnGroup) + require.Len(t, txnSQL.SQLs, 1) + + // Verify SQL format: should start with BEGIN and end with COMMIT + sql := txnSQL.SQLs[0] + require.True(t, strings.HasPrefix(sql, "BEGIN;")) + require.True(t, strings.HasSuffix(sql, ";COMMIT;")) + + // Should contain INSERT ON DUPLICATE KEY UPDATE + require.Contains(t, sql, "INSERT INTO") + require.Contains(t, sql, "ON DUPLICATE KEY UPDATE") +} + +func TestSQLGenerator_ConvertTxnGroupToSQL_EmptyGroup(t *testing.T) { + t.Parallel() + + generator := NewSQLGenerator() + + // Create empty transaction group + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: []*commonEvent.DMLEvent{}, + } + + // Convert to SQL + txnSQL, err := generator.ConvertTxnGroupToSQL(txnGroup) + require.NoError(t, err) + require.NotNil(t, txnSQL) + + // Should have empty SQL list + require.Len(t, txnSQL.SQLs, 0) +} + +func TestSQLGenerator_ConvertTxnGroupToSQL_MultiTable(t *testing.T) { + t.Parallel() + + generator := NewSQLGenerator() + helper := commonEvent.NewEventTestHelper(t) + defer helper.Close() + + // Setup test tables + helper.Tk().MustExec("use test") + createTableSQL1 := "create table t1 (id int primary key, name varchar(32));" + job1 := helper.DDL2Job(createTableSQL1) + require.NotNil(t, job1) + + createTableSQL2 := "create table t2 (id int primary key, age int);" + job2 := helper.DDL2Job(createTableSQL2) + require.NotNil(t, job2) + + // Create events for different tables + event1 := helper.DML2Event("test", "t1", "insert into t1 values (1, 'test1')") + event1.CommitTs = 100 + event1.StartTs = 50 + + event2 := helper.DML2Event("test", "t2", "insert into t2 values (1, 25)") + event2.CommitTs = 100 + event2.StartTs = 50 + + // Create transaction group + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: []*commonEvent.DMLEvent{event1, event2}, + } + + // Convert to SQL + txnSQL, err := generator.ConvertTxnGroupToSQL(txnGroup) + require.NoError(t, err) + require.NotNil(t, txnSQL) + + // Should have one transaction SQL + require.Len(t, txnSQL.SQLs, 1) + sql := txnSQL.SQLs[0] + + // Should contain both tables + require.Contains(t, sql, "`test`.`t1`") + require.Contains(t, sql, "`test`.`t2`") + require.True(t, strings.HasPrefix(sql, "BEGIN;")) + require.True(t, strings.HasSuffix(sql, ";COMMIT;")) +} + +func TestSQLGenerator_groupRowsByType(t *testing.T) { + t.Parallel() + + generator := NewSQLGenerator() + helper := commonEvent.NewEventTestHelper(t) + defer helper.Close() + + // Setup test table + helper.Tk().MustExec("use test") + createTableSQL := "create table t (id int primary key, name varchar(32));" + job := helper.DDL2Job(createTableSQL) + require.NotNil(t, job) + + // Create different types of events + insertEvent := helper.DML2Event("test", "t", "insert into t values (1, 'test1')") + updateEvent, _ := helper.DML2UpdateEvent("test", "t", "insert into t values (2, 'test2')", "update t set name = 'updated' where id = 2") + deleteEvent := helper.DML2DeleteEvent("test", "t", "insert into t values (3, 'test3')", "delete from t where id = 3") + + events := []*commonEvent.DMLEvent{insertEvent, updateEvent, deleteEvent} + tableInfo := insertEvent.TableInfo + + // Group rows by type + insertRows, updateRows, deleteRows := generator.groupRowsByType(events, tableInfo) + + // Verify grouping + require.Len(t, insertRows, 1) // One batch of insert rows + require.Len(t, updateRows, 1) // One batch of update rows + require.Len(t, deleteRows, 1) // One batch of delete rows + + // Verify row counts in each batch + require.Len(t, insertRows[0], 1) // One insert row + require.Len(t, updateRows[0], 1) // One update row + require.Len(t, deleteRows[0], 1) // One delete row +} + +func TestSQLGenerator_generateTableSQL(t *testing.T) { + t.Parallel() + + generator := NewSQLGenerator() + helper := commonEvent.NewEventTestHelper(t) + defer helper.Close() + + // Setup test table + helper.Tk().MustExec("use test") + createTableSQL := "create table t (id int primary key, name varchar(32));" + job := helper.DDL2Job(createTableSQL) + require.NotNil(t, job) + + // Create test events + insertEvent := helper.DML2Event("test", "t", "insert into t values (1, 'test1')") + deleteEvent := helper.DML2DeleteEvent("test", "t", "insert into t values (2, 'test2')", "delete from t where id = 2") + + events := []*commonEvent.DMLEvent{insertEvent, deleteEvent} + + // Generate table SQL + sqls, args, err := generator.generateTableSQL(events) + require.NoError(t, err) + + // Should generate 2 SQL statements (one DELETE, one INSERT ON DUPLICATE KEY UPDATE) + require.Len(t, sqls, 2) + require.Len(t, args, 2) + + // Verify SQL types + foundDelete := false + foundInsert := false + + for _, sql := range sqls { + if strings.Contains(sql, "DELETE FROM") { + foundDelete = true + } + if strings.Contains(sql, "INSERT INTO") && strings.Contains(sql, "ON DUPLICATE KEY UPDATE") { + foundInsert = true + } + } + + require.True(t, foundDelete, "Should generate DELETE SQL") + require.True(t, foundInsert, "Should generate INSERT ON DUPLICATE KEY UPDATE SQL") +} + +func TestSQLGenerator_generateTableSQL_EmptyEvents(t *testing.T) { + t.Parallel() + + generator := NewSQLGenerator() + + // Test with empty events + sqls, args, err := generator.generateTableSQL([]*commonEvent.DMLEvent{}) + require.NoError(t, err) + require.Len(t, sqls, 0) + require.Len(t, args, 0) +} + +func TestSQLGenerator_getArgsWithGeneratedColumn(t *testing.T) { + t.Parallel() + + generator := NewSQLGenerator() + helper := commonEvent.NewEventTestHelper(t) + defer helper.Close() + + // Setup test table + helper.Tk().MustExec("use test") + createTableSQL := "create table t (id int primary key, name varchar(32));" + job := helper.DDL2Job(createTableSQL) + require.NotNil(t, job) + + // Create test event + event := helper.DML2Event("test", "t", "insert into t values (1, 'test')") + tableInfo := event.TableInfo + + // Get first row + event.Rewind() + row, ok := event.GetNextRow() + require.True(t, ok) + + // Extract arguments + args := generator.getArgsWithGeneratedColumn(&row.Row, tableInfo) + + // Should extract arguments for all columns + require.Equal(t, len(tableInfo.GetColumns()), len(args)) +} + +// Test mixed operations in the same transaction +func TestSQLGenerator_MixedOperations(t *testing.T) { + t.Parallel() + + generator := NewSQLGenerator() + helper := commonEvent.NewEventTestHelper(t) + defer helper.Close() + + // Setup test table + helper.Tk().MustExec("use test") + createTableSQL := "create table t (id int primary key, name varchar(32));" + job := helper.DDL2Job(createTableSQL) + require.NotNil(t, job) + + // Create mixed events: insert, update, delete + insertEvent := helper.DML2Event("test", "t", "insert into t values (1, 'test1')") + updateEvent, _ := helper.DML2UpdateEvent("test", "t", "insert into t values (2, 'test2')", "update t set name = 'updated' where id = 2") + deleteEvent := helper.DML2DeleteEvent("test", "t", "insert into t values (3, 'test3')", "delete from t where id = 3") + + // Set same transaction timestamps + insertEvent.CommitTs = 100 + insertEvent.StartTs = 50 + updateEvent.CommitTs = 100 + updateEvent.StartTs = 50 + deleteEvent.CommitTs = 100 + deleteEvent.StartTs = 50 + + // Create transaction group + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: []*commonEvent.DMLEvent{insertEvent, updateEvent, deleteEvent}, + } + + // Convert to SQL + txnSQL, err := generator.ConvertTxnGroupToSQL(txnGroup) + require.NoError(t, err) + require.NotNil(t, txnSQL) + + // Should have one transaction SQL + require.Len(t, txnSQL.SQLs, 1) + sql := txnSQL.SQLs[0] + + // Verify transaction format + require.True(t, strings.HasPrefix(sql, "BEGIN;")) + require.True(t, strings.HasSuffix(sql, ";COMMIT;")) + + // Should contain all operation types + require.Contains(t, sql, "DELETE FROM") + require.Contains(t, sql, "INSERT INTO") + require.Contains(t, sql, "ON DUPLICATE KEY UPDATE") +} + +// Benchmark tests for SQL generation +func BenchmarkSQLGenerator_ConvertTxnGroupToSQL(b *testing.B) { + generator := NewSQLGenerator() + helper := commonEvent.NewEventTestHelper(b) + defer helper.Close() + + // Setup test table + helper.Tk().MustExec("use test") + createTableSQL := "create table t (id int primary key, name varchar(32));" + job := helper.DDL2Job(createTableSQL) + require.NotNil(b, job) + + // Create multiple events + events := make([]*commonEvent.DMLEvent, 100) + for i := 0; i < 100; i++ { + events[i] = helper.DML2Event("test", "t", "insert into t values (?, 'test')") + events[i].CommitTs = 100 + events[i].StartTs = 50 + } + + txnGroup := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: events, + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := generator.ConvertTxnGroupToSQL(txnGroup) + require.NoError(b, err) + } +} + +func BenchmarkSQLGenerator_groupRowsByType(b *testing.B) { + generator := NewSQLGenerator() + helper := commonEvent.NewEventTestHelper(b) + defer helper.Close() + + // Setup test table + helper.Tk().MustExec("use test") + createTableSQL := "create table t (id int primary key, name varchar(32));" + job := helper.DDL2Job(createTableSQL) + require.NotNil(b, job) + + // Create mixed events + events := make([]*commonEvent.DMLEvent, 1000) + for i := 0; i < 1000; i++ { + switch i % 3 { + case 0: + events[i] = helper.DML2Event("test", "t", "insert into t values (?, 'test')") + case 1: + events[i], _ = helper.DML2UpdateEvent("test", "t", "insert into t values (?, 'test')", "update t set name = 'updated' where id = ?") + case 2: + events[i] = helper.DML2DeleteEvent("test", "t", "insert into t values (?, 'test')", "delete from t where id = ?") + } + } + + tableInfo := events[0].TableInfo + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, _ = generator.groupRowsByType(events, tableInfo) + } +} diff --git a/downstreamadapter/sink/txnsink/types.go b/downstreamadapter/sink/txnsink/types.go new file mode 100644 index 000000000..a269c1c5b --- /dev/null +++ b/downstreamadapter/sink/txnsink/types.go @@ -0,0 +1,483 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package txnsink + +import ( + "context" + "encoding/binary" + "hash/fnv" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/downstreamadapter/sink/mysql/causality" + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/utils/chann" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/util/chunk" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" +) + +// TxnStore represents the in-memory store for DML events +// Structure: map[commitTs]map[startTs][]DMLEvent +type TxnStore struct { + store map[uint64]map[uint64][]*commonEvent.DMLEvent + mu sync.RWMutex +} + +// NewTxnStore creates a new TxnStore instance +func NewTxnStore() *TxnStore { + return &TxnStore{ + store: make(map[uint64]map[uint64][]*commonEvent.DMLEvent), + } +} + +// AddEvent adds a DML event to the store +func (ts *TxnStore) AddEvent(event *commonEvent.DMLEvent) { + ts.mu.Lock() + defer ts.mu.Unlock() + + commitTs := event.CommitTs + startTs := event.StartTs + + if ts.store[commitTs] == nil { + ts.store[commitTs] = make(map[uint64][]*commonEvent.DMLEvent) + } + ts.store[commitTs][startTs] = append(ts.store[commitTs][startTs], event) +} + +// GetEventsByCheckpointTs retrieves all events with commitTs <= checkpointTs +func (ts *TxnStore) GetEventsByCheckpointTs(checkpointTs uint64) []*TxnGroup { + ts.mu.Lock() + defer ts.mu.Unlock() + + var groups []*TxnGroup + for commitTs, startTsMap := range ts.store { + if commitTs <= checkpointTs { + for startTs, events := range startTsMap { + groups = append(groups, &TxnGroup{ + CommitTs: commitTs, + StartTs: startTs, + Events: events, + }) + } + } + } + return groups +} + +// RemoveEventsByCheckpointTs removes all events with commitTs <= checkpointTs +func (ts *TxnStore) RemoveEventsByCheckpointTs(checkpointTs uint64) { + ts.mu.Lock() + defer ts.mu.Unlock() + + for commitTs := range ts.store { + if commitTs <= checkpointTs { + delete(ts.store, commitTs) + } + } +} + +// TxnGroup represents a complete transaction +type TxnGroup struct { + CommitTs uint64 + StartTs uint64 + Events []*commonEvent.DMLEvent + + // PostFlushFuncs are functions to be executed after the transaction is flushed + PostFlushFuncs []func() +} + +// GetTxnKey returns a unique key for the transaction +func (tg *TxnGroup) GetTxnKey() string { + return strconv.FormatUint(tg.CommitTs, 10) + "_" + strconv.FormatUint(tg.StartTs, 10) +} + +// ExtractKeys extracts all affected keys from the transaction +func (tg *TxnGroup) ExtractKeys() map[string]struct{} { + keys := make(map[string]struct{}) + for _, event := range tg.Events { + for _, rowKey := range event.RowKeys { + keys[string(rowKey)] = struct{}{} + } + } + return keys +} + +// AddPostFlushFunc adds a function to be executed after the transaction is flushed +func (tg *TxnGroup) AddPostFlushFunc(f func()) { + tg.PostFlushFuncs = append(tg.PostFlushFuncs, f) +} + +// PostFlush executes all post-flush functions +func (tg *TxnGroup) PostFlush() { + for _, f := range tg.PostFlushFuncs { + f() + } +} + +// TxnSQL represents the SQL statements for a transaction +type TxnSQL struct { + TxnGroup *TxnGroup + SQLs []string + Keys map[string]struct{} +} + +// BlockStrategy is the strategy to handle the situation when the cache is full. +type BlockStrategy string + +const ( + // BlockStrategyWaitAvailable means the cache will block until there is an available slot. + BlockStrategyWaitAvailable BlockStrategy = "waitAvailable" + // BlockStrategyWaitEmpty means the cache will block until all cached txns are consumed. + BlockStrategyWaitEmpty = "waitEmpty" +) + +// TxnCacheOption is the option for creating a cache for resolved txns. +type TxnCacheOption struct { + // Count controls the number of caches, txns in different caches could be executed concurrently. + Count int + // Size controls the max number of txns a cache can hold. + Size int + // BlockStrategy controls the strategy when the cache is full. + BlockStrategy BlockStrategy +} + +// txnCache interface for TxnGroup +type txnCache interface { + // addTxnGroup adds a txn group to the Cache. + addTxnGroup(txnGroup *TxnGroup) bool + // out returns a unlimited channel to receive txn groups which are ready to be executed. + out() *chann.UnlimitedChannel[*TxnGroup, any] +} + +// boundedTxnCache is a cache which has a limit on the number of txn groups it can hold. +type boundedTxnCache struct { + ch *chann.UnlimitedChannel[*TxnGroup, any] + upperSize int +} + +func (w *boundedTxnCache) addTxnGroup(txnGroup *TxnGroup) bool { + if w.ch.Len() > w.upperSize { + return false + } + w.ch.Push(txnGroup) + return true +} + +func (w *boundedTxnCache) out() *chann.UnlimitedChannel[*TxnGroup, any] { + return w.ch +} + +// boundedTxnCacheWithBlock is a special bounded cache. Once the cache +// is full, it will block until all cached txn groups are consumed. +type boundedTxnCacheWithBlock struct { + ch *chann.UnlimitedChannel[*TxnGroup, any] + isBlocked atomic.Bool + upperSize int +} + +func (w *boundedTxnCacheWithBlock) addTxnGroup(txnGroup *TxnGroup) bool { + if w.isBlocked.Load() && w.ch.Len() <= 0 { + w.isBlocked.Store(false) + } + + if !w.isBlocked.Load() { + if w.ch.Len() > w.upperSize { + w.isBlocked.CompareAndSwap(false, true) + return false + } + w.ch.Push(txnGroup) + return true + } + return false +} + +func (w *boundedTxnCacheWithBlock) out() *chann.UnlimitedChannel[*TxnGroup, any] { + return w.ch +} + +func newTxnCache(opt TxnCacheOption) txnCache { + if opt.Size <= 0 { + log.Panic("TxnCacheOption.Size should be greater than 0, please report a bug") + } + + switch opt.BlockStrategy { + case BlockStrategyWaitAvailable: + return &boundedTxnCache{ch: chann.NewUnlimitedChannel[*TxnGroup, any](nil, nil), upperSize: opt.Size} + case BlockStrategyWaitEmpty: + return &boundedTxnCacheWithBlock{ch: chann.NewUnlimitedChannel[*TxnGroup, any](nil, nil), upperSize: opt.Size} + default: + return nil + } +} + +// ConflictKeysForTxnGroup generates conflict keys for a transaction group +func ConflictKeysForTxnGroup(txnGroup *TxnGroup) []uint64 { + if len(txnGroup.Events) == 0 { + return nil + } + + hashRes := make(map[uint64]struct{}) + hasher := fnv.New32a() + + // Iterate through all events in the transaction group + for _, event := range txnGroup.Events { + // Iterate through all rows in the event + event.Rewind() + for { + rowChange, ok := event.GetNextRow() + if !ok { + break + } + + // Generate keys for each row + keys := genRowKeysForTxnGroup(rowChange, event.TableInfo, event.DispatcherID) + for _, key := range keys { + if n, err := hasher.Write(key); n != len(key) || err != nil { + log.Panic("transaction key hash fail") + } + hashRes[uint64(hasher.Sum32())] = struct{}{} + hasher.Reset() + } + } + event.Rewind() + } + + keys := make([]uint64, 0, len(hashRes)) + for key := range hashRes { + keys = append(keys, key) + } + return keys +} + +// genRowKeysForTxnGroup generates row keys for a row change in transaction group +func genRowKeysForTxnGroup(rowChange commonEvent.RowChange, tableInfo *common.TableInfo, dispatcherID common.DispatcherID) [][]byte { + var keys [][]byte + + if !rowChange.Row.IsEmpty() { + for iIdx, idxColID := range tableInfo.GetIndexColumns() { + key := genKeyListForTxnGroup(&rowChange.Row, iIdx, idxColID, dispatcherID, tableInfo) + if len(key) == 0 { + continue + } + keys = append(keys, key) + } + } + if !rowChange.PreRow.IsEmpty() { + for iIdx, idxColID := range tableInfo.GetIndexColumns() { + key := genKeyListForTxnGroup(&rowChange.PreRow, iIdx, idxColID, dispatcherID, tableInfo) + if len(key) == 0 { + continue + } + keys = append(keys, key) + } + } + if len(keys) == 0 { + // use dispatcherID as key if no key generated (no PK/UK), + // no concurrence for rows in the same dispatcher. + log.Debug("Use dispatcherID as the key", zap.Any("dispatcherID", dispatcherID)) + tableKey := make([]byte, 8) + binary.BigEndian.PutUint64(tableKey, uint64(dispatcherID.GetLow())) + keys = [][]byte{tableKey} + } + return keys +} + +// genKeyListForTxnGroup generates a key list for a row in transaction group +func genKeyListForTxnGroup( + row *chunk.Row, iIdx int, idxColID []int64, dispatcherID common.DispatcherID, tableInfo *common.TableInfo, +) []byte { + var key []byte + for _, colID := range idxColID { + info, ok := tableInfo.GetColumnInfo(colID) + // If the index contain generated column, we can't use this key to detect conflict with other DML, + if !ok || info == nil || info.IsGenerated() { + return nil + } + offset, ok := tableInfo.GetRowColumnsOffset()[colID] + if !ok { + log.Warn("can't find column offset", zap.Int64("colID", colID), zap.String("colName", info.Name.O)) + return nil + } + value := common.ExtractColVal(row, info, offset) + // if a column value is null, we can ignore this index + if value == nil { + return nil + } + + val := common.ColumnValueString(value) + if columnNeeds2LowerCase(info.GetType(), info.GetCollate()) { + val = strings.ToLower(val) + } + + key = append(key, []byte(val)...) + key = append(key, 0) + } + if len(key) == 0 { + return nil + } + tableKey := make([]byte, 16) + binary.BigEndian.PutUint64(tableKey[:8], uint64(iIdx)) + binary.BigEndian.PutUint64(tableKey[8:], dispatcherID.GetLow()) + key = append(key, tableKey...) + return key +} + +// columnNeeds2LowerCase checks if a column needs to be converted to lowercase +func columnNeeds2LowerCase(mysqlType byte, collation string) bool { + switch mysqlType { + case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString, mysql.TypeTinyBlob, + mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob: + return collationNeeds2LowerCase(collation) + } + return false +} + +// collationNeeds2LowerCase checks if a collation needs to be converted to lowercase +func collationNeeds2LowerCase(collation string) bool { + return strings.HasSuffix(collation, "_ci") +} + +// ConflictDetector manages transaction conflicts for parallel processing +type ConflictDetector struct { + // resolvedTxnCaches are used to cache resolved transactions. + resolvedTxnCaches []txnCache + + // slots are used to find all unfinished transactions + // conflicting with an incoming transactions. + slots *causality.Slots + + // nextCacheID is used to dispatch transactions round-robin. + nextCacheID atomic.Int64 + + notifiedNodes *chann.DrainableChann[func()] + + changefeedID common.ChangeFeedID + metricConflictDetectDuration prometheus.Observer +} + +// NewConflictDetector creates a new ConflictDetector instance +func NewConflictDetector(changefeedID common.ChangeFeedID) *ConflictDetector { + opt := TxnCacheOption{ + Count: 10, // Default worker count + Size: 1024, + BlockStrategy: BlockStrategyWaitEmpty, + } + + ret := &ConflictDetector{ + resolvedTxnCaches: make([]txnCache, opt.Count), + slots: causality.NewSlots(16 * 1024), // Default slot count + notifiedNodes: chann.NewAutoDrainChann[func()](), + metricConflictDetectDuration: metrics.ConflictDetectDuration.WithLabelValues(changefeedID.Namespace(), changefeedID.Name()), + changefeedID: changefeedID, + } + for i := 0; i < opt.Count; i++ { + ret.resolvedTxnCaches[i] = newTxnCache(opt) + } + log.Info("txn conflict detector initialized", zap.Int("cacheCount", opt.Count), + zap.Int("cacheSize", opt.Size), zap.String("BlockStrategy", string(opt.BlockStrategy))) + return ret +} + +// AddTxnGroup adds a transaction group to the conflict detector +func (cd *ConflictDetector) AddTxnGroup(txnGroup *TxnGroup) { + start := time.Now() + hashes := ConflictKeysForTxnGroup(txnGroup) + node := cd.slots.AllocNode(hashes) + + txnGroup.AddPostFlushFunc(func() { + cd.slots.Remove(node) + }) + + node.TrySendToTxnCache = func(cacheID int64) bool { + // Try sending this txn group to related cache as soon as all dependencies are resolved. + ok := cd.sendTxnGroupToCache(txnGroup, cacheID) + if ok { + cd.metricConflictDetectDuration.Observe(time.Since(start).Seconds()) + } + return ok + } + node.RandCacheID = func() int64 { + return cd.nextCacheID.Add(1) % int64(len(cd.resolvedTxnCaches)) + } + node.OnNotified = func(callback func()) { + // TODO:find a better way to handle the panic + defer func() { + if r := recover(); r != nil { + log.Warn("failed to send notification, channel might be closed", zap.Any("error", r)) + } + }() + cd.notifiedNodes.In() <- callback + } + cd.slots.Add(node) +} + +// GetOutChByCacheID returns the output channel by cacheID +func (cd *ConflictDetector) GetOutChByCacheID(id int) *chann.UnlimitedChannel[*TxnGroup, any] { + return cd.resolvedTxnCaches[id].out() +} + +// Run starts the conflict detector +func (cd *ConflictDetector) Run(ctx context.Context) error { + defer func() { + metrics.ConflictDetectDuration.DeleteLabelValues(cd.changefeedID.Namespace(), cd.changefeedID.Name()) + cd.closeCache() + }() + + for { + select { + case <-ctx.Done(): + return errors.Trace(ctx.Err()) + case notifyCallback := <-cd.notifiedNodes.Out(): + if notifyCallback != nil { + notifyCallback() + } + } + } +} + +// Close closes the conflict detector +func (cd *ConflictDetector) Close() { + cd.notifiedNodes.CloseAndDrain() +} + +// sendTxnGroupToCache should not call txn.Callback if it returns an error. +func (cd *ConflictDetector) sendTxnGroupToCache(txnGroup *TxnGroup, id int64) bool { + cache := cd.resolvedTxnCaches[id] + ok := cache.addTxnGroup(txnGroup) + return ok +} + +func (cd *ConflictDetector) closeCache() { + // the unlimited channel should be closed when quit wait group, otherwise txnWorker will be blocked + for _, cache := range cd.resolvedTxnCaches { + cache.out().Close() + } +} + +// TxnSinkConfig represents the configuration for txnSink +type TxnSinkConfig struct { + MaxConcurrentTxns int + BatchSize int + FlushInterval int // milliseconds + MaxSQLBatchSize int // maximum size of SQL batch in bytes +} diff --git a/downstreamadapter/sink/txnsink/types_test.go b/downstreamadapter/sink/txnsink/types_test.go new file mode 100644 index 000000000..320831626 --- /dev/null +++ b/downstreamadapter/sink/txnsink/types_test.go @@ -0,0 +1,416 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package txnsink + +import ( + "sync" + "testing" + + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/stretchr/testify/require" +) + +func TestTxnStore_AddEvent(t *testing.T) { + t.Parallel() + + store := NewTxnStore() + + // Create test event + event := &commonEvent.DMLEvent{ + CommitTs: 100, + StartTs: 50, + Length: 1, + } + + // Add event to store + store.AddEvent(event) + + // Verify event is stored correctly + store.mu.Lock() + defer store.mu.Unlock() + + require.Contains(t, store.store, uint64(100)) + require.Contains(t, store.store[100], uint64(50)) + require.Len(t, store.store[100][50], 1) + require.Equal(t, event, store.store[100][50][0]) +} + +func TestTxnStore_AddMultipleEvents(t *testing.T) { + t.Parallel() + + store := NewTxnStore() + + // Create multiple events with same commitTs but different startTs + event1 := &commonEvent.DMLEvent{CommitTs: 100, StartTs: 50} + event2 := &commonEvent.DMLEvent{CommitTs: 100, StartTs: 60} + event3 := &commonEvent.DMLEvent{CommitTs: 100, StartTs: 50} // Same as event1 + + store.AddEvent(event1) + store.AddEvent(event2) + store.AddEvent(event3) + + store.mu.Lock() + defer store.mu.Unlock() + + // Should have one commitTs entry + require.Len(t, store.store, 1) + require.Contains(t, store.store, uint64(100)) + + // Should have two startTs entries under commitTs 100 + require.Len(t, store.store[100], 2) + require.Contains(t, store.store[100], uint64(50)) + require.Contains(t, store.store[100], uint64(60)) + + // Should have two events under startTs 50 + require.Len(t, store.store[100][50], 2) + require.Equal(t, event1, store.store[100][50][0]) + require.Equal(t, event3, store.store[100][50][1]) + + // Should have one event under startTs 60 + require.Len(t, store.store[100][60], 1) + require.Equal(t, event2, store.store[100][60][0]) +} + +func TestTxnStore_GetEventsByCheckpointTs(t *testing.T) { + t.Parallel() + + store := NewTxnStore() + + // Add events with different commitTs + events := []*commonEvent.DMLEvent{ + {CommitTs: 50, StartTs: 10}, + {CommitTs: 100, StartTs: 20}, + {CommitTs: 150, StartTs: 30}, + {CommitTs: 200, StartTs: 40}, + } + + for _, event := range events { + store.AddEvent(event) + } + + // Test getting events with checkpoint 100 + groups := store.GetEventsByCheckpointTs(100) + + // Should return 2 groups (commitTs 50 and 100) + require.Len(t, groups, 2) + + // Verify the groups + commitTsSet := make(map[uint64]bool) + for _, group := range groups { + commitTsSet[group.CommitTs] = true + require.True(t, group.CommitTs <= 100) + require.Len(t, group.Events, 1) + } + + require.True(t, commitTsSet[50]) + require.True(t, commitTsSet[100]) +} + +func TestTxnStore_RemoveEventsByCheckpointTs(t *testing.T) { + t.Parallel() + + store := NewTxnStore() + + // Add events with different commitTs + events := []*commonEvent.DMLEvent{ + {CommitTs: 50, StartTs: 10}, + {CommitTs: 100, StartTs: 20}, + {CommitTs: 150, StartTs: 30}, + {CommitTs: 200, StartTs: 40}, + } + + for _, event := range events { + store.AddEvent(event) + } + + // Remove events with checkpoint 100 + store.RemoveEventsByCheckpointTs(100) + + store.mu.Lock() + defer store.mu.Unlock() + + // Should only have commitTs 150 and 200 remaining + require.Len(t, store.store, 2) + require.Contains(t, store.store, uint64(150)) + require.Contains(t, store.store, uint64(200)) + require.NotContains(t, store.store, uint64(50)) + require.NotContains(t, store.store, uint64(100)) +} + +func TestTxnStore_ConcurrentAccess(t *testing.T) { + t.Parallel() + + store := NewTxnStore() + var wg sync.WaitGroup + + // Concurrent writes + numWriters := 10 + eventsPerWriter := 100 + + for i := 0; i < numWriters; i++ { + wg.Add(1) + go func(writerID int) { + defer wg.Done() + for j := 0; j < eventsPerWriter; j++ { + event := &commonEvent.DMLEvent{ + CommitTs: uint64(writerID*eventsPerWriter + j), + StartTs: uint64(j), + } + store.AddEvent(event) + } + }(i) + } + + // Concurrent reads + numReaders := 5 + for i := 0; i < numReaders; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 50; j++ { + _ = store.GetEventsByCheckpointTs(uint64(j * 10)) + } + }() + } + + wg.Wait() + + // Verify final state + store.mu.Lock() + defer store.mu.Unlock() + require.Len(t, store.store, numWriters*eventsPerWriter) +} + +func TestTxnGroup_GetTxnKey(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + commitTs uint64 + startTs uint64 + expected string + }{ + { + name: "normal case", + commitTs: 123, + startTs: 456, + expected: "123_456", + }, + { + name: "zero values", + commitTs: 0, + startTs: 0, + expected: "0_0", + }, + { + name: "large values", + commitTs: 18446744073709551615, // max uint64 + startTs: 18446744073709551614, + expected: "18446744073709551615_18446744073709551614", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + group := &TxnGroup{ + CommitTs: tt.commitTs, + StartTs: tt.startTs, + } + + result := group.GetTxnKey() + require.Equal(t, tt.expected, result) + }) + } +} + +func TestTxnGroup_ExtractKeys(t *testing.T) { + t.Parallel() + + // Create test events with row keys + events := []*commonEvent.DMLEvent{ + { + RowKeys: [][]byte{ + []byte("key1"), + []byte("key2"), + }, + }, + { + RowKeys: [][]byte{ + []byte("key2"), // Duplicate key + []byte("key3"), + }, + }, + } + + group := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: events, + } + + keys := group.ExtractKeys() + + // Should have 3 unique keys + require.Len(t, keys, 3) + require.Contains(t, keys, "key1") + require.Contains(t, keys, "key2") + require.Contains(t, keys, "key3") +} + +func TestTxnGroup_PostFlushFuncs(t *testing.T) { + t.Parallel() + + group := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + } + + var executed []int + + // Add post-flush functions + group.AddPostFlushFunc(func() { + executed = append(executed, 1) + }) + group.AddPostFlushFunc(func() { + executed = append(executed, 2) + }) + group.AddPostFlushFunc(func() { + executed = append(executed, 3) + }) + + // Execute post-flush functions + group.PostFlush() + + // Verify all functions were executed in order + require.Equal(t, []int{1, 2, 3}, executed) +} + +func TestTxnSinkConfig_Defaults(t *testing.T) { + t.Parallel() + + config := &TxnSinkConfig{} + + // Test default values are reasonable + require.Equal(t, 0, config.MaxConcurrentTxns) + require.Equal(t, 0, config.BatchSize) + require.Equal(t, 0, config.FlushInterval) +} + +func TestConflictDetector_Creation(t *testing.T) { + t.Parallel() + + changefeedID := common.NewChangefeedID4Test("test", "test") + detector := NewConflictDetector(changefeedID) + + require.NotNil(t, detector) + require.NotNil(t, detector.resolvedTxnCaches) + require.NotNil(t, detector.slots) + require.NotNil(t, detector.notifiedNodes) + require.NotNil(t, detector.metricConflictDetectDuration) + require.Equal(t, changefeedID, detector.changefeedID) +} + +func TestConflictDetector_GetOutChByCacheID(t *testing.T) { + t.Parallel() + + changefeedID := common.NewChangefeedID4Test("test", "test") + detector := NewConflictDetector(changefeedID) + + // Test valid cache ID + ch := detector.GetOutChByCacheID(0) + require.NotNil(t, ch) + + // Test another valid cache ID + ch2 := detector.GetOutChByCacheID(1) + require.NotNil(t, ch2) + + // Channels should be different + require.NotEqual(t, ch, ch2) + + // Test invalid cache ID (should still return a channel but may be nil depending on implementation) + ch3 := detector.GetOutChByCacheID(999) + // The behavior here depends on the underlying causality implementation + // We just verify it doesn't panic + _ = ch3 +} + +// Helper function to create a test DML event +func createTestDMLEvent(commitTs, startTs uint64, tableID int64) *commonEvent.DMLEvent { + return &commonEvent.DMLEvent{ + CommitTs: commitTs, + StartTs: startTs, + Length: 1, + RowKeys: [][]byte{[]byte("test_key")}, + } +} + +// Benchmark tests +func BenchmarkTxnStore_AddEvent(b *testing.B) { + store := NewTxnStore() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + event := &commonEvent.DMLEvent{ + CommitTs: uint64(i % 1000), // Reuse some commitTs values + StartTs: uint64(i), + } + store.AddEvent(event) + } +} + +func BenchmarkTxnStore_GetEventsByCheckpointTs(b *testing.B) { + store := NewTxnStore() + + // Pre-populate store + for i := 0; i < 10000; i++ { + event := &commonEvent.DMLEvent{ + CommitTs: uint64(i), + StartTs: uint64(i), + } + store.AddEvent(event) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = store.GetEventsByCheckpointTs(uint64(i % 5000)) + } +} + +func BenchmarkTxnGroup_ExtractKeys(b *testing.B) { + // Create a transaction group with many events + events := make([]*commonEvent.DMLEvent, 100) + for i := 0; i < 100; i++ { + events[i] = &commonEvent.DMLEvent{ + RowKeys: [][]byte{ + []byte("key_" + string(rune(i%10))), // Some duplicate keys + []byte("unique_key_" + string(rune(i))), + }, + } + } + + group := &TxnGroup{ + CommitTs: 100, + StartTs: 50, + Events: events, + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = group.ExtractKeys() + } +} diff --git a/pkg/common/types.go b/pkg/common/types.go index 10ec3f946..bfcdb5c19 100644 --- a/pkg/common/types.go +++ b/pkg/common/types.go @@ -301,4 +301,5 @@ const ( CloudStorageSinkType BlackHoleSinkType RedoSinkType + TxnSinkType ) From 08d3068676fe3880f2e9df146bb4dff56b17d0fe Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Sun, 24 Aug 2025 17:27:36 +0800 Subject: [PATCH 02/15] txnSink works well for sysbench workload --- coordinator/changefeed/changefeed.go | 3 +- downstreamadapter/sink/txnsink/db_executor.go | 168 ++++++---- .../sink/txnsink/db_executor_test.go | 59 ++-- .../sink/txnsink/event_processor.go | 33 +- .../sink/txnsink/progress_tracker.go | 192 +++++++++++ .../sink/txnsink/progress_tracker_test.go | 227 +++++++++++++ downstreamadapter/sink/txnsink/sink.go | 217 +++---------- .../sink/txnsink/sql_generator.go | 33 +- .../sink/txnsink/sql_generator_test.go | 16 +- downstreamadapter/sink/txnsink/types.go | 20 +- downstreamadapter/sink/txnsink/types_test.go | 4 +- downstreamadapter/sink/txnsink/worker.go | 306 ++++++++++++++++++ 12 files changed, 974 insertions(+), 304 deletions(-) create mode 100644 downstreamadapter/sink/txnsink/progress_tracker.go create mode 100644 downstreamadapter/sink/txnsink/progress_tracker_test.go create mode 100644 downstreamadapter/sink/txnsink/worker.go diff --git a/coordinator/changefeed/changefeed.go b/coordinator/changefeed/changefeed.go index dcfcb4764..da30e97ca 100644 --- a/coordinator/changefeed/changefeed.go +++ b/coordinator/changefeed/changefeed.go @@ -166,7 +166,8 @@ func (c *Changefeed) ForceUpdateStatus(newStatus *heartbeatpb.MaintainerStatus) } func (c *Changefeed) NeedCheckpointTsMessage() bool { - return c.sinkType == common.KafkaSinkType || c.sinkType == common.CloudStorageSinkType + // return c.sinkType != common.MysqlSinkType + return true } func (c *Changefeed) SetIsNew(isNew bool) { diff --git a/downstreamadapter/sink/txnsink/db_executor.go b/downstreamadapter/sink/txnsink/db_executor.go index 0096eedb1..1fcfb1d8b 100644 --- a/downstreamadapter/sink/txnsink/db_executor.go +++ b/downstreamadapter/sink/txnsink/db_executor.go @@ -3,14 +3,27 @@ package txnsink import ( "context" "database/sql" + "database/sql/driver" "strings" "time" + dmysql "github.com/go-sql-driver/mysql" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/retry" + "github.com/pingcap/tidb/pkg/parser/mysql" "go.uber.org/zap" ) +const ( + // BackoffBaseDelay indicates the base delay time for retrying. + BackoffBaseDelay = 500 * time.Millisecond + // BackoffMaxDelay indicates the max delay time for retrying. + BackoffMaxDelay = 60 * time.Second + // DefaultDMLMaxRetry is the default maximum number of retries for DML operations + DefaultDMLMaxRetry = 8 +) + // DBExecutor handles database execution for transaction SQL type DBExecutor struct { db *sql.DB @@ -23,99 +36,132 @@ func NewDBExecutor(db *sql.DB) *DBExecutor { } } -// ExecuteSQLBatch executes a batch of SQL transactions +// ExecuteSQLBatch executes a batch of SQL transactions with retry mechanism func (e *DBExecutor) ExecuteSQLBatch(batch []*TxnSQL) error { if len(batch) == 0 { return nil } - log.Debug("txnSink: executing SQL batch", + log.Info("txnSink: executing SQL batch", zap.Int("batchSize", len(batch))) - // If batch size is 1, execute directly (SQL already contains BEGIN/COMMIT) - if len(batch) == 1 { - txnSQL := batch[0] - for _, sql := range txnSQL.SQLs { + // Define the execution function that will be retried + tryExec := func() error { + // If batch size is 1, execute directly (SQL already contains BEGIN/COMMIT) + if len(batch) == 1 { + txnSQL := batch[0] + // Skip execution if SQL is empty + if txnSQL.SQL == "" { + log.Debug("txnSink: skipping empty SQL execution", + zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), + zap.Uint64("startTs", txnSQL.TxnGroup.StartTs)) + return nil + } + + log.Info("hyy execute single sql", + zap.String("sql", txnSQL.SQL), + zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), + zap.Uint64("startTs", txnSQL.TxnGroup.StartTs)) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - _, execErr := e.db.ExecContext(ctx, sql) + _, execErr := e.db.ExecContext(ctx, txnSQL.SQL, txnSQL.Args...) cancel() if execErr != nil { log.Error("txnSink: failed to execute single SQL", - zap.String("sql", sql), + zap.String("sql", txnSQL.SQL), zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), zap.Uint64("startTs", txnSQL.TxnGroup.StartTs), zap.Error(execErr)) return errors.Trace(execErr) } + + log.Info("txnSink: successfully executed single transaction", + zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), + zap.Uint64("startTs", txnSQL.TxnGroup.StartTs)) + return nil } - log.Debug("txnSink: successfully executed single transaction", - zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), - zap.Uint64("startTs", txnSQL.TxnGroup.StartTs)) - return nil - } + // For multiple transactions, combine them into a single SQL statement + var combinedSQL []string + var combinedArgs []interface{} - // For multiple transactions, use explicit transaction and combine SQLs - tx, err := e.db.Begin() - if err != nil { - return errors.Trace(err) - } - defer func() { - if err != nil { - tx.Rollback() - } - }() - - // Build combined SQL from all transactions - var combinedSQL strings.Builder - - // Collect all SQL statements from all transactions - for _, txnSQL := range batch { - for _, sql := range txnSQL.SQLs { - // Keep the original SQL with BEGIN/COMMIT - cleanSQL := strings.TrimSpace(sql) - if len(cleanSQL) > 0 { - combinedSQL.WriteString(cleanSQL) - if !strings.HasSuffix(cleanSQL, ";") { - combinedSQL.WriteString(";") - } + for _, txnSQL := range batch { + // Skip execution if SQL is empty + if txnSQL.SQL == "" { + log.Debug("txnSink: skipping empty SQL execution in batch", + zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), + zap.Uint64("startTs", txnSQL.TxnGroup.StartTs)) + continue } + + combinedSQL = append(combinedSQL, txnSQL.SQL) + combinedArgs = append(combinedArgs, txnSQL.Args...) } - } - finalSQL := combinedSQL.String() + if len(combinedSQL) == 0 { + log.Debug("txnSink: no valid SQL to execute in batch") + return nil + } - log.Debug("txnSink: executing combined SQL batch with explicit transaction", - zap.String("sql", finalSQL), - zap.Int("batchSize", len(batch))) + // Join all SQL statements with semicolons + finalSQL := strings.Join(combinedSQL, ";") - // Execute the combined SQL within transaction - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - _, execErr := tx.ExecContext(ctx, finalSQL) - cancel() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + _, execErr := e.db.ExecContext(ctx, finalSQL, combinedArgs...) + cancel() + + if execErr != nil { + log.Error("txnSink: failed to execute combined SQL batch", + zap.String("sql", finalSQL), + zap.Int("batchSize", len(batch)), + zap.Error(execErr)) + return errors.Trace(execErr) + } - if execErr != nil { - log.Error("txnSink: failed to execute SQL batch", + log.Debug("txnSink: successfully executed combined SQL batch", zap.String("sql", finalSQL), - zap.Int("batchSize", len(batch)), - zap.Error(execErr)) - return errors.Trace(execErr) + zap.Int("batchSize", len(batch))) + + return nil + } + + // Use retry mechanism + return retry.Do(context.Background(), func() error { + err := tryExec() + if err != nil { + log.Warn("txnSink: SQL execution failed, will retry", + zap.Int("batchSize", len(batch)), + zap.Error(err)) + } + return err + }, retry.WithBackoffBaseDelay(BackoffBaseDelay.Milliseconds()), + retry.WithBackoffMaxDelay(BackoffMaxDelay.Milliseconds()), + retry.WithMaxTries(DefaultDMLMaxRetry), + retry.WithIsRetryableErr(isRetryableDMLError)) +} + +// isRetryableDMLError determines if a DML error is retryable +func isRetryableDMLError(err error) bool { + // Check if it's a retryable error + if !errors.IsRetryableError(err) { + return false } - // Commit the transaction - if err = tx.Commit(); err != nil { - log.Error("txnSink: failed to commit batch transaction", - zap.Int("batchSize", len(batch)), - zap.Error(err)) - return errors.Trace(err) + // Check for specific MySQL error codes + if mysqlErr, ok := errors.Cause(err).(*dmysql.MySQLError); ok { + switch mysqlErr.Number { + case uint16(mysql.ErrNoSuchTable), uint16(mysql.ErrBadDB), uint16(mysql.ErrDupEntry): + return false + } } - log.Debug("txnSink: successfully executed SQL batch", - zap.Int("batchSize", len(batch)), - zap.Int("sqlLength", len(finalSQL))) + // Check for driver errors + if err == driver.ErrBadConn { + return true + } - return nil + return true } // Close closes the database connection diff --git a/downstreamadapter/sink/txnsink/db_executor_test.go b/downstreamadapter/sink/txnsink/db_executor_test.go index fc8250a3f..e596d5314 100644 --- a/downstreamadapter/sink/txnsink/db_executor_test.go +++ b/downstreamadapter/sink/txnsink/db_executor_test.go @@ -43,12 +43,13 @@ func TestDBExecutor_ExecuteTransaction(t *testing.T) { txnSQL := &TxnSQL{ TxnGroup: txnGroup, - SQLs: []string{"BEGIN;INSERT INTO test VALUES (1, 'test');UPDATE test SET name = 'updated' WHERE id = 1;COMMIT;"}, + SQL: "BEGIN;INSERT INTO test VALUES (1, 'test');UPDATE test SET name = 'updated' WHERE id = 1;COMMIT;", + Args: []interface{}{}, Keys: map[string]struct{}{"key1": {}}, } // Set up mock expectations - expect the full SQL with BEGIN/COMMIT - mock.ExpectExec("BEGIN;INSERT INTO test VALUES").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("BEGIN;INSERT INTO test VALUES").WithArgs().WillReturnResult(sqlmock.NewResult(1, 1)) // Execute transaction err = executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) @@ -77,7 +78,8 @@ func TestDBExecutor_ExecuteTransaction_EmptySQL(t *testing.T) { txnSQL := &TxnSQL{ TxnGroup: txnGroup, - SQLs: []string{}, + SQL: "", + Args: []interface{}{}, Keys: map[string]struct{}{}, } @@ -108,7 +110,8 @@ func TestDBExecutor_ExecuteTransaction_BeginError(t *testing.T) { txnSQL := &TxnSQL{ TxnGroup: txnGroup, - SQLs: []string{"BEGIN;INSERT INTO test VALUES (1, 'test');COMMIT;"}, + SQL: "BEGIN;INSERT INTO test VALUES (1, 'test');COMMIT;", + Args: []interface{}{}, Keys: map[string]struct{}{"key1": {}}, } @@ -143,7 +146,8 @@ func TestDBExecutor_ExecuteTransaction_ExecError(t *testing.T) { txnSQL := &TxnSQL{ TxnGroup: txnGroup, - SQLs: []string{"BEGIN;INSERT INTO test VALUES (1, 'test');UPDATE test SET name = 'updated' WHERE id = 1;COMMIT;"}, + SQL: "BEGIN;INSERT INTO test VALUES (1, 'test');UPDATE test SET name = 'updated' WHERE id = 1;COMMIT;", + Args: []interface{}{}, Keys: map[string]struct{}{"key1": {}}, } @@ -178,7 +182,8 @@ func TestDBExecutor_ExecuteTransaction_CommitError(t *testing.T) { txnSQL := &TxnSQL{ TxnGroup: txnGroup, - SQLs: []string{"BEGIN;INSERT INTO test VALUES (1, 'test');COMMIT;"}, + SQL: "BEGIN;INSERT INTO test VALUES (1, 'test');COMMIT;", + Args: []interface{}{}, Keys: map[string]struct{}{"key1": {}}, } @@ -213,7 +218,8 @@ func TestDBExecutor_ExecuteTransaction_Timeout(t *testing.T) { txnSQL := &TxnSQL{ TxnGroup: txnGroup, - SQLs: []string{"BEGIN;INSERT INTO test VALUES (1, 'test');COMMIT;"}, + SQL: "BEGIN;INSERT INTO test VALUES (1, 'test');COMMIT;", + Args: []interface{}{}, Keys: map[string]struct{}{"key1": {}}, } @@ -249,14 +255,13 @@ func TestDBExecutor_ExecuteTransaction_MultipleSQL(t *testing.T) { txnSQL := &TxnSQL{ TxnGroup: txnGroup, - SQLs: []string{ - "BEGIN;INSERT INTO users VALUES (1, 'alice');INSERT INTO users VALUES (2, 'bob');UPDATE users SET name = 'alice_updated' WHERE id = 1;DELETE FROM users WHERE id = 2;COMMIT;", - }, - Keys: map[string]struct{}{"user1": {}, "user2": {}}, + SQL: "BEGIN;INSERT INTO users VALUES (1, 'alice');INSERT INTO users VALUES (2, 'bob');UPDATE users SET name = 'alice_updated' WHERE id = 1;DELETE FROM users WHERE id = 2;COMMIT;", + Args: []interface{}{}, + Keys: map[string]struct{}{"user1": {}, "user2": {}}, } // Set up mock expectations for the combined SQL - mock.ExpectExec("BEGIN;INSERT INTO users VALUES").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("BEGIN;INSERT INTO users VALUES").WithArgs().WillReturnResult(sqlmock.NewResult(1, 1)) // Execute transaction err = executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) @@ -299,7 +304,7 @@ func createMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock, func()) { } // Test helper function to create a test TxnSQL -func createTestTxnSQL(sqls []string) *TxnSQL { +func createTestTxnSQL(sql string) *TxnSQL { txnGroup := &TxnGroup{ CommitTs: 100, StartTs: 50, @@ -308,7 +313,8 @@ func createTestTxnSQL(sqls []string) *TxnSQL { return &TxnSQL{ TxnGroup: txnGroup, - SQLs: sqls, + SQL: sql, + Args: []interface{}{}, Keys: map[string]struct{}{"test_key": {}}, } } @@ -321,24 +327,15 @@ func BenchmarkDBExecutor_ExecuteTransaction(b *testing.B) { executor := NewDBExecutor(db) // Create test transaction SQL - txnSQL := createTestTxnSQL([]string{ - "INSERT INTO test VALUES (1, 'test')", - "UPDATE test SET name = 'updated' WHERE id = 1", - }) + txnSQL := createTestTxnSQL("BEGIN;INSERT INTO test VALUES (1, 'test');UPDATE test SET name = 'updated' WHERE id = 1;COMMIT;") // Set up mock expectations - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO test VALUES").WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectExec("UPDATE test SET").WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() + mock.ExpectExec("BEGIN;INSERT INTO test VALUES").WithArgs().WillReturnResult(sqlmock.NewResult(1, 1)) b.ResetTimer() for i := 0; i < b.N; i++ { // Reset mock expectations for each iteration - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO test VALUES").WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectExec("UPDATE test SET").WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() + mock.ExpectExec("BEGIN;INSERT INTO test VALUES").WithArgs().WillReturnResult(sqlmock.NewResult(1, 1)) err := executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) require.NoError(b, err) @@ -352,19 +349,15 @@ func BenchmarkDBExecutor_ExecuteTransaction_SingleSQL(b *testing.B) { executor := NewDBExecutor(db) // Create test transaction SQL with single statement - txnSQL := createTestTxnSQL([]string{"INSERT INTO test VALUES (1, 'test')"}) + txnSQL := createTestTxnSQL("BEGIN;INSERT INTO test VALUES (1, 'test');COMMIT;") // Set up mock expectations - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO test VALUES").WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() + mock.ExpectExec("BEGIN;INSERT INTO test VALUES").WithArgs().WillReturnResult(sqlmock.NewResult(1, 1)) b.ResetTimer() for i := 0; i < b.N; i++ { // Reset mock expectations for each iteration - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO test VALUES").WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() + mock.ExpectExec("BEGIN;INSERT INTO test VALUES").WithArgs().WillReturnResult(sqlmock.NewResult(1, 1)) err := executor.ExecuteSQLBatch([]*TxnSQL{txnSQL}) require.NoError(b, err) diff --git a/downstreamadapter/sink/txnsink/event_processor.go b/downstreamadapter/sink/txnsink/event_processor.go index cf305c111..602e05eab 100644 --- a/downstreamadapter/sink/txnsink/event_processor.go +++ b/downstreamadapter/sink/txnsink/event_processor.go @@ -10,13 +10,15 @@ import ( // EventProcessor handles DML events and checkpoint processing type EventProcessor struct { - txnStore *TxnStore + txnStore *TxnStore + progressTracker *ProgressTracker } // NewEventProcessor creates a new event processor -func NewEventProcessor(txnStore *TxnStore) *EventProcessor { +func NewEventProcessor(txnStore *TxnStore, progressTracker *ProgressTracker) *EventProcessor { return &EventProcessor{ - txnStore: txnStore, + txnStore: txnStore, + progressTracker: progressTracker, } } @@ -57,7 +59,7 @@ func (p *EventProcessor) processDMLEvent(event *commonEvent.DMLEvent) { // Add event to the transaction store p.txnStore.AddEvent(event) - log.Debug("txnSink: processed DML event", + log.Info("txnSink: processed DML event", zap.Uint64("commitTs", event.CommitTs), zap.Uint64("startTs", event.StartTs), zap.Int64("tableID", event.GetTableID()), @@ -66,32 +68,39 @@ func (p *EventProcessor) processDMLEvent(event *commonEvent.DMLEvent) { // processCheckpoint processes a checkpoint timestamp func (p *EventProcessor) processCheckpoint(checkpointTs uint64, txnChan chan<- *TxnGroup) error { - // Get all events with commitTs <= checkpointTs - events := p.txnStore.GetEventsByCheckpointTs(checkpointTs) - if len(events) == 0 { - log.Debug("txnSink: no events to process for checkpoint", + log.Info("hyy process checkpoint", + zap.Uint64("checkpointTs", checkpointTs)) + // Get all events with commitTs <= checkpointTs (already sorted by commitTs) + txnGroups := p.txnStore.GetEventsByCheckpointTs(checkpointTs) + if len(txnGroups) == 0 { + log.Info("txnSink: no events to process for checkpoint", zap.Uint64("checkpointTs", checkpointTs)) + p.progressTracker.UpdateCheckpointTs(checkpointTs) return nil } - // Events are already grouped by TxnStore.GetEventsByCheckpointTs - txnGroups := events + // Add all transaction groups to pending transactions for progress tracking + for _, txnGroup := range txnGroups { + p.progressTracker.AddPendingTxn(txnGroup.CommitTs, txnGroup.StartTs) + } // Send transaction groups to the output channel for _, txnGroup := range txnGroups { txnChan <- txnGroup - log.Debug("txnSink: sent transaction group", + log.Info("txnSink: sent transaction group", zap.Uint64("commitTs", txnGroup.CommitTs), zap.Uint64("startTs", txnGroup.StartTs), zap.Int("eventCount", len(txnGroup.Events))) } + // Update checkpoint progress + p.progressTracker.UpdateCheckpointTs(checkpointTs) + // Remove processed events from the store p.txnStore.RemoveEventsByCheckpointTs(checkpointTs) log.Info("txnSink: processed checkpoint", zap.Uint64("checkpointTs", checkpointTs), - zap.Int("eventCount", len(events)), zap.Int("txnGroupCount", len(txnGroups))) return nil diff --git a/downstreamadapter/sink/txnsink/progress_tracker.go b/downstreamadapter/sink/txnsink/progress_tracker.go new file mode 100644 index 000000000..0ce0488cc --- /dev/null +++ b/downstreamadapter/sink/txnsink/progress_tracker.go @@ -0,0 +1,192 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package txnsink + +import ( + "container/list" + "context" + "sync" + "time" + + "github.com/pingcap/log" + appcontext "github.com/pingcap/ticdc/pkg/common/context" + "github.com/pingcap/ticdc/pkg/pdutil" + "github.com/tikv/client-go/v2/oracle" + "go.uber.org/zap" +) + +// TxnKey represents a unique transaction identifier +type TxnKey struct { + commitTs uint64 + startTs uint64 +} + +// ProgressTracker tracks the progress of data processing and flushing +type ProgressTracker struct { + // Current progress state + checkpointTs uint64 + + // Track pending transactions using list + map pattern (like tableProgress) + list *list.List // Maintains order (FIFO) + elemMap map[TxnKey]*list.Element // TxnKey -> list.Element for O(1) removal + + // Thread safety + mu sync.RWMutex + + // Monitoring + pdClock pdutil.Clock + cancelMonitor context.CancelFunc + changefeedName string +} + +// NewProgressTracker creates a new progress tracker +func NewProgressTracker() *ProgressTracker { + return &ProgressTracker{ + checkpointTs: 0, + list: list.New(), + elemMap: make(map[TxnKey]*list.Element), + } +} + +// NewProgressTrackerWithMonitor creates a new progress tracker with monitoring enabled +func NewProgressTrackerWithMonitor(changefeedName string) *ProgressTracker { + pt := &ProgressTracker{ + checkpointTs: 0, + list: list.New(), + elemMap: make(map[TxnKey]*list.Element), + changefeedName: changefeedName, + } + + pt.pdClock = appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock) + + // Start monitoring goroutine + ctx, cancel := context.WithCancel(context.Background()) + pt.cancelMonitor = cancel + go pt.runMonitor(ctx) + + return pt +} + +// AddPendingTxn adds a pending transaction to track +func (pt *ProgressTracker) AddPendingTxn(commitTs, startTs uint64) { + pt.mu.Lock() + defer pt.mu.Unlock() + + key := TxnKey{commitTs: commitTs, startTs: startTs} + // Add to list (maintains order) + elem := pt.list.PushBack(key) + pt.elemMap[key] = elem +} + +// RemoveCompletedTxn removes a completed transaction from pending list +func (pt *ProgressTracker) RemoveCompletedTxn(commitTs, startTs uint64) { + pt.mu.Lock() + defer pt.mu.Unlock() + + key := TxnKey{commitTs: commitTs, startTs: startTs} + if elem, ok := pt.elemMap[key]; ok { + pt.list.Remove(elem) + delete(pt.elemMap, key) + } +} + +// UpdateCheckpointTs updates the latest checkpoint TS received +func (pt *ProgressTracker) UpdateCheckpointTs(ts uint64) { + pt.mu.Lock() + defer pt.mu.Unlock() + + if ts > pt.checkpointTs { + pt.checkpointTs = ts + } +} + +// calculateEffectiveTs calculates the effective progress TS +// If there are pending transactions: effectiveTs = min(pendingTxns) - 1 +// If no pending transactions: effectiveTs = checkpointTs +func (pt *ProgressTracker) calculateEffectiveTs() uint64 { + if pt.list.Len() > 0 { + // Return min(pendingTxns) - 1 (first element in list) + key := pt.list.Front().Value.(TxnKey) + return key.commitTs - 1 + } + + // No pending transactions, use checkpointTs + return pt.checkpointTs +} + +// Reset resets the progress tracker to initial state +func (pt *ProgressTracker) Reset() { + pt.mu.Lock() + defer pt.mu.Unlock() + + pt.checkpointTs = 0 + pt.list.Init() // Clear list + pt.elemMap = make(map[TxnKey]*list.Element) // Clear map +} + +// Close stops the monitoring goroutine +func (pt *ProgressTracker) Close() { + if pt.cancelMonitor != nil { + pt.cancelMonitor() + } +} + +// runMonitor runs the monitoring goroutine that prints progress and lag every second +func (pt *ProgressTracker) runMonitor(ctx context.Context) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + log.Info("txnSink: ProgressTracker monitor started", zap.String("changefeed", pt.changefeedName)) + + for { + select { + case <-ctx.Done(): + log.Info("txnSink: ProgressTracker monitor stopped", zap.String("changefeed", pt.changefeedName)) + return + case <-ticker.C: + pt.printProgress() + } + } +} + +// printProgress prints current progress and lag information +func (pt *ProgressTracker) printProgress() { + pt.mu.RLock() + effectiveTs := pt.calculateEffectiveTs() + checkpointTs := pt.checkpointTs + pendingCount := pt.list.Len() + pt.mu.RUnlock() + + if pt.pdClock == nil { + log.Info("txnSink: Progress status", + zap.String("changefeed", pt.changefeedName), + zap.Uint64("effectiveTs", effectiveTs), + zap.Uint64("checkpointTs", checkpointTs), + zap.Int("pendingCount", pendingCount)) + return + } + + // Calculate lag using the same logic as maintainer + pdTime := pt.pdClock.CurrentTime() + phyEffectiveTs := oracle.ExtractPhysical(effectiveTs) + lag := float64(oracle.GetPhysical(pdTime)-phyEffectiveTs) / 1e3 // Convert to seconds + + log.Info("txnSink: Progress status", + zap.String("changefeed", pt.changefeedName), + zap.Uint64("effectiveTs", effectiveTs), + zap.Int64("phyEffectiveTs", phyEffectiveTs), + zap.Uint64("checkpointTs", checkpointTs), + zap.Int("pendingCount", pendingCount), + zap.Float64("lagSeconds", lag)) +} diff --git a/downstreamadapter/sink/txnsink/progress_tracker_test.go b/downstreamadapter/sink/txnsink/progress_tracker_test.go new file mode 100644 index 000000000..e5f360355 --- /dev/null +++ b/downstreamadapter/sink/txnsink/progress_tracker_test.go @@ -0,0 +1,227 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package txnsink + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestProgressTracker_Basic(t *testing.T) { + t.Parallel() + + tracker := NewProgressTracker() + + // Initial state + require.Equal(t, uint64(0), tracker.GetEffectiveTs()) + require.Equal(t, uint64(0), tracker.GetProgress()) +} + +func TestProgressTracker_AddPendingTxn(t *testing.T) { + t.Parallel() + + tracker := NewProgressTracker() + + // Add pending transaction + tracker.AddPendingTxn(100, 1) + require.Equal(t, uint64(99), tracker.GetEffectiveTs()) // min(pending) - 1 = 100 - 1 = 99 + + // Add another pending transaction + tracker.AddPendingTxn(200, 2) + require.Equal(t, uint64(99), tracker.GetEffectiveTs()) // Still 99, as 100 is still pending + + progress := tracker.GetProgress() + require.Equal(t, uint64(99), progress) +} + +func TestProgressTracker_RemoveCompletedTxn(t *testing.T) { + t.Parallel() + + tracker := NewProgressTracker() + + // Add pending transactions + tracker.AddPendingTxn(100, 1) + tracker.AddPendingTxn(200, 2) + + // Remove completed transaction (200, 2) + tracker.RemoveCompletedTxn(200, 2) + require.Equal(t, uint64(99), tracker.GetEffectiveTs()) // Still 99, as 100 is still pending + + // Remove completed transaction (100, 1) + tracker.RemoveCompletedTxn(100, 1) + require.Equal(t, uint64(0), tracker.GetEffectiveTs()) // No pending, use checkpointTs (0) + + progress := tracker.GetProgress() + require.Equal(t, uint64(0), progress) +} + +func TestProgressTracker_UpdateCheckpointTs(t *testing.T) { + t.Parallel() + + tracker := NewProgressTracker() + + // Update checkpoint TS + tracker.UpdateCheckpointTs(100) + require.Equal(t, uint64(100), tracker.GetEffectiveTs()) // No pending, use checkpointTs + + // Add pending transaction + tracker.AddPendingTxn(50, 1) + require.Equal(t, uint64(49), tracker.GetEffectiveTs()) // min(pending) - 1 = 50 - 1 = 49 + + // Update checkpoint TS (should not affect effective TS when there are pending transactions) + tracker.UpdateCheckpointTs(200) + require.Equal(t, uint64(49), tracker.GetEffectiveTs()) // Still 49, as 50 is still pending + + // Remove pending transaction + tracker.RemoveCompletedTxn(50, 1) + require.Equal(t, uint64(200), tracker.GetEffectiveTs()) // No pending, use checkpointTs (200) +} + +func TestProgressTracker_EffectiveTs(t *testing.T) { + t.Parallel() + + tracker := NewProgressTracker() + + // Initially should be 0 + require.Equal(t, uint64(0), tracker.GetEffectiveTs()) + + // Only checkpoint TS set + tracker.UpdateCheckpointTs(100) + require.Equal(t, uint64(100), tracker.GetEffectiveTs()) // Use checkpointTs + + // Add pending transactions + tracker.AddPendingTxn(50, 1) + tracker.AddPendingTxn(75, 2) + require.Equal(t, uint64(49), tracker.GetEffectiveTs()) // min(50, 75) - 1 = 49 + + // Remove first pending transaction + tracker.RemoveCompletedTxn(50, 1) + require.Equal(t, uint64(74), tracker.GetEffectiveTs()) // min(75) - 1 = 74 + + // Remove second pending transaction + tracker.RemoveCompletedTxn(75, 2) + require.Equal(t, uint64(100), tracker.GetEffectiveTs()) // No pending, use checkpointTs (100) +} + +func TestProgressTracker_SameCommitTsDifferentStartTs(t *testing.T) { + t.Parallel() + + tracker := NewProgressTracker() + + // Add transactions with same commitTs but different startTs + tracker.AddPendingTxn(100, 1) + tracker.AddPendingTxn(100, 2) + tracker.AddPendingTxn(100, 3) + + require.Equal(t, uint64(99), tracker.GetEffectiveTs()) // min(100) - 1 = 99 + + // Remove one transaction + tracker.RemoveCompletedTxn(100, 2) + require.Equal(t, uint64(99), tracker.GetEffectiveTs()) // Still 99, as 100 is still pending + + // Remove another transaction + tracker.RemoveCompletedTxn(100, 1) + require.Equal(t, uint64(99), tracker.GetEffectiveTs()) // Still 99, as 100 is still pending + + // Remove the last transaction + tracker.RemoveCompletedTxn(100, 3) + require.Equal(t, uint64(0), tracker.GetEffectiveTs()) // No pending, use checkpointTs (0) +} + +func TestProgressTracker_Reset(t *testing.T) { + t.Parallel() + + tracker := NewProgressTracker() + + // Set some values + tracker.AddPendingTxn(100, 1) + tracker.AddPendingTxn(200, 2) + tracker.UpdateCheckpointTs(300) + + // Reset + tracker.Reset() + + // Check all values are reset + require.Equal(t, uint64(0), tracker.GetEffectiveTs()) + require.Equal(t, uint64(0), tracker.GetProgress()) +} + +func TestProgressTracker_Concurrent(t *testing.T) { + t.Parallel() + + tracker := NewProgressTracker() + + // Test concurrent updates + done := make(chan bool, 3) + + // Add transactions + go func() { + for i := uint64(1); i <= 10; i++ { + tracker.AddPendingTxn(i, i) + } + done <- true + }() + + // Update checkpoint TS + go func() { + for i := uint64(1); i <= 10; i++ { + tracker.UpdateCheckpointTs(i) + } + done <- true + }() + + // Read progress concurrently + go func() { + for i := uint64(1); i <= 10; i++ { + tracker.GetEffectiveTs() + tracker.GetProgress() + } + done <- true + }() + + // Wait for all goroutines to complete + <-done + <-done + <-done + + // Final state should be consistent + effectiveTs := tracker.GetEffectiveTs() + progress := tracker.GetProgress() + require.Equal(t, effectiveTs, progress) +} + +func TestProgressTracker_Monitor(t *testing.T) { + t.Parallel() + + // Create tracker with monitor + tracker := NewProgressTrackerWithMonitor("test-changefeed") + defer tracker.Close() + + // Add some transactions and update checkpoint + tracker.AddPendingTxn(100, 1) + tracker.AddPendingTxn(200, 2) + tracker.UpdateCheckpointTs(150) + + // Wait for at least one monitor cycle + time.Sleep(1500 * time.Millisecond) + + // Remove a transaction and wait for another cycle + tracker.RemoveCompletedTxn(100, 1) + time.Sleep(1500 * time.Millisecond) + + // Verify the tracker is still working + require.Equal(t, uint64(199), tracker.GetEffectiveTs()) // min(200) - 1 = 199 +} diff --git a/downstreamadapter/sink/txnsink/sink.go b/downstreamadapter/sink/txnsink/sink.go index 06139d046..0ab1c8288 100644 --- a/downstreamadapter/sink/txnsink/sink.go +++ b/downstreamadapter/sink/txnsink/sink.go @@ -35,9 +35,11 @@ type Sink struct { // Core components txnStore *TxnStore conflictDetector *ConflictDetector - dbExecutor *DBExecutor - sqlGenerator *SQLGenerator eventProcessor *EventProcessor + progressTracker *ProgressTracker + + // Workers + workers []*Worker // Configuration config *TxnSinkConfig @@ -46,7 +48,6 @@ type Sink struct { dmlEventChan chan *commonEvent.DMLEvent checkpointChan chan uint64 txnChan chan *TxnGroup - sqlChan chan *TxnSQL // State management isNormal *atomic.Bool @@ -61,30 +62,42 @@ func New(ctx context.Context, changefeedID common.ChangeFeedID, db *sql.DB, conf if config == nil { config = &TxnSinkConfig{ MaxConcurrentTxns: 16, - BatchSize: 16, + BatchSize: 256, FlushInterval: 100, MaxSQLBatchSize: 1024 * 16, } } txnStore := NewTxnStore() - conflictDetector := NewConflictDetector(changefeedID) - dbExecutor := NewDBExecutor(db) - sqlGenerator := NewSQLGenerator() - eventProcessor := NewEventProcessor(txnStore) + conflictDetector := NewConflictDetector(changefeedID, config.MaxConcurrentTxns) + progressTracker := NewProgressTrackerWithMonitor(changefeedID.Name()) + eventProcessor := NewEventProcessor(txnStore, progressTracker) + + // Create workers + workers := make([]*Worker, config.MaxConcurrentTxns) + for i := 0; i < config.MaxConcurrentTxns; i++ { + inputCh := conflictDetector.GetOutChByCacheID(i) + if inputCh == nil { + log.Error("txnSink: failed to get output channel from conflict detector", + zap.String("namespace", changefeedID.Namespace()), + zap.String("changefeed", changefeedID.Name()), + zap.Int("workerID", i)) + continue + } + workers[i] = NewWorker(i, changefeedID, config, db, inputCh, progressTracker, metrics.NewStatistics(changefeedID, "txnsink")) + } return &Sink{ changefeedID: changefeedID, txnStore: txnStore, conflictDetector: conflictDetector, - dbExecutor: dbExecutor, - sqlGenerator: sqlGenerator, eventProcessor: eventProcessor, + progressTracker: progressTracker, + workers: workers, config: config, dmlEventChan: make(chan *commonEvent.DMLEvent, 10000), checkpointChan: make(chan uint64, 100), txnChan: make(chan *TxnGroup, 10000), - sqlChan: make(chan *TxnSQL, 10000), isNormal: atomic.NewBool(true), ctx: ctx, statistics: metrics.NewStatistics(changefeedID, "txnsink"), @@ -103,7 +116,9 @@ func (s *Sink) IsNormal() bool { // AddDMLEvent adds a DML event to the sink func (s *Sink) AddDMLEvent(event *commonEvent.DMLEvent) { + // Note: We don't add to pending here, as we need to wait for txnGroup formation s.dmlEventChan <- event + event.PostFlush() } // AddCheckpointTs adds a checkpoint timestamp to trigger transaction processing @@ -128,14 +143,20 @@ func (s *Sink) Close(removeChangefeed bool) { // Close conflict detector s.conflictDetector.Close() - // Close database executor - s.dbExecutor.Close() + // Close all workers + for _, worker := range s.workers { + if worker != nil { + worker.Close() + } + } + + // Close progress tracker + s.progressTracker.Close() // Close channels close(s.dmlEventChan) close(s.checkpointChan) close(s.txnChan) - close(s.sqlChan) log.Info("txnSink: closed", zap.String("namespace", s.changefeedID.Namespace()), @@ -152,16 +173,10 @@ func (s *Sink) Run(ctx context.Context) error { zap.String("changefeed", changefeed)) // Start conflict detector - s.conflictDetector.Run(ctx) - - // Start multiple transaction workers eg, ctx := errgroup.WithContext(ctx) - for i := 0; i < s.config.MaxConcurrentTxns; i++ { - workerID := i - eg.Go(func() error { - return s.runTxnWorker(ctx, workerID) - }) - } + eg.Go(func() error { + return s.conflictDetector.Run(ctx) + }) // Start event processor for DML events eg.Go(func() error { @@ -178,10 +193,15 @@ func (s *Sink) Run(ctx context.Context) error { return s.processTransactions(ctx) }) - // Start SQL batch processor - eg.Go(func() error { - return s.processSQLBatch(ctx) - }) + // Start all workers + for _, worker := range s.workers { + if worker != nil { + worker := worker + eg.Go(func() error { + return worker.Run(ctx) + }) + } + } err := eg.Wait() if err != nil { @@ -199,57 +219,6 @@ func (s *Sink) Run(ctx context.Context) error { return nil } -// runTxnWorker runs a transaction worker (similar to mysqlSink's runDMLWriter) -func (s *Sink) runTxnWorker(ctx context.Context, idx int) error { - namespace := s.changefeedID.Namespace() - changefeed := s.changefeedID.Name() - - log.Info("txnSink: starting txn worker", - zap.String("namespace", namespace), - zap.String("changefeed", changefeed), - zap.Int("workerID", idx)) - - inputCh := s.conflictDetector.GetOutChByCacheID(idx) - if inputCh == nil { - return errors.New("failed to get output channel from conflict detector") - } - - buffer := make([]*TxnGroup, 0, s.config.BatchSize) - for { - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - default: - // Get multiple txn groups from the channel - txnGroups, ok := inputCh.GetMultipleNoGroup(buffer) - if !ok { - return errors.Trace(ctx.Err()) - } - - if len(txnGroups) == 0 { - buffer = buffer[:0] - continue - } - - // Process each txn group - for _, txnGroup := range txnGroups { - if err := s.processTxnGroup(txnGroup); err != nil { - log.Error("txnSink: failed to process transaction group", - zap.String("namespace", namespace), - zap.String("changefeed", changefeed), - zap.Int("workerID", idx), - zap.Uint64("commitTs", txnGroup.CommitTs), - zap.Uint64("startTs", txnGroup.StartTs), - zap.Error(err)) - return err - } - } - - buffer = buffer[:0] - } - } -} - // processTransactions processes transactions from the transaction channel func (s *Sink) processTransactions(ctx context.Context) error { for { @@ -265,95 +234,3 @@ func (s *Sink) processTransactions(ctx context.Context) error { } } } - -// processTxnGroup processes a single transaction group -func (s *Sink) processTxnGroup(txnGroup *TxnGroup) error { - // Convert to SQL and send to SQL channel - txnSQL, err := s.sqlGenerator.ConvertTxnGroupToSQL(txnGroup) - if err != nil { - return err - } - - // Send to SQL channel for batch processing - select { - case s.sqlChan <- txnSQL: - return nil - default: - return errors.New("SQL channel is full") - } -} - -// processSQLBatch processes SQL batches from the SQL channel -func (s *Sink) processSQLBatch(ctx context.Context) error { - namespace := s.changefeedID.Namespace() - changefeed := s.changefeedID.Name() - - log.Info("txnSink: starting SQL batch processor", - zap.String("namespace", namespace), - zap.String("changefeed", changefeed)) - - batch := make([]*TxnSQL, 0, s.config.BatchSize) - currentBatchSize := 0 - - for { - select { - case <-ctx.Done(): - return ctx.Err() - case txnSQL, ok := <-s.sqlChan: - if !ok { - // Channel closed, flush remaining batch - if len(batch) > 0 { - if err := s.executeSQLBatch(batch); err != nil { - log.Error("txnSink: failed to execute final SQL batch", - zap.String("namespace", namespace), - zap.String("changefeed", changefeed), - zap.Error(err)) - return err - } - } - return nil - } - - // Calculate SQL size for this transaction - sqlSize := s.calculateSQLSize(txnSQL) - - // Check if adding this SQL would exceed batch size limit - if len(batch) > 0 && (currentBatchSize+sqlSize > s.config.MaxSQLBatchSize || len(batch) >= s.config.BatchSize) { - // Execute current batch before adding new SQL - if err := s.executeSQLBatch(batch); err != nil { - log.Error("txnSink: failed to execute SQL batch", - zap.String("namespace", namespace), - zap.String("changefeed", changefeed), - zap.Error(err)) - return err - } - - // Reset batch - batch = batch[:0] - currentBatchSize = 0 - } - - // Add SQL to batch - batch = append(batch, txnSQL) - currentBatchSize += sqlSize - } - } -} - -// calculateSQLSize calculates the total size of SQL statements in a transaction -func (s *Sink) calculateSQLSize(txnSQL *TxnSQL) int { - totalSize := 0 - for _, sql := range txnSQL.SQLs { - totalSize += len(sql) - } - return totalSize -} - -// executeSQLBatch executes a batch of SQL transactions -func (s *Sink) executeSQLBatch(batch []*TxnSQL) error { - if len(batch) == 0 { - return nil - } - - return s.dbExecutor.ExecuteSQLBatch(batch) -} diff --git a/downstreamadapter/sink/txnsink/sql_generator.go b/downstreamadapter/sink/txnsink/sql_generator.go index 7b9a50f0e..a80a37859 100644 --- a/downstreamadapter/sink/txnsink/sql_generator.go +++ b/downstreamadapter/sink/txnsink/sql_generator.go @@ -16,10 +16,12 @@ package txnsink import ( "strings" + "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/sink/sqlmodel" "github.com/pingcap/tidb/pkg/util/chunk" + "go.uber.org/zap" ) // SQLGenerator handles SQL generation for transaction groups @@ -57,27 +59,30 @@ func (g *SQLGenerator) ConvertTxnGroupToSQL(txnGroup *TxnGroup) (*TxnSQL, error) allSQLs = append(allSQLs, sqls...) allArgs = append(allArgs, args...) } - // Wrap in transaction - if len(allSQLs) > 0 { - transactionSQL := "BEGIN;" + strings.Join(allSQLs, ";") + ";COMMIT;" - transactionArgs := make([]interface{}, 0) - for _, arg := range allArgs { - transactionArgs = append(transactionArgs, arg...) - } - - return &TxnSQL{ - TxnGroup: txnGroup, - SQLs: []string{transactionSQL}, - Keys: txnGroup.ExtractKeys(), - }, nil + var transactionSQL string + if len(allSQLs) == 0 { + transactionSQL = "" + } else { + transactionSQL = "BEGIN;" + strings.Join(allSQLs, ";") + ";COMMIT;" } + transactionArgs := make([]interface{}, 0) + for _, arg := range allArgs { + transactionArgs = append(transactionArgs, arg...) + } + + log.Info("hyy generate sql", + zap.Any("commitTs", txnGroup.CommitTs), + zap.Any("startTs", txnGroup.StartTs), + zap.Any("sql", transactionSQL)) return &TxnSQL{ TxnGroup: txnGroup, - SQLs: []string{}, + SQL: transactionSQL, + Args: transactionArgs, Keys: txnGroup.ExtractKeys(), }, nil + } // generateTableSQL generates SQL statements for events of the same table diff --git a/downstreamadapter/sink/txnsink/sql_generator_test.go b/downstreamadapter/sink/txnsink/sql_generator_test.go index c60f111ff..5da68aa9a 100644 --- a/downstreamadapter/sink/txnsink/sql_generator_test.go +++ b/downstreamadapter/sink/txnsink/sql_generator_test.go @@ -57,10 +57,10 @@ func TestSQLGenerator_ConvertTxnGroupToSQL(t *testing.T) { // Verify transaction structure require.Equal(t, txnGroup, txnSQL.TxnGroup) - require.Len(t, txnSQL.SQLs, 1) + require.NotEmpty(t, txnSQL.SQL) // Verify SQL format: should start with BEGIN and end with COMMIT - sql := txnSQL.SQLs[0] + sql := txnSQL.SQL require.True(t, strings.HasPrefix(sql, "BEGIN;")) require.True(t, strings.HasSuffix(sql, ";COMMIT;")) @@ -86,8 +86,8 @@ func TestSQLGenerator_ConvertTxnGroupToSQL_EmptyGroup(t *testing.T) { require.NoError(t, err) require.NotNil(t, txnSQL) - // Should have empty SQL list - require.Len(t, txnSQL.SQLs, 0) + // Should have empty SQL + require.Empty(t, txnSQL.SQL) } func TestSQLGenerator_ConvertTxnGroupToSQL_MultiTable(t *testing.T) { @@ -129,8 +129,8 @@ func TestSQLGenerator_ConvertTxnGroupToSQL_MultiTable(t *testing.T) { require.NotNil(t, txnSQL) // Should have one transaction SQL - require.Len(t, txnSQL.SQLs, 1) - sql := txnSQL.SQLs[0] + require.NotEmpty(t, txnSQL.SQL) + sql := txnSQL.SQL // Should contain both tables require.Contains(t, sql, "`test`.`t1`") @@ -299,8 +299,8 @@ func TestSQLGenerator_MixedOperations(t *testing.T) { require.NotNil(t, txnSQL) // Should have one transaction SQL - require.Len(t, txnSQL.SQLs, 1) - sql := txnSQL.SQLs[0] + require.NotEmpty(t, txnSQL.SQL) + sql := txnSQL.SQL // Verify transaction format require.True(t, strings.HasPrefix(sql, "BEGIN;")) diff --git a/downstreamadapter/sink/txnsink/types.go b/downstreamadapter/sink/txnsink/types.go index a269c1c5b..a97c3db48 100644 --- a/downstreamadapter/sink/txnsink/types.go +++ b/downstreamadapter/sink/txnsink/types.go @@ -17,6 +17,7 @@ import ( "context" "encoding/binary" "hash/fnv" + "sort" "strconv" "strings" "sync" @@ -52,6 +53,11 @@ func NewTxnStore() *TxnStore { // AddEvent adds a DML event to the store func (ts *TxnStore) AddEvent(event *commonEvent.DMLEvent) { + log.Info("txnSink: add event", + zap.Uint64("commitTs", event.CommitTs), + zap.Uint64("startTs", event.StartTs), + zap.Int64("tableID", event.GetTableID()), + zap.Int32("rowCount", event.Len())) ts.mu.Lock() defer ts.mu.Unlock() @@ -65,6 +71,7 @@ func (ts *TxnStore) AddEvent(event *commonEvent.DMLEvent) { } // GetEventsByCheckpointTs retrieves all events with commitTs <= checkpointTs +// Returns txnGroups sorted by commitTs in ascending order func (ts *TxnStore) GetEventsByCheckpointTs(checkpointTs uint64) []*TxnGroup { ts.mu.Lock() defer ts.mu.Unlock() @@ -81,6 +88,12 @@ func (ts *TxnStore) GetEventsByCheckpointTs(checkpointTs uint64) []*TxnGroup { } } } + + // Sort groups by commitTs in ascending order + sort.Slice(groups, func(i, j int) bool { + return groups[i].CommitTs < groups[j].CommitTs + }) + return groups } @@ -137,7 +150,8 @@ func (tg *TxnGroup) PostFlush() { // TxnSQL represents the SQL statements for a transaction type TxnSQL struct { TxnGroup *TxnGroup - SQLs []string + SQL string + Args []interface{} Keys map[string]struct{} } @@ -376,9 +390,9 @@ type ConflictDetector struct { } // NewConflictDetector creates a new ConflictDetector instance -func NewConflictDetector(changefeedID common.ChangeFeedID) *ConflictDetector { +func NewConflictDetector(changefeedID common.ChangeFeedID, maxConcurrentTxns int) *ConflictDetector { opt := TxnCacheOption{ - Count: 10, // Default worker count + Count: maxConcurrentTxns, // Default worker count Size: 1024, BlockStrategy: BlockStrategyWaitEmpty, } diff --git a/downstreamadapter/sink/txnsink/types_test.go b/downstreamadapter/sink/txnsink/types_test.go index 320831626..e707bc6a7 100644 --- a/downstreamadapter/sink/txnsink/types_test.go +++ b/downstreamadapter/sink/txnsink/types_test.go @@ -315,7 +315,7 @@ func TestConflictDetector_Creation(t *testing.T) { t.Parallel() changefeedID := common.NewChangefeedID4Test("test", "test") - detector := NewConflictDetector(changefeedID) + detector := NewConflictDetector(changefeedID, 16) require.NotNil(t, detector) require.NotNil(t, detector.resolvedTxnCaches) @@ -329,7 +329,7 @@ func TestConflictDetector_GetOutChByCacheID(t *testing.T) { t.Parallel() changefeedID := common.NewChangefeedID4Test("test", "test") - detector := NewConflictDetector(changefeedID) + detector := NewConflictDetector(changefeedID, 16) // Test valid cache ID ch := detector.GetOutChByCacheID(0) diff --git a/downstreamadapter/sink/txnsink/worker.go b/downstreamadapter/sink/txnsink/worker.go new file mode 100644 index 000000000..44967973b --- /dev/null +++ b/downstreamadapter/sink/txnsink/worker.go @@ -0,0 +1,306 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package txnsink + +import ( + "context" + "database/sql" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/utils/chann" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +// Worker represents a worker that processes transaction groups and executes SQL +type Worker struct { + workerID int + changefeedID common.ChangeFeedID + config *TxnSinkConfig + + // Core components + sqlGenerator *SQLGenerator + dbExecutor *DBExecutor + progressTracker *ProgressTracker + + // Channels + inputCh *chann.UnlimitedChannel[*TxnGroup, any] + sqlChan *chann.UnlimitedChannel[*TxnSQL, any] // Simple FIFO channel for SQL batching + + // Statistics + statistics *metrics.Statistics +} + +// NewWorker creates a new worker instance +func NewWorker( + workerID int, + changefeedID common.ChangeFeedID, + config *TxnSinkConfig, + db *sql.DB, + inputCh *chann.UnlimitedChannel[*TxnGroup, any], + progressTracker *ProgressTracker, + statistics *metrics.Statistics, +) *Worker { + // Create unlimited channel for SQL batching + sqlChan := chann.NewUnlimitedChannel[*TxnSQL, any]( + nil, // No grouping function needed + func(txnSQL *TxnSQL) int { + // Calculate SQL size for batching + return len(txnSQL.SQL) + }, + ) + + return &Worker{ + workerID: workerID, + changefeedID: changefeedID, + config: config, + sqlGenerator: NewSQLGenerator(), + dbExecutor: NewDBExecutor(db), + progressTracker: progressTracker, + inputCh: inputCh, + sqlChan: sqlChan, + statistics: statistics, + } +} + +// Run starts the worker processing +func (w *Worker) Run(ctx context.Context) error { + namespace := w.changefeedID.Namespace() + changefeed := w.changefeedID.Name() + + log.Info("txnSink: starting worker", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Int("workerID", w.workerID)) + + // Start multiple goroutines for different responsibilities + eg, ctx := errgroup.WithContext(ctx) + + // Start transaction processor (converts TxnGroup to SQL) + eg.Go(func() error { + return w.processTransactions(ctx) + }) + + // Start SQL executor (executes SQL batches) + eg.Go(func() error { + return w.executeSQLBatches(ctx) + }) + + err := eg.Wait() + if err != nil { + log.Error("txnSink: worker stopped with error", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Int("workerID", w.workerID), + zap.Error(err)) + return err + } + + log.Info("txnSink: worker stopped normally", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Int("workerID", w.workerID)) + + return nil +} + +// processTransactions processes transaction groups from input channel and converts them to SQL +func (w *Worker) processTransactions(ctx context.Context) error { + namespace := w.changefeedID.Namespace() + changefeed := w.changefeedID.Name() + + log.Info("hyy processTransactions") + + buffer := make([]*TxnGroup, 0, w.config.BatchSize) + for { + select { + case <-ctx.Done(): + return errors.Trace(ctx.Err()) + default: + // Get multiple txn groups from the channel + txnGroups, ok := w.inputCh.GetMultipleNoGroup(buffer) + if !ok { + return errors.Trace(ctx.Err()) + } + log.Info("hyy get txn group", zap.Int("txnGroupSize", len(txnGroups))) + + if len(txnGroups) == 0 { + buffer = buffer[:0] + continue + } + + // Process each txn group + for _, txnGroup := range txnGroups { + if len(txnGroup.Events) == 0 { + continue + } + if err := w.processTxnGroup(txnGroup); err != nil { + log.Error("txnSink: failed to process transaction group", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Int("workerID", w.workerID), + zap.Uint64("commitTs", txnGroup.CommitTs), + zap.Uint64("startTs", txnGroup.StartTs), + zap.Error(err)) + return err + } + } + + buffer = buffer[:0] + } + } +} + +// processTxnGroup converts a transaction group to SQL and pushes to sqlChan +func (w *Worker) processTxnGroup(txnGroup *TxnGroup) error { + // Convert to SQL + txnSQL, err := w.sqlGenerator.ConvertTxnGroupToSQL(txnGroup) + if err != nil { + return err + } + + // Push to worker's own sqlChan + w.sqlChan.Push(txnSQL) + + return nil +} + +// executeSQLBatches processes SQL batches from the SQL channel and executes them +func (w *Worker) executeSQLBatches(ctx context.Context) error { + namespace := w.changefeedID.Namespace() + changefeed := w.changefeedID.Name() + + log.Info("txnSink: starting SQL batch executor", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Int("workerID", w.workerID)) + + buffer := make([]*TxnSQL, 0, w.config.BatchSize) + currentBatchSize := 0 + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + // Get multiple SQLs from the channel (no grouping needed) + sqlBatch, ok := w.sqlChan.GetMultipleNoGroup(buffer, w.config.MaxSQLBatchSize) + if !ok { + return nil + } + + if len(sqlBatch) == 0 { + buffer = buffer[:0] + continue + } + + batch := make([]*TxnSQL, 0, w.config.BatchSize) + + log.Debug("txnSink: got sql batch", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Int("workerID", w.workerID), + zap.Int("sqlBatchSize", len(sqlBatch)), + zap.Int("batchSize", len(batch)), + zap.Int("currentBatchSize", currentBatchSize)) + + // Process each SQL in the batch, respecting size and count limits + for _, txnSQL := range sqlBatch { + // Calculate SQL size for this transaction + sqlSize := w.calculateSQLSize(txnSQL) + + // Check if adding this SQL would exceed batch size limit + if len(batch) > 0 && (currentBatchSize+sqlSize > w.config.MaxSQLBatchSize || len(batch) >= w.config.BatchSize) { + // Execute current batch before adding new SQL + if err := w.executeSQLBatch(batch); err != nil { + log.Error("txnSink: failed to execute SQL batch", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Int("workerID", w.workerID), + zap.Error(err)) + return err + } + + // Reset batch + batch = batch[:0] + currentBatchSize = 0 + } + + // Add SQL to batch + log.Debug("txnSink: add sql to batch", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Int("workerID", w.workerID), + zap.Int("batchSize", len(batch)), + zap.Int("currentBatchSize", currentBatchSize)) + batch = append(batch, txnSQL) + currentBatchSize += sqlSize + } + + if len(batch) > 0 { + if err := w.executeSQLBatch(batch); err != nil { + log.Error("txnSink: failed to execute SQL batch", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Int("workerID", w.workerID), + zap.Error(err)) + return err + } + } + buffer = buffer[:0] + } + } +} + +// calculateSQLSize calculates the total size of SQL statements in a transaction +func (w *Worker) calculateSQLSize(txnSQL *TxnSQL) int { + return len(txnSQL.SQL) +} + +// executeSQLBatch executes a batch of SQL transactions +func (w *Worker) executeSQLBatch(batch []*TxnSQL) error { + namespace := w.changefeedID.Namespace() + changefeed := w.changefeedID.Name() + + log.Debug("txnSink: execute sql batch", + zap.String("namespace", namespace), + zap.String("changefeed", changefeed), + zap.Int("workerID", w.workerID), + zap.Int("batchSize", len(batch))) + + if len(batch) == 0 { + return nil + } + + err := w.dbExecutor.ExecuteSQLBatch(batch) + if err != nil { + return err + } + + // Update flushed progress for all transactions in the batch + for _, txnSQL := range batch { + w.progressTracker.RemoveCompletedTxn(txnSQL.TxnGroup.CommitTs, txnSQL.TxnGroup.StartTs) + } + + return nil +} + +// Close closes the worker and releases resources +func (w *Worker) Close() { + w.sqlChan.Close() + w.dbExecutor.Close() +} From b77ba12c17221b968122388bca22e92d6da4c190 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 25 Aug 2025 11:51:29 +0800 Subject: [PATCH 03/15] update --- downstreamadapter/sink/txnsink/db_executor.go | 69 ++++----------- .../sink/txnsink/sql_generator.go | 7 -- downstreamadapter/sink/txnsink/worker.go | 84 ++++++++++++++++--- pkg/metrics/sink.go | 42 ++++++++++ 4 files changed, 131 insertions(+), 71 deletions(-) diff --git a/downstreamadapter/sink/txnsink/db_executor.go b/downstreamadapter/sink/txnsink/db_executor.go index 1fcfb1d8b..aa2549246 100644 --- a/downstreamadapter/sink/txnsink/db_executor.go +++ b/downstreamadapter/sink/txnsink/db_executor.go @@ -47,79 +47,42 @@ func (e *DBExecutor) ExecuteSQLBatch(batch []*TxnSQL) error { // Define the execution function that will be retried tryExec := func() error { - // If batch size is 1, execute directly (SQL already contains BEGIN/COMMIT) - if len(batch) == 1 { - txnSQL := batch[0] - // Skip execution if SQL is empty - if txnSQL.SQL == "" { - log.Debug("txnSink: skipping empty SQL execution", - zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), - zap.Uint64("startTs", txnSQL.TxnGroup.StartTs)) - return nil - } - - log.Info("hyy execute single sql", - zap.String("sql", txnSQL.SQL), - zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), - zap.Uint64("startTs", txnSQL.TxnGroup.StartTs)) - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - _, execErr := e.db.ExecContext(ctx, txnSQL.SQL, txnSQL.Args...) - cancel() - - if execErr != nil { - log.Error("txnSink: failed to execute single SQL", - zap.String("sql", txnSQL.SQL), - zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), - zap.Uint64("startTs", txnSQL.TxnGroup.StartTs), - zap.Error(execErr)) - return errors.Trace(execErr) - } - - log.Info("txnSink: successfully executed single transaction", - zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), - zap.Uint64("startTs", txnSQL.TxnGroup.StartTs)) - return nil - } - - // For multiple transactions, combine them into a single SQL statement - var combinedSQL []string - var combinedArgs []interface{} + // Filter out empty SQL statements + var validSQLs []string + var validArgs []interface{} for _, txnSQL := range batch { - // Skip execution if SQL is empty - if txnSQL.SQL == "" { - log.Debug("txnSink: skipping empty SQL execution in batch", - zap.Uint64("commitTs", txnSQL.TxnGroup.CommitTs), - zap.Uint64("startTs", txnSQL.TxnGroup.StartTs)) - continue + if txnSQL.SQL != "" { + validSQLs = append(validSQLs, txnSQL.SQL) + validArgs = append(validArgs, txnSQL.Args...) } - - combinedSQL = append(combinedSQL, txnSQL.SQL) - combinedArgs = append(combinedArgs, txnSQL.Args...) } - if len(combinedSQL) == 0 { + if len(validSQLs) == 0 { log.Debug("txnSink: no valid SQL to execute in batch") return nil } - // Join all SQL statements with semicolons - finalSQL := strings.Join(combinedSQL, ";") + // Combine SQL statements with semicolons + finalSQL := strings.Join(validSQLs, ";") + + log.Info("executing transaction", + zap.String("sql", finalSQL), + zap.Any("args", validArgs)) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - _, execErr := e.db.ExecContext(ctx, finalSQL, combinedArgs...) + _, execErr := e.db.ExecContext(ctx, finalSQL, validArgs...) cancel() if execErr != nil { - log.Error("txnSink: failed to execute combined SQL batch", + log.Error("txnSink: failed to execute SQL batch", zap.String("sql", finalSQL), zap.Int("batchSize", len(batch)), zap.Error(execErr)) return errors.Trace(execErr) } - log.Debug("txnSink: successfully executed combined SQL batch", + log.Debug("txnSink: successfully executed SQL batch", zap.String("sql", finalSQL), zap.Int("batchSize", len(batch))) diff --git a/downstreamadapter/sink/txnsink/sql_generator.go b/downstreamadapter/sink/txnsink/sql_generator.go index a80a37859..2d306e657 100644 --- a/downstreamadapter/sink/txnsink/sql_generator.go +++ b/downstreamadapter/sink/txnsink/sql_generator.go @@ -16,12 +16,10 @@ package txnsink import ( "strings" - "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/sink/sqlmodel" "github.com/pingcap/tidb/pkg/util/chunk" - "go.uber.org/zap" ) // SQLGenerator handles SQL generation for transaction groups @@ -71,11 +69,6 @@ func (g *SQLGenerator) ConvertTxnGroupToSQL(txnGroup *TxnGroup) (*TxnSQL, error) transactionArgs = append(transactionArgs, arg...) } - log.Info("hyy generate sql", - zap.Any("commitTs", txnGroup.CommitTs), - zap.Any("startTs", txnGroup.StartTs), - zap.Any("sql", transactionSQL)) - return &TxnSQL{ TxnGroup: txnGroup, SQL: transactionSQL, diff --git a/downstreamadapter/sink/txnsink/worker.go b/downstreamadapter/sink/txnsink/worker.go index 44967973b..6905a1cbb 100644 --- a/downstreamadapter/sink/txnsink/worker.go +++ b/downstreamadapter/sink/txnsink/worker.go @@ -16,12 +16,15 @@ package txnsink import ( "context" "database/sql" + "strconv" + "time" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/utils/chann" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) @@ -43,6 +46,11 @@ type Worker struct { // Statistics statistics *metrics.Statistics + + // Monitoring metrics + workerFlushDuration prometheus.Observer + workerTotalDuration prometheus.Observer + workerHandledRows prometheus.Counter } // NewWorker creates a new worker instance @@ -64,16 +72,24 @@ func NewWorker( }, ) + // Initialize monitoring metrics + namespace := changefeedID.Namespace() + changefeed := changefeedID.Name() + workerIDStr := strconv.Itoa(workerID) + return &Worker{ - workerID: workerID, - changefeedID: changefeedID, - config: config, - sqlGenerator: NewSQLGenerator(), - dbExecutor: NewDBExecutor(db), - progressTracker: progressTracker, - inputCh: inputCh, - sqlChan: sqlChan, - statistics: statistics, + workerID: workerID, + changefeedID: changefeedID, + config: config, + sqlGenerator: NewSQLGenerator(), + dbExecutor: NewDBExecutor(db), + progressTracker: progressTracker, + inputCh: inputCh, + sqlChan: sqlChan, + statistics: statistics, + workerFlushDuration: metrics.WorkerFlushDuration.WithLabelValues(namespace, changefeed, workerIDStr), + workerTotalDuration: metrics.WorkerTotalDuration.WithLabelValues(namespace, changefeed, workerIDStr), + workerHandledRows: metrics.WorkerHandledRows.WithLabelValues(namespace, changefeed, workerIDStr), } } @@ -191,6 +207,7 @@ func (w *Worker) executeSQLBatches(ctx context.Context) error { buffer := make([]*TxnSQL, 0, w.config.BatchSize) currentBatchSize := 0 + totalStart := time.Now() for { select { @@ -261,6 +278,10 @@ func (w *Worker) executeSQLBatches(ctx context.Context) error { return err } } + + // Record total duration for worker busy ratio calculation + w.workerTotalDuration.Observe(time.Since(totalStart).Seconds()) + totalStart = time.Now() buffer = buffer[:0] } } @@ -271,7 +292,7 @@ func (w *Worker) calculateSQLSize(txnSQL *TxnSQL) int { return len(txnSQL.SQL) } -// executeSQLBatch executes a batch of SQL transactions +// executeSQLBatch executes a batch of SQL transactions with monitoring metrics func (w *Worker) executeSQLBatch(batch []*TxnSQL) error { namespace := w.changefeedID.Namespace() changefeed := w.changefeedID.Name() @@ -286,11 +307,43 @@ func (w *Worker) executeSQLBatch(batch []*TxnSQL) error { return nil } - err := w.dbExecutor.ExecuteSQLBatch(batch) + // Calculate total row count for this batch + totalRowCount := 0 + for _, txnSQL := range batch { + for _, event := range txnSQL.TxnGroup.Events { + totalRowCount += int(event.Len()) + } + } + + // Record batch size metric + metrics.TxnSinkBatchSize.WithLabelValues(namespace, changefeed).Observe(float64(len(batch))) + + // Record batch execution with monitoring + start := time.Now() + err := w.statistics.RecordBatchExecution(func() (int, int64, error) { + execErr := w.dbExecutor.ExecuteSQLBatch(batch) + if execErr != nil { + return 0, 0, execErr + } + // Return row count and approximate size (using SQL length as approximation) + approximateSize := int64(0) + for _, txnSQL := range batch { + approximateSize += int64(len(txnSQL.SQL)) + } + return totalRowCount, approximateSize, nil + }) + if err != nil { return err } + // Record flush duration and handled rows + w.workerFlushDuration.Observe(time.Since(start).Seconds()) + w.workerHandledRows.Add(float64(totalRowCount)) + + // Record batch duration + metrics.TxnSinkBatchDuration.WithLabelValues(namespace, changefeed).Observe(time.Since(start).Seconds()) + // Update flushed progress for all transactions in the batch for _, txnSQL := range batch { w.progressTracker.RemoveCompletedTxn(txnSQL.TxnGroup.CommitTs, txnSQL.TxnGroup.StartTs) @@ -303,4 +356,13 @@ func (w *Worker) executeSQLBatch(batch []*TxnSQL) error { func (w *Worker) Close() { w.sqlChan.Close() w.dbExecutor.Close() + + // Clean up monitoring metrics + namespace := w.changefeedID.Namespace() + changefeed := w.changefeedID.Name() + workerIDStr := strconv.Itoa(w.workerID) + + metrics.WorkerFlushDuration.DeleteLabelValues(namespace, changefeed, workerIDStr) + metrics.WorkerTotalDuration.DeleteLabelValues(namespace, changefeed, workerIDStr) + metrics.WorkerHandledRows.DeleteLabelValues(namespace, changefeed, workerIDStr) } diff --git a/pkg/metrics/sink.go b/pkg/metrics/sink.go index a915c6147..ae89a177a 100644 --- a/pkg/metrics/sink.go +++ b/pkg/metrics/sink.go @@ -124,6 +124,44 @@ var ( Help: "Busy ratio (X ms in 1s) for all workers.", }, []string{"namespace", "changefeed", "id"}) + // TxnSinkBatchSize records the batch size of SQL transactions + TxnSinkBatchSize = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "txn_sink_batch_size", + Help: "Batch size of SQL transactions in txnSink.", + Buckets: prometheus.ExponentialBuckets(1, 2, 16), // 1~65536 + }, []string{"namespace", "changefeed"}) + + // TxnSinkBatchDuration records the duration of processing a batch of transactions + TxnSinkBatchDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "txn_sink_batch_duration", + Help: "Duration of processing a batch of transactions in txnSink.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms~524s + }, []string{"namespace", "changefeed"}) + + // TxnSinkPendingTxns records the number of pending transactions + TxnSinkPendingTxns = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "txn_sink_pending_txns", + Help: "Number of pending transactions in txnSink.", + }, []string{"namespace", "changefeed"}) + + // TxnSinkTxnStoreSize records the size of transaction store + TxnSinkTxnStoreSize = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "txn_sink_txn_store_size", + Help: "Size of transaction store in txnSink.", + }, []string{"namespace", "changefeed"}) + SinkDMLBatchCommit = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "ticdc", @@ -215,6 +253,10 @@ func InitSinkMetrics(registry *prometheus.Registry) { registry.MustRegister(WorkerFlushDuration) registry.MustRegister(WorkerTotalDuration) registry.MustRegister(WorkerHandledRows) + registry.MustRegister(TxnSinkBatchSize) + registry.MustRegister(TxnSinkBatchDuration) + registry.MustRegister(TxnSinkPendingTxns) + registry.MustRegister(TxnSinkTxnStoreSize) registry.MustRegister(SinkDMLBatchCommit) registry.MustRegister(SinkDMLBatchCallback) registry.MustRegister(PrepareStatementErrors) From a50a9f75593ed14783856ab095d2c35a9d25b4f8 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 25 Aug 2025 14:44:26 +0800 Subject: [PATCH 04/15] update --- .../sink/txnsink/event_processor.go | 18 +- .../sink/txnsink/progress_tracker_test.go | 227 ------------------ downstreamadapter/sink/txnsink/sink.go | 1 - downstreamadapter/sink/txnsink/types.go | 6 - downstreamadapter/sink/txnsink/worker.go | 1 + 5 files changed, 2 insertions(+), 251 deletions(-) delete mode 100644 downstreamadapter/sink/txnsink/progress_tracker_test.go diff --git a/downstreamadapter/sink/txnsink/event_processor.go b/downstreamadapter/sink/txnsink/event_processor.go index 602e05eab..e76a34f64 100644 --- a/downstreamadapter/sink/txnsink/event_processor.go +++ b/downstreamadapter/sink/txnsink/event_processor.go @@ -58,18 +58,10 @@ func (p *EventProcessor) ProcessCheckpoints(ctx context.Context, checkpointChan func (p *EventProcessor) processDMLEvent(event *commonEvent.DMLEvent) { // Add event to the transaction store p.txnStore.AddEvent(event) - - log.Info("txnSink: processed DML event", - zap.Uint64("commitTs", event.CommitTs), - zap.Uint64("startTs", event.StartTs), - zap.Int64("tableID", event.GetTableID()), - zap.Int32("rowCount", event.Len())) } // processCheckpoint processes a checkpoint timestamp func (p *EventProcessor) processCheckpoint(checkpointTs uint64, txnChan chan<- *TxnGroup) error { - log.Info("hyy process checkpoint", - zap.Uint64("checkpointTs", checkpointTs)) // Get all events with commitTs <= checkpointTs (already sorted by commitTs) txnGroups := p.txnStore.GetEventsByCheckpointTs(checkpointTs) if len(txnGroups) == 0 { @@ -79,18 +71,10 @@ func (p *EventProcessor) processCheckpoint(checkpointTs uint64, txnChan chan<- * return nil } - // Add all transaction groups to pending transactions for progress tracking - for _, txnGroup := range txnGroups { - p.progressTracker.AddPendingTxn(txnGroup.CommitTs, txnGroup.StartTs) - } - // Send transaction groups to the output channel for _, txnGroup := range txnGroups { + p.progressTracker.AddPendingTxn(txnGroup.CommitTs, txnGroup.StartTs) txnChan <- txnGroup - log.Info("txnSink: sent transaction group", - zap.Uint64("commitTs", txnGroup.CommitTs), - zap.Uint64("startTs", txnGroup.StartTs), - zap.Int("eventCount", len(txnGroup.Events))) } // Update checkpoint progress diff --git a/downstreamadapter/sink/txnsink/progress_tracker_test.go b/downstreamadapter/sink/txnsink/progress_tracker_test.go deleted file mode 100644 index e5f360355..000000000 --- a/downstreamadapter/sink/txnsink/progress_tracker_test.go +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2024 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package txnsink - -import ( - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestProgressTracker_Basic(t *testing.T) { - t.Parallel() - - tracker := NewProgressTracker() - - // Initial state - require.Equal(t, uint64(0), tracker.GetEffectiveTs()) - require.Equal(t, uint64(0), tracker.GetProgress()) -} - -func TestProgressTracker_AddPendingTxn(t *testing.T) { - t.Parallel() - - tracker := NewProgressTracker() - - // Add pending transaction - tracker.AddPendingTxn(100, 1) - require.Equal(t, uint64(99), tracker.GetEffectiveTs()) // min(pending) - 1 = 100 - 1 = 99 - - // Add another pending transaction - tracker.AddPendingTxn(200, 2) - require.Equal(t, uint64(99), tracker.GetEffectiveTs()) // Still 99, as 100 is still pending - - progress := tracker.GetProgress() - require.Equal(t, uint64(99), progress) -} - -func TestProgressTracker_RemoveCompletedTxn(t *testing.T) { - t.Parallel() - - tracker := NewProgressTracker() - - // Add pending transactions - tracker.AddPendingTxn(100, 1) - tracker.AddPendingTxn(200, 2) - - // Remove completed transaction (200, 2) - tracker.RemoveCompletedTxn(200, 2) - require.Equal(t, uint64(99), tracker.GetEffectiveTs()) // Still 99, as 100 is still pending - - // Remove completed transaction (100, 1) - tracker.RemoveCompletedTxn(100, 1) - require.Equal(t, uint64(0), tracker.GetEffectiveTs()) // No pending, use checkpointTs (0) - - progress := tracker.GetProgress() - require.Equal(t, uint64(0), progress) -} - -func TestProgressTracker_UpdateCheckpointTs(t *testing.T) { - t.Parallel() - - tracker := NewProgressTracker() - - // Update checkpoint TS - tracker.UpdateCheckpointTs(100) - require.Equal(t, uint64(100), tracker.GetEffectiveTs()) // No pending, use checkpointTs - - // Add pending transaction - tracker.AddPendingTxn(50, 1) - require.Equal(t, uint64(49), tracker.GetEffectiveTs()) // min(pending) - 1 = 50 - 1 = 49 - - // Update checkpoint TS (should not affect effective TS when there are pending transactions) - tracker.UpdateCheckpointTs(200) - require.Equal(t, uint64(49), tracker.GetEffectiveTs()) // Still 49, as 50 is still pending - - // Remove pending transaction - tracker.RemoveCompletedTxn(50, 1) - require.Equal(t, uint64(200), tracker.GetEffectiveTs()) // No pending, use checkpointTs (200) -} - -func TestProgressTracker_EffectiveTs(t *testing.T) { - t.Parallel() - - tracker := NewProgressTracker() - - // Initially should be 0 - require.Equal(t, uint64(0), tracker.GetEffectiveTs()) - - // Only checkpoint TS set - tracker.UpdateCheckpointTs(100) - require.Equal(t, uint64(100), tracker.GetEffectiveTs()) // Use checkpointTs - - // Add pending transactions - tracker.AddPendingTxn(50, 1) - tracker.AddPendingTxn(75, 2) - require.Equal(t, uint64(49), tracker.GetEffectiveTs()) // min(50, 75) - 1 = 49 - - // Remove first pending transaction - tracker.RemoveCompletedTxn(50, 1) - require.Equal(t, uint64(74), tracker.GetEffectiveTs()) // min(75) - 1 = 74 - - // Remove second pending transaction - tracker.RemoveCompletedTxn(75, 2) - require.Equal(t, uint64(100), tracker.GetEffectiveTs()) // No pending, use checkpointTs (100) -} - -func TestProgressTracker_SameCommitTsDifferentStartTs(t *testing.T) { - t.Parallel() - - tracker := NewProgressTracker() - - // Add transactions with same commitTs but different startTs - tracker.AddPendingTxn(100, 1) - tracker.AddPendingTxn(100, 2) - tracker.AddPendingTxn(100, 3) - - require.Equal(t, uint64(99), tracker.GetEffectiveTs()) // min(100) - 1 = 99 - - // Remove one transaction - tracker.RemoveCompletedTxn(100, 2) - require.Equal(t, uint64(99), tracker.GetEffectiveTs()) // Still 99, as 100 is still pending - - // Remove another transaction - tracker.RemoveCompletedTxn(100, 1) - require.Equal(t, uint64(99), tracker.GetEffectiveTs()) // Still 99, as 100 is still pending - - // Remove the last transaction - tracker.RemoveCompletedTxn(100, 3) - require.Equal(t, uint64(0), tracker.GetEffectiveTs()) // No pending, use checkpointTs (0) -} - -func TestProgressTracker_Reset(t *testing.T) { - t.Parallel() - - tracker := NewProgressTracker() - - // Set some values - tracker.AddPendingTxn(100, 1) - tracker.AddPendingTxn(200, 2) - tracker.UpdateCheckpointTs(300) - - // Reset - tracker.Reset() - - // Check all values are reset - require.Equal(t, uint64(0), tracker.GetEffectiveTs()) - require.Equal(t, uint64(0), tracker.GetProgress()) -} - -func TestProgressTracker_Concurrent(t *testing.T) { - t.Parallel() - - tracker := NewProgressTracker() - - // Test concurrent updates - done := make(chan bool, 3) - - // Add transactions - go func() { - for i := uint64(1); i <= 10; i++ { - tracker.AddPendingTxn(i, i) - } - done <- true - }() - - // Update checkpoint TS - go func() { - for i := uint64(1); i <= 10; i++ { - tracker.UpdateCheckpointTs(i) - } - done <- true - }() - - // Read progress concurrently - go func() { - for i := uint64(1); i <= 10; i++ { - tracker.GetEffectiveTs() - tracker.GetProgress() - } - done <- true - }() - - // Wait for all goroutines to complete - <-done - <-done - <-done - - // Final state should be consistent - effectiveTs := tracker.GetEffectiveTs() - progress := tracker.GetProgress() - require.Equal(t, effectiveTs, progress) -} - -func TestProgressTracker_Monitor(t *testing.T) { - t.Parallel() - - // Create tracker with monitor - tracker := NewProgressTrackerWithMonitor("test-changefeed") - defer tracker.Close() - - // Add some transactions and update checkpoint - tracker.AddPendingTxn(100, 1) - tracker.AddPendingTxn(200, 2) - tracker.UpdateCheckpointTs(150) - - // Wait for at least one monitor cycle - time.Sleep(1500 * time.Millisecond) - - // Remove a transaction and wait for another cycle - tracker.RemoveCompletedTxn(100, 1) - time.Sleep(1500 * time.Millisecond) - - // Verify the tracker is still working - require.Equal(t, uint64(199), tracker.GetEffectiveTs()) // min(200) - 1 = 199 -} diff --git a/downstreamadapter/sink/txnsink/sink.go b/downstreamadapter/sink/txnsink/sink.go index 0ab1c8288..d0139a05e 100644 --- a/downstreamadapter/sink/txnsink/sink.go +++ b/downstreamadapter/sink/txnsink/sink.go @@ -63,7 +63,6 @@ func New(ctx context.Context, changefeedID common.ChangeFeedID, db *sql.DB, conf config = &TxnSinkConfig{ MaxConcurrentTxns: 16, BatchSize: 256, - FlushInterval: 100, MaxSQLBatchSize: 1024 * 16, } } diff --git a/downstreamadapter/sink/txnsink/types.go b/downstreamadapter/sink/txnsink/types.go index a97c3db48..10f54cf92 100644 --- a/downstreamadapter/sink/txnsink/types.go +++ b/downstreamadapter/sink/txnsink/types.go @@ -53,11 +53,6 @@ func NewTxnStore() *TxnStore { // AddEvent adds a DML event to the store func (ts *TxnStore) AddEvent(event *commonEvent.DMLEvent) { - log.Info("txnSink: add event", - zap.Uint64("commitTs", event.CommitTs), - zap.Uint64("startTs", event.StartTs), - zap.Int64("tableID", event.GetTableID()), - zap.Int32("rowCount", event.Len())) ts.mu.Lock() defer ts.mu.Unlock() @@ -492,6 +487,5 @@ func (cd *ConflictDetector) closeCache() { type TxnSinkConfig struct { MaxConcurrentTxns int BatchSize int - FlushInterval int // milliseconds MaxSQLBatchSize int // maximum size of SQL batch in bytes } diff --git a/downstreamadapter/sink/txnsink/worker.go b/downstreamadapter/sink/txnsink/worker.go index 6905a1cbb..89e064f17 100644 --- a/downstreamadapter/sink/txnsink/worker.go +++ b/downstreamadapter/sink/txnsink/worker.go @@ -346,6 +346,7 @@ func (w *Worker) executeSQLBatch(batch []*TxnSQL) error { // Update flushed progress for all transactions in the batch for _, txnSQL := range batch { + txnSQL.TxnGroup.PostFlush() w.progressTracker.RemoveCompletedTxn(txnSQL.TxnGroup.CommitTs, txnSQL.TxnGroup.StartTs) } From 86cacfd9d0a99daf1416b30a1cc564ec3252dd2e Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 25 Aug 2025 14:48:13 +0800 Subject: [PATCH 05/15] update --- downstreamadapter/sink/sink.go | 1 - 1 file changed, 1 deletion(-) diff --git a/downstreamadapter/sink/sink.go b/downstreamadapter/sink/sink.go index fdfc04402..45625aaa0 100644 --- a/downstreamadapter/sink/sink.go +++ b/downstreamadapter/sink/sink.go @@ -82,7 +82,6 @@ func newTxnSinkAdapter( txnConfig := &txnsink.TxnSinkConfig{ MaxConcurrentTxns: 16, BatchSize: 16, - FlushInterval: 100, MaxSQLBatchSize: 1024 * 16, // 1MB } From ba07ba57198220958ccbfbe6efc84ce755fa63f7 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 25 Aug 2025 15:44:04 +0800 Subject: [PATCH 06/15] update --- downstreamadapter/sink/sink.go | 2 +- downstreamadapter/sink/txnsink/worker.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/downstreamadapter/sink/sink.go b/downstreamadapter/sink/sink.go index 45625aaa0..c0bad3ef2 100644 --- a/downstreamadapter/sink/sink.go +++ b/downstreamadapter/sink/sink.go @@ -81,7 +81,7 @@ func newTxnSinkAdapter( // Create txnSink configuration txnConfig := &txnsink.TxnSinkConfig{ MaxConcurrentTxns: 16, - BatchSize: 16, + BatchSize: 256, MaxSQLBatchSize: 1024 * 16, // 1MB } diff --git a/downstreamadapter/sink/txnsink/worker.go b/downstreamadapter/sink/txnsink/worker.go index 89e064f17..3cb425dec 100644 --- a/downstreamadapter/sink/txnsink/worker.go +++ b/downstreamadapter/sink/txnsink/worker.go @@ -206,7 +206,6 @@ func (w *Worker) executeSQLBatches(ctx context.Context) error { zap.Int("workerID", w.workerID)) buffer := make([]*TxnSQL, 0, w.config.BatchSize) - currentBatchSize := 0 totalStart := time.Now() for { @@ -226,8 +225,9 @@ func (w *Worker) executeSQLBatches(ctx context.Context) error { } batch := make([]*TxnSQL, 0, w.config.BatchSize) + currentBatchSize := 0 - log.Debug("txnSink: got sql batch", + log.Info("txnSink: got sql batch", zap.String("namespace", namespace), zap.String("changefeed", changefeed), zap.Int("workerID", w.workerID), From b225361ffee497e2e0864d355c9016c949b10749 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 25 Aug 2025 15:56:09 +0800 Subject: [PATCH 07/15] update --- downstreamadapter/sink/sink.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/downstreamadapter/sink/sink.go b/downstreamadapter/sink/sink.go index c0bad3ef2..54c44b077 100644 --- a/downstreamadapter/sink/sink.go +++ b/downstreamadapter/sink/sink.go @@ -80,7 +80,7 @@ func newTxnSinkAdapter( // Create txnSink configuration txnConfig := &txnsink.TxnSinkConfig{ - MaxConcurrentTxns: 16, + MaxConcurrentTxns: 128, BatchSize: 256, MaxSQLBatchSize: 1024 * 16, // 1MB } From 709fda69eb62e39486320436ec5cad748ae151b3 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 25 Aug 2025 20:12:21 +0800 Subject: [PATCH 08/15] update --- downstreamadapter/sink/txnsink/progress_tracker.go | 8 +++++++- downstreamadapter/sink/txnsink/sink.go | 2 +- downstreamadapter/sink/txnsink/worker.go | 4 +++- pkg/metrics/sink.go | 10 ++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/downstreamadapter/sink/txnsink/progress_tracker.go b/downstreamadapter/sink/txnsink/progress_tracker.go index 0ce0488cc..07926aa06 100644 --- a/downstreamadapter/sink/txnsink/progress_tracker.go +++ b/downstreamadapter/sink/txnsink/progress_tracker.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/log" appcontext "github.com/pingcap/ticdc/pkg/common/context" + "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/pdutil" "github.com/tikv/client-go/v2/oracle" "go.uber.org/zap" @@ -47,6 +48,7 @@ type ProgressTracker struct { // Monitoring pdClock pdutil.Clock cancelMonitor context.CancelFunc + namespace string changefeedName string } @@ -60,11 +62,12 @@ func NewProgressTracker() *ProgressTracker { } // NewProgressTrackerWithMonitor creates a new progress tracker with monitoring enabled -func NewProgressTrackerWithMonitor(changefeedName string) *ProgressTracker { +func NewProgressTrackerWithMonitor(namespace, changefeedName string) *ProgressTracker { pt := &ProgressTracker{ checkpointTs: 0, list: list.New(), elemMap: make(map[TxnKey]*list.Element), + namespace: namespace, changefeedName: changefeedName, } @@ -182,6 +185,9 @@ func (pt *ProgressTracker) printProgress() { phyEffectiveTs := oracle.ExtractPhysical(effectiveTs) lag := float64(oracle.GetPhysical(pdTime)-phyEffectiveTs) / 1e3 // Convert to seconds + // Update monitoring metrics + metrics.TxnSinkProgressLagGauge.WithLabelValues(pt.namespace, pt.changefeedName).Set(lag) + log.Info("txnSink: Progress status", zap.String("changefeed", pt.changefeedName), zap.Uint64("effectiveTs", effectiveTs), diff --git a/downstreamadapter/sink/txnsink/sink.go b/downstreamadapter/sink/txnsink/sink.go index d0139a05e..be57858ce 100644 --- a/downstreamadapter/sink/txnsink/sink.go +++ b/downstreamadapter/sink/txnsink/sink.go @@ -69,7 +69,7 @@ func New(ctx context.Context, changefeedID common.ChangeFeedID, db *sql.DB, conf txnStore := NewTxnStore() conflictDetector := NewConflictDetector(changefeedID, config.MaxConcurrentTxns) - progressTracker := NewProgressTrackerWithMonitor(changefeedID.Name()) + progressTracker := NewProgressTrackerWithMonitor(changefeedID.Namespace(), changefeedID.Name()) eventProcessor := NewEventProcessor(txnStore, progressTracker) // Create workers diff --git a/downstreamadapter/sink/txnsink/worker.go b/downstreamadapter/sink/txnsink/worker.go index 3cb425dec..2c543a1fc 100644 --- a/downstreamadapter/sink/txnsink/worker.go +++ b/downstreamadapter/sink/txnsink/worker.go @@ -328,7 +328,9 @@ func (w *Worker) executeSQLBatch(batch []*TxnSQL) error { // Return row count and approximate size (using SQL length as approximation) approximateSize := int64(0) for _, txnSQL := range batch { - approximateSize += int64(len(txnSQL.SQL)) + for _, event := range txnSQL.TxnGroup.Events { + approximateSize += int64(event.GetSize()) + } } return totalRowCount, approximateSize, nil }) diff --git a/pkg/metrics/sink.go b/pkg/metrics/sink.go index ae89a177a..a35b81821 100644 --- a/pkg/metrics/sink.go +++ b/pkg/metrics/sink.go @@ -162,6 +162,15 @@ var ( Help: "Size of transaction store in txnSink.", }, []string{"namespace", "changefeed"}) + // TxnSinkProgressLagGauge records the lag of progress tracker in txnSink + TxnSinkProgressLagGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "txn_sink_progress_lag", + Help: "Progress lag of txnSink in seconds (based on effectiveTs).", + }, []string{"namespace", "changefeed"}) + SinkDMLBatchCommit = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "ticdc", @@ -257,6 +266,7 @@ func InitSinkMetrics(registry *prometheus.Registry) { registry.MustRegister(TxnSinkBatchDuration) registry.MustRegister(TxnSinkPendingTxns) registry.MustRegister(TxnSinkTxnStoreSize) + registry.MustRegister(TxnSinkProgressLagGauge) registry.MustRegister(SinkDMLBatchCommit) registry.MustRegister(SinkDMLBatchCallback) registry.MustRegister(PrepareStatementErrors) From 0eaebee2c2d89a95310e21104cdf2b79a3c77fc1 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 25 Aug 2025 20:13:33 +0800 Subject: [PATCH 09/15] update --- downstreamadapter/sink/txnsink/sink.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/downstreamadapter/sink/txnsink/sink.go b/downstreamadapter/sink/txnsink/sink.go index be57858ce..0e5b7670f 100644 --- a/downstreamadapter/sink/txnsink/sink.go +++ b/downstreamadapter/sink/txnsink/sink.go @@ -61,7 +61,7 @@ type Sink struct { func New(ctx context.Context, changefeedID common.ChangeFeedID, db *sql.DB, config *TxnSinkConfig) *Sink { if config == nil { config = &TxnSinkConfig{ - MaxConcurrentTxns: 16, + MaxConcurrentTxns: 32, BatchSize: 256, MaxSQLBatchSize: 1024 * 16, } From 53a409a4885c5dbb9ccd285c4c6e37c7987281c5 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 26 Aug 2025 14:40:59 +0800 Subject: [PATCH 10/15] support if statement --- .../sink/txnsink/sql_generator.go | 10 +++-- pkg/sink/mysql/mysql_writer_dml.go | 18 +++++--- pkg/sink/sqlmodel/multi_row.go | 45 +++++++++++++++---- 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/downstreamadapter/sink/txnsink/sql_generator.go b/downstreamadapter/sink/txnsink/sql_generator.go index 2d306e657..32fa7f343 100644 --- a/downstreamadapter/sink/txnsink/sql_generator.go +++ b/downstreamadapter/sink/txnsink/sql_generator.go @@ -85,6 +85,8 @@ func (g *SQLGenerator) generateTableSQL(events []*commonEvent.DMLEvent) ([]strin } tableInfo := events[0].TableInfo + startTs := events[0].StartTs + commitTs := events[0].CommitTs // Group rows by type (insert, update, delete) insertRows, updateRows, deleteRows := g.groupRowsByType(events, tableInfo) @@ -104,7 +106,7 @@ func (g *SQLGenerator) generateTableSQL(events []*commonEvent.DMLEvent) ([]strin // Handle update operations - use INSERT ON DUPLICATE KEY UPDATE if len(updateRows) > 0 { for _, rows := range updateRows { - sql, value := g.genInsertOnDuplicateUpdateSQL(rows...) + sql, value := g.genInsertOnDuplicateUpdateSQL(startTs, commitTs, rows...) sqls = append(sqls, sql) args = append(args, value) } @@ -113,7 +115,7 @@ func (g *SQLGenerator) generateTableSQL(events []*commonEvent.DMLEvent) ([]strin // Handle insert operations - use INSERT ON DUPLICATE KEY UPDATE if len(insertRows) > 0 { for _, rows := range insertRows { - sql, value := g.genInsertOnDuplicateUpdateSQL(rows...) + sql, value := g.genInsertOnDuplicateUpdateSQL(startTs, commitTs, rows...) sqls = append(sqls, sql) args = append(args, value) } @@ -196,8 +198,8 @@ func (g *SQLGenerator) genDeleteSQL(rows ...*sqlmodel.RowChange) (string, []inte } // genInsertOnDuplicateUpdateSQL generates INSERT ON DUPLICATE KEY UPDATE SQL -func (g *SQLGenerator) genInsertOnDuplicateUpdateSQL(rows ...*sqlmodel.RowChange) (string, []interface{}) { - return sqlmodel.GenInsertSQL(sqlmodel.DMLInsertOnDuplicateUpdate, rows...) +func (g *SQLGenerator) genInsertOnDuplicateUpdateSQL(startTs uint64, commitTs uint64, rows ...*sqlmodel.RowChange) (string, []interface{}) { + return sqlmodel.GenInsertSQLWithStartTs(sqlmodel.DMLInsertOnDuplicateUpdate, startTs, commitTs, rows...) } // getArgsWithGeneratedColumn extracts column values including generated columns diff --git a/pkg/sink/mysql/mysql_writer_dml.go b/pkg/sink/mysql/mysql_writer_dml.go index c0fa157ef..81f61b670 100644 --- a/pkg/sink/mysql/mysql_writer_dml.go +++ b/pkg/sink/mysql/mysql_writer_dml.go @@ -231,7 +231,9 @@ func (w *Writer) generateSQLForSingleEvent(event *commonEvent.DMLEvent, inDataSa } rowLists = append(rowLists, &row) } - return w.batchSingleTxnDmls(rowLists, tableInfo, inDataSafeMode) + startTs := event.StartTs + commitTs := event.CommitTs + return w.batchSingleTxnDmls(rowLists, tableInfo, inDataSafeMode, startTs, commitTs) } func (w *Writer) generateBatchSQLInSafeMode(events []*commonEvent.DMLEvent) ([]string, [][]interface{}) { @@ -404,7 +406,9 @@ func (w *Writer) generateBatchSQLInSafeMode(events []*commonEvent.DMLEvent) ([]s } // Step 3. generate sqls based on finalRowLists - return w.batchSingleTxnDmls(finalRowLists, tableInfo, true) + startTs := events[0].StartTs + commitTs := events[0].CommitTs + return w.batchSingleTxnDmls(finalRowLists, tableInfo, true, startTs, commitTs) } func (w *Writer) generateBatchSQLInUnsafeMode(events []*commonEvent.DMLEvent) ([]string, [][]interface{}) { @@ -497,7 +501,9 @@ func (w *Writer) generateBatchSQLInUnsafeMode(events []*commonEvent.DMLEvent) ([ rowsList = append(rowsList, rowChanges[len(rowChanges)-1]) } // step 3. generate sqls based on rowsList - return w.batchSingleTxnDmls(rowsList, tableInfo, false) + startTs := events[0].StartTs + commitTs := events[0].CommitTs + return w.batchSingleTxnDmls(rowsList, tableInfo, false, startTs, commitTs) } func (w *Writer) generateNormalSQLs(events []*commonEvent.DMLEvent) ([]string, [][]interface{}) { @@ -744,6 +750,8 @@ func (w *Writer) batchSingleTxnDmls( rows []*commonEvent.RowChange, tableInfo *common.TableInfo, translateToInsert bool, + startTs uint64, + commitTs uint64, ) (sqls []string, values [][]interface{}) { insertRows, updateRows, deleteRows := w.groupRowsByType(rows, tableInfo) @@ -782,11 +790,11 @@ func (w *Writer) batchSingleTxnDmls( if len(insertRows) > 0 { for _, rows := range insertRows { if translateToInsert { - sql, value := sqlmodel.GenInsertSQL(sqlmodel.DMLInsert, rows...) + sql, value := sqlmodel.GenInsertSQLWithStartTs(sqlmodel.DMLInsert, startTs, commitTs, rows...) sqls = append(sqls, sql) values = append(values, value) } else { - sql, value := sqlmodel.GenInsertSQL(sqlmodel.DMLReplace, rows...) + sql, value := sqlmodel.GenInsertSQLWithStartTs(sqlmodel.DMLReplace, startTs, commitTs, rows...) sqls = append(sqls, sql) values = append(values, value) } diff --git a/pkg/sink/sqlmodel/multi_row.go b/pkg/sink/sqlmodel/multi_row.go index 9291b2c76..1ea7e5ed9 100644 --- a/pkg/sink/sqlmodel/multi_row.go +++ b/pkg/sink/sqlmodel/multi_row.go @@ -227,6 +227,15 @@ func GenUpdateSQL(changes ...*RowChange) (string, []any) { // Input `changes` should have same target table and same modifiable columns, // otherwise the behaviour is undefined. func GenInsertSQL(tp DMLType, changes ...*RowChange) (string, []interface{}) { + return GenInsertSQLWithStartTs(tp, 0, 0, changes...) +} + +// GenInsertSQLWithStartTs generates the INSERT SQL and its arguments with optional start_ts column. +// Input `changes` should have same target table and same modifiable columns, +// otherwise the behaviour is undefined. +// If startTs is greater than 0, the last column (start_ts) will be set to startTs value instead of NULL. +// If commitTs is greater than 0, the last column (origin_ts) will be set to commitTs value instead of NULL. +func GenInsertSQLWithStartTs(tp DMLType, startTs uint64, commitTs uint64, changes ...*RowChange) (string, []interface{}) { if len(changes) == 0 { log.L().DPanic("row changes is empty") return "", nil @@ -260,6 +269,7 @@ func GenInsertSQL(tp DMLType, changes ...*RowChange) (string, []interface{}) { columnNum++ buf.WriteString(common.QuoteName(col.Name.O)) } + buf.WriteString(") VALUES ") holder := valuesHolder(columnNum) for i := range changes { @@ -285,23 +295,42 @@ func GenInsertSQL(tp DMLType, changes ...*RowChange) (string, []interface{}) { writtenFirstCol = true colName := common.QuoteName(col.Name.O) - buf.WriteString(colName + "=VALUES(" + colName + ")") + tableName := first.targetTable.QuoteString() + + // For all columns, use conditional update logic + buf.WriteString(colName + "=IF((@cond := (IFNULL(" + tableName + "." + colName + ", " + tableName + ".commit_ts) <= VALUES(" + colName + "))),VALUES(" + colName + "), " + tableName + "." + colName + ")") } } args := make([]interface{}, 0, len(changes)*(len(first.sourceTableInfo.GetColumns())-len(skipColIdx))) for _, change := range changes { - i := 0 // used as index of skipColIdx + i := 0 // used as index of skipColIdx + colIndex := 0 // used to track the actual column index for j, val := range change.postValues { - if i >= len(skipColIdx) { - args = append(args, change.postValues[j:]...) - break - } - if skipColIdx[i] == j { + if i < len(skipColIdx) && skipColIdx[i] == j { i++ continue } - args = append(args, val) + + // If this is the last column and commitTs is provided, replace NULL with commitTs + if commitTs > 0 && colIndex == len(first.sourceTableInfo.GetColumns())-len(skipColIdx)-1 { + // This is the last column (origin_ts), replace NULL with commitTs + if val == nil { + args = append(args, commitTs) + } else { + args = append(args, val) + } + } else if startTs > 0 && colIndex == len(first.sourceTableInfo.GetColumns())-len(skipColIdx)-1 { + // This is the last column (start_ts), replace NULL with startTs + if val == nil { + args = append(args, startTs) + } else { + args = append(args, val) + } + } else { + args = append(args, val) + } + colIndex++ } } return buf.String(), args From fa6cc4dae50ece8fc355a3d509d0d26a1a4545f1 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 27 Aug 2025 10:47:40 +0800 Subject: [PATCH 11/15] update --- downstreamadapter/sink/sink.go | 23 ++- downstreamadapter/sink/sink_test.go | 70 +++++++++ .../sink/txnsink/progress_tracker_test.go | 51 +++++++ .../sink/txnsink/sql_generator.go | 2 +- pkg/sink/mysql/mysql_writer_dml.go | 18 +-- pkg/sink/sqlmodel/multi_row.go | 135 ++++++++++++++++-- 6 files changed, 274 insertions(+), 25 deletions(-) create mode 100644 downstreamadapter/sink/sink_test.go create mode 100644 downstreamadapter/sink/txnsink/progress_tracker_test.go diff --git a/downstreamadapter/sink/sink.go b/downstreamadapter/sink/sink.go index 54c44b077..f8a878a56 100644 --- a/downstreamadapter/sink/sink.go +++ b/downstreamadapter/sink/sink.go @@ -16,6 +16,7 @@ package sink import ( "context" "net/url" + "strconv" "github.com/pingcap/ticdc/downstreamadapter/sink/blackhole" "github.com/pingcap/ticdc/downstreamadapter/sink/cloudstorage" @@ -52,7 +53,12 @@ func New(ctx context.Context, cfg *config.ChangefeedConfig, changefeedID common. scheme := config.GetScheme(sinkURI) switch scheme { case config.MySQLScheme, config.MySQLSSLScheme, config.TiDBScheme, config.TiDBSSLScheme: - return newTxnSinkAdapter(ctx, changefeedID, cfg, sinkURI) + // Check if enable-transaction-atomic is set to true + if isTransactionAtomicEnabled(sinkURI) { + return newTxnSinkAdapter(ctx, changefeedID, cfg, sinkURI) + } + // Use mysqlSink if enable-transaction-atomic is not set or set to false + return mysql.New(ctx, changefeedID, cfg, sinkURI) case config.KafkaScheme, config.KafkaSSLScheme: return kafka.New(ctx, changefeedID, sinkURI, cfg.SinkConfig) case config.PulsarScheme, config.PulsarSSLScheme, config.PulsarHTTPScheme, config.PulsarHTTPSScheme: @@ -65,6 +71,21 @@ func New(ctx context.Context, cfg *config.ChangefeedConfig, changefeedID common. return nil, errors.ErrSinkURIInvalid.GenWithStackByArgs(sinkURI) } +// isTransactionAtomicEnabled checks if enable-transaction-atomic parameter is set to true in sink URI +func isTransactionAtomicEnabled(sinkURI *url.URL) bool { + query := sinkURI.Query() + s := query.Get("enable-transaction-atomic") + if len(s) == 0 { + return false + } + enabled, err := strconv.ParseBool(s) + if err != nil { + // If the parameter value is invalid, default to false + return false + } + return enabled +} + // newTxnSinkAdapter creates a txnSink adapter that uses the same database connection as mysqlSink func newTxnSinkAdapter( ctx context.Context, diff --git a/downstreamadapter/sink/sink_test.go b/downstreamadapter/sink/sink_test.go new file mode 100644 index 000000000..8a8ef39e3 --- /dev/null +++ b/downstreamadapter/sink/sink_test.go @@ -0,0 +1,70 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package sink + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsTransactionAtomicEnabled(t *testing.T) { + tests := []struct { + name string + sinkURI string + expected bool + }{ + { + name: "enable-transaction-atomic=true", + sinkURI: "mysql://user:pass@localhost:3306/test?enable-transaction-atomic=true", + expected: true, + }, + { + name: "enable-transaction-atomic=false", + sinkURI: "mysql://user:pass@localhost:3306/test?enable-transaction-atomic=false", + expected: false, + }, + { + name: "enable-transaction-atomic not set", + sinkURI: "mysql://user:pass@localhost:3306/test", + expected: false, + }, + { + name: "enable-transaction-atomic with invalid value", + sinkURI: "mysql://user:pass@localhost:3306/test?enable-transaction-atomic=invalid", + expected: false, + }, + { + name: "enable-transaction-atomic=1", + sinkURI: "mysql://user:pass@localhost:3306/test?enable-transaction-atomic=1", + expected: true, + }, + { + name: "enable-transaction-atomic=0", + sinkURI: "mysql://user:pass@localhost:3306/test?enable-transaction-atomic=0", + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parsedURI, err := url.Parse(tt.sinkURI) + require.NoError(t, err) + + result := isTransactionAtomicEnabled(parsedURI) + require.Equal(t, tt.expected, result) + }) + } +} diff --git a/downstreamadapter/sink/txnsink/progress_tracker_test.go b/downstreamadapter/sink/txnsink/progress_tracker_test.go new file mode 100644 index 000000000..022985a3d --- /dev/null +++ b/downstreamadapter/sink/txnsink/progress_tracker_test.go @@ -0,0 +1,51 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package txnsink + +import ( + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func TestProgressTrackerMetrics(t *testing.T) { + // Create a new registry for testing + registry := prometheus.NewRegistry() + metrics.InitSinkMetrics(registry) + + // Create progress tracker with monitoring + pt := NewProgressTrackerWithMonitor("test_namespace", "test_changefeed") + defer pt.Close() + + // Add some pending transactions + pt.AddPendingTxn(1000, 900) + pt.AddPendingTxn(2000, 1900) + + // Update checkpoint ts + pt.UpdateCheckpointTs(500) + + // Wait a bit for the monitor to run + time.Sleep(2 * time.Second) + + // Check if the metric is registered and has a value + metricValue := testutil.ToFloat64(metrics.TxnSinkProgressLagGauge.WithLabelValues("test_namespace", "test_changefeed")) + require.Greater(t, metricValue, float64(0), "Progress lag metric should be greater than 0") + + // Verify the metric is working correctly + require.NotZero(t, metricValue, "Progress lag metric should not be zero") +} diff --git a/downstreamadapter/sink/txnsink/sql_generator.go b/downstreamadapter/sink/txnsink/sql_generator.go index 32fa7f343..a9a783fb4 100644 --- a/downstreamadapter/sink/txnsink/sql_generator.go +++ b/downstreamadapter/sink/txnsink/sql_generator.go @@ -199,7 +199,7 @@ func (g *SQLGenerator) genDeleteSQL(rows ...*sqlmodel.RowChange) (string, []inte // genInsertOnDuplicateUpdateSQL generates INSERT ON DUPLICATE KEY UPDATE SQL func (g *SQLGenerator) genInsertOnDuplicateUpdateSQL(startTs uint64, commitTs uint64, rows ...*sqlmodel.RowChange) (string, []interface{}) { - return sqlmodel.GenInsertSQLWithStartTs(sqlmodel.DMLInsertOnDuplicateUpdate, startTs, commitTs, rows...) + return sqlmodel.GenInsertSQLWithCommitTs(sqlmodel.DMLInsertOnDuplicateUpdate, startTs, commitTs, rows...) } // getArgsWithGeneratedColumn extracts column values including generated columns diff --git a/pkg/sink/mysql/mysql_writer_dml.go b/pkg/sink/mysql/mysql_writer_dml.go index 81f61b670..c0fa157ef 100644 --- a/pkg/sink/mysql/mysql_writer_dml.go +++ b/pkg/sink/mysql/mysql_writer_dml.go @@ -231,9 +231,7 @@ func (w *Writer) generateSQLForSingleEvent(event *commonEvent.DMLEvent, inDataSa } rowLists = append(rowLists, &row) } - startTs := event.StartTs - commitTs := event.CommitTs - return w.batchSingleTxnDmls(rowLists, tableInfo, inDataSafeMode, startTs, commitTs) + return w.batchSingleTxnDmls(rowLists, tableInfo, inDataSafeMode) } func (w *Writer) generateBatchSQLInSafeMode(events []*commonEvent.DMLEvent) ([]string, [][]interface{}) { @@ -406,9 +404,7 @@ func (w *Writer) generateBatchSQLInSafeMode(events []*commonEvent.DMLEvent) ([]s } // Step 3. generate sqls based on finalRowLists - startTs := events[0].StartTs - commitTs := events[0].CommitTs - return w.batchSingleTxnDmls(finalRowLists, tableInfo, true, startTs, commitTs) + return w.batchSingleTxnDmls(finalRowLists, tableInfo, true) } func (w *Writer) generateBatchSQLInUnsafeMode(events []*commonEvent.DMLEvent) ([]string, [][]interface{}) { @@ -501,9 +497,7 @@ func (w *Writer) generateBatchSQLInUnsafeMode(events []*commonEvent.DMLEvent) ([ rowsList = append(rowsList, rowChanges[len(rowChanges)-1]) } // step 3. generate sqls based on rowsList - startTs := events[0].StartTs - commitTs := events[0].CommitTs - return w.batchSingleTxnDmls(rowsList, tableInfo, false, startTs, commitTs) + return w.batchSingleTxnDmls(rowsList, tableInfo, false) } func (w *Writer) generateNormalSQLs(events []*commonEvent.DMLEvent) ([]string, [][]interface{}) { @@ -750,8 +744,6 @@ func (w *Writer) batchSingleTxnDmls( rows []*commonEvent.RowChange, tableInfo *common.TableInfo, translateToInsert bool, - startTs uint64, - commitTs uint64, ) (sqls []string, values [][]interface{}) { insertRows, updateRows, deleteRows := w.groupRowsByType(rows, tableInfo) @@ -790,11 +782,11 @@ func (w *Writer) batchSingleTxnDmls( if len(insertRows) > 0 { for _, rows := range insertRows { if translateToInsert { - sql, value := sqlmodel.GenInsertSQLWithStartTs(sqlmodel.DMLInsert, startTs, commitTs, rows...) + sql, value := sqlmodel.GenInsertSQL(sqlmodel.DMLInsert, rows...) sqls = append(sqls, sql) values = append(values, value) } else { - sql, value := sqlmodel.GenInsertSQLWithStartTs(sqlmodel.DMLReplace, startTs, commitTs, rows...) + sql, value := sqlmodel.GenInsertSQL(sqlmodel.DMLReplace, rows...) sqls = append(sqls, sql) values = append(values, value) } diff --git a/pkg/sink/sqlmodel/multi_row.go b/pkg/sink/sqlmodel/multi_row.go index 1ea7e5ed9..facb36997 100644 --- a/pkg/sink/sqlmodel/multi_row.go +++ b/pkg/sink/sqlmodel/multi_row.go @@ -227,15 +227,89 @@ func GenUpdateSQL(changes ...*RowChange) (string, []any) { // Input `changes` should have same target table and same modifiable columns, // otherwise the behaviour is undefined. func GenInsertSQL(tp DMLType, changes ...*RowChange) (string, []interface{}) { - return GenInsertSQLWithStartTs(tp, 0, 0, changes...) + if len(changes) == 0 { + log.L().DPanic("row changes is empty") + return "", nil + } + + first := changes[0] + + var buf strings.Builder + buf.Grow(1024) + if tp == DMLReplace { + buf.WriteString("REPLACE INTO ") + } else { + buf.WriteString("INSERT INTO ") + } + buf.WriteString(first.targetTable.QuoteString()) + buf.WriteString(" (") + columnNum := 0 + var skipColIdx []int + + // build generated columns lower name set to accelerate the following check + generatedColumns := generatedColumnsNameSet(first.targetTableInfo.GetColumns()) + for i, col := range first.sourceTableInfo.GetColumns() { + if _, ok := generatedColumns[col.Name.L]; ok { + skipColIdx = append(skipColIdx, i) + continue + } + + if columnNum != 0 { + buf.WriteByte(',') + } + columnNum++ + buf.WriteString(common.QuoteName(col.Name.O)) + } + + buf.WriteString(") VALUES ") + holder := valuesHolder(columnNum) + for i := range changes { + if i > 0 { + buf.WriteString(",") + } + buf.WriteString(holder) + } + if tp == DMLInsertOnDuplicateUpdate { + buf.WriteString(" ON DUPLICATE KEY UPDATE ") + i := 0 // used as index of skipColIdx + writtenFirstCol := false + + for j, col := range first.sourceTableInfo.GetColumns() { + if i < len(skipColIdx) && skipColIdx[i] == j { + i++ + continue + } + + if writtenFirstCol { + buf.WriteByte(',') + } + writtenFirstCol = true + + colName := common.QuoteName(col.Name.O) + buf.WriteString(colName + "=VALUES(" + colName + ")") + } + } + + args := make([]interface{}, 0, len(changes)*(len(first.sourceTableInfo.GetColumns())-len(skipColIdx))) + for _, change := range changes { + i := 0 // used as index of skipColIdx + for j, val := range change.postValues { + if i < len(skipColIdx) && skipColIdx[i] == j { + i++ + continue + } + args = append(args, val) + } + } + return buf.String(), args } -// GenInsertSQLWithStartTs generates the INSERT SQL and its arguments with optional start_ts column. +// GenInsertSQLWithCommitTs generates the INSERT SQL and its arguments with commitTs logic for txnSink. +// This function is specifically designed for txnSink to handle _tidb_origin_ts column. // Input `changes` should have same target table and same modifiable columns, // otherwise the behaviour is undefined. -// If startTs is greater than 0, the last column (start_ts) will be set to startTs value instead of NULL. -// If commitTs is greater than 0, the last column (origin_ts) will be set to commitTs value instead of NULL. -func GenInsertSQLWithStartTs(tp DMLType, startTs uint64, commitTs uint64, changes ...*RowChange) (string, []interface{}) { +// If commitTs is greater than 0, the _tidb_origin_ts column will be handled specially. +func GenInsertSQLWithCommitTs(tp DMLType, startTs uint64, commitTs uint64, changes ...*RowChange) (string, []interface{}) { if len(changes) == 0 { log.L().DPanic("row changes is empty") return "", nil @@ -270,6 +344,23 @@ func GenInsertSQLWithStartTs(tp DMLType, startTs uint64, commitTs uint64, change buf.WriteString(common.QuoteName(col.Name.O)) } + // Add _tidb_origin_ts column if it doesn't exist and commitTs is provided + hasOriginTsColumn := false + for _, col := range first.sourceTableInfo.GetColumns() { + if col.Name.L == "_tidb_origin_ts" { + hasOriginTsColumn = true + break + } + } + + if commitTs > 0 && !hasOriginTsColumn { + if columnNum != 0 { + buf.WriteByte(',') + } + columnNum++ + buf.WriteString(common.QuoteName("_tidb_origin_ts")) + } + buf.WriteString(") VALUES ") holder := valuesHolder(columnNum) for i := range changes { @@ -298,11 +389,30 @@ func GenInsertSQLWithStartTs(tp DMLType, startTs uint64, commitTs uint64, change tableName := first.targetTable.QuoteString() // For all columns, use conditional update logic - buf.WriteString(colName + "=IF((@cond := (IFNULL(" + tableName + "." + colName + ", " + tableName + ".commit_ts) <= VALUES(" + colName + "))),VALUES(" + colName + "), " + tableName + "." + colName + ")") + buf.WriteString(colName + "=IF((@cond := (IFNULL(" + tableName + "." + colName + ", " + tableName + "._tidb_commit_ts) <= VALUES(" + colName + "))),VALUES(" + colName + "), " + tableName + "." + colName + ")") + } + + // Add _tidb_origin_ts to ON DUPLICATE KEY UPDATE if it doesn't exist in source but we're adding it + if commitTs > 0 && !hasOriginTsColumn { + if writtenFirstCol { + buf.WriteByte(',') + } + tableName := first.targetTable.QuoteString() + buf.WriteString(common.QuoteName("_tidb_origin_ts") + "=IF((@cond := (IFNULL(" + tableName + "._tidb_origin_ts, " + tableName + "._tidb_commit_ts) <= VALUES(" + common.QuoteName("_tidb_origin_ts") + "))),VALUES(" + common.QuoteName("_tidb_origin_ts") + "), " + tableName + "._tidb_origin_ts)") + } + } + + args := make([]interface{}, 0, len(changes)*(len(first.sourceTableInfo.GetColumns())-len(skipColIdx)+1)) // +1 for potential _tidb_origin_ts column + + // Find the origin_ts column index + originTsColIndex := -1 + for i, col := range first.sourceTableInfo.GetColumns() { + if col.Name.L == "_tidb_origin_ts" { + originTsColIndex = i + break } } - args := make([]interface{}, 0, len(changes)*(len(first.sourceTableInfo.GetColumns())-len(skipColIdx))) for _, change := range changes { i := 0 // used as index of skipColIdx colIndex := 0 // used to track the actual column index @@ -312,9 +422,9 @@ func GenInsertSQLWithStartTs(tp DMLType, startTs uint64, commitTs uint64, change continue } - // If this is the last column and commitTs is provided, replace NULL with commitTs - if commitTs > 0 && colIndex == len(first.sourceTableInfo.GetColumns())-len(skipColIdx)-1 { - // This is the last column (origin_ts), replace NULL with commitTs + // If this is the origin_ts column and commitTs is provided, replace NULL with commitTs + if commitTs > 0 && j == originTsColIndex { + // This is the origin_ts column, replace NULL with commitTs if val == nil { args = append(args, commitTs) } else { @@ -332,6 +442,11 @@ func GenInsertSQLWithStartTs(tp DMLType, startTs uint64, commitTs uint64, change } colIndex++ } + + // Add _tidb_origin_ts value if the column doesn't exist in source but we're adding it + if commitTs > 0 && originTsColIndex == -1 { + args = append(args, commitTs) + } } return buf.String(), args } From f17301741e28805613b69aafa8c3ebe2f30406c0 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 27 Aug 2025 10:53:49 +0800 Subject: [PATCH 12/15] update --- pkg/sink/sqlmodel/multi_row.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/sink/sqlmodel/multi_row.go b/pkg/sink/sqlmodel/multi_row.go index facb36997..138b4f973 100644 --- a/pkg/sink/sqlmodel/multi_row.go +++ b/pkg/sink/sqlmodel/multi_row.go @@ -294,11 +294,14 @@ func GenInsertSQL(tp DMLType, changes ...*RowChange) (string, []interface{}) { for _, change := range changes { i := 0 // used as index of skipColIdx for j, val := range change.postValues { - if i < len(skipColIdx) && skipColIdx[i] == j { + if i < len(skipColIdx) { + args = append(args, change.postValues[j:]...) + break + } + if skipColIdx[i] == j { i++ continue } - args = append(args, val) } } return buf.String(), args From 81f743e337e3f3c65457fa4b0f14417b03975f9f Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 27 Aug 2025 10:54:45 +0800 Subject: [PATCH 13/15] update --- pkg/sink/sqlmodel/multi_row.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/sink/sqlmodel/multi_row.go b/pkg/sink/sqlmodel/multi_row.go index 138b4f973..71233cc3e 100644 --- a/pkg/sink/sqlmodel/multi_row.go +++ b/pkg/sink/sqlmodel/multi_row.go @@ -294,7 +294,7 @@ func GenInsertSQL(tp DMLType, changes ...*RowChange) (string, []interface{}) { for _, change := range changes { i := 0 // used as index of skipColIdx for j, val := range change.postValues { - if i < len(skipColIdx) { + if i >= len(skipColIdx) { args = append(args, change.postValues[j:]...) break } @@ -302,6 +302,7 @@ func GenInsertSQL(tp DMLType, changes ...*RowChange) (string, []interface{}) { i++ continue } + args = append(args, val) } } return buf.String(), args From 0988c8ffb610e68a307b7c94dadd605317493079 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 27 Aug 2025 11:11:45 +0800 Subject: [PATCH 14/15] fix sql --- pkg/sink/sqlmodel/multi_row.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/sink/sqlmodel/multi_row.go b/pkg/sink/sqlmodel/multi_row.go index 71233cc3e..54bb9d80c 100644 --- a/pkg/sink/sqlmodel/multi_row.go +++ b/pkg/sink/sqlmodel/multi_row.go @@ -392,8 +392,8 @@ func GenInsertSQLWithCommitTs(tp DMLType, startTs uint64, commitTs uint64, chang colName := common.QuoteName(col.Name.O) tableName := first.targetTable.QuoteString() - // For all columns, use conditional update logic - buf.WriteString(colName + "=IF((@cond := (IFNULL(" + tableName + "." + colName + ", " + tableName + "._tidb_commit_ts) <= VALUES(" + colName + "))),VALUES(" + colName + "), " + tableName + "." + colName + ")") + // For all columns, use _tidb_origin_ts as comparison basis + buf.WriteString(colName + "=IF((@cond := (IFNULL(" + tableName + "._tidb_origin_ts, " + tableName + "._tidb_commit_ts) <= VALUES(" + common.QuoteName("_tidb_origin_ts") + "))),VALUES(" + colName + "), " + tableName + "." + colName + ")") } // Add _tidb_origin_ts to ON DUPLICATE KEY UPDATE if it doesn't exist in source but we're adding it From 353d480e5e8e223d2c3e7d40f5bc33f509087012 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 27 Aug 2025 14:08:44 +0800 Subject: [PATCH 15/15] update --- pkg/sink/sqlmodel/multi_row.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pkg/sink/sqlmodel/multi_row.go b/pkg/sink/sqlmodel/multi_row.go index 54bb9d80c..93483c310 100644 --- a/pkg/sink/sqlmodel/multi_row.go +++ b/pkg/sink/sqlmodel/multi_row.go @@ -393,7 +393,7 @@ func GenInsertSQLWithCommitTs(tp DMLType, startTs uint64, commitTs uint64, chang tableName := first.targetTable.QuoteString() // For all columns, use _tidb_origin_ts as comparison basis - buf.WriteString(colName + "=IF((@cond := (IFNULL(" + tableName + "._tidb_origin_ts, " + tableName + "._tidb_commit_ts) <= VALUES(" + common.QuoteName("_tidb_origin_ts") + "))),VALUES(" + colName + "), " + tableName + "." + colName + ")") + buf.WriteString(colName + "=IF(((IFNULL(" + tableName + "._tidb_origin_ts, " + tableName + "._tidb_commit_ts) <= VALUES(" + common.QuoteName("_tidb_origin_ts") + "))),VALUES(" + colName + "), " + tableName + "." + colName + ")") } // Add _tidb_origin_ts to ON DUPLICATE KEY UPDATE if it doesn't exist in source but we're adding it @@ -402,7 +402,7 @@ func GenInsertSQLWithCommitTs(tp DMLType, startTs uint64, commitTs uint64, chang buf.WriteByte(',') } tableName := first.targetTable.QuoteString() - buf.WriteString(common.QuoteName("_tidb_origin_ts") + "=IF((@cond := (IFNULL(" + tableName + "._tidb_origin_ts, " + tableName + "._tidb_commit_ts) <= VALUES(" + common.QuoteName("_tidb_origin_ts") + "))),VALUES(" + common.QuoteName("_tidb_origin_ts") + "), " + tableName + "._tidb_origin_ts)") + buf.WriteString(common.QuoteName("_tidb_origin_ts") + "=IF(((IFNULL(" + tableName + "._tidb_origin_ts, " + tableName + "._tidb_commit_ts) <= VALUES(" + common.QuoteName("_tidb_origin_ts") + "))),VALUES(" + common.QuoteName("_tidb_origin_ts") + "), " + tableName + "._tidb_origin_ts)") } } @@ -434,13 +434,6 @@ func GenInsertSQLWithCommitTs(tp DMLType, startTs uint64, commitTs uint64, chang } else { args = append(args, val) } - } else if startTs > 0 && colIndex == len(first.sourceTableInfo.GetColumns())-len(skipColIdx)-1 { - // This is the last column (start_ts), replace NULL with startTs - if val == nil { - args = append(args, startTs) - } else { - args = append(args, val) - } } else { args = append(args, val) }