diff --git a/api/persistence/v1/executions.pb.go b/api/persistence/v1/executions.pb.go index 2000a2538d..d14e49bf16 100644 --- a/api/persistence/v1/executions.pb.go +++ b/api/persistence/v1/executions.pb.go @@ -2521,8 +2521,10 @@ type ActivityInfo struct { // set to true if reset heartbeat flag was set with an activity reset ResetHeartbeats bool `protobuf:"varint,48,opt,name=reset_heartbeats,json=resetHeartbeats,proto3" json:"reset_heartbeats,omitempty"` StartVersion int64 `protobuf:"varint,50,opt,name=start_version,json=startVersion,proto3" json:"start_version,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // The task queue on which the server will send control tasks to the worker running this activity. + WorkerControlTaskQueue string `protobuf:"bytes,51,opt,name=worker_control_task_queue,json=workerControlTaskQueue,proto3" json:"worker_control_task_queue,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ActivityInfo) Reset() { @@ -2895,6 +2897,13 @@ func (x *ActivityInfo) GetStartVersion() int64 { return 0 } +func (x *ActivityInfo) GetWorkerControlTaskQueue() string { + if x != nil { + return x.WorkerControlTaskQueue + } + return "" +} + type isActivityInfo_BuildIdInfo interface { isActivityInfo_BuildIdInfo() } @@ -4840,7 +4849,7 @@ const file_temporal_server_api_persistence_v1_executions_proto_rawDesc = "" + "\x17NexusInvocationTaskInfo\x12\x18\n" + "\aattempt\x18\x01 \x01(\x05R\aattempt\"4\n" + "\x18NexusCancelationTaskInfo\x12\x18\n" + - "\aattempt\x18\x01 \x01(\x05R\aattempt\"\x9e\x1b\n" + + "\aattempt\x18\x01 \x01(\x05R\aattempt\"\xd9\x1b\n" + "\fActivityInfo\x12\x18\n" + "\aversion\x18\x01 \x01(\x03R\aversion\x127\n" + "\x18scheduled_event_batch_id\x18\x02 \x01(\x03R\x15scheduledEventBatchId\x12A\n" + @@ -4893,7 +4902,8 @@ const file_temporal_server_api_persistence_v1_executions_proto_rawDesc = "" + "pause_info\x18. \x01(\v2:.temporal.server.api.persistence.v1.ActivityInfo.PauseInfoR\tpauseInfo\x12%\n" + "\x0eactivity_reset\x18/ \x01(\bR\ractivityReset\x12)\n" + "\x10reset_heartbeats\x180 \x01(\bR\x0fresetHeartbeats\x12#\n" + - "\rstart_version\x182 \x01(\x03R\fstartVersion\x1ay\n" + + "\rstart_version\x182 \x01(\x03R\fstartVersion\x129\n" + + "\x19worker_control_task_queue\x183 \x01(\tR\x16workerControlTaskQueue\x1ay\n" + "\x16UseWorkflowBuildIdInfo\x12+\n" + "\x12last_used_build_id\x18\x01 \x01(\tR\x0flastUsedBuildId\x122\n" + "\x15last_redirect_counter\x18\x02 \x01(\x03R\x13lastRedirectCounter\x1a\x89\x02\n" + diff --git a/go.mod b/go.mod index 60ad1aa45a..92b96de1dd 100644 --- a/go.mod +++ b/go.mod @@ -173,3 +173,5 @@ require ( modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect ) + +replace go.temporal.io/api => github.com/temporalio/api-go v1.62.2-0.20260217225453-e6a9241288b9 diff --git a/go.sum b/go.sum index 2cd1258aaf..204f76e6e4 100644 --- a/go.sum +++ b/go.sum @@ -310,6 +310,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/temporalio/api-go v1.62.2-0.20260217225453-e6a9241288b9 h1:nv1DjGfsfM/ITt5ehoHodVzCtTxv+mM+sMqM9Zgx0Tc= +github.com/temporalio/api-go v1.62.2-0.20260217225453-e6a9241288b9/go.mod h1:oewVgOWEx67DlpbXkEJl5PlcpDPXjR8h9+raDfl0fpo= github.com/temporalio/ringpop-go v0.0.0-20250130211428-b97329e994f7 h1:lEebX/hZss+TSH3EBwhztnBavJVj7pWGJOH8UgKHS0w= github.com/temporalio/ringpop-go v0.0.0-20250130211428-b97329e994f7/go.mod h1:RE+CHmY+kOZQk47AQaVzwrGmxpflnLgTd6EOK0853j4= github.com/temporalio/sqlparser v0.0.0-20231115171017-f4060bcfa6cb h1:YzHH/U/dN7vMP+glybzcXRTczTrgfdRisNTzAj7La04= @@ -375,8 +377,6 @@ go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= -go.temporal.io/api v1.62.2 h1:jFhIzlqNyJsJZTiCRQmTIMv6OTQ5BZ57z8gbgLGMaoo= -go.temporal.io/api v1.62.2/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM= go.temporal.io/sdk v1.38.0 h1:4Bok5LEdED7YKpsSjIa3dDqram5VOq+ydBf4pyx0Wo4= go.temporal.io/sdk v1.38.0/go.mod h1:a+R2Ej28ObvHoILbHaxMyind7M6D+W0L7edt5UJF4SE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= diff --git a/proto/internal/temporal/server/api/persistence/v1/executions.proto b/proto/internal/temporal/server/api/persistence/v1/executions.proto index 0a025649ae..fc3cf5b220 100644 --- a/proto/internal/temporal/server/api/persistence/v1/executions.proto +++ b/proto/internal/temporal/server/api/persistence/v1/executions.proto @@ -631,6 +631,9 @@ message ActivityInfo { bool reset_heartbeats = 48; int64 start_version = 50; + + // The task queue on which the server will send control tasks to the worker running this activity. + string worker_control_task_queue = 51; } // timer_map column diff --git a/service/history/api/recordactivitytaskstarted/api.go b/service/history/api/recordactivitytaskstarted/api.go index 05c65076b7..bb8fd4c56c 100644 --- a/service/history/api/recordactivitytaskstarted/api.go +++ b/service/history/api/recordactivitytaskstarted/api.go @@ -242,6 +242,7 @@ func recordActivityTaskStarted( if _, err := mutableState.AddActivityTaskStartedEvent( ai, scheduledEventID, requestID, request.PollRequest.GetIdentity(), versioningStamp, pollerDeployment, request.GetBuildIdRedirectInfo(), + request.PollRequest.GetWorkerControlTaskQueue(), ); err != nil { return nil, rejectCodeUndefined, err } diff --git a/service/history/api/respondactivitytaskcompleted/api.go b/service/history/api/respondactivitytaskcompleted/api.go index 69764f9fd3..99675b340a 100644 --- a/service/history/api/respondactivitytaskcompleted/api.go +++ b/service/history/api/respondactivitytaskcompleted/api.go @@ -98,6 +98,7 @@ func Invoke( // TODO (shahab): do we need to do anything with wf redirect in this case or any // other case where an activity starts? nil, + "", // workerControlTaskQueue not available for force complete ) if err != nil { return nil, err diff --git a/service/history/api/respondworkflowtaskcompleted/api.go b/service/history/api/respondworkflowtaskcompleted/api.go index 310dc4c1ae..39e7987745 100644 --- a/service/history/api/respondworkflowtaskcompleted/api.go +++ b/service/history/api/respondworkflowtaskcompleted/api.go @@ -389,6 +389,7 @@ func (handler *WorkflowTaskCompletedHandler) Invoke( workflowTaskHandler := newWorkflowTaskCompletedHandler( request.GetIdentity(), + request.GetWorkerControlTaskQueue(), completedEvent.GetEventId(), // If completedEvent is nil, then GetEventId() returns 0 and this value shouldn't be used in workflowTaskHandler. ms, updateRegistry, diff --git a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go index 0b526b4c7f..ee23a76893 100644 --- a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go +++ b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go @@ -53,6 +53,7 @@ type ( workflowTaskCompletedHandler struct { identity string + workerControlTaskQueue string workflowTaskCompletedID int64 // internal state @@ -104,6 +105,7 @@ type ( func newWorkflowTaskCompletedHandler( identity string, + workerControlTaskQueue string, workflowTaskCompletedID int64, mutableState historyi.MutableState, updateRegistry update.Registry, @@ -123,6 +125,7 @@ func newWorkflowTaskCompletedHandler( ) *workflowTaskCompletedHandler { return &workflowTaskCompletedHandler{ identity: identity, + workerControlTaskQueue: workerControlTaskQueue, workflowTaskCompletedID: workflowTaskCompletedID, // internal state @@ -559,6 +562,7 @@ func (handler *workflowTaskCompletedHandler) handlePostCommandEagerExecuteActivi stamp, nil, nil, + handler.workerControlTaskQueue, ); err != nil { return nil, err } diff --git a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler_test.go b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler_test.go index 0018795df3..c68b8d1478 100644 --- a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler_test.go +++ b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler_test.go @@ -84,6 +84,7 @@ func TestCommandProtocolMessage(t *testing.T) { ) out.handler = newWorkflowTaskCompletedHandler( // 😲 t.Name(), // identity + "", // workerControlTaskQueue 123, // workflowTaskCompletedID out.ms, out.updates, diff --git a/service/history/history_engine_test.go b/service/history/history_engine_test.go index 8b888ca4e1..165782b3ef 100644 --- a/service/history/history_engine_test.go +++ b/service/history/history_engine_test.go @@ -6679,6 +6679,7 @@ func addActivityTaskStartedEvent(ms historyi.MutableState, scheduledEventID int6 nil, nil, nil, + "", ) return event } diff --git a/service/history/interfaces/mutable_state.go b/service/history/interfaces/mutable_state.go index 68aedfd2ac..cd07f9ac69 100644 --- a/service/history/interfaces/mutable_state.go +++ b/service/history/interfaces/mutable_state.go @@ -57,6 +57,7 @@ type ( *commonpb.WorkerVersionStamp, *deploymentpb.Deployment, *taskqueuespb.BuildIdRedirectInfo, + string, // workerControlTaskQueue ) (*historypb.HistoryEvent, error) AddActivityTaskTimedOutEvent(int64, int64, *failurepb.Failure, enumspb.RetryState) (*historypb.HistoryEvent, error) AddChildWorkflowExecutionCanceledEvent(int64, *commonpb.WorkflowExecution, *historypb.WorkflowExecutionCanceledEventAttributes) (*historypb.HistoryEvent, error) diff --git a/service/history/interfaces/mutable_state_mock.go b/service/history/interfaces/mutable_state_mock.go index 103a53840e..9b89e38359 100644 --- a/service/history/interfaces/mutable_state_mock.go +++ b/service/history/interfaces/mutable_state_mock.go @@ -148,18 +148,18 @@ func (mr *MockMutableStateMockRecorder) AddActivityTaskScheduledEvent(arg0, arg1 } // AddActivityTaskStartedEvent mocks base method. -func (m *MockMutableState) AddActivityTaskStartedEvent(arg0 *persistence.ActivityInfo, arg1 int64, arg2, arg3 string, arg4 *common.WorkerVersionStamp, arg5 *deployment.Deployment, arg6 *taskqueue0.BuildIdRedirectInfo) (*history.HistoryEvent, error) { +func (m *MockMutableState) AddActivityTaskStartedEvent(arg0 *persistence.ActivityInfo, arg1 int64, arg2, arg3 string, arg4 *common.WorkerVersionStamp, arg5 *deployment.Deployment, arg6 *taskqueue0.BuildIdRedirectInfo, arg7 string) (*history.HistoryEvent, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddActivityTaskStartedEvent", arg0, arg1, arg2, arg3, arg4, arg5, arg6) + ret := m.ctrl.Call(m, "AddActivityTaskStartedEvent", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) ret0, _ := ret[0].(*history.HistoryEvent) ret1, _ := ret[1].(error) return ret0, ret1 } // AddActivityTaskStartedEvent indicates an expected call of AddActivityTaskStartedEvent. -func (mr *MockMutableStateMockRecorder) AddActivityTaskStartedEvent(arg0, arg1, arg2, arg3, arg4, arg5, arg6 any) *gomock.Call { +func (mr *MockMutableStateMockRecorder) AddActivityTaskStartedEvent(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddActivityTaskStartedEvent", reflect.TypeOf((*MockMutableState)(nil).AddActivityTaskStartedEvent), arg0, arg1, arg2, arg3, arg4, arg5, arg6) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddActivityTaskStartedEvent", reflect.TypeOf((*MockMutableState)(nil).AddActivityTaskStartedEvent), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) } // AddActivityTaskTimedOutEvent mocks base method. diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index e0b21c8342..0b92eb7ddc 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -4057,6 +4057,7 @@ func (ms *MutableStateImpl) AddActivityTaskStartedEvent( versioningStamp *commonpb.WorkerVersionStamp, deployment *deploymentpb.Deployment, redirectInfo *taskqueuespb.BuildIdRedirectInfo, + workerControlTaskQueue string, ) (*historypb.HistoryEvent, error) { opTag := tag.WorkflowActionActivityTaskStarted err := ms.checkMutability(opTag) @@ -4085,6 +4086,8 @@ func (ms *MutableStateImpl) AddActivityTaskStartedEvent( ai.LastDeploymentVersion = worker_versioning.ExternalWorkerDeploymentVersionFromDeployment(deployment) } + ai.WorkerControlTaskQueue = workerControlTaskQueue + if !ai.HasRetryPolicy { event := ms.hBuilder.AddActivityTaskStartedEvent( scheduledEventID, diff --git a/service/history/workflow/mutable_state_impl_restart_activity_test.go b/service/history/workflow/mutable_state_impl_restart_activity_test.go index 9337a98fff..3cb91b7fbd 100644 --- a/service/history/workflow/mutable_state_impl_restart_activity_test.go +++ b/service/history/workflow/mutable_state_impl_restart_activity_test.go @@ -419,6 +419,7 @@ func (s *retryActivitySuite) makeActivityAndPutIntoFailingState() *persistencesp nil, nil, nil, + "", ) s.NoError(err) diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index 2b302fd7da..62e48a43ed 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -2878,6 +2878,7 @@ func (s *mutableStateSuite) TestRetryActivity_TruncateRetryableFailure() { nil, nil, nil, + "", ) s.NoError(err) @@ -2943,6 +2944,7 @@ func (s *mutableStateSuite) TestRetryActivity_PausedIncrementsStamp() { nil, nil, nil, + "", ) s.NoError(err) @@ -6150,3 +6152,76 @@ func (s *mutableStateSuite) TestSetContextMetadata() { s.True(ok) s.Equal(taskQueue, tq) } + +func (s *mutableStateSuite) TestAddActivityTaskStartedEventStoresWorkerControlTaskQueue() { + s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() + + // Setup workflow execution + _, err := s.mutableState.AddWorkflowExecutionStartedEvent( + &commonpb.WorkflowExecution{WorkflowId: tests.WorkflowID, RunId: tests.RunID}, + &historyservice.StartWorkflowExecutionRequest{ + NamespaceId: tests.NamespaceID.String(), + StartRequest: &workflowservice.StartWorkflowExecutionRequest{ + WorkflowType: &commonpb.WorkflowType{Name: "workflow-type"}, + TaskQueue: &taskqueuepb.TaskQueue{Name: "task-queue"}, + WorkflowRunTimeout: durationpb.New(200 * time.Second), + WorkflowTaskTimeout: durationpb.New(1 * time.Second), + }, + }, + ) + s.NoError(err) + + di, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) + s.NoError(err) + _, _, err = s.mutableState.AddWorkflowTaskStartedEvent( + di.ScheduledEventID, + di.RequestID, + di.TaskQueue, + "identity", + nil, + nil, + nil, + false, + nil, + ) + s.NoError(err) + _, err = s.mutableState.AddWorkflowTaskCompletedEvent( + di, + &workflowservice.RespondWorkflowTaskCompletedRequest{Identity: "identity"}, + workflowTaskCompletionLimits, + ) + s.NoError(err) + + // Schedule activity + workflowTaskCompletedEventID := int64(4) + _, activityInfo, err := s.mutableState.AddActivityTaskScheduledEvent( + workflowTaskCompletedEventID, + &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: "test-activity-1", + ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + }, + false, + ) + s.NoError(err) + s.Empty(activityInfo.WorkerControlTaskQueue, "WorkerControlTaskQueue should be empty before activity starts") + + // Start activity with workerControlTaskQueue + expectedWorkerControlTaskQueue := "test-control-queue" + _, err = s.mutableState.AddActivityTaskStartedEvent( + activityInfo, + activityInfo.ScheduledEventId, + uuid.NewString(), + "worker-identity", + nil, + nil, + nil, + expectedWorkerControlTaskQueue, + ) + s.NoError(err) + + // Verify workerControlTaskQueue is stored + updatedActivityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) + s.True(ok) + s.Equal(expectedWorkerControlTaskQueue, updatedActivityInfo.WorkerControlTaskQueue) +} diff --git a/service/matching/forwarder.go b/service/matching/forwarder.go index 018aeffca1..6b893db2aa 100644 --- a/service/matching/forwarder.go +++ b/service/matching/forwarder.go @@ -249,6 +249,7 @@ func (fwdr *Forwarder) ForwardPoll(ctx context.Context, pollMetadata *pollMetada WorkerVersionCapabilities: pollMetadata.workerVersionCapabilities, DeploymentOptions: pollMetadata.deploymentOptions, WorkerInstanceKey: pollMetadata.workerInstanceKey, + WorkerControlTaskQueue: pollMetadata.workerControlTaskQueue, }, ForwardedSource: fwdr.partition.RpcName(), Conditions: pollMetadata.conditions, @@ -273,6 +274,7 @@ func (fwdr *Forwarder) ForwardPoll(ctx context.Context, pollMetadata *pollMetada WorkerVersionCapabilities: pollMetadata.workerVersionCapabilities, DeploymentOptions: pollMetadata.deploymentOptions, WorkerInstanceKey: pollMetadata.workerInstanceKey, + WorkerControlTaskQueue: pollMetadata.workerControlTaskQueue, }, ForwardedSource: fwdr.partition.RpcName(), Conditions: pollMetadata.conditions, diff --git a/service/matching/matching_engine.go b/service/matching/matching_engine.go index b09c10338c..0195923910 100644 --- a/service/matching/matching_engine.go +++ b/service/matching/matching_engine.go @@ -95,6 +95,7 @@ type ( forwardedFrom string localPollStartTime time.Time workerInstanceKey string + workerControlTaskQueue string } userDataUpdate struct { @@ -679,6 +680,7 @@ pollLoop: forwardedFrom: req.ForwardedSource, conditions: req.Conditions, workerInstanceKey: request.WorkerInstanceKey, + workerControlTaskQueue: request.WorkerControlTaskQueue, } task, versionSetUsed, err := e.pollTask(pollerCtx, partition, pollMetadata) if err != nil { @@ -984,6 +986,7 @@ pollLoop: forwardedFrom: req.ForwardedSource, conditions: req.Conditions, workerInstanceKey: request.WorkerInstanceKey, + workerControlTaskQueue: request.WorkerControlTaskQueue, } task, versionSetUsed, err := e.pollTask(pollerCtx, partition, pollMetadata) if err != nil { diff --git a/service/matching/pri_forwarder.go b/service/matching/pri_forwarder.go index ca1e1f6f5c..dd2ab8c8c3 100644 --- a/service/matching/pri_forwarder.go +++ b/service/matching/pri_forwarder.go @@ -219,6 +219,7 @@ func ForwardPollWithTarget( WorkerVersionCapabilities: pollMetadata.workerVersionCapabilities, DeploymentOptions: pollMetadata.deploymentOptions, WorkerInstanceKey: pollMetadata.workerInstanceKey, + WorkerControlTaskQueue: pollMetadata.workerControlTaskQueue, }, ForwardedSource: source.RpcName(), Conditions: pollMetadata.conditions, @@ -243,6 +244,7 @@ func ForwardPollWithTarget( WorkerVersionCapabilities: pollMetadata.workerVersionCapabilities, DeploymentOptions: pollMetadata.deploymentOptions, WorkerInstanceKey: pollMetadata.workerInstanceKey, + WorkerControlTaskQueue: pollMetadata.workerControlTaskQueue, }, ForwardedSource: source.RpcName(), Conditions: pollMetadata.conditions,