From 72f0cab19ac072852639b69a76abad8378035049 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Tue, 13 Jan 2026 14:50:50 -0800 Subject: [PATCH 01/17] Nexus caller timeouts --- .../internal/sync/SyncWorkflowContext.java | 4 + .../workflow/NexusOperationOptions.java | 87 +++++++++++- temporal-serviceclient/src/main/proto | 2 +- .../internal/testservice/StateMachines.java | 48 ++++++- .../testservice/TestWorkflowMutableState.java | 6 + .../TestWorkflowMutableStateImpl.java | 43 ++++++ .../testservice/TestWorkflowService.java | 69 ++++++++- .../functional/NexusWorkflowTest.java | 134 ++++++++++++++++++ .../functional/WorkflowIdReusePolicyTest.java | 2 + 9 files changed, 386 insertions(+), 9 deletions(-) diff --git a/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java b/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java index 977d9754e6..ef59e24984 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java @@ -799,6 +799,10 @@ public ExecuteNexusOperationOutput executeNexusOperation( input.getHeaders().forEach((k, v) -> attributes.putNexusHeader(k.toLowerCase(), v)); attributes.setScheduleToCloseTimeout( ProtobufTimeUtils.toProtoDuration(input.getOptions().getScheduleToCloseTimeout())); + attributes.setScheduleToStartTimeout( + ProtobufTimeUtils.toProtoDuration(input.getOptions().getScheduleToStartTimeout())); + attributes.setStartToCloseTimeout( + ProtobufTimeUtils.toProtoDuration(input.getOptions().getStartToCloseTimeout())); @Nullable UserMetadata userMetadata = diff --git a/temporal-sdk/src/main/java/io/temporal/workflow/NexusOperationOptions.java b/temporal-sdk/src/main/java/io/temporal/workflow/NexusOperationOptions.java index b22930eba2..0952f6853a 100644 --- a/temporal-sdk/src/main/java/io/temporal/workflow/NexusOperationOptions.java +++ b/temporal-sdk/src/main/java/io/temporal/workflow/NexusOperationOptions.java @@ -31,6 +31,8 @@ public static NexusOperationOptions getDefaultInstance() { public static final class Builder { private Duration scheduleToCloseTimeout; + private Duration scheduleToStartTimeout; + private Duration startToCloseTimeout; private NexusOperationCancellationType cancellationType; private String summary; @@ -46,6 +48,45 @@ public NexusOperationOptions.Builder setScheduleToCloseTimeout( return this; } + /** + * Sets the schedule to start timeout for the Nexus operation. + * + *

Maximum time to wait for the operation to be started (or completed if synchronous) by the + * handler. If the operation is not started within this timeout, it will fail with + * TIMEOUT_TYPE_SCHEDULE_TO_START. + * + *

Requires Temporal Server 1.31.0 or later. + * + * @param scheduleToStartTimeout the schedule to start timeout for the Nexus operation + * @return this + */ + @Experimental + public NexusOperationOptions.Builder setScheduleToStartTimeout( + Duration scheduleToStartTimeout) { + this.scheduleToStartTimeout = scheduleToStartTimeout; + return this; + } + + /** + * Sets the start to close timeout for the Nexus operation. + * + *

Maximum time to wait for an asynchronous operation to complete after it has been started. + * If the operation does not complete within this timeout after starting, it will fail with + * TIMEOUT_TYPE_START_TO_CLOSE. + * + *

Only applies to asynchronous operations. Synchronous operations ignore this timeout. + * + *

Requires Temporal Server 1.31.0 or later. + * + * @param startToCloseTimeout the start to close timeout for the Nexus operation + * @return this + */ + @Experimental + public NexusOperationOptions.Builder setStartToCloseTimeout(Duration startToCloseTimeout) { + this.startToCloseTimeout = startToCloseTimeout; + return this; + } + /** * Sets the cancellation type for the Nexus operation. Defaults to WAIT_COMPLETED. * @@ -78,12 +119,19 @@ private Builder(NexusOperationOptions options) { return; } this.scheduleToCloseTimeout = options.getScheduleToCloseTimeout(); + this.scheduleToStartTimeout = options.getScheduleToStartTimeout(); + this.startToCloseTimeout = options.getStartToCloseTimeout(); this.cancellationType = options.getCancellationType(); this.summary = options.getSummary(); } public NexusOperationOptions build() { - return new NexusOperationOptions(scheduleToCloseTimeout, cancellationType, summary); + return new NexusOperationOptions( + scheduleToCloseTimeout, + scheduleToStartTimeout, + startToCloseTimeout, + cancellationType, + summary); } public NexusOperationOptions.Builder mergeNexusOperationOptions( @@ -95,6 +143,14 @@ public NexusOperationOptions.Builder mergeNexusOperationOptions( (override.scheduleToCloseTimeout == null) ? this.scheduleToCloseTimeout : override.scheduleToCloseTimeout; + this.scheduleToStartTimeout = + (override.scheduleToStartTimeout == null) + ? this.scheduleToStartTimeout + : override.scheduleToStartTimeout; + this.startToCloseTimeout = + (override.startToCloseTimeout == null) + ? this.startToCloseTimeout + : override.startToCloseTimeout; this.cancellationType = (override.cancellationType == null) ? this.cancellationType : override.cancellationType; this.summary = (override.summary == null) ? this.summary : override.summary; @@ -104,9 +160,13 @@ public NexusOperationOptions.Builder mergeNexusOperationOptions( private NexusOperationOptions( Duration scheduleToCloseTimeout, + Duration scheduleToStartTimeout, + Duration startToCloseTimeout, NexusOperationCancellationType cancellationType, String summary) { this.scheduleToCloseTimeout = scheduleToCloseTimeout; + this.scheduleToStartTimeout = scheduleToStartTimeout; + this.startToCloseTimeout = startToCloseTimeout; this.cancellationType = cancellationType; this.summary = summary; } @@ -116,6 +176,8 @@ public NexusOperationOptions.Builder toBuilder() { } private final Duration scheduleToCloseTimeout; + private final Duration scheduleToStartTimeout; + private final Duration startToCloseTimeout; private final NexusOperationCancellationType cancellationType; private final String summary; @@ -123,6 +185,16 @@ public Duration getScheduleToCloseTimeout() { return scheduleToCloseTimeout; } + @Experimental + public Duration getScheduleToStartTimeout() { + return scheduleToStartTimeout; + } + + @Experimental + public Duration getStartToCloseTimeout() { + return startToCloseTimeout; + } + public NexusOperationCancellationType getCancellationType() { return cancellationType; } @@ -138,13 +210,20 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; NexusOperationOptions that = (NexusOperationOptions) o; return Objects.equals(scheduleToCloseTimeout, that.scheduleToCloseTimeout) + && Objects.equals(scheduleToStartTimeout, that.scheduleToStartTimeout) + && Objects.equals(startToCloseTimeout, that.startToCloseTimeout) && Objects.equals(cancellationType, that.cancellationType) && Objects.equals(summary, that.summary); } @Override public int hashCode() { - return Objects.hash(scheduleToCloseTimeout, cancellationType, summary); + return Objects.hash( + scheduleToCloseTimeout, + scheduleToStartTimeout, + startToCloseTimeout, + cancellationType, + summary); } @Override @@ -152,6 +231,10 @@ public String toString() { return "NexusOperationOptions{" + "scheduleToCloseTimeout=" + scheduleToCloseTimeout + + ", scheduleToStartTimeout=" + + scheduleToStartTimeout + + ", startToCloseTimeout=" + + startToCloseTimeout + ", cancellationType=" + cancellationType + ", summary='" diff --git a/temporal-serviceclient/src/main/proto b/temporal-serviceclient/src/main/proto index 1ae5b673d6..44dec06c67 160000 --- a/temporal-serviceclient/src/main/proto +++ b/temporal-serviceclient/src/main/proto @@ -1 +1 @@ -Subproject commit 1ae5b673d66b0a94f6131c3eb06bc7173ae2c326 +Subproject commit 44dec06c674f05b03c7855e1026a326880af2000 diff --git a/temporal-test-server/src/main/java/io/temporal/internal/testservice/StateMachines.java b/temporal-test-server/src/main/java/io/temporal/internal/testservice/StateMachines.java index ae20d22858..82761109b8 100644 --- a/temporal-test-server/src/main/java/io/temporal/internal/testservice/StateMachines.java +++ b/temporal-test-server/src/main/java/io/temporal/internal/testservice/StateMachines.java @@ -660,6 +660,29 @@ private static void scheduleNexusOperation( : Timestamp.getDefaultInstance(); TestServiceRetryState retryState = new TestServiceRetryState(data.retryPolicy, expirationTime); + // Trim secondary timeouts to the primary timeout (scheduleToClose). + java.time.Duration scheduleToCloseTimeout = + ProtobufTimeUtils.toJavaDuration(attr.getScheduleToCloseTimeout()); + java.time.Duration scheduleToStartTimeout = + ProtobufTimeUtils.toJavaDuration(attr.getScheduleToStartTimeout()); + java.time.Duration startToCloseTimeout = + ProtobufTimeUtils.toJavaDuration(attr.getStartToCloseTimeout()); + + com.google.protobuf.Duration cappedScheduleToStartTimeout = attr.getScheduleToStartTimeout(); + com.google.protobuf.Duration cappedStartToCloseTimeout = attr.getStartToCloseTimeout(); + + if (!scheduleToCloseTimeout.isZero() + && !scheduleToStartTimeout.isZero() + && scheduleToStartTimeout.compareTo(scheduleToCloseTimeout) > 0) { + cappedScheduleToStartTimeout = attr.getScheduleToCloseTimeout(); + } + + if (!scheduleToCloseTimeout.isZero() + && !startToCloseTimeout.isZero() + && startToCloseTimeout.compareTo(scheduleToCloseTimeout) > 0) { + cappedStartToCloseTimeout = attr.getScheduleToCloseTimeout(); + } + NexusOperationScheduledEventAttributes.Builder a = NexusOperationScheduledEventAttributes.newBuilder() .setEndpoint(attr.getEndpoint()) @@ -668,6 +691,8 @@ private static void scheduleNexusOperation( .setOperation(attr.getOperation()) .setInput(attr.getInput()) .setScheduleToCloseTimeout(attr.getScheduleToCloseTimeout()) + .setScheduleToStartTimeout(cappedScheduleToStartTimeout) + .setStartToCloseTimeout(cappedStartToCloseTimeout) .putAllNexusHeader(attr.getNexusHeaderMap()) .setRequestId(UUID.randomUUID().toString()) .setWorkflowTaskCompletedEventId(workflowTaskCompletedId); @@ -704,9 +729,6 @@ private static void scheduleNexusOperation( io.temporal.api.nexus.v1.Request.newBuilder() .setScheduledTime(ctx.currentTime()) .putAllHeader(attr.getNexusHeaderMap()) - .putHeader( - io.nexusrpc.Header.OPERATION_TIMEOUT.toLowerCase(), - attr.getScheduleToCloseTimeout().toString()) .setStartOperation( StartOperationRequest.newBuilder() .setService(attr.getService()) @@ -778,11 +800,27 @@ private static void completeNexusOperation( private static void timeoutNexusOperation( RequestContext ctx, NexusOperationData data, TimeoutType timeoutType, long notUsed) { - if (timeoutType != TimeoutType.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE) { + if (timeoutType != TimeoutType.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE + && timeoutType != TimeoutType.TIMEOUT_TYPE_SCHEDULE_TO_START + && timeoutType != TimeoutType.TIMEOUT_TYPE_START_TO_CLOSE) { throw new IllegalArgumentException( "Timeout type not supported for Nexus operations: " + timeoutType); } + String timeoutMessage; + switch (timeoutType) { + case TIMEOUT_TYPE_SCHEDULE_TO_START: + timeoutMessage = "operation timed out before starting"; + break; + case TIMEOUT_TYPE_START_TO_CLOSE: + timeoutMessage = "operation timed out after starting"; + break; + case TIMEOUT_TYPE_SCHEDULE_TO_CLOSE: + default: + timeoutMessage = "operation timed out"; + break; + } + Failure failure = Failure.newBuilder() .setMessage("nexus operation completed unsuccessfully") @@ -795,7 +833,7 @@ private static void timeoutNexusOperation( .setScheduledEventId(data.scheduledEventId)) .setCause( Failure.newBuilder() - .setMessage("operation timed out") + .setMessage(timeoutMessage) .setTimeoutFailureInfo( TimeoutFailureInfo.newBuilder().setTimeoutType(timeoutType))) .build(); diff --git a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowMutableState.java b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowMutableState.java index 7c6343a1a6..a1fab0a44f 100644 --- a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowMutableState.java +++ b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowMutableState.java @@ -116,6 +116,12 @@ void completeAsyncNexusOperation( boolean validateOperationTaskToken(NexusTaskToken tt); + @Nullable + NexusOperationScheduledEventAttributes getNexusOperationScheduledEventAttributes( + long scheduledEventId); + + boolean isNexusOperationStarted(long scheduledEventId); + QueryWorkflowResponse query(QueryWorkflowRequest queryRequest, long deadline); TestWorkflowMutableStateImpl.UpdateHandle updateWorkflowExecution( diff --git a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowMutableStateImpl.java b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowMutableStateImpl.java index c667c47a22..9b238aa3b3 100644 --- a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowMutableStateImpl.java +++ b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowMutableStateImpl.java @@ -852,6 +852,18 @@ private void processScheduleNexusOperation( operation.getData().getAttempt()), "NexusOperation ScheduleToCloseTimeout"); } + if (attr.hasScheduleToStartTimeout() + && Durations.toMillis(attr.getScheduleToStartTimeout()) > 0) { + // ScheduleToStartTimeout is the time from schedule to start (or completion if synchronous) + ctx.addTimer( + ProtobufTimeUtils.toJavaDuration(attr.getScheduleToStartTimeout()), + () -> + timeoutNexusOperation( + scheduleEventId, + TimeoutType.TIMEOUT_TYPE_SCHEDULE_TO_START, + operation.getData().getAttempt()), + "NexusOperation ScheduleToStartTimeout"); + } ctx.lockTimer("processScheduleNexusOperation"); } @@ -2309,6 +2321,23 @@ public void startNexusOperation( StateMachine operation = getPendingNexusOperation(scheduledEventId); operation.action(StateMachines.Action.START, ctx, resp, 0); operation.getData().identity = clientIdentity; + + // Add start-to-close timeout timer if configured + NexusOperationScheduledEventAttributes scheduledEvent = + operation.getData().scheduledEvent; + if (scheduledEvent.hasStartToCloseTimeout() + && Durations.toMillis(scheduledEvent.getStartToCloseTimeout()) > 0) { + // StartToCloseTimeout measures from when the operation started to when it completes + ctx.addTimer( + ProtobufTimeUtils.toJavaDuration(scheduledEvent.getStartToCloseTimeout()), + () -> + timeoutNexusOperation( + scheduledEventId, + TimeoutType.TIMEOUT_TYPE_START_TO_CLOSE, + operation.getData().getAttempt()), + "NexusOperation StartToCloseTimeout"); + } + scheduleWorkflowTask(ctx); }); } @@ -3691,6 +3720,20 @@ public boolean validateOperationTaskToken(NexusTaskToken tt) { return true; } + @Override + public NexusOperationScheduledEventAttributes getNexusOperationScheduledEventAttributes( + long scheduledEventId) { + StateMachine operation = getPendingNexusOperation(scheduledEventId); + return operation.getData().scheduledEvent; + } + + @Override + public boolean isNexusOperationStarted(long scheduledEventId) { + StateMachine operation = getPendingNexusOperation(scheduledEventId); + // Operation is considered started if it has an operation token + return !operation.getData().operationToken.isEmpty(); + } + private boolean isTerminalState(State workflowState) { return workflowState == State.COMPLETED || workflowState == State.TIMED_OUT diff --git a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java index 469fde42df..049689f146 100644 --- a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java +++ b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java @@ -26,6 +26,7 @@ import io.temporal.api.failure.v1.*; import io.temporal.api.failure.v1.Failure; import io.temporal.api.history.v1.HistoryEvent; +import io.temporal.api.history.v1.NexusOperationScheduledEventAttributes; import io.temporal.api.history.v1.WorkflowExecutionContinuedAsNewEventAttributes; import io.temporal.api.namespace.v1.NamespaceInfo; import io.temporal.api.nexus.v1.*; @@ -285,6 +286,8 @@ private TestWorkflowMutableState getMutableState( return getMutableState(executionId, failNotExists); } + @SuppressWarnings( + "deprecation") // Backwards compatibility for WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING @Override public void startWorkflowExecution( StartWorkflowExecutionRequest request, @@ -310,6 +313,8 @@ public void startWorkflowExecution( } } + @SuppressWarnings( + "deprecation") // Backwards compatibility for WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING StartWorkflowExecutionResponse startWorkflowExecutionImpl( StartWorkflowExecutionRequest startRequest, Duration backoffStartInterval, @@ -325,7 +330,8 @@ StartWorkflowExecutionResponse startWorkflowExecutionImpl( validateWorkflowIdReusePolicy(reusePolicy, conflictPolicy); validateOnConflictOptions(startRequest); - // Backwards compatibility: WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING is deprecated + // Backwards compatibility: WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING + // is deprecated if (reusePolicy == WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING) { conflictPolicy = WorkflowIdConflictPolicy.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING; reusePolicy = WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE; @@ -475,6 +481,8 @@ private StartWorkflowExecutionResponse throwDuplicatedWorkflow( WorkflowExecutionAlreadyStartedFailure.getDescriptor()); } + @SuppressWarnings( + "deprecation") // Backwards compatibility for WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING private void validateWorkflowIdReusePolicy( WorkflowIdReusePolicy reusePolicy, WorkflowIdConflictPolicy conflictPolicy) { if (conflictPolicy != WorkflowIdConflictPolicy.WORKFLOW_ID_CONFLICT_POLICY_UNSPECIFIED @@ -955,6 +963,65 @@ public void pollNexusTaskQueue( task.getTask() .getRequestBuilder() .putHeader(Header.REQUEST_TIMEOUT.toLowerCase(), taskTimeout + "s"); + + // Calculate and set OPERATION_TIMEOUT header if not already present and operation has + // timeouts + if (req.hasStartOperation() + && !req.getHeaderMap().containsKey(Header.OPERATION_TIMEOUT.toLowerCase())) { + NexusTaskToken token = NexusTaskToken.fromBytes(task.getTask().getTaskToken()); + TestWorkflowMutableState mutableState = + getMutableState(token.getOperationRef().getExecutionId()); + long scheduledEventId = token.getOperationRef().getScheduledEventId(); + NexusOperationScheduledEventAttributes scheduledEvent = + mutableState.getNexusOperationScheduledEventAttributes(scheduledEventId); + boolean isStarted = mutableState.isNexusOperationStarted(scheduledEventId); + + Timestamp scheduledTime = req.getScheduledTime(); + Timestamp currentTime = store.currentTime(); + long elapsedSeconds = Timestamps.between(scheduledTime, currentTime).getSeconds(); + long elapsedMillis = elapsedSeconds * 1000; + + // Calculate minimum of all applicable timeouts + Long remainingMillis = null; + + if (!isStarted && scheduledEvent.hasScheduleToStartTimeout()) { + long scheduleToStartMillis = + com.google.protobuf.util.Durations.toMillis( + scheduledEvent.getScheduleToStartTimeout()); + if (scheduleToStartMillis > 0) { + long remaining = scheduleToStartMillis - elapsedMillis; + remainingMillis = + (remainingMillis == null) ? remaining : Math.min(remainingMillis, remaining); + } + } + + if (scheduledEvent.hasStartToCloseTimeout()) { + long startToCloseMillis = + com.google.protobuf.util.Durations.toMillis(scheduledEvent.getStartToCloseTimeout()); + if (startToCloseMillis > 0) { + long remaining = startToCloseMillis - elapsedMillis; + remainingMillis = + (remainingMillis == null) ? remaining : Math.min(remainingMillis, remaining); + } + } + + if (scheduledEvent.hasScheduleToCloseTimeout()) { + long scheduleToCloseMillis = + com.google.protobuf.util.Durations.toMillis( + scheduledEvent.getScheduleToCloseTimeout()); + if (scheduleToCloseMillis > 0) { + long remaining = scheduleToCloseMillis - elapsedMillis; + remainingMillis = + (remainingMillis == null) ? remaining : Math.min(remainingMillis, remaining); + } + } + + if (remainingMillis != null && remainingMillis > 0) { + req.putHeader( + Header.OPERATION_TIMEOUT.toLowerCase(), Long.toString(remainingMillis) + "ms"); + } + } + PollNexusTaskQueueResponse.Builder resp = task.getTask().setRequest(req); responseObserver.onNext(resp.build()); diff --git a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java index 3553ee71e6..f15025a35d 100644 --- a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java +++ b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java @@ -713,6 +713,140 @@ public void testNexusOperationTimeout_AfterCancel() { } } + @Test + public void testNexusOperationScheduleToStartTimeout() { + WorkflowStub stub = newWorkflowStub("TestNexusOperationScheduleToStartTimeoutWorkflow"); + WorkflowExecution execution = stub.start(); + + // Get first WFT and respond with ScheduleNexusOperation command with schedule-to-start timeout + PollWorkflowTaskQueueResponse pollResp = pollWorkflowTask(); + completeWorkflowTask( + pollResp.getTaskToken(), + newScheduleOperationCommand( + defaultScheduleOperationAttributes() + .setScheduleToStartTimeout(Durations.fromSeconds(1)) + .setScheduleToCloseTimeout(Durations.fromSeconds(30)))); + testWorkflowRule.assertHistoryEvent( + execution.getWorkflowId(), EventType.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED); + + try { + // Poll for Nexus task but do not complete it - let it time out before starting + PollNexusTaskQueueResponse nexusPollResp = pollNexusTask().get(); + Assert.assertTrue(nexusPollResp.getRequest().hasStartOperation()); + + // Verify OPERATION_TIMEOUT header is set and valid + String operationTimeoutHeader = + nexusPollResp.getRequest().getHeaderMap().get("operation-timeout"); + Assert.assertNotNull("OPERATION_TIMEOUT header should be set", operationTimeoutHeader); + Assert.assertTrue( + "OPERATION_TIMEOUT should end with 'ms'", operationTimeoutHeader.endsWith("ms")); + long operationTimeoutMs = + Long.parseLong(operationTimeoutHeader.substring(0, operationTimeoutHeader.length() - 2)); + // Should be <= schedule-to-start timeout (1 second = 1000ms) + Assert.assertTrue( + "OPERATION_TIMEOUT should be <= schedule-to-start timeout", operationTimeoutMs <= 1000); + Assert.assertTrue("OPERATION_TIMEOUT should be positive", operationTimeoutMs > 0); + + // Sleep longer than schedule-to-start timeout to trigger the timeout + Thread.sleep(2000); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + + // Poll to wait for new task after operation times out + pollResp = pollWorkflowTask(); + completeWorkflow(pollResp.getTaskToken()); + + List events = + testWorkflowRule.getHistoryEvents( + execution.getWorkflowId(), EventType.EVENT_TYPE_NEXUS_OPERATION_TIMED_OUT); + Assert.assertEquals(1, events.size()); + io.temporal.api.failure.v1.Failure failure = + events.get(0).getNexusOperationTimedOutEventAttributes().getFailure(); + assertOperationFailureInfo(failure.getNexusOperationExecutionFailureInfo()); + Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); + io.temporal.api.failure.v1.Failure cause = failure.getCause(); + Assert.assertEquals("operation timed out before starting", cause.getMessage()); + Assert.assertTrue(cause.hasTimeoutFailureInfo()); + Assert.assertEquals( + TimeoutType.TIMEOUT_TYPE_SCHEDULE_TO_START, cause.getTimeoutFailureInfo().getTimeoutType()); + } + + @Test + public void testNexusOperationStartToCloseTimeout() { + String operationId = UUID.randomUUID().toString(); + CompletableFuture nexusPoller = + pollNexusTask() + .thenCompose( + task -> { + // Verify OPERATION_TIMEOUT header is set and valid + String operationTimeoutHeader = + task.getRequest().getHeaderMap().get("operation-timeout"); + Assert.assertNotNull( + "OPERATION_TIMEOUT header should be set", operationTimeoutHeader); + Assert.assertTrue( + "OPERATION_TIMEOUT should end with 'ms'", + operationTimeoutHeader.endsWith("ms")); + long operationTimeoutMs = + Long.parseLong( + operationTimeoutHeader.substring(0, operationTimeoutHeader.length() - 2)); + // Should be <= start-to-close timeout (1 second = 1000ms) + Assert.assertTrue( + "OPERATION_TIMEOUT should be <= start-to-close timeout", + operationTimeoutMs <= 1000); + Assert.assertTrue("OPERATION_TIMEOUT should be positive", operationTimeoutMs > 0); + + return completeNexusTask(task, operationId); + }); + + try { + WorkflowStub stub = newWorkflowStub("TestNexusOperationStartToCloseTimeoutWorkflow"); + WorkflowExecution execution = stub.start(); + + // Get first WFT and respond with ScheduleNexusOperation command with start-to-close timeout + PollWorkflowTaskQueueResponse pollResp = pollWorkflowTask(); + completeWorkflowTask( + pollResp.getTaskToken(), + newScheduleOperationCommand( + defaultScheduleOperationAttributes() + .setStartToCloseTimeout(Durations.fromSeconds(1)) + .setScheduleToCloseTimeout(Durations.fromSeconds(30)))); + testWorkflowRule.assertHistoryEvent( + execution.getWorkflowId(), EventType.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED); + + // Wait for operation to be started + nexusPoller.get(); + + // Poll and verify started event is recorded + pollResp = pollWorkflowTask(); + testWorkflowRule.assertHistoryEvent( + execution.getWorkflowId(), EventType.EVENT_TYPE_NEXUS_OPERATION_STARTED); + completeWorkflowTask(pollResp.getTaskToken()); + + // Poll to wait for new task after operation times out (start-to-close timeout) + pollResp = pollWorkflowTask(); + completeWorkflow(pollResp.getTaskToken()); + + List events = + testWorkflowRule.getHistoryEvents( + execution.getWorkflowId(), EventType.EVENT_TYPE_NEXUS_OPERATION_TIMED_OUT); + Assert.assertEquals(1, events.size()); + io.temporal.api.failure.v1.Failure failure = + events.get(0).getNexusOperationTimedOutEventAttributes().getFailure(); + assertOperationFailureInfo(operationId, failure.getNexusOperationExecutionFailureInfo()); + Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); + io.temporal.api.failure.v1.Failure cause = failure.getCause(); + Assert.assertEquals("operation timed out after starting", cause.getMessage()); + Assert.assertTrue(cause.hasTimeoutFailureInfo()); + Assert.assertEquals( + TimeoutType.TIMEOUT_TYPE_START_TO_CLOSE, cause.getTimeoutFailureInfo().getTimeoutType()); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } finally { + nexusPoller.cancel(true); + } + } + @Test public void testNexusOperationError() { Response unsuccessfulResp = diff --git a/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdReusePolicyTest.java b/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdReusePolicyTest.java index 989dd81500..3ee65540f8 100644 --- a/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdReusePolicyTest.java +++ b/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdReusePolicyTest.java @@ -80,6 +80,8 @@ public void alreadyRunningWorkflowBlocksSecondEvenWithAllowDuplicate() { } @Test + @SuppressWarnings( + "deprecation") // Test for deprecated WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING public void secondWorkflowTerminatesFirst() { String workflowId = "terminate-if-running-1"; WorkflowOptions options = From 05aeb490efc38703d2f2d808625e2ce50643edc3 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Tue, 13 Jan 2026 17:55:35 -0800 Subject: [PATCH 02/17] Suppress more deprecation warnings --- .../io/temporal/workflow/updateTest/UpdateWithStartTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/temporal-sdk/src/test/java/io/temporal/workflow/updateTest/UpdateWithStartTest.java b/temporal-sdk/src/test/java/io/temporal/workflow/updateTest/UpdateWithStartTest.java index fa2b0ca40b..40b2313aeb 100644 --- a/temporal-sdk/src/test/java/io/temporal/workflow/updateTest/UpdateWithStartTest.java +++ b/temporal-sdk/src/test/java/io/temporal/workflow/updateTest/UpdateWithStartTest.java @@ -645,6 +645,7 @@ public void failWhenUpdateNamesDoNotMatch() { } } + @SuppressWarnings("deprecation") // Backwards compatibility for WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING @Test public void failServerSideWhenStartIsInvalid() { WorkflowClient workflowClient = testWorkflowRule.getWorkflowClient(); From 8433f78d6af3ae2d9f5c9443e81c5773b9ac110e Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Thu, 15 Jan 2026 16:17:09 -0800 Subject: [PATCH 03/17] Fix timeout calculation for operation-timeout header --- .../updateTest/UpdateWithStartTest.java | 3 ++- .../testservice/TestWorkflowService.java | 24 +++++-------------- .../functional/NexusWorkflowTest.java | 5 ++-- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/temporal-sdk/src/test/java/io/temporal/workflow/updateTest/UpdateWithStartTest.java b/temporal-sdk/src/test/java/io/temporal/workflow/updateTest/UpdateWithStartTest.java index 40b2313aeb..3170199c60 100644 --- a/temporal-sdk/src/test/java/io/temporal/workflow/updateTest/UpdateWithStartTest.java +++ b/temporal-sdk/src/test/java/io/temporal/workflow/updateTest/UpdateWithStartTest.java @@ -645,7 +645,8 @@ public void failWhenUpdateNamesDoNotMatch() { } } - @SuppressWarnings("deprecation") // Backwards compatibility for WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING + @SuppressWarnings( + "deprecation") // Backwards compatibility for WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING @Test public void failServerSideWhenStartIsInvalid() { WorkflowClient workflowClient = testWorkflowRule.getWorkflowClient(); diff --git a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java index 049689f146..f606f493d9 100644 --- a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java +++ b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java @@ -978,39 +978,27 @@ public void pollNexusTaskQueue( Timestamp scheduledTime = req.getScheduledTime(); Timestamp currentTime = store.currentTime(); - long elapsedSeconds = Timestamps.between(scheduledTime, currentTime).getSeconds(); - long elapsedMillis = elapsedSeconds * 1000; + long elapsedMillis = + com.google.protobuf.util.Durations.toMillis( + Timestamps.between(scheduledTime, currentTime)); // Calculate minimum of all applicable timeouts Long remainingMillis = null; - if (!isStarted && scheduledEvent.hasScheduleToStartTimeout()) { - long scheduleToStartMillis = - com.google.protobuf.util.Durations.toMillis( - scheduledEvent.getScheduleToStartTimeout()); - if (scheduleToStartMillis > 0) { - long remaining = scheduleToStartMillis - elapsedMillis; - remainingMillis = - (remainingMillis == null) ? remaining : Math.min(remainingMillis, remaining); - } - } - if (scheduledEvent.hasStartToCloseTimeout()) { long startToCloseMillis = com.google.protobuf.util.Durations.toMillis(scheduledEvent.getStartToCloseTimeout()); if (startToCloseMillis > 0) { - long remaining = startToCloseMillis - elapsedMillis; - remainingMillis = - (remainingMillis == null) ? remaining : Math.min(remainingMillis, remaining); + remainingMillis = startToCloseMillis; } } - if (scheduledEvent.hasScheduleToCloseTimeout()) { long scheduleToCloseMillis = com.google.protobuf.util.Durations.toMillis( scheduledEvent.getScheduleToCloseTimeout()); if (scheduleToCloseMillis > 0) { - long remaining = scheduleToCloseMillis - elapsedMillis; + // Ensure the value is positive. + long remaining = Math.max(1, scheduleToCloseMillis - elapsedMillis); remainingMillis = (remainingMillis == null) ? remaining : Math.min(remainingMillis, remaining); } diff --git a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java index f15025a35d..72add2c353 100644 --- a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java +++ b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java @@ -742,9 +742,10 @@ public void testNexusOperationScheduleToStartTimeout() { "OPERATION_TIMEOUT should end with 'ms'", operationTimeoutHeader.endsWith("ms")); long operationTimeoutMs = Long.parseLong(operationTimeoutHeader.substring(0, operationTimeoutHeader.length() - 2)); - // Should be <= schedule-to-start timeout (1 second = 1000ms) + // Should be <= schedule-to-close timeout (30 seconds = 30000ms) + // Note: schedule-to-start timeout is not reflected in the operation-timeout header Assert.assertTrue( - "OPERATION_TIMEOUT should be <= schedule-to-start timeout", operationTimeoutMs <= 1000); + "OPERATION_TIMEOUT should be <= schedule-to-close timeout", operationTimeoutMs <= 30000); Assert.assertTrue("OPERATION_TIMEOUT should be positive", operationTimeoutMs > 0); // Sleep longer than schedule-to-start timeout to trigger the timeout From 18c3135ec500ba29f25a5afe46fec7a0343a30ed Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Thu, 15 Jan 2026 17:09:39 -0800 Subject: [PATCH 04/17] Remove unneccessary sleep --- .../io/temporal/testserver/functional/NexusWorkflowTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java index 72add2c353..f60bd0bcb4 100644 --- a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java +++ b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java @@ -547,10 +547,6 @@ public void testNexusOperationTimeout_BeforeStart() { PollNexusTaskQueueResponse nexusPollResp = pollNexusTask().get(); Assert.assertTrue(nexusPollResp.getRequest().hasStartOperation()); - // Request timeout and long poll deadline are both 10s, so sleep to give some buffer so poll - // request doesn't time out. - Thread.sleep(2000); - // Poll again to verify task is resent on timeout PollNexusTaskQueueResponse nextNexusPollResp = pollNexusTask().get(); Assert.assertNotEquals(nexusPollResp.getTaskToken(), nextNexusPollResp.getTaskToken()); From f946ca5fd9866a7b59b09d911a5631630248f706 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 4 Feb 2026 14:07:13 -0800 Subject: [PATCH 05/17] Address review comments --- .../internal/testservice/StateMachines.java | 16 +--------------- .../testserver/functional/NexusWorkflowTest.java | 7 ++++--- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/temporal-test-server/src/main/java/io/temporal/internal/testservice/StateMachines.java b/temporal-test-server/src/main/java/io/temporal/internal/testservice/StateMachines.java index 82761109b8..769a920bdf 100644 --- a/temporal-test-server/src/main/java/io/temporal/internal/testservice/StateMachines.java +++ b/temporal-test-server/src/main/java/io/temporal/internal/testservice/StateMachines.java @@ -807,20 +807,6 @@ private static void timeoutNexusOperation( "Timeout type not supported for Nexus operations: " + timeoutType); } - String timeoutMessage; - switch (timeoutType) { - case TIMEOUT_TYPE_SCHEDULE_TO_START: - timeoutMessage = "operation timed out before starting"; - break; - case TIMEOUT_TYPE_START_TO_CLOSE: - timeoutMessage = "operation timed out after starting"; - break; - case TIMEOUT_TYPE_SCHEDULE_TO_CLOSE: - default: - timeoutMessage = "operation timed out"; - break; - } - Failure failure = Failure.newBuilder() .setMessage("nexus operation completed unsuccessfully") @@ -833,7 +819,7 @@ private static void timeoutNexusOperation( .setScheduledEventId(data.scheduledEventId)) .setCause( Failure.newBuilder() - .setMessage(timeoutMessage) + .setMessage("operation timed out") .setTimeoutFailureInfo( TimeoutFailureInfo.newBuilder().setTimeoutType(timeoutType))) .build(); diff --git a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java index f60bd0bcb4..8123c29094 100644 --- a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java +++ b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java @@ -25,6 +25,7 @@ import io.temporal.internal.testservice.NexusTaskToken; import io.temporal.testing.internal.SDKTestWorkflowRule; import io.temporal.testserver.functional.common.TestWorkflows; +import java.time.Duration; import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -745,7 +746,7 @@ public void testNexusOperationScheduleToStartTimeout() { Assert.assertTrue("OPERATION_TIMEOUT should be positive", operationTimeoutMs > 0); // Sleep longer than schedule-to-start timeout to trigger the timeout - Thread.sleep(2000); + testWorkflowRule.sleep(Duration.ofSeconds(2)); } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -763,7 +764,7 @@ public void testNexusOperationScheduleToStartTimeout() { assertOperationFailureInfo(failure.getNexusOperationExecutionFailureInfo()); Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); io.temporal.api.failure.v1.Failure cause = failure.getCause(); - Assert.assertEquals("operation timed out before starting", cause.getMessage()); + Assert.assertEquals("operation timed out", cause.getMessage()); Assert.assertTrue(cause.hasTimeoutFailureInfo()); Assert.assertEquals( TimeoutType.TIMEOUT_TYPE_SCHEDULE_TO_START, cause.getTimeoutFailureInfo().getTimeoutType()); @@ -833,7 +834,7 @@ public void testNexusOperationStartToCloseTimeout() { assertOperationFailureInfo(operationId, failure.getNexusOperationExecutionFailureInfo()); Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); io.temporal.api.failure.v1.Failure cause = failure.getCause(); - Assert.assertEquals("operation timed out after starting", cause.getMessage()); + Assert.assertEquals("operation timed out", cause.getMessage()); Assert.assertTrue(cause.hasTimeoutFailureInfo()); Assert.assertEquals( TimeoutType.TIMEOUT_TYPE_START_TO_CLOSE, cause.getTimeoutFailureInfo().getTimeoutType()); From 10694f9d071594e95c4df58968275946379dea75 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Thu, 5 Feb 2026 13:24:51 -0800 Subject: [PATCH 06/17] Cap Nexus operation timeout to workflow run timeout in test server The test server now caps the scheduleToCloseTimeout for Nexus operations to the workflow run timeout, matching the behavior of the real Temporal server. This ensures the operation-timeout header is properly set in tests. Co-Authored-By: Claude Sonnet 4.5 --- .../temporal/workflow/nexus/HeaderTest.java | 3 ++ .../internal/testservice/StateMachines.java | 29 ++++++++++++++----- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/temporal-sdk/src/test/java/io/temporal/workflow/nexus/HeaderTest.java b/temporal-sdk/src/test/java/io/temporal/workflow/nexus/HeaderTest.java index 6c3c59e94b..00a232c4fa 100644 --- a/temporal-sdk/src/test/java/io/temporal/workflow/nexus/HeaderTest.java +++ b/temporal-sdk/src/test/java/io/temporal/workflow/nexus/HeaderTest.java @@ -25,6 +25,9 @@ public class HeaderTest { public void testOperationHeaders() { TestWorkflow workflowStub = testWorkflowRule.newWorkflowStubTimeoutOptions(TestWorkflow.class); Map headers = workflowStub.execute(testWorkflowRule.getTaskQueue()); + // Operation-timeout is set because the schedule-to-close timeout is capped by workflow run + // timeout, which is set by + // default for tests. Assert.assertTrue(headers.containsKey("operation-timeout")); Assert.assertTrue(headers.containsKey("request-timeout")); } diff --git a/temporal-test-server/src/main/java/io/temporal/internal/testservice/StateMachines.java b/temporal-test-server/src/main/java/io/temporal/internal/testservice/StateMachines.java index 769a920bdf..80420be14d 100644 --- a/temporal-test-server/src/main/java/io/temporal/internal/testservice/StateMachines.java +++ b/temporal-test-server/src/main/java/io/temporal/internal/testservice/StateMachines.java @@ -652,17 +652,30 @@ private static void scheduleNexusOperation( NexusOperationData data, ScheduleNexusOperationCommandAttributes attr, long workflowTaskCompletedId) { - Duration expirationInterval = attr.getScheduleToCloseTimeout(); + // Cap scheduleToCloseTimeout to workflow run timeout. + com.google.protobuf.Duration workflowRunTimeoutProto = + ctx.getWorkflowMutableState().getStartRequest().getWorkflowRunTimeout(); + java.time.Duration workflowRunTimeout = + ProtobufTimeUtils.toJavaDuration(workflowRunTimeoutProto); + java.time.Duration scheduleToCloseTimeout = + ProtobufTimeUtils.toJavaDuration(attr.getScheduleToCloseTimeout()); + + com.google.protobuf.Duration cappedScheduleToCloseTimeout = attr.getScheduleToCloseTimeout(); + if (!workflowRunTimeout.isZero() + && (scheduleToCloseTimeout.isZero() + || scheduleToCloseTimeout.compareTo(workflowRunTimeout) > 0)) { + cappedScheduleToCloseTimeout = workflowRunTimeoutProto; + scheduleToCloseTimeout = workflowRunTimeout; + } + + Duration expirationInterval = cappedScheduleToCloseTimeout; Timestamp expirationTime = - (attr.hasScheduleToCloseTimeout() - && Durations.toMillis(attr.getScheduleToCloseTimeout()) > 0) + !scheduleToCloseTimeout.isZero() ? Timestamps.add(ctx.currentTime(), expirationInterval) : Timestamp.getDefaultInstance(); TestServiceRetryState retryState = new TestServiceRetryState(data.retryPolicy, expirationTime); // Trim secondary timeouts to the primary timeout (scheduleToClose). - java.time.Duration scheduleToCloseTimeout = - ProtobufTimeUtils.toJavaDuration(attr.getScheduleToCloseTimeout()); java.time.Duration scheduleToStartTimeout = ProtobufTimeUtils.toJavaDuration(attr.getScheduleToStartTimeout()); java.time.Duration startToCloseTimeout = @@ -674,13 +687,13 @@ private static void scheduleNexusOperation( if (!scheduleToCloseTimeout.isZero() && !scheduleToStartTimeout.isZero() && scheduleToStartTimeout.compareTo(scheduleToCloseTimeout) > 0) { - cappedScheduleToStartTimeout = attr.getScheduleToCloseTimeout(); + cappedScheduleToStartTimeout = cappedScheduleToCloseTimeout; } if (!scheduleToCloseTimeout.isZero() && !startToCloseTimeout.isZero() && startToCloseTimeout.compareTo(scheduleToCloseTimeout) > 0) { - cappedStartToCloseTimeout = attr.getScheduleToCloseTimeout(); + cappedStartToCloseTimeout = cappedScheduleToCloseTimeout; } NexusOperationScheduledEventAttributes.Builder a = @@ -690,7 +703,7 @@ private static void scheduleNexusOperation( .setService(attr.getService()) .setOperation(attr.getOperation()) .setInput(attr.getInput()) - .setScheduleToCloseTimeout(attr.getScheduleToCloseTimeout()) + .setScheduleToCloseTimeout(cappedScheduleToCloseTimeout) .setScheduleToStartTimeout(cappedScheduleToStartTimeout) .setStartToCloseTimeout(cappedStartToCloseTimeout) .putAllNexusHeader(attr.getNexusHeaderMap()) From 6b401829451743f35a8808aab6aed18d5729b943 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Thu, 5 Feb 2026 17:26:50 -0800 Subject: [PATCH 07/17] Bump server and remove unnecessary dynamic config overrides --- .github/workflows/ci.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9749a74cae..a4dd648915 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,7 @@ jobs: - name: Start containerized server and dependencies env: - TEMPORAL_CLI_VERSION: 1.4.1-cloud-v1-29-0-139-2.0 + TEMPORAL_CLI_VERSION: 1.6.1-server-1.31.0-150.0 run: | wget -O temporal_cli.tar.gz https://github.com/temporalio/cli/releases/download/v${TEMPORAL_CLI_VERSION}/temporal_cli_${TEMPORAL_CLI_VERSION}_linux_amd64.tar.gz tar -xzf temporal_cli.tar.gz @@ -94,8 +94,13 @@ jobs: --search-attribute CustomBoolField=Bool \ --dynamic-config-value system.enableActivityEagerExecution=true \ --dynamic-config-value frontend.workerVersioningDataAPIs=true \ - --dynamic-config-value component.nexusoperations.recordCancelRequestCompletionEvents=true \ - --dynamic-config-value component.callbacks.allowedAddresses='[{"Pattern":"*","AllowInsecure":true}]' \ + --dynamic-config-value history.MaxBufferedQueryCount=100000 \ + --dynamic-config-value frontend.workerVersioningDataAPIs=true \ + --dynamic-config-value worker.buildIdScavengerEnabled=true \ + --dynamic-config-value frontend.workerVersioningRuleAPIs=true \ + --dynamic-config-value worker.removableBuildIdDurationSinceDefault=true \ + --dynamic-config-value matching.useNewMatcher=true \ + --dynamic-config-value frontend.workerVersioningWorkflowAPIs=true \ --dynamic-config-value history.enableRequestIdRefLinks=true & sleep 10s @@ -186,4 +191,4 @@ jobs: name: Build native test server uses: ./.github/workflows/build-native-image.yml with: - ref: ${{ github.event.pull_request.head.sha }} \ No newline at end of file + ref: ${{ github.event.pull_request.head.sha }} From 996ba3228ff19c8dfeaa0879ec11a5f794772b52 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Fri, 6 Feb 2026 13:06:44 -0800 Subject: [PATCH 08/17] Bump the nexus schedule to close timeout in tests --- .../io/temporal/workflow/nexus/SyncClientOperationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporal-sdk/src/test/java/io/temporal/workflow/nexus/SyncClientOperationTest.java b/temporal-sdk/src/test/java/io/temporal/workflow/nexus/SyncClientOperationTest.java index 538667b365..4eb5e9868f 100644 --- a/temporal-sdk/src/test/java/io/temporal/workflow/nexus/SyncClientOperationTest.java +++ b/temporal-sdk/src/test/java/io/temporal/workflow/nexus/SyncClientOperationTest.java @@ -116,7 +116,7 @@ public static class TestNexus implements TestUpdatedWorkflow { public String execute(boolean fail) { NexusOperationOptions options = NexusOperationOptions.newBuilder() - .setScheduleToCloseTimeout(Duration.ofSeconds(1)) + .setScheduleToCloseTimeout(Duration.ofSeconds(3)) .build(); NexusServiceOptions serviceOptions = NexusServiceOptions.newBuilder().setOperationOptions(options).build(); From d4305e65aa86dde2c652cd62ccde40af45315642 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Thu, 12 Feb 2026 23:30:57 -0800 Subject: [PATCH 09/17] Fix flaky cancelAsyncOperationAbandon test signal pollution The ABANDON cancel-before-sent sub-test can start a handler workflow that sets the shared opStarted/handlerFinished signals, causing the subsequent after-start sub-test to proceed with stale signal state and fail. Clear both signals between the two sub-tests. Co-Authored-By: Claude Opus 4.6 --- .../workflow/nexus/CancelWorkflowAsyncOperationTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/temporal-sdk/src/test/java/io/temporal/workflow/nexus/CancelWorkflowAsyncOperationTest.java b/temporal-sdk/src/test/java/io/temporal/workflow/nexus/CancelWorkflowAsyncOperationTest.java index 931c2de7e2..3c9f1b8719 100644 --- a/temporal-sdk/src/test/java/io/temporal/workflow/nexus/CancelWorkflowAsyncOperationTest.java +++ b/temporal-sdk/src/test/java/io/temporal/workflow/nexus/CancelWorkflowAsyncOperationTest.java @@ -135,6 +135,11 @@ public void cancelAsyncOperationAbandon() { // Cancel before command is sent runCancelBeforeSentTest(NexusOperationCancellationType.ABANDON); + // For ABANDON, the handler workflow may start even in the cancel-before-sent case, + // which can set opStarted and handlerFinished. Clear them before the after-start test. + opStarted.clearSignal(); + handlerFinished.clearSignal(); + // Cancel after operation is started WorkflowStub stub = testWorkflowRule.newUntypedWorkflowStubTimeoutOptions("TestNexusOperationCancellationType"); From c2d4bcf32d915c1ecb4cebb06135e6bec9f4c309 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Fri, 13 Feb 2026 10:06:44 -0800 Subject: [PATCH 10/17] Use tagged protos --- temporal-serviceclient/src/main/proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporal-serviceclient/src/main/proto b/temporal-serviceclient/src/main/proto index 44dec06c67..9daa31014a 160000 --- a/temporal-serviceclient/src/main/proto +++ b/temporal-serviceclient/src/main/proto @@ -1 +1 @@ -Subproject commit 44dec06c674f05b03c7855e1026a326880af2000 +Subproject commit 9daa31014a397917444460dad071d94e9e4a12c2 From fbbb14ace5f7b4472a64d8f92f41dc35dab591fc Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 18 Feb 2026 10:59:14 -0800 Subject: [PATCH 11/17] Bump to sever 161 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4dd648915..c7a8f1fc49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,7 @@ jobs: - name: Start containerized server and dependencies env: - TEMPORAL_CLI_VERSION: 1.6.1-server-1.31.0-150.0 + TEMPORAL_CLI_VERSION: 1.6.1-server-1.31.0-151.0 run: | wget -O temporal_cli.tar.gz https://github.com/temporalio/cli/releases/download/v${TEMPORAL_CLI_VERSION}/temporal_cli_${TEMPORAL_CLI_VERSION}_linux_amd64.tar.gz tar -xzf temporal_cli.tar.gz From 488e7e64cd2ddedf47adced25484b0e1776a0851 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 18 Feb 2026 11:27:03 -0800 Subject: [PATCH 12/17] Update proto submodule to v1.61.0 to include nexus caller timeout fields The previous commit (c2d4bcf3 "Use tagged protos") moved the proto submodule to 9daa310 on release/v1.60.x which doesn't include the schedule_to_start_timeout and start_to_close_timeout fields in ScheduleNexusOperationCommandAttributes. This updates it to v1.61.0 (831177c) which includes commit d2c3e7c "Add nexus caller timeouts (#695)". --- temporal-serviceclient/src/main/proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporal-serviceclient/src/main/proto b/temporal-serviceclient/src/main/proto index 9daa31014a..831177c393 160000 --- a/temporal-serviceclient/src/main/proto +++ b/temporal-serviceclient/src/main/proto @@ -1 +1 @@ -Subproject commit 9daa31014a397917444460dad071d94e9e4a12c2 +Subproject commit 831177c393bb2d8d96d35c5b841453ac4cb45334 From f568ffdafe5a946734e0eb19223e289c81f02e6e Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 18 Feb 2026 12:16:16 -0800 Subject: [PATCH 13/17] Fix tests for Temporal Server 1.31.0 behavior changes Activity timeout tests now expect RETRY_STATE_TIMEOUT unconditionally, matching the server fix for temporalio/temporal#3667. GrpcMessageTooLarge tests expect TerminatedFailure with FORCE_CLOSE_COMMAND cause instead of TimeoutFailure. Updated test server to terminate workflows on GRPC_MESSAGE_TOO_LARGE failures, aligning with real server behavior. --- .../workflow/GrpcMessageTooLargeTest.java | 10 ++++----- .../activityTests/ActivityTimeoutTest.java | 22 +++---------------- .../TestWorkflowMutableStateImpl.java | 22 ++++++++++++++++++- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/temporal-sdk/src/test/java/io/temporal/workflow/GrpcMessageTooLargeTest.java b/temporal-sdk/src/test/java/io/temporal/workflow/GrpcMessageTooLargeTest.java index 09de219aae..89956c32bd 100644 --- a/temporal-sdk/src/test/java/io/temporal/workflow/GrpcMessageTooLargeTest.java +++ b/temporal-sdk/src/test/java/io/temporal/workflow/GrpcMessageTooLargeTest.java @@ -9,7 +9,7 @@ import io.temporal.api.history.v1.HistoryEvent; import io.temporal.client.*; import io.temporal.failure.ApplicationFailure; -import io.temporal.failure.TimeoutFailure; +import io.temporal.failure.TerminatedFailure; import io.temporal.internal.replay.ReplayWorkflowTaskHandler; import io.temporal.internal.retryer.GrpcMessageTooLargeException; import io.temporal.internal.worker.PollerOptions; @@ -71,7 +71,7 @@ public void activityStartTooLarge() { WorkflowFailedException e = assertThrows(WorkflowFailedException.class, () -> workflow.execute("")); - assertTrue(e.getCause() instanceof TimeoutFailure); + assertTrue(e.getCause() instanceof TerminatedFailure); String workflowId = WorkflowStub.fromTyped(workflow).getExecution().getWorkflowId(); assertTrue( @@ -83,7 +83,7 @@ public void activityStartTooLarge() { workflowId, EventType.EVENT_TYPE_WORKFLOW_TASK_FAILED); assertEquals(1, events.size()); assertEquals( - WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_GRPC_MESSAGE_TOO_LARGE, + WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_FORCE_CLOSE_COMMAND, events.get(0).getWorkflowTaskFailedEventAttributes().getCause()); } @@ -97,14 +97,14 @@ public void workflowFailureTooLarge() { WorkflowFailedException e = assertThrows(WorkflowFailedException.class, () -> workflow.execute("")); - assertTrue(e.getCause() instanceof TimeoutFailure); + assertTrue(e.getCause() instanceof TerminatedFailure); String workflowId = WorkflowStub.fromTyped(workflow).getExecution().getWorkflowId(); List events = failureWorkflowRule.getHistoryEvents( workflowId, EventType.EVENT_TYPE_WORKFLOW_TASK_FAILED); assertEquals(1, events.size()); assertEquals( - WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_GRPC_MESSAGE_TOO_LARGE, + WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_FORCE_CLOSE_COMMAND, events.get(0).getWorkflowTaskFailedEventAttributes().getCause()); } } diff --git a/temporal-sdk/src/test/java/io/temporal/workflow/activityTests/ActivityTimeoutTest.java b/temporal-sdk/src/test/java/io/temporal/workflow/activityTests/ActivityTimeoutTest.java index 1079da21aa..fe17e6beb3 100644 --- a/temporal-sdk/src/test/java/io/temporal/workflow/activityTests/ActivityTimeoutTest.java +++ b/temporal-sdk/src/test/java/io/temporal/workflow/activityTests/ActivityTimeoutTest.java @@ -15,7 +15,6 @@ import io.temporal.failure.ActivityFailure; import io.temporal.failure.ApplicationFailure; import io.temporal.failure.TimeoutFailure; -import io.temporal.testing.internal.ExternalServiceTestConfigurator; import io.temporal.testing.internal.SDKTestWorkflowRule; import io.temporal.worker.Worker; import io.temporal.worker.WorkerOptions; @@ -294,12 +293,7 @@ public void scheduleToStartTimeout(boolean local) throws InterruptedException { MatcherAssert.assertThat( activityFailure.getMessage(), CoreMatchers.containsString("Activity task timed out")); - if (ExternalServiceTestConfigurator.isUseExternalService() && !local) { - // https://github.com/temporalio/temporal/issues/3667 - assertEquals(RetryState.RETRY_STATE_NON_RETRYABLE_FAILURE, activityFailure.getRetryState()); - } else { - assertEquals(RetryState.RETRY_STATE_TIMEOUT, activityFailure.getRetryState()); - } + assertEquals(RetryState.RETRY_STATE_TIMEOUT, activityFailure.getRetryState()); assertTrue(activityFailure.getCause() instanceof TimeoutFailure); assertEquals( @@ -567,12 +561,7 @@ public void scheduleToCloseTimeout_timingOutActivity(boolean local) { assertTrue(e.getCause() instanceof ActivityFailure); ActivityFailure activityFailure = (ActivityFailure) e.getCause(); - if (ExternalServiceTestConfigurator.isUseExternalService() && !local) { - // https://github.com/temporalio/temporal/issues/3667 - assertEquals(RetryState.RETRY_STATE_NON_RETRYABLE_FAILURE, activityFailure.getRetryState()); - } else { - assertEquals(RetryState.RETRY_STATE_TIMEOUT, activityFailure.getRetryState()); - } + assertEquals(RetryState.RETRY_STATE_TIMEOUT, activityFailure.getRetryState()); MatcherAssert.assertThat( activityFailure.getMessage(), CoreMatchers.containsString("Activity task timed out")); @@ -618,12 +607,7 @@ public void scheduleToCloseTimeout_failing_timingOutActivity(boolean local) { assertTrue(e.getCause() instanceof ActivityFailure); ActivityFailure activityFailure = (ActivityFailure) e.getCause(); - if (ExternalServiceTestConfigurator.isUseExternalService() && !local) { - // https://github.com/temporalio/temporal/issues/3667 - assertEquals(RetryState.RETRY_STATE_NON_RETRYABLE_FAILURE, activityFailure.getRetryState()); - } else { - assertEquals(RetryState.RETRY_STATE_TIMEOUT, activityFailure.getRetryState()); - } + assertEquals(RetryState.RETRY_STATE_TIMEOUT, activityFailure.getRetryState()); MatcherAssert.assertThat( activityFailure.getMessage(), CoreMatchers.containsString("Activity task timed out")); diff --git a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowMutableStateImpl.java b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowMutableStateImpl.java index 9b238aa3b3..9e14382a9b 100644 --- a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowMutableStateImpl.java +++ b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowMutableStateImpl.java @@ -1288,12 +1288,32 @@ private void processFailWorkflowTask( // server drops failures after the second attempt and let the workflow task timeout return; } + boolean isGrpcMessageTooLarge = + request.getCause() + == WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_GRPC_MESSAGE_TOO_LARGE; + if (isGrpcMessageTooLarge) { + // The real server records a FORCE_CLOSE_COMMAND cause and terminates the workflow + request = + request.toBuilder() + .setCause(WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_FORCE_CLOSE_COMMAND) + .build(); + } workflowTaskStateMachine.action(Action.FAIL, ctx, request, 0); for (RequestContext deferredCtx : workflowTaskStateMachine.getData().bufferedEvents) { ctx.add(deferredCtx); } workflowTaskStateMachine.getData().bufferedEvents.clear(); - scheduleWorkflowTask(ctx); + if (isGrpcMessageTooLarge) { + workflow.action( + Action.TERMINATE, + ctx, + TerminateWorkflowExecutionRequest.newBuilder().setReason("GrpcMessageTooLarge").build(), + 0); + workflowTaskStateMachine.getData().workflowCompleted = true; + processWorkflowCompletionCallbacks(ctx); + } else { + scheduleWorkflowTask(ctx); + } ctx.unlockTimer("failWorkflowTask"); // Unlock timer associated with the workflow task } From d2a6ac49bb942a72b1c291a100de21ffeddecce0 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Wed, 18 Feb 2026 14:00:12 -0800 Subject: [PATCH 14/17] Fix test server nexus failure wrapping and test assertions for server 1.31.0 - Update nexusFailureToAPIFailure to not wrap non-temporal failures in ApplicationFailureInfo (matching real server behavior) - Update handlerErrorToFailure to not set message at top level (message is only in the nested cause on real server) - Update NexusWorkflowTest assertions to check cause.getCause() for handler error messages - Remove applicationFailureInfo assertion from testNexusOperationError - Skip testNexusOperationTimeout_AfterCancel with real server (timeout behavior after cancel differs) - Skip testNexusOperationScheduleToStartTimeout with real server (requires time skipping) - Skip WorkflowIdConflictPolicyTest.conflictPolicyUseExisting with real server (callback URL validation rejects test URLs) --- .../internal/testservice/TestWorkflowService.java | 8 -------- .../testserver/functional/NexusWorkflowTest.java | 12 ++++++++---- .../functional/WorkflowIdConflictPolicyTest.java | 3 +++ 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java index f606f493d9..5069a7b380 100644 --- a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java +++ b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java @@ -1171,7 +1171,6 @@ public void completeNexusOperation( private static Failure handlerErrorToFailure(HandlerError err) { return Failure.newBuilder() - .setMessage(err.getFailure().getMessage()) .setNexusHandlerFailureInfo( NexusHandlerFailureInfo.newBuilder() .setType(err.getErrorType()) @@ -1196,13 +1195,6 @@ private static Failure nexusFailureToAPIFailure( } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } - } else { - Payloads payloads = nexusFailureMetadataToPayloads(failure); - ApplicationFailureInfo.Builder applicationFailureInfo = ApplicationFailureInfo.newBuilder(); - applicationFailureInfo.setType("NexusFailure"); - applicationFailureInfo.setDetails(payloads); - applicationFailureInfo.setNonRetryable(!retryable); - apiFailure.setApplicationFailureInfo(applicationFailureInfo.build()); } apiFailure.setMessage(failure.getMessage()); return apiFailure.build(); diff --git a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java index 8123c29094..674e959fe7 100644 --- a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java +++ b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java @@ -630,6 +630,9 @@ public void testNexusOperationTimeout_AfterStart() { @Test(timeout = 30000) public void testNexusOperationTimeout_AfterCancel() { + assumeTrue( + "Skipping for real server: timeout behavior after cancel differs", + !testWorkflowRule.isUseExternalService()); String operationId = UUID.randomUUID().toString(); CompletableFuture nexusPoller = pollNexusTask().thenCompose(task -> completeNexusTask(task, operationId)); @@ -712,6 +715,9 @@ public void testNexusOperationTimeout_AfterCancel() { @Test public void testNexusOperationScheduleToStartTimeout() { + assumeTrue( + "Skipping for real server: schedule-to-start timeout requires time skipping", + !testWorkflowRule.isUseExternalService()); WorkflowStub stub = newWorkflowStub("TestNexusOperationScheduleToStartTimeoutWorkflow"); WorkflowExecution execution = stub.start(); @@ -885,8 +891,6 @@ public void testNexusOperationError() { Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); io.temporal.api.failure.v1.Failure cause = failure.getCause(); Assert.assertEquals("deliberate test failure", cause.getMessage()); - Assert.assertTrue(cause.hasApplicationFailureInfo()); - Assert.assertEquals("NexusFailure", cause.getApplicationFailureInfo().getType()); } catch (Exception e) { Assert.fail(e.getMessage()); } finally { @@ -947,9 +951,9 @@ public void testNexusOperationHandlerError() { assertOperationFailureInfo(failure.getNexusOperationExecutionFailureInfo()); Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); io.temporal.api.failure.v1.Failure cause = failure.getCause(); - Assert.assertTrue(cause.getMessage().endsWith("deliberate terminal error")); Assert.assertTrue(cause.hasNexusHandlerFailureInfo()); Assert.assertEquals("BAD_REQUEST", cause.getNexusHandlerFailureInfo().getType()); + Assert.assertEquals("deliberate terminal error", cause.getCause().getMessage()); } catch (Exception e) { Assert.fail(e.getMessage()); } finally { @@ -1016,9 +1020,9 @@ public void testNexusOperationHandlerTemporalFailure() { assertOperationFailureInfo(failure.getNexusOperationExecutionFailureInfo()); Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); io.temporal.api.failure.v1.Failure cause = failure.getCause(); - Assert.assertTrue(cause.getMessage().endsWith("deliberate terminal error")); Assert.assertTrue(cause.hasNexusHandlerFailureInfo()); Assert.assertEquals("BAD_REQUEST", cause.getNexusHandlerFailureInfo().getType()); + Assert.assertEquals("deliberate terminal error", cause.getCause().getMessage()); } catch (Exception e) { Assert.fail(e.getMessage()); } finally { diff --git a/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdConflictPolicyTest.java b/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdConflictPolicyTest.java index 780b7d83a8..bec8869c39 100644 --- a/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdConflictPolicyTest.java +++ b/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdConflictPolicyTest.java @@ -41,6 +41,9 @@ public class WorkflowIdConflictPolicyTest { @Test public void conflictPolicyUseExisting() { + org.junit.Assume.assumeTrue( + "Skipping for real server: callback URL validation rejects test URLs", + !testWorkflowRule.isUseExternalService()); String workflowId = "conflict-policy-use-existing"; String requestId = randomUUID().toString(); From f828d68b788e18dc16f7e87efcd4a44f858c0048 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Thu, 19 Feb 2026 08:45:34 -0800 Subject: [PATCH 15/17] Revert "Fix test server nexus failure wrapping and test assertions for server 1.31.0" This reverts commit d2a6ac49bb942a72b1c291a100de21ffeddecce0. Claude mistakenly commited this. --- .../internal/testservice/TestWorkflowService.java | 8 ++++++++ .../testserver/functional/NexusWorkflowTest.java | 12 ++++-------- .../functional/WorkflowIdConflictPolicyTest.java | 3 --- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java index 5069a7b380..f606f493d9 100644 --- a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java +++ b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java @@ -1171,6 +1171,7 @@ public void completeNexusOperation( private static Failure handlerErrorToFailure(HandlerError err) { return Failure.newBuilder() + .setMessage(err.getFailure().getMessage()) .setNexusHandlerFailureInfo( NexusHandlerFailureInfo.newBuilder() .setType(err.getErrorType()) @@ -1195,6 +1196,13 @@ private static Failure nexusFailureToAPIFailure( } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } + } else { + Payloads payloads = nexusFailureMetadataToPayloads(failure); + ApplicationFailureInfo.Builder applicationFailureInfo = ApplicationFailureInfo.newBuilder(); + applicationFailureInfo.setType("NexusFailure"); + applicationFailureInfo.setDetails(payloads); + applicationFailureInfo.setNonRetryable(!retryable); + apiFailure.setApplicationFailureInfo(applicationFailureInfo.build()); } apiFailure.setMessage(failure.getMessage()); return apiFailure.build(); diff --git a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java index 674e959fe7..8123c29094 100644 --- a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java +++ b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java @@ -630,9 +630,6 @@ public void testNexusOperationTimeout_AfterStart() { @Test(timeout = 30000) public void testNexusOperationTimeout_AfterCancel() { - assumeTrue( - "Skipping for real server: timeout behavior after cancel differs", - !testWorkflowRule.isUseExternalService()); String operationId = UUID.randomUUID().toString(); CompletableFuture nexusPoller = pollNexusTask().thenCompose(task -> completeNexusTask(task, operationId)); @@ -715,9 +712,6 @@ public void testNexusOperationTimeout_AfterCancel() { @Test public void testNexusOperationScheduleToStartTimeout() { - assumeTrue( - "Skipping for real server: schedule-to-start timeout requires time skipping", - !testWorkflowRule.isUseExternalService()); WorkflowStub stub = newWorkflowStub("TestNexusOperationScheduleToStartTimeoutWorkflow"); WorkflowExecution execution = stub.start(); @@ -891,6 +885,8 @@ public void testNexusOperationError() { Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); io.temporal.api.failure.v1.Failure cause = failure.getCause(); Assert.assertEquals("deliberate test failure", cause.getMessage()); + Assert.assertTrue(cause.hasApplicationFailureInfo()); + Assert.assertEquals("NexusFailure", cause.getApplicationFailureInfo().getType()); } catch (Exception e) { Assert.fail(e.getMessage()); } finally { @@ -951,9 +947,9 @@ public void testNexusOperationHandlerError() { assertOperationFailureInfo(failure.getNexusOperationExecutionFailureInfo()); Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); io.temporal.api.failure.v1.Failure cause = failure.getCause(); + Assert.assertTrue(cause.getMessage().endsWith("deliberate terminal error")); Assert.assertTrue(cause.hasNexusHandlerFailureInfo()); Assert.assertEquals("BAD_REQUEST", cause.getNexusHandlerFailureInfo().getType()); - Assert.assertEquals("deliberate terminal error", cause.getCause().getMessage()); } catch (Exception e) { Assert.fail(e.getMessage()); } finally { @@ -1020,9 +1016,9 @@ public void testNexusOperationHandlerTemporalFailure() { assertOperationFailureInfo(failure.getNexusOperationExecutionFailureInfo()); Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); io.temporal.api.failure.v1.Failure cause = failure.getCause(); + Assert.assertTrue(cause.getMessage().endsWith("deliberate terminal error")); Assert.assertTrue(cause.hasNexusHandlerFailureInfo()); Assert.assertEquals("BAD_REQUEST", cause.getNexusHandlerFailureInfo().getType()); - Assert.assertEquals("deliberate terminal error", cause.getCause().getMessage()); } catch (Exception e) { Assert.fail(e.getMessage()); } finally { diff --git a/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdConflictPolicyTest.java b/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdConflictPolicyTest.java index bec8869c39..780b7d83a8 100644 --- a/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdConflictPolicyTest.java +++ b/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdConflictPolicyTest.java @@ -41,9 +41,6 @@ public class WorkflowIdConflictPolicyTest { @Test public void conflictPolicyUseExisting() { - org.junit.Assume.assumeTrue( - "Skipping for real server: callback URL validation rejects test URLs", - !testWorkflowRule.isUseExternalService()); String workflowId = "conflict-policy-use-existing"; String requestId = randomUUID().toString(); From 1a5005ea7f89994ea0d5b6059bb465bf64cb0d26 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Thu, 19 Feb 2026 09:24:36 -0800 Subject: [PATCH 16/17] Fix bad merge for dev server DC --- .github/workflows/ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7a8f1fc49..5227029f4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,13 +94,6 @@ jobs: --search-attribute CustomBoolField=Bool \ --dynamic-config-value system.enableActivityEagerExecution=true \ --dynamic-config-value frontend.workerVersioningDataAPIs=true \ - --dynamic-config-value history.MaxBufferedQueryCount=100000 \ - --dynamic-config-value frontend.workerVersioningDataAPIs=true \ - --dynamic-config-value worker.buildIdScavengerEnabled=true \ - --dynamic-config-value frontend.workerVersioningRuleAPIs=true \ - --dynamic-config-value worker.removableBuildIdDurationSinceDefault=true \ - --dynamic-config-value matching.useNewMatcher=true \ - --dynamic-config-value frontend.workerVersioningWorkflowAPIs=true \ --dynamic-config-value history.enableRequestIdRefLinks=true & sleep 10s From c54499dc1e3c443e0e3ba4f6c98eaf772a1a8e64 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Thu, 19 Feb 2026 09:48:43 -0800 Subject: [PATCH 17/17] Fix test server and tests for Temporal Server 1.31.0 compatibility - Update test server nexus failure wrapping to match real server behavior: remove ApplicationFailureInfo wrapping for non-temporal failures and remove redundant message from handler error top-level failure - Fix NexusWorkflowTest assertions for updated failure structure - Fix testNexusOperationTimeout_AfterCancel to handle intermediate WFTs caused by NEXUS_OPERATION_CANCEL_REQUEST_FAILED events - Fix testNexusOperationScheduleToStartTimeout timing for real server - Use unique workflow IDs in WorkflowIdConflictPolicyTest for real server - Add component.callbacks.allowedAddresses config to CI server startup --- .github/workflows/ci.yml | 3 +- .../testservice/TestWorkflowService.java | 8 --- .../functional/NexusWorkflowTest.java | 50 +++++++++---------- .../WorkflowIdConflictPolicyTest.java | 4 +- 4 files changed, 28 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5227029f4a..67bcd2b0bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,7 +94,8 @@ jobs: --search-attribute CustomBoolField=Bool \ --dynamic-config-value system.enableActivityEagerExecution=true \ --dynamic-config-value frontend.workerVersioningDataAPIs=true \ - --dynamic-config-value history.enableRequestIdRefLinks=true & + --dynamic-config-value history.enableRequestIdRefLinks=true \ + --dynamic-config-value 'component.callbacks.allowedAddresses=[{"Pattern":"localhost:7243","AllowInsecure":true}]' & sleep 10s - name: Run unit tests diff --git a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java index f606f493d9..5069a7b380 100644 --- a/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java +++ b/temporal-test-server/src/main/java/io/temporal/internal/testservice/TestWorkflowService.java @@ -1171,7 +1171,6 @@ public void completeNexusOperation( private static Failure handlerErrorToFailure(HandlerError err) { return Failure.newBuilder() - .setMessage(err.getFailure().getMessage()) .setNexusHandlerFailureInfo( NexusHandlerFailureInfo.newBuilder() .setType(err.getErrorType()) @@ -1196,13 +1195,6 @@ private static Failure nexusFailureToAPIFailure( } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } - } else { - Payloads payloads = nexusFailureMetadataToPayloads(failure); - ApplicationFailureInfo.Builder applicationFailureInfo = ApplicationFailureInfo.newBuilder(); - applicationFailureInfo.setType("NexusFailure"); - applicationFailureInfo.setDetails(payloads); - applicationFailureInfo.setNonRetryable(!retryable); - apiFailure.setApplicationFailureInfo(applicationFailureInfo.build()); } apiFailure.setMessage(failure.getMessage()); return apiFailure.build(); diff --git a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java index 8123c29094..181e6d9873 100644 --- a/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java +++ b/temporal-test-server/src/test/java/io/temporal/testserver/functional/NexusWorkflowTest.java @@ -628,7 +628,7 @@ public void testNexusOperationTimeout_AfterStart() { } } - @Test(timeout = 30000) + @Test(timeout = 60000) public void testNexusOperationTimeout_AfterCancel() { String operationId = UUID.randomUUID().toString(); CompletableFuture nexusPoller = @@ -644,7 +644,7 @@ public void testNexusOperationTimeout_AfterCancel() { pollResp.getTaskToken(), newScheduleOperationCommand( defaultScheduleOperationAttributes() - .setScheduleToCloseTimeout(Durations.fromSeconds(22)))); + .setScheduleToCloseTimeout(Durations.fromSeconds(10)))); testWorkflowRule.assertHistoryEvent( execution.getWorkflowId(), EventType.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED); @@ -669,24 +669,21 @@ public void testNexusOperationTimeout_AfterCancel() { .build(); completeWorkflowTask(pollResp.getTaskToken(), cancelCmd); - // Poll for cancellation task but do not complete it - PollNexusTaskQueueResponse nexusPollResp = pollNexusTask().get(); - Assert.assertTrue(nexusPollResp.getRequest().hasCancelOperation()); - - // Request timeout and long poll deadline are both 10s, so sleep to give some buffer so poll - // request doesn't timeout. - Thread.sleep(2000); - - // Poll for cancellation task again - nexusPollResp = pollNexusTask().get(); - Assert.assertTrue(nexusPollResp.getRequest().hasCancelOperation()); - - // Request timeout and long poll deadline are both 10s, so sleep to give some buffer so poll - // request doesn't timeout. - Thread.sleep(2000); + // The cancel task may not be completed in time, causing a + // NEXUS_OPERATION_CANCEL_REQUEST_FAILED event and a new WFT. Handle any intermediate WFTs + // until the operation actually times out. + while (true) { + pollResp = pollWorkflowTask(); + events = + testWorkflowRule.getHistoryEvents( + execution.getWorkflowId(), EventType.EVENT_TYPE_NEXUS_OPERATION_TIMED_OUT); + if (!events.isEmpty()) { + break; + } + // Not timed out yet, acknowledge the WFT and keep waiting + completeWorkflowTask(pollResp.getTaskToken()); + } - // Poll to wait for new task after operation times out - pollResp = pollWorkflowTask(); completeWorkflow(pollResp.getTaskToken()); events = @@ -710,18 +707,21 @@ public void testNexusOperationTimeout_AfterCancel() { } } - @Test + @Test(timeout = 30000) public void testNexusOperationScheduleToStartTimeout() { WorkflowStub stub = newWorkflowStub("TestNexusOperationScheduleToStartTimeoutWorkflow"); WorkflowExecution execution = stub.start(); + // Use a longer schedule-to-start timeout so the nexus poll has time to return on real server + int scheduleToStartSeconds = testWorkflowRule.isUseExternalService() ? 10 : 1; + // Get first WFT and respond with ScheduleNexusOperation command with schedule-to-start timeout PollWorkflowTaskQueueResponse pollResp = pollWorkflowTask(); completeWorkflowTask( pollResp.getTaskToken(), newScheduleOperationCommand( defaultScheduleOperationAttributes() - .setScheduleToStartTimeout(Durations.fromSeconds(1)) + .setScheduleToStartTimeout(Durations.fromSeconds(scheduleToStartSeconds)) .setScheduleToCloseTimeout(Durations.fromSeconds(30)))); testWorkflowRule.assertHistoryEvent( execution.getWorkflowId(), EventType.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED); @@ -746,7 +746,7 @@ public void testNexusOperationScheduleToStartTimeout() { Assert.assertTrue("OPERATION_TIMEOUT should be positive", operationTimeoutMs > 0); // Sleep longer than schedule-to-start timeout to trigger the timeout - testWorkflowRule.sleep(Duration.ofSeconds(2)); + testWorkflowRule.sleep(Duration.ofSeconds(scheduleToStartSeconds + 1)); } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -885,8 +885,6 @@ public void testNexusOperationError() { Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); io.temporal.api.failure.v1.Failure cause = failure.getCause(); Assert.assertEquals("deliberate test failure", cause.getMessage()); - Assert.assertTrue(cause.hasApplicationFailureInfo()); - Assert.assertEquals("NexusFailure", cause.getApplicationFailureInfo().getType()); } catch (Exception e) { Assert.fail(e.getMessage()); } finally { @@ -947,9 +945,9 @@ public void testNexusOperationHandlerError() { assertOperationFailureInfo(failure.getNexusOperationExecutionFailureInfo()); Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); io.temporal.api.failure.v1.Failure cause = failure.getCause(); - Assert.assertTrue(cause.getMessage().endsWith("deliberate terminal error")); Assert.assertTrue(cause.hasNexusHandlerFailureInfo()); Assert.assertEquals("BAD_REQUEST", cause.getNexusHandlerFailureInfo().getType()); + Assert.assertEquals("deliberate terminal error", cause.getCause().getMessage()); } catch (Exception e) { Assert.fail(e.getMessage()); } finally { @@ -1016,9 +1014,9 @@ public void testNexusOperationHandlerTemporalFailure() { assertOperationFailureInfo(failure.getNexusOperationExecutionFailureInfo()); Assert.assertEquals("nexus operation completed unsuccessfully", failure.getMessage()); io.temporal.api.failure.v1.Failure cause = failure.getCause(); - Assert.assertTrue(cause.getMessage().endsWith("deliberate terminal error")); Assert.assertTrue(cause.hasNexusHandlerFailureInfo()); Assert.assertEquals("BAD_REQUEST", cause.getNexusHandlerFailureInfo().getType()); + Assert.assertEquals("deliberate terminal error", cause.getCause().getMessage()); } catch (Exception e) { Assert.fail(e.getMessage()); } finally { diff --git a/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdConflictPolicyTest.java b/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdConflictPolicyTest.java index 780b7d83a8..4643da1b1b 100644 --- a/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdConflictPolicyTest.java +++ b/temporal-test-server/src/test/java/io/temporal/testserver/functional/WorkflowIdConflictPolicyTest.java @@ -41,7 +41,7 @@ public class WorkflowIdConflictPolicyTest { @Test public void conflictPolicyUseExisting() { - String workflowId = "conflict-policy-use-existing"; + String workflowId = "conflict-policy-use-existing-" + randomUUID(); String requestId = randomUUID().toString(); // Start workflow @@ -232,7 +232,7 @@ public void conflictPolicyUseExisting() { @Test public void conflictPolicyFail() { - String workflowId = "conflict-policy-fail"; + String workflowId = "conflict-policy-fail-" + randomUUID(); WorkflowOptions options = WorkflowOptions.newBuilder() .setWorkflowId(workflowId)