Skip to content

External Storage Integration: NexusWorker #3018

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cconstable wants to merge 1 commit into from
Open

External Storage Integration: NexusWorker #3018

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 63 additions & 27 deletions temporal-sdk/src/main/java/io/temporal/internal/worker/NexusWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import static io.temporal.serviceclient.MetricsTag.TASK_FAILURE_TYPE;

import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
import com.uber.m3.tally.Scope;
import com.uber.m3.tally.Stopwatch;
import com.uber.m3.util.Duration;
Expand All @@ -17,6 +18,7 @@
import io.temporal.internal.common.NexusUtil;
import io.temporal.internal.common.ProtobufTimeUtils;
import io.temporal.internal.logging.LoggerTag;
import io.temporal.internal.payload.storage.ExternalStorage;
import io.temporal.internal.retryer.GrpcRetryer;
import io.temporal.serviceclient.MetricsTag;
import io.temporal.serviceclient.WorkflowServiceStubs;
Expand Down Expand Up @@ -274,6 +276,16 @@ public String toString() {
options.getIdentity(), namespace, taskQueue);
}

/**
* Marks a failure raised by the external storage. Reported to the server as a retryable handler
* error.
*/
private static final class ExternalStorageTaskFailure extends RuntimeException {
ExternalStorageTaskFailure(String message, Throwable cause) {
super(message, cause);
}
}

private class TaskHandlerImpl implements PollTaskExecutor.TaskHandler<NexusTask> {

final NexusTaskHandler handler;
Expand Down Expand Up @@ -304,28 +316,30 @@ private String getNexusTaskOperation(PollNexusTaskQueueResponseOrBuilder pollRes

@Override
public void handle(NexusTask task) {
PollNexusTaskQueueResponseOrBuilder pollResponse = task.getResponse();
// Extract service and operation from the request and set them as MDC and metrics
// scope tags. If the request does not have a service or operation, do not set the tags.
// If we don't know how to handle the task, we will fail the task further down the line.
Scope metricsScope = workerMetricsScope;
String service = getNexusTaskService(pollResponse);
if (!service.isEmpty()) {
MDC.put(LoggerTag.NEXUS_SERVICE, service);
metricsScope = metricsScope.tagged(ImmutableMap.of(MetricsTag.NEXUS_SERVICE, service));
}
String operation = getNexusTaskOperation(pollResponse);
if (!operation.isEmpty()) {
MDC.put(LoggerTag.NEXUS_OPERATION, operation);
metricsScope = metricsScope.tagged(ImmutableMap.of(MetricsTag.NEXUS_OPERATION, operation));
}
slotSupplier.markSlotUsed(
new NexusSlotInfo(
service, operation, taskQueue, options.getIdentity(), options.getBuildId()),
task.getPermit());

boolean taskFailed = false;
try {
task = retrieveInboundPayloads(task);
PollNexusTaskQueueResponseOrBuilder pollResponse = task.getResponse();
// Extract service and operation from the request and set them as MDC and metrics
// scope tags. If the request does not have a service or operation, do not set the tags.
// If we don't know how to handle the task, we will fail the task further down the line.
Scope metricsScope = workerMetricsScope;
String service = getNexusTaskService(pollResponse);
if (!service.isEmpty()) {
MDC.put(LoggerTag.NEXUS_SERVICE, service);
metricsScope = metricsScope.tagged(ImmutableMap.of(MetricsTag.NEXUS_SERVICE, service));
}
String operation = getNexusTaskOperation(pollResponse);
if (!operation.isEmpty()) {
MDC.put(LoggerTag.NEXUS_OPERATION, operation);
metricsScope =
metricsScope.tagged(ImmutableMap.of(MetricsTag.NEXUS_OPERATION, operation));
}
slotSupplier.markSlotUsed(
new NexusSlotInfo(
service, operation, taskQueue, options.getIdentity(), options.getBuildId()),
task.getPermit());

taskFailed = handleNexusTask(task, metricsScope);
} catch (Throwable e) {
taskFailed = true;
Expand Down Expand Up @@ -485,12 +499,13 @@ private void sendReply(
taskResponse = getResponseForOldServer(taskResponse);
}
RespondNexusTaskCompletedRequest request =
RespondNexusTaskCompletedRequest.newBuilder()
.setTaskToken(taskToken)
.setIdentity(options.getIdentity())
.setNamespace(namespace)
.setResponse(taskResponse)
.build();
storeOutbound(
RespondNexusTaskCompletedRequest.newBuilder()
.setTaskToken(taskToken)
.setIdentity(options.getIdentity())
.setNamespace(namespace)
.setResponse(taskResponse)
.build());

grpcRetryer.retry(
() ->
Expand All @@ -512,17 +527,38 @@ private void sendReply(
} else {
request.setError(NexusUtil.handlerErrorToNexusError(handlerException, dataConverter));
}
RespondNexusTaskFailedRequest failedRequest = storeOutbound(request.build());
grpcRetryer.retry(
() ->
service
.blockingStub()
.withOption(METRICS_TAGS_CALL_OPTIONS_KEY, metricsScope)
.respondNexusTaskFailed(request.build()),
.respondNexusTaskFailed(failedRequest),
replyGrpcRetryerOptions);
} else {
throw new IllegalArgumentException("[BUG] Either response or failure must be set");
}
}
}

private NexusTask retrieveInboundPayloads(NexusTask task) {
ExternalStorage externalStorage = options.getExternalStorage();
PollNexusTaskQueueResponseOrBuilder response = task.getResponse();
PollNexusTaskQueueResponse built =
response instanceof PollNexusTaskQueueResponse
? (PollNexusTaskQueueResponse) response
: ((PollNexusTaskQueueResponse.Builder) response).build();
if (externalStorage == null) {
ExternalStorage.throwIfContainsReference(built);
return task;
}
return new NexusTask(
externalStorage.retrieveBlocking(built), task.getPermit(), task.getCompletionCallback());
}

private <T extends Message> T storeOutbound(T request) {
ExternalStorage externalStorage = options.getExternalStorage();
return externalStorage == null ? request : externalStorage.storeBlocking(request, null);
}
}
}
Loading