Skip to content

External Storage Integration: WorkflowWorker, replay, history #3017

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cconstable wants to merge 1 commit into
base: extstore/foundation
Choose a base branch
from
Open

External Storage Integration: WorkflowWorker, replay, history #3017

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import io.temporal.common.converter.DataConverter;
import io.temporal.internal.common.ProtobufTimeUtils;
import io.temporal.internal.common.WorkflowExecutionUtils;
import io.temporal.internal.payload.storage.ExternalStorage;
import io.temporal.internal.worker.*;
import io.temporal.payload.context.WorkflowSerializationContext;
import io.temporal.serviceclient.MetricsTag;
Expand Down Expand Up @@ -77,6 +78,12 @@ public WorkflowTaskHandler.Result handleWorkflowTask(PollWorkflowTaskQueueRespon
String workflowType = workflowTask.getWorkflowType().getName();
Scope metricsScope =
options.getMetricsScope().tagged(ImmutableMap.of(MetricsTag.WORKFLOW_TYPE, workflowType));
ExternalStorage externalStorage = options.getExternalStorage();
if (externalStorage == null) {
ExternalStorage.throwIfContainsReference(workflowTask);
} else {
workflowTask = externalStorage.retrieveBlocking(workflowTask);
}
return handleWorkflowTaskWithQuery(workflowTask.toBuilder(), metricsScope);
}

Expand All @@ -94,7 +101,8 @@ private Result handleWorkflowTaskWithQuery(
logWorkflowTaskToBeProcessed(workflowTask, createdNew);

ServiceWorkflowHistoryIterator historyIterator =
new ServiceWorkflowHistoryIterator(service, namespace, workflowTask, metricsScope);
new ServiceWorkflowHistoryIterator(
service, namespace, workflowTask, metricsScope, options.getExternalStorage());
boolean finalCommand;
Result result;

Expand Down
21 changes: 20 additions & 1 deletion temporal-sdk/src/main/java/io/temporal/internal/replay/ServiceWorkflowHistoryIterator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@
import io.temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryRequest;
import io.temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryResponse;
import io.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponseOrBuilder;
import io.temporal.internal.payload.storage.ExternalStorage;
import io.temporal.internal.retryer.GrpcRetryer;
import io.temporal.serviceclient.RpcRetryOptions;
import io.temporal.serviceclient.WorkflowServiceStubs;
import java.time.Duration;
import java.util.Iterator;
import java.util.NoSuchElementException;
import javax.annotation.Nullable;

/** Supports iteration over history while loading new pages through calls to the service. */
class ServiceWorkflowHistoryIterator implements WorkflowHistoryIterator {
Expand All @@ -29,6 +31,7 @@ class ServiceWorkflowHistoryIterator implements WorkflowHistoryIterator {
private final Scope metricsScope;
private final PollWorkflowTaskQueueResponseOrBuilder task;
private final GrpcRetryer grpcRetryer;
private final @Nullable ExternalStorage externalStorage;
private Deadline deadline;
private Iterator<HistoryEvent> current;
ByteString nextPageToken;
Expand All @@ -38,10 +41,20 @@ class ServiceWorkflowHistoryIterator implements WorkflowHistoryIterator {
String namespace,
PollWorkflowTaskQueueResponseOrBuilder task,
Scope metricsScope) {
this(service, namespace, task, metricsScope, null);
}

ServiceWorkflowHistoryIterator(
WorkflowServiceStubs service,
String namespace,
PollWorkflowTaskQueueResponseOrBuilder task,
Scope metricsScope,
@Nullable ExternalStorage externalStorage) {
this.service = service;
this.namespace = namespace;
this.task = task;
this.metricsScope = metricsScope;
this.externalStorage = externalStorage;
// TODO Refactor WorkflowHistoryIteratorTest or WorkflowHistoryIterator to remove this check.
// `service == null` shouldn't be allowed as it's needed for a normal functioning of this
// class.
Expand All @@ -64,7 +77,13 @@ public boolean hasNext() {
// true.
GetWorkflowExecutionHistoryResponse response = queryWorkflowExecutionHistory();

current = response.getHistory().getEventsList().iterator();
History history = response.getHistory();
if (externalStorage == null) {
ExternalStorage.throwIfContainsReference(history);
} else {
history = externalStorage.retrieveBlocking(history);
}
current = history.getEventsList().iterator();
nextPageToken = response.getNextPageToken();
// Server can return an empty page, but a valid nextPageToken that contains
// more events.
Expand Down
100 changes: 89 additions & 11 deletions temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,14 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.protobuf.ByteString;
import com.google.protobuf.MessageOrBuilder;
import com.uber.m3.tally.Scope;
import com.uber.m3.tally.Stopwatch;
import com.uber.m3.util.ImmutableMap;
import io.grpc.StatusRuntimeException;
import io.temporal.api.command.v1.ScheduleActivityTaskCommandAttributesOrBuilder;
import io.temporal.api.command.v1.SignalExternalWorkflowExecutionCommandAttributesOrBuilder;
import io.temporal.api.command.v1.StartChildWorkflowExecutionCommandAttributesOrBuilder;
import io.temporal.api.common.v1.WorkflowExecution;
import io.temporal.api.enums.v1.QueryResultType;
import io.temporal.api.enums.v1.TaskQueueKind;
Expand All @@ -18,9 +22,14 @@
import io.temporal.api.workflowservice.v1.*;
import io.temporal.failure.ApplicationFailure;
import io.temporal.internal.logging.LoggerTag;
import io.temporal.internal.payload.storage.ExternalStorage;
import io.temporal.internal.payload.visitor.MessageVisitor;
import io.temporal.internal.retryer.GrpcMessageTooLargeException;
import io.temporal.internal.retryer.GrpcRetryer;
import io.temporal.payload.context.WorkflowSerializationContext;
import io.temporal.payload.storage.StorageDriverActivityInfo;
import io.temporal.payload.storage.StorageDriverTargetInfo;
import io.temporal.payload.storage.StorageDriverWorkflowInfo;
import io.temporal.serviceclient.MetricsTag;
import io.temporal.serviceclient.RpcRetryOptions;
import io.temporal.serviceclient.WorkflowServiceStubs;
Expand Down Expand Up @@ -381,6 +390,55 @@ public String toString() {
options.getIdentity(), namespace, taskQueue);
}

private <T extends com.google.protobuf.Message> T storeOutboundPayloads(
T request, @Nullable StorageDriverTargetInfo target) {
ExternalStorage externalStorage = options.getExternalStorage();
return externalStorage == null ? request : externalStorage.storeBlocking(request, target);
}

private <T extends com.google.protobuf.Message> T storeOutboundPayloads(
T request,
@Nullable StorageDriverTargetInfo target,
MessageVisitor<StorageDriverTargetInfo> targetVisitor) {
ExternalStorage externalStorage = options.getExternalStorage();
return externalStorage == null
? request
: externalStorage.storeBlocking(request, target, targetVisitor);
}

@Nullable
private StorageDriverTargetInfo workflowStorageTarget(
WorkflowExecution execution, String workflowType) {
if (options.getExternalStorage() == null) {
return null;
}
return new StorageDriverWorkflowInfo(
namespace, execution.getWorkflowId(), execution.getRunId(), workflowType);
}

static StorageDriverTargetInfo refineStorageTarget(
String namespace, StorageDriverTargetInfo current, MessageOrBuilder message) {
if (message instanceof ScheduleActivityTaskCommandAttributesOrBuilder) {
ScheduleActivityTaskCommandAttributesOrBuilder attrs =
(ScheduleActivityTaskCommandAttributesOrBuilder) message;
return new StorageDriverActivityInfo(
namespace, attrs.getActivityId(), null, attrs.getActivityType().getName());
}
if (message instanceof StartChildWorkflowExecutionCommandAttributesOrBuilder) {
StartChildWorkflowExecutionCommandAttributesOrBuilder attrs =
(StartChildWorkflowExecutionCommandAttributesOrBuilder) message;
return new StorageDriverWorkflowInfo(
namespace, attrs.getWorkflowId(), null, attrs.getWorkflowType().getName());
}
if (message instanceof SignalExternalWorkflowExecutionCommandAttributesOrBuilder) {
WorkflowExecution execution =
((SignalExternalWorkflowExecutionCommandAttributesOrBuilder) message).getExecution();
return new StorageDriverWorkflowInfo(
namespace, execution.getWorkflowId(), execution.getRunId(), null);
}
return current;
}

private class TaskHandlerImpl implements PollTaskExecutor.TaskHandler<WorkflowTask> {

final WorkflowTaskHandler handler;
Expand Down Expand Up @@ -453,7 +511,10 @@ public void handle(WorkflowTask task) throws Exception {
if (queryCompleted != null) {
try {
sendDirectQueryCompletedResponse(
currentTask.getTaskToken(), queryCompleted.toBuilder(), workflowTypeScope);
currentTask.getTaskToken(),
queryCompleted.toBuilder(),
workflowTypeScope,
workflowStorageTarget(workflowExecution, workflowType));
} catch (StatusRuntimeException e) {
GrpcMessageTooLargeException tooLargeException =
GrpcMessageTooLargeException.tryWrap(e);
Expand All @@ -473,7 +534,10 @@ public void handle(WorkflowTask task) throws Exception {
.setErrorMessage(failure.getMessage())
.setFailure(failure);
sendDirectQueryCompletedResponse(
currentTask.getTaskToken(), queryFailedBuilder, workflowTypeScope);
currentTask.getTaskToken(),
queryFailedBuilder,
workflowTypeScope,
workflowStorageTarget(workflowExecution, workflowType));
}
} else {
try {
Expand All @@ -489,7 +553,8 @@ public void handle(WorkflowTask task) throws Exception {
currentTask.getTaskToken(),
requestBuilder,
result.getRequestRetryOptions(),
workflowTypeScope);
workflowTypeScope,
workflowStorageTarget(workflowExecution, workflowType));
// If we were processing a speculative WFT the server may instruct us that the
// task was dropped by resting out event ID.
long resetEventId = response.getResetHistoryEventId();
Expand All @@ -509,7 +574,8 @@ public void handle(WorkflowTask task) throws Exception {
currentTask.getTaskToken(),
taskFailed.toBuilder(),
result.getRequestRetryOptions(),
workflowTypeScope);
workflowTypeScope,
workflowStorageTarget(workflowExecution, workflowType));
}

// Apply post-completion metrics only if runnable present and the above succeeded
Expand Down Expand Up @@ -546,7 +612,8 @@ public void handle(WorkflowTask task) throws Exception {
currentTask.getTaskToken(),
taskFailedBuilder,
result.getRequestRetryOptions(),
workflowTypeScope);
workflowTypeScope,
workflowStorageTarget(workflowExecution, workflowType));
}
}
} catch (Exception e) {
Expand Down Expand Up @@ -651,7 +718,8 @@ private RespondWorkflowTaskCompletedResponse sendTaskCompleted(
ByteString taskToken,
RespondWorkflowTaskCompletedRequest.Builder taskCompleted,
RpcRetryOptions retryOptions,
Scope workflowTypeMetricsScope) {
Scope workflowTypeMetricsScope,
@Nullable StorageDriverTargetInfo storageTarget) {
GrpcRetryer.GrpcRetryerOptions grpcRetryOptions =
new GrpcRetryer.GrpcRetryerOptions(
RpcRetryOptions.newBuilder().buildWithDefaultsFrom(retryOptions), null);
Expand All @@ -674,12 +742,16 @@ private RespondWorkflowTaskCompletedResponse sendTaskCompleted(
taskCompleted.setBinaryChecksum(options.getBuildId());
}

MessageVisitor<StorageDriverTargetInfo> storageTargetVisitor =
(current, message) -> refineStorageTarget(namespace, current, message);
RespondWorkflowTaskCompletedRequest request =
storeOutboundPayloads(taskCompleted.build(), storageTarget, storageTargetVisitor);
return grpcRetryer.retryWithResult(
() ->
service
.blockingStub()
.withOption(METRICS_TAGS_CALL_OPTIONS_KEY, workflowTypeMetricsScope)
.respondWorkflowTaskCompleted(taskCompleted.build()),
.respondWorkflowTaskCompleted(request),
grpcRetryOptions);
}

Expand All @@ -688,7 +760,8 @@ private void sendTaskFailed(
ByteString taskToken,
RespondWorkflowTaskFailedRequest.Builder taskFailed,
RpcRetryOptions retryOptions,
Scope workflowTypeMetricsScope) {
Scope workflowTypeMetricsScope,
@Nullable StorageDriverTargetInfo storageTarget) {
GrpcRetryer.GrpcRetryerOptions grpcRetryOptions =
new GrpcRetryer.GrpcRetryerOptions(
RpcRetryOptions.newBuilder().buildWithDefaultsFrom(retryOptions), null);
Expand All @@ -702,25 +775,30 @@ private void sendTaskFailed(
taskFailed.setWorkerVersion(options.workerVersionStamp());
}

RespondWorkflowTaskFailedRequest request =
storeOutboundPayloads(taskFailed.build(), storageTarget);
grpcRetryer.retry(
() ->
service
.blockingStub()
.withOption(METRICS_TAGS_CALL_OPTIONS_KEY, workflowTypeMetricsScope)
.respondWorkflowTaskFailed(taskFailed.build()),
.respondWorkflowTaskFailed(request),
grpcRetryOptions);
}

private void sendDirectQueryCompletedResponse(
ByteString taskToken,
RespondQueryTaskCompletedRequest.Builder queryCompleted,
Scope workflowTypeMetricsScope) {
Scope workflowTypeMetricsScope,
@Nullable StorageDriverTargetInfo storageTarget) {
queryCompleted.setTaskToken(taskToken).setNamespace(namespace);
RespondQueryTaskCompletedRequest request =
storeOutboundPayloads(queryCompleted.build(), storageTarget);
// Do not retry query response
service
.blockingStub()
.withOption(METRICS_TAGS_CALL_OPTIONS_KEY, workflowTypeMetricsScope)
.respondQueryTaskCompleted(queryCompleted.build());
.respondQueryTaskCompleted(request);
}

private void logExceptionDuringResultReporting(
Expand Down
Loading
Loading