Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
e6889a8
Writing some intermediate tests for the multi-consumer pattern.
elliottslaughter Aug 20, 2026
8c50add
Add the subgradient ID.
elliottslaughter Aug 20, 2026
b2e314f
Tweak subgradient ID representation.
elliottslaughter Aug 20, 2026
5eae56b
More work on gradient reduction.
elliottslaughter Aug 21, 2026
74b9915
Fix up test case.
elliottslaughter Aug 21, 2026
32c30ce
Cleanup.
elliottslaughter Aug 21, 2026
7262ff4
Cleanup.
elliottslaughter Aug 21, 2026
bf9f779
Fix doxygen reference.
elliottslaughter Aug 21, 2026
7efcac0
Add a failing e2e test for gradient reduction.
elliottslaughter Aug 21, 2026
484d2de
Better formatting.
elliottslaughter Aug 22, 2026
90be077
Checkpoint.
elliottslaughter Aug 25, 2026
295bea6
Restructure pass expansion implementation.
elliottslaughter Aug 25, 2026
e9c68bc
Cleanup.
elliottslaughter Aug 26, 2026
b7b0c92
Work on mapping.
elliottslaughter Aug 26, 2026
d7db98b
Cleanup.
elliottslaughter Aug 26, 2026
1026ad2
Update test case.
elliottslaughter Aug 26, 2026
da55dd4
Fixes for pass expansion.
elliottslaughter Aug 26, 2026
f224ec6
Cleanup.
elliottslaughter Aug 26, 2026
7a43fba
Actually invoke mapping.
elliottslaughter Aug 26, 2026
5377009
Don't compute mapping if not present.
elliottslaughter Aug 26, 2026
1e6c58c
Header cleanup.
elliottslaughter Aug 26, 2026
0e9021f
Fix build after rebase.
elliottslaughter Sep 16, 2026
c84ee2e
Apply fixes in response to review feedback.
elliottslaughter Nov 13, 2026
0d571ec
More PR review response.
elliottslaughter Nov 13, 2026
3f78d4b
More fixes in response to review feedback.
elliottslaughter Nov 13, 2026
b2046a5
Clean up unneeded headers.
elliottslaughter Nov 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions lib/realm-execution/src/realm-execution/pcg_instance.cc
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,7 @@ static Realm::Event spawn_dynamic_node_invocation(
},
[&](LossAttrs const &) { return spawn_task(); },
[&](CopyAttrs const &) { return issue_copy(); },
[&](GradientReductionAttrs const &) { return issue_reduction(); },
});
}

Expand Down
243 changes: 243 additions & 0 deletions lib/realm-execution/test/src/realm-execution/test_e2e.cc
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,144 @@ MappedParallelComputationGraph
return mpcg;
}

MappedParallelComputationGraph
make_test_gradient_reduction_mpcg_for_device_type(DeviceType device_type) {
positive_int batch_size = 10_p;
positive_int data_dim = 16_p;
positive_int hidden_dim = 32_p;
positive_int output_dim = 1_p;

TensorShape output_tensor_shape = TensorShape{
TensorDims{FFOrdered{batch_size, output_dim}}, DataType::FLOAT};

TensorShape label_tensor_shape = TensorShape{
TensorDims{FFOrdered{batch_size, output_dim}}, DataType::FLOAT};

ParallelComputationGraph pcg = empty_parallel_computation_graph();

TensorShape input_tensor_shape =
TensorShape{TensorDims{FFOrdered{batch_size, data_dim}}, DataType::FLOAT};

ParallelLayerAddedResult inputs_layer =
pcg_add_input_layer(pcg, input_tensor_shape, CreateGrad::YES);
parallel_tensor_guid_t t_input =
require_only_key(inputs_layer.outputs, TensorSlotName::OUTPUT);

ParallelLayerAddedResult relu_operator_1 =
add_parallel_layer(pcg,
make_layer_attrs(make_relu_attrs()),
{
{
TensorSlotName::INPUT,
t_input,
},
},
/*weights=*/{});

parallel_tensor_guid_t t_relu_1 =
require_only_key(relu_operator_1.outputs, TensorSlotName::OUTPUT);

ParallelLayerAddedResult relu_operator_2 =
add_parallel_layer(pcg,
make_layer_attrs(make_relu_attrs()),
{
{
TensorSlotName::INPUT,
t_relu_1,
},
},
/*weights=*/{});

parallel_tensor_guid_t t_relu_2 =
require_only_key(relu_operator_2.outputs, TensorSlotName::OUTPUT);

ParallelLayerAddedResult relu_operator_3 =
add_parallel_layer(pcg,
make_layer_attrs(make_relu_attrs()),
{
{
TensorSlotName::INPUT,
t_relu_1,
},
},
/*weights=*/{});

parallel_tensor_guid_t t_relu_3 =
require_only_key(relu_operator_3.outputs, TensorSlotName::OUTPUT);

MachineSpaceCoordinate mc0{0_n, 0_n};
MachineSpaceCoordinate mc1{0_n, 1_n};
MachineSpaceCoordinate mc2{0_n, 2_n};

ParallelTensorSpaceCoordinate tensor_coord0{
/*sum_component=*/0_n,
/*discard_copy_component=*/0_n,
/*shard_component=*/FFOrdered{0_n}};

MappedParallelComputationGraph mpcg =
mapped_pcg_from_pcg_and_mapped_op_task_groups(
/*pcg=*/pcg,
/*mapped_op_task_groups=*/{
{
inputs_layer.parallel_layer,
MappedOperatorTaskGroup{
{
{
mc0,
OperatorAtomicTaskShardBinding{{
{TensorSlotName::OUTPUT, tensor_coord0},
}},
},
},
},
},
{
relu_operator_1.parallel_layer,
MappedOperatorTaskGroup{
{
{
mc0,
OperatorAtomicTaskShardBinding{{
{TensorSlotName::INPUT, tensor_coord0},
{TensorSlotName::OUTPUT, tensor_coord0},
}},
},
},
},
},
{
relu_operator_2.parallel_layer,
MappedOperatorTaskGroup{
{
{
mc1,
OperatorAtomicTaskShardBinding{{
{TensorSlotName::INPUT, tensor_coord0},
{TensorSlotName::OUTPUT, tensor_coord0},
}},
},
},
},
},
{
relu_operator_3.parallel_layer,
MappedOperatorTaskGroup{
{
{
mc2,
OperatorAtomicTaskShardBinding{{
{TensorSlotName::INPUT, tensor_coord0},
{TensorSlotName::OUTPUT, tensor_coord0},
}},
},
},
},
},
});

return mpcg;
}

TEST_SUITE(FF_TEST_SUITE) {
TEST_CASE("RealmBackend e2e Training (CPU Model Parallelism)") {
std::vector<char *> fake_args =
Expand Down Expand Up @@ -533,6 +671,58 @@ TEST_SUITE(FF_TEST_SUITE) {
});
result.wait();
}

TEST_CASE("RealmBackend e2e Training Gradient Reduction Op (CPU Model "
"Parallelism)") {
std::vector<char *> fake_args =
make_fake_realm_args(/*num_cpus=*/3_p, /*num_gpus=*/0_n);
int fake_argc = fake_args.size();
char **fake_argv = fake_args.data();

RealmManager manager = RealmManager{&fake_argc, &fake_argv};
ControllerTaskResult result =
manager.start_controller([](RealmContext &ctx) {
Allocator allocator = ctx.get_current_device_allocator();

MappedParallelComputationGraph mpcg =
make_test_gradient_reduction_mpcg_for_device_type(
DeviceType::CPU);

std::map<DynamicValueAttrs, DynamicTensorAccessor> input_tensors;

OptimizerAttrs optimizer_attrs = OptimizerAttrs{
SGDOptimizerAttrs{
/*lr=*/0.001,
/*momentum=*/0.9,
/*nesterov=*/false,
/*weight_decay=*/0.001,
},
};

DistributedFfHandle device_handle = create_distributed_ff_handle(
ctx,
/*workSpaceSize=*/1024 * 1024,
/*allowTensorOpMathConversion=*/true);

PCGInstance pcg_instance = create_pcg_instance(
/*ctx=*/ctx,
/*mpcg=*/mpcg,
/*optimizer=*/optimizer_attrs,
/*loss=*/std::nullopt,
/*input_tensors=*/input_tensors,
/*device_handle=*/device_handle,
/*device_type=*/DeviceType::CPU);

// begin training loop
int num_epochs = 1;
for (int i = 0; i < num_epochs; i++) {
perform_all_passes_for_pcg_instance(
/*instance=*/pcg_instance,
/*device_handle=*/device_handle);
}
});
result.wait();
}
}

TEST_SUITE(FF_CUDA_TEST_SUITE) {
Expand Down Expand Up @@ -664,6 +854,59 @@ TEST_SUITE(FF_CUDA_TEST_SUITE) {
});
result.wait();
}

TEST_CASE("RealmBackend e2e Training Gradient Reduction Op (GPU Model "
"Parallelism)") {
std::vector<char *> fake_args =
make_fake_realm_args(/*num_cpus=*/1_p, /*num_gpus=*/3_n);
int fake_argc = fake_args.size();
char **fake_argv = fake_args.data();

RealmManager manager = RealmManager{&fake_argc, &fake_argv};

ControllerTaskResult result =
manager.start_controller([](RealmContext &ctx) {
Allocator allocator = ctx.get_current_device_allocator();

MappedParallelComputationGraph mpcg =
make_test_gradient_reduction_mpcg_for_device_type(
DeviceType::GPU);

OptimizerAttrs optimizer_attrs = OptimizerAttrs{
SGDOptimizerAttrs{
/*lr=*/0.001,
/*momentum=*/0.9,
/*nesterov=*/false,
/*weight_decay=*/0.001,
},
};

std::map<DynamicValueAttrs, DynamicTensorAccessor> input_tensors;

DistributedFfHandle device_handle = create_distributed_ff_handle(
ctx,
/*workSpaceSize=*/1024 * 1024,
/*allowTensorOpMathConversion=*/true);

PCGInstance pcg_instance = create_pcg_instance(
/*ctx=*/ctx,
/*mpcg=*/mpcg,
/*optimizer=*/optimizer_attrs,
/*loss=*/std::nullopt,
/*input_tensors=*/input_tensors,
/*device_handle=*/device_handle,
/*device_type=*/DeviceType::GPU);

// begin training loop
int num_epochs = 1;
for (int i = 0; i < num_epochs; i++) {
perform_all_passes_for_pcg_instance(
/*instance=*/pcg_instance,
/*device_handle=*/device_handle);
}
});
result.wait();
}
}

} // namespace test
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
namespace = "FlexFlow"
name = "dynamic_gradient_reduction_layer_guid_t"
type = "struct"
features = [
"eq",
"ord",
"hash",
"json",
"fmt",
"rapidcheck",
]

fields = []
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ includes = [
"pcg/parallel_computation_graph/parallel_layer_guid_t.dtg.h",
"task-spec/dynamic_graph/dynamic_loss_layer_guid_t.dtg.h",
"task-spec/dynamic_graph/dynamic_copy_layer_guid_t.dtg.h",
"task-spec/dynamic_graph/dynamic_gradient_reduction_layer_guid_t.dtg.h",
]

[[values]]
Expand All @@ -31,3 +32,7 @@ key = "loss_layer_guid"
[[values]]
type = "::FlexFlow::dynamic_copy_layer_guid_t"
key = "copy_layer_guid"

[[values]]
type = "::FlexFlow::dynamic_gradient_reduction_layer_guid_t"
key = "gradient_reduction_layer_guid"
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
#define _FLEXFLOW_LIB_TASK_SPEC_INCLUDE_TASK_SPEC_DYNAMIC_GRAPH_DYNAMIC_NODE_MAPPING_H

#include "task-spec/dynamic_graph/dynamic_node_mapping.dtg.h"
#include "task-spec/dynamic_graph/dynamic_tensor_slot.dtg.h"
#include "task-spec/dynamic_graph/parallel_tensor_mapping.dtg.h"
#include "task-spec/global_device_id_t.dtg.h"

namespace FlexFlow {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ includes = [
"<optional>",
"task-spec/dynamic_graph/dynamic_tensor_guid_t.dtg.h",
"op-attrs/parallel_tensor_shape.dtg.h",
"task-spec/dynamic_graph/subgradient_id_t.dtg.h",
"op-attrs/parallel_tensor_space_coordinate.dtg.h",
"pcg/machine_space_coordinate.dtg.h",
"utils/bidict/bidict.h",
Expand Down Expand Up @@ -48,6 +49,15 @@ For a \ref DynamicOpenDataflowGraph originating form a \ref MappedParallelComput
name = "create_grad"
type = "std::optional<bool>"

[[fields]]
name = "subgradient_id"
type = "std::optional<::FlexFlow::subgradient_id_t>"
docstring = '''
\brief The unique identity of the subgradient represented by this value.

This field is filled in by \ref pass_expansion.h iff a value is consumed multiple times, requiring a \ref GradientReductionAttrs in the backward pass.
'''

[[fields]]
name = "shard_coord"
type = "std::optional<::FlexFlow::ParallelTensorSpaceCoordinate>"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,17 @@

#include "task-spec/dynamic_graph/dynamic_value_attrs.dtg.h"
#include "task-spec/dynamic_graph/parallel_tensor_mapping.dtg.h"
#include "task-spec/dynamic_graph/subgradient_id_t.dtg.h"

namespace FlexFlow {

DynamicValueAttrs decide_dynamic_value_attrs_role(DynamicValueAttrs const &,
DynamicTensorRole);

DynamicValueAttrs
decide_dynamic_value_attrs_subgradient_id(DynamicValueAttrs const &,
subgradient_id_t const &);

DynamicValueAttrs
decide_dynamic_value_attrs_mapping(DynamicValueAttrs const &,
ParallelTensorMapping const &);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
namespace = "FlexFlow"
name = "GradientReductionAttrs"
type = "struct"
features = [
"eq",
"ord",
"hash",
"json",
"fmt",
"rapidcheck",
]

fields = []
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name = "InternalDynamicSlotSite"
type = "struct"
features = [
"eq",
"ord",
"ord",
"hash",
"fmt",
"json",
Expand Down
Loading
Loading