Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions lib/kernels/include/kernels/batch_matmul_kernels.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,20 @@
#define _FLEXFLOW_LIB_KERNELS_INCLUDE_KERNELS_BATCH_MATMUL_KERNELS_H

#include "kernels/accessor.h"
#include "kernels/device_handle_t.dtg.h"
#include "kernels/device_stream_t.dtg.h"
#include "op-attrs/ops/batch_matmul_attrs.dtg.h"

namespace FlexFlow {

void batch_matmul_forward_kernel(device_stream_t const &stream,
device_handle_t const &handle,
GenericTensorAccessorR const &input_lhs,
GenericTensorAccessorR const &input_rhs,
GenericTensorAccessorW const &output);

void batch_matmul_backward_kernel(device_stream_t const &stream,
device_handle_t const &handle,
GenericTensorAccessorR const &output,
GenericTensorAccessorR const &output_grad,
GenericTensorAccessorR const &input_lhs,
Expand Down
3 changes: 3 additions & 0 deletions lib/kernels/include/kernels/batch_matmul_kernels_gpu.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,20 @@
#define _FLEXFLOW_LIB_KERNELS_INCLUDE_KERNELS_BATCH_MATMUL_KERNELS_GPU_H

#include "kernels/accessor.h"
#include "kernels/device.h"
#include "op-attrs/ops/batch_matmul_attrs.dtg.h"

namespace FlexFlow {

void batch_matmul_gpu_forward_kernel(ffStream_t stream,
PerDeviceFFHandle const &handle,
GenericTensorAccessorR const &input_lhs,
GenericTensorAccessorR const &input_rhs,
GenericTensorAccessorW const &output);

void batch_matmul_gpu_backward_kernel(
ffStream_t stream,
PerDeviceFFHandle const &handle,
GenericTensorAccessorR const &output,
GenericTensorAccessorR const &output_grad,
GenericTensorAccessorR const &input_lhs,
Expand Down
5 changes: 3 additions & 2 deletions lib/kernels/include/kernels/batch_norm_kernels.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ std::optional<BatchNormPerDeviceState>
int output_c,
int output_h,
int output_w,
std::optional<Activation> const &activation);
std::optional<Activation> const &activation,
float eps = 1e-5f);

void forward_kernel(device_stream_t const &stream,
BatchNormPerDeviceState const &per_device_state,
Expand All @@ -40,7 +41,7 @@ void backward_kernel(device_stream_t const &stream,

void cleanup_kernel(DeviceType device_type,
Allocator &allocator,
std::optional<Activation> const &activation);
std::optional<BatchNormPerDeviceState> &per_device_state);

} // namespace FlexFlow::Kernels::BatchNorm
#endif
3 changes: 2 additions & 1 deletion lib/kernels/include/kernels/batch_norm_kernels_gpu.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ BatchNormPerDeviceState
int output_c,
int output_h,
int output_w,
std::optional<Activation> const &activation);
std::optional<Activation> const &activation,
float eps = 1e-5f);

void gpu_forward_kernel(ffStream_t stream,
BatchNormPerDeviceState const &per_device_statem,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,3 +69,7 @@ type = "int"
[[fields]]
name = "activation"
type = "std::optional<::FlexFlow::Activation>"

[[fields]]
name = "eps"
type = "double"
2 changes: 2 additions & 0 deletions lib/kernels/include/kernels/conv_2d_kernels_gpu.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#ifndef _FLEXFLOW_LIB_KERNELS_INCLUDE_KERNELS_CONV_2D_KERNELS_GPU_H
#define _FLEXFLOW_LIB_KERNELS_INCLUDE_KERNELS_CONV_2D_KERNELS_GPU_H

#include "kernels/conv_2d_kernels.h"

namespace FlexFlow {

Conv2DPerDeviceState
Expand Down
11 changes: 8 additions & 3 deletions lib/kernels/include/kernels/create_accessor_with_contents.h
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,8 @@ GenericTensorAccessorW create_4d_accessor_w_with_contents(
type_to_data_type_enum_v<T>,
};

GenericTensorAccessorW accessor = allocator.allocate_tensor(shape);
Allocator cpu_allocator = create_local_cpu_memory_allocator();
GenericTensorAccessorW cpu_accessor = cpu_allocator.allocate_tensor(shape);

for (nonnegative_int dim0_idx :
nonnegative_range(dim0_size.nonnegative_int_from_positive_int())) {
Expand All @@ -165,7 +166,7 @@ GenericTensorAccessorW create_4d_accessor_w_with_contents(
nonnegative_range(dim2_size.nonnegative_int_from_positive_int())) {
for (nonnegative_int dim3_idx :
nonnegative_range(dim3_size.nonnegative_int_from_positive_int())) {
accessor.at<type_to_data_type_enum_v<T>>(TensorDimsCoord{
cpu_accessor.at<type_to_data_type_enum_v<T>>(TensorDimsCoord{
FFOrdered{dim0_idx, dim1_idx, dim2_idx, dim3_idx}}) =
contents.at(dim0_idx.unwrap_nonnegative())
.at(dim1_idx.unwrap_nonnegative())
Expand All @@ -176,7 +177,11 @@ GenericTensorAccessorW create_4d_accessor_w_with_contents(
}
}

return accessor;
GenericTensorAccessorW result = allocator.allocate_tensor(shape);
copy_accessor_data_to_l_from_r_strict(
result, read_only_accessor_from_write_accessor(cpu_accessor));

return result;
}

template <typename T>
Expand Down
5 changes: 3 additions & 2 deletions lib/kernels/include/kernels/pool_2d_kernels.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,9 @@ void pool2d_backward_kernel(
GenericTensorAccessorR const &input,
GenericTensorAccessorW const &input_grad);

void cleanup_kernel(DeviceType device_type,
std::optional<Pool2DPerDeviceState> &per_device_state);
void pool2d_cleanup_kernel(
DeviceType device_type,
std::optional<Pool2DPerDeviceState> &per_device_state);

} // namespace FlexFlow

Expand Down
7 changes: 4 additions & 3 deletions lib/kernels/include/kernels/tensor_accessor_binary_ops.h
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,10 @@ void tensor_accessor_matmul_to(GenericTensorAccessorR const &lhs,
GenericTensorAccessorR const &rhs,
GenericTensorAccessorW const &output);

void tensor_accessor_batch_matmul(GenericTensorAccessorR const &lhs,
GenericTensorAccessorR const &rhs,
GenericTensorAccessorW const &output);
GenericTensorAccessorW
tensor_accessor_batch_matmul(GenericTensorAccessorR const &lhs,
GenericTensorAccessorR const &rhs,
Allocator &output_allocator);

void tensor_accessor_batch_matmul_to(GenericTensorAccessorR const &lhs,
GenericTensorAccessorR const &rhs,
Expand Down
162 changes: 156 additions & 6 deletions lib/kernels/src/cuda/ops/batch_matmul_kernels.cu
Original file line number Diff line number Diff line change
@@ -1,25 +1,175 @@
#include "internal/device.h"
#include "kernels/upsample_kernels_gpu.h"
#include "utils/not_implemented.h"
#include "kernels/batch_matmul_kernels_gpu.h"
#include "op-attrs/ff_ordered/ff_ordered_slice.h"
#include "op-attrs/tensor_dims.h"
#include "utils/containers/product.h"
#include "utils/containers/require_same.h"
#include "utils/containers/vector_of.h"

namespace FlexFlow {

void batch_matmul_gpu_forward_kernel(ffStream_t stream,
// The dimensions of a batched matmul `output[b] = input_lhs[b] * input_rhs[b]`,
// where input_lhs is (batch_size, lhs_rows, inner), input_rhs is (batch_size,
// inner, rhs_cols) and output is (batch_size, lhs_rows, rhs_cols).
//
// Inputs may have more than one leading (batch) dimension, matching
// @ref batch_matmul_get_output_shape. Since the tensors are contiguous and
// row-major, all leading dimensions fold into a single cuBLAS batch count
// without any data movement.
struct BatchMatmulDims {
int batch_size;
int lhs_rows;
int inner;
int rhs_cols;
};

// The dimensions preceding the trailing (rows, cols) pair, which together make
// up the batch of matrices being multiplied.
static FFOrdered<positive_int> get_leading_dims(TensorShape const &shape) {
return ff_ordered_slice(
shape.dims.ff_ordered, relative_ff_dim_t{0}, relative_ff_dim_t{-2});
}

static BatchMatmulDims get_batch_matmul_dims(TensorShape const &lhs_shape,
TensorShape const &rhs_shape,
TensorShape const &output_shape) {
ASSERT(get_num_dims(lhs_shape.dims) >= num_tensor_dims_t{3_n},
"BatchMatmul expects tensors of at least 3 dimensions",
lhs_shape);

require_same(lhs_shape.data_type, rhs_shape.data_type);
ASSERT(require_same(lhs_shape.data_type, output_shape.data_type) ==
DataType::FLOAT,
"BatchMatmul currently only supports data_type = FLOAT. "
"If you need this feature, please create an issue.",
lhs_shape.data_type);

FFOrdered<positive_int> leading_dims =
require_same(get_leading_dims(lhs_shape),
get_leading_dims(rhs_shape),
get_leading_dims(output_shape));

positive_int batch_size = product(vector_of(leading_dims));

positive_int lhs_rows =
require_same(dim_at_idx(lhs_shape.dims, relative_ff_dim_t{-2}),
dim_at_idx(output_shape.dims, relative_ff_dim_t{-2}));

positive_int inner =
require_same(dim_at_idx(lhs_shape.dims, relative_ff_dim_t{-1}),
dim_at_idx(rhs_shape.dims, relative_ff_dim_t{-2}));

positive_int rhs_cols =
require_same(dim_at_idx(rhs_shape.dims, relative_ff_dim_t{-1}),
dim_at_idx(output_shape.dims, relative_ff_dim_t{-1}));

return BatchMatmulDims{
/*batch_size=*/batch_size.int_from_positive_int(),
/*lhs_rows=*/lhs_rows.int_from_positive_int(),
/*inner=*/inner.int_from_positive_int(),
/*rhs_cols=*/rhs_cols.int_from_positive_int(),
};
}

void batch_matmul_gpu_forward_kernel(cudaStream_t stream,
PerDeviceFFHandle const &handle,
GenericTensorAccessorR const &input_lhs,
GenericTensorAccessorR const &input_rhs,
GenericTensorAccessorW const &output) {
NOT_IMPLEMENTED();
BatchMatmulDims dims =
get_batch_matmul_dims(input_lhs.shape, input_rhs.shape, output.shape);

checkCUBLAS(cublasSetStream(handle.blas, stream));

// Our tensors are row-major but cuBLAS is column-major, so we compute
// `output^T = input_rhs^T * input_lhs^T` instead. A row-major (r, c) matrix
// is bit-for-bit a column-major (c, r) matrix, so this needs no transposes
// and no data movement: it is just a matter of swapping the operands.
float alpha = 1.0f, beta = 0.0f;
checkCUBLAS(cublasSgemmStridedBatched(handle.blas,
CUBLAS_OP_N,
CUBLAS_OP_N,
/*m=*/dims.rhs_cols,
/*n=*/dims.lhs_rows,
/*k=*/dims.inner,
&alpha,
/*A=*/input_rhs.get_float_ptr(),
/*lda=*/dims.rhs_cols,
/*strideA=*/dims.inner * dims.rhs_cols,
/*B=*/input_lhs.get_float_ptr(),
/*ldb=*/dims.inner,
/*strideB=*/dims.lhs_rows * dims.inner,
&beta,
/*C=*/output.get_float_ptr(),
/*ldc=*/dims.rhs_cols,
/*strideC=*/
dims.lhs_rows * dims.rhs_cols,
/*batchCount=*/dims.batch_size));
}

void batch_matmul_gpu_backward_kernel(
ffStream_t stream,
cudaStream_t stream,
PerDeviceFFHandle const &handle,
GenericTensorAccessorR const &output,
GenericTensorAccessorR const &output_grad,
GenericTensorAccessorR const &input_lhs,
GenericTensorAccessorW const &input_lhs_grad,
GenericTensorAccessorR const &input_rhs,
GenericTensorAccessorW const &input_rhs_grad) {
NOT_IMPLEMENTED();
require_same(input_lhs.shape, input_lhs_grad.shape);
require_same(input_rhs.shape, input_rhs_grad.shape);
require_same(output.shape, output_grad.shape);

BatchMatmulDims dims = get_batch_matmul_dims(
input_lhs.shape, input_rhs.shape, output_grad.shape);

checkCUBLAS(cublasSetStream(handle.blas, stream));

// NOTE: beta is 0 so that the gradients are overwritten rather than
// accumulated into, matching batch_matmul_cpu_backward_kernel.
float alpha = 1.0f, beta = 0.0f;

// input_lhs_grad = output_grad * input_rhs^T
checkCUBLAS(cublasSgemmStridedBatched(handle.blas,
CUBLAS_OP_T,
CUBLAS_OP_N,
/*m=*/dims.inner,
/*n=*/dims.lhs_rows,
/*k=*/dims.rhs_cols,
&alpha,
/*A=*/input_rhs.get_float_ptr(),
/*lda=*/dims.rhs_cols,
/*strideA=*/dims.inner * dims.rhs_cols,
/*B=*/output_grad.get_float_ptr(),
/*ldb=*/dims.rhs_cols,
/*strideB=*/
dims.lhs_rows * dims.rhs_cols,
&beta,
/*C=*/input_lhs_grad.get_float_ptr(),
/*ldc=*/dims.inner,
/*strideC=*/dims.lhs_rows * dims.inner,
/*batchCount=*/dims.batch_size));

// input_rhs_grad = input_lhs^T * output_grad
checkCUBLAS(cublasSgemmStridedBatched(handle.blas,
CUBLAS_OP_N,
CUBLAS_OP_T,
/*m=*/dims.rhs_cols,
/*n=*/dims.inner,
/*k=*/dims.lhs_rows,
&alpha,
/*A=*/output_grad.get_float_ptr(),
/*lda=*/dims.rhs_cols,
/*strideA=*/
dims.lhs_rows * dims.rhs_cols,
/*B=*/input_lhs.get_float_ptr(),
/*ldb=*/dims.inner,
/*strideB=*/dims.lhs_rows * dims.inner,
&beta,
/*C=*/input_rhs_grad.get_float_ptr(),
/*ldc=*/dims.rhs_cols,
/*strideC=*/dims.inner * dims.rhs_cols,
/*batchCount=*/dims.batch_size));
}

} // namespace FlexFlow
Loading
Loading