diff --git a/lib/kernels/include/kernels/batch_matmul_kernels.h b/lib/kernels/include/kernels/batch_matmul_kernels.h index b2902e3bdf..b6a6b8e970 100644 --- a/lib/kernels/include/kernels/batch_matmul_kernels.h +++ b/lib/kernels/include/kernels/batch_matmul_kernels.h @@ -2,17 +2,20 @@ #define _FLEXFLOW_LIB_KERNELS_INCLUDE_KERNELS_BATCH_MATMUL_KERNELS_H #include "kernels/accessor.h" +#include "kernels/device_handle_t.dtg.h" #include "kernels/device_stream_t.dtg.h" #include "op-attrs/ops/batch_matmul_attrs.dtg.h" namespace FlexFlow { void batch_matmul_forward_kernel(device_stream_t const &stream, + device_handle_t const &handle, GenericTensorAccessorR const &input_lhs, GenericTensorAccessorR const &input_rhs, GenericTensorAccessorW const &output); void batch_matmul_backward_kernel(device_stream_t const &stream, + device_handle_t const &handle, GenericTensorAccessorR const &output, GenericTensorAccessorR const &output_grad, GenericTensorAccessorR const &input_lhs, diff --git a/lib/kernels/include/kernels/batch_matmul_kernels_gpu.h b/lib/kernels/include/kernels/batch_matmul_kernels_gpu.h index c8bb9432da..83c5390de4 100644 --- a/lib/kernels/include/kernels/batch_matmul_kernels_gpu.h +++ b/lib/kernels/include/kernels/batch_matmul_kernels_gpu.h @@ -2,17 +2,20 @@ #define _FLEXFLOW_LIB_KERNELS_INCLUDE_KERNELS_BATCH_MATMUL_KERNELS_GPU_H #include "kernels/accessor.h" +#include "kernels/device.h" #include "op-attrs/ops/batch_matmul_attrs.dtg.h" namespace FlexFlow { void batch_matmul_gpu_forward_kernel(ffStream_t stream, + PerDeviceFFHandle const &handle, GenericTensorAccessorR const &input_lhs, GenericTensorAccessorR const &input_rhs, GenericTensorAccessorW const &output); void batch_matmul_gpu_backward_kernel( ffStream_t stream, + PerDeviceFFHandle const &handle, GenericTensorAccessorR const &output, GenericTensorAccessorR const &output_grad, GenericTensorAccessorR const &input_lhs, diff --git a/lib/kernels/include/kernels/batch_norm_kernels.h b/lib/kernels/include/kernels/batch_norm_kernels.h index b713452eb5..0cf5c59c6e 100644 --- a/lib/kernels/include/kernels/batch_norm_kernels.h +++ b/lib/kernels/include/kernels/batch_norm_kernels.h @@ -18,7 +18,8 @@ std::optional int output_c, int output_h, int output_w, - std::optional const &activation); + std::optional const &activation, + float eps = 1e-5f); void forward_kernel(device_stream_t const &stream, BatchNormPerDeviceState const &per_device_state, @@ -40,7 +41,7 @@ void backward_kernel(device_stream_t const &stream, void cleanup_kernel(DeviceType device_type, Allocator &allocator, - std::optional const &activation); + std::optional &per_device_state); } // namespace FlexFlow::Kernels::BatchNorm #endif diff --git a/lib/kernels/include/kernels/batch_norm_kernels_gpu.h b/lib/kernels/include/kernels/batch_norm_kernels_gpu.h index 931b471a6c..6d24c26f5a 100644 --- a/lib/kernels/include/kernels/batch_norm_kernels_gpu.h +++ b/lib/kernels/include/kernels/batch_norm_kernels_gpu.h @@ -16,7 +16,8 @@ BatchNormPerDeviceState int output_c, int output_h, int output_w, - std::optional const &activation); + std::optional const &activation, + float eps = 1e-5f); void gpu_forward_kernel(ffStream_t stream, BatchNormPerDeviceState const &per_device_statem, diff --git a/lib/kernels/include/kernels/batch_norm_per_device_state.dtg.toml b/lib/kernels/include/kernels/batch_norm_per_device_state.dtg.toml index 6ee67aca3c..55f7031eda 100644 --- a/lib/kernels/include/kernels/batch_norm_per_device_state.dtg.toml +++ b/lib/kernels/include/kernels/batch_norm_per_device_state.dtg.toml @@ -69,3 +69,7 @@ type = "int" [[fields]] name = "activation" type = "std::optional<::FlexFlow::Activation>" + +[[fields]] +name = "eps" +type = "double" diff --git a/lib/kernels/include/kernels/conv_2d_kernels_gpu.h b/lib/kernels/include/kernels/conv_2d_kernels_gpu.h index 614ae63b05..f2cfe2ef18 100644 --- a/lib/kernels/include/kernels/conv_2d_kernels_gpu.h +++ b/lib/kernels/include/kernels/conv_2d_kernels_gpu.h @@ -1,6 +1,8 @@ #ifndef _FLEXFLOW_LIB_KERNELS_INCLUDE_KERNELS_CONV_2D_KERNELS_GPU_H #define _FLEXFLOW_LIB_KERNELS_INCLUDE_KERNELS_CONV_2D_KERNELS_GPU_H +#include "kernels/conv_2d_kernels.h" + namespace FlexFlow { Conv2DPerDeviceState diff --git a/lib/kernels/include/kernels/create_accessor_with_contents.h b/lib/kernels/include/kernels/create_accessor_with_contents.h index b0417f867d..d445411047 100644 --- a/lib/kernels/include/kernels/create_accessor_with_contents.h +++ b/lib/kernels/include/kernels/create_accessor_with_contents.h @@ -155,7 +155,8 @@ GenericTensorAccessorW create_4d_accessor_w_with_contents( type_to_data_type_enum_v, }; - GenericTensorAccessorW accessor = allocator.allocate_tensor(shape); + Allocator cpu_allocator = create_local_cpu_memory_allocator(); + GenericTensorAccessorW cpu_accessor = cpu_allocator.allocate_tensor(shape); for (nonnegative_int dim0_idx : nonnegative_range(dim0_size.nonnegative_int_from_positive_int())) { @@ -165,7 +166,7 @@ GenericTensorAccessorW create_4d_accessor_w_with_contents( nonnegative_range(dim2_size.nonnegative_int_from_positive_int())) { for (nonnegative_int dim3_idx : nonnegative_range(dim3_size.nonnegative_int_from_positive_int())) { - accessor.at>(TensorDimsCoord{ + cpu_accessor.at>(TensorDimsCoord{ FFOrdered{dim0_idx, dim1_idx, dim2_idx, dim3_idx}}) = contents.at(dim0_idx.unwrap_nonnegative()) .at(dim1_idx.unwrap_nonnegative()) @@ -176,7 +177,11 @@ GenericTensorAccessorW create_4d_accessor_w_with_contents( } } - return accessor; + GenericTensorAccessorW result = allocator.allocate_tensor(shape); + copy_accessor_data_to_l_from_r_strict( + result, read_only_accessor_from_write_accessor(cpu_accessor)); + + return result; } template diff --git a/lib/kernels/include/kernels/pool_2d_kernels.h b/lib/kernels/include/kernels/pool_2d_kernels.h index 9da5a5eff3..9d049bc82d 100644 --- a/lib/kernels/include/kernels/pool_2d_kernels.h +++ b/lib/kernels/include/kernels/pool_2d_kernels.h @@ -34,8 +34,9 @@ void pool2d_backward_kernel( GenericTensorAccessorR const &input, GenericTensorAccessorW const &input_grad); -void cleanup_kernel(DeviceType device_type, - std::optional &per_device_state); +void pool2d_cleanup_kernel( + DeviceType device_type, + std::optional &per_device_state); } // namespace FlexFlow diff --git a/lib/kernels/include/kernels/tensor_accessor_binary_ops.h b/lib/kernels/include/kernels/tensor_accessor_binary_ops.h index b841d51ed2..7cabbefb4c 100644 --- a/lib/kernels/include/kernels/tensor_accessor_binary_ops.h +++ b/lib/kernels/include/kernels/tensor_accessor_binary_ops.h @@ -74,9 +74,10 @@ void tensor_accessor_matmul_to(GenericTensorAccessorR const &lhs, GenericTensorAccessorR const &rhs, GenericTensorAccessorW const &output); -void tensor_accessor_batch_matmul(GenericTensorAccessorR const &lhs, - GenericTensorAccessorR const &rhs, - GenericTensorAccessorW const &output); +GenericTensorAccessorW + tensor_accessor_batch_matmul(GenericTensorAccessorR const &lhs, + GenericTensorAccessorR const &rhs, + Allocator &output_allocator); void tensor_accessor_batch_matmul_to(GenericTensorAccessorR const &lhs, GenericTensorAccessorR const &rhs, diff --git a/lib/kernels/src/cuda/ops/batch_matmul_kernels.cu b/lib/kernels/src/cuda/ops/batch_matmul_kernels.cu index 5209957325..291a9809a4 100644 --- a/lib/kernels/src/cuda/ops/batch_matmul_kernels.cu +++ b/lib/kernels/src/cuda/ops/batch_matmul_kernels.cu @@ -1,25 +1,175 @@ #include "internal/device.h" -#include "kernels/upsample_kernels_gpu.h" -#include "utils/not_implemented.h" +#include "kernels/batch_matmul_kernels_gpu.h" +#include "op-attrs/ff_ordered/ff_ordered_slice.h" +#include "op-attrs/tensor_dims.h" +#include "utils/containers/product.h" +#include "utils/containers/require_same.h" +#include "utils/containers/vector_of.h" namespace FlexFlow { -void batch_matmul_gpu_forward_kernel(ffStream_t stream, +// The dimensions of a batched matmul `output[b] = input_lhs[b] * input_rhs[b]`, +// where input_lhs is (batch_size, lhs_rows, inner), input_rhs is (batch_size, +// inner, rhs_cols) and output is (batch_size, lhs_rows, rhs_cols). +// +// Inputs may have more than one leading (batch) dimension, matching +// @ref batch_matmul_get_output_shape. Since the tensors are contiguous and +// row-major, all leading dimensions fold into a single cuBLAS batch count +// without any data movement. +struct BatchMatmulDims { + int batch_size; + int lhs_rows; + int inner; + int rhs_cols; +}; + +// The dimensions preceding the trailing (rows, cols) pair, which together make +// up the batch of matrices being multiplied. +static FFOrdered get_leading_dims(TensorShape const &shape) { + return ff_ordered_slice( + shape.dims.ff_ordered, relative_ff_dim_t{0}, relative_ff_dim_t{-2}); +} + +static BatchMatmulDims get_batch_matmul_dims(TensorShape const &lhs_shape, + TensorShape const &rhs_shape, + TensorShape const &output_shape) { + ASSERT(get_num_dims(lhs_shape.dims) >= num_tensor_dims_t{3_n}, + "BatchMatmul expects tensors of at least 3 dimensions", + lhs_shape); + + require_same(lhs_shape.data_type, rhs_shape.data_type); + ASSERT(require_same(lhs_shape.data_type, output_shape.data_type) == + DataType::FLOAT, + "BatchMatmul currently only supports data_type = FLOAT. " + "If you need this feature, please create an issue.", + lhs_shape.data_type); + + FFOrdered leading_dims = + require_same(get_leading_dims(lhs_shape), + get_leading_dims(rhs_shape), + get_leading_dims(output_shape)); + + positive_int batch_size = product(vector_of(leading_dims)); + + positive_int lhs_rows = + require_same(dim_at_idx(lhs_shape.dims, relative_ff_dim_t{-2}), + dim_at_idx(output_shape.dims, relative_ff_dim_t{-2})); + + positive_int inner = + require_same(dim_at_idx(lhs_shape.dims, relative_ff_dim_t{-1}), + dim_at_idx(rhs_shape.dims, relative_ff_dim_t{-2})); + + positive_int rhs_cols = + require_same(dim_at_idx(rhs_shape.dims, relative_ff_dim_t{-1}), + dim_at_idx(output_shape.dims, relative_ff_dim_t{-1})); + + return BatchMatmulDims{ + /*batch_size=*/batch_size.int_from_positive_int(), + /*lhs_rows=*/lhs_rows.int_from_positive_int(), + /*inner=*/inner.int_from_positive_int(), + /*rhs_cols=*/rhs_cols.int_from_positive_int(), + }; +} + +void batch_matmul_gpu_forward_kernel(cudaStream_t stream, + PerDeviceFFHandle const &handle, GenericTensorAccessorR const &input_lhs, GenericTensorAccessorR const &input_rhs, GenericTensorAccessorW const &output) { - NOT_IMPLEMENTED(); + BatchMatmulDims dims = + get_batch_matmul_dims(input_lhs.shape, input_rhs.shape, output.shape); + + checkCUBLAS(cublasSetStream(handle.blas, stream)); + + // Our tensors are row-major but cuBLAS is column-major, so we compute + // `output^T = input_rhs^T * input_lhs^T` instead. A row-major (r, c) matrix + // is bit-for-bit a column-major (c, r) matrix, so this needs no transposes + // and no data movement: it is just a matter of swapping the operands. + float alpha = 1.0f, beta = 0.0f; + checkCUBLAS(cublasSgemmStridedBatched(handle.blas, + CUBLAS_OP_N, + CUBLAS_OP_N, + /*m=*/dims.rhs_cols, + /*n=*/dims.lhs_rows, + /*k=*/dims.inner, + &alpha, + /*A=*/input_rhs.get_float_ptr(), + /*lda=*/dims.rhs_cols, + /*strideA=*/dims.inner * dims.rhs_cols, + /*B=*/input_lhs.get_float_ptr(), + /*ldb=*/dims.inner, + /*strideB=*/dims.lhs_rows * dims.inner, + &beta, + /*C=*/output.get_float_ptr(), + /*ldc=*/dims.rhs_cols, + /*strideC=*/ + dims.lhs_rows * dims.rhs_cols, + /*batchCount=*/dims.batch_size)); } void batch_matmul_gpu_backward_kernel( - ffStream_t stream, + cudaStream_t stream, + PerDeviceFFHandle const &handle, GenericTensorAccessorR const &output, GenericTensorAccessorR const &output_grad, GenericTensorAccessorR const &input_lhs, GenericTensorAccessorW const &input_lhs_grad, GenericTensorAccessorR const &input_rhs, GenericTensorAccessorW const &input_rhs_grad) { - NOT_IMPLEMENTED(); + require_same(input_lhs.shape, input_lhs_grad.shape); + require_same(input_rhs.shape, input_rhs_grad.shape); + require_same(output.shape, output_grad.shape); + + BatchMatmulDims dims = get_batch_matmul_dims( + input_lhs.shape, input_rhs.shape, output_grad.shape); + + checkCUBLAS(cublasSetStream(handle.blas, stream)); + + // NOTE: beta is 0 so that the gradients are overwritten rather than + // accumulated into, matching batch_matmul_cpu_backward_kernel. + float alpha = 1.0f, beta = 0.0f; + + // input_lhs_grad = output_grad * input_rhs^T + checkCUBLAS(cublasSgemmStridedBatched(handle.blas, + CUBLAS_OP_T, + CUBLAS_OP_N, + /*m=*/dims.inner, + /*n=*/dims.lhs_rows, + /*k=*/dims.rhs_cols, + &alpha, + /*A=*/input_rhs.get_float_ptr(), + /*lda=*/dims.rhs_cols, + /*strideA=*/dims.inner * dims.rhs_cols, + /*B=*/output_grad.get_float_ptr(), + /*ldb=*/dims.rhs_cols, + /*strideB=*/ + dims.lhs_rows * dims.rhs_cols, + &beta, + /*C=*/input_lhs_grad.get_float_ptr(), + /*ldc=*/dims.inner, + /*strideC=*/dims.lhs_rows * dims.inner, + /*batchCount=*/dims.batch_size)); + + // input_rhs_grad = input_lhs^T * output_grad + checkCUBLAS(cublasSgemmStridedBatched(handle.blas, + CUBLAS_OP_N, + CUBLAS_OP_T, + /*m=*/dims.rhs_cols, + /*n=*/dims.inner, + /*k=*/dims.lhs_rows, + &alpha, + /*A=*/output_grad.get_float_ptr(), + /*lda=*/dims.rhs_cols, + /*strideA=*/ + dims.lhs_rows * dims.rhs_cols, + /*B=*/input_lhs.get_float_ptr(), + /*ldb=*/dims.inner, + /*strideB=*/dims.lhs_rows * dims.inner, + &beta, + /*C=*/input_rhs_grad.get_float_ptr(), + /*ldc=*/dims.rhs_cols, + /*strideC=*/dims.inner * dims.rhs_cols, + /*batchCount=*/dims.batch_size)); } } // namespace FlexFlow diff --git a/lib/kernels/src/cuda/ops/batch_norm_kernels.cu b/lib/kernels/src/cuda/ops/batch_norm_kernels.cu index cd08e2833c..c5124ebaa4 100644 --- a/lib/kernels/src/cuda/ops/batch_norm_kernels.cu +++ b/lib/kernels/src/cuda/ops/batch_norm_kernels.cu @@ -15,10 +15,12 @@ #include "internal/device.h" #include "kernels/allocation.h" -#include "kernels/batch_norm_kernels.h" +#include "kernels/batch_norm_kernels_gpu.h" #include "kernels/ff_handle.h" #include "utils/integer_conversions.h" #include "utils/optional.h" +#include +#include namespace FlexFlow { namespace Kernels { @@ -47,9 +49,19 @@ void gpu_forward_kernel(cudaStream_t stream, 1.0, m.runningMean, m.runningVar, - CUDNN_BN_MIN_EPSILON, + m.eps, m.saveMean, m.saveVar)); + if (m.activation.has_value()) { + checkCUDNN(cudnnActivationForward(m.handle.dnn, + m.actiDesc, + &alpha, + m.outputTensor, + output_ptr, + &beta, + m.outputTensor, + output_ptr)); + } } void gpu_backward_kernel(cudaStream_t stream, @@ -64,7 +76,7 @@ void gpu_backward_kernel(cudaStream_t stream, size_t numElements) { checkCUDNN(cudnnSetStream(m.handle.dnn, stream)); - float alpha = 1.0f; + float alpha = 1.0f, beta = 0.0f; if (m.activation.has_value()) { switch (m.activation.value()) { case Activation::RELU: @@ -80,9 +92,9 @@ void gpu_backward_kernel(cudaStream_t stream, checkCUDNN(cudnnBatchNormalizationBackward(m.handle.dnn, m.mode, &alpha, + &beta, &alpha, - &alpha, - &alpha, + &beta, m.inputTensor, input_ptr, m.outputTensor, @@ -93,7 +105,7 @@ void gpu_backward_kernel(cudaStream_t stream, scale_ptr, scale_grad_ptr, bias_grad_ptr, - CUDNN_BN_MIN_EPSILON, + m.eps, m.saveMean, m.saveVar)); } @@ -106,11 +118,15 @@ BatchNormPerDeviceState int output_c, int output_h, int output_w, - std::optional const &activation) { + std::optional const &activation, + float eps) { + ASSERT(eps >= static_cast(CUDNN_BN_MIN_EPSILON), + "cuDNN requires BatchNorm eps to be at least CUDNN_BN_MIN_EPSILON", + eps); ffTensorDescriptor_t inputTensor; ffTensorDescriptor_t outputTensor; ffTensorDescriptor_t biasTensor; - ffActivationDescriptor_t actiDesc; + ffActivationDescriptor_t actiDesc{}; ffBatchNormMode_t mode; checkCUDNN(cudnnCreateTensorDescriptor(&inputTensor)); checkCUDNN(cudnnCreateTensorDescriptor(&biasTensor)); @@ -141,13 +157,15 @@ BatchNormPerDeviceState float *runningVar = (float *)runningMean + output_c; float *saveMean = (float *)runningVar + output_c; float *saveVar = (float *)saveMean + output_c; - cudaStream_t stream; - checkCUDA(get_legion_stream(&stream)); - - assign_kernel<<>>( - runningMean, size_t_from_int(output_c), 0.0f); - assign_kernel<<>>( - runningVar, size_t_from_int(output_c), 0.0f); + // Initialize on the host so standalone kernels do not need a Legion stream. + std::vector initial_running_stats(output_c * 2, 0.0f); + std::fill(initial_running_stats.begin() + output_c, + initial_running_stats.end(), + 1.0f); + checkCUDA(cudaMemcpy(runningMean, + initial_running_stats.data(), + sizeof(float) * output_c * 2, + cudaMemcpyHostToDevice)); if (activation.has_value()) { switch (activation.value()) { @@ -178,9 +196,10 @@ BatchNormPerDeviceState output_h, output_w, activation, + // A float epsilon of 1e-5 rounds just below cuDNN's double minimum. + std::max(static_cast(eps), CUDNN_BN_MIN_EPSILON), }; - checkCUDA(cudaStreamDestroy(stream)); return per_device_state; } diff --git a/lib/kernels/src/cuda/ops/concat_kernels.cu b/lib/kernels/src/cuda/ops/concat_kernels.cu index 94fa73b16c..c6e670c48d 100644 --- a/lib/kernels/src/cuda/ops/concat_kernels.cu +++ b/lib/kernels/src/cuda/ops/concat_kernels.cu @@ -81,14 +81,14 @@ void concat_gpu_backward_kernel( int blocks_to_add = (output_blk_size == input_blk_size) ? input_num_blocks : num_blocks; - add_with_stride<<>>(input_grad.get_float_ptr(), - output_grad.get_float_ptr() + offset, - blocks_to_add, - input_blk_size, - output_blk_size); + copy_with_stride<<>>(input_grad.get_float_ptr(), + output_grad.get_float_ptr() + offset, + blocks_to_add, + input_blk_size, + output_blk_size); offset += (output_blk_size == input_blk_size) ? input_blk_size * input_num_blocks diff --git a/lib/kernels/src/cuda/ops/conv_2d_kernels.cu b/lib/kernels/src/cuda/ops/conv_2d_kernels.cu index 74ed85154c..6dd3bf6101 100644 --- a/lib/kernels/src/cuda/ops/conv_2d_kernels.cu +++ b/lib/kernels/src/cuda/ops/conv_2d_kernels.cu @@ -327,8 +327,7 @@ void conv2d_gpu_backward_kernel(ffStream_t stream, std::optional activation) { checkCUDNN(cudnnSetStream(m.handle.dnn, stream)); - float alpha = 1.0f; - // float beta = 0.0f; + float alpha = 1.0f, beta = 0.0f; ASSERT(!activation.has_value(), ("Currently activation functions are unsupported for the Conv2D " @@ -354,7 +353,6 @@ void conv2d_gpu_backward_kernel(ffStream_t stream, */ // Compute filter gradiant - // NOTE: we use alpha for kernel_grad to accumulate gradients checkCUDNN(cudnnConvolutionBackwardFilter(m.handle.dnn, &alpha, m.inputTensor, @@ -365,22 +363,20 @@ void conv2d_gpu_backward_kernel(ffStream_t stream, m.bwdFilterAlgo, m.handle.workSpace, m.handle.workSpaceSize, - &alpha, + &beta, m.filterDesc, filter_grad_ptr)); // Compute bias gradiant - // NOTE: we use alpha for bias_grad to accumulate gradients if (bias_grad_ptr != NULL) { checkCUDNN(cudnnConvolutionBackwardBias(m.handle.dnn, &alpha, m.outputTensor, output_grad_ptr, - &alpha, + &beta, m.biasTensor, bias_grad_ptr)); } // Compute data gradiant - // NOTE: we use alpha for input_grad to accumulate gradients if (input_grad_ptr != NULL) { checkCUDNN(cudnnConvolutionBackwardData(m.handle.dnn, &alpha, @@ -392,14 +388,19 @@ void conv2d_gpu_backward_kernel(ffStream_t stream, m.bwdDataAlgo, m.handle.workSpace, m.handle.workSpaceSize, - &alpha, + &beta, m.inputTensor, input_grad_ptr)); } } void conv2d_gpu_cleanup_kernel(Conv2DPerDeviceState &per_device_state) { - NOT_IMPLEMENTED(); + checkCUDNN(cudnnDestroyTensorDescriptor(per_device_state.inputTensor)); + checkCUDNN(cudnnDestroyTensorDescriptor(per_device_state.biasTensor)); + checkCUDNN(cudnnDestroyTensorDescriptor(per_device_state.outputTensor)); + checkCUDNN(cudnnDestroyFilterDescriptor(per_device_state.filterDesc)); + checkCUDNN(cudnnDestroyActivationDescriptor(per_device_state.actiDesc)); + checkCUDNN(cudnnDestroyConvolutionDescriptor(per_device_state.convDesc)); } } // namespace FlexFlow diff --git a/lib/kernels/src/cuda/ops/element_unary_kernels.cu b/lib/kernels/src/cuda/ops/element_unary_kernels.cu index 6107f49646..6851812447 100644 --- a/lib/kernels/src/cuda/ops/element_unary_kernels.cu +++ b/lib/kernels/src/cuda/ops/element_unary_kernels.cu @@ -41,12 +41,25 @@ static bool use_scalar(OperatorType op_type) { case OperatorType::SCALAR_SUB: case OperatorType::SCALAR_TRUE_DIV: case OperatorType::POW: + case OperatorType::SILU: return true; default: return false; } } +// SILU's scalar (beta) is optional and defaults to 1, matching the CPU +// kernel. The other scalar operators require a scalar. +static float require_scalar(OperatorType op_type, + std::optional const &scalar) { + if (op_type == OperatorType::SILU) { + return scalar.value_or(1.0f); + } + + ASSERT(scalar.has_value(), op_type); + return scalar.value(); +} + ElementUnaryPerDeviceState element_unary_gpu_init_kernel(ElementUnaryAttrs const &attrs, TensorShape const &input_shape, @@ -118,6 +131,10 @@ __global__ void elewise_scalar_unary_forward_kernel( out[i] = (T)(powf(in[i], scalar)); break; } + case OperatorType::SILU: { + out[i] = (T)(in[i] / (1.0f + expf(-1.0f * scalar * in[i]))); + break; + } default: assert(false); } @@ -172,19 +189,19 @@ __global__ void elewise_scalar_unary_backward_kernel(coord_t volume, CUDA_KERNEL_LOOP(i, volume) { switch (type) { case OperatorType::SCALAR_MULTIPLY: { - input_grad[i] += output_grad[i] * scalar; + input_grad[i] = output_grad[i] * scalar; break; } case OperatorType::SCALAR_ADD: { - input_grad[i] += output_grad[i]; + input_grad[i] = output_grad[i]; break; } case OperatorType::SCALAR_SUB: { - input_grad[i] += output_grad[i]; + input_grad[i] = output_grad[i]; break; } case OperatorType::SCALAR_TRUE_DIV: { - input_grad[i] += output_grad[i] / scalar; + input_grad[i] = output_grad[i] / scalar; break; } case OperatorType::POW: { @@ -192,6 +209,13 @@ __global__ void elewise_scalar_unary_backward_kernel(coord_t volume, (T)(output_grad[i] * scalar * powf(input[i], scalar - 1)); break; } + case OperatorType::SILU: { + float e_to_bx = expf(scalar * input[i]); + input_grad[i] = (T)(output_grad[i] * + (e_to_bx * (scalar * input[i] + e_to_bx + 1.0f)) / + ((e_to_bx + 1.0f) * (e_to_bx + 1.0f))); + break; + } default: assert(false); } @@ -209,11 +233,11 @@ __global__ void elewise_unary_backward_kernel(coord_t volume, switch (type) { case OperatorType::EXP: { // TODO: change to use output instead of recomputing - input_grad[i] += (T)(output_grad[i] * exp((float)input[i])); + input_grad[i] = (T)(output_grad[i] * exp((float)input[i])); break; } case OperatorType::IDENTITY: { - input_grad[i] += output_grad[i]; + input_grad[i] = output_grad[i]; break; } case OperatorType::GELU: { @@ -229,11 +253,11 @@ __global__ void elewise_unary_backward_kernel(coord_t volume, break; } case OperatorType::SIN: { - input_grad[i] += (T)(output_grad[i] * cos((float)input[i])); + input_grad[i] = (T)(output_grad[i] * cos((float)input[i])); break; } case OperatorType::COS: { - input_grad[i] += (T)(output_grad[i] * -sin((float)input[i])); + input_grad[i] = (T)(output_grad[i] * -sin((float)input[i])); break; } default: @@ -267,11 +291,10 @@ struct ForwardKernel { m.outputTensor, output.get())); } else if (use_scalar(op_type)) { - assert(scalar.has_value()); elewise_scalar_unary_forward_kernel> <<>>( num_elements, - static_cast>(scalar.value()), + static_cast>(require_scalar(op_type, scalar)), op_type, input.get(), output.get()); @@ -299,7 +322,7 @@ struct BackwardKernel { get_num_elements(input.shape.dims).int_from_positive_int(); if (use_cudnn(op_type)) { - float alpha = 1.0f; + float alpha = 1.0f, beta = 0.0f; checkCUDNN(cudnnActivationBackward(handle.dnn, m.actiDesc, &alpha, @@ -309,15 +332,14 @@ struct BackwardKernel { output_grad.get(), m.inputTensor, input.get(), - &alpha, + &beta, m.inputTensor, input_grad.get())); } else if (use_scalar(op_type)) { - assert(scalar.has_value()); elewise_scalar_unary_backward_kernel> <<>>( num_elements, - static_cast>(scalar.value()), + static_cast>(require_scalar(op_type, scalar)), op_type, output.get(), output_grad.get(), diff --git a/lib/kernels/src/cuda/ops/pool_2d_kernels.cu b/lib/kernels/src/cuda/ops/pool_2d_kernels.cu index 281e6521b6..afc04e2808 100644 --- a/lib/kernels/src/cuda/ops/pool_2d_kernels.cu +++ b/lib/kernels/src/cuda/ops/pool_2d_kernels.cu @@ -124,7 +124,7 @@ void pool2d_gpu_backward_kernel(cudaStream_t stream, checkCUDNN(cudnnSetStream(m.handle.dnn, stream)); - float alpha = 1.0f; + float alpha = 1.0f, beta = 0.0f; checkCUDNN(cudnnPoolingBackward(m.handle.dnn, m.poolDesc, &alpha, @@ -134,13 +134,16 @@ void pool2d_gpu_backward_kernel(cudaStream_t stream, output_grad_ptr, m.inputTensor, input_ptr, - &alpha, + &beta, m.inputTensor, input_grad_ptr)); } void pool2d_gpu_cleanup_kernel(Pool2DPerDeviceState &per_device_state) { - NOT_IMPLEMENTED(); + checkCUDNN(cudnnDestroyTensorDescriptor(per_device_state.inputTensor)); + checkCUDNN(cudnnDestroyTensorDescriptor(per_device_state.outputTensor)); + checkCUDNN(cudnnDestroyActivationDescriptor(per_device_state.actiDesc)); + checkCUDNN(cudnnDestroyPoolingDescriptor(per_device_state.poolDesc)); } } // namespace FlexFlow diff --git a/lib/kernels/src/cuda/ops/reshape_kernels.cu b/lib/kernels/src/cuda/ops/reshape_kernels.cu index c05a76fe9d..898d8f8a72 100644 --- a/lib/kernels/src/cuda/ops/reshape_kernels.cu +++ b/lib/kernels/src/cuda/ops/reshape_kernels.cu @@ -16,6 +16,7 @@ #include "internal/device.h" #include "kernels/datatype_dispatch.h" #include "kernels/reshape_kernels_gpu.h" +#include "op-attrs/tensor_shape.h" namespace FlexFlow { @@ -25,7 +26,7 @@ __global__ void apply_add_with_scale2(DT *data_ptr, size_t size, DT scale) { CUDA_KERNEL_LOOP(i, size) { - data_ptr[i] += grad_ptr[i] * scale; + data_ptr[i] = grad_ptr[i] * scale; } } @@ -50,7 +51,11 @@ struct BackwardKernel { void reshape_gpu_forward_kernel(cudaStream_t stream, GenericTensorAccessorR const &input, GenericTensorAccessorW const &output) { - copy_accessor_data_to_l_from_r(output, input); + size_t num_bytes = + get_size_in_bytes(output.shape).unwrap_num_bytes().unwrap_nonnegative(); + ASSERT(get_size_in_bytes(input.shape) == get_size_in_bytes(output.shape)); + checkCUDA(cudaMemcpyAsync( + output.ptr, input.ptr, num_bytes, cudaMemcpyDeviceToDevice, stream)); } void reshape_gpu_backward_kernel(cudaStream_t stream, diff --git a/lib/kernels/src/cuda/ops/split_kernels.cu b/lib/kernels/src/cuda/ops/split_kernels.cu index c99237830d..5b1937186f 100644 --- a/lib/kernels/src/cuda/ops/split_kernels.cu +++ b/lib/kernels/src/cuda/ops/split_kernels.cu @@ -45,10 +45,10 @@ void split_gpu_backward_kernel(cudaStream_t stream, int numOutputs) { for (int i = 0; i < numOutputs; i++) { - add_with_stride<<>>( + copy_with_stride<<>>( in_grad_ptr, out_grad_ptr[i], num_blks, in_blk_size, out_blk_sizes[i]); in_grad_ptr += out_blk_sizes[i]; } diff --git a/lib/kernels/src/cuda/ops/transpose_kernels.cu b/lib/kernels/src/cuda/ops/transpose_kernels.cu index 398acd5990..7c8124d609 100644 --- a/lib/kernels/src/cuda/ops/transpose_kernels.cu +++ b/lib/kernels/src/cuda/ops/transpose_kernels.cu @@ -17,6 +17,8 @@ #include "kernels/accessor.h" #include "kernels/legion_ordered/legion_ordered_transform.h" #include "kernels/transpose_kernels_gpu.h" +#include "op-attrs/tensor_dim_permutation.h" +#include "utils/exception.h" #include "utils/nonnegative_int/num_elements.h" namespace FlexFlow { @@ -27,11 +29,12 @@ struct TransposeStrides { perm[MAX_TENSOR_DIM]; }; +// Computes out_ptr[o] = in_ptr[i], where the coordinates +// of o and i are related by `info.perm` (see `make_strides`). __global__ void transpose_simple_kernel(std::size_t volume, float const *in_ptr, float *out_ptr, - const TransposeStrides info, - float const beta) { + const TransposeStrides info) { CUDA_KERNEL_LOOP(o_idx, volume) { size_t i_idx = 0; size_t t = o_idx; @@ -40,17 +43,21 @@ __global__ void transpose_simple_kernel(std::size_t volume, t -= ratio * info.out_strides[i]; i_idx += ratio * info.in_strides[info.perm[i]]; } - out_ptr[o_idx] += out_ptr[o_idx] * beta + in_ptr[i_idx]; + out_ptr[o_idx] = in_ptr[i_idx]; } } -static TransposeStrides make_strides(TransposeAttrs const &m, +// Builds the stride/permutation info needed to compute +// `output[coord] = input[permuted_coord]`, where +// `permuted_coord[permutation.at_l(d)] = coord[d]`. +static TransposeStrides make_strides(TensorDimPermutation const &permutation, TensorDims const &input_dims, TensorDims const &output_dims) { - ASSERT(get_num_dims(input_dims) == m.permutation.num_tensor_dims()); + ASSERT(get_num_dims(input_dims) == permutation.num_tensor_dims()); + ASSERT(get_num_dims(output_dims) == permutation.num_tensor_dims()); TransposeStrides info; - num_tensor_dims_t num_dims = m.permutation.num_tensor_dims(); + num_tensor_dims_t num_dims = permutation.num_tensor_dims(); info.num_dim = num_dims.int_from_num_tensor_dims(); for (int i = 0; i < info.num_dim; i++) { @@ -61,15 +68,16 @@ static TransposeStrides make_strides(TransposeAttrs const &m, info.in_strides[i] = 1; info.out_strides[i] = 1; } else { + legion_dim_t prev_legion_dim = legion_dim_t{nonnegative_int{i - 1}}; int in_dim_size = - dim_at_idx(input_dims, legion_dim).int_from_positive_int(); + dim_at_idx(input_dims, prev_legion_dim).int_from_positive_int(); int out_dim_size = - dim_at_idx(output_dims, legion_dim).int_from_positive_int(); + dim_at_idx(output_dims, prev_legion_dim).int_from_positive_int(); info.in_strides[i] = info.in_strides[i - 1] * in_dim_size; info.out_strides[i] = info.out_strides[i - 1] * out_dim_size; } - ff_dim_t ff_permuted_dim = m.permutation.at_l(ff_dim); + ff_dim_t ff_permuted_dim = permutation.at_l(ff_dim); legion_dim_t legion_permuted_dim = legion_dim_from_ff_dim(ff_permuted_dim, num_dims); info.perm[i] = legion_permuted_dim.value.unwrap_nonnegative(); @@ -79,35 +87,40 @@ static TransposeStrides make_strides(TransposeAttrs const &m, } void transpose_gpu_forward_kernel(cudaStream_t stream, - TransposeAttrs const &m, + TransposeAttrs const &attrs, GenericTensorAccessorR const &input, GenericTensorAccessorW const &output) { - transpose_simple_kernel<<< - GET_BLOCKS(get_num_elements(output.shape.dims).int_from_positive_int()), - CUDA_NUM_THREADS, - 0, - stream>>>(get_num_elements(output.shape.dims).int_from_positive_int(), - input.get_float_ptr(), - output.get_float_ptr(), - make_strides(m, input.shape.dims, output.shape.dims), - /*beta=*/0.0f); + ASSERT(permute_tensor_dims(attrs.permutation, input.shape.dims) == + output.shape.dims); + + int volume = get_num_elements(output.shape.dims).int_from_positive_int(); + + transpose_simple_kernel<<>>( + volume, + input.get_float_ptr(), + output.get_float_ptr(), + make_strides(attrs.permutation, input.shape.dims, output.shape.dims)); } void transpose_gpu_backward_kernel(cudaStream_t stream, - TransposeAttrs const &m, - GenericTensorAccessorR const &out_grad, - GenericTensorAccessorW const &in_grad) { - ASSERT(get_num_dims(in_grad.shape.dims) == m.permutation.num_tensor_dims()); - - transpose_simple_kernel<<< - GET_BLOCKS(get_num_elements(in_grad.shape.dims).int_from_positive_int()), - CUDA_NUM_THREADS, - 0, - stream>>>(get_num_elements(in_grad.shape.dims).int_from_positive_int(), - out_grad.get_float_ptr(), - in_grad.get_float_ptr(), - make_strides(m, out_grad.shape.dims, in_grad.shape.dims), - /*beta=*/1.0f); + TransposeAttrs const &attrs, + GenericTensorAccessorR const &output_grad, + GenericTensorAccessorW const &input_grad) { + ASSERT(permute_tensor_dims(attrs.permutation, input_grad.shape.dims) == + output_grad.shape.dims); + + // The gradient of a permutation is the inverse permutation. + TensorDimPermutation inverse_permutation = + invert_tensor_dim_permutation(attrs.permutation); + + int volume = get_num_elements(input_grad.shape.dims).int_from_positive_int(); + + transpose_simple_kernel<<>>( + volume, + output_grad.get_float_ptr(), + input_grad.get_float_ptr(), + make_strides( + inverse_permutation, output_grad.shape.dims, input_grad.shape.dims)); } } // namespace FlexFlow diff --git a/lib/kernels/src/cuda/ops/upsample_kernels.cu b/lib/kernels/src/cuda/ops/upsample_kernels.cu index b34cbc3997..43826ba064 100644 --- a/lib/kernels/src/cuda/ops/upsample_kernels.cu +++ b/lib/kernels/src/cuda/ops/upsample_kernels.cu @@ -1,23 +1,175 @@ #include "internal/device.h" +#include "kernels/datatype_dispatch.h" #include "kernels/upsample_kernels_gpu.h" -#include "utils/not_implemented.h" +#include "op-attrs/tensor_dims.h" +#include "utils/containers/require_same.h" namespace FlexFlow { -void upsample_gpu_forward_kernel(ffStream_t stream, +// The shape of an NCHW tensor, flattened into the quantities the kernels below +// need in order to turn a linear index into coordinates. +struct UpsampleNCHWDims { + int num_samples; + int num_channels; + int height; + int width; +}; + +static UpsampleNCHWDims get_nchw_dims(TensorShape const &shape) { + return UpsampleNCHWDims{ + /*num_samples=*/ + dim_at_idx(shape.dims, ff_dim_t{0_n}).int_from_positive_int(), + /*num_channels=*/ + dim_at_idx(shape.dims, ff_dim_t{1_n}).int_from_positive_int(), + /*height=*/dim_at_idx(shape.dims, ff_dim_t{2_n}).int_from_positive_int(), + /*width=*/dim_at_idx(shape.dims, ff_dim_t{3_n}).int_from_positive_int(), + }; +} + +static void check_shapes(UpsampleAttrs const &attrs, + TensorShape const &input_shape, + TensorShape const &output_shape) { + ASSERT(get_num_dims(input_shape.dims) == num_tensor_dims_t{4_n}, + "Currently Upsample only supports 4-dimensional input tensors (i.e., " + "NCHW tensors). " + "If you need other support for other tensor shapes, please create an " + "issue.", + input_shape); + + ASSERT(attrs.mode == UpsampleMode::NEAREST, + "Currently Upsample is only supports mode {}. " + "If you need other support for other modes, please create an issue.", + UpsampleMode::NEAREST); + + UpsampleNCHWDims input_dims = get_nchw_dims(input_shape); + UpsampleNCHWDims output_dims = get_nchw_dims(output_shape); + int scale_factor = attrs.scale_factor.int_from_int_ge_two(); + + ASSERT(output_dims.num_samples == input_dims.num_samples); + ASSERT(output_dims.num_channels == input_dims.num_channels); + ASSERT(output_dims.height == input_dims.height * scale_factor); + ASSERT(output_dims.width == input_dims.width * scale_factor); +} + +// Each thread computes one output element by reading the input element the +// output coordinate maps back to. +template +__global__ void upsample_nearest_forward_kernel(size_t num_output_elements, + T const *input, + T *output, + UpsampleNCHWDims input_dims, + int scale_factor) { + CUDA_KERNEL_LOOP(output_idx, num_output_elements) { + int output_width = input_dims.width * scale_factor; + int output_height = input_dims.height * scale_factor; + + int w = output_idx % output_width; + int h = (output_idx / output_width) % output_height; + int nc = output_idx / (output_width * output_height); + + int input_idx = + (nc * input_dims.height + h / scale_factor) * input_dims.width + + w / scale_factor; + + output[output_idx] = input[input_idx]; + } +} + +// Each thread accumulates one input element by summing over the block of +// output elements that read from it. Iterating over inputs (rather than +// outputs) keeps the accumulation deterministic and avoids atomics. +template +__global__ void upsample_nearest_backward_kernel(size_t num_input_elements, + T const *output_grad, + T *input_grad, + UpsampleNCHWDims input_dims, + int scale_factor) { + CUDA_KERNEL_LOOP(input_idx, num_input_elements) { + int output_width = input_dims.width * scale_factor; + int output_height = input_dims.height * scale_factor; + + int w = input_idx % input_dims.width; + int h = (input_idx / input_dims.width) % input_dims.height; + int nc = input_idx / (input_dims.width * input_dims.height); + + T sum = 0; + for (int dh = 0; dh < scale_factor; dh++) { + for (int dw = 0; dw < scale_factor; dw++) { + int output_idx = + (nc * output_height + h * scale_factor + dh) * output_width + + w * scale_factor + dw; + sum += output_grad[output_idx]; + } + } + + input_grad[input_idx] = sum; + } +} + +template +struct UpsampleGPUForwardKernel { + void operator()(cudaStream_t stream, + UpsampleAttrs const &attrs, + GenericTensorAccessorR const &input, + GenericTensorAccessorW const &output) const { + int num_output_elements = + get_num_elements(output.shape.dims).int_from_positive_int(); + + upsample_nearest_forward_kernel> + <<>>( + num_output_elements, + input.get
(), + output.get
(), + get_nchw_dims(input.shape), + attrs.scale_factor.int_from_int_ge_two()); + } +}; + +void upsample_gpu_forward_kernel(cudaStream_t stream, UpsampleAttrs const &attrs, GenericTensorAccessorR const &input, GenericTensorAccessorW const &output) { - NOT_IMPLEMENTED(); + check_shapes(attrs, input.shape, output.shape); + + DataType data_type = + require_same(input.shape.data_type, output.shape.data_type); + + DataTypeDispatch1{}( + data_type, stream, attrs, input, output); } -void upsample_gpu_backward_kernel(ffStream_t stream, +template +struct UpsampleGPUBackwardKernel { + void operator()(cudaStream_t stream, + UpsampleAttrs const &attrs, + GenericTensorAccessorR const &output_grad, + GenericTensorAccessorW const &input_grad) const { + int num_input_elements = + get_num_elements(input_grad.shape.dims).int_from_positive_int(); + + upsample_nearest_backward_kernel> + <<>>( + num_input_elements, + output_grad.get
(), + input_grad.get
(), + get_nchw_dims(input_grad.shape), + attrs.scale_factor.int_from_int_ge_two()); + } +}; + +void upsample_gpu_backward_kernel(cudaStream_t stream, UpsampleAttrs const &attrs, GenericTensorAccessorR const &output, GenericTensorAccessorR const &output_grad, GenericTensorAccessorR const &input, GenericTensorAccessorW const &input_grad) { - NOT_IMPLEMENTED(); + check_shapes(attrs, input_grad.shape, output_grad.shape); + + DataType data_type = + require_same(output_grad.shape.data_type, input_grad.shape.data_type); + + DataTypeDispatch1{}( + data_type, stream, attrs, output_grad, input_grad); } } // namespace FlexFlow diff --git a/lib/kernels/src/kernels/accessors_are_approximately_equal.cc b/lib/kernels/src/kernels/accessors_are_approximately_equal.cc index 5bd25d52ef..ef78a5bbf0 100644 --- a/lib/kernels/src/kernels/accessors_are_approximately_equal.cc +++ b/lib/kernels/src/kernels/accessors_are_approximately_equal.cc @@ -1,5 +1,6 @@ #include "kernels/accessors_are_approximately_equal.h" -#include "kernels/map_tensor_accessors.h" +#include "kernels/copy_tensor_accessor.h" +#include "kernels/local_cpu_allocator.h" #include "kernels/tensor_accessor_binary_ops.h" #include "kernels/tensor_accessor_reductions.h" #include "kernels/values_are_approximately_equal.h" @@ -14,9 +15,15 @@ bool accessors_are_approximately_equal( ASSERT(shape.data_type == DataType::FLOAT); + Allocator cpu_allocator = create_local_cpu_memory_allocator(); + GenericTensorAccessorR cpu_a = + copy_tensor_accessor_r_to_cpu_if_necessary(accessor_a, cpu_allocator); + GenericTensorAccessorR cpu_b = + copy_tensor_accessor_r_to_cpu_if_necessary(accessor_b, cpu_allocator); + for (TensorDimsCoord const &coord : get_tensor_dims_coord_set(shape.dims)) { - float val_a = accessor_a.at(coord); - float val_b = accessor_b.at(coord); + float val_a = cpu_a.at(coord); + float val_b = cpu_b.at(coord); if (!values_are_approximately_equal(val_a, val_b, relative_error_allowed)) { return false; } diff --git a/lib/kernels/src/kernels/batch_matmul_kernels.cc b/lib/kernels/src/kernels/batch_matmul_kernels.cc index 724bff7136..f30bfa3f34 100644 --- a/lib/kernels/src/kernels/batch_matmul_kernels.cc +++ b/lib/kernels/src/kernels/batch_matmul_kernels.cc @@ -5,17 +5,20 @@ namespace FlexFlow { void batch_matmul_forward_kernel(device_stream_t const &stream, + device_handle_t const &handle, GenericTensorAccessorR const &input_lhs, GenericTensorAccessorR const &input_rhs, GenericTensorAccessorW const &output) { if (stream.is_gpu()) { batch_matmul_gpu_forward_kernel( /*stream=*/stream.require_gpu(), + /*handle=*/handle.require_for_gpu(), /*input_lhs=*/input_lhs, /*input_rhs=*/input_rhs, /*output=*/output); } else { ASSERT(stream.is_cpu()); + ASSERT(handle.is_for_cpu()); batch_matmul_cpu_forward_kernel( /*input_lhs=*/input_lhs, @@ -26,6 +29,7 @@ void batch_matmul_forward_kernel(device_stream_t const &stream, void batch_matmul_backward_kernel( device_stream_t const &stream, + device_handle_t const &handle, GenericTensorAccessorR const &output, GenericTensorAccessorR const &output_grad, GenericTensorAccessorR const &input_lhs, @@ -35,6 +39,7 @@ void batch_matmul_backward_kernel( if (stream.is_gpu()) { batch_matmul_gpu_backward_kernel( /*stream=*/stream.require_gpu(), + /*handle=*/handle.require_for_gpu(), /*output=*/output, /*output_grad=*/output_grad, /*input_lhs=*/input_lhs, @@ -43,6 +48,7 @@ void batch_matmul_backward_kernel( /*input_rhs_grad=*/input_rhs_grad); } else { ASSERT(stream.is_cpu()); + ASSERT(handle.is_for_cpu()); batch_matmul_cpu_backward_kernel( /*output=*/output, diff --git a/lib/kernels/src/kernels/batch_norm_kernels.cc b/lib/kernels/src/kernels/batch_norm_kernels.cc index 41540dc7f3..90920ab314 100644 --- a/lib/kernels/src/kernels/batch_norm_kernels.cc +++ b/lib/kernels/src/kernels/batch_norm_kernels.cc @@ -13,7 +13,8 @@ std::optional int output_c, int output_h, int output_w, - std::optional const &activation) { + std::optional const &activation, + float eps) { if (device_type == DeviceType::GPU) { return gpu_init_kernel( /*handle=*/handle.require_for_gpu(), @@ -23,7 +24,8 @@ std::optional /*output_c=*/output_c, /*output_h=*/output_h, /*output_w=*/output_w, - /*activation=*/activation); + /*activation=*/activation, + /*eps=*/eps); } else { ASSERT(device_type == DeviceType::CPU); ASSERT(handle.is_for_cpu()); diff --git a/lib/kernels/src/kernels/split_kernels.cc b/lib/kernels/src/kernels/split_kernels.cc index 9ca2df9b76..c6e1e161a3 100644 --- a/lib/kernels/src/kernels/split_kernels.cc +++ b/lib/kernels/src/kernels/split_kernels.cc @@ -1,23 +1,17 @@ #include "kernels/split_kernels.h" #include "kernels/split_kernels_cpu.h" #include "kernels/split_kernels_gpu.h" +#include "op-attrs/tensor_dims.h" #include namespace FlexFlow { static std::pair calc_block_size(TensorShape const &tensor_shape, ff_dim_t axis) { - positive_int num_blocks = 1_p; - positive_int block_size = 1_p; - for (nonnegative_int d : - nonnegative_range(get_num_elements(tensor_shape.dims) - .nonnegative_int_from_positive_int())) { - if (d <= axis.value) { - block_size *= dim_at_idx(tensor_shape.dims, legion_dim_t{d}); - } else { - num_blocks *= dim_at_idx(tensor_shape.dims, legion_dim_t{d}); - } - } + positive_int block_size = get_num_elements( + slice_tensor_dims(tensor_shape.dims, axis, std::nullopt)); + positive_int num_blocks = get_num_elements( + slice_tensor_dims(tensor_shape.dims, ff_dim_t{0_n}, axis)); return {num_blocks, block_size}; } diff --git a/lib/kernels/src/kernels/transpose_kernels.cc b/lib/kernels/src/kernels/transpose_kernels.cc index 2979567256..0dc1010faa 100644 --- a/lib/kernels/src/kernels/transpose_kernels.cc +++ b/lib/kernels/src/kernels/transpose_kernels.cc @@ -35,7 +35,7 @@ void transpose_backward_kernel(device_stream_t const &stream, /*in_grad=*/in_grad); } else { ASSERT(stream.is_cpu()); - transpose_cpu_forward_kernel( + transpose_cpu_backward_kernel( /*attrs=*/attrs, /*out_grad=*/out_grad, /*in_grad=*/in_grad); diff --git a/lib/kernels/src/kernels/upsample_kernels_cpu.cc b/lib/kernels/src/kernels/upsample_kernels_cpu.cc index 83def0cf86..5152fb9e58 100644 --- a/lib/kernels/src/kernels/upsample_kernels_cpu.cc +++ b/lib/kernels/src/kernels/upsample_kernels_cpu.cc @@ -56,21 +56,25 @@ struct UpsampleCPUBackwardKernel { void operator()(UpsampleAttrs const &attrs, GenericTensorAccessorR const &output_grad, GenericTensorAccessorW const &input_grad) { - auto input_coord_from_output_coord = - [&](TensorDimsCoord const &output_coord) -> TensorDimsCoord { - TensorDimsCoord input_coord = output_coord; - tensor_dims_coord_at_rel_idx(input_coord, relative_ff_dim_t{-1}) /= - attrs.scale_factor; - tensor_dims_coord_at_rel_idx(input_coord, relative_ff_dim_t{-2}) /= - attrs.scale_factor; - return input_coord; - }; - - for (TensorDimsCoord output_coord : - get_tensor_dims_coord_set(output_grad.shape.dims)) { - TensorDimsCoord input_coord = input_coord_from_output_coord(output_coord); - - input_grad.at
(output_coord) += output_grad.at
(input_coord); + for (TensorDimsCoord const &input_coord : + get_tensor_dims_coord_set(input_grad.shape.dims)) { + real_type_t
sum = 0; + int scale = attrs.scale_factor.int_from_int_ge_two(); + for (int h = 0; h < scale; h++) { + for (int w = 0; w < scale; w++) { + TensorDimsCoord output_coord = input_coord; + tensor_dims_coord_at_rel_idx(output_coord, relative_ff_dim_t{-2}) = + tensor_dims_coord_at_rel_idx(input_coord, relative_ff_dim_t{-2}) * + attrs.scale_factor + + nonnegative_int{h}; + tensor_dims_coord_at_rel_idx(output_coord, relative_ff_dim_t{-1}) = + tensor_dims_coord_at_rel_idx(input_coord, relative_ff_dim_t{-1}) * + attrs.scale_factor + + nonnegative_int{w}; + sum += output_grad.at
(output_coord); + } + } + input_grad.at
(input_coord) = sum; } } }; diff --git a/lib/kernels/test/src/kernels/accessors_are_approximately_equal.cc b/lib/kernels/test/src/kernels/accessors_are_approximately_equal.cc index 8514860fa0..ebec8fc3cd 100644 --- a/lib/kernels/test/src/kernels/accessors_are_approximately_equal.cc +++ b/lib/kernels/test/src/kernels/accessors_are_approximately_equal.cc @@ -1,5 +1,6 @@ #include "kernels/accessors_are_approximately_equal.h" #include "kernels/create_accessor_with_contents.h" +#include "kernels/local_cuda_allocator.h" #include using namespace ::FlexFlow; @@ -42,3 +43,33 @@ TEST_SUITE(FF_TEST_SUITE) { } } } + +TEST_SUITE(FF_TEST_SUITE) { + TEST_CASE( + "accessors_are_approximately_equal (zeros and relative tolerance)") { + Allocator allocator = create_local_cpu_memory_allocator(); + GenericTensorAccessorR lhs = + create_1d_accessor_r_with_contents({0, 1000, -2}, allocator); + GenericTensorAccessorR rhs = + create_1d_accessor_r_with_contents({0, 1001, -2.001}, allocator); + + CHECK(accessors_are_approximately_equal(lhs, lhs)); + CHECK(accessors_are_approximately_equal(lhs, rhs, 0.01f)); + CHECK_FALSE(accessors_are_approximately_equal(lhs, rhs, 0.0001f)); + } +} + +TEST_SUITE(FF_CUDA_TEST_SUITE) { + TEST_CASE("accessors_are_approximately_equal (GPU and CPU accessors)") { + Allocator gpu_allocator = create_local_cuda_memory_allocator(); + Allocator cpu_allocator = create_local_cpu_memory_allocator(); + GenericTensorAccessorR lhs = + create_1d_accessor_r_with_contents({0, 1000, -2}, gpu_allocator); + GenericTensorAccessorR rhs = create_1d_accessor_r_with_contents( + {0, 1001, -2.001}, cpu_allocator); + + CHECK(accessors_are_approximately_equal(lhs, lhs)); + CHECK(accessors_are_approximately_equal(lhs, rhs, 0.01f)); + CHECK_FALSE(accessors_are_approximately_equal(lhs, rhs, 0.0001f)); + } +} diff --git a/lib/kernels/test/src/kernels/batch_matmul_kernels_gpu.cc b/lib/kernels/test/src/kernels/batch_matmul_kernels_gpu.cc new file mode 100644 index 0000000000..fa974032b0 --- /dev/null +++ b/lib/kernels/test/src/kernels/batch_matmul_kernels_gpu.cc @@ -0,0 +1,246 @@ +#include "kernels/batch_matmul_kernels_gpu.h" +#include "kernels/accessors_are_approximately_equal.h" +#include "kernels/accessors_are_equal.h" +#include "kernels/create_accessor_with_contents.h" +#include "kernels/create_random_filled_accessor.h" +#include "kernels/format_accessor_contents.h" +#include "kernels/local_cuda_allocator.h" +#include "kernels/managed_ff_stream.h" +#include "kernels/managed_per_device_ff_handle.h" +#include "op-attrs/tensor_dims.h" +#include "test/utils/doctest/check_kv.h" +#include + +using namespace ::FlexFlow; + +static GenericTensorAccessorR make_input_lhs(Allocator &allocator) { + return create_3d_accessor_r_with_contents( + { + { + {3, 3, 6}, + {2, 1, 5}, + {1, 2, -2}, + {8, 0.5, -3}, + }, + { + {5, 1, -2}, + {-8, 0, -1}, + {0.25, -0.3, -2}, + {0, -1, -5}, + }, + }, + allocator); +} + +static GenericTensorAccessorR make_input_rhs(Allocator &allocator) { + return create_3d_accessor_r_with_contents( + { + { + {1.0, 0.5}, + {2.0, 4.0}, + {1.5, -1.0}, + }, + { + {-6.0, -0.5}, + {1.0, -2.0}, + {0.4, -3.5}, + }, + }, + allocator); +} + +static GenericTensorAccessorR make_output(Allocator &allocator) { + return create_3d_accessor_r_with_contents( + { + { + {18.0, 7.5}, + {11.5, 0.0}, + {2.0, 10.5}, + {4.5, 9.0}, + }, + { + {-29.799999237060547, 2.5}, + {47.599998474121094, 7.5}, + {-2.5999999046325684, 7.474999904632568}, + {-3.0, 19.5}, + }, + }, + allocator); +} + +static TensorShape with_batch_rank(TensorShape shape, int rank) { + positive_int batch = dim_at_idx(shape.dims, ff_dim_t{0_n}); + positive_int rows = dim_at_idx(shape.dims, ff_dim_t{1_n}); + positive_int cols = dim_at_idx(shape.dims, ff_dim_t{2_n}); + if (rank == 4) { + shape.dims = TensorDims{FFOrdered{1_p, batch, rows, cols}}; + } else if (rank == 5) { + shape.dims = TensorDims{FFOrdered{1_p, batch, 1_p, rows, cols}}; + } + return shape; +} + +TEST_SUITE(FF_CUDA_TEST_SUITE) { + TEST_CASE("batch_matmul_gpu_forward_kernel") { + int rank = 3; + SUBCASE("3D") {} + SUBCASE("4D") { + rank = 4; + } + SUBCASE("5D") { + rank = 5; + } + + ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( + /*workSpaceSize=*/1024 * 1024, + /*allowTensorOpMathConversion=*/true); + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + GenericTensorAccessorR input_lhs = make_input_lhs(allocator); + GenericTensorAccessorR input_rhs = make_input_rhs(allocator); + input_lhs.shape = with_batch_rank(input_lhs.shape, rank); + input_rhs.shape = with_batch_rank(input_rhs.shape, rank); + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{2_p, 4_p, 2_p}}, + DataType::FLOAT, + }; + + output_shape = with_batch_rank(output_shape, rank); + + // Intentionally randomize this tensor so we can be confident we never read + // it + GenericTensorAccessorW result = + create_random_filled_accessor_w(output_shape, allocator); + + batch_matmul_gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*handle=*/managed_handle.raw_handle(), + /*input_lhs=*/input_lhs, + /*input_rhs=*/input_rhs, + /*output=*/result); + + GenericTensorAccessorR correct = make_output(allocator); + correct.shape = with_batch_rank(correct.shape, rank); + + CHECK_MESSAGE(accessors_are_approximately_equal(result, correct), + check_kv("result", format_accessor_w_contents(result))); + } + + TEST_CASE("batch_matmul_gpu_backward_kernel") { + int rank = 3; + SUBCASE("3D") {} + SUBCASE("4D") { + rank = 4; + } + SUBCASE("5D") { + rank = 5; + } + + ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( + /*workSpaceSize=*/1024 * 1024, + /*allowTensorOpMathConversion=*/true); + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + GenericTensorAccessorR input_lhs = make_input_lhs(allocator); + GenericTensorAccessorR input_rhs = make_input_rhs(allocator); + input_lhs.shape = with_batch_rank(input_lhs.shape, rank); + input_rhs.shape = with_batch_rank(input_rhs.shape, rank); + GenericTensorAccessorR output = make_output(allocator); + output.shape = with_batch_rank(output.shape, rank); + + GenericTensorAccessorR output_grad = + create_3d_accessor_r_with_contents( + { + { + {1.0, 2.0}, + {0.1, -1.0}, + {0.5, 1.5}, + {4.5, -3.0}, + }, + { + {0.0, -2.5}, + {1.0, -0.5}, + {-2.0, -0.5}, + {-3.0, 0.25}, + }, + }, + allocator); + + output_grad.shape = with_batch_rank(output_grad.shape, rank); + + // The gradients are overwritten rather than accumulated into (matching + // batch_matmul_cpu_backward_kernel), so randomize them to be confident + // their previous contents are never read + GenericTensorAccessorW input_lhs_grad = create_random_filled_accessor_w( + get_tensor_shape_for_accessor_r(input_lhs), allocator); + + GenericTensorAccessorW input_rhs_grad = create_random_filled_accessor_w( + get_tensor_shape_for_accessor_r(input_rhs), allocator); + + batch_matmul_gpu_backward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*handle=*/managed_handle.raw_handle(), + /*output=*/output, + /*output_grad=*/output_grad, + /*input_lhs=*/input_lhs, + /*input_lhs_grad=*/input_lhs_grad, + /*input_rhs=*/input_rhs, + /*input_rhs_grad=*/input_rhs_grad); + + GenericTensorAccessorR correct_input_lhs_grad = + create_3d_accessor_r_with_contents( + { + { + {2.0, 10.0, -0.5}, + {-0.4000000059604645, + -3.799999952316284, + 1.149999976158142}, + {1.25, 7.0, -0.75}, + {3.0, -3.0, 9.75}, + }, + { + {1.25, 5.0, 8.75}, + {-5.75, 2.0, 2.1500000953674316}, + {12.25, -1.0, 0.949999988079071}, + {17.875, -3.5, -2.075000047683716}, + }, + }, + allocator); + + GenericTensorAccessorR correct_input_rhs_grad = + create_3d_accessor_r_with_contents( + { + { + {39.70000076293945, -18.5}, + {6.349999904632568, 6.5}, + {-8.0, 13.0}, + }, + { + {-8.5, -8.625}, + {3.5999999046325684, -2.5999999046325684}, + {18.0, 5.25}, + }, + }, + allocator); + + correct_input_lhs_grad.shape = + with_batch_rank(correct_input_lhs_grad.shape, rank); + correct_input_rhs_grad.shape = + with_batch_rank(correct_input_rhs_grad.shape, rank); + + CHECK_MESSAGE( + accessors_are_approximately_equal(input_lhs_grad, + correct_input_lhs_grad), + check_kv("input_lhs_grad", format_accessor_w_contents(input_lhs_grad))); + + CHECK_MESSAGE( + accessors_are_approximately_equal(input_rhs_grad, + correct_input_rhs_grad), + check_kv("input_rhs_grad", format_accessor_w_contents(input_rhs_grad))); + } +} diff --git a/lib/kernels/test/src/kernels/batch_norm_kernels_gpu.cc b/lib/kernels/test/src/kernels/batch_norm_kernels_gpu.cc new file mode 100644 index 0000000000..e9494c63ea --- /dev/null +++ b/lib/kernels/test/src/kernels/batch_norm_kernels_gpu.cc @@ -0,0 +1,326 @@ +#include "kernels/batch_norm_kernels_gpu.h" +#include "kernels/accessors_are_approximately_equal.h" +#include "kernels/accessors_are_equal.h" +#include "kernels/create_accessor_with_contents.h" +#include "kernels/create_random_filled_accessor.h" +#include "kernels/format_accessor_contents.h" +#include "kernels/local_cuda_allocator.h" +#include "kernels/managed_ff_stream.h" +#include "kernels/managed_per_device_ff_handle.h" +#include "op-attrs/ops/batch_norm_attrs.dtg.h" +#include "test/utils/doctest/check_kv.h" +#include + +using namespace ::FlexFlow; +using namespace ::FlexFlow::Kernels::BatchNorm; + +static BatchNormAttrs make_attrs() { + return BatchNormAttrs{ + /*activation=*/std::nullopt, + /*affine=*/true, + /*eps=*/1e-5, + /*momentum=*/0.1, + }; +} + +// NCHW +static GenericTensorAccessorR make_input(Allocator &allocator) { + return create_4d_accessor_r_with_contents( + { + { + {{1, 2}, {3, 4}}, + {{-1, -2}, {0.5, 0.25}}, + }, + { + {{5, 6}, {7, 8}}, + {{2, 0.125}, {-3, 1}}, + }, + }, + allocator); +} + +static GenericTensorAccessorR make_gamma(Allocator &allocator) { + return create_1d_accessor_r_with_contents({2, 0.5}, allocator); +} + +static GenericTensorAccessorR make_beta(Allocator &allocator) { + return create_1d_accessor_r_with_contents({-1, 3}, allocator); +} + +static GenericTensorAccessorR make_output(Allocator &allocator) { + return create_4d_accessor_r_with_contents( + { + { + {{-4.055047512054443, -3.1821770668029785}, + {-2.3093061447143555, -1.4364354610443115}}, + {{2.760241985321045, 2.433763027191162}, + {3.249960422515869, 3.1683406829833984}}, + }, + { + {{-0.5635647177696228, 0.3093060255050659}, + {1.1821768283843994, 2.0550475120544434}}, + {{3.7396788597106934, 3.127530813217163}, + {2.1072840690612793, 3.4131999015808105}}, + }, + }, + allocator); +} + +TEST_SUITE(FF_CUDA_TEST_SUITE) { + TEST_CASE("batch_norm_gpu_forward_kernel") { + ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( + /*workSpaceSize=*/1024 * 1024, + /*allowTensorOpMathConversion=*/true); + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + BatchNormAttrs attrs = make_attrs(); + + GenericTensorAccessorR input = make_input(allocator); + GenericTensorAccessorR gamma = make_gamma(allocator); + GenericTensorAccessorR beta = make_beta(allocator); + + // Intentionally randomize this tensor so we can be confident we never read + // it + GenericTensorAccessorW output = + create_random_filled_accessor_w(input.shape, allocator); + + BatchNormPerDeviceState per_device_state = + gpu_init_kernel(managed_handle.raw_handle(), + allocator, + nullptr, + 2, + 2, + 2, + 2, + attrs.activation, + attrs.eps); + + gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*per_device_state=*/per_device_state, + /*input_ptr=*/input.get_float_ptr(), + /*output_ptr=*/output.get_float_ptr(), + /*scale_ptr=*/gamma.get_float_ptr(), + /*bias_ptr=*/beta.get_float_ptr()); + + GenericTensorAccessorR correct = make_output(allocator); + + CHECK_MESSAGE(accessors_are_approximately_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + + gpu_cleanup_kernel(allocator, per_device_state); + } + + TEST_CASE("batch_norm_gpu_backward_kernel") { + ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( + /*workSpaceSize=*/1024 * 1024, + /*allowTensorOpMathConversion=*/true); + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + BatchNormAttrs attrs = make_attrs(); + + GenericTensorAccessorR input = make_input(allocator); + GenericTensorAccessorR gamma = make_gamma(allocator); + GenericTensorAccessorR beta = make_beta(allocator); + + GenericTensorAccessorW forward_output = + create_random_filled_accessor_w(input.shape, allocator); + + BatchNormPerDeviceState per_device_state = + gpu_init_kernel(managed_handle.raw_handle(), + allocator, + nullptr, + 2, + 2, + 2, + 2, + attrs.activation, + attrs.eps); + + // cudnnBatchNormalizationBackward reads the batch statistics saved by the + // forward pass, so the forward kernel has to run first + gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*per_device_state=*/per_device_state, + /*input_ptr=*/input.get_float_ptr(), + /*output_ptr=*/forward_output.get_float_ptr(), + /*scale_ptr=*/gamma.get_float_ptr(), + /*bias_ptr=*/beta.get_float_ptr()); + + GenericTensorAccessorR output = make_output(allocator); + + GenericTensorAccessorW output_grad = + create_4d_accessor_w_with_contents( + { + { + {{-0.875, -0.75}, {-0.625, -0.5}}, + {{-0.375, -0.25}, {-0.125, 0}}, + }, + { + {{0.125, 0.25}, {0.375, 0.5}}, + {{0.625, 0.75}, {0.875, 1}}, + }, + }, + allocator); + + // Initialize with nonzero values to verify replacement. + GenericTensorAccessorW input_grad = + create_4d_accessor_w_with_contents( + { + { + {{-1.75, -1.5}, {-1.25, -1}}, + {{-0.75, -0.5}, {-0.25, 0}}, + }, + { + {{0.25, 0.5}, {0.75, 1}}, + {{1.25, 1.5}, {1.75, 2}}, + }, + }, + allocator); + + GenericTensorAccessorW gamma_grad = + create_1d_accessor_w_with_contents({7, -3}, allocator); + + GenericTensorAccessorW beta_grad = + create_1d_accessor_w_with_contents({0.5, 11}, allocator); + + gpu_backward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*per_device_state=*/per_device_state, + /*output_ptr=*/output.get_float_ptr(), + /*output_grad_ptr=*/output_grad.get_float_ptr(), + /*input_ptr=*/input.get_float_ptr(), + /*input_grad_ptr=*/input_grad.get_float_ptr(), + /*scale_ptr=*/gamma.get_float_ptr(), + /*scale_grad_ptr=*/gamma_grad.get_float_ptr(), + /*bias_grad_ptr=*/beta_grad.get_float_ptr(), + /*numElements=*/ + get_num_elements(input.shape.dims).int_from_positive_int()); + + GenericTensorAccessorR correct_input_grad = + create_4d_accessor_r_with_contents( + { + { + {{0.0727379322052002, -0.01039218902587891}, + {-0.09352242946624756, -0.1766525506973267}}, + {{-0.2091808915138245, -0.1475732326507568}, + {-0.1587578058242798, -0.1127485111355782}}, + }, + { + {{0.1766526103019714, 0.09352242946624756}, + {0.01039218902587891, -0.07273799180984497}}, + {{0.05490469932556152, 0.134710431098938}, + {0.2405133247375488, 0.1981320381164551}}, + }, + }, + allocator); + + GenericTensorAccessorR correct_gamma_grad = + create_1d_accessor_r_with_contents( + {4.037027359008789, 0.7804887294769287}, allocator); + + GenericTensorAccessorR correct_beta_grad = + create_1d_accessor_r_with_contents({-1.5, 2.5}, allocator); + + CHECK_MESSAGE( + // cuDNN's batch norm data gradient is not bit-identical to PyTorch's + accessors_are_approximately_equal(input_grad, correct_input_grad, 1e-4), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + + CHECK_MESSAGE( + accessors_are_approximately_equal(gamma_grad, correct_gamma_grad), + check_kv("gamma_grad", format_accessor_w_contents(gamma_grad))); + + CHECK_MESSAGE(accessors_are_equal(beta_grad, correct_beta_grad), + check_kv("beta_grad", format_accessor_w_contents(beta_grad))); + + gpu_cleanup_kernel(allocator, per_device_state); + } +} + +TEST_SUITE(FF_CUDA_TEST_SUITE) { + TEST_CASE("BatchNorm configurable epsilon and fused ReLU") { + ManagedPerDeviceFFHandle handle = initialize_single_gpu_handle( + /*workSpaceSize=*/1024 * 1024, + /*allowTensorOpMathConversion=*/true); + ManagedFFStream stream{}; + Allocator allocator = create_local_cuda_memory_allocator(); + std::optional activation = std::nullopt; + SUBCASE("without activation") {} + SUBCASE("ReLU") { + activation = Activation::RELU; + } + + GenericTensorAccessorR input = create_4d_accessor_r_with_contents( + {{{{1}}}, {{{3}}}}, allocator); + GenericTensorAccessorR scale = + create_1d_accessor_r_with_contents({2}, allocator); + GenericTensorAccessorR bias = + create_1d_accessor_r_with_contents({0.5}, allocator); + GenericTensorAccessorW output = + create_random_filled_accessor_w(input.shape, allocator); + BatchNormPerDeviceState state = gpu_init_kernel( + handle.raw_handle(), allocator, nullptr, 2, 1, 1, 1, activation, 0.25f); + + TensorShape channel_shape{TensorDims{FFOrdered{1_p}}, DataType::FLOAT}; + GenericTensorAccessorR running_var{ + channel_shape, state.runningVar, DeviceType::GPU}; + GenericTensorAccessorR one = + create_1d_accessor_r_with_contents({1}, allocator); + CHECK(accessors_are_equal(running_var, one)); + + gpu_forward_kernel(stream.raw_stream(), + state, + input.get_float_ptr(), + output.get_float_ptr(), + scale.get_float_ptr(), + bias.get_float_ptr()); + GenericTensorAccessorR expected = create_4d_accessor_r_with_contents( + {{{{activation.has_value() ? 0.0f : -1.2888543819998317f}}}, + {{{2.2888543819998317f}}}}, + allocator); + CHECK(accessors_are_approximately_equal(output, expected)); + + GenericTensorAccessorW output_grad = + create_4d_accessor_w_with_contents({{{{1}}}, {{{2}}}}, + allocator); + GenericTensorAccessorW input_grad = + create_random_filled_accessor_w(input.shape, allocator); + GenericTensorAccessorW scale_grad = + create_random_filled_accessor_w(channel_shape, allocator); + GenericTensorAccessorW bias_grad = + create_random_filled_accessor_w(channel_shape, allocator); + gpu_backward_kernel(stream.raw_stream(), + state, + output.get_float_ptr(), + output_grad.get_float_ptr(), + input.get_float_ptr(), + input_grad.get_float_ptr(), + scale.get_float_ptr(), + scale_grad.get_float_ptr(), + bias_grad.get_float_ptr(), + 2); + + float dx = + activation.has_value() ? 0.35777087639996635f : 0.17888543819998318f; + GenericTensorAccessorR expected_input_grad = + create_4d_accessor_r_with_contents({{{{-dx}}}, {{{dx}}}}, + allocator); + GenericTensorAccessorR expected_scale_grad = + create_1d_accessor_r_with_contents({activation.has_value() + ? 1.7888543819998317f + : 0.8944271909999159f}, + allocator); + GenericTensorAccessorR expected_bias_grad = + create_1d_accessor_r_with_contents( + {activation.has_value() ? 2.0f : 3.0f}, allocator); + CHECK(accessors_are_approximately_equal(input_grad, expected_input_grad)); + CHECK(accessors_are_approximately_equal(scale_grad, expected_scale_grad)); + CHECK(accessors_are_equal(bias_grad, expected_bias_grad)); + gpu_cleanup_kernel(allocator, state); + } +} diff --git a/lib/kernels/test/src/kernels/concat_kernels_cpu.cc b/lib/kernels/test/src/kernels/concat_kernels_cpu.cc index 2bc64379fd..97f7562714 100644 --- a/lib/kernels/test/src/kernels/concat_kernels_cpu.cc +++ b/lib/kernels/test/src/kernels/concat_kernels_cpu.cc @@ -1,6 +1,7 @@ #include "kernels/concat_kernels_cpu.h" #include "kernels/accessors_are_equal.h" #include "kernels/create_accessor_with_contents.h" +#include "kernels/create_random_filled_accessor.h" #include "kernels/create_zero_filled_accessor.h" #include "kernels/format_accessor_contents.h" #include "kernels/local_cpu_allocator.h" @@ -114,3 +115,257 @@ TEST_SUITE(FF_TEST_SUITE) { check_kv("input_grad_2", format_accessor_w_contents(input_grad_2))); } } + +TEST_SUITE(FF_TEST_SUITE) { + TEST_CASE("concat_cpu_forward_kernel (additional shapes)") { + Allocator allocator = create_local_cpu_memory_allocator(); + + std::vector inputs = { + create_2d_accessor_r_with_contents( + { + {1, 2, 3}, + {4, 5, 6}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-1, -2, -3}, + {-4, -5, -6}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {0.5, 0.25, 0.125}, + {10, 20, 30}, + }, + allocator), + }; + + SUBCASE("axis = 0") { + ff_dim_t axis = ff_dim_t{0_n}; + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{6_p, 3_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never + // read it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + concat_cpu_forward_kernel( + /*output=*/output, + /*inputs=*/inputs, + /*axis=*/axis); + + GenericTensorAccessorR correct = + create_2d_accessor_r_with_contents( + { + {1, 2, 3}, + {4, 5, 6}, + {-1, -2, -3}, + {-4, -5, -6}, + {0.5, 0.25, 0.125}, + {10, 20, 30}, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + } + + SUBCASE("axis = 1") { + ff_dim_t axis = ff_dim_t{1_n}; + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{2_p, 9_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never + // read it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + concat_cpu_forward_kernel( + /*output=*/output, + /*inputs=*/inputs, + /*axis=*/axis); + + GenericTensorAccessorR correct = + create_2d_accessor_r_with_contents( + { + {1, 2, 3, -1, -2, -3, 0.5, 0.25, 0.125}, + {4, 5, 6, -4, -5, -6, 10, 20, 30}, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + } + } + + TEST_CASE("concat_cpu_backward_kernel (replacement, additional shapes)") { + Allocator allocator = create_local_cpu_memory_allocator(); + + std::vector inputs = { + create_2d_accessor_r_with_contents( + { + {1, 2, 3}, + {4, 5, 6}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-1, -2, -3}, + {-4, -5, -6}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {0.5, 0.25, 0.125}, + {10, 20, 30}, + }, + allocator), + }; + + // Initialize with nonzero values to verify replacement. + auto make_input_grads = [&]() { + return std::vector{ + create_2d_accessor_w_with_contents( + { + {100, 101, 102}, + {103, 104, 105}, + }, + allocator), + create_2d_accessor_w_with_contents( + { + {200, 201, 202}, + {203, 204, 205}, + }, + allocator), + create_2d_accessor_w_with_contents( + { + {300, 301, 302}, + {303, 304, 305}, + }, + allocator), + }; + }; + + SUBCASE("axis = 0") { + ff_dim_t axis = ff_dim_t{0_n}; + + GenericTensorAccessorR output = create_2d_accessor_r_with_contents( + { + {1, 2, 3}, + {4, 5, 6}, + {-1, -2, -3}, + {-4, -5, -6}, + {0.5, 0.25, 0.125}, + {10, 20, 30}, + }, + allocator); + + GenericTensorAccessorR output_grad = + create_2d_accessor_r_with_contents( + { + {-1.75, -1.5, -1.25}, + {-1, -0.75, -0.5}, + {-0.25, 0, 0.25}, + {0.5, 0.75, 1}, + {1.25, 1.5, 1.75}, + {2, 2.25, 2.5}, + }, + allocator); + + std::vector input_grads = make_input_grads(); + + concat_cpu_backward_kernel( + /*output_grad=*/output_grad, + /*input_grads=*/input_grads, + /*axis=*/axis); + + std::vector correct = { + create_2d_accessor_r_with_contents( + { + {-1.75, -1.5, -1.25}, + {-1, -0.75, -0.5}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-0.25, 0, 0.25}, + {0.5, 0.75, 1}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {1.25, 1.5, 1.75}, + {2, 2.25, 2.5}, + }, + allocator), + }; + + for (int i = 0; i < input_grads.size(); i++) { + CHECK_MESSAGE(accessors_are_equal(input_grads.at(i), correct.at(i)), + check_kv("input_grad", + format_accessor_w_contents(input_grads.at(i)))); + } + } + + SUBCASE("axis = 1") { + ff_dim_t axis = ff_dim_t{1_n}; + + GenericTensorAccessorR output = create_2d_accessor_r_with_contents( + { + {1, 2, 3, -1, -2, -3, 0.5, 0.25, 0.125}, + {4, 5, 6, -4, -5, -6, 10, 20, 30}, + }, + allocator); + + GenericTensorAccessorR output_grad = + create_2d_accessor_r_with_contents( + { + {-1.75, -1.5, -1.25, -1, -0.75, -0.5, -0.25, 0, 0.25}, + {0.5, 0.75, 1, 1.25, 1.5, 1.75, 2, 2.25, 2.5}, + }, + allocator); + + std::vector input_grads = make_input_grads(); + + concat_cpu_backward_kernel( + /*output_grad=*/output_grad, + /*input_grads=*/input_grads, + /*axis=*/axis); + + std::vector correct = { + create_2d_accessor_r_with_contents( + { + {-1.75, -1.5, -1.25}, + {0.5, 0.75, 1}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-1, -0.75, -0.5}, + {1.25, 1.5, 1.75}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-0.25, 0, 0.25}, + {2, 2.25, 2.5}, + }, + allocator), + }; + + for (int i = 0; i < input_grads.size(); i++) { + CHECK_MESSAGE(accessors_are_equal(input_grads.at(i), correct.at(i)), + check_kv("input_grad", + format_accessor_w_contents(input_grads.at(i)))); + } + } + } +} diff --git a/lib/kernels/test/src/kernels/concat_kernels_gpu.cc b/lib/kernels/test/src/kernels/concat_kernels_gpu.cc new file mode 100644 index 0000000000..9daf2db89e --- /dev/null +++ b/lib/kernels/test/src/kernels/concat_kernels_gpu.cc @@ -0,0 +1,274 @@ +#include "kernels/concat_kernels_gpu.h" +#include "kernels/accessors_are_approximately_equal.h" +#include "kernels/accessors_are_equal.h" +#include "kernels/create_accessor_with_contents.h" +#include "kernels/create_random_filled_accessor.h" +#include "kernels/format_accessor_contents.h" +#include "kernels/local_cuda_allocator.h" +#include "kernels/managed_ff_stream.h" +#include "test/utils/doctest/check_kv.h" +#include + +using namespace ::FlexFlow; + +TEST_SUITE(FF_CUDA_TEST_SUITE) { + TEST_CASE("concat_gpu_forward_kernel") { + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + std::vector inputs = { + create_2d_accessor_r_with_contents( + { + {1, 2, 3}, + {4, 5, 6}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-1, -2, -3}, + {-4, -5, -6}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {0.5, 0.25, 0.125}, + {10, 20, 30}, + }, + allocator), + }; + + SUBCASE("axis = 0") { + ff_dim_t axis = ff_dim_t{0_n}; + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{6_p, 3_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never + // read it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + concat_gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*output=*/output, + /*inputs=*/inputs, + /*axis=*/axis); + + GenericTensorAccessorR correct = + create_2d_accessor_r_with_contents( + { + {1, 2, 3}, + {4, 5, 6}, + {-1, -2, -3}, + {-4, -5, -6}, + {0.5, 0.25, 0.125}, + {10, 20, 30}, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + } + + SUBCASE("axis = 1") { + ff_dim_t axis = ff_dim_t{1_n}; + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{2_p, 9_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never + // read it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + concat_gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*output=*/output, + /*inputs=*/inputs, + /*axis=*/axis); + + GenericTensorAccessorR correct = + create_2d_accessor_r_with_contents( + { + {1, 2, 3, -1, -2, -3, 0.5, 0.25, 0.125}, + {4, 5, 6, -4, -5, -6, 10, 20, 30}, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + } + } + + TEST_CASE("concat_gpu_backward_kernel") { + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + std::vector inputs = { + create_2d_accessor_r_with_contents( + { + {1, 2, 3}, + {4, 5, 6}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-1, -2, -3}, + {-4, -5, -6}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {0.5, 0.25, 0.125}, + {10, 20, 30}, + }, + allocator), + }; + + // Initialize with nonzero values to verify replacement. + auto make_input_grads = [&]() { + return std::vector{ + create_2d_accessor_w_with_contents( + { + {100, 101, 102}, + {103, 104, 105}, + }, + allocator), + create_2d_accessor_w_with_contents( + { + {200, 201, 202}, + {203, 204, 205}, + }, + allocator), + create_2d_accessor_w_with_contents( + { + {300, 301, 302}, + {303, 304, 305}, + }, + allocator), + }; + }; + + SUBCASE("axis = 0") { + ff_dim_t axis = ff_dim_t{0_n}; + + GenericTensorAccessorR output = create_2d_accessor_r_with_contents( + { + {1, 2, 3}, + {4, 5, 6}, + {-1, -2, -3}, + {-4, -5, -6}, + {0.5, 0.25, 0.125}, + {10, 20, 30}, + }, + allocator); + + GenericTensorAccessorR output_grad = + create_2d_accessor_r_with_contents( + { + {-1.75, -1.5, -1.25}, + {-1, -0.75, -0.5}, + {-0.25, 0, 0.25}, + {0.5, 0.75, 1}, + {1.25, 1.5, 1.75}, + {2, 2.25, 2.5}, + }, + allocator); + + std::vector input_grads = make_input_grads(); + + concat_gpu_backward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*output_grad=*/output_grad, + /*input_grads=*/input_grads, + /*axis=*/axis); + + std::vector correct = { + create_2d_accessor_r_with_contents( + { + {-1.75, -1.5, -1.25}, + {-1, -0.75, -0.5}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-0.25, 0, 0.25}, + {0.5, 0.75, 1}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {1.25, 1.5, 1.75}, + {2, 2.25, 2.5}, + }, + allocator), + }; + + for (int i = 0; i < input_grads.size(); i++) { + CHECK_MESSAGE(accessors_are_equal(input_grads.at(i), correct.at(i)), + check_kv("input_grad", + format_accessor_w_contents(input_grads.at(i)))); + } + } + + SUBCASE("axis = 1") { + ff_dim_t axis = ff_dim_t{1_n}; + + GenericTensorAccessorR output = create_2d_accessor_r_with_contents( + { + {1, 2, 3, -1, -2, -3, 0.5, 0.25, 0.125}, + {4, 5, 6, -4, -5, -6, 10, 20, 30}, + }, + allocator); + + GenericTensorAccessorR output_grad = + create_2d_accessor_r_with_contents( + { + {-1.75, -1.5, -1.25, -1, -0.75, -0.5, -0.25, 0, 0.25}, + {0.5, 0.75, 1, 1.25, 1.5, 1.75, 2, 2.25, 2.5}, + }, + allocator); + + std::vector input_grads = make_input_grads(); + + concat_gpu_backward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*output_grad=*/output_grad, + /*input_grads=*/input_grads, + /*axis=*/axis); + + std::vector correct = { + create_2d_accessor_r_with_contents( + { + {-1.75, -1.5, -1.25}, + {0.5, 0.75, 1}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-1, -0.75, -0.5}, + {1.25, 1.5, 1.75}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-0.25, 0, 0.25}, + {2, 2.25, 2.5}, + }, + allocator), + }; + + for (int i = 0; i < input_grads.size(); i++) { + CHECK_MESSAGE(accessors_are_equal(input_grads.at(i), correct.at(i)), + check_kv("input_grad", + format_accessor_w_contents(input_grads.at(i)))); + } + } + } +} diff --git a/lib/kernels/test/src/kernels/conv_2d_kernels_gpu.cc b/lib/kernels/test/src/kernels/conv_2d_kernels_gpu.cc new file mode 100644 index 0000000000..027861a005 --- /dev/null +++ b/lib/kernels/test/src/kernels/conv_2d_kernels_gpu.cc @@ -0,0 +1,370 @@ +#include "kernels/conv_2d_kernels_gpu.h" +#include "kernels/accessors_are_approximately_equal.h" +#include "kernels/accessors_are_equal.h" +#include "kernels/create_accessor_with_contents.h" +#include "kernels/create_random_filled_accessor.h" +#include "kernels/format_accessor_contents.h" +#include "kernels/local_cuda_allocator.h" +#include "kernels/managed_ff_stream.h" +#include "kernels/managed_per_device_ff_handle.h" +#include "test/utils/doctest/check_kv.h" +#include + +using namespace ::FlexFlow; + +static Conv2DAttrs make_attrs() { + return Conv2DAttrs{ + /*out_channels=*/3_p, + /*kernel_h=*/3_p, + /*kernel_w=*/3_p, + /*stride_h=*/1_p, + /*stride_w=*/1_p, + /*padding_h=*/1_n, + /*padding_w=*/1_n, + /*groups=*/1_p, + /*activation=*/std::nullopt, + /*use_bias=*/true, + }; +} + +// NCHW +static GenericTensorAccessorR make_input(Allocator &allocator) { + return create_4d_accessor_r_with_contents( + { + { + { + {1, 2, 3, 4}, + {0.5, 0.25, -1, -2}, + {-3, 0.125, 2, 1}, + {4, -0.5, 0.75, 0}, + }, + { + {-1, 0.5, 2, -2}, + {3, 1, 0.25, 0.5}, + {-0.25, -1.5, 1.25, 2.5}, + {0.75, 3, -4, 1}, + }, + }, + }, + allocator); +} + +static GenericTensorAccessorR make_filter(Allocator &allocator) { + return create_4d_accessor_r_with_contents( + { + { + {{1, 0, -1}, {0.5, 1, 0.5}, {-1, 0, 1}}, + {{0.25, -0.5, 0.75}, {1, -1, 0.5}, {0, 0.25, -0.25}}, + }, + { + {{-1, 2, 0.5}, {0, 1, -1}, {0.25, 0.5, 0.75}}, + {{1, 1, 1}, {-0.5, -0.5, -0.5}, {0.25, 0, -0.25}}, + }, + { + {{0.5, 0.5, 0.5}, {0.25, 0.25, 0.25}, {-1, -1, -1}}, + {{2, -1, 0}, {0.5, 0.75, -0.25}, {1, 0.5, 0.25}}, + }, + }, + allocator); +} + +static GenericTensorAccessorR make_bias(Allocator &allocator) { + return create_1d_accessor_r_with_contents({0.5, -1, 2}, allocator); +} + +static GenericTensorAccessorR make_output_grad(Allocator &allocator) { + return create_4d_accessor_r_with_contents( + { + { + { + {-0.9375, -0.875, -0.8125, -0.75}, + {-0.6875, -0.625, -0.5625, -0.5}, + {-0.4375, -0.375, -0.3125, -0.25}, + {-0.1875, -0.125, -0.0625, 0}, + }, + { + {0.0625, 0.125, 0.1875, 0.25}, + {0.3125, 0.375, 0.4375, 0.5}, + {0.5625, 0.625, 0.6875, 0.75}, + {0.8125, 0.875, 0.9375, 1}, + }, + { + {1.0625, 1.125, 1.1875, 1.25}, + {1.3125, 1.375, 1.4375, 1.5}, + {1.5625, 1.625, 1.6875, 1.75}, + {1.8125, 1.875, 1.9375, 2}, + }, + }, + }, + allocator); +} + +TEST_SUITE(FF_CUDA_TEST_SUITE) { + TEST_CASE("conv2d_gpu_forward_kernel") { + ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( + /*workSpaceSize=*/1024 * 1024, + /*allowTensorOpMathConversion=*/true); + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + Conv2DAttrs attrs = make_attrs(); + + GenericTensorAccessorR input = make_input(allocator); + GenericTensorAccessorR filter = make_filter(allocator); + GenericTensorAccessorR bias = make_bias(allocator); + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{1_p, 3_p, 4_p, 4_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never read + // it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + GenericTensorAccessorW init_input = allocator.allocate_tensor(input.shape); + copy_accessor_data_to_l_from_r(init_input, input); + GenericTensorAccessorW init_output = + allocator.allocate_tensor(output.shape); + GenericTensorAccessorW init_filter_grad = + allocator.allocate_tensor(filter.shape); + Conv2DPerDeviceState per_device_state = + conv2d_gpu_init_kernel(managed_handle.raw_handle(), + attrs.activation, + 3, + 3, + 1, + 1, + 1, + 1, + 1, + init_input, + init_output, + filter.get_float_ptr(), + init_filter_grad.get_float_ptr()); + + conv2d_gpu_forward_kernel(managed_stream.raw_stream(), + per_device_state, + input.get_float_ptr(), + output.get_float_ptr(), + filter.get_float_ptr(), + bias.get_float_ptr(), + attrs.activation); + + GenericTensorAccessorR correct = create_4d_accessor_r_with_contents( + { + { + { + {4.5, 2.6875, 1.6875, 11.125}, + {-2.0625, 5.9375, -4.1875, 0.875}, + {-5, 2.4375, 3.5625, -0.4375}, + {3.875, -5.25, 8.5, -3.0625}, + }, + { + {-1.5625, -2.5625, -4.0625, 1.8125}, + {-1.28125, 4.5625, 6.40625, 2.9375}, + {2.75, 3.625, -1.875, -4.9375}, + {-6.0625, 1.625, 6.375, 4.25}, + }, + { + {2.875, 6.6875, 10.5, 6.75}, + {9.0625, 4.8125, 2, 10.75}, + {-3.53125, 2.09375, 3.71875, -0.5}, + {1.5, 7.25, -2.375, 2.4375}, + }, + }, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + + conv2d_gpu_cleanup_kernel(per_device_state); + } + + TEST_CASE("conv2d_gpu_backward_kernel") { + ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( + /*workSpaceSize=*/1024 * 1024, + /*allowTensorOpMathConversion=*/true); + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + Conv2DAttrs attrs = make_attrs(); + + GenericTensorAccessorR input = make_input(allocator); + GenericTensorAccessorR filter = make_filter(allocator); + GenericTensorAccessorR output_grad = make_output_grad(allocator); + + GenericTensorAccessorR output = create_4d_accessor_r_with_contents( + { + { + { + {4.5, 2.6875, 1.6875, 11.125}, + {-2.0625, 5.9375, -4.1875, 0.875}, + {-5, 2.4375, 3.5625, -0.4375}, + {3.875, -5.25, 8.5, -3.0625}, + }, + { + {-1.5625, -2.5625, -4.0625, 1.8125}, + {-1.28125, 4.5625, 6.40625, 2.9375}, + {2.75, 3.625, -1.875, -4.9375}, + {-6.0625, 1.625, 6.375, 4.25}, + }, + { + {2.875, 6.6875, 10.5, 6.75}, + {9.0625, 4.8125, 2, 10.75}, + {-3.53125, 2.09375, 3.71875, -0.5}, + {1.5, 7.25, -2.375, 2.4375}, + }, + }, + }, + allocator); + + // Initialize with nonzero values to verify replacement. + GenericTensorAccessorW input_grad = + create_4d_accessor_w_with_contents( + { + { + { + {-2.75, -2.5, -2.25, -2}, + {-1.75, -1.5, -1.25, -1}, + {-0.75, -0.5, -0.25, 0}, + {0.25, 0.5, 0.75, 1}, + }, + { + {1.25, 1.5, 1.75, 2}, + {2.25, 2.5, 2.75, 3}, + {3.25, 3.5, 3.75, 4}, + {4.25, 4.5, 4.75, 5}, + }, + }, + }, + allocator); + + GenericTensorAccessorW filter_grad = create_4d_accessor_w_with_contents< + float>( + { + { + {{-1.875, -1.75, -1.625}, + {-1.5, -1.375, -1.25}, + {-1.125, -1, -0.875}}, + {{-0.75, -0.625, -0.5}, + {-0.375, -0.25, -0.125}, + {0, 0.125, 0.25}}, + }, + { + {{0.375, 0.5, 0.625}, {0.75, 0.875, 1}, {1.125, 1.25, 1.375}}, + {{1.5, 1.625, 1.75}, {1.875, 2, 2.125}, {2.25, 2.375, 2.5}}, + }, + { + {{2.625, 2.75, 2.875}, {3, 3.125, 3.25}, {3.375, 3.5, 3.625}}, + {{3.75, 3.875, 4}, {4.125, 4.25, 4.375}, {4.5, 4.625, 4.75}}, + }, + }, + allocator); + + GenericTensorAccessorW bias_grad = + create_1d_accessor_w_with_contents({10, -5, 0.25}, allocator); + + GenericTensorAccessorW init_input = allocator.allocate_tensor(input.shape); + copy_accessor_data_to_l_from_r(init_input, input); + GenericTensorAccessorW init_output = + allocator.allocate_tensor(output.shape); + GenericTensorAccessorW init_filter_grad = + allocator.allocate_tensor(filter.shape); + Conv2DPerDeviceState per_device_state = + conv2d_gpu_init_kernel(managed_handle.raw_handle(), + attrs.activation, + 3, + 3, + 1, + 1, + 1, + 1, + 1, + init_input, + init_output, + filter.get_float_ptr(), + init_filter_grad.get_float_ptr()); + + conv2d_gpu_backward_kernel(managed_stream.raw_stream(), + per_device_state, + output.get_float_ptr(), + output_grad.get_float_ptr(), + input.get_float_ptr(), + input_grad.get_float_ptr(), + filter.get_float_ptr(), + filter_grad.get_float_ptr(), + bias_grad.get_float_ptr(), + attrs.activation); + + GenericTensorAccessorR correct_input_grad = + create_4d_accessor_r_with_contents( + { + { + { + {0.203125, 1.8125, 2.171875, 2.765625}, + {0.453125, -0.09375, 0.171875, 0.90625}, + {1.390625, 0.96875, 1.234375, 2.09375}, + {-0.890625, -2.875, -2.796875, -1.84375}, + }, + { + {3.640625, 2.859375, 3.140625, 0.03125}, + {5.84375, 6.046875, 6.4375, 0.984375}, + {7.09375, 7.609375, 8, 1.171875}, + {3.96875, 3.515625, 3.625, 1.15625}, + }, + }, + }, + allocator); + + GenericTensorAccessorR correct_filter_grad = + create_4d_accessor_r_with_contents( + { + { + {{-2.8984375, -4.703125, -4.9453125}, + {-4.5859375, -7.40625, -7.2890625}, + {-0.6171875, 0.1875, 0.3046875}}, + {{-2.03125, -1.484375, -1.1875}, + {-3.625, -3.3125, -2.4375}, + {-3.46875, -5.5625, -2.84375}}, + }, + { + {{1.9765625, 3.171875, 4.4296875}, + {4.5390625, 4.71875, 2.3359375}, + {2.5078125, 2.3125, 0.9296875}}, + {{3.21875, 4.765625, 3.3125}, + {1.375, 3.6875, 2.0625}, + {0.03125, 1.9375, 1.15625}}, + }, + { + {{6.8515625, 11.046875, 13.8046875}, + {13.6640625, 16.84375, 11.9609375}, + {5.6328125, 4.4375, 1.5546875}}, + {{8.46875, 11.015625, 7.8125}, + {6.375, 10.6875, 6.5625}, + {3.53125, 9.4375, 5.15625}}, + }, + }, + allocator); + + GenericTensorAccessorR correct_bias_grad = + create_1d_accessor_r_with_contents({-7.5, 8.5, 24.5}, allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct_input_grad), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + + CHECK_MESSAGE( + accessors_are_equal(filter_grad, correct_filter_grad), + check_kv("filter_grad", format_accessor_w_contents(filter_grad))); + + CHECK_MESSAGE(accessors_are_equal(bias_grad, correct_bias_grad), + check_kv("bias_grad", format_accessor_w_contents(bias_grad))); + + conv2d_gpu_cleanup_kernel(per_device_state); + } +} diff --git a/lib/kernels/test/src/kernels/element_unary_kernels_gpu.cc b/lib/kernels/test/src/kernels/element_unary_kernels_gpu.cc index 378a4483a3..38f140e4f2 100644 --- a/lib/kernels/test/src/kernels/element_unary_kernels_gpu.cc +++ b/lib/kernels/test/src/kernels/element_unary_kernels_gpu.cc @@ -1,9 +1,9 @@ +#include "kernels/element_unary_kernels_gpu.h" +#include "kernels/accessors_are_approximately_equal.h" #include "kernels/accessors_are_equal.h" #include "kernels/create_accessor_with_contents.h" #include "kernels/create_random_filled_accessor.h" -#include "kernels/element_unary_kernels_cpu.h" #include "kernels/format_accessor_contents.h" -#include "kernels/local_cpu_allocator.h" #include "kernels/local_cuda_allocator.h" #include "kernels/managed_ff_stream.h" #include "kernels/managed_per_device_ff_handle.h" @@ -13,32 +13,61 @@ using namespace ::FlexFlow; +static ElementUnaryAttrs make_silu_attrs() { + return ElementUnaryAttrs{ + /*op=*/ElementUnaryOp::SILU, + /*scalar=*/std::nullopt, + }; +} + +static GenericTensorAccessorR make_input(Allocator &allocator) { + return create_2d_accessor_r_with_contents( + { + {3, -3, 6}, + {0, 1, 5}, + {1, 2, -2}, + {-8, 0.5, -3}, + }, + allocator); +} + +static GenericTensorAccessorR make_output_grad(Allocator &allocator) { + return create_2d_accessor_r_with_contents( + { + {1, 2, -1}, + {6, 4, 2}, + {0.5, 0.1, -2}, + {0, 0.5, 0}, + }, + allocator); +} + TEST_SUITE(FF_CUDA_TEST_SUITE) { TEST_CASE("element_unary_gpu_forward_kernel") { - SUBCASE("relu") { - ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( - /*workSpaceSize=*/1024 * 1024, - /*allowTensorOpMathConversion=*/true); - ManagedFFStream managed_stream{}; + ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( + /*workSpaceSize=*/1024 * 1024, + /*allowTensorOpMathConversion=*/true); + ManagedFFStream managed_stream{}; - Allocator allocator = create_local_cuda_memory_allocator(); + Allocator allocator = create_local_cuda_memory_allocator(); - ElementUnaryAttrs attrs = make_relu_attrs(); + GenericTensorAccessorR input = make_input(allocator); - GenericTensorAccessorR input = create_2d_accessor_r_with_contents( - { - {3, -3, 6}, - {0, 1, 5}, - {1, 2, -2}, - {-8, 0.5, -3}, - }, - allocator); + SUBCASE("relu") { + ElementUnaryAttrs attrs = make_relu_attrs(); - // Intentionally randomize this tensor so we can be confident we never read it + // Intentionally randomize this tensor so we can be confident we never + // read it GenericTensorAccessorW result = create_random_filled_accessor_w(input.shape, allocator); - element_unary_cpu_forward_kernel( + ElementUnaryPerDeviceState per_device_state = + element_unary_gpu_init_kernel(attrs, input.shape, result.shape); + + element_unary_gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*handle=*/managed_handle.raw_handle(), + /*per_device_state=*/per_device_state, /*attrs=*/attrs, /*input=*/input, /*output=*/result); @@ -56,52 +85,128 @@ TEST_SUITE(FF_CUDA_TEST_SUITE) { CHECK_MESSAGE(accessors_are_equal(result, correct), check_kv("result", format_accessor_w_contents(result))); } + + SUBCASE("silu") { + ElementUnaryAttrs attrs = make_silu_attrs(); + + // Intentionally randomize this tensor so we can be confident we never + // read it + GenericTensorAccessorW result = + create_random_filled_accessor_w(input.shape, allocator); + + ElementUnaryPerDeviceState per_device_state = + element_unary_gpu_init_kernel(attrs, input.shape, result.shape); + + element_unary_gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*handle=*/managed_handle.raw_handle(), + /*per_device_state=*/per_device_state, + /*attrs=*/attrs, + /*input=*/input, + /*output=*/result); + + GenericTensorAccessorR correct = create_2d_accessor_r_with_contents< + float>( + { + {2.857722520828247, -0.14227761328220367, 5.985164642333984}, + {0.0, 0.7310585975646973, 4.966535568237305}, + {0.7310585975646973, 1.7615940570831299, -0.23840583860874176}, + {-0.00268280110321939, 0.3112296760082245, -0.14227761328220367}, + }, + allocator); + + // CUDA's expf is not bit-identical to PyTorch's CPU implementation + CHECK_MESSAGE(accessors_are_approximately_equal(result, correct, 1e-6), + check_kv("result", format_accessor_w_contents(result))); + } } TEST_CASE("element_unary_gpu_backward_kernel") { - SUBCASE("relu") { - ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( - /*workSpaceSize=*/1024 * 1024, - /*allowTensorOpMathConversion=*/true); - ManagedFFStream managed_stream{}; + ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( + /*workSpaceSize=*/1024 * 1024, + /*allowTensorOpMathConversion=*/true); + ManagedFFStream managed_stream{}; - Allocator allocator = create_local_cuda_memory_allocator(); + Allocator allocator = create_local_cuda_memory_allocator(); - ElementUnaryAttrs attrs = make_relu_attrs(); + GenericTensorAccessorR input = make_input(allocator); + GenericTensorAccessorR output_grad = make_output_grad(allocator); - GenericTensorAccessorR input = create_2d_accessor_r_with_contents( + // Initialize with nonzero values to verify replacement. + auto make_input_grad = [&]() { + return create_2d_accessor_w_with_contents( { - {3, -3, 6}, - {0, 1, 5}, - {1, 2, -2}, - {-8, 0.5, -3}, + {10, 11, 12}, + {13, 14, 15}, + {16, 17, 18}, + {19, 20, 21}, }, allocator); + }; - // Intentionally randomize this tensor so we can be confident we never read it - GenericTensorAccessorW input_grad = create_random_filled_accessor_w( - get_tensor_shape_for_accessor_r(input), allocator); + SUBCASE("relu") { + ElementUnaryAttrs attrs = make_relu_attrs(); GenericTensorAccessorR output = create_2d_accessor_r_with_contents( { {3, 0, 6}, {0, 1, 5}, - {1, 2, -2}, + {1, 2, 0}, {0, 0.5, 0}, }, allocator); - GenericTensorAccessorR output_grad = + GenericTensorAccessorW input_grad = make_input_grad(); + + ElementUnaryPerDeviceState per_device_state = + element_unary_gpu_init_kernel(attrs, input.shape, output.shape); + + element_unary_gpu_backward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*handle=*/managed_handle.raw_handle(), + /*per_device_state=*/per_device_state, + /*attrs=*/attrs, + /*output=*/output, + /*output_grad=*/output_grad, + /*input=*/input, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct_input_grad = create_2d_accessor_r_with_contents( { - {1, 2, -1}, - {6, 4, 2}, - {0.5, 0.1, -2}, + {1, 0, -1}, + {0, 4, 2}, + {0.5, 0.1, 0}, {0, 0.5, 0}, }, allocator); - element_unary_cpu_backward_kernel( + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct_input_grad), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } + + SUBCASE("silu") { + ElementUnaryAttrs attrs = make_silu_attrs(); + + GenericTensorAccessorR output = create_2d_accessor_r_with_contents( + { + {2.857722520828247, -0.14227761328220367, 5.985164642333984}, + {0.0, 0.7310585975646973, 4.966535568237305}, + {0.7310585975646973, 1.7615940570831299, -0.23840583860874176}, + {-0.00268280110321939, 0.3112296760082245, -0.14227761328220367}, + }, + allocator); + + GenericTensorAccessorW input_grad = make_input_grad(); + + ElementUnaryPerDeviceState per_device_state = + element_unary_gpu_init_kernel(attrs, input.shape, output.shape); + + element_unary_gpu_backward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*handle=*/managed_handle.raw_handle(), + /*per_device_state=*/per_device_state, /*attrs=*/attrs, /*output=*/output, /*output_grad=*/output_grad, @@ -111,15 +216,16 @@ TEST_SUITE(FF_CUDA_TEST_SUITE) { GenericTensorAccessorR correct_input_grad = create_2d_accessor_r_with_contents( { - {1.0f, 0.0f, -1.0f}, - {0.0f, 4.0f, 2.0f}, - {0.5f, 0.1f, 0.0f}, - {0.0f, 0.5f, 0.0f}, + {1.088104106015169, -0.1762082120303393, -1.012326432591525}, + {3, 3.710682047485947, 2.053094864859331}, + {0.4638352559357434, 0.1090784248784896, 0.1815684975697909}, + {-0, 0.3699805936513259, -0}, }, allocator); CHECK_MESSAGE( - accessors_are_equal(input_grad, correct_input_grad), + accessors_are_approximately_equal( + input_grad, correct_input_grad, 1e-5), check_kv("input_grad", format_accessor_w_contents(input_grad))); } } diff --git a/lib/kernels/test/src/kernels/pool_2d_kernels_gpu.cc b/lib/kernels/test/src/kernels/pool_2d_kernels_gpu.cc new file mode 100644 index 0000000000..4539080eb0 --- /dev/null +++ b/lib/kernels/test/src/kernels/pool_2d_kernels_gpu.cc @@ -0,0 +1,349 @@ +#include "kernels/pool_2d_kernels_gpu.h" +#include "kernels/accessors_are_approximately_equal.h" +#include "kernels/accessors_are_equal.h" +#include "kernels/create_accessor_with_contents.h" +#include "kernels/create_random_filled_accessor.h" +#include "kernels/format_accessor_contents.h" +#include "kernels/local_cuda_allocator.h" +#include "kernels/managed_ff_stream.h" +#include "kernels/managed_per_device_ff_handle.h" +#include "test/utils/doctest/check_kv.h" +#include + +using namespace ::FlexFlow; + +// NCHW +static GenericTensorAccessorR make_input(Allocator &allocator) { + return create_4d_accessor_r_with_contents( + { + { + { + {1, 2, 3, 4}, + {5, 6, 7, 8}, + {-1, -2, -3, -4}, + {0.5, 0.25, 0.125, 0.0625}, + }, + { + {9, 1, 0.5, -7}, + {2, 3, 0.25, 0.125}, + {-8, 4, 10, 20}, + {6, -5, 30, 40}, + }, + }, + }, + allocator); +} + +// Initialize with nonzero values to verify replacement. +static GenericTensorAccessorW make_input_grad(Allocator &allocator) { + return create_4d_accessor_w_with_contents( + { + { + { + {-4.5, -4, -3.5, -3}, + {-2.5, -2, -1.5, -1}, + {-0.5, 0, 0.5, 1}, + {1.5, 2, 2.5, 3}, + }, + { + {3.5, 4, 4.5, 5}, + {5.5, 6, 6.5, 7}, + {7.5, 8, 8.5, 9}, + {9.5, 10, 10.5, 11}, + }, + }, + }, + allocator); +} + +static GenericTensorAccessorR make_output_grad(Allocator &allocator) { + return create_4d_accessor_r_with_contents( + { + { + { + {-0.75, -0.5}, + {-0.25, 0}, + }, + { + {0.25, 0.5}, + {0.75, 1}, + }, + }, + }, + allocator); +} + +static Pool2DAttrs make_attrs(PoolOp pool_type) { + return Pool2DAttrs{ + /*kernel_h=*/2_p, + /*kernel_w=*/2_p, + /*stride_h=*/2_p, + /*stride_w=*/2_p, + /*padding_h=*/0_n, + /*padding_w=*/0_n, + /*pool_type=*/pool_type, + /*activation=*/std::nullopt, + }; +} + +static TensorShape output_shape() { + return TensorShape{ + TensorDims{FFOrdered{1_p, 2_p, 2_p, 2_p}}, + DataType::FLOAT, + }; +} + +TEST_SUITE(FF_CUDA_TEST_SUITE) { + TEST_CASE("pool2d_gpu_forward_kernel") { + ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( + /*workSpaceSize=*/1024 * 1024, + /*allowTensorOpMathConversion=*/true); + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + GenericTensorAccessorR input = make_input(allocator); + + SUBCASE("PoolOp::MAX") { + Pool2DAttrs attrs = make_attrs(PoolOp::MAX); + + // Intentionally randomize this tensor so we can be confident we never + // read it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape(), allocator); + + Pool2DPerDeviceState per_device_state = + pool2d_gpu_init_kernel(managed_handle.raw_handle(), + attrs.activation, + 4, + 4, + 2, + 1, + 2, + 2, + 2, + 1, + 0, + 0, + 2, + 2, + 2, + 2, + attrs.pool_type); + + pool2d_gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*per_device_state=*/per_device_state, + /*input_ptr=*/input.ptr, + /*output_ptr=*/output.ptr); + + GenericTensorAccessorR correct = + create_4d_accessor_r_with_contents( + { + { + {{6, 8}, {0.5, 0.125}}, + {{9, 0.5}, {6, 40}}, + }, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + pool2d_gpu_cleanup_kernel(per_device_state); + } + + SUBCASE("PoolOp::AVG") { + Pool2DAttrs attrs = make_attrs(PoolOp::AVG); + + // Intentionally randomize this tensor so we can be confident we never + // read it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape(), allocator); + + Pool2DPerDeviceState per_device_state = + pool2d_gpu_init_kernel(managed_handle.raw_handle(), + attrs.activation, + 4, + 4, + 2, + 1, + 2, + 2, + 2, + 1, + 0, + 0, + 2, + 2, + 2, + 2, + attrs.pool_type); + + pool2d_gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*per_device_state=*/per_device_state, + /*input_ptr=*/input.ptr, + /*output_ptr=*/output.ptr); + + GenericTensorAccessorR correct = + create_4d_accessor_r_with_contents( + { + { + {{3.5, 5.5}, {-0.5625, -1.703125}}, + {{3.75, -1.53125}, {-0.75, 25}}, + }, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + pool2d_gpu_cleanup_kernel(per_device_state); + } + } + + TEST_CASE("pool2d_gpu_backward_kernel") { + ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( + /*workSpaceSize=*/1024 * 1024, + /*allowTensorOpMathConversion=*/true); + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + GenericTensorAccessorR input = make_input(allocator); + GenericTensorAccessorR output_grad = make_output_grad(allocator); + + SUBCASE("PoolOp::MAX") { + Pool2DAttrs attrs = make_attrs(PoolOp::MAX); + + GenericTensorAccessorR output = create_4d_accessor_r_with_contents( + { + { + {{6, 8}, {0.5, 0.125}}, + {{9, 0.5}, {6, 40}}, + }, + }, + allocator); + + GenericTensorAccessorW input_grad = make_input_grad(allocator); + + Pool2DPerDeviceState per_device_state = + pool2d_gpu_init_kernel(managed_handle.raw_handle(), + attrs.activation, + 4, + 4, + 2, + 1, + 2, + 2, + 2, + 1, + 0, + 0, + 2, + 2, + 2, + 2, + attrs.pool_type); + + pool2d_gpu_backward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*per_device_state=*/per_device_state, + /*output_ptr=*/output.ptr, + /*output_grad_ptr=*/output_grad.ptr, + /*input_ptr=*/input.ptr, + /*input_grad_ptr=*/input_grad.ptr); + + GenericTensorAccessorR correct = + create_4d_accessor_r_with_contents( + { + { + { + {0, 0, 0, 0}, + {0, -0.75, 0, -0.5}, + {0, 0, 0, 0}, + {-0.25, 0, 0, 0}, + }, + { + {0.25, 0, 0.5, 0}, + {0, 0, 0, 0}, + {0, 0, 0, 0}, + {0.75, 0, 0, 1}, + }, + }, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + pool2d_gpu_cleanup_kernel(per_device_state); + } + + SUBCASE("PoolOp::AVG") { + Pool2DAttrs attrs = make_attrs(PoolOp::AVG); + + GenericTensorAccessorR output = create_4d_accessor_r_with_contents( + { + { + {{3.5, 5.5}, {-0.5625, -1.703125}}, + {{3.75, -1.53125}, {-0.75, 25}}, + }, + }, + allocator); + + GenericTensorAccessorW input_grad = make_input_grad(allocator); + + Pool2DPerDeviceState per_device_state = + pool2d_gpu_init_kernel(managed_handle.raw_handle(), + attrs.activation, + 4, + 4, + 2, + 1, + 2, + 2, + 2, + 1, + 0, + 0, + 2, + 2, + 2, + 2, + attrs.pool_type); + + pool2d_gpu_backward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*per_device_state=*/per_device_state, + /*output_ptr=*/output.ptr, + /*output_grad_ptr=*/output_grad.ptr, + /*input_ptr=*/input.ptr, + /*input_grad_ptr=*/input_grad.ptr); + + GenericTensorAccessorR correct = + create_4d_accessor_r_with_contents( + { + { + { + {-0.1875, -0.1875, -0.125, -0.125}, + {-0.1875, -0.1875, -0.125, -0.125}, + {-0.0625, -0.0625, 0, 0}, + {-0.0625, -0.0625, 0, 0}, + }, + { + {0.0625, 0.0625, 0.125, 0.125}, + {0.0625, 0.0625, 0.125, 0.125}, + {0.1875, 0.1875, 0.25, 0.25}, + {0.1875, 0.1875, 0.25, 0.25}, + }, + }, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + pool2d_gpu_cleanup_kernel(per_device_state); + } + } +} diff --git a/lib/kernels/test/src/kernels/reshape_kernels_cpu.cc b/lib/kernels/test/src/kernels/reshape_kernels_cpu.cc index 7f98d0bac5..5293f53808 100644 --- a/lib/kernels/test/src/kernels/reshape_kernels_cpu.cc +++ b/lib/kernels/test/src/kernels/reshape_kernels_cpu.cc @@ -3,6 +3,7 @@ #include "kernels/create_accessor_with_contents.h" #include "kernels/create_random_filled_accessor.h" #include "kernels/format_accessor_contents.h" +#include "kernels/local_cpu_allocator.h" #include "test/utils/doctest/check_kv.h" #include @@ -149,3 +150,93 @@ TEST_SUITE(FF_TEST_SUITE) { check_kv("result", format_accessor_w_contents(result))); } } + +TEST_SUITE(FF_TEST_SUITE) { + TEST_CASE("reshape_cpu_forward_kernel (additional shapes)") { + Allocator allocator = create_local_cpu_memory_allocator(); + + GenericTensorAccessorR input = create_2d_accessor_r_with_contents( + { + {1, 2, 3}, + {4, 5, 6}, + }, + allocator); + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{3_p, 2_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never read + // it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + reshape_cpu_forward_kernel( + /*input=*/input, + /*output=*/output); + + GenericTensorAccessorR correct = create_2d_accessor_r_with_contents( + { + {1, 2}, + {3, 4}, + {5, 6}, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + } + + TEST_CASE("reshape_cpu_backward_kernel (replacement, additional shapes)") { + Allocator allocator = create_local_cpu_memory_allocator(); + + GenericTensorAccessorR input = create_2d_accessor_r_with_contents( + { + {1, 2, 3}, + {4, 5, 6}, + }, + allocator); + + GenericTensorAccessorR output = create_2d_accessor_r_with_contents( + { + {1, 2}, + {3, 4}, + {5, 6}, + }, + allocator); + + GenericTensorAccessorR output_grad = + create_2d_accessor_r_with_contents( + { + {1, 2}, + {-3, 4}, + {0.5, 0.25}, + }, + allocator); + + // Initialize with nonzero values to verify replacement. + GenericTensorAccessorW input_grad = + create_2d_accessor_w_with_contents( + { + {10, 20, 30}, + {40, 50, 60}, + }, + allocator); + + reshape_cpu_backward_kernel( + /*output_grad=*/output_grad, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct = create_2d_accessor_r_with_contents( + { + {1, 2, -3}, + {4, 0.5, 0.25}, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } +} diff --git a/lib/kernels/test/src/kernels/reshape_kernels_gpu.cc b/lib/kernels/test/src/kernels/reshape_kernels_gpu.cc new file mode 100644 index 0000000000..dee08a89da --- /dev/null +++ b/lib/kernels/test/src/kernels/reshape_kernels_gpu.cc @@ -0,0 +1,108 @@ +#include "kernels/reshape_kernels_gpu.h" +#include "kernels/accessors_are_approximately_equal.h" +#include "kernels/accessors_are_equal.h" +#include "kernels/create_accessor_with_contents.h" +#include "kernels/create_random_filled_accessor.h" +#include "kernels/format_accessor_contents.h" +#include "kernels/local_cuda_allocator.h" +#include "kernels/managed_ff_stream.h" +#include "test/utils/doctest/check_kv.h" +#include + +using namespace ::FlexFlow; + +TEST_SUITE(FF_CUDA_TEST_SUITE) { + TEST_CASE("reshape_gpu_forward_kernel") { + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + GenericTensorAccessorR input = create_2d_accessor_r_with_contents( + { + {1, 2, 3}, + {4, 5, 6}, + }, + allocator); + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{3_p, 2_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never read + // it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + reshape_gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*input=*/input, + /*output=*/output); + + GenericTensorAccessorR correct = create_2d_accessor_r_with_contents( + { + {1, 2}, + {3, 4}, + {5, 6}, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + } + + TEST_CASE("reshape_gpu_backward_kernel") { + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + GenericTensorAccessorR input = create_2d_accessor_r_with_contents( + { + {1, 2, 3}, + {4, 5, 6}, + }, + allocator); + + GenericTensorAccessorR output = create_2d_accessor_r_with_contents( + { + {1, 2}, + {3, 4}, + {5, 6}, + }, + allocator); + + GenericTensorAccessorR output_grad = + create_2d_accessor_r_with_contents( + { + {1, 2}, + {-3, 4}, + {0.5, 0.25}, + }, + allocator); + + // Initialize with nonzero values to verify replacement. + GenericTensorAccessorW input_grad = + create_2d_accessor_w_with_contents( + { + {10, 20, 30}, + {40, 50, 60}, + }, + allocator); + + reshape_gpu_backward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*output_grad=*/output_grad, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct = create_2d_accessor_r_with_contents( + { + {1, 2, -3}, + {4, 0.5, 0.25}, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } +} diff --git a/lib/kernels/test/src/kernels/split_kernels_cpu.cc b/lib/kernels/test/src/kernels/split_kernels_cpu.cc index bccd5c32d9..ed025b07cd 100644 --- a/lib/kernels/test/src/kernels/split_kernels_cpu.cc +++ b/lib/kernels/test/src/kernels/split_kernels_cpu.cc @@ -1,9 +1,11 @@ #include "kernels/split_kernels_cpu.h" #include "kernels/accessors_are_equal.h" #include "kernels/create_accessor_with_contents.h" +#include "kernels/create_random_filled_accessor.h" #include "kernels/create_zero_filled_accessor.h" #include "kernels/format_accessor_contents.h" #include "kernels/local_cpu_allocator.h" +#include "kernels/split_kernels.h" #include "test/utils/doctest/check_kv.h" #include @@ -136,3 +138,283 @@ TEST_SUITE(FF_TEST_SUITE) { check_kv("input_grad", format_accessor_w_contents(input_grad))); } } + +static GenericTensorAccessorR make_input(Allocator &allocator) { + return create_2d_accessor_r_with_contents( + { + {1, 2, 3, 4, 5, 6}, + {-1, -2, -3, -4, -5, -6}, + {0.5, 0.25, 0.125, 10, 20, 30}, + {7, 8, 9, 0.75, 0.5, 0.25}, + }, + allocator); +} + +// Initialize with nonzero values to verify replacement. +static GenericTensorAccessorW make_input_grad(Allocator &allocator) { + return create_2d_accessor_w_with_contents( + { + {-3, -1, 1, 3, 5, 7}, + {9, 11, 13, 15, 17, 19}, + {21, 23, 25, 27, 29, 31}, + {33, 35, 37, 39, 41, 43}, + }, + allocator); +} + +TEST_SUITE(FF_TEST_SUITE) { + TEST_CASE("split_cpu_forward_kernel (additional shapes)") { + Allocator allocator = create_local_cpu_memory_allocator(); + + GenericTensorAccessorR input = make_input(allocator); + + SUBCASE("axis = 0") { + SplitAttrs attrs = SplitAttrs{ + /*splits=*/{1_p, 3_p}, + /*axis=*/ff_dim_t{0_n}, + }; + + // Intentionally randomize these tensors so we can be confident we never + // read them + std::vector outputs = { + create_random_filled_accessor_w( + TensorShape{TensorDims{FFOrdered{1_p, 6_p}}, DataType::FLOAT}, + allocator), + create_random_filled_accessor_w( + TensorShape{TensorDims{FFOrdered{3_p, 6_p}}, DataType::FLOAT}, + allocator), + }; + + split_forward_kernel(device_stream_t{std::monostate{}}, + /*attrs=*/attrs, + /*input=*/input, + /*outputs=*/outputs); + + std::vector correct = { + create_2d_accessor_r_with_contents( + { + {1, 2, 3, 4, 5, 6}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-1, -2, -3, -4, -5, -6}, + {0.5, 0.25, 0.125, 10, 20, 30}, + {7, 8, 9, 0.75, 0.5, 0.25}, + }, + allocator), + }; + + for (int i = 0; i < outputs.size(); i++) { + CHECK_MESSAGE( + accessors_are_equal(outputs.at(i), correct.at(i)), + check_kv("output", format_accessor_w_contents(outputs.at(i)))); + } + } + + SUBCASE("axis = 1") { + SplitAttrs attrs = SplitAttrs{ + /*splits=*/{2_p, 1_p, 3_p}, + /*axis=*/ff_dim_t{1_n}, + }; + + // Intentionally randomize these tensors so we can be confident we never + // read them + std::vector outputs = { + create_random_filled_accessor_w( + TensorShape{TensorDims{FFOrdered{4_p, 2_p}}, DataType::FLOAT}, + allocator), + create_random_filled_accessor_w( + TensorShape{TensorDims{FFOrdered{4_p, 1_p}}, DataType::FLOAT}, + allocator), + create_random_filled_accessor_w( + TensorShape{TensorDims{FFOrdered{4_p, 3_p}}, DataType::FLOAT}, + allocator), + }; + + split_forward_kernel(device_stream_t{std::monostate{}}, + /*attrs=*/attrs, + /*input=*/input, + /*outputs=*/outputs); + + std::vector correct = { + create_2d_accessor_r_with_contents( + { + {1, 2}, + {-1, -2}, + {0.5, 0.25}, + {7, 8}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {3}, + {-3}, + {0.125}, + {9}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {4, 5, 6}, + {-4, -5, -6}, + {10, 20, 30}, + {0.75, 0.5, 0.25}, + }, + allocator), + }; + + for (int i = 0; i < outputs.size(); i++) { + CHECK_MESSAGE( + accessors_are_equal(outputs.at(i), correct.at(i)), + check_kv("output", format_accessor_w_contents(outputs.at(i)))); + } + } + } + + TEST_CASE("split_cpu_backward_kernel (replacement, additional shapes)") { + Allocator allocator = create_local_cpu_memory_allocator(); + + GenericTensorAccessorR input = make_input(allocator); + + SUBCASE("axis = 0") { + SplitAttrs attrs = SplitAttrs{ + /*splits=*/{1_p, 3_p}, + /*axis=*/ff_dim_t{0_n}, + }; + + std::vector outputs = { + create_2d_accessor_r_with_contents( + { + {1, 2, 3, 4, 5, 6}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-1, -2, -3, -4, -5, -6}, + {0.5, 0.25, 0.125, 10, 20, 30}, + {7, 8, 9, 0.75, 0.5, 0.25}, + }, + allocator), + }; + + std::vector output_grads = { + create_2d_accessor_r_with_contents( + { + {-0.75, -0.5, -0.25, 0, 0.25, 0.5}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {9.25, 9.5, 9.75, 10, 10.25, 10.5}, + {10.75, 11, 11.25, 11.5, 11.75, 12}, + {12.25, 12.5, 12.75, 13, 13.25, 13.5}, + }, + allocator), + }; + + GenericTensorAccessorW input_grad = make_input_grad(allocator); + + split_backward_kernel(device_stream_t{std::monostate{}}, + /*attrs=*/attrs, + /*output_grads=*/output_grads, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct = + create_2d_accessor_r_with_contents( + { + {-0.75, -0.5, -0.25, 0, 0.25, 0.5}, + {9.25, 9.5, 9.75, 10, 10.25, 10.5}, + {10.75, 11, 11.25, 11.5, 11.75, 12}, + {12.25, 12.5, 12.75, 13, 13.25, 13.5}, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } + + SUBCASE("axis = 1") { + SplitAttrs attrs = SplitAttrs{ + /*splits=*/{2_p, 1_p, 3_p}, + /*axis=*/ff_dim_t{1_n}, + }; + + std::vector outputs = { + create_2d_accessor_r_with_contents( + { + {1, 2}, + {-1, -2}, + {0.5, 0.25}, + {7, 8}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {3}, + {-3}, + {0.125}, + {9}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {4, 5, 6}, + {-4, -5, -6}, + {10, 20, 30}, + {0.75, 0.5, 0.25}, + }, + allocator), + }; + + std::vector output_grads = { + create_2d_accessor_r_with_contents( + { + {-0.75, -0.5}, + {-0.25, 0}, + {0.25, 0.5}, + {0.75, 1}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {9.25}, + {9.5}, + {9.75}, + {10}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {19.25, 19.5, 19.75}, + {20, 20.25, 20.5}, + {20.75, 21, 21.25}, + {21.5, 21.75, 22}, + }, + allocator), + }; + + GenericTensorAccessorW input_grad = make_input_grad(allocator); + + split_backward_kernel(device_stream_t{std::monostate{}}, + /*attrs=*/attrs, + /*output_grads=*/output_grads, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct = + create_2d_accessor_r_with_contents( + { + {-0.75, -0.5, 9.25, 19.25, 19.5, 19.75}, + {-0.25, 0, 9.5, 20, 20.25, 20.5}, + {0.25, 0.5, 9.75, 20.75, 21, 21.25}, + {0.75, 1, 10, 21.5, 21.75, 22}, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } + } +} diff --git a/lib/kernels/test/src/kernels/split_kernels_gpu.cc b/lib/kernels/test/src/kernels/split_kernels_gpu.cc new file mode 100644 index 0000000000..f16ef997a5 --- /dev/null +++ b/lib/kernels/test/src/kernels/split_kernels_gpu.cc @@ -0,0 +1,300 @@ +#include "kernels/accessors_are_approximately_equal.h" +#include "kernels/accessors_are_equal.h" +#include "kernels/create_accessor_with_contents.h" +#include "kernels/create_random_filled_accessor.h" +#include "kernels/format_accessor_contents.h" +#include "kernels/local_cuda_allocator.h" +#include "kernels/managed_ff_stream.h" +#include "kernels/split_kernels.h" +#include "test/utils/doctest/check_kv.h" +#include + +using namespace ::FlexFlow; + +static GenericTensorAccessorR make_input(Allocator &allocator) { + return create_2d_accessor_r_with_contents( + { + {1, 2, 3, 4, 5, 6}, + {-1, -2, -3, -4, -5, -6}, + {0.5, 0.25, 0.125, 10, 20, 30}, + {7, 8, 9, 0.75, 0.5, 0.25}, + }, + allocator); +} + +// Initialize with nonzero values to verify replacement. +static GenericTensorAccessorW make_input_grad(Allocator &allocator) { + return create_2d_accessor_w_with_contents( + { + {-3, -1, 1, 3, 5, 7}, + {9, 11, 13, 15, 17, 19}, + {21, 23, 25, 27, 29, 31}, + {33, 35, 37, 39, 41, 43}, + }, + allocator); +} + +TEST_SUITE(FF_CUDA_TEST_SUITE) { + TEST_CASE("split_gpu_forward_kernel") { + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + GenericTensorAccessorR input = make_input(allocator); + + SUBCASE("axis = 0") { + SplitAttrs attrs = SplitAttrs{ + /*splits=*/{1_p, 3_p}, + /*axis=*/ff_dim_t{0_n}, + }; + + // Intentionally randomize these tensors so we can be confident we never + // read them + std::vector outputs = { + create_random_filled_accessor_w( + TensorShape{TensorDims{FFOrdered{1_p, 6_p}}, DataType::FLOAT}, + allocator), + create_random_filled_accessor_w( + TensorShape{TensorDims{FFOrdered{3_p, 6_p}}, DataType::FLOAT}, + allocator), + }; + + split_forward_kernel( + /*stream=*/device_stream_t{managed_stream.raw_stream()}, + /*attrs=*/attrs, + /*input=*/input, + /*outputs=*/outputs); + + std::vector correct = { + create_2d_accessor_r_with_contents( + { + {1, 2, 3, 4, 5, 6}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-1, -2, -3, -4, -5, -6}, + {0.5, 0.25, 0.125, 10, 20, 30}, + {7, 8, 9, 0.75, 0.5, 0.25}, + }, + allocator), + }; + + for (int i = 0; i < outputs.size(); i++) { + CHECK_MESSAGE( + accessors_are_equal(outputs.at(i), correct.at(i)), + check_kv("output", format_accessor_w_contents(outputs.at(i)))); + } + } + + SUBCASE("axis = 1") { + SplitAttrs attrs = SplitAttrs{ + /*splits=*/{2_p, 1_p, 3_p}, + /*axis=*/ff_dim_t{1_n}, + }; + + // Intentionally randomize these tensors so we can be confident we never + // read them + std::vector outputs = { + create_random_filled_accessor_w( + TensorShape{TensorDims{FFOrdered{4_p, 2_p}}, DataType::FLOAT}, + allocator), + create_random_filled_accessor_w( + TensorShape{TensorDims{FFOrdered{4_p, 1_p}}, DataType::FLOAT}, + allocator), + create_random_filled_accessor_w( + TensorShape{TensorDims{FFOrdered{4_p, 3_p}}, DataType::FLOAT}, + allocator), + }; + + split_forward_kernel( + /*stream=*/device_stream_t{managed_stream.raw_stream()}, + /*attrs=*/attrs, + /*input=*/input, + /*outputs=*/outputs); + + std::vector correct = { + create_2d_accessor_r_with_contents( + { + {1, 2}, + {-1, -2}, + {0.5, 0.25}, + {7, 8}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {3}, + {-3}, + {0.125}, + {9}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {4, 5, 6}, + {-4, -5, -6}, + {10, 20, 30}, + {0.75, 0.5, 0.25}, + }, + allocator), + }; + + for (int i = 0; i < outputs.size(); i++) { + CHECK_MESSAGE( + accessors_are_equal(outputs.at(i), correct.at(i)), + check_kv("output", format_accessor_w_contents(outputs.at(i)))); + } + } + } + + TEST_CASE("split_gpu_backward_kernel") { + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + GenericTensorAccessorR input = make_input(allocator); + + SUBCASE("axis = 0") { + SplitAttrs attrs = SplitAttrs{ + /*splits=*/{1_p, 3_p}, + /*axis=*/ff_dim_t{0_n}, + }; + + std::vector outputs = { + create_2d_accessor_r_with_contents( + { + {1, 2, 3, 4, 5, 6}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {-1, -2, -3, -4, -5, -6}, + {0.5, 0.25, 0.125, 10, 20, 30}, + {7, 8, 9, 0.75, 0.5, 0.25}, + }, + allocator), + }; + + std::vector output_grads = { + create_2d_accessor_r_with_contents( + { + {-0.75, -0.5, -0.25, 0, 0.25, 0.5}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {9.25, 9.5, 9.75, 10, 10.25, 10.5}, + {10.75, 11, 11.25, 11.5, 11.75, 12}, + {12.25, 12.5, 12.75, 13, 13.25, 13.5}, + }, + allocator), + }; + + GenericTensorAccessorW input_grad = make_input_grad(allocator); + + split_backward_kernel( + /*stream=*/device_stream_t{managed_stream.raw_stream()}, + /*attrs=*/attrs, + /*output_grads=*/output_grads, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct = + create_2d_accessor_r_with_contents( + { + {-0.75, -0.5, -0.25, 0, 0.25, 0.5}, + {9.25, 9.5, 9.75, 10, 10.25, 10.5}, + {10.75, 11, 11.25, 11.5, 11.75, 12}, + {12.25, 12.5, 12.75, 13, 13.25, 13.5}, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } + + SUBCASE("axis = 1") { + SplitAttrs attrs = SplitAttrs{ + /*splits=*/{2_p, 1_p, 3_p}, + /*axis=*/ff_dim_t{1_n}, + }; + + std::vector outputs = { + create_2d_accessor_r_with_contents( + { + {1, 2}, + {-1, -2}, + {0.5, 0.25}, + {7, 8}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {3}, + {-3}, + {0.125}, + {9}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {4, 5, 6}, + {-4, -5, -6}, + {10, 20, 30}, + {0.75, 0.5, 0.25}, + }, + allocator), + }; + + std::vector output_grads = { + create_2d_accessor_r_with_contents( + { + {-0.75, -0.5}, + {-0.25, 0}, + {0.25, 0.5}, + {0.75, 1}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {9.25}, + {9.5}, + {9.75}, + {10}, + }, + allocator), + create_2d_accessor_r_with_contents( + { + {19.25, 19.5, 19.75}, + {20, 20.25, 20.5}, + {20.75, 21, 21.25}, + {21.5, 21.75, 22}, + }, + allocator), + }; + + GenericTensorAccessorW input_grad = make_input_grad(allocator); + + split_backward_kernel( + /*stream=*/device_stream_t{managed_stream.raw_stream()}, + /*attrs=*/attrs, + /*output_grads=*/output_grads, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct = + create_2d_accessor_r_with_contents( + { + {-0.75, -0.5, 9.25, 19.25, 19.5, 19.75}, + {-0.25, 0, 9.5, 20, 20.25, 20.5}, + {0.25, 0.5, 9.75, 20.75, 21, 21.25}, + {0.75, 1, 10, 21.5, 21.75, 22}, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } + } +} diff --git a/lib/kernels/test/src/kernels/transpose_kernels_cpu.cc b/lib/kernels/test/src/kernels/transpose_kernels_cpu.cc index 19004b559a..c5dab18205 100644 --- a/lib/kernels/test/src/kernels/transpose_kernels_cpu.cc +++ b/lib/kernels/test/src/kernels/transpose_kernels_cpu.cc @@ -3,6 +3,8 @@ #include "kernels/create_accessor_with_contents.h" #include "kernels/create_random_filled_accessor.h" #include "kernels/format_accessor_contents.h" +#include "kernels/local_cpu_allocator.h" +#include "kernels/transpose_kernels.h" #include "test/utils/doctest/check_kv.h" #include @@ -177,3 +179,149 @@ TEST_SUITE(FF_TEST_SUITE) { check_kv("result", format_accessor_w_contents(result))); } } + +// The permutation below maps each output dim to the input dim it comes from +// (i.e. `output[d] = input[permutation.at_l(d)]`), so it is equivalent to +// torch.permute(input, (1, 2, 0)). +static TransposeAttrs make_test_attrs() { + return TransposeAttrs{ + TensorDimPermutation{ + bidict{ + {ff_dim_t{0_n}, ff_dim_t{1_n}}, + {ff_dim_t{1_n}, ff_dim_t{2_n}}, + {ff_dim_t{2_n}, ff_dim_t{0_n}}, + }, + }, + }; +} + +TEST_SUITE(FF_TEST_SUITE) { + TEST_CASE("transpose_cpu_forward_kernel (additional shapes)") { + Allocator allocator = create_local_cpu_memory_allocator(); + + TransposeAttrs attrs = make_test_attrs(); + + GenericTensorAccessorR input = create_3d_accessor_r_with_contents( + { + { + {0.25, 0.5, 0.75, 1.0}, + {1.25, 1.5, 1.75, 2.0}, + {2.25, 2.5, 2.75, 3.0}, + }, + { + {3.25, 3.5, 3.75, 4.0}, + {4.25, 4.5, 4.75, 5.0}, + {5.25, 5.5, 5.75, 6.0}, + }, + }, + allocator); + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{3_p, 4_p, 2_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never read + // it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + transpose_forward_kernel(device_stream_t{std::monostate{}}, + /*attrs=*/attrs, + /*input=*/input, + /*output=*/output); + + GenericTensorAccessorR correct = create_3d_accessor_r_with_contents( + { + {{0.25, 3.25}, {0.5, 3.5}, {0.75, 3.75}, {1.0, 4.0}}, + {{1.25, 4.25}, {1.5, 4.5}, {1.75, 4.75}, {2.0, 5.0}}, + {{2.25, 5.25}, {2.5, 5.5}, {2.75, 5.75}, {3.0, 6.0}}, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + } + + TEST_CASE("transpose_cpu_backward_kernel (replacement, additional shapes)") { + Allocator allocator = create_local_cpu_memory_allocator(); + + TransposeAttrs attrs = make_test_attrs(); + + GenericTensorAccessorR input = create_3d_accessor_r_with_contents( + { + { + {0.25, 0.5, 0.75, 1.0}, + {1.25, 1.5, 1.75, 2.0}, + {2.25, 2.5, 2.75, 3.0}, + }, + { + {3.25, 3.5, 3.75, 4.0}, + {4.25, 4.5, 4.75, 5.0}, + {5.25, 5.5, 5.75, 6.0}, + }, + }, + allocator); + + GenericTensorAccessorR output = create_3d_accessor_r_with_contents( + { + {{0.25, 3.25}, {0.5, 3.5}, {0.75, 3.75}, {1.0, 4.0}}, + {{1.25, 4.25}, {1.5, 4.5}, {1.75, 4.75}, {2.0, 5.0}}, + {{2.25, 5.25}, {2.5, 5.5}, {2.75, 5.75}, {3.0, 6.0}}, + }, + allocator); + + GenericTensorAccessorR output_grad = + create_3d_accessor_r_with_contents( + { + {{-0.875, -0.75}, + {-0.625, -0.5}, + {-0.375, -0.25}, + {-0.125, 0.0}}, + {{0.125, 0.25}, {0.375, 0.5}, {0.625, 0.75}, {0.875, 1.0}}, + {{1.125, 1.25}, {1.375, 1.5}, {1.625, 1.75}, {1.875, 2.0}}, + }, + allocator); + + // Initialize with nonzero values to verify replacement. + GenericTensorAccessorW input_grad = + create_3d_accessor_w_with_contents( + { + { + {-2.5, -2.0, -1.5, -1.0}, + {-0.5, 0.0, 0.5, 1.0}, + {1.5, 2.0, 2.5, 3.0}, + }, + { + {3.5, 4.0, 4.5, 5.0}, + {5.5, 6.0, 6.5, 7.0}, + {7.5, 8.0, 8.5, 9.0}, + }, + }, + allocator); + + transpose_backward_kernel(device_stream_t{std::monostate{}}, + /*attrs=*/attrs, + /*output_grad=*/output_grad, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct = create_3d_accessor_r_with_contents( + { + { + {-0.875, -0.625, -0.375, -0.125}, + {0.125, 0.375, 0.625, 0.875}, + {1.125, 1.375, 1.625, 1.875}, + }, + { + {-0.75, -0.5, -0.25, 0}, + {0.25, 0.5, 0.75, 1}, + {1.25, 1.5, 1.75, 2}, + }, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } +} diff --git a/lib/kernels/test/src/kernels/transpose_kernels_gpu.cc b/lib/kernels/test/src/kernels/transpose_kernels_gpu.cc new file mode 100644 index 0000000000..17bd31622f --- /dev/null +++ b/lib/kernels/test/src/kernels/transpose_kernels_gpu.cc @@ -0,0 +1,164 @@ +#include "kernels/transpose_kernels_gpu.h" +#include "kernels/accessors_are_approximately_equal.h" +#include "kernels/accessors_are_equal.h" +#include "kernels/create_accessor_with_contents.h" +#include "kernels/create_random_filled_accessor.h" +#include "kernels/format_accessor_contents.h" +#include "kernels/local_cuda_allocator.h" +#include "kernels/managed_ff_stream.h" +#include "test/utils/doctest/check_kv.h" +#include + +using namespace ::FlexFlow; + +// The permutation below maps each output dim to the input dim it comes from +// (i.e. `output[d] = input[permutation.at_l(d)]`), so it is equivalent to +// torch.permute(input, (1, 2, 0)). +static TransposeAttrs make_test_attrs() { + return TransposeAttrs{ + TensorDimPermutation{ + bidict{ + {ff_dim_t{0_n}, ff_dim_t{1_n}}, + {ff_dim_t{1_n}, ff_dim_t{2_n}}, + {ff_dim_t{2_n}, ff_dim_t{0_n}}, + }, + }, + }; +} + +TEST_SUITE(FF_CUDA_TEST_SUITE) { + TEST_CASE("transpose_gpu_forward_kernel") { + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + TransposeAttrs attrs = make_test_attrs(); + + GenericTensorAccessorR input = create_3d_accessor_r_with_contents( + { + { + {0.25, 0.5, 0.75, 1.0}, + {1.25, 1.5, 1.75, 2.0}, + {2.25, 2.5, 2.75, 3.0}, + }, + { + {3.25, 3.5, 3.75, 4.0}, + {4.25, 4.5, 4.75, 5.0}, + {5.25, 5.5, 5.75, 6.0}, + }, + }, + allocator); + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{3_p, 4_p, 2_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never read + // it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + transpose_gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*attrs=*/attrs, + /*input=*/input, + /*output=*/output); + + GenericTensorAccessorR correct = create_3d_accessor_r_with_contents( + { + {{0.25, 3.25}, {0.5, 3.5}, {0.75, 3.75}, {1.0, 4.0}}, + {{1.25, 4.25}, {1.5, 4.5}, {1.75, 4.75}, {2.0, 5.0}}, + {{2.25, 5.25}, {2.5, 5.5}, {2.75, 5.75}, {3.0, 6.0}}, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + } + + TEST_CASE("transpose_gpu_backward_kernel") { + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + TransposeAttrs attrs = make_test_attrs(); + + GenericTensorAccessorR input = create_3d_accessor_r_with_contents( + { + { + {0.25, 0.5, 0.75, 1.0}, + {1.25, 1.5, 1.75, 2.0}, + {2.25, 2.5, 2.75, 3.0}, + }, + { + {3.25, 3.5, 3.75, 4.0}, + {4.25, 4.5, 4.75, 5.0}, + {5.25, 5.5, 5.75, 6.0}, + }, + }, + allocator); + + GenericTensorAccessorR output = create_3d_accessor_r_with_contents( + { + {{0.25, 3.25}, {0.5, 3.5}, {0.75, 3.75}, {1.0, 4.0}}, + {{1.25, 4.25}, {1.5, 4.5}, {1.75, 4.75}, {2.0, 5.0}}, + {{2.25, 5.25}, {2.5, 5.5}, {2.75, 5.75}, {3.0, 6.0}}, + }, + allocator); + + GenericTensorAccessorR output_grad = + create_3d_accessor_r_with_contents( + { + {{-0.875, -0.75}, + {-0.625, -0.5}, + {-0.375, -0.25}, + {-0.125, 0.0}}, + {{0.125, 0.25}, {0.375, 0.5}, {0.625, 0.75}, {0.875, 1.0}}, + {{1.125, 1.25}, {1.375, 1.5}, {1.625, 1.75}, {1.875, 2.0}}, + }, + allocator); + + // Initialize with nonzero values to verify replacement. + GenericTensorAccessorW input_grad = + create_3d_accessor_w_with_contents( + { + { + {-2.5, -2.0, -1.5, -1.0}, + {-0.5, 0.0, 0.5, 1.0}, + {1.5, 2.0, 2.5, 3.0}, + }, + { + {3.5, 4.0, 4.5, 5.0}, + {5.5, 6.0, 6.5, 7.0}, + {7.5, 8.0, 8.5, 9.0}, + }, + }, + allocator); + + transpose_gpu_backward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*attrs=*/attrs, + /*output_grad=*/output_grad, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct = create_3d_accessor_r_with_contents( + { + { + {-0.875, -0.625, -0.375, -0.125}, + {0.125, 0.375, 0.625, 0.875}, + {1.125, 1.375, 1.625, 1.875}, + }, + { + {-0.75, -0.5, -0.25, 0}, + {0.25, 0.5, 0.75, 1}, + {1.25, 1.5, 1.75, 2}, + }, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } +} diff --git a/lib/kernels/test/src/kernels/upsample_kernels_cpu.cc b/lib/kernels/test/src/kernels/upsample_kernels_cpu.cc new file mode 100644 index 0000000000..d8c13282b5 --- /dev/null +++ b/lib/kernels/test/src/kernels/upsample_kernels_cpu.cc @@ -0,0 +1,313 @@ +#include "kernels/upsample_kernels_cpu.h" +#include "kernels/accessors_are_equal.h" +#include "kernels/create_accessor_with_contents.h" +#include "kernels/create_random_filled_accessor.h" +#include "kernels/format_accessor_contents.h" +#include "kernels/local_cpu_allocator.h" +#include "test/utils/doctest/check_kv.h" +#include + +using namespace ::FlexFlow; + +TEST_SUITE(FF_TEST_SUITE) { + TEST_CASE("upsample_cpu_forward_kernel") { + Allocator allocator = create_local_cpu_memory_allocator(); + + UpsampleAttrs attrs = UpsampleAttrs{ + /*scale_factor=*/int_ge_two{2}, + /*mode=*/UpsampleMode::NEAREST, + }; + + // NCHW + GenericTensorAccessorR input = create_4d_accessor_r_with_contents( + { + { + { + {1, 2, 3}, + {4, 5, 6}, + }, + { + {-1, -2, -3}, + {0.5, 0.25, 0.125}, + }, + }, + }, + allocator); + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{1_p, 2_p, 4_p, 6_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never read + // it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + upsample_cpu_forward_kernel( + /*attrs=*/attrs, + /*input=*/input, + /*output=*/output); + + GenericTensorAccessorR correct = create_4d_accessor_r_with_contents( + { + { + { + {1, 1, 2, 2, 3, 3}, + {1, 1, 2, 2, 3, 3}, + {4, 4, 5, 5, 6, 6}, + {4, 4, 5, 5, 6, 6}, + }, + { + {-1, -1, -2, -2, -3, -3}, + {-1, -1, -2, -2, -3, -3}, + {0.5, 0.5, 0.25, 0.25, 0.125, 0.125}, + {0.5, 0.5, 0.25, 0.25, 0.125, 0.125}, + }, + }, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + } + + TEST_CASE("upsample_cpu_backward_kernel") { + Allocator allocator = create_local_cpu_memory_allocator(); + + UpsampleAttrs attrs = UpsampleAttrs{ + /*scale_factor=*/int_ge_two{2}, + /*mode=*/UpsampleMode::NEAREST, + }; + + GenericTensorAccessorR input = create_4d_accessor_r_with_contents( + { + { + { + {1, 2, 3}, + {4, 5, 6}, + }, + { + {-1, -2, -3}, + {0.5, 0.25, 0.125}, + }, + }, + }, + allocator); + + GenericTensorAccessorR output = create_4d_accessor_r_with_contents( + { + { + { + {1, 1, 2, 2, 3, 3}, + {1, 1, 2, 2, 3, 3}, + {4, 4, 5, 5, 6, 6}, + {4, 4, 5, 5, 6, 6}, + }, + { + {-1, -1, -2, -2, -3, -3}, + {-1, -1, -2, -2, -3, -3}, + {0.5, 0.5, 0.25, 0.25, 0.125, 0.125}, + {0.5, 0.5, 0.25, 0.25, 0.125, 0.125}, + }, + }, + }, + allocator); + + GenericTensorAccessorR output_grad = + create_4d_accessor_r_with_contents( + { + { + { + {-0.875, -0.75, -0.625, -0.5, -0.375, -0.25}, + {-0.125, 0, 0.125, 0.25, 0.375, 0.5}, + {0.625, 0.75, 0.875, 1, 1.125, 1.25}, + {1.375, 1.5, 1.625, 1.75, 1.875, 2}, + }, + { + {2.125, 2.25, 2.375, 2.5, 2.625, 2.75}, + {2.875, 3, 3.125, 3.25, 3.375, 3.5}, + {3.625, 3.75, 3.875, 4, 4.125, 4.25}, + {4.375, 4.5, 4.625, 4.75, 4.875, 5}, + }, + }, + }, + allocator); + + // Initialize with nonzero values to verify replacement. + GenericTensorAccessorW input_grad = + create_4d_accessor_w_with_contents( + { + { + { + {-7, -4, -1}, + {2, 5, 8}, + }, + { + {11, 14, 17}, + {20, 23, 26}, + }, + }, + }, + allocator); + + upsample_cpu_backward_kernel( + /*attrs=*/attrs, + /*output_grad=*/output_grad, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct = create_4d_accessor_r_with_contents( + { + { + { + {-1.75, -0.75, 0.25}, + {4.25, 5.25, 6.25}, + }, + { + {10.25, 11.25, 12.25}, + {16.25, 17.25, 18.25}, + }, + }, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } +} + +TEST_SUITE(FF_TEST_SUITE) { + TEST_CASE("upsample_cpu_forward_kernel (scale_factor = 3)") { + + Allocator allocator = create_local_cpu_memory_allocator(); + + UpsampleAttrs attrs{int_ge_two{3}, UpsampleMode::NEAREST}; + + GenericTensorAccessorR input = create_4d_accessor_r_with_contents( + { + { + { + {1, 2}, + {-3, 0.5}, + }, + }, + }, + allocator); + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{1_p, 1_p, 6_p, 6_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never read + // it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + upsample_cpu_forward_kernel( + /*attrs=*/attrs, + /*input=*/input, + /*output=*/output); + + GenericTensorAccessorR correct = create_4d_accessor_r_with_contents( + { + { + { + {1, 1, 1, 2, 2, 2}, + {1, 1, 1, 2, 2, 2}, + {1, 1, 1, 2, 2, 2}, + {-3, -3, -3, 0.5, 0.5, 0.5}, + {-3, -3, -3, 0.5, 0.5, 0.5}, + {-3, -3, -3, 0.5, 0.5, 0.5}, + }, + }, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + } + + TEST_CASE("upsample_cpu_backward_kernel (scale_factor = 3)") { + + Allocator allocator = create_local_cpu_memory_allocator(); + + UpsampleAttrs attrs{int_ge_two{3}, UpsampleMode::NEAREST}; + + GenericTensorAccessorR input = create_4d_accessor_r_with_contents( + { + { + { + {1, 2}, + {-3, 0.5}, + }, + }, + }, + allocator); + + GenericTensorAccessorR output = create_4d_accessor_r_with_contents( + { + { + { + {1, 1, 1, 2, 2, 2}, + {1, 1, 1, 2, 2, 2}, + {1, 1, 1, 2, 2, 2}, + {-3, -3, -3, 0.5, 0.5, 0.5}, + {-3, -3, -3, 0.5, 0.5, 0.5}, + {-3, -3, -3, 0.5, 0.5, 0.5}, + }, + }, + }, + allocator); + + GenericTensorAccessorR output_grad = + create_4d_accessor_r_with_contents( + { + { + { + {-3.75, -3.5, -3.25, -3, -2.75, -2.5}, + {-2.25, -2, -1.75, -1.5, -1.25, -1}, + {-0.75, -0.5, -0.25, 0, 0.25, 0.5}, + {0.75, 1, 1.25, 1.5, 1.75, 2}, + {2.25, 2.5, 2.75, 3, 3.25, 3.5}, + {3.75, 4, 4.25, 4.5, 4.75, 5}, + }, + }, + }, + allocator); + + // Initialize with nonzero values to verify replacement. + GenericTensorAccessorW input_grad = + create_4d_accessor_w_with_contents( + { + { + { + {100, 200}, + {300, 400}, + }, + }, + }, + allocator); + + upsample_cpu_backward_kernel( + /*attrs=*/attrs, + /*output_grad=*/output_grad, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct = create_4d_accessor_r_with_contents( + { + { + { + {-18, -11.25}, + {22.5, 29.25}, + }, + }, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } +} diff --git a/lib/kernels/test/src/kernels/upsample_kernels_gpu.cc b/lib/kernels/test/src/kernels/upsample_kernels_gpu.cc new file mode 100644 index 0000000000..10920bfd53 --- /dev/null +++ b/lib/kernels/test/src/kernels/upsample_kernels_gpu.cc @@ -0,0 +1,305 @@ +#include "kernels/upsample_kernels_gpu.h" +#include "kernels/accessors_are_approximately_equal.h" +#include "kernels/accessors_are_equal.h" +#include "kernels/create_accessor_with_contents.h" +#include "kernels/create_random_filled_accessor.h" +#include "kernels/format_accessor_contents.h" +#include "kernels/local_cuda_allocator.h" +#include "kernels/managed_ff_stream.h" +#include "test/utils/doctest/check_kv.h" +#include + +using namespace ::FlexFlow; + +static UpsampleAttrs make_attrs(int scale_factor) { + return UpsampleAttrs{ + /*scale_factor=*/int_ge_two{scale_factor}, + /*mode=*/UpsampleMode::NEAREST, + }; +} + +// NCHW +static GenericTensorAccessorR make_input(Allocator &allocator) { + return create_4d_accessor_r_with_contents( + { + { + { + {1, 2, 3}, + {4, 5, 6}, + }, + { + {-1, -2, -3}, + {0.5, 0.25, 0.125}, + }, + }, + }, + allocator); +} + +static GenericTensorAccessorR make_output(Allocator &allocator) { + return create_4d_accessor_r_with_contents( + { + { + { + {1, 1, 2, 2, 3, 3}, + {1, 1, 2, 2, 3, 3}, + {4, 4, 5, 5, 6, 6}, + {4, 4, 5, 5, 6, 6}, + }, + { + {-1, -1, -2, -2, -3, -3}, + {-1, -1, -2, -2, -3, -3}, + {0.5, 0.5, 0.25, 0.25, 0.125, 0.125}, + {0.5, 0.5, 0.25, 0.25, 0.125, 0.125}, + }, + }, + }, + allocator); +} + +TEST_SUITE(FF_CUDA_TEST_SUITE) { + TEST_CASE("upsample_gpu_forward_kernel") { + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + UpsampleAttrs attrs = make_attrs(/*scale_factor=*/2); + + GenericTensorAccessorR input = make_input(allocator); + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{1_p, 2_p, 4_p, 6_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never read + // it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + upsample_gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*attrs=*/attrs, + /*input=*/input, + /*output=*/output); + + GenericTensorAccessorR correct = make_output(allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + } + + TEST_CASE("upsample_gpu_backward_kernel") { + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + UpsampleAttrs attrs = make_attrs(/*scale_factor=*/2); + + GenericTensorAccessorR input = make_input(allocator); + GenericTensorAccessorR output = make_output(allocator); + + GenericTensorAccessorR output_grad = + create_4d_accessor_r_with_contents( + { + { + { + {-0.875, -0.75, -0.625, -0.5, -0.375, -0.25}, + {-0.125, 0, 0.125, 0.25, 0.375, 0.5}, + {0.625, 0.75, 0.875, 1, 1.125, 1.25}, + {1.375, 1.5, 1.625, 1.75, 1.875, 2}, + }, + { + {2.125, 2.25, 2.375, 2.5, 2.625, 2.75}, + {2.875, 3, 3.125, 3.25, 3.375, 3.5}, + {3.625, 3.75, 3.875, 4, 4.125, 4.25}, + {4.375, 4.5, 4.625, 4.75, 4.875, 5}, + }, + }, + }, + allocator); + + // Initialize with nonzero values to verify replacement. + GenericTensorAccessorW input_grad = + create_4d_accessor_w_with_contents( + { + { + { + {-7, -4, -1}, + {2, 5, 8}, + }, + { + {11, 14, 17}, + {20, 23, 26}, + }, + }, + }, + allocator); + + upsample_gpu_backward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*attrs=*/attrs, + /*output=*/output, + /*output_grad=*/output_grad, + /*input=*/input, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct = create_4d_accessor_r_with_contents( + { + { + { + {-1.75, -0.75, 0.25}, + {4.25, 5.25, 6.25}, + }, + { + {10.25, 11.25, 12.25}, + {16.25, 17.25, 18.25}, + }, + }, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } + + TEST_CASE("upsample_gpu_forward_kernel (scale_factor = 3)") { + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + UpsampleAttrs attrs = make_attrs(/*scale_factor=*/3); + + GenericTensorAccessorR input = create_4d_accessor_r_with_contents( + { + { + { + {1, 2}, + {-3, 0.5}, + }, + }, + }, + allocator); + + TensorShape output_shape = TensorShape{ + TensorDims{FFOrdered{1_p, 1_p, 6_p, 6_p}}, + DataType::FLOAT, + }; + + // Intentionally randomize this tensor so we can be confident we never read + // it + GenericTensorAccessorW output = + create_random_filled_accessor_w(output_shape, allocator); + + upsample_gpu_forward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*attrs=*/attrs, + /*input=*/input, + /*output=*/output); + + GenericTensorAccessorR correct = create_4d_accessor_r_with_contents( + { + { + { + {1, 1, 1, 2, 2, 2}, + {1, 1, 1, 2, 2, 2}, + {1, 1, 1, 2, 2, 2}, + {-3, -3, -3, 0.5, 0.5, 0.5}, + {-3, -3, -3, 0.5, 0.5, 0.5}, + {-3, -3, -3, 0.5, 0.5, 0.5}, + }, + }, + }, + allocator); + + CHECK_MESSAGE(accessors_are_equal(output, correct), + check_kv("output", format_accessor_w_contents(output))); + } + + TEST_CASE("upsample_gpu_backward_kernel (scale_factor = 3)") { + ManagedFFStream managed_stream{}; + + Allocator allocator = create_local_cuda_memory_allocator(); + + UpsampleAttrs attrs = make_attrs(/*scale_factor=*/3); + + GenericTensorAccessorR input = create_4d_accessor_r_with_contents( + { + { + { + {1, 2}, + {-3, 0.5}, + }, + }, + }, + allocator); + + GenericTensorAccessorR output = create_4d_accessor_r_with_contents( + { + { + { + {1, 1, 1, 2, 2, 2}, + {1, 1, 1, 2, 2, 2}, + {1, 1, 1, 2, 2, 2}, + {-3, -3, -3, 0.5, 0.5, 0.5}, + {-3, -3, -3, 0.5, 0.5, 0.5}, + {-3, -3, -3, 0.5, 0.5, 0.5}, + }, + }, + }, + allocator); + + GenericTensorAccessorR output_grad = + create_4d_accessor_r_with_contents( + { + { + { + {-3.75, -3.5, -3.25, -3, -2.75, -2.5}, + {-2.25, -2, -1.75, -1.5, -1.25, -1}, + {-0.75, -0.5, -0.25, 0, 0.25, 0.5}, + {0.75, 1, 1.25, 1.5, 1.75, 2}, + {2.25, 2.5, 2.75, 3, 3.25, 3.5}, + {3.75, 4, 4.25, 4.5, 4.75, 5}, + }, + }, + }, + allocator); + + // Initialize with nonzero values to verify replacement. + GenericTensorAccessorW input_grad = + create_4d_accessor_w_with_contents( + { + { + { + {100, 200}, + {300, 400}, + }, + }, + }, + allocator); + + upsample_gpu_backward_kernel( + /*stream=*/managed_stream.raw_stream(), + /*attrs=*/attrs, + /*output=*/output, + /*output_grad=*/output_grad, + /*input=*/input, + /*input_grad=*/input_grad); + + GenericTensorAccessorR correct = create_4d_accessor_r_with_contents( + { + { + { + {-18, -11.25}, + {22.5, 29.25}, + }, + }, + }, + allocator); + + CHECK_MESSAGE( + accessors_are_equal(input_grad, correct), + check_kv("input_grad", format_accessor_w_contents(input_grad))); + } +} diff --git a/lib/kernels/test/src/test_batch_norm_kernel.cc b/lib/kernels/test/src/test_batch_norm_kernel.cc index 43660b1bff..9978d1db54 100644 --- a/lib/kernels/test/src/test_batch_norm_kernel.cc +++ b/lib/kernels/test/src/test_batch_norm_kernel.cc @@ -83,6 +83,15 @@ TEST_SUITE(FF_CUDA_TEST_SUITE) { GenericTensorAccessorW bias_grad_accessor = create_random_filled_accessor_w(bias_shape, allocator); + GenericTensorAccessorW bias_accessor = create_constant_filled_accessor_w( + bias_shape, allocator, make_float_data_type_value(0)); + Kernels::BatchNorm::gpu_forward_kernel(managed_stream.raw_stream(), + state, + input_accessor.get_float_ptr(), + output_accessor.get_float_ptr(), + scale_accessor.get_float_ptr(), + bias_accessor.get_float_ptr()); + Kernels::BatchNorm::gpu_backward_kernel( /*stream=*/managed_stream.raw_stream(), /*per_device_state=*/state, diff --git a/lib/kernels/test/src/test_concat_kernel.cc b/lib/kernels/test/src/test_concat_kernel.cc index 034d463995..8dd36ec260 100644 --- a/lib/kernels/test/src/test_concat_kernel.cc +++ b/lib/kernels/test/src/test_concat_kernel.cc @@ -4,16 +4,12 @@ #include "kernels/create_zero_filled_accessor.h" #include "kernels/local_cuda_allocator.h" #include "kernels/managed_ff_stream.h" -#include "kernels/managed_per_device_ff_handle.h" #include "utils/containers/repeat.h" #include using namespace ::FlexFlow; TEST_SUITE(FF_CUDA_TEST_SUITE) { TEST_CASE("Test concat kernel forward and backward") { - ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( - /*workSpaceSize=*/1024 * 1024, - /*allowTensorOpMathConversion=*/true); ManagedFFStream managed_stream{}; Allocator allocator = create_local_cuda_memory_allocator(); diff --git a/lib/kernels/test/src/test_flat_kernel.cc b/lib/kernels/test/src/test_flat_kernel.cc index c9a5146b38..2b65515679 100644 --- a/lib/kernels/test/src/test_flat_kernel.cc +++ b/lib/kernels/test/src/test_flat_kernel.cc @@ -3,7 +3,6 @@ #include "kernels/flat_kernels_gpu.h" #include "kernels/local_cuda_allocator.h" #include "kernels/managed_ff_stream.h" -#include "kernels/managed_per_device_ff_handle.h" #include "op-attrs/datatype_value.h" #include @@ -12,9 +11,6 @@ TEST_SUITE(FF_CUDA_TEST_SUITE) { TEST_CASE("Test Flat Kernel") { Allocator allocator = create_local_cuda_memory_allocator(); - ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( - /*workSpaceSize=*/1024 * 1024, - /*allowTensorOpMathConversion=*/true); ManagedFFStream managed_stream{}; TensorShape input_shape = TensorShape{ diff --git a/lib/kernels/test/src/test_reshape_kernel.cc b/lib/kernels/test/src/test_reshape_kernel.cc index a0a55daded..3e8a57468d 100644 --- a/lib/kernels/test/src/test_reshape_kernel.cc +++ b/lib/kernels/test/src/test_reshape_kernel.cc @@ -2,16 +2,12 @@ #include "kernels/create_random_filled_accessor.h" #include "kernels/local_cuda_allocator.h" #include "kernels/managed_ff_stream.h" -#include "kernels/managed_per_device_ff_handle.h" #include "kernels/reshape_kernels_gpu.h" #include using namespace ::FlexFlow; TEST_SUITE(FF_CUDA_TEST_SUITE) { TEST_CASE("Test Reshape Forward and Backward") { - ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( - /*workSpaceSize=*/1024 * 1024, - /*allowTensorOpMathConversion=*/true); ManagedFFStream managed_stream{}; Allocator allocator = create_local_cuda_memory_allocator(); diff --git a/lib/kernels/test/src/test_reverse_kernels.cc b/lib/kernels/test/src/test_reverse_kernels.cc index 0b7f29d95d..928c044ddb 100644 --- a/lib/kernels/test/src/test_reverse_kernels.cc +++ b/lib/kernels/test/src/test_reverse_kernels.cc @@ -7,7 +7,6 @@ #include "kernels/local_cpu_allocator.h" #include "kernels/local_cuda_allocator.h" #include "kernels/managed_ff_stream.h" -#include "kernels/managed_per_device_ff_handle.h" #include "kernels/reverse_kernels_cpu.h" #include "kernels/reverse_kernels_gpu.h" #include "op-attrs/datatype_value.h" @@ -22,9 +21,6 @@ TEST_SUITE(FF_CUDA_TEST_SUITE) { }; TensorShape output_shape = input_shape; - ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( - /*workSpaceSize=*/1024 * 1024, - /*allowTensorOpMathConversion=*/true); ManagedFFStream managed_stream{}; Allocator allocator = create_local_cuda_memory_allocator(); @@ -69,9 +65,6 @@ TEST_SUITE(FF_CUDA_TEST_SUITE) { }; TensorShape output_shape = input_shape; - ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( - /*workSpaceSize=*/1024 * 1024, - /*allowTensorOpMathConversion=*/true); ManagedFFStream managed_stream{}; Allocator gpu_allocator = create_local_cuda_memory_allocator(); diff --git a/lib/kernels/test/src/test_split_kernel.cc b/lib/kernels/test/src/test_split_kernel.cc index 5d29cbd2e9..569e5777ca 100644 --- a/lib/kernels/test/src/test_split_kernel.cc +++ b/lib/kernels/test/src/test_split_kernel.cc @@ -2,7 +2,6 @@ #include "kernels/create_random_filled_accessor.h" #include "kernels/local_cuda_allocator.h" #include "kernels/managed_ff_stream.h" -#include "kernels/managed_per_device_ff_handle.h" #include "kernels/split_kernels_gpu.h" #include "op-attrs/datatype_value.h" #include "utils/containers/repeat.h" @@ -17,9 +16,6 @@ TEST_SUITE(FF_CUDA_TEST_SUITE) { int in_blk_size = 100; int num_blks = 1; - ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( - /*workSpaceSize=*/1024 * 1024, - /*allowTensorOpMathConversion=*/true); ManagedFFStream managed_stream{}; Allocator allocator = create_local_cuda_memory_allocator(); diff --git a/lib/kernels/test/src/test_transpose_kernel.cc b/lib/kernels/test/src/test_transpose_kernel.cc index 07698dcd2c..d36ae89a6c 100644 --- a/lib/kernels/test/src/test_transpose_kernel.cc +++ b/lib/kernels/test/src/test_transpose_kernel.cc @@ -2,7 +2,6 @@ #include "kernels/create_random_filled_accessor.h" #include "kernels/local_cuda_allocator.h" #include "kernels/managed_ff_stream.h" -#include "kernels/managed_per_device_ff_handle.h" #include "kernels/transpose_kernels_gpu.h" #include @@ -18,9 +17,6 @@ TEST_SUITE(FF_CUDA_TEST_SUITE) { }, }; - ManagedPerDeviceFFHandle managed_handle = initialize_single_gpu_handle( - /*workSpaceSize=*/1024 * 1024, - /*allowTensorOpMathConversion=*/true); ManagedFFStream managed_stream{}; Allocator allocator = create_local_cuda_memory_allocator(); diff --git a/lib/task-spec/src/task-spec/ops/impl/batch_matmul.cc b/lib/task-spec/src/task-spec/ops/impl/batch_matmul.cc index 6427ceaf22..eebf87831e 100644 --- a/lib/task-spec/src/task-spec/ops/impl/batch_matmul.cc +++ b/lib/task-spec/src/task-spec/ops/impl/batch_matmul.cc @@ -9,6 +9,7 @@ static std::optional std::optional profiling = acc.get_profiling_settings(); DeviceType kernel_device_type = acc.get_kernel_device_type(); + device_handle_t handle = acc.get_ff_handle(); auto lhs_input = acc.get_tensor(TensorSlotName::LHS_INPUT); auto rhs_input = acc.get_tensor(TensorSlotName::RHS_INPUT); auto output = acc.get_tensor(TensorSlotName::OUTPUT); @@ -17,6 +18,7 @@ static std::optional profiling, kernel_device_type, "[BatchMatmul] forward_time = {:.2lf}ms\n", + handle, lhs_input, rhs_input, output); @@ -26,14 +28,15 @@ static std::optional backward_task_impl(TaskArgumentAccessor const &acc) { std::optional profiling = acc.get_profiling_settings(); DeviceType kernel_device_type = acc.get_kernel_device_type(); + device_handle_t handle = acc.get_ff_handle(); auto lhs_input = acc.get_tensor(TensorSlotName::LHS_INPUT); auto lhs_input_grad = - acc.get_tensor_grad(TensorSlotName::LHS_INPUT); + acc.get_tensor_grad(TensorSlotName::LHS_INPUT); auto rhs_input = acc.get_tensor(TensorSlotName::RHS_INPUT); auto rhs_input_grad = - acc.get_tensor_grad(TensorSlotName::RHS_INPUT); + acc.get_tensor_grad(TensorSlotName::RHS_INPUT); auto output = acc.get_tensor(TensorSlotName::OUTPUT); auto output_grad = @@ -43,6 +46,7 @@ static std::optional profiling, kernel_device_type, "[BatchMatmul] backward_time = {:.2lf}ms\n", + handle, output, output_grad, lhs_input, diff --git a/lib/task-spec/src/task-spec/ops/impl/batch_norm.cc b/lib/task-spec/src/task-spec/ops/impl/batch_norm.cc index faac3c7551..99396516bc 100644 --- a/lib/task-spec/src/task-spec/ops/impl/batch_norm.cc +++ b/lib/task-spec/src/task-spec/ops/impl/batch_norm.cc @@ -36,7 +36,8 @@ static DeviceSpecificPerDeviceOpState positive_int output_c = dim_at_idx(output.shape.dims, legion_dim_t{2_n}); positive_int output_n = dim_at_idx(output.shape.dims, legion_dim_t{3_n}); - float *runningMean; + ASSERT(attrs.affine, "BatchNorm currently only supports attrs.affine = true"); + float *runningMean = nullptr; std::optional per_device_state = init_kernel( /*device_type=*/kernel_device_type, @@ -47,7 +48,8 @@ static DeviceSpecificPerDeviceOpState /*output_c=*/output_c.int_from_positive_int(), /*output_h=*/output_h.int_from_positive_int(), /*output_w=*/output_w.int_from_positive_int(), - /*activation=*/attrs.activation); + /*activation=*/attrs.activation, + /*eps=*/attrs.eps); return DeviceSpecificPerDeviceOpState{ acc.make_device_specific(per_device_state), @@ -63,8 +65,8 @@ static std::optional auto input = acc.get_tensor(TensorSlotName::INPUT); auto output = acc.get_tensor(TensorSlotName::OUTPUT); - auto scale = acc.get_tensor(TensorSlotName::SCALE); - auto bias = acc.get_tensor(TensorSlotName::BIAS); + auto scale = acc.get_tensor(TensorSlotName::GAMMA); + auto bias = acc.get_tensor(TensorSlotName::BETA); return profile(forward_kernel, profiling, @@ -85,13 +87,13 @@ static std::optional DeviceType kernel_device_type = acc.get_kernel_device_type(); auto input = acc.get_tensor(TensorSlotName::INPUT); - auto input_grad = acc.get_tensor_grad(TensorSlotName::INPUT); + auto input_grad = acc.get_tensor_grad(TensorSlotName::INPUT); auto output = acc.get_tensor(TensorSlotName::OUTPUT); auto output_grad = acc.get_tensor_grad(TensorSlotName::OUTPUT); - auto scale = acc.get_tensor(TensorSlotName::SCALE); - auto scale_grad = acc.get_tensor_grad(TensorSlotName::SCALE); - auto bias_grad = acc.get_tensor_grad(TensorSlotName::BIAS); + auto scale = acc.get_tensor(TensorSlotName::GAMMA); + auto scale_grad = acc.get_tensor_grad(TensorSlotName::GAMMA); + auto bias_grad = acc.get_tensor_grad(TensorSlotName::BETA); return profile(backward_kernel, profiling, diff --git a/lib/task-spec/src/task-spec/ops/impl/concat.cc b/lib/task-spec/src/task-spec/ops/impl/concat.cc index f6bc18a9aa..95efb5d433 100644 --- a/lib/task-spec/src/task-spec/ops/impl/concat.cc +++ b/lib/task-spec/src/task-spec/ops/impl/concat.cc @@ -67,7 +67,7 @@ static std::optional std::vector input_grads = transform(input_slots, [&](TensorSlotName input_slot_name) -> GenericTensorAccessorW { - return acc.get_tensor_grad(input_slot_name); + return acc.get_tensor_grad(input_slot_name); }); ASSERT(input_grads.size() <= MAX_NUM_INPUTS); diff --git a/lib/task-spec/src/task-spec/ops/impl/conv_2d.cc b/lib/task-spec/src/task-spec/ops/impl/conv_2d.cc index 43b44ab080..41d86d1874 100644 --- a/lib/task-spec/src/task-spec/ops/impl/conv_2d.cc +++ b/lib/task-spec/src/task-spec/ops/impl/conv_2d.cc @@ -14,7 +14,7 @@ static DeviceSpecificPerDeviceOpState auto output = acc.get_tensor(TensorSlotName::OUTPUT); auto filter = acc.get_tensor(TensorSlotName::FILTER); auto filter_grad = - acc.get_tensor_grad(TensorSlotName::FILTER); + acc.get_tensor_grad(TensorSlotName::FILTER); std::optional per_device_state = conv2d_init_kernel( /*device_type=*/kernel_device_type, @@ -41,8 +41,8 @@ static std::optional forward_task_impl(TaskArgumentAccessor const &acc) { std::optional profiling = acc.get_profiling_settings(); DeviceType kernel_device_type = acc.get_kernel_device_type(); - Conv2DPerDeviceState per_device_state = - acc.get_per_device_op_state().require_conv2d().value(); + std::optional per_device_state = + acc.get_per_device_op_state().require_conv2d(); Conv2DAttrs attrs = acc.get_op_attrs().require_conv2d(); auto input = acc.get_tensor(TensorSlotName::INPUT); @@ -71,8 +71,8 @@ static std::optional backward_task_impl(TaskArgumentAccessor const &acc) { std::optional profiling = acc.get_profiling_settings(); DeviceType kernel_device_type = acc.get_kernel_device_type(); - Conv2DPerDeviceState per_device_state = - acc.get_per_device_op_state().require_conv2d().value(); + std::optional per_device_state = + acc.get_per_device_op_state().require_conv2d(); Conv2DAttrs attrs = acc.get_op_attrs().require_conv2d(); auto output = acc.get_tensor(TensorSlotName::OUTPUT); @@ -81,16 +81,16 @@ static std::optional auto output_grad = acc.get_tensor_grad(TensorSlotName::OUTPUT); - auto input_grad = acc.get_tensor_grad(TensorSlotName::INPUT); + auto input_grad = acc.get_tensor_grad(TensorSlotName::INPUT); auto filter_grad = - acc.get_tensor_grad(TensorSlotName::FILTER); + acc.get_tensor_grad(TensorSlotName::FILTER); std::optional bias = std::nullopt; std::optional bias_grad = std::nullopt; if (attrs.use_bias) { - bias = acc.get_tensor_grad(TensorSlotName::BIAS); - bias_grad = acc.get_tensor_grad(TensorSlotName::BIAS); + bias = acc.get_tensor(TensorSlotName::BIAS); + bias_grad = acc.get_tensor_grad(TensorSlotName::BIAS); } return profile(conv2d_backward_kernel, diff --git a/lib/task-spec/src/task-spec/ops/impl/element_unary.cc b/lib/task-spec/src/task-spec/ops/impl/element_unary.cc index a370cec621..ed3f4f5c92 100644 --- a/lib/task-spec/src/task-spec/ops/impl/element_unary.cc +++ b/lib/task-spec/src/task-spec/ops/impl/element_unary.cc @@ -55,7 +55,7 @@ static std::optional GenericTensorAccessorR input = acc.get_tensor(TensorSlotName::INPUT); GenericTensorAccessorW input_grad = - acc.get_tensor_grad(TensorSlotName::INPUT); + acc.get_tensor_grad(TensorSlotName::INPUT); GenericTensorAccessorR output = acc.get_tensor(TensorSlotName::OUTPUT); GenericTensorAccessorR output_grad = diff --git a/lib/task-spec/src/task-spec/ops/impl/pool_2d.cc b/lib/task-spec/src/task-spec/ops/impl/pool_2d.cc index 18b824662c..e75df472e0 100644 --- a/lib/task-spec/src/task-spec/ops/impl/pool_2d.cc +++ b/lib/task-spec/src/task-spec/ops/impl/pool_2d.cc @@ -27,8 +27,8 @@ static std::optional std::optional profiling = acc.get_profiling_settings(); Pool2DAttrs attrs = acc.get_op_attrs().require_pool2d(); DeviceType kernel_device_type = acc.get_kernel_device_type(); - Pool2DPerDeviceState state = - acc.get_per_device_op_state().require_pool_2d().value(); + std::optional state = + acc.get_per_device_op_state().require_pool_2d(); auto input = acc.get_tensor(TensorSlotName::INPUT); auto output = acc.get_tensor(TensorSlotName::OUTPUT); @@ -48,13 +48,14 @@ static std::optional std::optional profiling = acc.get_profiling_settings(); Pool2DAttrs attrs = acc.get_op_attrs().require_pool2d(); DeviceType kernel_device_type = acc.get_kernel_device_type(); - Pool2DPerDeviceState state = - acc.get_per_device_op_state().require_pool_2d().value(); + std::optional state = + acc.get_per_device_op_state().require_pool_2d(); auto output = acc.get_tensor(TensorSlotName::OUTPUT); - auto output_grad = acc.get_tensor(TensorSlotName::OUTPUT); + auto output_grad = + acc.get_tensor_grad(TensorSlotName::OUTPUT); auto input = acc.get_tensor(TensorSlotName::INPUT); - auto input_grad = acc.get_tensor(TensorSlotName::INPUT); + auto input_grad = acc.get_tensor_grad(TensorSlotName::INPUT); return profile(pool2d_backward_kernel, profiling, diff --git a/lib/task-spec/src/task-spec/ops/impl/reshape.cc b/lib/task-spec/src/task-spec/ops/impl/reshape.cc index 15b3d29dd9..5e60aebbed 100644 --- a/lib/task-spec/src/task-spec/ops/impl/reshape.cc +++ b/lib/task-spec/src/task-spec/ops/impl/reshape.cc @@ -42,7 +42,7 @@ static std::optional DeviceType kernel_device_type = acc.get_kernel_device_type(); ReshapeAttrs attrs = acc.get_op_attrs().require_reshape(); - auto input_grad = acc.get_tensor_grad(TensorSlotName::INPUT); + auto input_grad = acc.get_tensor_grad(TensorSlotName::INPUT); auto output_grad = acc.get_tensor_grad(TensorSlotName::OUTPUT); diff --git a/lib/task-spec/src/task-spec/ops/impl/split.cc b/lib/task-spec/src/task-spec/ops/impl/split.cc index ccc40694e2..2d38912f2a 100644 --- a/lib/task-spec/src/task-spec/ops/impl/split.cc +++ b/lib/task-spec/src/task-spec/ops/impl/split.cc @@ -51,7 +51,7 @@ static std::optional DeviceType kernel_device_type = acc.get_kernel_device_type(); SplitAttrs attrs = acc.get_op_attrs().require_split(); - auto input_grad = acc.get_tensor_grad(TensorSlotName::INPUT); + auto input_grad = acc.get_tensor_grad(TensorSlotName::INPUT); auto output_grads = transform(split_get_output_slot_names(attrs), diff --git a/lib/task-spec/src/task-spec/ops/impl/upsample.cc b/lib/task-spec/src/task-spec/ops/impl/upsample.cc index 71ce0ab687..f9e015ea30 100644 --- a/lib/task-spec/src/task-spec/ops/impl/upsample.cc +++ b/lib/task-spec/src/task-spec/ops/impl/upsample.cc @@ -32,7 +32,7 @@ static std::optional auto output = acc.get_tensor(TensorSlotName::OUTPUT); auto output_grad = acc.get_tensor_grad(TensorSlotName::OUTPUT); - auto input_grad = acc.get_tensor_grad(TensorSlotName::INPUT); + auto input_grad = acc.get_tensor_grad(TensorSlotName::INPUT); return profile(upsample_backward_kernel, profiling,