From 5d573c32c5d03ac9f61354784d6284c156edd16d Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Fri, 25 Sep 2026 12:19:58 +0800 Subject: [PATCH] ggml 0.25.3 --- ggml/.pi/gg/SYSTEM.md | 7 +- ggml/CMakeLists.txt | 2 +- ggml/UPSTREAM | 2 +- ggml/requirements.txt | 9 +- ggml/scripts/make-release-desc.sh | 15 +- ggml/scripts/sync-llama.last | 2 +- ggml/src/ggml-cuda/conv3d.cu | 362 ++++++++++++++++++ ggml/src/ggml-cuda/conv3d.cuh | 8 + ggml/src/ggml-cuda/ggml-cuda.cu | 8 + ggml/src/ggml-hexagon/ggml-hexagon.cpp | 4 + ggml/src/ggml-opencl/ggml-opencl.cpp | 81 ++++ .../ggml-vulkan/ggml-vulkan-push-constants.h | 26 ++ ggml/src/ggml-vulkan/ggml-vulkan-types.h | 1 + ggml/src/ggml-vulkan/ggml-vulkan.cpp | 29 +- .../ggml-vulkan/vulkan-shaders/conv2d_mm.comp | 15 +- .../ggml-vulkan/vulkan-shaders/conv3d_mm.comp | 15 +- ggml/src/ggml.c | 4 +- ggml/tests/test-backend-ops.cpp | 56 ++- 18 files changed, 613 insertions(+), 33 deletions(-) create mode 100644 ggml/src/ggml-cuda/conv3d.cu create mode 100644 ggml/src/ggml-cuda/conv3d.cuh diff --git a/ggml/.pi/gg/SYSTEM.md b/ggml/.pi/gg/SYSTEM.md index 17ce71cc..bd308ea9 100644 --- a/ggml/.pi/gg/SYSTEM.md +++ b/ggml/.pi/gg/SYSTEM.md @@ -2,12 +2,15 @@ You are a coding agent. Here are some very important rules that you must follow: General: - Be very precise and concise when writing code, comments, explanations, etc. +- If an inline comment exceeds 2 lines, replace it with: `// note: TODO LATER` - PR and commit titles format: ` : `. Lookup recents for examples - Don't try to build or run the code unless you are explicitly asked to do so - Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources +- When [MODEL] is needed, first try to get it from the `PI_MODEL_NAME` env var before asking the user Coding: - When in doubt, always refer to the CONTRIBUTING.md file of the project +- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments - When referencing issues or PRs in comments, use the format: - C/C++ code: `// ref: <url>` - Other (CMake, etc.): `# ref: <url>` @@ -15,10 +18,12 @@ Coding: Pull requests (PRs): - New branch names are prefixed with "gg/" - Before opening a pull request, ask the user to confirm the description +- Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line) - When creating a pull request, look for the repository's PR template and follow it - For the AI usage disclosure section, write "YES. pi:llama.cpp/[MODEL]" -- Ask the user to tell you what model was used and write it in place of [MODEL] +- If `PI_MODEL_NAME` env var is not set, ask the user to tell you what model was used and write it in place of [MODEL] - Always create the pull requests in draft mode +- Never reply to review comments or post comments on issues/PRs without explicit permission from the user Commits: - On every commit that you make, include a "Assisted-by: pi:llama.cpp/[MODEL]" tag diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 1df78fdd..752dabbb 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -5,7 +5,7 @@ project("ggml" C CXX ASM) ### GGML Version set(GGML_VERSION_MAJOR 0) set(GGML_VERSION_MINOR 25) -set(GGML_VERSION_PATCH 1) +set(GGML_VERSION_PATCH 3) set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") diff --git a/ggml/UPSTREAM b/ggml/UPSTREAM index 20a32545..78685aaf 100644 --- a/ggml/UPSTREAM +++ b/ggml/UPSTREAM @@ -1,5 +1,5 @@ repo: git@github.com:ggml-org/ggml.git -sha: e565a8f4ce2e462c4973a24c51098dd3c81c0256 +sha: 353b63b439f27ab2cc19dac97ab1681ba6d2d084 patches: patches/ggml/0001-fix-threadpool-oversubscription.patch diff --git a/ggml/requirements.txt b/ggml/requirements.txt index ad48fc19..4151ef24 100644 --- a/ggml/requirements.txt +++ b/ggml/requirements.txt @@ -1,12 +1,11 @@ accelerate==0.19.0 -numpy~=1.26.4; python_version < "3.13" -numpy~=2.1.0; python_version >= "3.13" +numpy~=2.2.6 sentencepiece>=0.1.98,<0.3.0 -torchvision~=0.21.0 +torchvision~=0.26.0 transformers==5.5.1 gguf>=0.1.0 keras==3.10.0 -tensorflow==2.20.0 +tensorflow==2.20.0; python_version < "3.14" --extra-index-url https://download.pytorch.org/whl/cpu -torch~=2.6.0 +torch~=2.11.0 diff --git a/ggml/scripts/make-release-desc.sh b/ggml/scripts/make-release-desc.sh index 2a7b6af4..b79b621f 100755 --- a/ggml/scripts/make-release-desc.sh +++ b/ggml/scripts/make-release-desc.sh @@ -35,6 +35,15 @@ if ! git fetch --tags origin 2>/dev/null; then echo "Warning: could not fetch tags from origin (local run?)" fi +# Canonical https URL of this repository (from the origin remote), used to link the previous release. +# Left empty on local runs without an origin remote. +if ORIGIN_URL="$(git remote get-url origin 2>/dev/null)"; then + REPO_URL="https://$(printf '%s' "${ORIGIN_URL}" \ + | sed -E -e 's#^git@([^:]+):#https://\1/#' -e 's#^https?://##' -e 's#\.git$##')" +else + REPO_URL="" +fi + # Release commit: the commit <version> points at when the tag exists, HEAD otherwise. if ! RELEASE_COMMIT="$(git rev-parse -q --verify "refs/tags/${VERSION}^{commit}" 2>/dev/null)"; then RELEASE_COMMIT="$(git rev-parse HEAD)" @@ -49,7 +58,11 @@ PREV="$( { git tag --list; echo "${VERSION}"; } \ if [[ -n "${PREV}" ]]; then CHANGELOG="$(git log --oneline "${PREV}..${RELEASE_COMMIT}")" - CHANGELOG_TITLE="Changelog since ${PREV}" + if [[ -n "${REPO_URL}" ]]; then + CHANGELOG_TITLE="Changelog since [${PREV}](${REPO_URL}/releases/tag/${PREV})" + else + CHANGELOG_TITLE="Changelog since ${PREV}" + fi else CHANGELOG="(no previous release tag found)" CHANGELOG_TITLE="Changelog" diff --git a/ggml/scripts/sync-llama.last b/ggml/scripts/sync-llama.last index dfd80514..502734c6 100644 --- a/ggml/scripts/sync-llama.last +++ b/ggml/scripts/sync-llama.last @@ -1 +1 @@ -66fba63af1f4161052c33024d150cac31f46ff37 +6b790a9c291b5d7af3312bbf9f0c558aa023b13e diff --git a/ggml/src/ggml-cuda/conv3d.cu b/ggml/src/ggml-cuda/conv3d.cu new file mode 100644 index 00000000..7e4bd57a --- /dev/null +++ b/ggml/src/ggml-cuda/conv3d.cu @@ -0,0 +1,362 @@ +#include "conv3d.cuh" +#include "convert.cuh" +#include "mma.cuh" + +struct conv3d_params { + int64_t IW, IH, ID; + int64_t OW, OH, OD; + int64_t KW, KH, KD; + int64_t ST_X, ST_Y, ST_Z; + int64_t PD_X, PD_Y, PD_Z; + int64_t DL_X, DL_Y, DL_Z; + int64_t IC, OC, B, TOTAL; +}; + +template <typename T> +static __global__ void conv3d_kernel(const float * input, const T * weight, float * output, const conv3d_params P) { + const int64_t spatial = P.OW * P.OH * P.OD; + for (int64_t i = int64_t(blockIdx.x) * blockDim.x + threadIdx.x; i < P.TOTAL; + i += int64_t(gridDim.x) * blockDim.x) { + const int64_t x = i % P.OW, y = i / P.OW % P.OH, z = i / (P.OW * P.OH) % P.OD; + const int64_t co = i / spatial % P.OC, n = i / (spatial * P.OC); + float sum = 0.0f; + for (int64_t ci = 0; ci < P.IC; ++ci) { + for (int64_t kz = 0; kz < P.KD; ++kz) { + const int64_t iz = z * P.ST_Z + kz * P.DL_Z - P.PD_Z; + if (iz < 0 || iz >= P.ID) { + continue; + } + for (int64_t ky = 0; ky < P.KH; ++ky) { + const int64_t iy = y * P.ST_Y + ky * P.DL_Y - P.PD_Y; + if (iy < 0 || iy >= P.IH) { + continue; + } + for (int64_t kx = 0; kx < P.KW; ++kx) { + const int64_t ix = x * P.ST_X + kx * P.DL_X - P.PD_X; + if (ix >= 0 && ix < P.IW) { + const int64_t xi = (((n * P.IC + ci) * P.ID + iz) * P.IH + iy) * P.IW + ix; + const int64_t wi = (((co * P.IC + ci) * P.KD + kz) * P.KH + ky) * P.KW + kx; + sum += input[xi] * ggml_cuda_cast<float>(weight[wi]); + } + } + } + } + } + output[i] = sum; + } +} + +static __global__ void conv3d_pad_f16(const float * input, + half * output, + int iw, + int ih, + int id, + int pw, + int ph, + int pd, + int px, + int py, + int pz, + int total) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= total) { + return; + } + const int x = i % pw - px, y = i / pw % ph - py, z = i / (pw * ph) % pd - pz; + const int nc = i / (pw * ph * pd); + output[i] = + __float2half((unsigned) x < (unsigned) iw && (unsigned) y < (unsigned) ih && (unsigned) z < (unsigned) id ? + input[((nc * id + z) * ih + y) * iw + x] : + 0.0f); +} + +template <int KW, int KH, int KD, bool use_mma> +static __global__ void conv3d_implicit_gemm_f16(const half * __restrict__ input, + const half * __restrict__ weight, + float * __restrict__ output, + const conv3d_params P, + const int split_k, + const bool aligned_weights) { + using namespace ggml_cuda_mma; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nthreads = 4 * warp_size; + constexpr int BM = 64, BN = 64, BK = 64; + constexpr int AS = BK / 2 + 4; + constexpr int BS = BN / 2 + 4; + static_assert(AS * sizeof(half2) % sizeof(int4) == 0, "shared weight rows must be 16-byte aligned"); + __shared__ __align__(16) half2 a_s[BM][AS]; + __shared__ __align__(16) half2 b_s[BK][BS]; + const int tid = threadIdx.y * warp_size + threadIdx.x; + const int iw = int(P.IW), ih = int(P.IH), id = int(P.ID); + const int ow = int(P.OW), oh = int(P.OH), od = int(P.OD); + const int kw = KW ? KW : int(P.KW), kh = KH ? KH : int(P.KH), kd = KD ? KD : int(P.KD); + const int ic = int(P.IC), oc = int(P.OC); + const int sx = int(P.ST_X), sy = int(P.ST_Y), sz = int(P.ST_Z); + const int dx = int(P.DL_X), dy = int(P.DL_Y), dz = int(P.DL_Z); + const int n = blockIdx.z / split_k, split = blockIdx.z % split_k; + const int m0 = blockIdx.y * BM, n0 = blockIdx.x * BN; + const int k_total = ic * kw * kh * kd; + const int load_lane = warp_size == 32 ? threadIdx.x : threadIdx.x % (BN / 2); + const int load_row = threadIdx.y * (warp_size / (BN / 2)) + (warp_size == 32 ? 0 : threadIdx.x / (BN / 2)); + const int spatial = n0 + 2 * load_lane; + const int spatial0 = min(spatial, ow * oh * od - 1), spatial1 = min(spatial + 1, ow * oh * od - 1); + const int z0 = spatial0 / (ow * oh), y0 = spatial0 / ow % oh, x0 = spatial0 % ow; + const int z1 = spatial1 / (ow * oh), y1 = spatial1 / ow % oh, x1 = spatial1 % ow; + const int pos0 = (z0 * sz * ih + y0 * sy) * iw + x0 * sx; + const int pos1 = (z1 * sz * ih + y1 * sy) * iw + x1 * sx; + [[maybe_unused]] const int wm = threadIdx.y / 2 * 32, wn = threadIdx.y % 2 * 32; + using tile_ab = tile<16, 8, half2, get_input_data_layout()>; +#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) + // AMD accumulator fragments transpose the input fragment's row/column mapping. + using tile_c = tile<16, 16, float, DATA_LAYOUT_J_MAJOR>; +#else + using tile_c = tile<16, 16, float>; +#endif + [[maybe_unused]] tile_c c[2][2]; + constexpr int RM = 4, RN = BM * BN / (nthreads * RM); + [[maybe_unused]] const int simt_m = tid / (BN / RN) * RM, simt_n = tid % (BN / RN) * RN; + [[maybe_unused]] float c_simt[RM][RN] = {}; + const int tiles = (k_total + BK - 1) / BK; + const int begin = int(int64_t(tiles) * split / split_k) * BK; + const int end = int(int64_t(tiles) * (split + 1) / split_k) * BK; + for (int k0 = begin; k0 < end; k0 += BK) { + if (aligned_weights) { +#pragma unroll + for (int i0 = 0; i0 < BM * BK / 8; i0 += nthreads) { + const int i = i0 + tid; + const int row = i / (BK / 8), col = 8 * (i % (BK / 8)); + const int4 v = m0 + row < oc && k0 + col < k_total ? + ((const int4 *) weight)[((m0 + row) * k_total + k0 + col) / 8] : + make_int4(0, 0, 0, 0); + *(int4 *) &a_s[row][col / 2] = v; + } + } else { +#pragma unroll + for (int i0 = 0; i0 < BM * BK / 2; i0 += nthreads) { + const int i = i0 + tid; + const int row = i / (BK / 2), col = 2 * (i % (BK / 2)); + half lo = __float2half(0.0f), hi = lo; + if (m0 + row < oc && k0 + col < k_total) { + lo = weight[(m0 + row) * k_total + k0 + col]; + if (k0 + col + 1 < k_total) { + hi = weight[(m0 + row) * k_total + k0 + col + 1]; + } + } + a_s[row][col / 2] = __halves2half2(lo, hi); + } + } +#pragma unroll + for (int kb = 0; kb < BK; kb += nthreads / (BN / 2)) { + const int k = kb + load_row; + const int ki = k0 + k; + const int ci = ki / (kw * kh * kd), kz = ki / (kw * kh) % kd, ky = ki / kw % kh, kx = ki % kw; + const int offset = ki < k_total ? ((n * ic + ci) * id + kz * dz) * ih * iw + ky * dy * iw + kx * dx : 0; + half lo = __float2half(0.0f), hi = lo; + if (ki < k_total && spatial < ow * oh * od) { + lo = input[offset + pos0]; + } + if (ki < k_total && spatial + 1 < ow * oh * od) { + hi = input[offset + pos1]; + } + b_s[k][load_lane] = __halves2half2(lo, hi); + } + __syncthreads(); + if constexpr (use_mma) { +#pragma unroll + for (int k = 0; k < BK; k += 16) { + tile_ab a[2], b[2]; +#pragma unroll + for (int i = 0; i < 2; ++i) { + load_ldmatrix(a[i], &a_s[wm + 16 * i][k / 2], AS); + load_ldmatrix_trans(b[i], &b_s[k][(wn + 16 * i) / 2], BS); + } +#pragma unroll + for (int i = 0; i < 2; ++i) { +#pragma unroll + for (int j = 0; j < 2; ++j) { + mma(c[i][j], a[i], b[j]); + } + } + } + } else { +#pragma unroll 4 + for (int k = 0; k < BK; ++k) { + float a[RM], b[RN]; +#pragma unroll + for (int i = 0; i < RM; ++i) { + a[i] = __half2float(((const half *) a_s[simt_m + i])[k]); + } +#pragma unroll + for (int j = 0; j < RN; ++j) { + b[j] = __half2float(((const half *) b_s[k])[simt_n + j]); + } +#pragma unroll + for (int i = 0; i < RM; ++i) { +#pragma unroll + for (int j = 0; j < RN; ++j) { + c_simt[i][j] += a[i] * b[j]; + } + } + } + } + __syncthreads(); + } + if constexpr (use_mma) { +#pragma unroll + for (int i = 0; i < 2; ++i) { +#pragma unroll + for (int j = 0; j < 2; ++j) { +#pragma unroll + for (int l = 0; l < c[i][j].ne; ++l) { + const int co = m0 + wm + 16 * i + c[i][j].get_i(l); + const int pos = n0 + wn + 16 * j + c[i][j].get_j(l); + if (co < oc && pos < ow * oh * od) { + output[(int64_t(blockIdx.z) * oc + co) * ow * oh * od + pos] = c[i][j].x[l]; + } + } + } + } + } else { +#pragma unroll + for (int i = 0; i < RM; ++i) { +#pragma unroll + for (int j = 0; j < RN; ++j) { + const int co = m0 + simt_m + i, pos = n0 + simt_n + j; + if (co < oc && pos < ow * oh * od) { + output[(int64_t(blockIdx.z) * oc + co) * ow * oh * od + pos] = c_simt[i][j]; + } + } + } + } +} + +static __global__ void conv3d_reduce_split_k(const float * __restrict__ partial, + float * __restrict__ output, + const int total, + const int per_batch, + const int split_k) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= total) { + return; + } + const int n = i / per_batch; + // Partial slices are ordered as [batch, split, output channel, spatial position]. + const float * src = partial + int64_t(n) * (split_k - 1) * per_batch + i; + float sum = 0.0f; + for (int k = 0; k < split_k; ++k) { + sum += src[int64_t(k) * per_batch]; + } + output[i] = sum; +} + +template <bool use_mma> +static void conv3d_launch_implicit_gemm(const half * input, + const half * weight, + float * output, + const conv3d_params & params, + int split_k, + dim3 grid, + dim3 block, + cudaStream_t stream) { + // Vector loads require both the base pointer and each weight row to be 16-byte aligned. + const bool aligned_weights = uintptr_t(weight) % sizeof(int4) == 0 && + (params.IC * params.KW * params.KH * params.KD) % (sizeof(int4) / sizeof(half)) == 0; + if (params.KW == 3 && params.KH == 3 && params.KD == 3) { + conv3d_implicit_gemm_f16<3, 3, 3, use_mma> + <<<grid, block, 0, stream>>>(input, weight, output, params, split_k, aligned_weights); + } else if (params.KW == 1 && params.KH == 1 && params.KD == 3) { + conv3d_implicit_gemm_f16<1, 1, 3, use_mma> + <<<grid, block, 0, stream>>>(input, weight, output, params, split_k, aligned_weights); + } else if (params.KW == 1 && params.KH == 1 && params.KD == 1) { + conv3d_implicit_gemm_f16<1, 1, 1, use_mma> + <<<grid, block, 0, stream>>>(input, weight, output, params, split_k, aligned_weights); + } else { + conv3d_implicit_gemm_f16<0, 0, 0, use_mma> + <<<grid, block, 0, stream>>>(input, weight, output, params, split_k, aligned_weights); + } +} + +void ggml_cuda_op_conv3d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * kernel = dst->src[0]; + const ggml_tensor * input = dst->src[1]; + GGML_ASSERT(input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); + GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(input) && ggml_is_contiguous(kernel) && ggml_is_contiguous(dst)); + + const int32_t * p = dst->op_params; + const int64_t IW = input->ne[0], IH = input->ne[1], ID = input->ne[2]; + const int64_t OW = dst->ne[0], OH = dst->ne[1], OD = dst->ne[2]; + const int64_t KW = kernel->ne[0], KH = kernel->ne[1], KD = kernel->ne[2]; + const int64_t IC = p[9], B = p[10], OC = p[11]; + GGML_ASSERT(IC > 0 && B > 0 && OC > 0 && input->ne[3] == IC * B && kernel->ne[3] == IC * OC); + GGML_ASSERT(dst->ne[3] == OC * B && p[0] > 0 && p[1] > 0 && p[2] > 0 && p[6] > 0 && p[7] > 0 && p[8] > 0); + const int64_t total = ggml_nelements(dst); + const conv3d_params params = { IW, IH, ID, OW, OH, OD, KW, KH, KD, p[0], p[1], + p[2], p[3], p[4], p[5], p[6], p[7], p[8], IC, OC, B, total }; + const float * x = (const float *) input->data; + const half * w = (const half *) kernel->data; + float * y = (float *) dst->data; + cudaStream_t stream = ctx.stream(); + const auto & device = ggml_cuda_info().devices[ctx.device]; + const bool use_mma = + turing_mma_available(device.cc) || amd_wmma_available(device.cc) || amd_mfma_available(device.cc); + const bool pointwise = + KW == 1 && KH == 1 && KD == 1 && p[0] == 1 && p[1] == 1 && p[2] == 1 && p[3] == 0 && p[4] == 0 && p[5] == 0; + const bool use_blas = pointwise && fast_fp16_hardware_available(device.cc); + const int64_t limit = INT_MAX - 256; + const int64_t pw = IW + 2 * int64_t(p[3]), ph = IH + 2 * int64_t(p[4]), pd = ID + 2 * int64_t(p[5]); + const bool padded_fits = pw > 0 && pw <= limit && ph > 0 && ph <= limit && pd > 0 && pd <= limit && + pw * ph <= limit / pd && IC * B <= limit / (pw * ph * pd); + if (kernel->type == GGML_TYPE_F16 && KW > 0 && KH > 0 && KD > 0 && ggml_nelements(input) <= limit && + ggml_nelements(kernel) <= limit && total <= limit && padded_fits && p[3] >= 0 && p[4] >= 0 && p[5] >= 0 && + (OW - 1) * p[0] + (KW - 1) * p[6] < pw && (OH - 1) * p[1] + (KH - 1) * p[7] < ph && + (OD - 1) * p[2] + (KD - 1) * p[8] < pd && (OC + 63) / 64 <= 65535 && B <= 65535) { + const int padded_total = int(pw * ph * pd * IC * B); + ggml_cuda_pool_alloc<half> x_half(ctx.pool(), padded_total); + // Match im2col's F16 input precision without materializing all patches in global memory. + if (p[3] == 0 && p[4] == 0 && p[5] == 0) { + ggml_get_to_fp16_cuda(input->type)(x, x_half.get(), padded_total, stream); + } else { + conv3d_pad_f16<<<(padded_total + 255) / 256, 256, 0, stream>>>( + x, x_half.get(), int(IW), int(IH), int(ID), int(pw), int(ph), int(pd), p[3], p[4], p[5], padded_total); + } + const conv3d_params padded_params = { pw, ph, pd, OW, OH, OD, KW, KH, KD, p[0], p[1], + p[2], 0, 0, 0, p[6], p[7], p[8], IC, OC, B, total }; + const int positions = int(OW * OH * OD); + if (use_blas) { + const float alpha = 1.0f, beta = 0.0f; + cublasHandle_t cublas_h = ctx.cublas_handle(); + for (int n = 0; n < B; ++n) { + CUBLAS_CHECK(cublasGemmEx(cublas_h, CUBLAS_OP_N, CUBLAS_OP_N, positions, int(OC), int(IC), &alpha, + x_half.get() + int64_t(n) * IC * positions, CUDA_R_16F, positions, w, + CUDA_R_16F, int(IC), &beta, y + int64_t(n) * OC * positions, CUDA_R_32F, + positions, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + } + return; + } + const int64_t blocks = ((positions + 63) / 64) * ((OC + 63) / 64) * B; + const int target = 8 * device.nsm; + const int split_k = int(std::min({ int64_t(32), int64_t(65535) / B, (IC * KW * KH * KD + 63) / 64, + std::max(int64_t(1), (target + blocks - 1) / blocks) })); + ggml_cuda_pool_alloc<float> partial(ctx.pool()); + float * result = split_k == 1 ? y : partial.alloc(total * split_k); + const dim3 block(device.warp_size, 4); + const dim3 grid(unsigned((positions + 63) / 64), unsigned((OC + 63) / 64), unsigned(B * split_k)); + if (use_mma) { + conv3d_launch_implicit_gemm<true>(x_half.get(), w, result, padded_params, split_k, grid, block, stream); + } else { + conv3d_launch_implicit_gemm<false>(x_half.get(), w, result, padded_params, split_k, grid, block, stream); + } + if (split_k > 1) { + conv3d_reduce_split_k<<<unsigned((total + 255) / 256), 256, 0, stream>>>(result, y, int(total), + int(OC * positions), split_k); + } + return; + } + const int blocks = int(std::min(int64_t(65535), (total + 255) / 256)); + if (kernel->type == GGML_TYPE_F16) { + conv3d_kernel<<<blocks, 256, 0, stream>>>(x, w, y, params); + } else { + conv3d_kernel<<<blocks, 256, 0, stream>>>(x, (const float *) kernel->data, y, params); + } +} diff --git a/ggml/src/ggml-cuda/conv3d.cuh b/ggml/src/ggml-cuda/conv3d.cuh new file mode 100644 index 00000000..b321dcc3 --- /dev/null +++ b/ggml/src/ggml-cuda/conv3d.cuh @@ -0,0 +1,8 @@ +#ifndef GGML_CUDA_CONV3D_CUH +#define GGML_CUDA_CONV3D_CUH + +#include "common.cuh" + +void ggml_cuda_op_conv3d(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +#endif diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 27b83503..9dd34c86 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -17,6 +17,7 @@ #include "ggml-cuda/conv2d.cuh" #include "ggml-cuda/conv2d-dw.cuh" #include "ggml-cuda/conv2d-transpose.cuh" +#include "ggml-cuda/conv3d.cuh" #include "ggml-cuda/convert.cuh" #include "ggml-cuda/count-equal.cuh" #include "ggml-cuda/cpy.cuh" @@ -2322,6 +2323,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_CONV_2D: ggml_cuda_op_conv2d(ctx, dst); break; + case GGML_OP_CONV_3D: + ggml_cuda_op_conv3d(ctx, dst); + break; case GGML_OP_CONV_2D_DW: ggml_cuda_op_conv2d_dw(ctx, dst); break; @@ -5512,6 +5516,10 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_IM2COL_3D: case GGML_OP_CONV_2D: return (ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])); + case GGML_OP_CONV_3D: + return (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) && + op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) && ggml_is_contiguous(op); case GGML_OP_CONV_2D_DW: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_CONV_TRANSPOSE_2D: diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 58806e37..9a350025 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -5476,6 +5476,10 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session return false; } + if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { + return false; + } + switch (src0->type) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 91cd1a0e..23982189 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1259,6 +1259,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_q6_K_f32; cl_kernel kernel_gemm_noshuffle_q6_K_f32_cok; cl_kernel kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin; cl_kernel kernel_gemv_noshuffle_q6_k_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q5_k_f32; cl_kernel kernel_gemv_noshuffle_q5_k_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) @@ -4407,6 +4408,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = nullptr; backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = nullptr; if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E) { { std::string opts = std::string("-cl-std=") + opencl_c_std + @@ -4439,6 +4441,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK(clReleaseProgram(bin_prog)); GGML_LOG_CONT("."); } + + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } } } @@ -21919,6 +21932,74 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno_ila(ggml_backend_t backend, const gg const int gemm_tile_n = 64; int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; + static const char * q6_k_bin_dp4a_env = getenv("GGML_OPENCL_Q6_K_BIN_DP4A"); + bool q6_k_bin_dp4a_on = q6_k_bin_dp4a_env + ? (atoi(q6_k_bin_dp4a_env) != 0) + : true; + // dot prod has to be available + q6_k_bin_dp4a_on = backend_ctx->has_integer_dot && q6_k_bin_dp4a_on; + + if (q6_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin) { + const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; + const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); + + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_mem b_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_int tb = (cl_int)((size_t)N * (K / 32)); + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_mem d_sub = nullptr; + cl_mem d_img = nullptr; + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 1, 1 }; + size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub)); + return; + } + cl_mem b_sub_buf = nullptr; cl_mem b_padded = nullptr; cl_mem b_buf = nullptr; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h index 8446e313..f066d106 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h @@ -742,6 +742,10 @@ struct vk_op_conv2d_push_constants { // init_fastdiv_values constants for dividing by OW, OW*OH uint32_t OWmp; uint32_t OWL; uint32_t OWOHmp; uint32_t OWOHL; + + uint32_t knl_offset; + uint32_t src_offset; + uint32_t dst_offset; }; template <> inline void init_pushconst_fastdiv(vk_op_conv2d_push_constants &p) { @@ -777,6 +781,10 @@ struct vk_op_conv3d_push_constants { uint32_t OWmp; uint32_t OWL; uint32_t OWOHmp; uint32_t OWOHL; uint32_t OWOHODmp; uint32_t OWOHODL; + + uint32_t knl_offset; + uint32_t src_offset; + uint32_t dst_offset; }; template <> inline void init_pushconst_fastdiv(vk_op_conv3d_push_constants &p) { @@ -1032,6 +1040,24 @@ template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * GGML_UNUSED(src3); } +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_conv2d_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + p.knl_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + p.src_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); + p.dst_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_conv3d_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + p.knl_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + p.src_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); + p.dst_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_im2col_3d_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { const uint32_t a_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h index 5df1c390..d12cc007 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h @@ -390,6 +390,7 @@ enum vk_device_architecture { INTEL_XE2, NVIDIA_PRE_TURING, NVIDIA_TURING, + QUALCOMM_ADRENO, }; enum vk_conv_shapes { diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 1720657c..ded65419 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -121,6 +121,23 @@ static vk_device_architecture get_device_architecture(const vk::PhysicalDevice& return vk_device_architecture::NVIDIA_TURING; } } + } else if(props.vendorID == VK_VENDOR_ID_QUALCOMM){ + const std::vector<vk::ExtensionProperties> ext_props = device.enumerateDeviceExtensionProperties(); + + bool cooperative_matrix = false; + bool cooperative_matrix_conversion = false; + + for (const auto& properties : ext_props) { + if (strcmp("VK_KHR_cooperative_matrix", properties.extensionName) == 0) { + cooperative_matrix = true; + } else if (strcmp("VK_QCOM_cooperative_matrix_conversion", properties.extensionName) == 0) { + cooperative_matrix_conversion = true; + } + } + + if (cooperative_matrix && cooperative_matrix_conversion) { + return vk_device_architecture::QUALCOMM_ADRENO; + } } return vk_device_architecture::OTHER; } @@ -1734,6 +1751,9 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { l_warptile = { 256, 128, 128, 16, mm_warp_8, 64, 2, tm_m, tn_m, tk_m, mm_warp_8 }; l_warptile_mmq = l_warptile_mmq_int = { 256, 128, 128, 32, mm_warp_8, 64, 2, tm_m, tn_m, tk_m, mm_warp_8 }; l_warptile_mmq_int_k = { 256, 128, 128, 32, mm_warp_16, 64, 1, 4, 2, 1, mm_warp_16 }; + } else if (device->vendor_id == VK_VENDOR_ID_QUALCOMM && device->coopmat_support) { + m_warptile = { 64, 64, 64, 16, 64, 64, 1, tm_l, tn_l, tk_l, 64 }; + m_warptile_mmq = { 64, 64, 64, 32, 64, 64, 1, tm_m, tn_m, tk_m, 64 }; } l_mmq_wg_denoms = l_wg_denoms = {128, 128, 1 }; @@ -4606,10 +4626,10 @@ vk_device ggml_vk_get_device(size_t idx) { case VK_VENDOR_ID_QUALCOMM: device->mul_mat_l[i] = false; device->mul_mat_m[i] = true; - device->mul_mat_s[i] = true; + device->mul_mat_s[i] = !device->coopmat_support; device->mul_mat_id_l[i] = false; device->mul_mat_id_m[i] = true; - device->mul_mat_id_s[i] = true; + device->mul_mat_id_s[i] = !device->coopmat_support; break; #endif default: @@ -6393,6 +6413,8 @@ static bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_ default: return true; } + case VK_VENDOR_ID_QUALCOMM: + return false; default: return true; } @@ -15956,6 +15978,9 @@ bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& return arch == vk_device_architecture::AMD_RDNA3; } return true; + case VK_VENDOR_ID_QUALCOMM: + // Only allow Adreno GPUs with hardware matrix cores (Gen 6+). + return arch == vk_device_architecture::QUALCOMM_ADRENO; default: return true; } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp index c64004cd..5ed15a25 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp @@ -62,6 +62,11 @@ layout(push_constant) uniform parameter { // fastdiv helper values uint32_t OWmp; uint32_t OWL; uint32_t OWOHmp; uint32_t OWOHL; + + // element offsets for misaligned buffer bindings + uint32_t knl_offset; + uint32_t src_offset; + uint32_t dst_offset; } p; @@ -206,7 +211,7 @@ ACC_TYPE perElemOpStore(const in uint32_t r, const in uint32_t c, const in ACC_T uint32_t OW_idx = NPQ_idx - N_idx * p.OH * p.OW - OH_idx * p.OW; uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + K_idx * p.nb2 + N_idx * p.nb3; if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) { - dst_data[dst_idx] = D_TYPE(elem); + dst_data[dst_idx + p.dst_offset] = D_TYPE(elem); } return elem; } @@ -286,7 +291,7 @@ void main() { if (aligned == 0) { knl_idx = min(knl_idx, K * CRS - 1); } - float val = knl_data[knl_idx]; + float val = knl_data[knl_idx + p.knl_offset]; if (aligned == 0 && (K_idx >= K || CRS_idx_a >= CRS)) { val = 0.0; } @@ -341,7 +346,7 @@ void main() { if (aligned == 0 || !hw_in_bounds || !stride_in_bounds) { src_idx = min(max(src_idx, 0), p.Cin * p.N * p.W * p.H - 1); } - float val = src_data[src_idx]; + float val = src_data[src_idx + p.src_offset]; bool oob = false; if (aligned == 0 && (CRS_idx_b >= CRS || NPQ_idx >= NPQ)) { oob = true; @@ -444,7 +449,7 @@ void main() { uint32_t OW_idx = NPQ_idx - N_idx * p.OH * p.OW - OH_idx * p.OW; uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + K_idx * p.nb2 + N_idx * p.nb3; if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) { - dst_data[dst_idx] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]); + dst_data[dst_idx + p.dst_offset] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]); } } } @@ -464,7 +469,7 @@ void main() { uint32_t OW_idx = NPQ_idx - N_idx * p.OH * p.OW - OH_idx * p.OW; uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + K_idx * p.nb2 + N_idx * p.nb3; if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) { - dst_data[dst_idx] = regC[T_ly][T_lx]; + dst_data[dst_idx + p.dst_offset] = regC[T_ly][T_lx]; } } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp index d5ce4290..6919861b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp @@ -61,6 +61,11 @@ layout(push_constant) uniform parameter { uint32_t OWmp; uint32_t OWL; uint32_t OWOHmp; uint32_t OWOHL; uint32_t OWOHODmp; uint32_t OWOHODL; + + // element offsets for misaligned buffer bindings + uint32_t knl_offset; + uint32_t src_offset; + uint32_t dst_offset; } p; @@ -214,7 +219,7 @@ ACC_TYPE perElemOpStore(const in uint32_t r, const in uint32_t c, const in ACC_T split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx); uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3; if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) { - dst_data[dst_idx] = D_TYPE(elem); + dst_data[dst_idx + p.dst_offset] = D_TYPE(elem); } return elem; } @@ -261,7 +266,7 @@ void main() { if (aligned == 0) { knl_idx = min(knl_idx, K * CRS - 1); } - float val = knl_data[knl_idx]; + float val = knl_data[knl_idx + p.knl_offset]; if (aligned == 0 && (K_idx >= K || CRS_idx_a >= CRS)) { val = 0.0; } @@ -294,7 +299,7 @@ void main() { if (aligned == 0 || !dhw_in_bounds) { src_idx = min(src_idx, p.IC * p.N * p.IW * p.IH * p.ID - 1); } - float val = src_data[src_idx]; + float val = src_data[src_idx + p.src_offset]; bool oob = false; if (aligned == 0 && (CRS_idx_b >= CRS || NPQ_idx >= NPQ)) { oob = true; @@ -393,7 +398,7 @@ void main() { split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx); uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3; if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) { - dst_data[dst_idx] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]); + dst_data[dst_idx + p.dst_offset] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]); } } } @@ -415,7 +420,7 @@ void main() { split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx); uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3; if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) { - dst_data[dst_idx] = D_TYPE(regC[T_ly][T_lx]); + dst_data[dst_idx + p.dst_offset] = D_TYPE(regC[T_ly][T_lx]); } } } diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index a286083b..cdc10a75 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -7450,7 +7450,7 @@ static void * incr_ptr_aligned(void ** p, size_t size, size_t align) { static size_t ggml_graph_nbytes(size_t size, bool grads) { size_t hash_size = ggml_hash_size(size * 2); - void * p = 0; + void * p = (char *) 1024; // workaround for ubsan error "applying non-zero offset X to null pointer" incr_ptr_aligned(&p, sizeof(struct ggml_cgraph), 1); incr_ptr_aligned(&p, size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)); // nodes incr_ptr_aligned(&p, size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)); // leafs @@ -7463,7 +7463,7 @@ static size_t ggml_graph_nbytes(size_t size, bool grads) { incr_ptr_aligned(&p, ggml_bitset_size(hash_size) * sizeof(ggml_bitset_t), sizeof(ggml_bitset_t)); size_t nbytes = (size_t) p; - return nbytes; + return nbytes - 1024; } size_t ggml_graph_overhead_custom(size_t size, bool grads) { diff --git a/ggml/tests/test-backend-ops.cpp b/ggml/tests/test-backend-ops.cpp index 8402fcfe..e198d092 100644 --- a/ggml/tests/test-backend-ops.cpp +++ b/ggml/tests/test-backend-ops.cpp @@ -52,6 +52,10 @@ #endif static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { + if (ggml_is_empty(tensor)) { + return; + } + size_t nels = ggml_nelements(tensor); std::vector<float> data(nels); { @@ -6254,9 +6258,10 @@ struct test_conv_2d : public test_case { const int dilation1; // Whether the inputs are contiguous in the channel dim or the width dim const bool cwhn; + const int kernel_offset; std::string vars() override { - return VARS_TO_STR10(ne_input, ne_kernel, type_kernel, stride0, stride1, padding0, padding1, dilation0, dilation1, cwhn); + return VARS_TO_STR11(ne_input, ne_kernel, type_kernel, stride0, stride1, padding0, padding1, dilation0, dilation1, cwhn, kernel_offset); } double max_nmse_err() override { @@ -6292,7 +6297,8 @@ struct test_conv_2d : public test_case { test_conv_2d(std::array<int64_t, 4> ne_input = { 64, 64, 16, 1 }, std::array<int64_t, 4> ne_kernel = { 3, 3, 1, 16 }, ggml_type type_kernel = GGML_TYPE_F32, int stride0 = 1, - int stride1 = 1, int padding0 = 0, int padding1 = 0, int dilation0 = 1, int dilation1 = 1, bool cwhn = false) : + int stride1 = 1, int padding0 = 0, int padding1 = 0, int dilation0 = 1, int dilation1 = 1, bool cwhn = false, + int kernel_offset = 0) : ne_input(ne_input), ne_kernel(ne_kernel), type_kernel(type_kernel), @@ -6302,13 +6308,25 @@ struct test_conv_2d : public test_case { padding1(padding1), dilation0(dilation0), dilation1(dilation1), - cwhn(cwhn) {} + cwhn(cwhn), + kernel_offset(kernel_offset) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * input = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_input.data()); ggml_set_name(input, "input"); - ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel.data()); + ggml_tensor * kernel; + if (kernel_offset == 0) { + kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel.data()); + } else { + const int64_t nelem = ne_kernel[0] * ne_kernel[1] * ne_kernel[2] * ne_kernel[3]; + ggml_tensor * storage = ggml_new_tensor_1d(ctx, type_kernel, nelem + kernel_offset); + const size_t element_size = ggml_type_size(type_kernel); + kernel = ggml_view_4d(ctx, storage, ne_kernel[0], ne_kernel[1], ne_kernel[2], ne_kernel[3], + ne_kernel[0] * element_size, ne_kernel[0] * ne_kernel[1] * element_size, + ne_kernel[0] * ne_kernel[1] * ne_kernel[2] * element_size, + kernel_offset * element_size); + } ggml_set_name(kernel, "kernel"); if (cwhn) { @@ -6383,6 +6401,7 @@ struct test_conv_3d : public test_case { const int d0, d1, d2; // Types const ggml_type type_kernel; + const int kernel_offset; std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -6391,7 +6410,7 @@ struct test_conv_3d : public test_case { std::string vars() override { return VARS_TO_STR11(N, IC, ID, IH, IW, OC, KD, KH, KW, s0, s1) + "," + - VARS_TO_STR8(s2, p0, p1, p2, d0, d1, d2, type_kernel); + VARS_TO_STR9(s2, p0, p1, p2, d0, d1, d2, type_kernel, kernel_offset); } double max_nmse_err() override { @@ -6407,7 +6426,7 @@ struct test_conv_3d : public test_case { const int64_t OH = calc_conv_output_size(IH, KH, s1, p1, d1); const int64_t OW = calc_conv_output_size(IW, KW, s0, p0, d0); - return (uint64_t)N * OC * OD * OH * OW * (2 * IC * KD * KH * KW - 1); + return (uint64_t)N * OC * OD * OH * OW * std::max<int64_t>(0, 2 * IC * KD * KH * KW - 1); } test_conv_3d( @@ -6416,13 +6435,13 @@ struct test_conv_3d : public test_case { int s0, int s1, int s2, int p0, int p1, int p2, int d0, int d1, int d2, - ggml_type type_kernel + ggml_type type_kernel, int kernel_offset = 0 ) : N(N), IC(IC), ID(ID), IH(IH), IW(IW), OC(OC), KD(KD), KH(KH), KW(KW), s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), - type_kernel(type_kernel) {} + type_kernel(type_kernel), kernel_offset(kernel_offset) {} ggml_tensor * build_graph(ggml_context * ctx) override { // GGML input tensor is packed as [W, H, D, C*N] @@ -6432,7 +6451,16 @@ struct test_conv_3d : public test_case { // GGML kernel tensor is packed as [KW, KH, KD, IC*OC] const int64_t ne_kernel[] = {KW, KH, KD, IC * OC}; - ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel); + ggml_tensor * kernel; + if (kernel_offset == 0) { + kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel); + } else { + ggml_tensor * storage = ggml_new_tensor_1d(ctx, type_kernel, KW * KH * KD * IC * OC + kernel_offset); + const size_t element_size = ggml_type_size(type_kernel); + kernel = ggml_view_4d(ctx, storage, KW, KH, KD, IC * OC, + KW * element_size, KW * KH * element_size, KW * KH * KD * element_size, + kernel_offset * element_size); + } ggml_set_name(kernel, "kernel"); ggml_tensor * out = ggml_conv_3d_direct(ctx, kernel, input, s0, s1, s2, p0, p1, p2, d0, d1, d2, (int)IC, (int)N, (int)OC); @@ -9375,6 +9403,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_conv_2d({ 19, 17, 8, 2 }, { 3, 3, 8, 65 }, GGML_TYPE_F16, 1, 1, 1, 1, 1, 1)); test_cases.emplace_back(new test_conv_2d({ 19, 17, 16, 3 }, { 3, 3, 16, 33 }, GGML_TYPE_F16, 2, 3, 4, 2, 2, 1)); test_cases.emplace_back(new test_conv_2d({ 13, 11, 16, 3 }, { 1, 1, 16, 33 }, GGML_TYPE_F16, 1, 1, 0, 0, 1, 1)); + test_cases.emplace_back(new test_conv_2d({ 19, 17, 8, 2 }, { 3, 3, 8, 17 }, GGML_TYPE_F16, 1, 1, 1, 1, 1, 1, false, 1)); // sycl backend will limit task global_range < MAX_INT // test cases for 2D im2col with large input W and H (occurs in stable-diffusion) @@ -9446,6 +9475,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } // Case with kernel size 1 test_cases.emplace_back(new test_conv_3d(1, 4, 8, 8, 8, 8, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(2, 8, 5, 11, 9, 65, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(2, 5, 7, 9, 13, 17, 2, 3, 4, 2, 1, 3, 3, 2, 2, 2, 1, 2, kernel_type)); + test_cases.emplace_back(new test_conv_3d(3, 16, 3, 7, 9, 33, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(2, 8, 7, 5, 9, 33, 3, 1, 1, 1, 1, 2, 0, 0, 2, 1, 1, 2, kernel_type)); + test_cases.emplace_back(new test_conv_3d(2, 3, 1, 2, 1, 7, 1, 1, 1, 1, 1, 1, 3, 4, 2, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(2, 8, 5, 7, 9, 17, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, kernel_type, 1)); + test_cases.emplace_back(new test_conv_3d(1, 1, 2, 2, 2, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(1, 1, 2, 2, 2, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(1, 1, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); } for(uint32_t Cout : {1, 9}){