From 27a62e041a6771ac8bd19ec835cdbe49fa7bfe94 Mon Sep 17 00:00:00 2001 From: mudler <2420543+mudler@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:29:24 +0000 Subject: [PATCH 1/2] :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- backend/cpp/llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index bcd388182d78..b8b1b08be4ef 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=a37cc46104eb50900d5ab4a9ba02a0abdac039db +LLAMA_VERSION?=78a8ae14be559b45f5f13ddee82c188b683d5d8d LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= From aab56052f0a13da862f71e791eb6aabaffd0c5eb Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 25 Oct 2026 07:59:47 +0000 Subject: [PATCH 2/2] fix(llama-cpp): adapt to the multimodal rerank signature Upstream llama.cpp added query_files and doc_files parameters to format_prompt_rerank() (ggml-org/llama.cpp#29625). The gRPC rerank call passed the init options in their place, so grpc-server.cpp no longer compiled against the new pin. prepare.sh now probes server-common.h for query_files and writes LOCALAI_HAS_RERANK_FILES into llama_compat.h. When it is set, the rerank call passes two empty media lists. The rerank API is text only, so behaviour is unchanged. Forks that still use the old signature (turboquant, bonsai) compile as before. Assisted-by: Claude:claude-sonnet-5-5 Signed-off-by: Ettore Di Giacinto --- backend/cpp/llama-cpp/grpc-server.cpp | 10 +++++++++- backend/cpp/llama-cpp/prepare.sh | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 21e556beff8b..55f777edb3f5 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -107,6 +107,14 @@ using grpc::Status; #define LOCALAI_MTMD_INIT_OPT_ARG(value) #endif +// Newer format_prompt_rerank() takes the decoded media of the query and of the +// document. The gRPC rerank API is text only, so pass empty lists. +#if LOCALAI_HAS_RERANK_FILES +#define LOCALAI_RERANK_FILES_ARGS , std::vector(), std::vector() +#else +#define LOCALAI_RERANK_FILES_ARGS +#endif + // gRPC bearer token auth for distributed mode. // Reads LOCALAI_GRPC_AUTH_TOKEN from the environment. When set, rejects // requests without a matching "authorization: Bearer " metadata header. @@ -3114,7 +3122,7 @@ class BackendServiceImpl final : public backend::Backend::Service { tasks.reserve(documents.size()); for (size_t i = 0; i < documents.size(); i++) { - auto tmp = format_prompt_rerank(ctx_server.impl->model_tgt, ctx_server.impl->vocab, ctx_server.impl->mctx, request->query(), documents[i] LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)); + auto tmp = format_prompt_rerank(ctx_server.impl->model_tgt, ctx_server.impl->vocab, ctx_server.impl->mctx, request->query(), documents[i] LOCALAI_RERANK_FILES_ARGS LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)); server_task task = server_task(SERVER_TASK_TYPE_RERANK); task.id = rd.queue_tasks.get_new_id(); task.index = i; diff --git a/backend/cpp/llama-cpp/prepare.sh b/backend/cpp/llama-cpp/prepare.sh index 82d13b4fa153..3d70f5447d43 100644 --- a/backend/cpp/llama-cpp/prepare.sh +++ b/backend/cpp/llama-cpp/prepare.sh @@ -134,6 +134,13 @@ if grep -q "common_chat_session & out_session" llama.cpp/tools/server/server-com else HAS_CHAT_SESSION=0 fi +# Multimodal rerank (query_files/doc_files on format_prompt_rerank, +# ggml-org/llama.cpp#29625). Forks that branched earlier keep the old signature. +if grep -q "query_files" llama.cpp/tools/server/server-common.h; then + HAS_RERANK_FILES=1 +else + HAS_RERANK_FILES=0 +fi cat > llama.cpp/tools/grpc-server/llama_compat.h < llama.cpp/tools/grpc-server/llama_compat.h <