Add idle servable management preview

docs/parameters.md

@@ -59,6 +59,16 @@ Configuration options for the server are defined only via command-line options a
 | `allowed_media_domains` | `string` | Comma separated list of media domains from which URLs can be used as input for LLMs. Set to \"all\" to disable this restrictions. If unset, URLs in requests are not supported."
 | `verbose_response` | `NA` | When enabled, responses include an extra `__verbose` object with additional debug information. Applies for text generation models |
 | `disable_input_count_validation` | `bool` (default: false) | Disables enforcement for the KServe requests to match all the model inputs. It ignores all inputs which are not used in the model. Not recommended for performance reasons but in some cases might simplify the client. |
+| `idle_unload_timeout_seconds` | `integer` (default: 0) | Unloads servable resources after this many seconds with no inference requests, freeing GPU/CPU memory. Resources are reloaded automatically on the next request. Set to `0` to disable the feature. See [Idle servable unload](#idle-servable-unload). |
+
+### Idle servable unload
+
+When `idle_unload_timeout_seconds` is set to a positive value, the model server unloads the active servable group's heavy resources after the configured period without any inference requests. The first request after an unload transparently reloads the requested servable and is served once it is ready, so the GPU can be used by other workloads while a servable is idle.
+
+Notes:
+- Only inference requests reset the idle timer; status, metrics, and health endpoints do not keep a servable loaded.
+- The first request after an idle unload pays the reload latency. Combine with [model caching](model_cache.md) (`--cache_dir`) so the reload is a fast cache import rather than a full recompile.
+- The servable reports as `AVAILABLE` while idle-unloaded and reloads on demand. The `ovms_graph_loaded` metric reports `1` when loaded and `0` when idle-unloaded.
 
 ## Config management mode options
 

spelling-whitelist.txt

@@ -42,3 +42,9 @@ src/test/llm/output_parsers/gemma4_output_parser_test.cpp
 src/test/llm/output_parsers/qwen3_output_parser_test.cpp:697: thi ==> the, this
 extras/chat_template_examples/chat_template_onyx.jinja
 src/test/llm/chat_templates/chat_template_onyx.jinja
+src/mediapipe_internal/mediapipegraphdefinition.cpp
+src/mediapipe_internal/mediapipegraphdefinition.hpp
+src/test/llm/llmnode_test.cpp
+nowNs ==> knowns, nouns
+src/test/servable_group_manager_test.cpp:229: groupD
+src/test/servable_group_manager_test.cpp:231: groupD

src/BUILD

@@ -158,6 +158,7 @@ ovms_cc_library(
 ovms_cc_library(
     name = "libovms_servable_definition",
     hdrs = ["servable_definition.hpp"],
+    visibility = ["//visibility:public"],
 )
 ovms_cc_library(
     name = "libovms_single_version_servable_definition",
@@ -575,48 +576,6 @@ ovms_cc_library(
     ],
     visibility = ["//visibility:public"],
 )
-ovms_cc_library(
-    name = "modelmanager",
-    hdrs = ["modelmanager.hpp"],
-    srcs = ["modelmanager.cpp"],
-    deps = select({
-        "//conditions:default": [],
-        "//:not_disable_mediapipe" : [
-            "//src/mediapipe_internal:libovms_mediapipe",
-        ],
-    }) + [
-        "cleaner_utils",
-        "customloaders",
-        "libovms_config",
-        "libovms_model_instance_provider",
-        "libovms_ov_utils",
-        "libovms_servable_definition",
-        "libovms_servable_name_checker",
-        "libovmslogging",
-        "libovmsschema",
-        "libovmsstring_utils",
-        "libovmsstatus",
-        "model",
-        "modelconfig",
-        "modelinstance",
-        "modelinstanceunloadguard",
-        "resources_cleaner",
-        "//src/dags:custom_node_library_manager",
-        "//src/dags:dag_resource_manager",
-        "//src/dags:pipeline_config_parser",
-        "//src/dags:pipeline_factory",
-        "//src/dags:pipelinedefinition",
-        "//src/filesystem:libovmsfilesystem",
-        "//src/filesystem:libovmsfilesystemfactory",
-        "//src/graph_export:graph_export",
-        "//src/metrics:libovms_metric_provider",
-        "//src/metrics:libovmsmetrics",
-        "@com_github_tencent_rapidjson//:rapidjson",
-        "//src/port:rapidjson_stringbuffer",
-        "//src/port:rapidjson_writer",
-    ],
-    visibility = ["//visibility:public"],
-)
 ovms_cc_library(
     name = "rest_parser_utils",
     hdrs = [
@@ -701,24 +660,6 @@ ovms_cc_library(
     ],
     visibility = ["//visibility:public"],
 )
-ovms_cc_library(
-    name = "servablemanagermodule",
-    hdrs = ["servablemanagermodule.hpp"],
-    srcs = ["servablemanagermodule.cpp"],
-    deps = select({
-        "//:not_disable_python": [
-            "//src/python:libovmspythonmodule",
-        ],
-        "//:disable_python": []
-    }) + [
-        "cpp_headers",
-        "libovms_module",
-        "libovmslogging",
-        "modelmanager",
-        "//src/metrics:libovmsmetrics",
-    ],
-    visibility = ["//visibility:public"],
-)
 ovms_cc_library(
     name = "ovms_lib",
     hdrs = [
@@ -832,8 +773,8 @@ ovms_cc_library(
             "libovms_kfs_utils",
             "libovms_kfs_grpc_inference_service_h",
             "modelchangesubscription",
-            "modelmanager",
-            "servablemanagermodule",
+            "//src/servable_management:modelmanager",
+            "//src/servable_management:servablemanagermodule",
             "//src/filesystem:libovmslocalfilesystem", # indirectly & directly through factory
             "libovmslogging",
             "//src/metrics:libovmsmetrics",
@@ -1561,6 +1502,13 @@ ovms_cc_library(
     ],
     visibility = ["//visibility:public"],
 )
+ovms_cc_library(
+    name = "libovms_time_utils",
+    hdrs = ["time_utils.hpp"],
+    deps = [
+    ],
+    visibility = ["//visibility:public"],
+)
 ovms_cc_library(
     name = "libovmslayout",
     hdrs = ["layout.hpp",],
@@ -2262,6 +2210,9 @@ cc_test(
             ":test_test_models_configs",
             ":test_cmd_exec",
             ":test_modelinstance_test",
+            ":servable_loading_queue_test",
+            ":test_idle_model_test",
+            ":test_servable_group_manager",
             ] + select({
                 "//conditions:default": [
                     ":openvino_remote_tensors_tests",
@@ -2286,6 +2237,7 @@ cc_test(
                 ":text2image_test",
                 "//src/rerank:rerank_api_handler",
                 ":embeddings_handler_tests",
+                ":test_idle_mediapipe_test",
                 "libovms_mediapipe_kfs_executor",
                 "//src/mediapipe_internal:mediapipe_utils",
                 "tensorflow_type_utils",
@@ -2374,6 +2326,57 @@ cc_library(
     linkopts = COMMON_STATIC_LIBS_LINKOPTS,
 )
 
+ovms_cc_test_library(
+    name = "servable_loading_queue_test",
+    srcs = ["test/servable_loading_queue_test.cpp"],
+    deps = [
+        "//src/servable_management:servable_loading_queue",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+ovms_cc_test_library(
+    name = "test_idle_model_test",
+    srcs = ["test/idle_model_test.cpp"],
+    deps = [
+        ":libtest_environment",
+        ":test_constructor_enabled_model_manager",
+        ":test_test_models",
+        ":test_test_models_configs",
+        ":test_test_with_temp_dir",
+        ":test_utils",
+        "//third_party:openvino",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+ovms_cc_test_library(
+    name = "test_servable_group_manager",
+    srcs = ["test/servable_group_manager_test.cpp"],
+    deps = [
+        ":libtest_environment",
+        ":test_constructor_enabled_model_manager",
+        ":test_test_models",
+        ":test_test_with_temp_dir",
+        "//src/servable_management:modelmanager",
+        "//src:modelconfig",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+ovms_cc_test_library(
+    name = "test_idle_mediapipe_test",
+    srcs = ["test/idle_mediapipe_test.cpp"],
+    deps = [
+        ":test_constructor_enabled_model_manager",
+        ":test_utils",
+        "//src/dags:pipelinedefinitionstatus",
+        "//src/mediapipe_internal:libovms_mediapipe",
+        "//src/mediapipe_internal:mediapipegraphconfig",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 cc_library(
     name = "test_utils",
     linkstatic = 1,

src/capi_frontend/capi.cpp

@@ -40,13 +40,13 @@
 #include "../deserialization_main.hpp"
 #include "../inference_executor.hpp"
 #include "../modelinstanceunloadguard.hpp"
-#include "../modelmanager.hpp"
+#include "src/servable_management/modelmanager.hpp"
 #include "../module_names.hpp"
 #include "../ovms.h"  // NOLINT
 #include "../profiler.hpp"
 #include "../dags/pipelinedefinitionstatus.hpp"
 #include "../servable_definition.hpp"
-#include "../servablemanagermodule.hpp"
+#include "src/servable_management/servablemanagermodule.hpp"
 #include "../server.hpp"
 #include "../single_version_servable_definition.hpp"
 #include "../status.hpp"

src/capi_frontend/capi_dag_utils.cpp

@@ -41,7 +41,9 @@ OVMS_ServableState convertToServableState(ovms::PipelineDefinitionStateCode code
     case ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED:
     case ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION:
         return OVMS_ServableState::OVMS_STATE_LOADING_FAILED;
-    }
+    case ovms::PipelineDefinitionStateCode::SLEEPING:
+        return OVMS_ServableState::OVMS_STATE_AVAILABLE;
+    }  // TODO #atobiszei idle management C-API change - new value in enum?
     throw new std::exception();
 }
 

src/capi_frontend/server_settings.hpp

@@ -246,6 +246,7 @@ struct ServerSettingsImpl {
     std::string grpcChannelArguments;
     uint32_t filesystemPollWaitMilliseconds = 1000;
     uint32_t resourcesCleanerPollWaitSeconds = 300;
+    uint32_t idleUnloadTimeoutSeconds = 0;
     std::string cacheDir;
     bool withPython = false;
     bool startedWithCLI = false;
@@ -268,6 +269,7 @@ struct ModelsSettingsImpl {
     uint32_t nireq = 0;
     std::string targetDevice;
     std::string pluginConfig;
+    std::optional<std::string> groupName;
     std::vector<std::string> userSetSingleModelArguments;
 
     std::string configPath;

src/cli_parser.cpp

@@ -72,7 +72,7 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
     std::stringstream ss;
     try {
         options = std::make_unique<cxxopts::Options>(argv[0], "OpenVINO Model Server");
-        auto configOptions = std::make_unique<cxxopts::Options>("ovms --add_to_config --config_path <CONFIG_PATH> --model_name <MODEL_NAME> --model_repository_path <MODEL_REPO_PATH> \n  ovms --add_to_config --config_path <CONFIG_PATH> --model_path <MODEL_PATH> --model_name <MODEL_NAME>  \n  ovms --remove_from_config --config_path <CONFIG_PATH> --model_name <MODEL_NAME>", "config management commands:");
+        auto configOptions = std::make_unique<cxxopts::Options>("ovms --add_to_config --config_path <CONFIG_PATH> --model_name <MODEL_NAME> --model_repository_path <MODEL_REPO_PATH> \n  ovms --add_to_config --config_path <CONFIG_PATH> --model_path <MODEL_PATH> --model_name <MODEL_NAME> --group_name <GROUP> \n  ovms --remove_from_config --config_path <CONFIG_PATH> --model_name <MODEL_NAME>", "config management commands:");
         // Adding this option to parse unrecognised options in another parser
         options->allow_unrecognised_options();
 
@@ -141,6 +141,10 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
                 "Time interval between config and model versions changes detection. Default is 1. Zero or negative value disables changes monitoring.",
                 cxxopts::value<uint32_t>()->default_value("1"),
                 "FILE_SYSTEM_POLL_WAIT_SECONDS")
+            ("idle_unload_timeout_seconds",
+                "Idle timeout in seconds for model group unloading. When > 0, models not in the 'permanent' group are loaded on demand and unloaded after this idle period. Only effective with config.json multi-model setup. Default is 0 (disabled).",
+                cxxopts::value<uint32_t>()->default_value("0"),
+                "IDLE_UNLOAD_TIMEOUT_SECONDS")
             ("custom_node_resources_cleaner_interval_seconds",
                 "Time interval between two consecutive resources cleanup scans. Default is 300. Zero value disables resources cleaner.",
                 cxxopts::value<uint32_t>()->default_value("300"),
@@ -213,7 +217,11 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
             ("remove_from_config",
                 "Directive to remove a model from configuration file. This parameter should be executed with --config_path and --model_name to specify which model to remove.",
                 cxxopts::value<bool>()->default_value("false"),
-                "REMOVE_FROM_CONFIG");
+                "REMOVE_FROM_CONFIG")
+            ("group_name",
+                "Optional group name for idle model group management. Used with --add_to_config.",
+                cxxopts::value<std::string>(),
+                "GROUP_NAME");
 
         // Set default value for model_repository_path from environment variable if it exists and is not empty
         std::string defaultModelRepoPath = "";
@@ -347,6 +355,10 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
                 "Name of the model",
                 cxxopts::value<std::string>(),
                 "MODEL_NAME")
+            ("group_name",
+                "Optional group name for idle model group management",
+                cxxopts::value<std::string>(),
+                "GROUP_NAME")
             ("config_path",
                 "Path to json configuration file",
                 cxxopts::value<std::string>()->default_value(defaultConfigPath),
@@ -573,6 +585,7 @@ void CLIParser::prepareServer(ServerSettingsImpl& serverSettings) {
     serverSettings.filesystemPollWaitMilliseconds = result->operator[]("file_system_poll_wait_seconds").as<uint32_t>() * 1000;
 
     serverSettings.resourcesCleanerPollWaitSeconds = result->operator[]("custom_node_resources_cleaner_interval_seconds").as<uint32_t>();
+    serverSettings.idleUnloadTimeoutSeconds = result->operator[]("idle_unload_timeout_seconds").as<uint32_t>();
     serverSettings.grpcWorkers = result->operator[]("grpc_workers").as<uint32_t>();
 
     if (result->count("log_level"))
@@ -927,6 +940,10 @@ void CLIParser::prepareConfigExport(ModelsSettingsImpl& modelsSettings) {
     } else if (!result->operator[]("model_repository_path").as<std::string>().empty() && result->count("model_name")) {
         modelsSettings.modelPath = FileSystem::joinPath({result->operator[]("model_repository_path").as<std::string>(), modelsSettings.modelName});
     }
+    if (result->count("group_name")) {
+        modelsSettings.groupName = result->operator[]("group_name").as<std::string>();
+        modelsSettings.userSetSingleModelArguments.push_back("group_name");
+    }
     std::string defaultConfigPath = "";
     const char* envModelRepoPath = std::getenv("OVMS_MODEL_REPOSITORY_PATH");
     if (envModelRepoPath != nullptr && std::string(envModelRepoPath).length() > 0) {

src/config.cpp

@@ -141,7 +141,7 @@ bool Config::validateUserSettingsInConfigAddRemoveModel(const ModelsSettingsImpl
     static const std::vector<std::string> allowedForRemove = {"model_name", "config_path"};
     static const std::vector<std::string> allowedForAdd = {"model_name", "model_path", "config_path",
         "batch_size", "shape", "layout", "mean", "scale", "color_format", "precision",
-        "model_version_policy", "nireq", "target_device", "plugin_config"};
+        "model_version_policy", "nireq", "target_device", "plugin_config", "group_name"};
 
     const auto& allowedUserSettings = (exportType == ENABLE_MODEL) ? allowedForAdd : allowedForRemove;
     std::vector<std::string> usedButDisallowedUserSettings;
@@ -432,6 +432,7 @@ const std::string& Config::tracePath() const { return this->serverSettings.trace
 const std::string& Config::grpcChannelArguments() const { return this->serverSettings.grpcChannelArguments; }
 uint32_t Config::filesystemPollWaitMilliseconds() const { return this->serverSettings.filesystemPollWaitMilliseconds; }
 uint32_t Config::resourcesCleanerPollWaitSeconds() const { return this->serverSettings.resourcesCleanerPollWaitSeconds; }
+uint32_t Config::idleUnloadTimeoutSeconds() const { return this->serverSettings.idleUnloadTimeoutSeconds; }
 bool Config::disableInputCountValidation() const { return this->serverSettings.disableInputCountValidation; }
 bool Config::allowCredentials() const { return this->serverSettings.allowCredentials; }
 const std::string& Config::allowedOrigins() const { return this->serverSettings.allowedOrigins; }

src/config.hpp

@@ -312,6 +312,8 @@ class Config {
      */
     uint32_t resourcesCleanerPollWaitSeconds() const;
 
+    uint32_t idleUnloadTimeoutSeconds() const;
+
     bool disableInputCountValidation() const;
     bool allowCredentials() const;
     const std::string& allowedOrigins() const;

src/config_export_module/config_export.cpp

@@ -56,6 +56,8 @@ static void addOptionalModelFields(rapidjson::Value& configObj, const ModelsSett
         configObj.AddMember("target_device", rapidjson::Value(modelSettings.targetDevice.c_str(), alloc), alloc);
     if (!modelSettings.pluginConfig.empty())
         addJsonOrStringMember(configObj, "plugin_config", modelSettings.pluginConfig, alloc);
+    if (modelSettings.groupName.has_value())
+        configObj.AddMember("group_name", rapidjson::Value(modelSettings.groupName.value().c_str(), alloc), alloc);
 }
 
 Status loadJsonConfig(const std::string& jsonFilename, rapidjson::Document& configJson) {

src/dags/pipelinedefinition.cpp

@@ -138,7 +138,7 @@ Status PipelineDefinition::reload(ModelInstanceProvider& modelInstanceProvider,
     // block creating new unloadGuards
     this->status.handle(ReloadEvent());
     resetSubscriptions(modelInstanceProvider);
-    while (requestsHandlesCounter > 0) {
+    while (pendingCreateExecutorCount > 0) {
         std::this_thread::sleep_for(std::chrono::microseconds(1));
     }
     // deinitialize all resources that are associated with nodes that are currently in PipelineDefinition, but not in nodeInfos
@@ -153,7 +153,7 @@ Status PipelineDefinition::reload(ModelInstanceProvider& modelInstanceProvider,
 void PipelineDefinition::retire(ModelInstanceProvider& modelInstanceProvider) {
     resetSubscriptions(modelInstanceProvider);
     this->status.handle(RetireEvent());
-    while (requestsHandlesCounter > 0) {
+    while (pendingCreateExecutorCount > 0) {
         std::this_thread::sleep_for(std::chrono::microseconds(1));
     }
     // deinitalize all resources