feat: on-demand training and SentenceTransformer CS on dev-swift-v5

.dev_scripts/ci_container_test.sh

@@ -26,7 +26,7 @@ if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
     pip uninstall autoawq -y
     pip uninstall lmdeploy -y
     pip uninstall tensorflow -y
-    pip install ray==2.48
+    pip install ray==2.55.1
     pip install optimum
 
     # test with install

.gitignore

@@ -169,3 +169,8 @@ tests/server/contract/client_api_baseline.json
 
 # Redis dump file produced by a local redis-server (test infra), never source.
 *.rdb
+
+# Local ST dependency reproducers and acceptance notes.
+/cookbook/client/swift_model/README-st.md
+/cookbook/client/swift_model/reproduce_st_dependency.py
+/cookbook/client/swift_model/requirements-st.txt

cookbook/client/async_rl/server_config.yaml

@@ -64,6 +64,10 @@ applications:
     args:
       backend: transformers
       model_id: ${oc.env:TWINKLE_LOCAL_MODEL_PATH}
+      min_replicas: 1
+      max_replicas: 1
+      adapter_config:
+        adapter_timeout: 30
       max_loras: 8
       max_length: 10240
       data_plane_url: http://127.0.0.1:8000/api/v1/data-plane
@@ -78,8 +82,6 @@ applications:
       queue_config:
         rps_limit: 100
         tps_limit: 100000
-      adapter_config:
-        adapter_timeout: 30
     deployments:
       - name: ModelManagement
         autoscaling_config:

cookbook/client/server/megatron/server_config.yaml

@@ -1,4 +1,8 @@
-# Twinkle Server Configuration - Megatron Backend
+# Twinkle Server Configuration - Megatron LoRA Backend
+# Candidate for 8 x L20D (~269 GiB each): 2-4 single-GPU sampler instances
+# plus up to 2 two-GPU LoRA training instances (3 handles each, 6 total).
+# Validate 27B/32K peak memory, throughput and optimizer resume on L20D
+# before production rollout; local smaller-model tests do not prove those limits.
 
 # proxy_location: determines where the HTTP proxy runs.
 # "EveryNode" means each Ray node runs its own proxy (good for multi-node).
@@ -26,6 +30,13 @@ persistence:
   mode: redis
   redis_url: redis://localhost:6380
 
+# Independent allocation, startup and confirmed-shutdown budgets.
+resources:
+  allocation_timeout_s: 3
+  startup_timeout_s: 1800
+  shutdown_timeout_s: 180
+  sweep_interval_s: 5
+
 # Applications: each entry defines a service component deployed on the server
 applications:
 
@@ -36,7 +47,7 @@ applications:
     import_path: server            # Python module to import
     args:
       server_config:
-        per_token_model_limit: 3      # Maximum number of models (adapters) per token (server-globally enforced)
+        per_token_model_limit: 3      # Global per-token limit; shared-token clients share these 3 slots
       supported_models:
         - Qwen/Qwen3.8-27B
 
@@ -56,30 +67,33 @@ applications:
 
   # 3. Sampler Service - Runs inference / sampling using vLLM engine
   #    Used for generating text from the model (e.g., evaluating LoRA results).
-  #    Config: TP=2 x DP=2 on 4 GPUs, ~27GB weights/GPU, ~37GB for KV cache + LoRA
+  #    Each instance has DP=1 and one GPU. Two are preloaded; scale to four.
+  #    Memory usage includes each engine's preallocated KV cache, not just weights.
   - name: sampler-Qwen3.8-27B
     route_prefix: /api/v1/sampler/Qwen/Qwen3.8-27B
     import_path: sampler
     args:
       model_id: "ms://Qwen/Qwen3.8-27B"  # ModelScope model identifier
-      nproc_per_node: 4               # Number of GPU processes per node
+      nproc_per_node: 1               # One GPU per independently allocated instance
+      min_replicas: 2
+      max_replicas: 4
+      target_ongoing_requests: 16
+      idle_timeout_s: 600
       sampler_type: vllm              # Inference engine: vllm
       engine_args:                    # vLLM engine-specific settings
         max_model_len: 32768           # Maximum sequence length the engine supports
-        gpu_memory_utilization: 0.75   # 80% utilization, ~64GB/GPU, leaves buffer for safety
+        gpu_memory_utilization: 0.75   # 75% engine memory budget; does not free Ray GPU reservations
         enable_lora: true             # Allow loading LoRA adapters during inference
         max_loras: 5                  # Max allowed loras working on vLLM at the same time
-        max_lora_rank: 32             # Support up to rank 64 LoRA adapters
+        max_lora_rank: 32             # Maximum supported LoRA rank
         enable_tower_connector_lora: true
       device_group:                   # Logical device group for the sampler
         name: sampler
-        gpus_per_worker: 2
-        ranks: 4                   # GPU rank indices to use
+        ranks: 1                   # Per-instance device count, not fixed physical GPU IDs
         device_type: cuda
       device_mesh:
         device_type: cuda
-        dp_size: 2
-        tp_size: 2                  # 2 TP replicas for multi-tenant throughput
+        dp_size: 1                 # Pool instance count is controlled by args.min/max_replicas
       queue_config:
         rps_limit: 20                               # Max requests per second
         tps_limit: 131072                            # Max tokens per second
@@ -97,33 +111,38 @@ applications:
               TWINKLE_TRUST_REMOTE_CODE: "0"
               TWINKLE_LONG_POLL_TIMEOUT: "120"
 
-  # 2. Model Service - Hosts the base model for training.
-  #    Config: PP=2 x DP=2 on 4 GPUs, ~27GB weights/GPU, comfortable for LoRA training
+  # 2. Model Service - LoRA-only, independently queued two-GPU instances.
+  #    Megatron instances require at least two devices; PP=2 shards each model.
+  #    min=0 enables cold startup; max=2 limits total training devices to four.
   - name: models-Qwen3.8-27B
     route_prefix: /api/v1/model/Qwen/Qwen3.8-27B
     import_path: model
     args:
       backend: megatron                          # Use Megatron-LM backend
       model_id: "ms://Qwen/Qwen3.8-27B"          # ModelScope model identifier
       max_length: 32768                           # model max length
-      max_loras: 3                                # model max loras
-      nproc_per_node: 4                           # Number of GPU processes per node
+      train_mode: lora
+      adapter_config:
+        adapter_timeout: 120                       # Seconds without a session heartbeat before release
+        adapter_max_lifetime: 36000               # Maximum lifetime of an adapter in seconds (e.g., 10 hours)
+      max_loras: 3                                # Handles per instance, not simultaneous backward calls
+      min_replicas: 0                             # Do not reserve training GPUs at startup
+      max_replicas: 2                             # Up to 2 independent two-GPU instances
+      idle_timeout_s: 600                         # Business idle; heartbeats/polling do not renew it
+      nproc_per_node: 2                           # Two workers per training instance
       device_group:
         name: model
-        ranks: 4
+        ranks: 2
         device_type: cuda
       device_mesh:
         device_type: cuda
-        dp_size: 2
+        dp_size: 1
         pp_size: 2
 
       queue_config:
         rps_limit: 20                               # Max requests per second
         tps_limit: 131072                            # Max tokens per second
         max_input_tokens: 32768
-      adapter_config:
-        adapter_timeout: 120                       # Seconds before idle adapter unload
-        adapter_max_lifetime: 36000               # Maximum lifetime of an adapter in seconds (e.g., 10 hours)
     deployments:
       - name: ModelManagement
         autoscaling_config:

cookbook/client/server/megatron/server_config_4b.yaml

@@ -39,6 +39,8 @@ applications:
     args:
       backend: megatron
       model_id: "ms://Qwen/Qwen3.5-4B" # ModelScope model identifier
+      min_replicas: 1
+      max_replicas: 1
       max_length: 10240
       nproc_per_node: 2                            # Number of GPU processes per node
       device_group:
@@ -53,7 +55,7 @@ applications:
         tps_limit: 100000                           # Max tokens per second for a single user
         max_input_tokens: 60000                    # Maximum input tokens per request
       adapter_config:
-        adapter_timeout: 30                        # Seconds before idle adapter unload
+        adapter_timeout: 30                        # Seconds without a session heartbeat before release
         adapter_max_lifetime: 36000                # Maximum lifetime of an adapter in seconds (e.g., 10 hours)
       max_loras: 5                                 # Maximum number of LoRA adapters per model
     deployments:

cookbook/client/server/transformer/server_config.yaml

@@ -55,6 +55,8 @@ applications:
     args:
       backend: transformers                        # Model backend: transformers | megatron
       model_id: "ms://Qwen/Qwen3.5-4B"    # ModelScope model identifier
+      min_replicas: 1
+      max_replicas: 1
       max_length: 10240
       nproc_per_node: 1                            # Number of GPU processes per node
       device_group:
@@ -68,7 +70,7 @@ applications:
         rps_limit: 100                              # Max requests per second
         tps_limit: 100000                           # Max tokens per second
       adapter_config:
-        adapter_timeout: 30                      # Seconds before idle adapter unload
+        adapter_timeout: 30                      # Seconds without a session heartbeat before release
     deployments:
       - name: ModelManagement
         autoscaling_config:

cookbook/client/swift_model/sentence_transformer_client.py

@@ -0,0 +1,86 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+"""ST LoRA over HTTP: train, read embeddings, save, recreate, resume and release."""
+
+import argparse
+
+import json
+import numpy as np
+from peft import LoraConfig
+from transformers import AutoTokenizer
+
+from twinkle import init_twinkle_client
+
+MODEL = "sentence-transformers/all-MiniLM-L6-v2"
+TEXTS = [
+    "A cat sits on a mat.",
+    "There is a cat on the mat.",
+    "A plane is flying.",
+    "The dog runs in a park.",
+    "A dog is running outdoors.",
+    "The stock market closed.",
+]
+
+
+def encoded_batch(tokenizer):
+    rows = [dict(tokenizer(text, truncation=True, max_length=128)) for text in TEXTS]
+    for index, row in enumerate(rows):
+        row["labels"] = [int(index % 3 == 0)]
+    return rows
+
+
+def configure(model, *, lr=1e-3, max_length=128):
+    model.set_template("Template", max_length=max_length)
+    model.set_processor("InputProcessor", padding_side="right")
+    model.set_loss("InfonceLoss", temperature=0.1)
+    model.set_optimizer("AdamW", lr=lr)
+
+
+def vectors(model, batch):
+    return np.asarray(model.forward_only(inputs=batch).result["embeddings"], dtype=float)
+
+
+def step(model, batch):
+    model.forward_backward(inputs=batch, task="embedding")
+    model.clip_grad_and_step()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--base-url", default="http://127.0.0.1:8000")
+    parser.add_argument("--model", default=MODEL, help="Public model name in the server YAML")
+    parser.add_argument("--tokenizer", default=MODEL, help="Original ST model or local cache directory")
+    parser.add_argument("--api-key", default="st-example")
+    args = parser.parse_args()
+    batch = encoded_batch(AutoTokenizer.from_pretrained(args.tokenizer))
+    client = init_twinkle_client(base_url=args.base_url, api_key=args.api_key)
+    active = None
+    try:
+        active = client.model(args.model)
+        active.add_adapter_to_model("first", LoraConfig(r=4, target_modules=["query", "value"]))
+        configure(active)
+        before = vectors(active, batch)
+        for _ in range(2):
+            step(active, batch)
+        trained = vectors(active, batch)
+        delta = float(np.max(np.abs(trained - before)))
+        assert delta > 1e-6, "Training did not change embeddings"
+        saved = active.save("st-example", save_optimizer=True, consumed_train_samples=2 * len(batch))
+        active.remove_adapter()
+        active = client.model(args.model)
+        active.add_adapter_to_model("restored", LoraConfig(r=4, target_modules=["query", "value"]))
+        configure(active)
+        progress = active.resume_from_checkpoint(saved.twinkle_path)
+        np.testing.assert_allclose(vectors(active, batch), trained, rtol=0, atol=2e-5)
+        assert progress["consumed_train_samples"] == 2 * len(batch)
+        step(active, batch)
+        print(json.dumps({"checkpoint": saved.twinkle_path, "progress": progress, "embedding_delta": delta}))  # noqa: T201
+    finally:
+        try:
+            if active is not None:
+                active.remove_adapter()
+        finally:
+            client.close()
+
+
+if __name__ == "__main__":
+    main()