diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 964fd8483fc..9cb32886994 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,12 @@ Changelog **Backward Breaking Changes** - Move the Mistral Medium 3.5 checkpoint-mirror recipe from ``huggingface/models/nvidia/Mistral-Medium-3.5-128B-NVFP4/ptq/nvfp4-max-calib`` to ``huggingface/models/mistralai/Mistral-Medium-3.5-128B/ptq/nvfp4-max-calib``, keying it by the canonical Hugging Face base model. Update any saved ``--recipe`` paths to the new location. +- Remove the ``--auto_quantize_bits``, ``--auto_quantize_method``, ``--auto_quantize_score_size``, ``--auto_quantize_cost_model`` and ``--auto_quantize_active_moe_expert_ratio`` flags from ``examples/hf_ptq`` (deprecated in 0.46). Use an AutoQuantize ``--recipe`` from ``modelopt_recipes/general/auto_quantize/`` instead. Those recipes now also splice in the shared base ``cost_excluded_layers`` unit, which the removed CLI applied unconditionally, so a VL model keeps its vision tower and MTP layers out of the effective-bits denominator. +- Remove the ``examples/llm_ptq`` symlink and the ``examples/vlm_ptq`` forwarder (both deprecated in 0.46). Use ``examples/hf_ptq``, passing ``--vlm`` for vision-language models. +- Remove the backward-compat ``--qformat`` / ``--quant_cfg`` short names ``int8_sq``, ``int8_wo``, ``w4a8_awq``, ``nvfp4_awq``, ``nvfp4_mse``, ``nvfp4_local_hessian``, ``fp8_pb_wo`` and ``fp8_pc_pt`` (deprecated in 0.45). Use the preset basename under ``modelopt_recipes/configs/ptq/presets/model/`` instead: ``int8_smoothquant``, ``int8_weight_only``, ``w4a8_awq_beta``, ``nvfp4_awq_lite``, ``nvfp4_w4a4_weight_mse_fp8_sweep``, ``nvfp4_w4a4_weight_local_hessian``, ``fp8_2d_blockwise_weight_only`` and ``fp8_per_channel_per_token``. The ``modelopt.recipe.presets.QFORMAT_ALIASES`` table and the ``aliases`` argument of ``load_quant_cfg_choices()`` are removed along with them. +- Remove the legacy ``layerwise`` bool form, its ``use_sequential`` alias, and the top-level ``layerwise_checkpoint_dir`` key from calibration algorithm configs (deprecated in 0.45). Use the nested form, e.g. ``layerwise: {enable: true, checkpoint_dir: /path}``. A pre-0.45 ``modelopt_state`` carrying either legacy key now fails validation on restore instead of being migrated; re-save it with a 0.45/0.46 release first. +- Remove in-trainer quantization via ``QuantizationArguments.quant_cfg`` / ``--quant_cfg`` (deprecated in 0.45); use ``--recipe``. New recipes ``general/ptq/mxfp4_mlp_weight_only`` and ``general/ptq/nvfp4_mlp_weight_only`` replace ``MXFP4_MLP_WEIGHT_ONLY_CFG`` / ``NVFP4_MLP_WEIGHT_ONLY_CFG`` in the ``examples/gpt-oss`` QAT flow. +- Remove the ``QuantizationArgumentsWithConfig`` alias in ``modelopt.torch.quantization.plugins.transformers_trainer`` (deprecated in 0.45). Use ``QuantizationArguments``. - Transformer Engine ``TEGroupedMLP`` (fused MoE experts) now uses **per-expert** weight quantization (one ``amax`` per expert) instead of a single shared ``amax``, so ModelOpt checkpoints containing quantized ``TEGroupedMLP`` modules saved before 0.47 are **not compatible** with 0.47. Re-run PTQ to regenerate compatible checkpoints. **Deprecations** diff --git a/examples/gpt-oss/README.md b/examples/gpt-oss/README.md index 94e9881b3bd..be3dea6b558 100644 --- a/examples/gpt-oss/README.md +++ b/examples/gpt-oss/README.md @@ -57,11 +57,12 @@ If you are training Huggingface models with trainer classes from Huggingface suc A real end-to-end example for this is in `sft.py` in this folder. To perform QAT with full parameter SFT on GPT-OSS 20B model, run: ```sh -# Other supported quantization configs include NVFP4_MLP_WEIGHT_ONLY_CFG, NVFP4_MLP_ONLY_CFG etc. +# Other supported quantization recipes include general/ptq/nvfp4_mlp_weight_only, or +# general/ptq/nvfp4_mlp_only-kv_fp8 (also quantizes activations and the KV cache to FP8, which needs calibration). # [Optional] For faster FlashAttention3, add '--attn_implementation kernels-community/vllm-flash-attn3' accelerate launch --config_file configs/zero3.yaml sft.py \ --config configs/sft_full.yaml --model_name_or_path openai/gpt-oss-20b \ - --quant_cfg MXFP4_MLP_WEIGHT_ONLY_CFG \ + --recipe general/ptq/mxfp4_mlp_weight_only \ --output_dir gpt-oss-20b-qat ``` @@ -89,7 +90,7 @@ accelerate launch --config_file configs/zero3.yaml sft.py \ # Step 2: Perform QAT on the high precision SFT checkpoint accelerate launch --config_file configs/zero3.yaml sft.py \ --config configs/sft_full.yaml --model_name_or_path gpt-oss-20b-sft \ - --quant_cfg MXFP4_MLP_WEIGHT_ONLY_CFG \ + --recipe general/ptq/mxfp4_mlp_weight_only \ --output_dir gpt-oss-20b-qat \ ``` @@ -160,7 +161,7 @@ Here is how to run LoRA QAT for GPT OSS 120B model: ```bash python sft.py --config configs/sft_lora.yaml \ --model_name_or_path openai/gpt-oss-120b \ - --quant_cfg MXFP4_MLP_WEIGHT_ONLY_CFG \ + --recipe general/ptq/mxfp4_mlp_weight_only \ --output_dir gpt-oss-120b-lora-qat ``` diff --git a/examples/gpt-oss/configs/sft_full.yaml b/examples/gpt-oss/configs/sft_full.yaml index c3ba873be28..34732956a0b 100644 --- a/examples/gpt-oss/configs/sft_full.yaml +++ b/examples/gpt-oss/configs/sft_full.yaml @@ -30,6 +30,6 @@ eval_steps: 8 dataset_test_split: test # ModelOpt Quantization Parameters -quant_cfg: # Examples: MXFP4_MLP_WEIGHT_ONLY_CFG, NVFP4_MLP_WEIGHT_ONLY_CFG, NVFP4_MLP_ONLY_CFG - # For the full list of supported configs, do: mtq.config.choices +recipe: # Examples: general/ptq/mxfp4_mlp_weight_only, general/ptq/nvfp4_mlp_weight_only + # For the full list of built-in recipes, see modelopt_recipes/general/ptq/ calib_size: 128 diff --git a/examples/gpt-oss/configs/sft_lora.yaml b/examples/gpt-oss/configs/sft_lora.yaml index 4f35c36182b..9d298ba9a15 100644 --- a/examples/gpt-oss/configs/sft_lora.yaml +++ b/examples/gpt-oss/configs/sft_lora.yaml @@ -35,6 +35,6 @@ eval_steps: 8 dataset_test_split: test # ModelOpt Quantization Parameters -quant_cfg: # Examples: MXFP4_MLP_WEIGHT_ONLY_CFG, NVFP4_MLP_WEIGHT_ONLY_CFG, NVFP4_MLP_ONLY_CFG - # For the full list of supported configs, do: mtq.config.choices +recipe: # Examples: general/ptq/mxfp4_mlp_weight_only, general/ptq/nvfp4_mlp_weight_only + # For the full list of built-in recipes, see modelopt_recipes/general/ptq/ calib_size: 128 diff --git a/examples/gpt-oss/sft.py b/examples/gpt-oss/sft.py index 494d89f72df..991b793c07c 100644 --- a/examples/gpt-oss/sft.py +++ b/examples/gpt-oss/sft.py @@ -37,7 +37,7 @@ --packing true packing_strategy wrapped \ --run_name 20b-full-qat \ --attn_implementation kernels-community/vllm-flash-attn3 - --quant_cfg MXFP4_MLP_WEIGHT_ONLY_CFG + --recipe general/ptq/mxfp4_mlp_weight_only """ from transformers import AutoModelForCausalLM, AutoTokenizer, Mxfp4Config diff --git a/examples/hf_ptq/README.md b/examples/hf_ptq/README.md index 72b2e2348ab..fea69221825 100755 --- a/examples/hf_ptq/README.md +++ b/examples/hf_ptq/README.md @@ -97,7 +97,7 @@ Please reference our [framework scripts](#framework-scripts) and our [docs](http ### Hugging Face Supported Models -| Model | fp8 | int8_sq | int4_awq | w4a8_awq1 | nvfp45 | +| Model | fp8 | int8_smoothquant | int4_awq | w4a8_awq_beta1 | nvfp45 | | :---: | :---: | :---: | :---: | :---: | :---: | | LLAMA 3.x | ✅ | ❌ | ✅ | ✅3 | ✅ | | LLAMA 4 6 | ✅ | ❌ | ❌ | ❌ | ✅ | @@ -125,7 +125,7 @@ Please reference our [framework scripts](#framework-scripts) and our [docs](http > *This is a subset of the models supported. For the full list please check the [TensorRT-LLM support matrix](https://nvidia.github.io/TensorRT-LLM/reference/precision.html#support-matrix)* -> *1.The w4a8_awq is an experimental quantization scheme that may result in a higher accuracy penalty.* \ +> *1.The w4a8_awq_beta is an experimental quantization scheme that may result in a higher accuracy penalty.* \ > *2.For some models, there is only support for exporting quantized checkpoints.* \ > *3.W4A8_AWQ is only available on some models but not all* \ > *4.For some models, KV cache quantization may result in a higher accuracy penalty.* \ @@ -136,7 +136,7 @@ Please reference our [framework scripts](#framework-scripts) and our [docs](http > *9.Running Whisper model with transformers>=5.0 requires [torchcodec](https://github.com/meta-pytorch/torchcodec?tab=readme-ov-file#installing-cuda-enabled-torchcodec) and other system packages (e.g. ffmpeg).* \ > *10.GPT-OSS ships with native MXFP4 weights; NVFP4 export is produced via the closed-form `--cast_mxfp4_to_nvfp4` cast (see [MXFP4 → NVFP4 cast](#mxfp4--nvfp4-cast-for-gpt-oss)).* \ > *11.Vision-language model (VLM): only the language model is quantized while the vision encoder is kept in high precision. Pass `--vlm` to the shell script (see [VLM quantization](#vlm-quantization)).* \ -> *12.For VLMs, `int8_sq` only supports TensorRT-LLM checkpoint export and is not compatible with the TensorRT-LLM torch backend.* \ +> *12.For VLMs, `int8_smoothquant` only supports TensorRT-LLM checkpoint export and is not compatible with the TensorRT-LLM torch backend.* \ > *13.Nemotron VL automatically calibrates with image-text pairs; see [VLM calibration with image-text pairs](#vlm-calibration-with-image-text-pairs-eg-nemotron-vl).* > *The accuracy loss after PTQ may vary depending on the actual model and the quantization method. Different models may have different accuracy loss and usually the accuracy loss is more significant when the base model is small. If the accuracy after PTQ is not meeting the requirement, please try either modifying [hf_ptq.py](./hf_ptq.py) and disabling the KV cache quantization or using the [QAT](./../llm_qat/README.md) instead. For NVFP4 quantization specifically, we recommend `nvfp4_mlp_only`, `nvfp4_experts_only`, or `nvfp4_omlp_only` to achieve higher accuracy by restricting quantization to the MLP/expert layers (and optionally the `o_proj` layer) while keeping the attention QKV projections unquantized.* @@ -162,7 +162,7 @@ export HF_PATH= --tp [1|2|4|8] ``` -Supported `QFORMAT` values: `fp8`, `fp8_pc_pt`, `fp8_pb_wo`, `int8`, `int8_sq`, `int8_wo`, `int4_awq`, `w4a8_awq`, `nvfp4`, `nvfp4_awq`, `nvfp4_mse`, `nvfp4_mlp_only`, `nvfp4_experts_only`, `nvfp4_omlp_only`, `nvfp4_svdquant`, `nvfp4_local_hessian`, `w4a8_nvfp4_fp8`, `w4a8_mxfp4_fp8`, `mxfp8`. +`QFORMAT` accepts any preset basename under [`modelopt_recipes/configs/ptq/presets/model/`](../../modelopt_recipes/configs/ptq/presets/model) — e.g. `fp8`, `fp8_per_channel_per_token`, `fp8_2d_blockwise_weight_only`, `int8`, `int8_smoothquant`, `int8_weight_only`, `int4_awq`, `w4a8_awq_beta`, `nvfp4`, `nvfp4_awq_lite`, `nvfp4_w4a4_weight_mse_fp8_sweep`, `nvfp4_mlp_only`, `nvfp4_experts_only`, `nvfp4_omlp_only`, `nvfp4_svdquant`, `nvfp4_w4a4_weight_local_hessian`, `w4a8_nvfp4_fp8`, `w4a8_mxfp4_fp8`, `mxfp8`. > *By default `trust_remote_code` is set to false. Please turn it on if model calibration and eval requires it using `--trust_remote_code`.* @@ -265,10 +265,8 @@ TensorRT-LLM multimodal quickstart as the deploy smoke test instead of the text- scripts/huggingface_example.sh --model --quant fp8 --vlm ``` -Supported `--quant` values for VLMs are `fp8`, `nvfp4`, `int8_sq`, `int4_awq`, and `w4a8_awq` (see -the `(VLM)` rows in the [Support Matrix](#hugging-face-supported-models)). - -> *This consolidates the former `examples/vlm_ptq` example, which now forwards here.* +Supported `--quant` values for VLMs are `fp8`, `nvfp4`, `int8_smoothquant`, `int4_awq`, and +`w4a8_awq_beta` (see the `(VLM)` rows in the [Support Matrix](#hugging-face-supported-models)). #### VLM calibration with image-text pairs (e.g., Nemotron VL) @@ -359,17 +357,6 @@ search-disabled layers, and cost-excluded layers — see recipes (carrying architecture-specific disabled layers — e.g. VL vision towers) live under `modelopt_recipes/huggingface//auto_quantize/`. -> *Migration: prefer an AutoQuantize `--recipe`. The `--auto_quantize_bits`, `--auto_quantize_method`, -> `--auto_quantize_score_size`, `--auto_quantize_cost_model`, and `--auto_quantize_active_moe_expert_ratio` -> CLI flags are **deprecated but still work** — they are converted into an `AutoQuantizeConfig` on the fly -> (with a `DeprecationWarning`) and will be removed in a future release. They map to recipe fields: -> `--auto_quantize_bits` → `constraints.effective_bits`, `--auto_quantize_method` → `auto_quantize_method`, -> `--auto_quantize_score_size` → `score_size`, `--auto_quantize_cost_model` → `constraints.cost_model`, -> `--auto_quantize_active_moe_expert_ratio` → `constraints.cost.active_moe_expert_ratio`, and the -> `--qformat fp8,nvfp4` candidate list → `candidate_formats`. When converted, the shared base -> `disabled_layers` and `cost_excluded_layers` patterns are appended automatically. `--auto_quantize_checkpoint` -> is unchanged. Start from a shipped recipe under `modelopt_recipes/general/auto_quantize/`.* - [Script](./scripts/huggingface_example.sh) ```bash diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index 34fa5a975aa..965dffdffb8 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -1110,37 +1110,30 @@ def copy_custom_model_files(source_path: str, export_path: str, trust_remote_cod print("No custom model files found to copy") -def _layerwise_checkpoint_dir_location(algorithm) -> tuple[str, str] | None: - """Return ``("flat"/"nested", checkpoint_dir)`` for the layerwise checkpoint dir, or None.""" +def _layerwise_checkpoint_dir(algorithm) -> str | None: + """Return the nested ``layerwise.checkpoint_dir``, or None.""" if not isinstance(algorithm, dict): return None - flat = algorithm.get("layerwise_checkpoint_dir") - if flat is not None: - return "flat", flat nested = algorithm.get("layerwise") or {} - ckpt = nested.get("checkpoint_dir") if isinstance(nested, dict) else None - return ("nested", ckpt) if ckpt is not None else None + return nested.get("checkpoint_dir") if isinstance(nested, dict) else None def needs_checkpoint_path_update(quant_cfg: dict) -> bool: """Check if quant_cfg has a layerwise checkpoint_dir that should be auto-resolved to a unique subpath.""" - return _layerwise_checkpoint_dir_location(quant_cfg.get("algorithm")) is not None + return _layerwise_checkpoint_dir(quant_cfg.get("algorithm")) is not None def resolve_checkpoint_dir(quant_cfg: dict, model_path: str) -> tuple[dict, str]: """Append a unique ``_`` subdirectory to the layerwise checkpoint_dir. Allows a single recipe to be reused across models without checkpoint collisions. - Supports both the legacy flat ``layerwise_checkpoint_dir`` and the nested - ``layerwise.checkpoint_dir`` shape, writing back to whichever the user provided. Must only be called when :func:`needs_checkpoint_path_update` returns True. Returns ``(updated_quant_cfg, resolved_path)`` so the caller can log or reference the resolved path without re-deriving the dict shape. """ - location = _layerwise_checkpoint_dir_location(quant_cfg["algorithm"]) - assert location is not None # guaranteed by needs_checkpoint_path_update - shape, base_dir = location + base_dir = _layerwise_checkpoint_dir(quant_cfg["algorithm"]) + assert base_dir is not None # guaranteed by needs_checkpoint_path_update name = model_path.rstrip("/") if "/" in name and not os.path.isabs(name): @@ -1152,11 +1145,7 @@ def resolve_checkpoint_dir(quant_cfg: dict, model_path: str) -> tuple[dict, str] resolved = os.path.join(base_dir, f"{name}_{config_hash}") quant_cfg = copy.deepcopy(quant_cfg) - algo = quant_cfg["algorithm"] - if "layerwise_checkpoint_dir" in algo: - algo["layerwise_checkpoint_dir"] = resolved - if isinstance(algo.get("layerwise"), dict) and "checkpoint_dir" in algo["layerwise"]: - algo["layerwise"]["checkpoint_dir"] = resolved + quant_cfg["algorithm"]["layerwise"]["checkpoint_dir"] = resolved return quant_cfg, resolved diff --git a/examples/hf_ptq/hf_ptq.py b/examples/hf_ptq/hf_ptq.py index 0790f644308..d57d857824f 100755 --- a/examples/hf_ptq/hf_ptq.py +++ b/examples/hf_ptq/hf_ptq.py @@ -391,47 +391,6 @@ def _mtq_inputs_from_auto_quantize_config( } -def _auto_quantize_config_from_cli(args: argparse.Namespace): - """Convert the deprecated ``--auto_quantize_*`` flags into an AutoQuantizeConfig on the fly. - - Backward-compat shim: old CLI invocations are turned into the same config object the recipe - path consumes, so the rest of the flow is recipe-driven. Layer patterns come from the shared - base sets loaded once in modelopt.recipe.config (no model introspection, no new CLI flags): the - base disabled set, and the base cost-excluded set — the latter is appended unconditionally - because it is harmless on non-VL models (nothing matches → cost_weight 0 is a no-op) and correct - on VL models. - """ - from modelopt.recipe.config import ( - AUTOQUANT_BASE_COST_EXCLUDED_LAYERS, - AUTOQUANT_BASE_DISABLED_LAYERS, - AutoQuantizeConfig, - AutoQuantizeConstraints, - AutoQuantizeCost, - ) - from modelopt.torch.quantization.config import QuantizeConfig - - disabled_layers = list(AUTOQUANT_BASE_DISABLED_LAYERS) - cost_excluded_layers = list(AUTOQUANT_BASE_COST_EXCLUDED_LAYERS) - - cost = ( - AutoQuantizeCost(active_moe_expert_ratio=args.auto_quantize_active_moe_expert_ratio) - if args.auto_quantize_cost_model == "active_moe" - else None - ) - return AutoQuantizeConfig( - constraints=AutoQuantizeConstraints( - effective_bits=args.auto_quantize_bits, - cost_model=args.auto_quantize_cost_model, - cost=cost, - ), - candidate_formats=[QuantizeConfig(**QUANT_CFG_CHOICES[q]) for q in args.qformat.split(",")], - auto_quantize_method=args.auto_quantize_method, - score_size=args.auto_quantize_score_size, - disabled_layers=disabled_layers, - cost_excluded_layers=cost_excluded_layers, - ) - - def auto_quantize( args: argparse.Namespace, language_model: torch.nn.Module, @@ -625,15 +584,13 @@ def load_model(args: argparse.Namespace): is_nemotron_vl_model = is_nemotron_vl(full_model) - # Default to image-text calibration for VLM models. Skip for either AutoQuantize path (recipe or - # the deprecated --auto_quantize_bits CLI), whose text-only path does not support image-text - # calibration yet (auto_quantize() would raise); auto-enabling it here would make Nemotron-VL - # AutoQuantize fail unconditionally. + # Default to image-text calibration for VLM models. Skip for the AutoQuantize recipe path, whose + # text-only path does not support image-text calibration yet (auto_quantize() would raise); + # auto-enabling it here would make Nemotron-VL AutoQuantize fail unconditionally. if ( is_nemotron_vl_model and not args.calib_with_images and not _recipe_is_auto_quantize(args.recipe) - and args.auto_quantize_bits is None ): print("Nemotron VL model detected. Enabling image-text calibration by default.") args.calib_with_images = True @@ -686,11 +643,11 @@ def load_model(args: argparse.Namespace): : len(args.dataset) ] - # Plain PTQ quantizes only the extracted language model. Recipe and AutoQuantize paths - # (incl. the deprecated --auto_quantize_bits CLI) keep the outer CausalLM so recipes / - # search can see the Qwen3.5/3.6-MoE VLM lm_head; extracting here would leave modelopt - # state on the ancestors and make auto_quantize() fail with "multiple modelopt states". - if args.recipe is None and args.auto_quantize_bits is None: + # Plain PTQ quantizes only the extracted language model. The recipe path keeps the outer + # CausalLM so recipes / search can see the Qwen3.5/3.6-MoE VLM lm_head; extracting here + # would leave modelopt state on the ancestors and make auto_quantize() fail with + # "multiple modelopt states". + if args.recipe is None: extracted_lm, extracted_model_type = extract_and_prepare_language_model_from_vl( full_model ) @@ -864,7 +821,7 @@ def export_quantized( if ( model_type in ["t5", "bart", "whisper"] or args.sparsity_fmt != "dense" - or "int8_sq" in args.qformat + or "int8_smoothquant" in args.qformat ): if ( args.inference_tensor_parallel != 1 or args.inference_pipeline_parallel != 1 @@ -1156,19 +1113,10 @@ def quantize_main( f"from {args.recipe}" ) - # Resolve the AutoQuantizeConfig from either source: a recipe, or the deprecated - # --auto_quantize_* CLI flags converted on the fly. Everything downstream is recipe-driven. + # AutoQuantize is recipe-driven: everything downstream reads the resolved AutoQuantizeConfig. if isinstance(recipe, ModelOptAutoQuantizeRecipe): aq_config = recipe.auto_quantize fixed_quantize_config = recipe.quantize - elif args.recipe is None and args.auto_quantize_bits is not None: - warnings.warn( - "The --auto_quantize_* CLI flags are deprecated; use an AutoQuantize --recipe instead. " - "They are converted to an AutoQuantizeConfig on the fly for now.", - DeprecationWarning, - ) - aq_config = _auto_quantize_config_from_cli(args) - fixed_quantize_config = None else: aq_config = None fixed_quantize_config = None @@ -1207,7 +1155,9 @@ def _is_layerwise(obj): # Calibration/sparsification will actually take much more memory than regular inference # due to intermediate tensors for fake quantization. Setting sample_memory_usage_ratio # to 2 to avoid OOM for AWQ/SmoothQuant fake quantization as it will take more memory than inference. - sample_memory_usage_ratio = 2 if "awq" in args.qformat or "sq" in args.qformat else 1.1 + sample_memory_usage_ratio = ( + 2 if "awq" in args.qformat or "smoothquant" in args.qformat else 1.1 + ) # Whisper model expects mel-spectrogram input features of length 3000 # Whisper model needs input of shape (batch_size, num_mel_bins, 3000) # As the encoder of Whisper doesn't have embedding layer, input dtype has to be float @@ -1257,9 +1207,9 @@ def _is_layerwise(obj): ) if aq_config is not None: - # AutoQuantize (recipe or the deprecated --auto_quantize_* CLI, converted on the fly). For - # VL models the search walks the OUTER CausalLM (which carries lm_head and the LM-head - # forward path); architecture-specific exclusions come from aq_config.disabled_layers. + # AutoQuantize (recipe-driven). For VL models the search walks the OUTER CausalLM (which + # carries lm_head and the LM-head forward path); architecture-specific exclusions come + # from aq_config.disabled_layers. auto_quantize( args, full_model, @@ -1556,47 +1506,9 @@ def parse_args() -> argparse.Namespace: default=None, help=( "Path to checkpoint file for saving/restoring auto_quantize search state " - "(sensitivity scores, costs, etc.). Used with an AutoQuantize --recipe or the " - "deprecated --auto_quantize_bits CLI path." + "(sensitivity scores, costs, etc.). Used with an AutoQuantize --recipe." ), ) - # Deprecated AutoQuantize CLI flags: kept as a backward-compat shim that converts them into an - # AutoQuantizeConfig on the fly (see _auto_quantize_config_from_cli). Prefer --recipe. The old - # CLI also lives on the 0.45 branch. - parser.add_argument( - "--auto_quantize_bits", - type=float, - default=None, - help="[Deprecated: use an AutoQuantize --recipe] Effective-bits target; also enables the " - "AutoQuantize CLI path. Candidate formats are taken from --qformat (comma-separated).", - ) - parser.add_argument( - "--auto_quantize_method", - type=str, - default="gradient", - choices=["gradient", "kl_div"], - help="[Deprecated: use an AutoQuantize --recipe] Sensitivity scoring method.", - ) - parser.add_argument( - "--auto_quantize_score_size", - type=int, - default=128, - help="[Deprecated: use an AutoQuantize --recipe] Number of samples for sensitivity scoring.", - ) - parser.add_argument( - "--auto_quantize_cost_model", - type=str, - default="weight", - choices=["weight", "active_moe"], - help="[Deprecated: use an AutoQuantize --recipe] Cost model for the effective-bits search.", - ) - parser.add_argument( - "--auto_quantize_active_moe_expert_ratio", - type=float, - default=None, - help="[Deprecated: use an AutoQuantize --recipe] Routed-expert active ratio for the " - "'active_moe' cost model.", - ) parser.add_argument( "--moe_calib_experts_ratio", type=float, @@ -1677,10 +1589,10 @@ def parse_args() -> argparse.Namespace: # via init_quantized_weights(), so it cannot honor a --recipe (which is authoritative # for the quant layout in quantize_main). Reject the combination rather than silently # instrumenting a layout that diverges from the recipe. - if args.low_memory_mode and (args.recipe is not None or args.auto_quantize_bits is not None): + if args.low_memory_mode and args.recipe is not None: parser.error( - "--low_memory_mode does not support --recipe or AutoQuantize (--auto_quantize_bits); " - "the low-memory loader initializes quantizers from --qformat/--kv_cache_qformat." + "--low_memory_mode does not support --recipe; the low-memory loader initializes " + "quantizers from --qformat/--kv_cache_qformat." ) if args.use_fsdp2 and args.use_seq_device_map: warnings.warn("--use_seq_device_map is ignored when --use_fsdp2 is set.") diff --git a/examples/hf_ptq/scripts/huggingface_example.sh b/examples/hf_ptq/scripts/huggingface_example.sh index 44363cb012e..adc4cf22ca3 100755 --- a/examples/hf_ptq/scripts/huggingface_example.sh +++ b/examples/hf_ptq/scripts/huggingface_example.sh @@ -94,9 +94,9 @@ if [ "$LOW_MEMORY_MODE" = "true" ]; then PTQ_ARGS+=" --low_memory_mode " fi -# AutoQuantize runs via an AutoQuantize --recipe or the deprecated --auto_quantize_bits CLI path. -# Auto-generate a checkpoint path (to save/restore the search state) when the user didn't supply one. -if [ -z "$AUTO_QUANTIZE_CHECKPOINT" ] && { [[ "$RECIPE" == *auto_quantize* ]] || [ -n "$AUTO_QUANTIZE_BITS" ]; }; then +# AutoQuantize runs via an AutoQuantize --recipe. Auto-generate a checkpoint path (to save/restore +# the search state) when the user didn't supply one. +if [ -z "$AUTO_QUANTIZE_CHECKPOINT" ] && [[ "$RECIPE" == *auto_quantize* ]]; then AUTO_QUANTIZE_CHECKPOINT="${ROOT_SAVE_PATH}/auto_quantize_checkpoints/${MODEL_NAME}.pth" mkdir -p "$(dirname "$AUTO_QUANTIZE_CHECKPOINT")" echo "Auto-generated auto_quantize checkpoint path: $AUTO_QUANTIZE_CHECKPOINT" @@ -105,18 +105,6 @@ if [ -n "$AUTO_QUANTIZE_CHECKPOINT" ]; then PTQ_ARGS+=" --auto_quantize_checkpoint=$AUTO_QUANTIZE_CHECKPOINT " fi -# Deprecated AutoQuantize CLI flags: passed through to hf_ptq.py, which converts them into an -# AutoQuantizeConfig on the fly. Prefer an AutoQuantize --recipe. -if [ -n "$AUTO_QUANTIZE_BITS" ]; then - PTQ_ARGS+=" --auto_quantize_bits=$AUTO_QUANTIZE_BITS " - PTQ_ARGS+=" --auto_quantize_method=${AUTO_QUANTIZE_METHOD:-gradient} " - PTQ_ARGS+=" --auto_quantize_score_size=${AUTO_QUANTIZE_SCORE_SIZE:-128} " - PTQ_ARGS+=" --auto_quantize_cost_model=${AUTO_QUANTIZE_COST_MODEL:-weight} " - if [ -n "$AUTO_QUANTIZE_ACTIVE_MOE_EXPERT_RATIO" ]; then - PTQ_ARGS+=" --auto_quantize_active_moe_expert_ratio=$AUTO_QUANTIZE_ACTIVE_MOE_EXPERT_RATIO " - fi -fi - if [ -n "$CALIB_DATASET" ]; then PTQ_ARGS+=" --dataset=$CALIB_DATASET " fi diff --git a/examples/hf_ptq/scripts/parser.sh b/examples/hf_ptq/scripts/parser.sh index af54832dfd9..e3eb0b18b63 100644 --- a/examples/hf_ptq/scripts/parser.sh +++ b/examples/hf_ptq/scripts/parser.sh @@ -41,7 +41,7 @@ parse_options() { CALIB_WITH_IMAGES=false # Parse command-line options - ARGS=$(getopt -o "" -l "model:,quant:,recipe:,kv_cache_quant:,tp:,pp:,sparsity:,awq_block_size:,calib:,calib_batch_size:,input:,output:,batch:,tasks:,lm_eval_tasks:,lm_eval_limit:,simple_eval_tasks:,simple_eval_limit:,mmlu_limit:,trust_remote_code,use_seq_device_map,gpu_max_mem_percentage:,kv_cache_free_gpu_memory_fraction:,low_memory_mode,no-verbose,calib_dataset:,calib_seq:,auto_quantize_checkpoint:,auto_quantize_bits:,auto_quantize_method:,auto_quantize_score_size:,auto_quantize_cost_model:,auto_quantize_active_moe_expert_ratio:,moe_calib_experts_ratio:,cast_mxfp4_to_nvfp4,vlm,calib_with_images" -n "$0" -- "$@") + ARGS=$(getopt -o "" -l "model:,quant:,recipe:,kv_cache_quant:,tp:,pp:,sparsity:,awq_block_size:,calib:,calib_batch_size:,input:,output:,batch:,tasks:,lm_eval_tasks:,lm_eval_limit:,simple_eval_tasks:,simple_eval_limit:,mmlu_limit:,trust_remote_code,use_seq_device_map,gpu_max_mem_percentage:,kv_cache_free_gpu_memory_fraction:,low_memory_mode,no-verbose,calib_dataset:,calib_seq:,auto_quantize_checkpoint:,moe_calib_experts_ratio:,cast_mxfp4_to_nvfp4,vlm,calib_with_images" -n "$0" -- "$@") eval set -- "$ARGS" while true; do @@ -74,11 +74,6 @@ parse_options() { --calib_dataset ) CALIB_DATASET="$2"; shift 2;; --calib_seq ) CALIB_SEQ="$2"; shift 2;; --auto_quantize_checkpoint ) AUTO_QUANTIZE_CHECKPOINT="$2"; shift 2;; - --auto_quantize_bits ) AUTO_QUANTIZE_BITS="$2"; shift 2;; - --auto_quantize_method ) AUTO_QUANTIZE_METHOD="$2"; shift 2;; - --auto_quantize_score_size ) AUTO_QUANTIZE_SCORE_SIZE="$2"; shift 2;; - --auto_quantize_cost_model ) AUTO_QUANTIZE_COST_MODEL="$2"; shift 2;; - --auto_quantize_active_moe_expert_ratio ) AUTO_QUANTIZE_ACTIVE_MOE_EXPERT_RATIO="$2"; shift 2;; --moe_calib_experts_ratio ) MOE_CALIB_EXPERTS_RATIO="$2"; shift 2;; --cast_mxfp4_to_nvfp4 ) CAST_MXFP4_TO_NVFP4=true; shift;; --vlm ) VLM=true; shift;; @@ -182,11 +177,6 @@ parse_options() { echo "calib_dataset: $CALIB_DATASET" echo "calib_seq: $CALIB_SEQ" echo "auto_quantize_checkpoint: $AUTO_QUANTIZE_CHECKPOINT" - echo "auto_quantize_bits: $AUTO_QUANTIZE_BITS" - echo "auto_quantize_method: $AUTO_QUANTIZE_METHOD" - echo "auto_quantize_score_size: $AUTO_QUANTIZE_SCORE_SIZE" - echo "auto_quantize_cost_model: $AUTO_QUANTIZE_COST_MODEL" - echo "auto_quantize_active_moe_expert_ratio: $AUTO_QUANTIZE_ACTIVE_MOE_EXPERT_RATIO" echo "moe_calib_experts_ratio: $MOE_CALIB_EXPERTS_RATIO" echo "cast_mxfp4_to_nvfp4: $CAST_MXFP4_TO_NVFP4" echo "vlm: $VLM" diff --git a/examples/llm_ptq b/examples/llm_ptq deleted file mode 120000 index a314d334fb1..00000000000 --- a/examples/llm_ptq +++ /dev/null @@ -1 +0,0 @@ -hf_ptq \ No newline at end of file diff --git a/examples/llm_qat/ARGUMENTS.md b/examples/llm_qat/ARGUMENTS.md index 35f788318c9..0b244cadc93 100644 --- a/examples/llm_qat/ARGUMENTS.md +++ b/examples/llm_qat/ARGUMENTS.md @@ -49,8 +49,7 @@ | Argument | Type | Default | Description | |----------|------|---------|-------------| -| `--recipe` | `str` | `None` | Path to a quantization recipe YAML file (built-in or custom). Built-in recipes can be specified by relative path, e.g. 'general/ptq/nvfp4_default-kv_fp8'. Replaces the deprecated --quant_cfg flag. | -| `--quant_cfg` | `str` | `None` | Deprecated: pre-quantize the model with a separate quantization step instead. Specify the quantization format for PTQ/QAT by name (e.g. NVFP4_DEFAULT_CFG). | +| `--recipe` | `str` | `None` | Path to a quantization recipe YAML file (built-in or custom). Built-in recipes can be specified by relative path, e.g. 'general/ptq/nvfp4_default-kv_fp8'. | | `--calib_size` | `int` | `512` | Specify the calibration size for quantization. The calibration dataset is used to setup the quantization scale parameters for PTQ/QAT. | | `--compress` | `bool` | `False` | Whether to compress the model weights after quantization for QLoRA. This is useful for reducing the model size. | | `--calib_batch_size` | `int` | `1` | Batch size for calibration data during quantization. | diff --git a/examples/llm_qat/quantize.py b/examples/llm_qat/quantize.py index 87a092e38ee..5f12105b8d9 100644 --- a/examples/llm_qat/quantize.py +++ b/examples/llm_qat/quantize.py @@ -64,7 +64,7 @@ def quantize(): print_rank_0(f"Loading quantization recipe: {quant_args.recipe}") ptq_cfg = resolve_quant_cfg_from_args(quant_args) if ptq_cfg is None: - raise ValueError("--recipe or --quant_cfg is required for quantization.") + raise ValueError("--recipe is required for quantization.") # Load model and tokenizer print_rank_0(f"Loading model: {model_args.model_name_or_path}") diff --git a/examples/megatron_bridge/README.md b/examples/megatron_bridge/README.md index f247a0b6137..9b0eb160cd1 100644 --- a/examples/megatron_bridge/README.md +++ b/examples/megatron_bridge/README.md @@ -68,7 +68,7 @@ This section shows how to quantize a HuggingFace model using ModelOpt in the Meg 1. [quantize.py](quantize.py) applies post-training quantization (PTQ) with calibration and saves a **Megatron checkpoint** (with ModelOpt state). Tensor / pipeline / expert parallelism are all supported, and the checkpoint can be reloaded for further training (Quantization Aware Training / Quantization Aware Distillation). 2. [export_quantized_megatron_to_hf.py](export_quantized_megatron_to_hf.py) converts that Megatron checkpoint to a **HuggingFace (unified) checkpoint** that deploys directly with TensorRT-LLM, vLLM, or SGLang. -`quantize.py` supports the following formats via `--quant_cfg` (e.g. `fp8`, `nvfp4`, `int8_sq`, `int4_awq`, `w4a8_awq`, ...). You can also pass any full config name exposed by ModelOpt (e.g. `NVFP4_DEFAULT_CFG`) or a YAML `--recipe` (e.g. `general/ptq/nvfp4_default-kv_fp8`, authoritative for quant_cfg + algorithm + KV-cache). KV-cache quantization can be enabled on top via `--kv_cache_quant` (e.g. `fp8`, `nvfp4`). +`quantize.py` supports the following formats via `--quant_cfg` (e.g. `fp8`, `nvfp4`, `int8_smoothquant`, `int4_awq`, `w4a8_awq_beta`, ...). You can also pass any full config name exposed by ModelOpt (e.g. `NVFP4_DEFAULT_CFG`) or a YAML `--recipe` (e.g. `general/ptq/nvfp4_default-kv_fp8`, authoritative for quant_cfg + algorithm + KV-cache). KV-cache quantization can be enabled on top via `--kv_cache_quant` (e.g. `fp8`, `nvfp4`). **Step 1 — quantize** Qwen3-8B to NVFP4 on 2 GPUs (Tensor Parallelism = 2) using 1024 samples from default dataset (Mix of [`cnn_dailymail`](https://huggingface.co/datasets/abisee/cnn_dailymail) and [`nemotron-post-training-dataset-v2`](https://huggingface.co/datasets/nvidia/Nemotron-Post-Training-Dataset-v2)) for calibration (sequence length = 4096): diff --git a/examples/megatron_bridge/quantize.py b/examples/megatron_bridge/quantize.py index 1e4ee79f574..fedd7bf62d0 100644 --- a/examples/megatron_bridge/quantize.py +++ b/examples/megatron_bridge/quantize.py @@ -126,7 +126,7 @@ def get_args() -> argparse.Namespace: type=str, default=None, help=( - f"Quantization config. Preset names / short aliases: {', '.join(QUANT_CFG_CHOICES)}. " + f"Quantization config. Preset names: {', '.join(QUANT_CFG_CHOICES)}. " "You can also pass any full config name exposed by modelopt (e.g. FP8_DEFAULT_CFG). " "Ignored when --recipe is set." ), @@ -240,7 +240,7 @@ def get_quant_config(args: argparse.Namespace) -> dict: mtq_config = getattr(mtq, args.quant_cfg) else: raise ValueError( - f"Unsupported --quant_cfg '{args.quant_cfg}'. Choose a preset name / short alias " + f"Unsupported --quant_cfg '{args.quant_cfg}'. Choose a preset name " f"({', '.join(QUANT_CFG_CHOICES)}) or a full config name from {mtq.config.choices}." ) diff --git a/examples/vlm_ptq/.gitignore b/examples/vlm_ptq/.gitignore deleted file mode 100644 index 1ad143e5f03..00000000000 --- a/examples/vlm_ptq/.gitignore +++ /dev/null @@ -1 +0,0 @@ -saved_models_* diff --git a/examples/vlm_ptq/README.md b/examples/vlm_ptq/README.md deleted file mode 100644 index 1823a21e6c0..00000000000 --- a/examples/vlm_ptq/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# [Deprecated] Post-training quantization (PTQ) for Vision Language Models - -> **This example has been consolidated into [`examples/hf_ptq`](../hf_ptq/README.md) and is -> deprecated.** It will be removed in a future release. VLM PTQ now shares the same entry point -> (`hf_ptq.py`) and shell script as LLM PTQ. - -## Migration - -Use the `hf_ptq` script with the `--vlm` flag: - -```bash -cd examples/hf_ptq -scripts/huggingface_example.sh --model --quant [fp8|nvfp4|int8_sq|int4_awq|w4a8_awq] --vlm -``` - -The previous `examples/vlm_ptq/scripts/huggingface_example.sh` entry point still works: it now -prints a deprecation warning and forwards to the command above. - -## Where things moved - -| Topic | New location | -| :--- | :--- | -| Supported VLMs / support matrix | [hf_ptq/README.md#hugging-face-supported-models](../hf_ptq/README.md#hugging-face-supported-models) | -| VLM quantization workflow (`--vlm`) | [hf_ptq/README.md#vlm-quantization](../hf_ptq/README.md#vlm-quantization) | -| Image-text calibration (`--calib_with_images`) | [hf_ptq/README.md#vlm-calibration-with-image-text-pairs-eg-nemotron-vl](../hf_ptq/README.md#vlm-calibration-with-image-text-pairs-eg-nemotron-vl) | -| Megatron-Bridge VLM PTQ | [examples/megatron_bridge/](../megatron_bridge/README.md) | - -## Resources - -- 📖 [Documentation](https://nvidia.github.io/Model-Optimizer) -- 💡 [Release Notes](https://nvidia.github.io/Model-Optimizer/reference/0_changelog.html) diff --git a/examples/vlm_ptq/scripts/huggingface_example.sh b/examples/vlm_ptq/scripts/huggingface_example.sh deleted file mode 100755 index 946d7baec55..00000000000 --- a/examples/vlm_ptq/scripts/huggingface_example.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# DEPRECATED: examples/vlm_ptq has been consolidated into examples/hf_ptq. -# This shim forwards all arguments to the hf_ptq script with the --vlm flag so existing -# commands keep working. Please migrate to: -# -# cd examples/hf_ptq -# scripts/huggingface_example.sh --model --quant --vlm -# -# See examples/hf_ptq/README.md#vlm-quantization for details. - -set -e - -echo "WARNING: examples/vlm_ptq is deprecated and will be removed in a future release." >&2 -echo " Forwarding to examples/hf_ptq/scripts/huggingface_example.sh --vlm" >&2 -echo " See examples/hf_ptq/README.md#vlm-quantization" >&2 - -script_dir="$(dirname "$(readlink -f "$0")")" - -exec "$script_dir/../../hf_ptq/scripts/huggingface_example.sh" --vlm "$@" diff --git a/modelopt/recipe/config.py b/modelopt/recipe/config.py index a16cfe4401a..2ca8f4f462b 100644 --- a/modelopt/recipe/config.py +++ b/modelopt/recipe/config.py @@ -24,7 +24,6 @@ from pydantic import Field, field_validator, model_validator from modelopt.torch.opt.config import ModeloptBaseConfig, ModeloptField -from modelopt.torch.opt.config_loader import load_config from modelopt.torch.quantization.config import QuantizeConfig # noqa: TC001 from modelopt.torch.speculative.config import DFlashConfig, EagleConfig, MedusaConfig from modelopt.torch.speculative.plugins.hf_training_args import DataArguments as SpecDataArgs @@ -130,24 +129,6 @@ class ModelOptPTQRecipe(ModelOptRecipeBase): LayerPatternList = list[str] -def _load_layer_pattern_list(config_path: str) -> list[str]: - """Load a ``list[str]`` layer-pattern unit (e.g. AutoQuantize base disabled/cost-excluded). - - Relies on the unit's ``modelopt-schema: ...LayerPatternList`` comment (like - _load_quantizer_cfg_dict_list) rather than an explicit ``list[str]`` schema_type. - """ - return list(load_config(config_path)) - - -# Base AutoQuantize layer-pattern sets, loaded once (used by the deprecated --auto_quantize_* CLI shim). -AUTOQUANT_BASE_DISABLED_LAYERS: list[str] = _load_layer_pattern_list( - "configs/auto_quantize/units/base_disabled_layers" -) -AUTOQUANT_BASE_COST_EXCLUDED_LAYERS: list[str] = _load_layer_pattern_list( - "configs/auto_quantize/units/base_cost_excluded_layers" -) - - class AutoQuantizeCost(ModeloptBaseConfig): """Cost-model parameters (the ``cost`` sub-dict of ``mtq.auto_quantize`` constraints).""" diff --git a/modelopt/recipe/presets.py b/modelopt/recipe/presets.py index bb4183c790a..90f36b8f8d7 100644 --- a/modelopt/recipe/presets.py +++ b/modelopt/recipe/presets.py @@ -32,7 +32,6 @@ ``mtq.*_CFG`` module constants — themselves eagerly-loaded shared dicts — are used). """ -from collections.abc import Mapping from typing import Any from modelopt.torch.opt.config_loader import BUILTIN_CONFIG_ROOT, load_config @@ -43,7 +42,6 @@ "KV_QUANT_CFG_CHOICES", "KV_QUANT_PRESET_DIR", "MODEL_QUANT_PRESET_DIR", - "QFORMAT_ALIASES", "QUANT_CFG_CHOICES", "load_quant_cfg_choices", ] @@ -61,73 +59,38 @@ # the scripts outside the discovered presets; guarded below against a ``none.yaml`` clash. KV_CACHE_NONE = "none" -# Backward-compat short names → canonical preset basename. These aliases predate the -# YAML-driven discovery and remain accepted so existing scripts/docs keep working. -# -# DO NOT add new entries here. New quantization formats must be exposed via their YAML -# basename under ``modelopt_recipes/configs/ptq/presets/model/`` — the directory listing -# is the canonical CLI vocabulary. This table exists solely to keep pre-existing short -# names working through deprecation and should only ever shrink. -QFORMAT_ALIASES: dict[str, str] = { - "int8_sq": "int8_smoothquant", - "int8_wo": "int8_weight_only", - "w4a8_awq": "w4a8_awq_beta", - "nvfp4_awq": "nvfp4_awq_lite", - "nvfp4_mse": "nvfp4_w4a4_weight_mse_fp8_sweep", - "nvfp4_local_hessian": "nvfp4_w4a4_weight_local_hessian", - "fp8_pb_wo": "fp8_2d_blockwise_weight_only", - "fp8_pc_pt": "fp8_per_channel_per_token", -} - - -def load_quant_cfg_choices( - subdir: str, aliases: Mapping[str, str] | None = None -) -> dict[str, dict[str, Any]]: + +def load_quant_cfg_choices(subdir: str) -> dict[str, dict[str, Any]]: """Build a ``{qformat_name: quant_cfg_dict}`` mapping from preset YAMLs. Every ``*.yaml`` under ``modelopt_recipes//`` is loaded and keyed by its - basename — the directory listing is the CLI vocabulary. ``aliases`` adds extra - short names pointing at canonical basenames; a stale alias raises here (at load - time) rather than failing silently at lookup time. + basename — the directory listing is the CLI vocabulary. Args: subdir: Preset directory relative to ``modelopt_recipes/`` (e.g. :data:`MODEL_QUANT_PRESET_DIR`). - aliases: Optional ``short_name -> canonical_basename`` deprecation map. Returns: - Mapping from format name (preset basename or alias) to the loaded - ``QuantizeConfig`` dict. Configs are loaded eagerly; callers that mutate a - returned config must deepcopy it first. + Mapping from preset basename to the loaded ``QuantizeConfig`` dict. Configs are + loaded eagerly; callers that mutate a returned config must deepcopy it first. """ - aliases = aliases or {} basenames = sorted( entry.name.rsplit(".", 1)[0] for entry in BUILTIN_CONFIG_ROOT.joinpath(subdir).iterdir() if entry.name.endswith((".yaml", ".yml")) ) - choices: dict[str, dict[str, Any]] = { + return { name: load_config(f"{subdir}/{name}", schema_type=QuantizeConfig).model_dump( exclude_unset=True ) for name in basenames } - for alias, target in sorted(aliases.items()): - if target not in choices: - raise ValueError( - f"Alias {alias!r} points at preset {target!r} which is not present " - f"under modelopt_recipes/{subdir}/." - ) - choices[alias] = choices[target] - return choices - - -QUANT_CFG_CHOICES: dict[str, dict[str, Any]] = load_quant_cfg_choices( - MODEL_QUANT_PRESET_DIR, QFORMAT_ALIASES -) + + +QUANT_CFG_CHOICES: dict[str, dict[str, Any]] = load_quant_cfg_choices(MODEL_QUANT_PRESET_DIR) KV_QUANT_CFG_CHOICES: dict[str, dict[str, Any]] = load_quant_cfg_choices(KV_QUANT_PRESET_DIR) -# Guard against a future ``none.yaml`` (or alias) colliding with the disable sentinel: +# Guard against a future ``none.yaml`` colliding with the disable sentinel: # the runtime branch on ``!= KV_CACHE_NONE`` would otherwise become ambiguous. assert KV_CACHE_NONE not in KV_QUANT_CFG_CHOICES, ( f"KV_CACHE_NONE sentinel {KV_CACHE_NONE!r} collides with a KV preset; rename the preset." diff --git a/modelopt/torch/quantization/backends/nvfp4_gemm.py b/modelopt/torch/quantization/backends/nvfp4_gemm.py index b62834c8935..bc6d8b8a86e 100644 --- a/modelopt/torch/quantization/backends/nvfp4_gemm.py +++ b/modelopt/torch/quantization/backends/nvfp4_gemm.py @@ -192,7 +192,8 @@ def apply(cls, *args, **kwargs): def _nvfp4_availability_check(module, input, args, kwargs): """Comprehensive check for FP4 GEMM availability.""" - # NOTE: Having the import at the top causes mpirun commands inside pytest (vlm_ptq) to fail without any error + # NOTE: Having the import at the top causes mpirun commands inside pytest (hf_ptq VLM + # tests) to fail without any error try: import tensorrt_llm # noqa: F401 except ImportError: diff --git a/modelopt/torch/quantization/config.py b/modelopt/torch/quantization/config.py index 88927fc7895..460a914f9ef 100644 --- a/modelopt/torch/quantization/config.py +++ b/modelopt/torch/quantization/config.py @@ -155,14 +155,7 @@ from collections.abc import Mapping, Sequence from typing import Any, ClassVar, Literal, TypeAlias -from pydantic import ( - AliasChoices, - Field, - ValidationInfo, - field_serializer, - field_validator, - model_validator, -) +from pydantic import Field, ValidationInfo, field_serializer, field_validator, model_validator from modelopt.torch.opt.config import ModeloptBaseConfig, ModeloptField from modelopt.torch.opt.config_loader import load_config @@ -774,15 +767,7 @@ class LayerwiseConfig(ModeloptBaseConfig): def _coerce_layerwise_input(value): - """Normalize a raw ``layerwise`` value to a dict; warn on deprecated bool.""" - if isinstance(value, bool): - warnings.warn( - "Passing the layerwise field as a bool is deprecated; use a dict, " - "e.g. `{'enable': True}`.", - DeprecationWarning, - stacklevel=2, - ) - return {"enable": value} + """Normalize a raw ``layerwise`` value to a dict.""" if value is None: return {} if isinstance(value, LayerwiseConfig): @@ -822,50 +807,17 @@ class QuantizeAlgorithmConfig(ModeloptBaseConfig): layerwise: LayerwiseConfig = Field( default_factory=LayerwiseConfig, - validation_alias=AliasChoices("layerwise", "use_sequential"), title="Layerwise calibration configuration.", description=( "Nested config controlling layer-by-layer calibration. Pass a dict, " - "e.g. ``{'enable': True, 'checkpoint_dir': '/path'}``. Bool input is " - "accepted for backward compatibility but deprecated." + "e.g. ``{'enable': True, 'checkpoint_dir': '/path'}``." ), ) - @model_validator(mode="before") - @classmethod - def _migrate_layerwise_checkpoint_dir(cls, data): - """Merge the legacy flat ``layerwise_checkpoint_dir`` key into ``layerwise``. - - Raises if both the flat key and a nested ``checkpoint_dir`` are set with conflicting values. - """ - if not isinstance(data, dict) or "layerwise_checkpoint_dir" not in data: - return data - warnings.warn( - "Passing `layerwise_checkpoint_dir` at the top level is deprecated; " - "nest it under `layerwise.checkpoint_dir` instead.", - DeprecationWarning, - stacklevel=2, - ) - data = dict(data) - flat_dir = data.pop("layerwise_checkpoint_dir") - # Resolve the legacy ``use_sequential`` alias before writing ``layerwise``, - # otherwise the alias value is silently dropped when AliasChoices picks the - # newly-written ``layerwise`` key over ``use_sequential``. - raw_layerwise = data.pop("layerwise", data.pop("use_sequential", None)) - layerwise = _coerce_layerwise_input(raw_layerwise) - existing = layerwise.get("checkpoint_dir") - if existing is not None and existing != flat_dir: - raise ValueError( - f"Conflicting checkpoint_dir: layerwise_checkpoint_dir={flat_dir!r} " - f"differs from layerwise.checkpoint_dir={existing!r}. Set only one." - ) - data["layerwise"] = {**layerwise, "checkpoint_dir": flat_dir} - return data - @field_validator("layerwise", mode="before") @classmethod def _coerce_layerwise(cls, value): - """Coerce ``layerwise=bool/None`` to dict form; also handles the alias path.""" + """Coerce ``layerwise=None``/``LayerwiseConfig`` to dict form.""" return _coerce_layerwise_input(value) @model_validator(mode="after") diff --git a/modelopt/torch/quantization/plugins/transformers_trainer.py b/modelopt/torch/quantization/plugins/transformers_trainer.py index 981f3d990d4..b3d5619beb2 100644 --- a/modelopt/torch/quantization/plugins/transformers_trainer.py +++ b/modelopt/torch/quantization/plugins/transformers_trainer.py @@ -19,7 +19,6 @@ import gc import os import types -import warnings from dataclasses import field import torch @@ -53,16 +52,7 @@ class QuantizationArguments(ModelOptHFArguments): "help": ( "Path to a quantization recipe YAML file (built-in or custom). " "Built-in recipes can be specified by relative path, e.g. " - "'general/ptq/nvfp4_default-kv_fp8'. Replaces the deprecated --quant_cfg flag." - ), - }, - ) - quant_cfg: str | None = field( - default=None, - metadata={ - "help": ( - "Deprecated: pre-quantize the model with a separate quantization step instead. " - "Specify the quantization format for PTQ/QAT by name (e.g. NVFP4_DEFAULT_CFG)." + "'general/ptq/nvfp4_default-kv_fp8'." ), }, ) @@ -86,41 +76,18 @@ class QuantizationArguments(ModelOptHFArguments): ) -# Backwards-compat alias for the pre-refactor public name; remove in a future release. -QuantizationArgumentsWithConfig = QuantizationArguments - - -def resolve_quant_cfg_from_args( - quant_args: QuantizationArguments | None, - *, - warn_on_quant_cfg: bool = False, -): - """Resolve a ModelOpt quantization config from recipe or legacy quant_cfg arguments.""" - if quant_args is None: +def resolve_quant_cfg_from_args(quant_args: QuantizationArguments | None): + """Resolve a ModelOpt quantization config from the recipe argument.""" + recipe_path = getattr(quant_args, "recipe", None) if quant_args is not None else None + if not recipe_path: return None - recipe_path = getattr(quant_args, "recipe", None) - if recipe_path: - from modelopt.recipe import ModelOptPTQRecipe, load_recipe - - recipe = load_recipe(recipe_path) - if not isinstance(recipe, ModelOptPTQRecipe): - raise ValueError( - f"Expected PTQ recipe, but got {type(recipe).__name__} from {recipe_path}" - ) - return recipe.quantize + from modelopt.recipe import ModelOptPTQRecipe, load_recipe - quant_cfg = getattr(quant_args, "quant_cfg", None) - if quant_cfg is None: - return None - if warn_on_quant_cfg: - warnings.warn( - "In-trainer quantization via quant_args is deprecated and will be removed in a " - "future release. Pre-quantize your model with a separate quantization step instead.", - DeprecationWarning, - stacklevel=3, - ) - return getattr(mtq, quant_cfg) if isinstance(quant_cfg, str) else quant_cfg + recipe = load_recipe(recipe_path) + if not isinstance(recipe, ModelOptPTQRecipe): + raise ValueError(f"Expected PTQ recipe, but got {type(recipe).__name__} from {recipe_path}") + return recipe.quantize def _patch_fsdp2_post_backward(): @@ -201,7 +168,7 @@ def __init__( super().__init__(*args, **kwargs) self.quant_args = quant_args - self.quant_cfg = resolve_quant_cfg_from_args(quant_args, warn_on_quant_cfg=True) + self.quant_cfg = resolve_quant_cfg_from_args(quant_args) # Add lora adapter before quantizing the model if getattr(self.args, "lora_config", None) is not None and not hasattr( diff --git a/modelopt/torch/utils/dataset_utils.py b/modelopt/torch/utils/dataset_utils.py index 789308a0b5b..280e881f016 100644 --- a/modelopt/torch/utils/dataset_utils.py +++ b/modelopt/torch/utils/dataset_utils.py @@ -1260,7 +1260,7 @@ def model_type_is_enc_dec(model): # `model.generate` to run the full denoising loop. # # Note: this list intentionally diverges from ``is_enc_dec`` in - # ``examples/llm_ptq/example_utils.py`` (which keys by ``model_type`` + # ``examples/hf_ptq/example_utils.py`` (which keys by ``model_type`` # string and is used for preview-decode slicing). DiffusionGemma is # included here so calibration uses ``.generate()`` end-to-end, but # deliberately excluded there so the preview decode treats its diff --git a/modelopt_recipes/configs/auto_quantize/units/base_cost_excluded_layers.yaml b/modelopt_recipes/configs/auto_quantize/units/base_cost_excluded_layers.yaml index 15437e25fbb..25ce5161d4a 100644 --- a/modelopt_recipes/configs/auto_quantize/units/base_cost_excluded_layers.yaml +++ b/modelopt_recipes/configs/auto_quantize/units/base_cost_excluded_layers.yaml @@ -13,11 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Base cost-excluded layer patterns for AutoQuantize (spliced into a recipe's cost_excluded_layers -# via ``$import``, and appended by the deprecated-CLI shim). These are VL patterns; appending them -# unconditionally is safe: on a non-VL model nothing matches, so cost_weight 0 is a no-op, while on -# a VL model it keeps the vision tower / MTP out of the bit-budget denominator. This avoids the old -# model-introspection path while preserving VL cost behavior. +# Base cost-excluded layer patterns for AutoQuantize, spliced into a recipe's cost_excluded_layers +# via ``$import``. These are VL patterns; including them unconditionally is safe: on a non-VL model +# nothing matches, so cost_weight 0 is a no-op, while on a VL model it keeps the vision tower / MTP +# out of the bit-budget denominator. # modelopt-schema: modelopt.recipe.config.LayerPatternList - "*visual*" diff --git a/modelopt_recipes/general/auto_quantize/nvfp4_fp8_at_5p4bits.yaml b/modelopt_recipes/general/auto_quantize/nvfp4_fp8_at_5p4bits.yaml index 7f4f337100d..f1b5c22251b 100644 --- a/modelopt_recipes/general/auto_quantize/nvfp4_fp8_at_5p4bits.yaml +++ b/modelopt_recipes/general/auto_quantize/nvfp4_fp8_at_5p4bits.yaml @@ -18,6 +18,7 @@ # modelopt-schema: modelopt.recipe.config.ModelOptAutoQuantizeRecipe imports: base_disabled_layers: configs/auto_quantize/units/base_disabled_layers + base_cost_excluded_layers: configs/auto_quantize/units/base_cost_excluded_layers nvfp4: configs/ptq/presets/model/nvfp4 fp8: configs/ptq/presets/model/fp8 @@ -40,3 +41,8 @@ auto_quantize: # models use a recipe under huggingface//auto_quantize/ that appends to this set. disabled_layers: - $import: base_disabled_layers + + # Base (model-agnostic) cost-excluded layers, spliced from the shared unit: VL vision + # towers / MTP are kept out of the effective-bits denominator. No-op on non-VL models. + cost_excluded_layers: + - $import: base_cost_excluded_layers diff --git a/modelopt_recipes/general/auto_quantize/nvfp4_fp8_kl_div_at_5p4bits.yaml b/modelopt_recipes/general/auto_quantize/nvfp4_fp8_kl_div_at_5p4bits.yaml index aac822ed0c0..51a6ba87f23 100644 --- a/modelopt_recipes/general/auto_quantize/nvfp4_fp8_kl_div_at_5p4bits.yaml +++ b/modelopt_recipes/general/auto_quantize/nvfp4_fp8_kl_div_at_5p4bits.yaml @@ -20,6 +20,7 @@ # modelopt-schema: modelopt.recipe.config.ModelOptAutoQuantizeRecipe imports: base_disabled_layers: configs/auto_quantize/units/base_disabled_layers + base_cost_excluded_layers: configs/auto_quantize/units/base_cost_excluded_layers nvfp4: configs/ptq/presets/model/nvfp4 fp8: configs/ptq/presets/model/fp8 @@ -42,3 +43,8 @@ auto_quantize: # models use a recipe under huggingface//auto_quantize/ that appends to this set. disabled_layers: - $import: base_disabled_layers + + # Base (model-agnostic) cost-excluded layers, spliced from the shared unit: VL vision + # towers / MTP are kept out of the effective-bits denominator. No-op on non-VL models. + cost_excluded_layers: + - $import: base_cost_excluded_layers diff --git a/modelopt_recipes/general/auto_quantize/nvfp4_mse_fp8_at_6p0bits.yaml b/modelopt_recipes/general/auto_quantize/nvfp4_mse_fp8_at_6p0bits.yaml index ac9546d049d..35b87ec8e98 100644 --- a/modelopt_recipes/general/auto_quantize/nvfp4_mse_fp8_at_6p0bits.yaml +++ b/modelopt_recipes/general/auto_quantize/nvfp4_mse_fp8_at_6p0bits.yaml @@ -18,6 +18,7 @@ # modelopt-schema: modelopt.recipe.config.ModelOptAutoQuantizeRecipe imports: base_disabled_layers: configs/auto_quantize/units/base_disabled_layers + base_cost_excluded_layers: configs/auto_quantize/units/base_cost_excluded_layers nvfp4_mse: configs/ptq/presets/model/nvfp4_w4a4_weight_mse_fp8_sweep fp8: configs/ptq/presets/model/fp8 @@ -40,3 +41,8 @@ auto_quantize: # models use a recipe under huggingface//auto_quantize/ that appends to this set. disabled_layers: - $import: base_disabled_layers + + # Base (model-agnostic) cost-excluded layers, spliced from the shared unit: VL vision + # towers / MTP are kept out of the effective-bits denominator. No-op on non-VL models. + cost_excluded_layers: + - $import: base_cost_excluded_layers diff --git a/modelopt_recipes/general/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml b/modelopt_recipes/general/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml index 56fc5fd789a..f7aa45a14a3 100644 --- a/modelopt_recipes/general/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml +++ b/modelopt_recipes/general/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml @@ -14,13 +14,12 @@ # limitations under the License. # AutoQuantize recipe: per-layer search over {FP8 (W8A8), NVFP4 weight-only (W4A16)} -# at 6.0 effective bits, active-MoE cost model. Recipe form of the CLI command: -# --qformat fp8,w4a16_nvfp4 --auto_quantize_bits 6.0 --auto_quantize_method gradient -# --auto_quantize_cost_model active_moe --auto_quantize_active_moe_expert_ratio 0.03125 +# at 6.0 effective bits, active-MoE cost model with a 0.03125 routed-expert active ratio. # modelopt-schema: modelopt.recipe.config.ModelOptAutoQuantizeRecipe imports: base_disabled_layers: configs/auto_quantize/units/base_disabled_layers + base_cost_excluded_layers: configs/auto_quantize/units/base_cost_excluded_layers fp8: configs/ptq/presets/model/fp8 w4a16_nvfp4: configs/ptq/presets/model/w4a16_nvfp4 @@ -51,3 +50,8 @@ auto_quantize: # huggingface//auto_quantize/ that extends this set. disabled_layers: - $import: base_disabled_layers + + # Base (model-agnostic) cost-excluded layers, spliced from the shared unit: VL vision + # towers / MTP are kept out of the effective-bits denominator. No-op on non-VL models. + cost_excluded_layers: + - $import: base_cost_excluded_layers diff --git a/modelopt_recipes/general/auto_quantize/w4a8_awq_beta_fp8_at_6p0bits.yaml b/modelopt_recipes/general/auto_quantize/w4a8_awq_beta_fp8_at_6p0bits.yaml index d0135f52a4d..69e0616f225 100644 --- a/modelopt_recipes/general/auto_quantize/w4a8_awq_beta_fp8_at_6p0bits.yaml +++ b/modelopt_recipes/general/auto_quantize/w4a8_awq_beta_fp8_at_6p0bits.yaml @@ -18,6 +18,7 @@ # modelopt-schema: modelopt.recipe.config.ModelOptAutoQuantizeRecipe imports: base_disabled_layers: configs/auto_quantize/units/base_disabled_layers + base_cost_excluded_layers: configs/auto_quantize/units/base_cost_excluded_layers w4a8_awq_beta: configs/ptq/presets/model/w4a8_awq_beta fp8: configs/ptq/presets/model/fp8 @@ -40,3 +41,8 @@ auto_quantize: # models use a recipe under huggingface//auto_quantize/ that appends to this set. disabled_layers: - $import: base_disabled_layers + + # Base (model-agnostic) cost-excluded layers, spliced from the shared unit: VL vision + # towers / MTP are kept out of the effective-bits denominator. No-op on non-VL models. + cost_excluded_layers: + - $import: base_cost_excluded_layers diff --git a/modelopt_recipes/general/ptq/mxfp4_mlp_weight_only.yaml b/modelopt_recipes/general/ptq/mxfp4_mlp_weight_only.yaml new file mode 100644 index 00000000000..c78ab4aa14f --- /dev/null +++ b/modelopt_recipes/general/ptq/mxfp4_mlp_weight_only.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# PTQ recipe wrapping the shipped mxfp4_mlp_weight_only preset (the config behind +# mtq.MXFP4_MLP_WEIGHT_ONLY_CFG), so the two cannot drift apart. + +imports: + preset: configs/ptq/presets/model/mxfp4_mlp_weight_only + +metadata: + recipe_type: ptq + description: >- + Applies dynamic MXFP4 to MLP/MoE weight quantizers only (weight-only, activations untouched); + no calibration forward pass is required. +quantize: + $import: preset diff --git a/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_cast.yaml b/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_cast.yaml index 01b497d7f1f..ea9fa1f60d9 100644 --- a/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_cast.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_cast.yaml @@ -30,9 +30,9 @@ quantize: algorithm: method: max # Max calibration is fast and does not typically need checkpointing. - # layerwise=false required for VLMs where the decoder layers are nested under + # layerwise.enable=false required for VLMs where the decoder layers are nested under # `model.language_model.layers` (layerwise_calibrate can't find them otherwise). - layerwise: false + layerwise: {enable: false} quant_cfg: - $import: base_disable_all - quantizer_name: '*.experts.*weight_quantizer' diff --git a/modelopt_recipes/general/ptq/nvfp4_experts_only_input_scale1-kv_fp8_cast.yaml b/modelopt_recipes/general/ptq/nvfp4_experts_only_input_scale1-kv_fp8_cast.yaml index 9d237eef31c..1e0e527b50a 100644 --- a/modelopt_recipes/general/ptq/nvfp4_experts_only_input_scale1-kv_fp8_cast.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_experts_only_input_scale1-kv_fp8_cast.yaml @@ -38,9 +38,9 @@ quantize: algorithm: method: max # Max calibration is fast and does not typically need checkpointing. - # layerwise=false required for VLMs where the decoder layers are nested under + # layerwise.enable=false required for VLMs where the decoder layers are nested under # `model.language_model.layers` (layerwise_calibrate can't find them otherwise). - layerwise: false + layerwise: {enable: false} # Every quantized activation quantizer here is either constant_amax (experts/block-sparse # inputs) or use_constant_amax (KV cast), so no data-driven activation stats are collected. # Skip the calibration forward and do weight-only calibration instead. diff --git a/modelopt_recipes/general/ptq/nvfp4_mlp_weight_only.yaml b/modelopt_recipes/general/ptq/nvfp4_mlp_weight_only.yaml new file mode 100644 index 00000000000..bfeddd7f163 --- /dev/null +++ b/modelopt_recipes/general/ptq/nvfp4_mlp_weight_only.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# PTQ recipe wrapping the shipped nvfp4_mlp_weight_only preset (the config behind +# mtq.NVFP4_MLP_WEIGHT_ONLY_CFG), so the two cannot drift apart. + +imports: + preset: configs/ptq/presets/model/nvfp4_mlp_weight_only + +metadata: + recipe_type: ptq + description: >- + Applies NVFP4 (block size 32) to MLP/MoE weight quantizers only (weight-only, activations + untouched); uses max calibration. +quantize: + $import: preset diff --git a/modelopt_recipes/huggingface/models/mistralai/Mistral-Medium-3.5-128B/ptq/nvfp4-max-calib.yaml b/modelopt_recipes/huggingface/models/mistralai/Mistral-Medium-3.5-128B/ptq/nvfp4-max-calib.yaml index 43798137f0f..3474315285b 100644 --- a/modelopt_recipes/huggingface/models/mistralai/Mistral-Medium-3.5-128B/ptq/nvfp4-max-calib.yaml +++ b/modelopt_recipes/huggingface/models/mistralai/Mistral-Medium-3.5-128B/ptq/nvfp4-max-calib.yaml @@ -28,7 +28,7 @@ metadata: quantize: algorithm: method: max - layerwise: false + layerwise: {enable: false} quant_cfg: - $import: base_disable_all # NVFP4 on MLP and expert weight matrices. diff --git a/modelopt_recipes/huggingface/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml b/modelopt_recipes/huggingface/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml index 9beadeafa7a..3279e96d995 100644 --- a/modelopt_recipes/huggingface/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml +++ b/modelopt_recipes/huggingface/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml @@ -39,6 +39,6 @@ quantize: algorithm: method: mse fp8_scale_sweep: true - layerwise: false + layerwise: {enable: false} quant_cfg: - $import: shared_quant_cfg diff --git a/modelopt_recipes/huggingface/qwen3_5_moe/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8.yaml b/modelopt_recipes/huggingface/qwen3_5_moe/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8.yaml index e9f45334176..9ef16f06e7e 100644 --- a/modelopt_recipes/huggingface/qwen3_5_moe/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8.yaml +++ b/modelopt_recipes/huggingface/qwen3_5_moe/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8.yaml @@ -37,7 +37,7 @@ quantize: algorithm: method: mse fp8_scale_sweep: true # MSE refines only static NVFP4 weights; FP8 layers stay max-calibrated - layerwise: false # Qwen3.5 decoder layers nested under model.language_model.layers + layerwise: {enable: false} # Qwen3.5 decoder layers nested under model.language_model.layers quant_cfg: - $import: base_disable_all # FP8 base on every weight/input quantizer: diff --git a/modelopt_recipes/huggingface/qwen3_5_moe/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml b/modelopt_recipes/huggingface/qwen3_5_moe/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml index 6b818d59310..44a213a3e15 100644 --- a/modelopt_recipes/huggingface/qwen3_5_moe/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml +++ b/modelopt_recipes/huggingface/qwen3_5_moe/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml @@ -39,6 +39,6 @@ quantize: algorithm: method: mse fp8_scale_sweep: true - layerwise: false + layerwise: {enable: false} quant_cfg: - $import: shared_quant_cfg diff --git a/modelopt_recipes/huggingface/qwen3_6_moe/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml b/modelopt_recipes/huggingface/qwen3_6_moe/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml index 201a70614eb..938664a1d7e 100644 --- a/modelopt_recipes/huggingface/qwen3_6_moe/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml +++ b/modelopt_recipes/huggingface/qwen3_6_moe/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml @@ -15,8 +15,7 @@ # Qwen3.6 MoE AutoQuantize: mixed FP8 + NVFP4 weight-only at 6.0 effective bits, active-MoE # cost model. Carries the architecture's disabled-layer patterns explicitly in the recipe -# (the set kept in sync with example_utils._get_auto_quantize_disabled_layers for a Qwen model; -# pinned by tests/examples/llm_ptq/test_hf_ptq_args.py). +# (pinned by tests/examples/hf_ptq/test_hf_ptq_args.py). # modelopt-schema: modelopt.recipe.config.ModelOptAutoQuantizeRecipe imports: diff --git a/modelopt_recipes/ptq.md b/modelopt_recipes/ptq.md index a1fb2d94c57..53c4ef99504 100644 --- a/modelopt_recipes/ptq.md +++ b/modelopt_recipes/ptq.md @@ -29,7 +29,7 @@ supported combinations. ### The shipped recipes
-All 22 general/ptq/ recipes (click to expand) +All 24 general/ptq/ recipes (click to expand) | Recipe | Model body | KV cache | Calibration | |--------|-----------|----------|-------------| @@ -55,6 +55,8 @@ supported combinations. | `nvfp4_weight_only-kv_fp16` | NVFP4 W4A16, weights only | none (BF16/FP16) | max | | `nvfp4_weight_only-kv_fp8_cast` | NVFP4 W4A16, weights only | FP8 (constant amax) | max | | `int4_blockwise_weight_only` | INT4 W4A16, block 128, weights only | none | max | +| `nvfp4_mlp_weight_only` | NVFP4 W4A16 (block 32), MLP + MoE weights only | none | max | +| `mxfp4_mlp_weight_only` | MXFP4 W4A16, MLP + MoE weights only | none | none (no calibration) |
@@ -128,6 +130,11 @@ activations and tensor-core math are what deliver the throughput. - **`int4_blockwise_weight_only`** — INT4 weights, block size 128, BF16 activations. Classic W4A16 weight compression; works without NVFP4-class hardware. +- **`nvfp4_mlp_weight_only`** — NVFP4 (block size 32) weights on MLP/MoE layers + only, BF16 activations. +- **`mxfp4_mlp_weight_only`** — MXFP4 weights on MLP/MoE layers only, BF16 + activations. Needs no calibration forward pass; the QAT starting point for the + GPT-OSS family (see `examples/gpt-oss`). --- diff --git a/plugins/modelopt/skills/deployment/references/trtllm.md b/plugins/modelopt/skills/deployment/references/trtllm.md index 74895b30dab..27185793307 100644 --- a/plugins/modelopt/skills/deployment/references/trtllm.md +++ b/plugins/modelopt/skills/deployment/references/trtllm.md @@ -51,7 +51,7 @@ directly together with a ModelOpt-quantized checkpoint. ### Workflow 1. Quantize the checkpoint with ModelOpt PTQ (including AutoQuant / mixed precision) via - `examples/llm_ptq` (`hf_ptq.py` / `scripts/huggingface_example.sh`), which produces a + `examples/hf_ptq` (`hf_ptq.py` / `scripts/huggingface_example.sh`), which produces a unified HuggingFace checkpoint with `hf_quant_config.json`. 2. Deploy that checkpoint with TensorRT-LLM's AutoDeploy backend (see the upstream `examples/auto_deploy` docs for the current API and `trtllm-serve` flags). diff --git a/plugins/modelopt/skills/deployment/scripts/deploy.sh b/plugins/modelopt/skills/deployment/scripts/deploy.sh index 1244b49e2c3..a514c7d1e43 100755 --- a/plugins/modelopt/skills/deployment/scripts/deploy.sh +++ b/plugins/modelopt/skills/deployment/scripts/deploy.sh @@ -338,7 +338,7 @@ start_trtllm() { cat <= 2 assert recipe.auto_quantize.auto_quantize_method in ("gradient", "kl_div") + # Both shared base units must be spliced in: the removed --auto_quantize_* CLI shim appended + # them unconditionally, so a general recipe is the migration target and must match it. Without + # cost_excluded_layers a VL/MTP model counts its vision tower in the effective-bits denominator. + assert "*output_layer*" in recipe.auto_quantize.disabled_layers + assert recipe.auto_quantize.cost_excluded_layers == ["*visual*", "*mtp*", "*vision_tower*"] + + +def _all_shipped_ptq_recipe_paths(): + """Every shipped PTQ recipe, discovered from disk rather than a hardcoded list.""" + root = files("modelopt_recipes") + paths = [] + for path in sorted(Path(str(root)).rglob("*.yaml")): + rel = path.relative_to(str(root)) + # Units/presets under configs/ are fragments, not standalone recipes. + if rel.parts[0] == "configs": + continue + raw = _load_raw_config(path) + # List-shaped fragments (layer-pattern units) are not recipes. + if not isinstance(raw, dict): + continue + if (raw.get("metadata") or {}).get("recipe_type") == "ptq": + paths.append(str(rel.with_suffix(""))) + return paths + + +@pytest.mark.parametrize("recipe_path", _all_shipped_ptq_recipe_paths()) +def test_shipped_ptq_recipe_algorithm_config_constructs(recipe_path): + """Every shipped PTQ recipe's ``algorithm`` must build its calibration config class. + + ``QuantizeConfig.algorithm`` accepts a bare dict, so ``load_recipe`` alone never constructs + ``QuantizeAlgorithmConfig`` — a malformed algorithm block loads fine here and only blows up + later inside ``mtq.quantize``. This walks the same path ``apply_mode`` does so a schema break + (e.g. a legacy ``layerwise: false`` bool) fails at test time instead of at calibration time. + """ + algorithm = load_recipe(recipe_path).quantize.algorithm + for mode_name, mode_cfg in get_modelike_from_algo_cfg(algorithm): + CalibrateModeRegistry[mode_name].config_class(**mode_cfg) diff --git a/tests/unit/recipe/test_presets.py b/tests/unit/recipe/test_presets.py index 64d011f8ef0..89645773f93 100644 --- a/tests/unit/recipe/test_presets.py +++ b/tests/unit/recipe/test_presets.py @@ -15,16 +15,18 @@ """Smoke tests for ``modelopt.recipe.presets`` preset discovery. Guards the eager import-time load shared by the PTQ example scripts: every preset -under the model/KV dirs must load into a usable ``quant_cfg`` dict, deprecation -aliases must resolve to their canonical preset, and the KV ``none`` sentinel must -not collide with a discovered preset. A single malformed preset YAML would -otherwise break ``import modelopt.recipe.presets`` (and every PTQ example). +under the model/KV dirs must load into a usable ``quant_cfg`` dict, and the KV +``none`` sentinel must not collide with a discovered preset. A single malformed +preset YAML would otherwise break ``import modelopt.recipe.presets`` (and every +PTQ example). """ import pytest -from modelopt.recipe import presets +import modelopt.torch.quantization as mtq +from modelopt.recipe import load_recipe, presets from modelopt.torch.opt.config_loader import BUILTIN_CONFIG_ROOT +from modelopt.torch.quantization.config import QuantizeConfig def _yaml_basenames(subdir: str) -> set[str]: @@ -55,13 +57,6 @@ def test_every_discovered_preset_loads(choices, preset_dir): assert "quant_cfg" in cfg, f"{name} is missing the 'quant_cfg' key" -def test_aliases_resolve_to_their_canonical_preset(): - for alias, target in presets.QFORMAT_ALIASES.items(): - assert alias in presets.QUANT_CFG_CHOICES, f"alias {alias!r} not exposed" - assert target in presets.QUANT_CFG_CHOICES, f"alias target {target!r} missing" - assert presets.QUANT_CFG_CHOICES[alias] == presets.QUANT_CFG_CHOICES[target] - - def test_kv_none_sentinel_is_not_a_discovered_preset(): # The scripts branch on ``kv_cache_qformat != KV_CACHE_NONE``; a real preset named # "none" would make that branch ambiguous. @@ -83,8 +78,16 @@ def test_w4a16_nvfp4_preset_disables_vllm_marlin_incompatible_projections(): } <= disabled_quantizers -def test_load_quant_cfg_choices_rejects_stale_alias(): - with pytest.raises(ValueError, match="does-not-exist"): - presets.load_quant_cfg_choices( - presets.MODEL_QUANT_PRESET_DIR, {"bad_alias": "does-not-exist"} - ) +@pytest.mark.parametrize( + ("recipe_name", "cfg_name"), + [ + ("general/ptq/mxfp4_mlp_weight_only", "MXFP4_MLP_WEIGHT_ONLY_CFG"), + ("general/ptq/nvfp4_mlp_weight_only", "NVFP4_MLP_WEIGHT_ONLY_CFG"), + ], +) +def test_mlp_weight_only_recipe_matches_its_mtq_cfg(recipe_name, cfg_name): + # examples/gpt-oss migrated from --quant_cfg to --recipe ; pin the + # equality so the recipe and the mtq constant cannot drift apart silently. + recipe_cfg = load_recipe(recipe_name).quantize.model_dump(exclude_unset=True) + mtq_cfg = QuantizeConfig(**getattr(mtq, cfg_name)).model_dump(exclude_unset=True) + assert recipe_cfg == mtq_cfg diff --git a/tests/unit/torch/quantization/test_config_validation.py b/tests/unit/torch/quantization/test_config_validation.py index 4b969d3259c..546dd21f85f 100644 --- a/tests/unit/torch/quantization/test_config_validation.py +++ b/tests/unit/torch/quantization/test_config_validation.py @@ -583,77 +583,35 @@ def test_validate_quant_cfg_entries_accepts_valid_cfg(self): assert len(cfg.quant_cfg) == 2 -class TestLayerwiseUseSequentialAlias: - """`use_sequential` is the legacy alias for `layerwise` (pre-#1251 checkpoints).""" - - @pytest.mark.parametrize("value", [True, False]) - def test_use_sequential_resolves_to_layerwise(self, value): - with pytest.warns(DeprecationWarning): - cfg = MaxCalibConfig(use_sequential=value) - assert cfg.layerwise.enable is value - - def test_serializes_under_layerwise_not_alias(self): - with pytest.warns(DeprecationWarning): - dumped = MaxCalibConfig(use_sequential=True).model_dump() - assert dumped["layerwise"]["enable"] is True - assert "use_sequential" not in dumped - - class TestLayerwiseNestedConfig: - """Layerwise expands from a bool to a nested ``LayerwiseConfig``. - - Backward compatibility: bool input is coerced with a DeprecationWarning, and - the legacy flat ``layerwise_checkpoint_dir`` key is silently absorbed. - """ + """Layerwise calibration options live in a nested ``LayerwiseConfig``.""" def test_nested_form_accepted(self): cfg = MaxCalibConfig(layerwise={"enable": True, "checkpoint_dir": "/x"}) assert cfg.layerwise.enable is True assert cfg.layerwise.checkpoint_dir == "/x" - def test_bool_form_deprecated_but_accepted(self): - with pytest.warns(DeprecationWarning, match="bool is deprecated"): - cfg = MaxCalibConfig(layerwise=True) - assert cfg.layerwise.enable is True + @pytest.mark.parametrize( + "kwargs", + [ + {"layerwise": True}, + {"use_sequential": True}, + {"layerwise": {"enable": True}, "layerwise_checkpoint_dir": "/x"}, + ], + ) + def test_legacy_forms_rejected(self, kwargs): + """The bool form, the ``use_sequential`` alias and the flat checkpoint-dir key are gone.""" + with pytest.raises(ValidationError): + MaxCalibConfig(**kwargs) def test_dict_form_no_deprecation(self): with warnings.catch_warnings(): warnings.simplefilter("error", DeprecationWarning) MaxCalibConfig(layerwise={"enable": True}) - def test_flat_checkpoint_dir_migrated_with_deprecation(self): - """Legacy ``layerwise_checkpoint_dir`` is migrated into the nested config - and emits a deprecation warning naming the flat key (independent of the - bool-form deprecation tested above). - """ - with pytest.warns(DeprecationWarning, match="layerwise_checkpoint_dir.*deprecated"): - cfg = MaxCalibConfig(layerwise={"enable": True}, layerwise_checkpoint_dir="/x") - assert cfg.layerwise.checkpoint_dir == "/x" - - def test_use_sequential_alias_survives_flat_checkpoint_migration(self): - """``use_sequential`` + flat ``layerwise_checkpoint_dir`` must not drop the alias value.""" - with pytest.warns(DeprecationWarning): - cfg = MaxCalibConfig(use_sequential=True, layerwise_checkpoint_dir="/x") - assert cfg.layerwise.enable is True - assert cfg.layerwise.checkpoint_dir == "/x" - - def test_conflicting_flat_and_nested_checkpoint_dir_raises(self): - with pytest.raises(ValidationError, match="Conflicting checkpoint_dir"): - MaxCalibConfig( - layerwise={"enable": True, "checkpoint_dir": "/a"}, - layerwise_checkpoint_dir="/b", - ) - - @pytest.mark.parametrize( - "kwargs", - [ - {"layerwise": {"checkpoint_dir": "/x"}}, - {"layerwise_checkpoint_dir": "/x"}, - ], - ) - def test_checkpoint_dir_requires_enable(self, kwargs): + def test_checkpoint_dir_requires_enable(self): with pytest.raises(ValidationError, match=r"requires layerwise.enable=True"): - MaxCalibConfig(**kwargs) + MaxCalibConfig(layerwise={"checkpoint_dir": "/x"}) @pytest.mark.parametrize( ("cfg_cls", "expected_qdq"), @@ -667,10 +625,6 @@ def test_per_algorithm_qdq_default(self, cfg_cls, expected_qdq): [ # GPTQ default kicks in for user dict that doesn't mention qdq. ({"enable": True}, True), - # GPTQ default kicks in for legacy bool form too. - pytest.param( - True, True, marks=pytest.mark.filterwarnings("ignore::DeprecationWarning") - ), # User-explicit False overrides the GPTQ default. ({"enable": True, "get_qdq_activations_from_prev_layer": False}, False), # ``LayerwiseConfig`` instance: ``_coerce_layerwise_input`` must diff --git a/tests/unit/torch/quantization/test_layerwise_calibrate.py b/tests/unit/torch/quantization/test_layerwise_calibrate.py index 1c8234fdb2f..bda8c6029b1 100644 --- a/tests/unit/torch/quantization/test_layerwise_calibrate.py +++ b/tests/unit/torch/quantization/test_layerwise_calibrate.py @@ -669,7 +669,7 @@ def _awq_layerwise_config() -> dict: for entry in cfg["quant_cfg"]: if entry.get("quantizer_name") == "*weight_quantizer": entry.setdefault("cfg", {})["block_sizes"] = {-1: 8, "type": "static"} - cfg["algorithm"] = {"method": "awq_lite", "alpha_step": 0.5, "layerwise": True} + cfg["algorithm"] = {"method": "awq_lite", "alpha_step": 0.5, "layerwise": {"enable": True}} return cfg @@ -679,12 +679,12 @@ def _svdquant_layerwise_config() -> dict: for entry in cfg["quant_cfg"]: if entry.get("quantizer_name") == "*weight_quantizer": entry.setdefault("cfg", {})["block_sizes"] = {-1: 8, "type": "static"} - cfg["algorithm"] = {"method": "svdquant", "lowrank": 4, "layerwise": True} + cfg["algorithm"] = {"method": "svdquant", "lowrank": 4, "layerwise": {"enable": True}} return cfg def test_mtq_quantize_layerwise_e2e_max(monkeypatch): - """End-to-end: mtq.quantize with layerwise=True produces populated amax values. + """End-to-end: mtq.quantize with layerwise enabled produces populated amax values. ``max`` is the representative algorithm for the layerwise happy path because every other algorithm seeds amax via max_calibrate first — if max works, the @@ -693,7 +693,7 @@ def test_mtq_quantize_layerwise_e2e_max(monkeypatch): CUDA) or unnecessary duplication. """ _register_test_discoverer(monkeypatch) - config = _int8_cfg_with_algorithm({"method": "max", "layerwise": True}) + config = _int8_cfg_with_algorithm({"method": "max", "layerwise": {"enable": True}}) torch.manual_seed(0) model = _SimpleTransformerModel(n_layers=3, dim=16) @@ -747,7 +747,7 @@ def stub(model, forward_loop, calib_func, **kwargs): if algorithm == "awq_lite": config = _awq_layerwise_config() else: - config = _int8_cfg_with_algorithm({"method": algorithm, "layerwise": True}) + config = _int8_cfg_with_algorithm({"method": algorithm, "layerwise": {"enable": True}}) torch.manual_seed(0) model = _SimpleTransformerModel(n_layers=2, dim=16)