diff --git a/keras/src/saving/saving_lib.py b/keras/src/saving/saving_lib.py
index c16d2ffafb4..7e858025963 100644
--- a/keras/src/saving/saving_lib.py
+++ b/keras/src/saving/saving_lib.py
@@ -3,6 +3,7 @@
 import datetime
 import io
 import json
+import pathlib
 import tempfile
 import warnings
 import zipfile
@@ -32,6 +33,8 @@
 _CONFIG_FILENAME = "config.json"
 _METADATA_FILENAME = "metadata.json"
 _VARS_FNAME = "model.weights"  # Will become e.g. "model.weights.h5"
+_VARS_FNAME_H5 = _VARS_FNAME + ".h5"
+_VARS_FNAME_NPZ = _VARS_FNAME + ".npz"
 _ASSETS_DIRNAME = "assets"

@@ -109,30 +112,50 @@ def _save_model_to_fileobj(model, fileobj, weights_format):
         with zf.open(_CONFIG_FILENAME, "w") as f:
             f.write(config_json.encode())

-        if weights_format == "h5":
-            weights_store = H5IOStore(_VARS_FNAME + ".h5", archive=zf, mode="w")
-        elif weights_format == "npz":
-            weights_store = NpzIOStore(
-                _VARS_FNAME + ".npz", archive=zf, mode="w"
-            )
-        else:
-            raise ValueError(
-                "Unknown `weights_format` argument. "
-                "Expected 'h5' or 'npz'. "
-                f"Received: weights_format={weights_format}"
-            )
+        weights_file_path = None
+        try:
+            if weights_format == "h5":
+                if isinstance(fileobj, io.BufferedWriter):
+                    # First, open the .h5 file, then write it to `zf` at the end
+                    # of the function call.
+                    working_dir = pathlib.Path(fileobj.name).parent
+                    weights_file_path = working_dir / _VARS_FNAME_H5
+                    weights_store = H5IOStore(weights_file_path, mode="w")
+                else:
+                    # Fall back when `fileobj` is an `io.BytesIO`. Typically,
+                    # this usage is for pickling.
+                    weights_store = H5IOStore(
+                        _VARS_FNAME_H5, archive=zf, mode="w"
+                    )
+            elif weights_format == "npz":
+                weights_store = NpzIOStore(
+                    _VARS_FNAME_NPZ, archive=zf, mode="w"
+                )
+            else:
+                raise ValueError(
+                    "Unknown `weights_format` argument. "
+                    "Expected 'h5' or 'npz'. "
+                    f"Received: weights_format={weights_format}"
+                )

-        asset_store = DiskIOStore(_ASSETS_DIRNAME, archive=zf, mode="w")
+            asset_store = DiskIOStore(_ASSETS_DIRNAME, archive=zf, mode="w")

-        _save_state(
-            model,
-            weights_store=weights_store,
-            assets_store=asset_store,
-            inner_path="",
-            visited_saveables=set(),
-        )
-        weights_store.close()
-        asset_store.close()
+            _save_state(
+                model,
+                weights_store=weights_store,
+                assets_store=asset_store,
+                inner_path="",
+                visited_saveables=set(),
+            )
+        except:
+            # Skip the final `zf.write` if any exception is raised
+            weights_file_path = None
+            raise
+        finally:
+            weights_store.close()
+            asset_store.close()
+            if weights_file_path:
+                zf.write(weights_file_path, weights_file_path.name)

 def load_model(filepath, custom_objects=None, compile=True, safe_mode=True):
@@ -172,36 +195,49 @@ def _load_model_from_fileobj(fileobj, custom_objects, compile, safe_mode):
             )

         all_filenames = zf.namelist()
-        if _VARS_FNAME + ".h5" in all_filenames:
-            weights_store = H5IOStore(_VARS_FNAME + ".h5", archive=zf, mode="r")
-        elif _VARS_FNAME + ".npz" in all_filenames:
-            weights_store = NpzIOStore(
-                _VARS_FNAME + ".npz", archive=zf, mode="r"
-            )
-        else:
-            raise ValueError(
-                f"Expected a {_VARS_FNAME}.h5 or {_VARS_FNAME}.npz file."
-            )
+        weights_file_path = None
+        try:
+            if _VARS_FNAME_H5 in all_filenames:
+                if isinstance(fileobj, io.BufferedReader):
+                    # First, extract the model.weights.h5 file, then load it
+                    # using h5py.
+                    working_dir = pathlib.Path(fileobj.name).parent
+                    zf.extract(_VARS_FNAME_H5, working_dir)
+                    weights_file_path = working_dir / _VARS_FNAME_H5
+                    weights_store = H5IOStore(weights_file_path, mode="r")
+                else:
+                    # Fall back when `fileobj` is an `io.BytesIO`. Typically,
+                    # this usage is for pickling.
+                    weights_store = H5IOStore(_VARS_FNAME_H5, zf, mode="r")
+            elif _VARS_FNAME_NPZ in all_filenames:
+                weights_store = NpzIOStore(_VARS_FNAME_NPZ, zf, mode="r")
+            else:
+                raise ValueError(
+                    f"Expected a {_VARS_FNAME_H5} or {_VARS_FNAME_NPZ} file."
+                )

-        if len(all_filenames) > 3:
-            asset_store = DiskIOStore(_ASSETS_DIRNAME, archive=zf, mode="r")
-        else:
-            asset_store = None
-
-        failed_saveables = set()
-        error_msgs = {}
-        _load_state(
-            model,
-            weights_store=weights_store,
-            assets_store=asset_store,
-            inner_path="",
-            visited_saveables=set(),
-            failed_saveables=failed_saveables,
-            error_msgs=error_msgs,
-        )
-        weights_store.close()
-        if asset_store:
-            asset_store.close()
+            if len(all_filenames) > 3:
+                asset_store = DiskIOStore(_ASSETS_DIRNAME, archive=zf, mode="r")
+            else:
+                asset_store = None
+
+            failed_saveables = set()
+            error_msgs = {}
+            _load_state(
+                model,
+                weights_store=weights_store,
+                assets_store=asset_store,
+                inner_path="",
+                visited_saveables=set(),
+                failed_saveables=failed_saveables,
+                error_msgs=error_msgs,
+            )
+        finally:
+            weights_store.close()
+            if asset_store:
+                asset_store.close()
+            if weights_file_path:
+                weights_file_path.unlink()

         if failed_saveables:
             _raise_loading_failure(error_msgs)
@@ -250,9 +286,7 @@ def load_weights_only(
         weights_store = H5IOStore(filepath, mode="r")
     elif filepath.endswith(".keras"):
         archive = zipfile.ZipFile(filepath, "r")
-        weights_store = H5IOStore(
-            _VARS_FNAME + ".h5", archive=archive, mode="r"
-        )
+        weights_store = H5IOStore(_VARS_FNAME_H5, archive=archive, mode="r")

     failed_saveables = set()
     if objects_to_skip is not None:

diff --git a/keras/src/saving/saving_lib_test.py b/keras/src/saving/saving_lib_test.py
index 80f53608f6c..91e40426285 100644
--- a/keras/src/saving/saving_lib_test.py
+++ b/keras/src/saving/saving_lib_test.py
@@ -612,6 +612,53 @@ def test_save_to_fileobj(self) -> None:

         self.assertAllClose(pred1, pred2, atol=1e-5)

+    def test_save_model_exception_raised(self):
+        # Assume we have an error in `save_own_variables`.
+        class RaiseErrorLayer(keras.layers.Layer):
+            def __init__(self, **kwargs):
+                super().__init__(**kwargs)
+
+            def call(self, inputs):
+                return inputs
+
+            def save_own_variables(self, store):
+                raise ValueError
+
+        model = keras.Sequential([keras.Input([1]), RaiseErrorLayer()])
+        filepath = f"{self.get_temp_dir()}/model.keras"
+        with self.assertRaises(ValueError):
+            saving_lib.save_model(model, filepath)
+
+        # Ensure we don't have a bad "model.weights.h5" inside the zip file.
+        self.assertTrue(Path(filepath).exists())
+        with zipfile.ZipFile(filepath) as zf:
+            all_filenames = zf.namelist()
+            self.assertNotIn("model.weights.h5", all_filenames)
+
+    def test_load_model_exception_raised(self):
+        # Assume we have an error in `load_own_variables`.
+        class RaiseErrorLayer(keras.layers.Layer):
+            def __init__(self, **kwargs):
+                super().__init__(**kwargs)
+
+            def call(self, inputs):
+                return inputs
+
+            def load_own_variables(self, store):
+                raise ValueError
+
+        model = keras.Sequential([keras.Input([1]), RaiseErrorLayer()])
+        filepath = f"{self.get_temp_dir()}/model.keras"
+        saving_lib.save_model(model, filepath)
+        with self.assertRaises(ValueError):
+            saving_lib.load_model(
+                filepath, custom_objects={"RaiseErrorLayer": RaiseErrorLayer}
+            )
+
+        # Ensure we don't leave a bad "model.weights.h5" on the filesystem.
+        self.assertTrue(Path(filepath).exists())
+        self.assertFalse(Path(filepath).with_name("model.weights.h5").exists())
+

 @pytest.mark.requires_trainable_backend
 class SavingAPITest(testing.TestCase):

model.keras format much slower to load
Anyone experiencing unreasonably slow load times when loading a keras-format saved model? I have noticed this repeated when working in ipython, where simply instantiating a model via `Model.from_config` then calling `model.load_weights` is much (several factors) faster than loading a `model.keras` file.

My understanding is the keras format is simply a zip file with the config.json file and weights h5 (iirc) but weirdly enough, there's something not right going on while loading.

Could you please provide the comparison and the time difference in loading the model with .keras and and other format.

For more details on the changes included with .keras format and why it is preferred over other format, refer https://keras.io/guides/serialization_and_saving/
I don't have an example at the moment but recently we updated our prod system from keras 2 to keras 3 so we converted all legacy saved models to the new keras 3 format which lead to our service to take over 12 minutes to load all models (>15 models loading in subprocesses in parallel). Moving to `from_config` + `load_weights` reduced the time to ~2 minutes (which is on par with what we had before).

For what it's worth, before we did that migration, I was already working on GPT2Backbone models with keras-nlp and noticed the same issue were loading the .keras model was really slow (but didn't give it much thought at the time)
What you're using is actually the same as what `load_model` is using except for the interaction with the zip file. So perhaps the zip file reading is the issue.
100% which is why I find this very odd
I encountered this issue before when trying to quantize Gemma

I have created this script to demonstrate the issue (using GPT-2)

`check_loading.py`
```python
import argparse
import json

import keras
import keras_nlp

def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-m",
        "--mode",
        default="save",
        choices=["save", "load", "load_weights"],
    )
    args = parser.parse_args()
    return args

def main(args):
    if args.mode == "save":
        model = keras_nlp.models.GPT2CausalLM.from_preset("gpt2_base_en")
        # Save keras file
        model.save("model.keras")
        # Save serialized config and weights
        config = keras.saving.serialize_keras_object(model)
        with open("model.json", "w") as f:
            json.dump(config, f)
        model.save_weights("model.weights.h5")
    elif args.mode == "load":
        model = keras.saving.load_model("model.keras")
    else:
        with open("model.json", "r") as f:
            config = json.load(f)
        model = keras.saving.deserialize_keras_object(config)
        model.load_weights("model.weights.h5")

if __name__ == "__main__":
    keras.config.disable_traceback_filtering()
    main(get_args())

```

Usage:
```bash
# 1. Save the model
python check_loading.py -m save
# 2. Profile `load_model`
pyinstrument python check_loading.py -m load
# 3. Profile `deserialize_keras_object` and `load_weights`
pyinstrument python check_loading.py -m load_weights
```

The result:

|Method|Cost Time|
|-|-|
|`load_model`|27.861s|
|`deserialize_keras_object` + `load_weights`|3.166s|

Logs:

<details>

```console
  _     ._   __/__   _ _  _  _ _/_   Recorded: 10:05:02  Samples:  10954
 /_//_/// /_\ / //_// / //_'/ //     Duration: 27.861    CPU time: 30.009
/   _/                      v4.6.2

Program: /home/hongyu/miniconda3/envs/kimm/bin/pyinstrument check_loading.py -m load

27.861 <module>  check_loading.py:1
├─ 25.635 main  check_loading.py:20
│  └─ 25.635 load_model  keras/src/saving/saving_api.py:116
│     └─ 25.634 load_model  keras/src/saving/saving_lib.py:138
│        └─ 25.634 _load_model_from_fileobj  keras/src/saving/saving_lib.py:157
│           ├─ 24.319 _load_state  keras/src/saving/saving_lib.py:395
│           │  ├─ 23.507 _load_container_state  keras/src/saving/saving_lib.py:510
│           │  │  └─ 23.507 _load_state  keras/src/saving/saving_lib.py:395
│           │  │     └─ 23.505 _load_container_state  keras/src/saving/saving_lib.py:510
│           │  │        └─ 23.504 _load_state  keras/src/saving/saving_lib.py:395
│           │  │           ├─ 21.286 _load_state  keras/src/saving/saving_lib.py:395
│           │  │           │  ├─ 9.102 _load_state  keras/src/saving/saving_lib.py:395
│           │  │           │  │  ├─ 5.381 H5IOStore.get  keras/src/saving/saving_lib.py:632
│           │  │           │  │  │  └─ 5.381 H5Entry.__init__  keras/src/saving/saving_lib.py:646
│           │  │           │  │  │     ├─ 3.618 Group.__contains__  h5py/_hl/group.py:508
│           │  │           │  │  │     │     [6 frames hidden]  h5py, zipfile, <built-in>
│           │  │           │  │  │     └─ 1.763 File.__getitem__  h5py/_hl/group.py:348
│           │  │           │  │  │           [6 frames hidden]  h5py, zipfile, <built-in>
│           │  │           │  │  └─ 3.717 EinsumDense.load_own_variables  keras/src/layers/core/einsum_dense.py:279
│           │  │           │  │     └─ 3.579 H5Entry.__getitem__  keras/src/saving/saving_lib.py:702
│           │  │           │  │        └─ 3.577 Group.__getitem__  h5py/_hl/group.py:348
│           │  │           │  │              [6 frames hidden]  h5py, zipfile, <built-in>
│           │  │           │  ├─ 7.054 H5IOStore.get  keras/src/saving/saving_lib.py:632
│           │  │           │  │  └─ 7.054 H5Entry.__init__  keras/src/saving/saving_lib.py:646
│           │  │           │  │     ├─ 4.377 Group.__contains__  h5py/_hl/group.py:508
│           │  │           │  │     │     [9 frames hidden]  h5py, zipfile, <built-in>
│           │  │           │  │     └─ 2.677 Group.__getitem__  h5py/_hl/group.py:348
│           │  │           │  │           [6 frames hidden]  h5py, zipfile, <built-in>
│           │  │           │  ├─ 3.121 LayerNormalization.load_own_variables  keras/src/layers/layer.py:1187
│           │  │           │  │  ├─ 1.936 H5Entry.__getitem__  keras/src/saving/saving_lib.py:702
│           │  │           │  │  │  └─ 1.935 Group.__getitem__  h5py/_hl/group.py:348
│           │  │           │  │  │        [6 frames hidden]  h5py, zipfile, <built-in>
│           │  │           │  │  └─ 0.967 Variable.assign  keras/src/backend/common/variables.py:223
│           │  │           │  │     └─ 0.962 Variable._convert_to_tensor  keras/src/backend/tensorflow/core.py:53
│           │  │           │  │        └─ 0.962 convert_to_tensor  keras/src/backend/tensorflow/core.py:102
│           │  │           │  │           └─ 0.961 error_handler  tensorflow/python/util/traceback_utils.py:138
│           │  │           │  │                 [18 frames hidden]  tensorflow, h5py, zipfile, <built-in>
│           │  │           │  └─ 1.978 Dense.load_own_variables  keras/src/layers/core/dense.py:224
│           │  │           │     └─ 1.690 H5Entry.__getitem__  keras/src/saving/saving_lib.py:702
│           │  │           │        └─ 1.690 Group.__getitem__  h5py/_hl/group.py:348
│           │  │           │              [6 frames hidden]  h5py, zipfile, <built-in>
│           │  │           ├─ 1.576 H5IOStore.get  keras/src/saving/saving_lib.py:632
│           │  │           │  └─ 1.576 H5Entry.__init__  keras/src/saving/saving_lib.py:646
│           │  │           │     └─ 1.391 Group.__contains__  h5py/_hl/group.py:508
│           │  │           │           [6 frames hidden]  h5py, zipfile, <built-in>
│           │  │           ├─ 0.344 ReversibleEmbedding.load_own_variables  keras_nlp/src/layers/modeling/reversible_embedding.py:151
│           │  │           │  └─ 0.344 ReversibleEmbedding.load_own_variables  keras/src/layers/core/embedding.py:214
│           │  │           │     └─ 0.288 Variable.assign  keras/src/backend/common/variables.py:223
│           │  │           │        └─ 0.288 Variable._convert_to_tensor  keras/src/backend/tensorflow/core.py:53
│           │  │           │           └─ 0.288 convert_to_tensor  keras/src/backend/tensorflow/core.py:102
│           │  │           │              └─ 0.288 error_handler  tensorflow/python/util/traceback_utils.py:138
│           │  │           │                    [11 frames hidden]  tensorflow
│           │  │           └─ 0.298 TransformerDecoder.load_own_variables  keras/src/layers/layer.py:1187
│           │  └─ 0.809 _load_state  keras/src/saving/saving_lib.py:395
│           │     └─ 0.586 _load_state  keras/src/saving/saving_lib.py:395
│           │        └─ 0.467 GPT2Tokenizer.load_assets  keras_nlp/src/tokenizers/byte_pair_tokenizer.py:327
│           │              [3 frames hidden]  keras_nlp
│           ├─ 0.534 deserialize_keras_object  keras/src/saving/serialization_lib.py:393
│           │  └─ 0.534 GPT2CausalLM.from_config  keras_nlp/src/models/task.py:143
│           │     └─ 0.522 deserialize  keras/src/layers/__init__.py:153
│           │        └─ 0.522 deserialize_keras_object  keras/src/saving/serialization_lib.py:393
│           │           └─ 0.516 GPT2Backbone.from_config  keras_nlp/src/models/backbone.py:139
│           │                 [3 frames hidden]  keras_nlp
│           │                    0.293 TransformerDecoder.__call__  keras_nlp/src/layers/modeling/transformer_decoder.py:253
│           │                    └─ 0.288 build_wrapper  keras/src/layers/layer.py:220
│           │                       └─ 0.288 TransformerDecoder.build  keras_nlp/src/layers/modeling/transformer_decoder.py:134
│           └─ 0.458 DiskIOStore.__init__  keras/src/saving/saving_lib.py:556
│              └─ 0.458 ZipFile.extractall  zipfile.py:1666
│                    [4 frames hidden]  zipfile, shutil, <built-in>
└─ 2.121 <module>  keras/__init__.py:1
   └─ 2.121 <module>  keras/api/__init__.py:1
      └─ 2.120 <module>  keras/api/_tf_keras/__init__.py:1
         └─ 2.120 <module>  keras/api/_tf_keras/keras/__init__.py:1
            └─ 2.110 <module>  keras/api/activations/__init__.py:1
               └─ 2.110 <module>  keras/src/__init__.py:1
                  └─ 2.109 <module>  keras/src/activations/__init__.py:1
                     └─ 1.921 <module>  keras/src/activations/activations.py:1
                        └─ 1.917 <module>  keras/src/backend/__init__.py:1
                           └─ 1.824 <module>  keras/src/backend/common/__init__.py:1
                              └─ 1.823 <module>  keras/src/backend/common/dtypes.py:1
                                 └─ 1.823 <module>  keras/src/backend/common/variables.py:1
                                    └─ 1.758 <module>  keras/src/utils/__init__.py:1
                                       └─ 1.721 <module>  keras/src/utils/model_visualization.py:1
                                          └─ 1.705 <module>  keras/src/tree/__init__.py:1
                                             └─ 1.705 <module>  keras/src/tree/tree_api.py:1
                                                └─ 1.699 <module>  keras/src/tree/optree_impl.py:1
                                                   └─ 1.698 <module>  tensorflow/__init__.py:1
                                                         [23 frames hidden]  tensorflow

  _     ._   __/__   _ _  _  _ _/_   Recorded: 10:05:39  Samples:  2266
 /_//_/// /_\ / //_// / //_'/ //     Duration: 3.166     CPU time: 5.276
/   _/                      v4.6.2

Program: /home/hongyu/miniconda3/envs/kimm/bin/pyinstrument check_loading.py -m load_weights

3.165 <module>  check_loading.py:1
├─ 2.121 <module>  keras/__init__.py:1
│  └─ 2.121 <module>  keras/api/__init__.py:1
│     └─ 2.120 <module>  keras/api/_tf_keras/__init__.py:1
│        └─ 2.120 <module>  keras/api/_tf_keras/keras/__init__.py:1
│           └─ 2.110 <module>  keras/api/activations/__init__.py:1
│              └─ 2.110 <module>  keras/src/__init__.py:1
│                 └─ 2.109 <module>  keras/src/activations/__init__.py:1
│                    ├─ 1.922 <module>  keras/src/activations/activations.py:1
│                    │  └─ 1.917 <module>  keras/src/backend/__init__.py:1
│                    │     ├─ 1.825 <module>  keras/src/backend/common/__init__.py:1
│                    │     │  └─ 1.824 <module>  keras/src/backend/common/dtypes.py:1
│                    │     │     └─ 1.824 <module>  keras/src/backend/common/variables.py:1
│                    │     │        ├─ 1.760 <module>  keras/src/utils/__init__.py:1
│                    │     │        │  └─ 1.722 <module>  keras/src/utils/model_visualization.py:1
│                    │     │        │     └─ 1.707 <module>  keras/src/tree/__init__.py:1
│                    │     │        │        └─ 1.707 <module>  keras/src/tree/tree_api.py:1
│                    │     │        │           └─ 1.701 <module>  keras/src/tree/optree_impl.py:1
│                    │     │        │              └─ 1.701 <module>  tensorflow/__init__.py:1
│                    │     │        │                    [116 frames hidden]  tensorflow, <built-in>, inspect, requ...
│                    │     │        └─ 0.063 <module>  numpy/__init__.py:1
│                    │     └─ 0.091 <module>  keras/src/backend/tensorflow/__init__.py:1
│                    │        └─ 0.089 <module>  keras/src/backend/tensorflow/numpy.py:1
│                    │           └─ 0.089 elementwise_unary  keras/src/backend/tensorflow/sparse.py:348
│                    │              └─ 0.089 update_wrapper  functools.py:35
│                    └─ 0.186 <module>  keras/src/saving/__init__.py:1
│                       └─ 0.186 <module>  keras/src/saving/saving_api.py:1
│                          └─ 0.186 <module>  keras/src/legacy/saving/legacy_h5_format.py:1
│                             └─ 0.182 <module>  keras/src/legacy/saving/saving_utils.py:1
│                                ├─ 0.139 <module>  keras/src/models/__init__.py:1
│                                │  └─ 0.138 <module>  keras/src/models/functional.py:1
│                                │     └─ 0.138 <module>  keras/src/models/model.py:1
│                                │        └─ 0.135 <module>  keras/src/trainers/trainer.py:1
│                                │           └─ 0.135 <module>  keras/src/trainers/data_adapters/__init__.py:1
│                                │              └─ 0.134 <module>  keras/src/trainers/data_adapters/array_data_adapter.py:1
│                                │                 └─ 0.133 <module>  keras/src/trainers/data_adapters/array_slicing.py:1
│                                │                    └─ 0.133 <module>  pandas/__init__.py:1
│                                │                          [5 frames hidden]  pandas
│                                └─ 0.043 <module>  keras/src/layers/__init__.py:1
├─ 0.940 main  check_loading.py:20
│  ├─ 0.540 deserialize_keras_object  keras/src/saving/serialization_lib.py:393
│  │  └─ 0.539 GPT2CausalLM.from_config  keras_nlp/src/models/task.py:143
│  │     └─ 0.528 deserialize  keras/src/layers/__init__.py:153
│  │        └─ 0.528 deserialize_keras_object  keras/src/saving/serialization_lib.py:393
│  │           └─ 0.522 GPT2Backbone.from_config  keras_nlp/src/models/backbone.py:139
│  │                 [3 frames hidden]  keras_nlp
│  │                    0.522 GPT2Backbone.__init__  keras_nlp/src/models/gpt2/gpt2_backbone.py:92
│  │                    ├─ 0.294 TransformerDecoder.__call__  keras_nlp/src/layers/modeling/transformer_decoder.py:253
│  │                    │  └─ 0.289 build_wrapper  keras/src/layers/layer.py:220
│  │                    │     └─ 0.289 TransformerDecoder.build  keras_nlp/src/layers/modeling/transformer_decoder.py:134
│  │                    │        └─ 0.223 build_wrapper  keras/src/layers/layer.py:220
│  │                    │           ├─ 0.138 CachedMultiHeadAttention.build  keras/src/layers/attention/multi_head_attention.py:199
│  │                    │           │  └─ 0.091 build_wrapper  keras/src/layers/layer.py:220
│  │                    │           │     └─ 0.088 EinsumDense.build  keras/src/layers/core/einsum_dense.py:147
│  │                    │           │        └─ 0.082 EinsumDense.add_weight  keras/src/layers/layer.py:455
│  │                    │           │           └─ 0.080 Variable.__init__  keras/src/backend/common/variables.py:80
│  │                    │           │              └─ 0.046 Variable._initialize  keras/src/backend/tensorflow/core.py:30
│  │                    │           │                 └─ 0.045 error_handler  tensorflow/python/util/traceback_utils.py:138
│  │                    │           │                       [14 frames hidden]  tensorflow, <built-in>
│  │                    │           ├─ 0.046 Dense.build  keras/src/layers/core/dense.py:102
│  │                    │           │  └─ 0.045 Dense.add_weight  keras/src/layers/layer.py:455
│  │                    │           │     └─ 0.045 Variable.__init__  keras/src/backend/common/variables.py:80
│  │                    │           └─ 0.033 LayerNormalization.build  keras/src/layers/normalization/layer_normalization.py:147
│  │                    │              └─ 0.033 LayerNormalization.add_weight  keras/src/layers/layer.py:455
│  │                    │                 └─ 0.033 Variable.__init__  keras/src/backend/common/variables.py:80
│  │                    └─ 0.199 Dropout.__init__  keras/src/layers/regularization/dropout.py:41
│  │                       └─ 0.199 SeedGenerator.__init__  keras/src/random/seed_generator.py:48
│  │                          └─ 0.199 Variable.__init__  keras/src/backend/common/variables.py:80
│  │                             └─ 0.198 seed_initializer  keras/src/random/seed_generator.py:70
│  │                                └─ 0.198 convert_to_tensor  keras/src/backend/tensorflow/core.py:102
│  │                                   └─ 0.198 error_handler  tensorflow/python/util/traceback_utils.py:138
│  │                                         [17 frames hidden]  tensorflow, <built-in>
│  └─ 0.399 error_handler  keras/src/utils/traceback_utils.py:110
│     └─ 0.399 GPT2CausalLM.load_weights  keras/src/models/model.py:321
│        └─ 0.399 load_weights  keras/src/saving/saving_api.py:226
│           └─ 0.399 load_weights_only  keras/src/saving/saving_lib.py:239
│              └─ 0.399 _load_state  keras/src/saving/saving_lib.py:395
│                 └─ 0.392 _load_container_state  keras/src/saving/saving_lib.py:510
│                    └─ 0.392 _load_state  keras/src/saving/saving_lib.py:395
│                       └─ 0.391 _load_container_state  keras/src/saving/saving_lib.py:510
│                          └─ 0.390 _load_state  keras/src/saving/saving_lib.py:395
│                             ├─ 0.209 _load_state  keras/src/saving/saving_lib.py:395
│                             │  ├─ 0.088 Dense.load_own_variables  keras/src/layers/core/dense.py:224
│                             │  │  └─ 0.086 Variable.assign  keras/src/backend/common/variables.py:223
│                             │  │     └─ 0.086 Variable._convert_to_tensor  keras/src/backend/tensorflow/core.py:53
│                             │  │        └─ 0.086 convert_to_tensor  keras/src/backend/tensorflow/core.py:102
│                             │  │           └─ 0.085 error_handler  tensorflow/python/util/traceback_utils.py:138
│                             │  │                 [14 frames hidden]  tensorflow, h5py
│                             │  └─ 0.077 _load_state  keras/src/saving/saving_lib.py:395
│                             │     └─ 0.060 EinsumDense.load_own_variables  keras/src/layers/core/einsum_dense.py:279
│                             │        └─ 0.059 Variable.assign  keras/src/backend/common/variables.py:223
│                             │           └─ 0.046 Variable._convert_to_tensor  keras/src/backend/tensorflow/core.py:53
│                             │              └─ 0.046 convert_to_tensor  keras/src/backend/tensorflow/core.py:102
│                             │                 └─ 0.046 error_handler  tensorflow/python/util/traceback_utils.py:138
│                             │                       [11 frames hidden]  tensorflow
│                             └─ 0.172 ReversibleEmbedding.load_own_variables  keras_nlp/src/layers/modeling/reversible_embedding.py:151
│                                └─ 0.172 ReversibleEmbedding.load_own_variables  keras/src/layers/core/embedding.py:214
│                                   └─ 0.164 Variable.assign  keras/src/backend/common/variables.py:223
│                                      └─ 0.164 Variable._convert_to_tensor  keras/src/backend/tensorflow/core.py:53
│                                         └─ 0.164 convert_to_tensor  keras/src/backend/tensorflow/core.py:102
│                                            └─ 0.164 error_handler  tensorflow/python/util/traceback_utils.py:138
│                                                  [14 frames hidden]  tensorflow, h5py
└─ 0.102 <module>  keras_nlp/__init__.py:1
      [18 frames hidden]  keras_nlp, tensorflow_text
```

</details>
By diving into the example provided by @james77777778 ,
in the hidden frames, there's a call: `Group.__getitem__` -> `ZipExtFile.seek`
This makes sense when we are using archive.

in python stdlib `zipfile.ZipExtFile`: `seek` -> `read` -> `_read1` -> `_update_crc`
The overhead caused by `_update_crc` during each `seek()` call is significant.
reference:
https://github.com/python/cpython/blob/f878d46e5614f08a9302fcb6fc611ef49e9acf2f/Lib/zipfile/__init__.py#L1133
A simple way to deal with it, which will work fine:

https://github.com/keras-team/keras/blob/a2df0f9ac595639aa2d3a0359122b030d934389e/keras/src/saving/saving_lib.py#L620-L627

by changing line 624 to `self.io_file = io.BytesIO(self.archive.open(self.root_path, "r").read()) `
That probably fixes the speed issue but would lead to unwanted extra memory usage which is undesirable
> That probably fixes the speed issue but would lead to unwanted extra memory usage which is undesirable

Is that a good tradeoff? Should we instead unzip on disk then load from the h5 file? What do you think @james77777778 @Grvzard ?
> Is that a good tradeoff?

Generally, It should be okay to load the entire h5 into memory before loading. This is the case when saving:

1. write into memory first:
https://github.com/keras-team/keras/blob/77ef1792201cd30b52146c71dd0380786512ac84/keras/src/saving/saving_lib.py#L620-L622
2. write to disk when closing:
https://github.com/keras-team/keras/blob/77ef1792201cd30b52146c71dd0380786512ac84/keras/src/saving/saving_lib.py#L635-L638

We can also provide an option to let users decide whether to use a faster but more memory-intensive approach.

> Should we instead unzip on disk then load from the h5 file?

Actually, `h5py` doesn't recommend using file-like object. https://docs.h5py.org/en/stable/high/file.html#python-file-like-objects
So, unzipping and then loading from the H5 file might be a better approach, IMO.

> So, unzipping and then loading from the H5 file might be a better approach

Same.


diff --git a/src/accelerate/hooks.py b/src/accelerate/hooks.py
index 14d57e33661..50098eaac01 100644
--- a/src/accelerate/hooks.py
+++ b/src/accelerate/hooks.py
@@ -436,7 +436,13 @@ def attach_execution_device_hook(
         return

     for child in module.children():
-        attach_execution_device_hook(child, execution_device, skip_keys=skip_keys, tied_params_map=tied_params_map)
+        attach_execution_device_hook(
+            child,
+            execution_device,
+            skip_keys=skip_keys,
+            preload_module_classes=preload_module_classes,
+            tied_params_map=tied_params_map,
+        )

 def attach_align_device_hook(

diff --git a/tests/test_accelerator.py b/tests/test_accelerator.py
index 9b18fe5c909..651dc17da5e 100644
--- a/tests/test_accelerator.py
+++ b/tests/test_accelerator.py
@@ -30,6 +30,7 @@
 from accelerate.state import GradientState, PartialState
 from accelerate.test_utils import (
     require_bnb,
+    require_huggingface_suite,
     require_multi_gpu,
     require_non_cpu,
     require_transformer_engine,
@@ -762,3 +763,65 @@ def test_save_model_with_stateful_dataloader(self, use_safetensors, tied_weights
             assert torch.allclose(original_linear1, new_linear1)
             assert torch.allclose(original_batchnorm, new_batchnorm)
             assert torch.allclose(original_linear2, new_linear2)
+
+    @require_cuda
+    @require_huggingface_suite
+    def test_nested_hook(self, use_safetensors):
+        from transformers.modeling_utils import PretrainedConfig, PreTrainedModel
+
+        class MyLinear(torch.nn.Module):
+            def __init__(self, device=None, dtype=None):
+                factory_kwargs = {"device": device, "dtype": dtype}
+                super().__init__()
+                self.centroid = torch.nn.Embedding(1, 2)
+                self.indices = torch.nn.parameter(torch.empty((1, 2, 2), **factory_kwargs))
+
+            def forward(self, x):
+                orig_shape = x.shape
+                x = torch.abs(x + self.indices).long()
+                x = x % 2
+                x = x.sum(-1)
+                x = (self.centroid.weight + x).reshape(orig_shape)
+                return x
+
+        class MySubModel(torch.nn.Module):
+            def __init__(self):
+                super().__init__()
+                self.layer = MyLinear()
+
+            def forward(self, x):
+                return self.layer(x)
+
+        class MyModel(PreTrainedModel):
+            def __init__(self, config):
+                super().__init__(config)
+                self.layer = torch.nn.ModuleList([MySubModel() for i in range(4)])
+
+            def forward(self, x):
+                for layer in self.layer:
+                    x = layer(x)
+                return x
+
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            check_point = tmpdirname
+            offload_folder = check_point + "/offload"
+            os.makedirs(offload_folder, exist_ok=True)
+            config = PretrainedConfig()
+            m = MyModel(config)
+            m.save_pretrained(check_point)
+
+            with init_empty_weights():
+                my_model = MyModel(config)
+            my_model = load_checkpoint_and_dispatch(
+                my_model,
+                checkpoint=check_point,
+                max_memory={"cpu": 60, 0: 60},
+                device_map="auto",
+                no_split_module_classes=["MySubModel"],
+                offload_folder=offload_folder,
+                preload_module_classes=["MyLinear"],
+            )
+        # before fix, this would raise an error
+        #       weight is on the meta device, we need a `value` to put in on 0
+        x = torch.randn(1, 2)
+        my_model(x)

`preload_module_classes` is lost in attach_execution_device_hook.
### System Info

https://github.com/huggingface/accelerate/blob/3fcc9461c4fcb7228df5e5246809ba09cfbb232e/src/accelerate/hooks.py#L439
Should we pass preload_module_classes  to the next calling? such as
```
        attach_execution_device_hook(child, execution_device, tied_params_map=tied_params_map,
 preload_module_classes=preload_module_classes)
```

### Information

- [ ] The official example scripts
- [ ] My own modified scripts

### Tasks

- [ ] One of the scripts in the examples/ folder of Accelerate or an officially supported `no_trainer` script in the `examples` folder of the `transformers` repo (such as `run_no_trainer_glue.py`)
- [ ] My own task or dataset (give details below)

### Reproduction

Step into the code is enough.

### Expected behavior

Identify the bug.

Hi @muellerzr
Could you please take a look at it? Sorry if I ping the wrong contributor could you please help to recommend someone? Thanks
This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/accelerate/blob/main/CONTRIBUTING.md) are likely to be ignored.
Could anyone help to look at this issue？
This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/accelerate/blob/main/CONTRIBUTING.md) are likely to be ignored.
Hi @SunMarc

Could you please take a look at this issue when you get a time?
Thanks.
This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/accelerate/blob/main/CONTRIBUTING.md) are likely to be ignored.
Oh my bad sorry for the delay @wejoncy, I totally missed it. We indeed need to pass it. Could you open a PR to fix it ?


diff --git a/vllm/attention/backends/flashinfer.py b/vllm/attention/backends/flashinfer.py
index 107e3bbf79666..b61c660e3e280 100644
--- a/vllm/attention/backends/flashinfer.py
+++ b/vllm/attention/backends/flashinfer.py
@@ -757,9 +757,8 @@ def __init__(
         if alibi_slopes is not None:
             alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
         self.alibi_slopes = alibi_slopes
-        if sliding_window is not None:
-            raise ValueError("Sliding window is not supported in FlashInfer.")
-        self.sliding_window = (-1, -1)
+        self.sliding_window = ((sliding_window - 1,
+                                0) if sliding_window is not None else (-1, -1))
         self.kv_cache_dtype = kv_cache_dtype
         self.logits_soft_cap = logits_soft_cap

@@ -865,6 +864,8 @@ def unified_flash_infer(
     assert query.shape[0] == num_prefill_tokens
     assert decode_query.shape[0] == num_decode_tokens

+    window_left = window_size[0] if window_size is not None else -1
+
     prefill_output: Optional[torch.Tensor] = None
     decode_output: Optional[torch.Tensor] = None
     if prefill_meta := attn_metadata.prefill_metadata:
@@ -895,7 +896,8 @@ def unified_flash_infer(
                 logits_soft_cap=logits_soft_cap,
                 causal=True,
                 k_scale=k_scale,
-                v_scale=v_scale)
+                v_scale=v_scale,
+                window_left=window_left)
     if decode_meta := attn_metadata.decode_metadata:
         assert attn_metadata.decode_metadata is not None
         assert attn_metadata.decode_metadata.decode_wrapper is not None
@@ -905,7 +907,8 @@ def unified_flash_infer(
             sm_scale=softmax_scale,
             logits_soft_cap=logits_soft_cap,
             k_scale=k_scale,
-            v_scale=v_scale)
+            v_scale=v_scale,
+            window_left=window_left)

     if prefill_output is None and decode_output is not None:
         # Decode only batch.

diff --git a/tests/core/block/e2e/test_correctness_sliding_window.py b/tests/core/block/e2e/test_correctness_sliding_window.py
index 9320a9ef62314..415d0bd8237df 100644
--- a/tests/core/block/e2e/test_correctness_sliding_window.py
+++ b/tests/core/block/e2e/test_correctness_sliding_window.py
@@ -3,6 +3,7 @@

 import pytest

+from tests.kernels.utils import override_backend_env_variable
 from vllm import LLM, SamplingParams

 from .conftest import get_text_from_llm_generator
@@ -28,8 +29,9 @@
 @pytest.mark.parametrize("test_llm_kwargs", [{}])
 @pytest.mark.parametrize("batch_size", [5])
 @pytest.mark.parametrize("seed", [1])
+@pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
 def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
-                                 batch_size, seed):
+                                 batch_size, seed, backend, monkeypatch):
     """
     The test does a bunch of assignments "x1 = 10\nx2 = 33\n..." and then
     asks for value of one of them (which is outside the sliding window).
@@ -38,6 +40,8 @@ def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,

     Additionally, we compare the results of the v1 and v2 managers.
     """
+    override_backend_env_variable(monkeypatch, backend)
+
     sampling_params = SamplingParams(
         max_tokens=1024,
         ignore_eos=True,
@@ -84,7 +88,9 @@ def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
 @pytest.mark.parametrize("test_llm_kwargs", [{"enable_chunked_prefill": True}])
 @pytest.mark.parametrize("batch_size", [5])
 @pytest.mark.parametrize("seed", [1])
-def test_sliding_window_chunked_prefill(test_llm_generator, batch_size, seed):
+@pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
+def test_sliding_window_chunked_prefill(test_llm_generator, batch_size, seed,
+                                        backend, monkeypatch):
     """
     This is similar to test_sliding_window_retrival, however, it doesn't
     compare against the v1 block manager since v1 doesn't support
@@ -93,6 +99,8 @@ def test_sliding_window_chunked_prefill(test_llm_generator, batch_size, seed):
     The results with and without chunked prefill are not the same due to
     numerical instabilities.
     """
+    override_backend_env_variable(monkeypatch, backend)
+
     sampling_params = SamplingParams(
         max_tokens=10,
         ignore_eos=True,

[help wanted]: add sliding window support for flashinfer
### Anything you want to discuss about vllm.

flashinfer already supports sliding window in https://github.com/flashinfer-ai/flashinfer/issues/159 , and we should update our code and pass sliding window to flashinfer , to remove this restriction:

https://github.com/vllm-project/vllm/blob/5f8d8075f957d5376b2f1cc451e35a2a757e95a5/vllm/attention/backends/flashinfer.py#L739-L740

### Before submitting a new issue...

- [X] Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://docs.vllm.ai/en/latest/), which can answer lots of frequently asked questions.

@elfiegg I see your great contribution in https://github.com/vllm-project/vllm/pull/9781 , would you like to take this?

if anyone else is interested, contribution is extremely welcome.
Hey Kaichao - sorry for the delay! I can help take a look later this week if it's not urgent.
@elfiegg thank you!


diff --git a/vllm/spec_decode/batch_expansion.py b/vllm/spec_decode/batch_expansion.py
index 25ef27b8378f0..01b9cdad963da 100644
--- a/vllm/spec_decode/batch_expansion.py
+++ b/vllm/spec_decode/batch_expansion.py
@@ -307,28 +307,16 @@ def _create_target_seq_group_metadata(
         token_ids_to_score = self._get_token_ids_to_score(
             proposal_token_ids[batch_index])

-        # Use simpler sampling parameters apart from for final token
-        # (in particular don't do seeded sampling) since those sampled tokens
-        # aren't used.
-        # We don't replace the sampling_params in the greedy case because
-        # this also controls whether the probs get modified in the sampler
-        # (see use of _modify_greedy_probs_inplace there).
         sampling_params = input_seq_group_metadata.sampling_params
-        non_bonus_sampling_params = DEFAULT_SIMPLE_SAMPLING_PARAMS \
-            if sampling_params.temperature else sampling_params
-
         target_seq_group_metadata_list: List[SequenceGroupMetadata] = []
-        last_index = len(token_ids_to_score) - 1
         for i, token_ids in enumerate(token_ids_to_score):
-            target_sampling_params = sampling_params if i == last_index \
-                else non_bonus_sampling_params
             target_seq_group_metadata_list.append(
                 self._create_single_target_seq_group_metadata(
                     input_seq_group_metadata,
                     input_seq_id,
                     next(target_seq_ids_iter),
                     token_ids,
-                    sampling_params=target_sampling_params,
+                    sampling_params=sampling_params,
                 ))

         return target_seq_group_metadata_list

diff --git a/tests/spec_decode/e2e/test_mlp_correctness.py b/tests/spec_decode/e2e/test_mlp_correctness.py
index 5ecc0d4e95719..183ff2f5db274 100644
--- a/tests/spec_decode/e2e/test_mlp_correctness.py
+++ b/tests/spec_decode/e2e/test_mlp_correctness.py
@@ -203,7 +203,7 @@ def test_mlp_e2e_acceptance_rate(vllm_runner, common_llm_kwargs,
 @pytest.mark.parametrize("test_llm_kwargs", [{"seed": 5}])
 @pytest.mark.parametrize("output_len", [64])
 @pytest.mark.parametrize("batch_size", [1, 32])
-@pytest.mark.parametrize("temperature", [0.1, 1.0])
+@pytest.mark.parametrize("temperature", [1.0])
 @pytest.mark.parametrize("seed", [1])
 def test_mlp_e2e_seeded_correctness(vllm_runner, common_llm_kwargs,
                                     per_test_common_llm_kwargs,
diff --git a/tests/spec_decode/test_batch_expansion.py b/tests/spec_decode/test_batch_expansion.py
index 0d6aaa449d856..3504fcf43e361 100644
--- a/tests/spec_decode/test_batch_expansion.py
+++ b/tests/spec_decode/test_batch_expansion.py
@@ -90,6 +90,14 @@ def test_create_single_target_seq_group_metadata(k: int):
     )

     assert output.request_id == input_seq_group_metadata.request_id
+    assert output.sampling_params.repetition_penalty == \
+        input_seq_group_metadata.sampling_params.repetition_penalty
+    assert output.sampling_params.temperature == \
+        input_seq_group_metadata.sampling_params.temperature
+    assert output.sampling_params.top_p == \
+        input_seq_group_metadata.sampling_params.top_p
+    assert output.sampling_params.top_k == \
+        input_seq_group_metadata.sampling_params.top_k
     assert len(output.seq_data) == 1
     assert output.seq_data[target_seq_id].get_prompt_token_ids() == tuple(
         prompt_tokens)

[Bug]: Sampling parameter fixed issue while doing speculative sampling verification step
### Your current environment

The output of `python collect_env.py`

```text
Your output of `python collect_env.py` here
```

PyTorch version: 2.4.0+cu121
Is debug build: False
CUDA used to build PyTorch: 12.1
ROCM used to build PyTorch: N/A

OS: Ubuntu 22.04.3 LTS (x86_64)
GCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0
Clang version: Could not collect
CMake version: version 3.27.9
Libc version: glibc-2.35

Python version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)
Python platform: Linux-5.4.239-1.el7.elrepo.x86_64-x86_64-with-glibc2.35
Is CUDA available: True
CUDA runtime version: 12.3.107
CUDA_MODULE_LOADING set to: LAZY
GPU models and configuration:
GPU 0: NVIDIA A100-SXM4-80GB
GPU 1: NVIDIA A100-SXM4-80GB
GPU 2: NVIDIA A100-SXM4-80GB
GPU 3: NVIDIA A100-SXM4-80GB
GPU 4: NVIDIA A100-SXM4-80GB
GPU 5: NVIDIA A100-SXM4-80GB
GPU 6: NVIDIA A100-SXM4-80GB
GPU 7: NVIDIA A100-SXM4-80GB

Nvidia driver version: 535.54.03
cuDNN version: Probably one of the following:
/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.7
/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.7
/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.7
/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.7
/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.7
/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.7
/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.7
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: True

CPU:
Architecture:                    x86_64
CPU op-mode(s):                  32-bit, 64-bit
Address sizes:                   43 bits physical, 48 bits virtual
Byte Order:                      Little Endian
CPU(s):                          64
On-line CPU(s) list:             0-63
Vendor ID:                       AuthenticAMD
Model name:                      AMD EPYC 7302 16-Core Processor
CPU family:                      23
Model:                           49
Thread(s) per core:              2
Core(s) per socket:              16
Socket(s):                       2
Stepping:                        0
BogoMIPS:                        5989.21
Flags:                           fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate sme ssbd mba sev ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif umip rdpid overflow_recov succor smca
Virtualization:                  AMD-V
L1d cache:                       1 MiB (32 instances)
L1i cache:                       1 MiB (32 instances)
L2 cache:                        16 MiB (32 instances)
L3 cache:                        256 MiB (16 instances)
NUMA node(s):                    2
NUMA node0 CPU(s):               0-15,32-47
NUMA node1 CPU(s):               16-31,48-63
Vulnerability Itlb multihit:     Not affected
Vulnerability L1tf:              Not affected
Vulnerability Mds:               Not affected
Vulnerability Meltdown:          Not affected
Vulnerability Mmio stale data:   Not affected
Vulnerability Retbleed:          Vulnerable
Vulnerability Spec store bypass: Vulnerable
Vulnerability Spectre v1:        Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers
Vulnerability Spectre v2:        Vulnerable, IBPB: disabled, STIBP: disabled, PBRSB-eIBRS: Not affected
Vulnerability Srbds:             Not affected
Vulnerability Tsx async abort:   Not affected

Versions of relevant libraries:
[pip3] numpy==1.24.4
[pip3] nvidia-cublas-cu12==12.1.3.1
[pip3] nvidia-cuda-cupti-cu12==12.1.105
[pip3] nvidia-cuda-nvrtc-cu12==12.1.105
[pip3] nvidia-cuda-runtime-cu12==12.1.105
[pip3] nvidia-cudnn-cu12==9.1.0.70
[pip3] nvidia-cufft-cu12==11.0.2.54
[pip3] nvidia-curand-cu12==10.3.2.106
[pip3] nvidia-cusolver-cu12==11.4.5.107
[pip3] nvidia-cusparse-cu12==12.1.0.106
[pip3] nvidia-dali-cuda120==1.32.0
[pip3] nvidia-ml-py==12.560.30
[pip3] nvidia-nccl-cu12==2.20.5
[pip3] nvidia-nvjitlink-cu12==12.6.77
[pip3] nvidia-nvtx-cu12==12.1.105
[pip3] nvidia-pyindex==1.0.9
[pip3] onnx==1.15.0rc2
[pip3] optree==0.10.0
[pip3] pynvml==11.4.1
[pip3] pytorch-quantization==2.1.2
[pip3] pyzmq==25.1.2
[pip3] torch==2.4.0
[pip3] torch-tensorrt==2.2.0a0
[pip3] torchdata==0.7.0a0
[pip3] torchtext==0.17.0a0
[pip3] torchvision==0.19.0
[pip3] transformers==4.39.3
[pip3] triton==3.0.0
[conda] Could not collect
ROCM Version: Could not collect
Neuron SDK Version: N/A
vLLM Version: 0.5.4
vLLM Build Flags:
CUDA Archs: 5.2 6.0 6.1 7.0 7.2 7.5 8.0 8.6 8.7 9.0+PTX; ROCm: Disabled; Neuron: Disabled
GPU Topology:
GPU0    GPU1    GPU2    GPU3    GPU4    GPU5    GPU6    GPU7    NIC0    CPU Affinity    NUMA Affinity   GPU NUMA ID
GPU0     X      NV12    NV12    NV12    NV12    NV12    NV12    NV12    NODE    0-15,32-47      0         N/A
GPU1    NV12     X      NV12    NV12    NV12    NV12    NV12    NV12    NODE    0-15,32-47      0         N/A
GPU2    NV12    NV12     X      NV12    NV12    NV12    NV12    NV12    PXB     0-15,32-47      0         N/A
GPU3    NV12    NV12    NV12     X      NV12    NV12    NV12    NV12    PXB     0-15,32-47      0         N/A
GPU4    NV12    NV12    NV12    NV12     X      NV12    NV12    NV12    SYS     16-31,48-63     1         N/A
GPU5    NV12    NV12    NV12    NV12    NV12     X      NV12    NV12    SYS     16-31,48-63     1         N/A
GPU6    NV12    NV12    NV12    NV12    NV12    NV12     X      NV12    SYS     16-31,48-63     1         N/A
GPU7    NV12    NV12    NV12    NV12    NV12    NV12    NV12     X      SYS     16-31,48-63     1         N/A
NIC0    NODE    NODE    PXB     PXB     SYS     SYS     SYS     SYS      X

Legend:

  X    = Self
  SYS  = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)
  NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node
  PHB  = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)
  PXB  = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)
  PIX  = Connection traversing at most a single PCIe bridge
  NV#  = Connection traversing a bonded set of # NVLinks

NIC Legend:

  NIC0: mlx5_0

### Model Input Dumps

_No response_

### 🐛 Describe the bug

**Example code**

1. server
`python vllm/vllm/entrypoints/openai/api_server.py -tp 4 --model meta-llama/Llama-2-70b-hf --port 8000 --gpu-memory-utilization 0.8 --trust-remote-code --speculative-model meta-llama/Llama-2-7b-hf --use-v2-block-manager --num_speculative_tokens 1`

2. completion
```
openai_api_base = "http://0.0.0.0:8041/v1"
client = OpenAI(base_url=openai_api_base,)

models = client.models.list()
model = models.data[0].id

completion = client.completions.create(
        model=model,
        prompt="Hello, my name is",
        echo=False,
        n=1,
        stream=False,
        temperature=0.6,
        top_p=0.6,
        max_tokens=128,
)
print(completion)
```

**Bug**
- Setting
  I am running **speculative sampling** with num_speculative_tokens = 1.
In this setting, verification with target model should sample two logits in parallel.

- Problem
  - The bug here is that when I print out the top_p value of sampling parameters,
**the first top_p is always fixed to 1,**
which affects the acceptance of the token.
(FYI, same with the example above, I used top_p as 0.6)
  -

  -  It is also same when the num_speculative_tokens value is different.
 The first top_p value is always fixed to 1.

 - The first top_p value should also be set to the specific sampling parameter value that is given as input.

- Test setting
  - sampling param: top_p = 0.9 / temperature = 0.6
  - Target model / Draft model : LLaMA3-70B / LLaMA3-8B

- As-Is
  - sampling param: top_p = 0.9
  - Speculative metrics: Draft acceptance rate: 0.651, System efficiency: 0.825, Number of speculative tokens: 1, Number of accepted tokens: 13496, Number of draft tokens: 20747, Number of emitted tokens: 34243.
  - Draft probs are filtered, and target probs are not filtered when comparing the probabilities.
  -

- Modified
  -  Speculative metrics: Draft acceptance rate: 0.819, System efficiency: 0.910, Number of speculative tokens: 1, Number of accepted tokens: 15311, Number of draft tokens: 18694, Number of emitted tokens: 34005.

   - Both draft probs and target probs are filtered
   -

### Before submitting a new issue...

- [X] Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://docs.vllm.ai/en/latest/), which can answer lots of frequently asked questions.


diff --git a/examples/offline_inference_vision_language.py b/examples/offline_inference_vision_language.py
index 83d2548a506e4..60cdb186331fe 100644
--- a/examples/offline_inference_vision_language.py
+++ b/examples/offline_inference_vision_language.py
@@ -262,10 +262,9 @@ def run_qwen2_vl(question: str, modality: str):

     model_name = "Qwen/Qwen2-VL-7B-Instruct"

-    # Tested on L40
     llm = LLM(
         model=model_name,
-        max_model_len=8192,
+        max_model_len=4096,
         max_num_seqs=5,
         # Note - mm_processor_kwargs can also be passed to generate/chat calls
         mm_processor_kwargs={

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index 32eed1a771718..9444dc43ea97e 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -9,6 +9,7 @@
 # label(str): the name of the test. emoji allowed.
 # fast_check(bool): whether to run this on each commit on fastcheck pipeline.
 # fast_check_only(bool): run this test on fastcheck pipeline only
+# nightly(bool): run this test in nightly pipeline only
 # optional(bool): never run this test by default (i.e. need to unblock manually)
 # command(str): the single command to run for tests. incompatible with commands.
 # commands(list): the list of commands to run for test. incompatbile with command.
@@ -330,18 +331,28 @@ steps:
   commands:
     - pytest -v -s models/decoder_only/language --ignore=models/decoder_only/language/test_models.py --ignore=models/decoder_only/language/test_big_models.py

-- label: Decoder-only Multi-Modal Models Test # 1h31min
+- label: Decoder-only Multi-Modal Models Test (Standard)
   #mirror_hardwares: [amd]
   source_file_dependencies:
   - vllm/
   - tests/models/decoder_only/audio_language
   - tests/models/decoder_only/vision_language
   commands:
-    - pytest -v -s models/decoder_only/audio_language
+    - pytest -v -s models/decoder_only/audio_language -m core_model
+    - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m core_model
+
+- label: Decoder-only Multi-Modal Models Test (Extended)
+  nightly: true
+  source_file_dependencies:
+  - vllm/
+  - tests/models/decoder_only/audio_language
+  - tests/models/decoder_only/vision_language
+  commands:
+    - pytest -v -s models/decoder_only/audio_language -m 'not core_model'
     # HACK - run phi3v tests separately to sidestep this transformers bug
     # https://github.com/huggingface/transformers/issues/34307
     - pytest -v -s models/decoder_only/vision_language/test_phi3v.py
-    - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language
+    - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model'

 - label: Other Models Test # 6min
   #mirror_hardwares: [amd]
diff --git a/tests/models/decoder_only/audio_language/test_ultravox.py b/tests/models/decoder_only/audio_language/test_ultravox.py
index ad6c2d854d1f0..b9089e75ffab8 100644
--- a/tests/models/decoder_only/audio_language/test_ultravox.py
+++ b/tests/models/decoder_only/audio_language/test_ultravox.py
@@ -158,6 +158,7 @@ def run_multi_audio_test(
     assert all(tokens for tokens, *_ in vllm_outputs)

+@pytest.mark.core_model
 @pytest.mark.parametrize("dtype", ["half"])
 @pytest.mark.parametrize("max_tokens", [128])
 @pytest.mark.parametrize("num_logprobs", [5])
@@ -178,6 +179,7 @@ def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
     )

+@pytest.mark.core_model
 @pytest.mark.parametrize("dtype", ["half"])
 @pytest.mark.parametrize("max_tokens", [128])
 @pytest.mark.parametrize("num_logprobs", [5])
diff --git a/tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py b/tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py
index 5c90e7f7a267c..c23fbedf0c6ae 100644
--- a/tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py
+++ b/tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py
@@ -17,7 +17,7 @@

 # Fixtures lazy import to avoid initializing CUDA during test collection
-# NOTE: Qwen2vl supports multiple input modalities, so it registers multiple
+# NOTE: Qwen2VL supports multiple input modalities, so it registers multiple
 # input mappers.
 @pytest.fixture()
 def image_input_mapper_for_qwen2_vl():
diff --git a/tests/models/decoder_only/vision_language/test_models.py b/tests/models/decoder_only/vision_language/test_models.py
index 9370527e3cd57..d738647c91b66 100644
--- a/tests/models/decoder_only/vision_language/test_models.py
+++ b/tests/models/decoder_only/vision_language/test_models.py
@@ -75,6 +75,63 @@
 # this is a good idea for checking your command first, since tests are slow.

 VLM_TEST_SETTINGS = {
+    #### Core tests to always run in the CI
+    "llava": VLMTestInfo(
+        models=["llava-hf/llava-1.5-7b-hf"],
+        test_type=(
+            VLMTestType.EMBEDDING,
+            VLMTestType.IMAGE,
+            VLMTestType.CUSTOM_INPUTS
+        ),
+        prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
+        convert_assets_to_embeddings=model_utils.get_llava_embeddings,
+        max_model_len=4096,
+        auto_cls=AutoModelForVision2Seq,
+        vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
+        custom_test_opts=[CustomTestOptions(
+            inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
+                formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:"
+            ),
+            limit_mm_per_prompt={"image": 4},
+        )],
+        marks=[pytest.mark.core_model],
+    ),
+    "paligemma": VLMTestInfo(
+        models=["google/paligemma-3b-mix-224"],
+        test_type=VLMTestType.IMAGE,
+        prompt_formatter=identity,
+        img_idx_to_prompt = lambda idx: "",
+        # Paligemma uses its own sample prompts because the default one fails
+        single_image_prompts=IMAGE_ASSETS.prompts({
+            "stop_sign": "caption es",
+            "cherry_blossom": "What is in the picture?",
+        }),
+        auto_cls=AutoModelForVision2Seq,
+        postprocess_inputs=model_utils.get_key_type_post_processor(
+            "pixel_values"
+        ),
+        vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
+        dtype="half" if current_platform.is_rocm() else ("half", "float"),
+        marks=[pytest.mark.core_model],
+    ),
+    "qwen2_vl": VLMTestInfo(
+        models=["Qwen/Qwen2-VL-2B-Instruct"],
+        test_type=(
+            VLMTestType.IMAGE,
+            VLMTestType.MULTI_IMAGE,
+            VLMTestType.VIDEO
+        ),
+        prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
+        img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
+        video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
+        max_model_len=4096,
+        max_num_seqs=2,
+        auto_cls=AutoModelForVision2Seq,
+        vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
+        marks=[pytest.mark.core_model],
+        image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
+    ),
+    #### Extended model tests
     "blip2": VLMTestInfo(
         models=["Salesforce/blip2-opt-2.7b"],
         test_type=VLMTestType.IMAGE,
@@ -151,25 +208,6 @@
         use_tokenizer_eos=True,
         patch_hf_runner=model_utils.internvl_patch_hf_runner,
     ),
-    "llava": VLMTestInfo(
-        models=["llava-hf/llava-1.5-7b-hf"],
-        test_type=(
-            VLMTestType.EMBEDDING,
-            VLMTestType.IMAGE,
-            VLMTestType.CUSTOM_INPUTS
-        ),
-        prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
-        convert_assets_to_embeddings=model_utils.get_llava_embeddings,
-        max_model_len=4096,
-        auto_cls=AutoModelForVision2Seq,
-        vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
-        custom_test_opts=[CustomTestOptions(
-            inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
-                formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:"
-            ),
-            limit_mm_per_prompt={"image": 4},
-        )],
-    ),
     "llava_next": VLMTestInfo(
         models=["llava-hf/llava-v1.6-mistral-7b-hf"],
         test_type=(VLMTestType.IMAGE, VLMTestType.CUSTOM_INPUTS),
@@ -200,12 +238,12 @@
         vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
         # Llava-one-vision tests fixed sizes & the default size factors
         image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
-        runner_mm_key="videos",
         custom_test_opts=[CustomTestOptions(
             inputs=custom_inputs.multi_video_multi_aspect_ratio_inputs(
                 formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n",   # noqa: E501
             ),
             limit_mm_per_prompt={"video": 4},
+            runner_mm_key="videos",
         )],
     ),
     # FIXME
@@ -218,9 +256,11 @@
         auto_cls=AutoModelForVision2Seq,
         vllm_output_post_proc=model_utils.llava_video_vllm_to_hf_output,
         image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
-        runner_mm_key="videos",
         marks=[
-            pytest.mark.skip(reason="LLava next video tests currently fail.")
+            pytest.mark.skipif(
+                transformers.__version__.startswith("4.46"),
+                reason="Model broken with changes in transformers 4.46"
+            )
         ],
     ),
     "minicpmv": VLMTestInfo(
@@ -234,23 +274,6 @@
         postprocess_inputs=model_utils.wrap_inputs_post_processor,
         hf_output_post_proc=model_utils.minicmpv_trunc_hf_output,
     ),
-    "paligemma": VLMTestInfo(
-        models=["google/paligemma-3b-mix-224"],
-        test_type=VLMTestType.IMAGE,
-        prompt_formatter=identity,
-        img_idx_to_prompt = lambda idx: "",
-        # Paligemma uses its own sample prompts because the default one fails
-        single_image_prompts=IMAGE_ASSETS.prompts({
-            "stop_sign": "caption es",
-            "cherry_blossom": "What is in the picture?",
-        }),
-        auto_cls=AutoModelForVision2Seq,
-        postprocess_inputs=model_utils.get_key_type_post_processor(
-            "pixel_values"
-        ),
-        vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
-        dtype="half" if current_platform.is_rocm() else ("half", "float"),
-    ),
     # Tests for phi3v currently live in another file because of a bug in
     # transformers. Once this issue is fixed, we can enable them here instead.
     # https://github.com/huggingface/transformers/issues/34307
diff --git a/tests/models/decoder_only/vision_language/vlm_utils/model_utils.py b/tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
index 6856e8df81a13..e925934db0e7c 100644
--- a/tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+++ b/tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
@@ -56,6 +56,17 @@ def qwen_vllm_to_hf_output(
     return output_ids, hf_output_str, out_logprobs

+def qwen2_vllm_to_hf_output(
+        vllm_output: RunnerOutput,
+        model: str) -> Tuple[List[int], str, Optional[SampleLogprobs]]:
+    """Sanitize vllm output [qwen2 models] to be comparable with hf output."""
+    output_ids, output_str, out_logprobs = vllm_output
+
+    hf_output_str = output_str + "<|im_end|>"
+
+    return output_ids, hf_output_str, out_logprobs
+
+
 def llava_image_vllm_to_hf_output(vllm_output: RunnerOutput,
                                   model: str) -> RunnerOutput:
     config = AutoConfig.from_pretrained(model)
diff --git a/tests/models/decoder_only/vision_language/vlm_utils/runners.py b/tests/models/decoder_only/vision_language/vlm_utils/runners.py
index 5a3f9e820dad0..2d3b39fe3594e 100644
--- a/tests/models/decoder_only/vision_language/vlm_utils/runners.py
+++ b/tests/models/decoder_only/vision_language/vlm_utils/runners.py
@@ -29,6 +29,7 @@ def run_single_image_test(*, tmp_path: PosixPath, model_test_info: VLMTestInfo,
         num_logprobs=test_case.num_logprobs,
         limit_mm_per_prompt={"image": 1},
         distributed_executor_backend=test_case.distributed_executor_backend,
+        runner_mm_key="images",
         **model_test_info.get_non_parametrized_runner_kwargs())

@@ -51,6 +52,7 @@ def run_multi_image_test(*, tmp_path: PosixPath, model_test_info: VLMTestInfo,
         num_logprobs=test_case.num_logprobs,
         limit_mm_per_prompt={"image": len(image_assets)},
         distributed_executor_backend=test_case.distributed_executor_backend,
+        runner_mm_key="images",
         **model_test_info.get_non_parametrized_runner_kwargs())

@@ -74,6 +76,7 @@ def run_embedding_test(*, model_test_info: VLMTestInfo,
         limit_mm_per_prompt={"image": 1},
         vllm_embeddings=vllm_embeddings,
         distributed_executor_backend=test_case.distributed_executor_backend,
+        runner_mm_key="images",
         **model_test_info.get_non_parametrized_runner_kwargs())

@@ -101,6 +104,7 @@ def run_video_test(
         num_logprobs=test_case.num_logprobs,
         limit_mm_per_prompt={"video": len(video_assets)},
         distributed_executor_backend=test_case.distributed_executor_backend,
+        runner_mm_key="videos",
         **model_test_info.get_non_parametrized_runner_kwargs())

@@ -115,7 +119,11 @@ def run_custom_inputs_test(*, model_test_info: VLMTestInfo,

     inputs = test_case.custom_test_opts.inputs
     limit_mm_per_prompt = test_case.custom_test_opts.limit_mm_per_prompt
-    assert inputs is not None and limit_mm_per_prompt is not None
+    runner_mm_key = test_case.custom_test_opts.runner_mm_key
+    # Inputs, limit_mm_per_prompt, and runner_mm_key should all be set
+    assert inputs is not None
+    assert limit_mm_per_prompt is not None
+    assert runner_mm_key is not None

     core.run_test(
         hf_runner=hf_runner,
@@ -127,4 +135,5 @@ def run_custom_inputs_test(*, model_test_info: VLMTestInfo,
         num_logprobs=test_case.num_logprobs,
         limit_mm_per_prompt=limit_mm_per_prompt,
         distributed_executor_backend=test_case.distributed_executor_backend,
+        runner_mm_key=runner_mm_key,
         **model_test_info.get_non_parametrized_runner_kwargs())
diff --git a/tests/models/decoder_only/vision_language/vlm_utils/types.py b/tests/models/decoder_only/vision_language/vlm_utils/types.py
index 4d18d53af30fa..fd18c7c8346f0 100644
--- a/tests/models/decoder_only/vision_language/vlm_utils/types.py
+++ b/tests/models/decoder_only/vision_language/vlm_utils/types.py
@@ -52,6 +52,8 @@ class SizeType(Enum):
 class CustomTestOptions(NamedTuple):
     inputs: List[Tuple[List[str], List[Union[List[Image], Image]]]]
     limit_mm_per_prompt: Dict[str, int]
+    # kwarg to pass multimodal data in as to vllm/hf runner instances.
+    runner_mm_key: str = "images"

 class ImageSizeWrapper(NamedTuple):
@@ -141,9 +143,6 @@ class VLMTestInfo(NamedTuple):
         Callable[[PosixPath, str, Union[List[ImageAsset], _ImageAssets]],
                  str]] = None  # noqa: E501

-    # kwarg to pass multimodal data in as to vllm/hf runner instances
-    runner_mm_key: str = "images"
-
     # Allows configuring a test to run with custom inputs
     custom_test_opts: Optional[List[CustomTestOptions]] = None

@@ -168,7 +167,6 @@ def get_non_parametrized_runner_kwargs(self):
             "get_stop_token_ids": self.get_stop_token_ids,
             "model_kwargs": self.model_kwargs,
             "patch_hf_runner": self.patch_hf_runner,
-            "runner_mm_key": self.runner_mm_key,
         }

diff --git a/tests/models/embedding/vision_language/test_llava_next.py b/tests/models/embedding/vision_language/test_llava_next.py
index a8d0ac4fc160d..9fab5898a06ba 100644
--- a/tests/models/embedding/vision_language/test_llava_next.py
+++ b/tests/models/embedding/vision_language/test_llava_next.py
@@ -2,6 +2,7 @@

 import pytest
 import torch.nn.functional as F
+import transformers
 from transformers import AutoModelForVision2Seq

 from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
@@ -85,8 +86,8 @@ def _run_test(
     )

-# FIXME
-@pytest.mark.skip(reason="LLava next embedding tests currently fail")
+@pytest.mark.skipif(transformers.__version__.startswith("4.46"),
+                    reason="Model broken with changes in transformers 4.46")
 @pytest.mark.parametrize("model", MODELS)
 @pytest.mark.parametrize("dtype", ["half"])
 def test_models_text(

[Feature]: CI - Split up "Models Test" and "Vision Language Models Test"
### 🚀 The feature, motivation and pitch

Takes 1 hour+ on CI compared to others, which take <~30 min. Thus, ends up being a bottleneck

So, should be split up similar to kernels

CC: @khluu

root for this!
This sounds great! However, we should still avoid regressions. I suggest testing a representative subset per PR and perform full testing once in a while.

Any ideas on which models we should always test?
A more concrete plan:

Let's add a pytest marker `core_model` to explicitly tag model tests to be tested in each PR. By default, unmarked model tests will only be tested daily.

Separately, we can organize the models tests as follows:
- `models/embedding/language`
- `models/encoder_decoder/language`
- `models/decoder_only/language`
- `models/decoder_only/vision`
- `models/decoder_only/audio` (once #7615 is merged)

The first three are currently included in Models test while the fourth one is currently in VLM test. They are split apart further so that model PRs can easily run only the tests that are relevant to their model.

(By the way, the models directory in main vLLM is pretty cluttered... perhaps we can also organize the model files as above?)

In the normal CI, we can add the pytest flag `-m core_model` so that only the tagged tests are run. The full CI (run daily) omits this flag to run all tests.

Regarding which model tests to mark as core_model:
- We should select one model per family of architectures.
- For multi-modal models, we should consider one model with multi-modal encoder defined by Transformers, and another one that is defined by ourselves.
- There is no need to consider LoRA, TP/PP and quantization behavior since their basic implementations are covered by other tests already. The full CI should be able to catch any regressions of those features for individual models.
Hmm, it sounds unreliable to ask users to run specified model tests... in pytorch for instance it's mandatory to run quite comprehensive test suite in every PR

I think that changing this behaviour will potentially introduce uncaught bugs.

> Separately, we can organize the models tests as follows:
>
> models/embedding/language
> models/encoder_decoder/language
> models/decoder_only/language
> models/decoder_only/vision
> models/decoder_only/audio (once https://github.com/vllm-project/vllm/pull/7615 is merged)

I think that first step of splitting up like this is a good idea.
After some offline discussion, we have decided to **cut down on the number of models to regularly test in CI**. This is due to the following reasons:

- Most of the model tests aim to test the correctness of the model implementation. However, there is no need to repeatedly test this unless the model file itself is updated. Instead, we can cover the code paths in model loader, layers, executor and runner (which are shared between models) using a relatively small number of tests.
- Some of the model PRs are from the same organization who originally developed the model. In this case, we trust the authors to implement their own model correctly.
- With the growing number of models supported by vLLM, it becomes increasingly expensive to test them all in the long run.

For encoder-decoder and embedding models, we will preserve all tests since there are only a few tests involved.

For multimodal models, @ywang96 and I have decided to only test the following models regularly in CI:

- LLaVA w/ our implementation of `CLIPEncoder`
- PaliGemma w/ our implementation of `SiglipEncoder`
- Ultravox w/ HuggingFace's implementation of `WhisperEncoder`
- (A model that supports video input, once it's implemented in vLLM. Qwen2-VL will add M-ROPE so we should test that)

Meanwhile, @simon-mo will help narrow down the list of decoder-only language models to test.

**Note that this does not remove the need to test new models.** New model PRs should still implement tests so we can verify their implementation, as well as re-run those tests whenever a future PR updates related code. Until @khluu figures out how to create a separate test item for each model without clogging up the web UI, these tests will remain excluded from CI; instead, we can run these tests locally as necessary.


diff --git a/vllm/model_executor/models/phi3v.py b/vllm/model_executor/models/phi3v.py
index 2e52531989232..4872929ec36cc 100644
--- a/vllm/model_executor/models/phi3v.py
+++ b/vllm/model_executor/models/phi3v.py
@@ -13,6 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import itertools
 import re
 from functools import lru_cache
 from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
@@ -37,11 +38,11 @@
 from vllm.model_executor.models.llama import LlamaModel
 from vllm.model_executor.sampling_metadata import SamplingMetadata
 from vllm.multimodal import MULTIMODAL_REGISTRY
-from vllm.multimodal.utils import cached_get_tokenizer
+from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
 from vllm.sequence import IntermediateTensors, SamplerOutput
+from vllm.utils import is_list_of

-from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
-                   input_processor_for_clip)
+from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
 from .interfaces import SupportsMultiModal
 from .utils import merge_multimodal_embeddings

@@ -400,9 +401,20 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
     image_data = multi_modal_data["image"]
     if isinstance(image_data, Image.Image):
         w, h = image_data.size
-        image_feature_size = get_phi3v_image_feature_size(hf_config,
-                                                          input_width=w,
-                                                          input_height=h)
+        image_feature_size = [
+            get_phi3v_image_feature_size(hf_config,
+                                         input_width=w,
+                                         input_height=h)
+        ]
+        image_data = [image_data]
+    elif is_list_of(image_data, Image.Image):
+        image_feature_size = []
+        for image in image_data:
+            w, h = image.size
+            image_feature_size.append(
+                get_phi3v_image_feature_size(hf_config,
+                                             input_width=w,
+                                             input_height=h))
     elif isinstance(image_data, torch.Tensor):
         image_feature_size = image_data.shape[0]
     else:
@@ -410,45 +422,61 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):

     prompt = llm_inputs.get("prompt")
     if prompt is None:
+        image_idx = []
         new_prompt = None
     else:
+        image_idx = sorted(map(int, re.findall(r"<\|image_(\d+)\|>+", prompt)))
         if prompt.count("<|image|>") > 0:
             logger.warning("Please follow the prompt format that is "
                            "documented on HuggingFace which does not involve "
                            "repeating <|image|> tokens.")
-        elif len(re.findall(r"(<\|image_\d+\|>)+", prompt)) > 1:
-            logger.warning("Multiple image input is not supported yet, "
-                           "so any extra image tokens will be treated "
-                           "as plain text.")
-
+        elif (num_image_tags := len(image_idx)) > 1:
+            assert num_image_tags == len(
+                image_data), "The count of image_placeholder not match image's"
         new_prompt = prompt

-    prompt_token_ids = llm_inputs["prompt_token_ids"]
-    image_1_token_ids = _get_image_placeholder_token_ids(model_config, idx=1)
+    prompt_token_ids = llm_inputs["prompt_token_ids"].copy()
+
+    # masked place_holder with image token id
+    for idx in image_idx:
+        image_token_ids = _get_image_placeholder_token_ids(model_config,
+                                                           idx=idx)
+        for i in range(len(prompt_token_ids) - len(image_token_ids) + 1):
+            if prompt_token_ids[i:i + len(image_token_ids)] == image_token_ids:
+                prompt_token_ids[i:i + len(image_token_ids)] = [
+                    _IMAGE_TOKEN_ID
+                ] * len(image_token_ids)
+                break
+
+    # merge consecutive tag ids
+    merged_token_ids: List[int] = []
+    for is_placeholder, token_ids in itertools.groupby(
+            prompt_token_ids, lambda x: x == _IMAGE_TOKEN_ID):
+        if is_placeholder:
+            merged_token_ids.append(_IMAGE_TOKEN_ID)
+        else:
+            merged_token_ids.extend(list(token_ids))

+    # TODO: Move this to utils or integrate with clip.
     new_token_ids: List[int] = []
-    for i in range(len(prompt_token_ids) - len(image_1_token_ids) + 1):
-        if prompt_token_ids[i:i + len(image_1_token_ids)] == image_1_token_ids:
-            new_token_ids.append(_IMAGE_TOKEN_ID)
-
-            # No need to further scan the list since we only replace once
-            new_token_ids.extend(prompt_token_ids[i + len(image_1_token_ids):])
-            break
+    placeholder_idx = 0
+    while merged_token_ids:
+        token_id = merged_token_ids.pop(0)
+        if token_id == _IMAGE_TOKEN_ID:
+            new_token_ids.extend(
+                repeat_and_pad_token(
+                    _IMAGE_TOKEN_ID,
+                    repeat_count=image_feature_size[placeholder_idx],
+                ))
+            placeholder_idx += 1
         else:
-            new_token_ids.append(prompt_token_ids[i])
+            new_token_ids.append(token_id)

     # NOTE: Create a defensive copy of the original inputs
     llm_inputs = LLMInputs(prompt_token_ids=new_token_ids,
                            prompt=new_prompt,
                            multi_modal_data=multi_modal_data)
-
-    return input_processor_for_clip(
-        model_config,
-        CLIP_VIT_LARGE_PATCH14_336_CONFIG,
-        llm_inputs,
-        image_token_id=_IMAGE_TOKEN_ID,
-        image_feature_size_override=image_feature_size,
-    )
+    return llm_inputs

 @MULTIMODAL_REGISTRY.register_image_input_mapper()

diff --git a/tests/models/test_phi3v.py b/tests/models/test_phi3v.py
index 40829785d3214..259cbe515066d 100644
--- a/tests/models/test_phi3v.py
+++ b/tests/models/test_phi3v.py
@@ -21,6 +21,7 @@
     "cherry_blossom":
     "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n",
 })
+HF_MULTIIMAGE_IMAGE_PROMPT = "<|user|>\n<|image_1|>\n<|image_2|>\nDescribe these images.<|end|>\n<|assistant|>\n"  # noqa: E501

 models = ["microsoft/Phi-3.5-vision-instruct"]

@@ -184,3 +185,113 @@ def test_regression_7840(hf_runner, vllm_runner, image_assets, model,
         num_logprobs=10,
         tensor_parallel_size=1,
     )
+
+
+def run_multi_image_test(
+    hf_runner: Type[HfRunner],
+    vllm_runner: Type[VllmRunner],
+    images: List[Image.Image],
+    model: str,
+    *,
+    size_factors: List[float],
+    dtype: str,
+    max_tokens: int,
+    num_logprobs: int,
+    tensor_parallel_size: int,
+    distributed_executor_backend: Optional[str] = None,
+):
+    """Inference result should be the same between hf and vllm.
+
+    All the image fixtures for the test is under tests/images.
+    For huggingface runner, we provide the PIL images as input.
+    For vllm runner, we provide MultiModalDataDict objects
+    and corresponding MultiModalConfig as input.
+    Note, the text input is also adjusted to abide by vllm contract.
+    The text output is sanitized to be able to compare with hf.
+    """
+
+    inputs_per_case = [
+        ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
+         [[rescale_image_size(image, factor) for image in images]
+          for factor in size_factors])
+    ]
+
+    # NOTE: take care of the order. run vLLM first, and then run HF.
+    # vLLM needs a fresh new process without cuda initialization.
+    # if we run HF first, the cuda initialization will be done and it
+    # will hurt multiprocessing backend with fork method (the default method).
+
+    # max_model_len should be greater than image_feature_size
+    with vllm_runner(model,
+                     max_model_len=4096,
+                     max_num_seqs=1,
+                     limit_mm_per_prompt={"image": len(images)},
+                     dtype=dtype,
+                     tensor_parallel_size=tensor_parallel_size,
+                     distributed_executor_backend=distributed_executor_backend,
+                     enforce_eager=True) as vllm_model:
+        vllm_outputs_per_case = [
+            vllm_model.generate_greedy_logprobs(prompts,
+                                                max_tokens,
+                                                num_logprobs=num_logprobs,
+                                                images=images)
+            for prompts, images in inputs_per_case
+        ]
+
+    hf_model_kwargs = {"_attn_implementation": "eager"}
+    with hf_runner(model, dtype=dtype,
+                   model_kwargs=hf_model_kwargs) as hf_model:
+        eos_token_id = hf_model.processor.tokenizer.eos_token_id
+        hf_outputs_per_case = [
+            hf_model.generate_greedy_logprobs_limit(prompts,
+                                                    max_tokens,
+                                                    num_logprobs=num_logprobs,
+                                                    images=images,
+                                                    eos_token_id=eos_token_id)
+            for prompts, images in inputs_per_case
+        ]
+
+    for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
+                                        vllm_outputs_per_case):
+        check_logprobs_close(
+            outputs_0_lst=hf_outputs,
+            outputs_1_lst=[
+                vllm_to_hf_output(vllm_output, model)
+                for vllm_output in vllm_outputs
+            ],
+            name_0="hf",
+            name_1="vllm",
+        )
+
+
+@pytest.mark.parametrize("model", models)
+@pytest.mark.parametrize(
+    "size_factors",
+    [
+        # No image
+        [],
+        # Single-scale
+        [1.0],
+        # Single-scale, batched
+        [1.0, 1.0, 1.0],
+        # Multi-scale
+        [0.25, 0.5, 1.0],
+    ],
+)
+@pytest.mark.parametrize("dtype", [target_dtype])
+@pytest.mark.parametrize("max_tokens", [128])
+@pytest.mark.parametrize("num_logprobs", [5])
+def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
+                             size_factors, dtype: str, max_tokens: int,
+                             num_logprobs: int) -> None:
+    run_multi_image_test(
+        hf_runner,
+        vllm_runner,
+        [asset.pil_image for asset in image_assets],
+        model,
+        size_factors=size_factors,
+        dtype=dtype,
+        max_tokens=max_tokens,
+        num_logprobs=num_logprobs,
+        tensor_parallel_size=1,
+    )

[Feature]: phi-3.5 is a strong model for its size, including vision support.  Has multi-image support, but vllm does not support
### 🚀 The feature, motivation and pitch

phi-3.5 is a strong model for its size, including strong multi-image vision support.   But vllm does not support the multi-image case.

https://github.com/vllm-project/vllm/blob/03b7bfb79b1edf54511fd1b12acc9a875cee5656/vllm/model_executor/models/phi3v.py#L421-L425

### Alternatives

Only other models

### Additional context

_No response_


diff --git a/pydantic/_internal/_generate_schema.py b/pydantic/_internal/_generate_schema.py
index bebe58c6c0..c9bceddb99 100644
--- a/pydantic/_internal/_generate_schema.py
+++ b/pydantic/_internal/_generate_schema.py
@@ -1892,34 +1892,27 @@ def _call_schema(self, function: ValidateCallSupportedTypes) -> core_schema.Call
         )

     def _unsubstituted_typevar_schema(self, typevar: typing.TypeVar) -> core_schema.CoreSchema:
-        assert isinstance(typevar, typing.TypeVar)
-
-        bound = typevar.__bound__
-        constraints = typevar.__constraints__
-
         try:
-            typevar_has_default = typevar.has_default()  # type: ignore
+            has_default = typevar.has_default()
         except AttributeError:
-            # could still have a default if it's an old version of typing_extensions.TypeVar
-            typevar_has_default = getattr(typevar, '__default__', None) is not None
+            # Happens if using `typing.TypeVar` on Python < 3.13
+            pass
+        else:
+            if has_default:
+                return self.generate_schema(typevar.__default__)

-        if (bound is not None) + (len(constraints) != 0) + typevar_has_default > 1:
-            raise NotImplementedError(
-                'Pydantic does not support mixing more than one of TypeVar bounds, constraints and defaults'
-            )
+        if constraints := typevar.__constraints__:
+            return self._union_schema(typing.Union[constraints])

-        if typevar_has_default:
-            return self.generate_schema(typevar.__default__)  # type: ignore
-        elif constraints:
-            return self._union_schema(typing.Union[constraints])  # type: ignore
-        elif bound:
+        if bound := typevar.__bound__:
             schema = self.generate_schema(bound)
             schema['serialization'] = core_schema.wrap_serializer_function_ser_schema(
-                lambda x, h: h(x), schema=core_schema.any_schema()
+                lambda x, h: h(x),
+                schema=core_schema.any_schema(),
             )
             return schema
-        else:
-            return core_schema.any_schema()
+
+        return core_schema.any_schema()

     def _computed_field_schema(
         self,

diff --git a/tests/test_generics.py b/tests/test_generics.py
index 2e11d871f8..ea92e7aba7 100644
--- a/tests/test_generics.py
+++ b/tests/test_generics.py
@@ -2836,14 +2836,93 @@ class MyErrorDetails(ErrorDetails):
     }

-def test_mix_default_and_constraints() -> None:
-    T = TypingExtensionsTypeVar('T', str, int, default=str)
+def test_serialize_typevars_default_and_bound_with_user_model() -> None:
+    class MyErrorDetails(BaseModel):
+        bar: str
+
+    class ExtendedMyErrorDetails(MyErrorDetails):
+        foo: str

-    msg = 'Pydantic does not support mixing more than one of TypeVar bounds, constraints and defaults'
-    with pytest.raises(NotImplementedError, match=msg):
+    class MoreExtendedMyErrorDetails(ExtendedMyErrorDetails):
+        suu: str

-        class _(BaseModel, Generic[T]):
-            x: T
+    T = TypingExtensionsTypeVar('T', bound=MyErrorDetails, default=ExtendedMyErrorDetails)
+
+    class Error(BaseModel, Generic[T]):
+        message: str
+        details: T
+
+    # bound small parent model
+    sample_error = Error[MyErrorDetails](
+        message='We just had an error',
+        details=MyErrorDetails(foo='var', bar='baz', suu='suu'),
+    )
+
+    assert sample_error.details.model_dump() == {
+        'bar': 'baz',
+    }
+    assert sample_error.model_dump() == {
+        'message': 'We just had an error',
+        'details': {
+            'bar': 'baz',
+        },
+    }
+
+    # default middle child model
+    sample_error = Error(
+        message='We just had an error',
+        details=MoreExtendedMyErrorDetails(foo='var', bar='baz', suu='suu'),
+    )
+
+    assert sample_error.details.model_dump() == {
+        'foo': 'var',
+        'bar': 'baz',
+        'suu': 'suu',
+    }
+    assert sample_error.model_dump() == {
+        'message': 'We just had an error',
+        'details': {'foo': 'var', 'bar': 'baz'},
+    }
+
+    # bound big child model
+    sample_error = Error[MoreExtendedMyErrorDetails](
+        message='We just had an error',
+        details=MoreExtendedMyErrorDetails(foo='var', bar='baz', suu='suu'),
+    )
+
+    assert sample_error.details.model_dump() == {
+        'foo': 'var',
+        'bar': 'baz',
+        'suu': 'suu',
+    }
+    assert sample_error.model_dump() == {
+        'message': 'We just had an error',
+        'details': {
+            'foo': 'var',
+            'bar': 'baz',
+            'suu': 'suu',
+        },
+    }
+
+
+def test_typevars_default_model_validation_error() -> None:
+    class MyErrorDetails(BaseModel):
+        bar: str
+
+    class ExtendedMyErrorDetails(MyErrorDetails):
+        foo: str
+
+    T = TypingExtensionsTypeVar('T', bound=MyErrorDetails, default=ExtendedMyErrorDetails)
+
+    class Error(BaseModel, Generic[T]):
+        message: str
+        details: T
+
+    with pytest.raises(ValidationError):
+        Error(
+            message='We just had an error',
+            details=MyErrorDetails(foo='var', bar='baz'),
+        )

 def test_generic_with_not_required_in_typed_dict() -> None:

Support TypeVar with Identical Bound and Default
### Initial Checks

- [X] I have searched Google & GitHub for similar requests and couldn't find anything
- [X] I have read and followed [the docs](https://docs.pydantic.dev) and still think this feature is missing

### Description

# The Problem

Currently Pydantic [does not support](https://github.com/pydantic/pydantic/blob/07b64739a251760a71082324e358c9d8291b7ab6/pydantic/_internal/_generate_schema.py#L1667) `TypeVar` with more than one of a bound, default, or constraints declared. This is a problem because users are forced to choose between better usability or greater correctness. As described in the [motivations](https://peps.python.org/pep-0696/#motivation) for the PEP which added `TypeVar` defaults declaring both a bound a default is relatively common:

```python
from typing_extensions import TypeVar

T = TypeVar("T", bound=int, default=int)

def f(x: T = 1) -> T:
    return x

f("hello")  # error because str not subtype of int
```

If we're forced to only declare a `default` we won't get appropriate type warnings:

```python
from typing_extensions import TypeVar

T = TypeVar("T", default=int)

def f(x: T = 1) -> T:
    return x

f("hello")  # no error
```

# Partial Solution

After inspecting the downstream code which [handles bounds and defaults](https://github.com/pydantic/pydantic/blob/07b64739a251760a71082324e358c9d8291b7ab6/pydantic/_internal/_generate_schema.py#L1670-L1681), it appears that it would relatively easy to at least support the case of a `TypeVar` with an identical bound and default declared. From my anecdotal experience, this likely covers the majority of cases where someone might want to specify both a bound and a default.

Supporting this would involve rewriting:

```python
        if (bound is not None) + (len(constraints) != 0) + (default is not None) > 1:
            raise NotImplementedError(
                'Pydantic does not support mixing more than one of TypeVar bounds, constraints and defaults'
            )

        if default is not None:
            return self.generate_schema(default)
        elif constraints:
            return self._union_schema(typing.Union[constraints])  # type: ignore
        elif bound:
            schema = self.generate_schema(bound)
            schema['serialization'] = core_schema.wrap_serializer_function_ser_schema(
                lambda x, h: h(x), schema=core_schema.any_schema()
            )
            return schema
        else:
            return core_schema.any_schema()
```

To look more like:

```python
        if len(constraints) != 0:
            if (bound is not None or default is not None):
                raise RuntimeError(...)
            return self._union_schema(typing.Union[constraints])
        if bound is not None and default is not None and bound != default:
            raise RuntimeError(...)
        else:
            schema = generate_schema(bound or default)
            if bound is not None:
                schema['serialization'] = core_schema.wrap_serializer_function_ser_schema(
                    lambda x, h: h(x), schema=core_schema.any_schema()
                )
            return schema
```

The main question I have is how feasible it is to actually check `bound != default`. Besides that, this seems fairly sound.

### Affected Components

- [ ] [Compatibility between releases](https://docs.pydantic.dev/changelog/)
- [X] [Data validation/parsing](https://docs.pydantic.dev/concepts/models/#basic-model-usage)
- [ ] [Data serialization](https://docs.pydantic.dev/concepts/serialization/) - `.model_dump()` and `.model_dump_json()`
- [ ] [JSON Schema](https://docs.pydantic.dev/concepts/json_schema/)
- [ ] [Dataclasses](https://docs.pydantic.dev/concepts/dataclasses/)
- [ ] [Model Config](https://docs.pydantic.dev/concepts/config/)
- [ ] [Field Types](https://docs.pydantic.dev/api/types/) - adding or changing a particular data type
- [ ] [Function validation decorator](https://docs.pydantic.dev/concepts/validation_decorator/)
- [X] [Generic Models](https://docs.pydantic.dev/concepts/models/#generic-models)
- [ ] [Other Model behaviour](https://docs.pydantic.dev/concepts/models/) - `model_construct()`, pickling, private attributes, ORM mode
- [ ] [Plugins](https://docs.pydantic.dev/) and integration with other tools - mypy, FastAPI, python-devtools, Hypothesis, VS Code, PyCharm, etc.

@rmorshea,

Thanks for the feature request. I've added the label `change` here as well.

At a first glance, your approach sounds good. Feel free to open a PR with the changes, and I can do a more thorough review! We'll want to test this pretty thoroughly before we introduce the change.
I have a similar case - a fix would be most helpful!
```python
from pydantic import BaseModel
from pint import Quantity
from pydantic_pint import PydanticPintQuantity
from typing_extensions import Generic

DimensionlessQuantity = Annotated[Quantity, PydanticPintQuantity("dimensionless", strict=False)]
QuantityType = TypeVar("QuantityType", bound=Quantity, default=DimensionlessQuantity)

class GenericModel(BaseModel, Generic[QuantityType]):
    some_field: QuantityType
```
voting up for this feature 👍
I have quite complext structure with many generics, so pydantic supporting default would be most helpful :)

awaiting author revision
relnotes-feature