diff --git a/src/diffusers/pipelines/pipeline_utils.py b/src/diffusers/pipelines/pipeline_utils.py
index a504184ea2f2..c505c5a262a3 100644
--- a/src/diffusers/pipelines/pipeline_utils.py
+++ b/src/diffusers/pipelines/pipeline_utils.py
@@ -13,6 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import enum
 import fnmatch
 import importlib
 import inspect
@@ -811,6 +812,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P
         # in this case they are already instantiated in `kwargs`
         # extract them here
         expected_modules, optional_kwargs = cls._get_signature_keys(pipeline_class)
+        expected_types = pipeline_class._get_signature_types()
         passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs}
         passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs}
         init_dict, unused_kwargs, _ = pipeline_class.extract_init_dict(config_dict, **kwargs)
@@ -833,6 +835,26 @@ def load_module(name, value):

         init_dict = {k: v for k, v in init_dict.items() if load_module(k, v)}

+        for key in init_dict.keys():
+            if key not in passed_class_obj:
+                continue
+            if "scheduler" in key:
+                continue
+
+            class_obj = passed_class_obj[key]
+            _expected_class_types = []
+            for expected_type in expected_types[key]:
+                if isinstance(expected_type, enum.EnumMeta):
+                    _expected_class_types.extend(expected_type.__members__.keys())
+                else:
+                    _expected_class_types.append(expected_type.__name__)
+
+            _is_valid_type = class_obj.__class__.__name__ in _expected_class_types
+            if not _is_valid_type:
+                logger.warning(
+                    f"Expected types for {key}: {_expected_class_types}, got {class_obj.__class__.__name__}."
+                )
+
         # Special case: safety_checker must be loaded separately when using `from_flax`
         if from_flax and "safety_checker" in init_dict and "safety_checker" not in passed_class_obj:
             raise NotImplementedError(

diff --git a/tests/pipelines/test_pipelines.py b/tests/pipelines/test_pipelines.py
index 43b01c40f5bb..423c82e0602e 100644
--- a/tests/pipelines/test_pipelines.py
+++ b/tests/pipelines/test_pipelines.py
@@ -1802,6 +1802,16 @@ def test_pipe_same_device_id_offload(self):
         sd.maybe_free_model_hooks()
         assert sd._offload_gpu_id == 5

+    def test_wrong_model(self):
+        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")
+        with self.assertRaises(ValueError) as error_context:
+            _ = StableDiffusionPipeline.from_pretrained(
+                "hf-internal-testing/diffusers-stable-diffusion-tiny-all", text_encoder=tokenizer
+            )
+
+        assert "is of type" in str(error_context.exception)
+        assert "but should be" in str(error_context.exception)
+

 @slow
 @require_torch_gpu

[pipelines] add better checking when a wrong model is passed when initializing a pipeline
With this code:

```py
from diffusers import DiffusionPipeline
from transformers import T5EncoderModel
import torch

repo_id = "black-forest-labs/FLUX.1-dev"
text_encoder_2 = T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_2")
pipe = DiffusionPipeline.from_pretrained(
    repo_id, text_encoder=text_encoder_2, torch_dtype=torch.bfloat16
).to("cuda")

image = pipe("A dog").images[0]
```

where we're doing `text_encoder=text_encoder_2` (should have been `text_encoder_2=text_encoder_2`), we get an extremely uninformative error:

<details>
<summary>Error</summary>

```bash
...
... [51,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
../aten/src/ATen/native/cuda/Indexing.cu:1308: indexSelectLargeIndex: block: [95,0,0], thread: [52,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
../aten/src/ATen/native/cuda/Indexing.cu:1308: indexSelectLargeIndex: block: [95,0,0], thread: [53,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
../aten/src/ATen/native/cuda/Indexing.cu:1308: indexSelectLargeIndex: block: [95,0,0], thread: [54,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
../aten/src/ATen/native/cuda/Indexing.cu:1308: indexSelectLargeIndex: block: [95,0,0], thread: [55,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
../aten/src/ATen/native/cuda/Indexing.cu:1308: indexSelectLargeIndex: block: [95,0,0], thread: [56,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
../aten/src/ATen/native/cuda/Indexing.cu:1308: indexSelectLargeIndex: block: [95,0,0], thread: [57,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
../aten/src/ATen/native/cuda/Indexing.cu:1308: indexSelectLargeIndex: block: [95,0,0], thread: [58,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
../aten/src/ATen/native/cuda/Indexing.cu:1308: indexSelectLargeIndex: block: [95,0,0], thread: [59,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
../aten/src/ATen/native/cuda/Indexing.cu:1308: indexSelectLargeIndex: block: [95,0,0], thread: [60,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
../aten/src/ATen/native/cuda/Indexing.cu:1308: indexSelectLargeIndex: block: [95,0,0], thread: [61,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
../aten/src/ATen/native/cuda/Indexing.cu:1308: indexSelectLargeIndex: block: [95,0,0], thread: [62,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
../aten/src/ATen/native/cuda/Indexing.cu:1308: indexSelectLargeIndex: block: [95,0,0], thread: [63,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
Traceback (most recent call last):
  File "/home/sayak/diffusers/check_incompatible_flux.py", line 11, in <module>
    image = pipe("A dog").images[0]
  File "/home/sayak/.pyenv/versions/3.10.12/envs/diffusers/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
  File "/home/sayak/diffusers/src/diffusers/pipelines/flux/pipeline_flux.py", line 677, in __call__
    ) = self.encode_prompt(
  File "/home/sayak/diffusers/src/diffusers/pipelines/flux/pipeline_flux.py", line 353, in encode_prompt
    pooled_prompt_embeds = self._get_clip_prompt_embeds(
  File "/home/sayak/diffusers/src/diffusers/pipelines/flux/pipeline_flux.py", line 289, in _get_clip_prompt_embeds
    prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False)
  File "/home/sayak/.pyenv/versions/3.10.12/envs/diffusers/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/sayak/.pyenv/versions/3.10.12/envs/diffusers/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/sayak/.pyenv/versions/3.10.12/envs/diffusers/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1972, in forward
    encoder_outputs = self.encoder(
  File "/home/sayak/.pyenv/versions/3.10.12/envs/diffusers/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/sayak/.pyenv/versions/3.10.12/envs/diffusers/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/sayak/.pyenv/versions/3.10.12/envs/diffusers/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1029, in forward
    attention_mask = torch.ones(batch_size, mask_seq_length, device=inputs_embeds.device)
RuntimeError: CUDA error: device-side assert triggered
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
```

</details>

I know it's on the users to ensure that they're initializing a pipeline properly with the right components. However, the thrown error makes it worse to do any further debugging on the user side. IMO.

It would be greatly helpful from the perspective of a user if we could throw a better warning/error instead. As soon as we detect wrong types (I think `model_index.json` info can be used in this case), we throw a warning/error. Is this something doable?

Cc: @DN6 @yiyixuxu
Ccing @hlky as well in case you're interested.

@sayakpaul
1. adding a `validate_pipeline()` method in pipeline loading class if not validated throw simple yet understandable errors like these not sure how good these are:
```
`ValueError: Mismatched model type for component 'text_encoder_2'.
Expected types: ['T5EncoderModel'], but got: 'T5Model'
```
Ensure the component matches the expected type as specified in the `model_index.json``

2. While using `model_index.json() `If a mismatch is detected, throw a clear and actionable error, if strict validation isn’t ideal, show warnings instead of hard errors.


diff --git a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py
index 96d53663b867..e7345ad96624 100644
--- a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py
+++ b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py
@@ -13,7 +13,7 @@
 # limitations under the License.

 import inspect
-from typing import Callable, Dict, List, Optional, Union
+from typing import Any, Callable, Dict, List, Optional, Union

 import PIL.Image
 import torch
@@ -25,7 +25,7 @@
 )

 from ...image_processor import PipelineImageInput, VaeImageProcessor
-from ...loaders import SD3LoraLoaderMixin
+from ...loaders import FromSingleFileMixin, SD3LoraLoaderMixin
 from ...models.autoencoders import AutoencoderKL
 from ...models.transformers import SD3Transformer2DModel
 from ...schedulers import FlowMatchEulerDiscreteScheduler
@@ -149,7 +149,7 @@ def retrieve_timesteps(
     return timesteps, num_inference_steps

-class StableDiffusion3Img2ImgPipeline(DiffusionPipeline):
+class StableDiffusion3Img2ImgPipeline(DiffusionPipeline, SD3LoraLoaderMixin, FromSingleFileMixin):
     r"""
     Args:
         transformer ([`SD3Transformer2DModel`]):
@@ -680,6 +680,10 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt
     def guidance_scale(self):
         return self._guidance_scale

+    @property
+    def joint_attention_kwargs(self):
+        return self._joint_attention_kwargs
+
     @property
     def clip_skip(self):
         return self._clip_skip
@@ -723,6 +727,7 @@ def __call__(
         negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
         output_type: Optional[str] = "pil",
         return_dict: bool = True,
+        joint_attention_kwargs: Optional[Dict[str, Any]] = None,
         clip_skip: Optional[int] = None,
         callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
         callback_on_step_end_tensor_inputs: List[str] = ["latents"],
@@ -797,6 +802,10 @@ def __call__(
             return_dict (`bool`, *optional*, defaults to `True`):
                 Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead
                 of a plain tuple.
+            joint_attention_kwargs (`dict`, *optional*):
+               A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
+                `self.processor` in
+                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
             callback_on_step_end (`Callable`, *optional*):
                 A function that calls at the end of each denoising steps during the inference. The function is called
                 with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
@@ -835,6 +844,7 @@ def __call__(

         self._guidance_scale = guidance_scale
         self._clip_skip = clip_skip
+        self._joint_attention_kwargs = joint_attention_kwargs
         self._interrupt = False

         # 2. Define call parameters
@@ -847,6 +857,10 @@ def __call__(

         device = self._execution_device

+        lora_scale = (
+            self.joint_attention_kwargs.get("scale", None) if self.joint_attention_kwargs is not None else None
+        )
+
         (
             prompt_embeds,
             negative_prompt_embeds,
@@ -868,6 +882,7 @@ def __call__(
             clip_skip=self.clip_skip,
             num_images_per_prompt=num_images_per_prompt,
             max_sequence_length=max_sequence_length,
+            lora_scale=lora_scale,
         )

         if self.do_classifier_free_guidance:
@@ -912,6 +927,7 @@ def __call__(
                     timestep=timestep,
                     encoder_hidden_states=prompt_embeds,
                     pooled_projections=pooled_prompt_embeds,
+                    joint_attention_kwargs=self.joint_attention_kwargs,
                     return_dict=False,
                 )[0]

diff --git a/tests/lora/test_lora_layers_sd3.py b/tests/lora/test_lora_layers_sd3.py
index 8f61c95c2fc8..78d4b786d21b 100644
--- a/tests/lora/test_lora_layers_sd3.py
+++ b/tests/lora/test_lora_layers_sd3.py
@@ -12,13 +12,29 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import gc
 import sys
 import unittest

+import numpy as np
+import torch
 from transformers import AutoTokenizer, CLIPTextModelWithProjection, CLIPTokenizer, T5EncoderModel

-from diffusers import FlowMatchEulerDiscreteScheduler, SD3Transformer2DModel, StableDiffusion3Pipeline
-from diffusers.utils.testing_utils import is_peft_available, require_peft_backend, require_torch_gpu, torch_device
+from diffusers import (
+    FlowMatchEulerDiscreteScheduler,
+    SD3Transformer2DModel,
+    StableDiffusion3Img2ImgPipeline,
+    StableDiffusion3Pipeline,
+)
+from diffusers.utils import load_image
+from diffusers.utils.import_utils import is_accelerate_available
+from diffusers.utils.testing_utils import (
+    is_peft_available,
+    numpy_cosine_similarity_distance,
+    require_peft_backend,
+    require_torch_gpu,
+    torch_device,
+)

 if is_peft_available():
@@ -29,6 +45,10 @@
 from utils import PeftLoraLoaderMixinTests  # noqa: E402

+if is_accelerate_available():
+    from accelerate.utils import release_memory
+
+
 @require_peft_backend
 class SD3LoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
     pipeline_class = StableDiffusion3Pipeline
@@ -108,3 +128,88 @@ def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(se
     @unittest.skip("Not supported in SD3.")
     def test_modify_padding_mode(self):
         pass
+
+
+@require_torch_gpu
+@require_peft_backend
+class LoraSD3IntegrationTests(unittest.TestCase):
+    pipeline_class = StableDiffusion3Img2ImgPipeline
+    repo_id = "stabilityai/stable-diffusion-3-medium-diffusers"
+
+    def setUp(self):
+        super().setUp()
+        gc.collect()
+        torch.cuda.empty_cache()
+
+    def tearDown(self):
+        super().tearDown()
+        gc.collect()
+        torch.cuda.empty_cache()
+
+    def get_inputs(self, device, seed=0):
+        init_image = load_image(
+            "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png"
+        )
+        if str(device).startswith("mps"):
+            generator = torch.manual_seed(seed)
+        else:
+            generator = torch.Generator(device="cpu").manual_seed(seed)
+
+        return {
+            "prompt": "corgi",
+            "num_inference_steps": 2,
+            "guidance_scale": 5.0,
+            "output_type": "np",
+            "generator": generator,
+            "image": init_image,
+        }
+
+    def test_sd3_img2img_lora(self):
+        pipe = self.pipeline_class.from_pretrained(self.repo_id, torch_dtype=torch.float16)
+        pipe.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors")
+        pipe.enable_sequential_cpu_offload()
+
+        inputs = self.get_inputs(torch_device)
+
+        image = pipe(**inputs).images[0]
+        image_slice = image[0, :10, :10]
+        expected_slice = np.array(
+            [
+                0.47827148,
+                0.5,
+                0.71972656,
+                0.3955078,
+                0.4194336,
+                0.69628906,
+                0.37036133,
+                0.40820312,
+                0.6923828,
+                0.36450195,
+                0.40429688,
+                0.6904297,
+                0.35595703,
+                0.39257812,
+                0.68652344,
+                0.35498047,
+                0.3984375,
+                0.68310547,
+                0.34716797,
+                0.3996582,
+                0.6855469,
+                0.3388672,
+                0.3959961,
+                0.6816406,
+                0.34033203,
+                0.40429688,
+                0.6845703,
+                0.34228516,
+                0.4086914,
+                0.6870117,
+            ]
+        )
+
+        max_diff = numpy_cosine_similarity_distance(expected_slice.flatten(), image_slice.flatten())
+
+        assert max_diff < 1e-4, f"Outputs are not close enough, got {max_diff}"
+        pipe.unload_lora_weights()
+        release_memory(pipe)

add load_lora_weights to  pipeline_stable_diffusion_3_img2img.py
**Is your feature request related to a problem? Please describe.*"
Yes, there is no effective way to load lora weights with SD3

A clear and concise description of what the problem is. Ex. I'm always frustrated when [...].
I can't get LoRa weights to load despite the fact there is a thorough and easy way to train them

**Describe the solution you'd like.**
the function load_lora_weights() should be added to pipeline_stable_diffusion_3_img2img.py

**Describe alternatives you've considered.**
A clear and concise description of any alternative solutions or features you've considered.
Any sort of work around using another form of loader or model merging etc.

**Additional context.**
Add any other context or screenshots about the feature request here.
https://huggingface.co/Mujeeb603/SD3-medium-Geometry-Diagrams-Lora-1

Would you like to open a PR for this?

It should be as simple as deriving from the lora loader class and supporting the lora parameters in the pipeline I believe. Relevant bits:

https://github.com/huggingface/diffusers/blob/31058cdaef63ca660a1a045281d156239fba8192/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py#L131

https://github.com/huggingface/diffusers/blob/31058cdaef63ca660a1a045281d156239fba8192/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py#L338

https://github.com/huggingface/diffusers/blob/31058cdaef63ca660a1a045281d156239fba8192/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py#L826

https://github.com/huggingface/diffusers/blob/31058cdaef63ca660a1a045281d156239fba8192/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py#L850

LMK if you need any help if you'd like to open the PR!
Yes, thank you, I would like to open a PR for this.
I'm still learning what the correct etiquette is for Github.
I suppose the approach here would be to fork diffusers and then copy in the relevant code bits?
I'm still trying to diagnose the problem. Gemini insists it is an environment problem. I'm running my code in Google colab with full updates so I can't see how that is possible.


diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 9071395e3d9..3b4f932a014 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -38,9 +38,7 @@ def add_pre_hook(self, request: Request, results: TestResult) -> Request:
             assert cb is not None

             @wraps(cb)
-            def wrapper(  # pylint: disable=inconsistent-return-statements
-                response: Response, **cb_kwargs: Any
-            ) -> list[Any]:
+            def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]:
                 try:
                     results.startTest(self.testcase_pre)
                     self.pre_process(response)
@@ -51,13 +49,10 @@ def wrapper(  # pylint: disable=inconsistent-return-statements
                     results.addError(self.testcase_pre, sys.exc_info())
                 else:
                     results.addSuccess(self.testcase_pre)
-                finally:
-                    cb_result = cb(response, **cb_kwargs)
-                    if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
-                        raise TypeError("Contracts don't support async callbacks")
-                    return list(  # pylint: disable=return-in-finally
-                        cast(Iterable[Any], iterate_spider_output(cb_result))
-                    )
+                cb_result = cb(response, **cb_kwargs)
+                if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+                    raise TypeError("Contracts don't support async callbacks")
+                return list(cast(Iterable[Any], iterate_spider_output(cb_result)))

             request.callback = wrapper

@@ -69,9 +64,7 @@ def add_post_hook(self, request: Request, results: TestResult) -> Request:
             assert cb is not None

             @wraps(cb)
-            def wrapper(  # pylint: disable=inconsistent-return-statements
-                response: Response, **cb_kwargs: Any
-            ) -> list[Any]:
+            def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]:
                 cb_result = cb(response, **cb_kwargs)
                 if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
                     raise TypeError("Contracts don't support async callbacks")
@@ -86,8 +79,7 @@ def wrapper(  # pylint: disable=inconsistent-return-statements
                     results.addError(self.testcase_post, sys.exc_info())
                 else:
                     results.addSuccess(self.testcase_post)
-                finally:
-                    return output  # pylint: disable=return-in-finally
+                return output

             request.callback = wrapper

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index d578b3af450..b0cb92d12d9 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -556,3 +556,61 @@ def test_inherited_contracts(self):

         requests = self.conman.from_spider(spider, self.results)
         self.assertTrue(requests)
+
+
+class CustomFailContractPreProcess(Contract):
+    name = "test_contract"
+
+    def pre_process(self, response):
+        raise KeyboardInterrupt("Pre-process exception")
+
+
+class CustomFailContractPostProcess(Contract):
+    name = "test_contract"
+
+    def post_process(self, response):
+        raise KeyboardInterrupt("Post-process exception")
+
+
+class CustomContractPrePostProcess(unittest.TestCase):
+
+    def setUp(self):
+        self.results = TextTestResult(stream=None, descriptions=False, verbosity=0)
+
+    def test_pre_hook_keyboard_interrupt(self):
+        spider = TestSpider()
+        response = ResponseMock()
+        contract = CustomFailContractPreProcess(spider.returns_request)
+        conman = ContractsManager([contract])
+
+        try:
+            request = conman.from_method(spider.returns_request, self.results)
+            contract.add_pre_hook(request, self.results)
+            # Expect this to raise a KeyboardInterrupt
+            request.callback(response, **request.cb_kwargs)
+        except KeyboardInterrupt as e:
+            self.assertEqual(str(e), "Pre-process exception")
+        else:
+            self.fail("KeyboardInterrupt not raised")
+
+        self.assertFalse(self.results.failures)
+        self.assertFalse(self.results.errors)
+
+    def test_post_hook_keyboard_interrupt(self):
+        spider = TestSpider()
+        response = ResponseMock()
+        contract = CustomFailContractPostProcess(spider.returns_request)
+        conman = ContractsManager([contract])
+
+        try:
+            request = conman.from_method(spider.returns_request, self.results)
+            contract.add_post_hook(request, self.results)
+            # Expect this to raise a KeyboardInterrupt
+            request.callback(response, **request.cb_kwargs)
+        except KeyboardInterrupt as e:
+            self.assertEqual(str(e), "Post-process exception")
+        else:
+            self.fail("KeyboardInterrupt not raised")
+
+        self.assertFalse(self.results.failures)
+        self.assertFalse(self.results.errors)

return in finally can swallow exception
### Description

There are two places in `scrapy/contracts/__init__.py` where a `finally:` body contains a `return` statement, which would swallow any in-flight exception.

This means that if a `BaseException` (such as `KeyboardInterrupt`) is raised from the body, or any exception is raised from one of the `except:` clause, it will not propagate on as expected.

The pylint warning about this was suppressed in [this commit](https://github.com/scrapy/scrapy/commit/991121fa91aee4d428ae09e75427d4e91970a41b) but it doesn't seem like there was justification for that.

These are the two locations:
https://github.com/scrapy/scrapy/blame/b4bad97eae6bcce790a626d244c63589f4134408/scrapy/contracts/__init__.py#L56

https://github.com/scrapy/scrapy/blame/b4bad97eae6bcce790a626d244c63589f4134408/scrapy/contracts/__init__.py#L86

See also https://docs.python.org/3/tutorial/errors.html#defining-clean-up-actions.

> If the finally clause executes a [break](https://docs.python.org/3/reference/simple_stmts.html#break), [continue](https://docs.python.org/3/reference/simple_stmts.html#continue) or [return](https://docs.python.org/3/reference/simple_stmts.html#return) statement, exceptions are not re-raised.

TIL
Hey,
What about re-raising the issue with a general raise statement in every except block along with putting the return statement outside the finally block?
If this solution seems promising, I'd like to contribute to the same. I would appreciate your insights.
I don't remember why I silenced them :-/
Is this issue still open?
@divyranjan17 it is.

@AdityaS8804 I don't think that makes sense to me.
From a quick look, it seems to me that simply taking the return part out of finally, and put it after the try-except blocks, could be all that’s needed. But we should also first write tests that detect the issues before we address those issues.
> From a quick look, it seems to me that simply taking the return part out of finally, and put it after the try-except blocks, could be all that’s needed.

This matches my first impression.
Is there a way to reproduce a this failure?
> Is there a way to reproduce a this failure?

For the first issue, for example, it seems like raising `KeyboardInterrupt` from an implementation of https://docs.scrapy.org/en/2.11/topics/contracts.html#scrapy.contracts.Contract.pre_process should see that exception raise, but will instead silence it.

I can think of 3 ways to tackle this issue

#### Solution 1: Using a Temporary Variable for Return Value
We can capture the callback result in a variable outside the `finally` block and then return it at the end of the function. By avoiding `return` inside `finally`, exceptions propagate naturally, allowing errors to be handled as expected.

**Simply:**

- Store the callback output in a variable (e.g., `cb_result`).
- Avoid using `return` in the `finally` block.
- Return `cb_result` at the end of the function, outside of any `try/finally` structure.

```python
cb_result = None
try:
    cb_result = cb(response, **cb_kwargs)
finally:
    pass  # Any final cleanup here
return list(iterate_spider_output(cb_result))
```

#### Solution 2: Separating Error Logging and Result Processing
- Create a helper function, `log_results()`, to handle logging outcomes.
- Call `log_results()` within `try/except` blocks to process success or errors.
- Return `cb_result` outside of the `try` block without `finally`.

```python
def log_results(testcase, exception_info=None):
    if exception_info:
        # Log failure
```

#### Solution 3: Wrapping Return Values with Exception Handling
- Define `process_result` to manage callback outputs while capturing exceptions.
- Invoke `process_result` instead of a direct return in the callback.
- Ensure all exception info is handled without using a return in `finally`.

```python
def process_result(cb, response, **cb_kwargs):
    try:
        cb_result = cb(response, **cb_kwargs)
    except Exception as exc:
        log_error(exc)
    return list(iterate_spider_output(cb_result))
```

Is this AI-generated?
yes Sol.1 and Sol.3 were suggested by github copilot
That's unfortunate, especially as the way forward was already suggested earlier.
[Here](https://github.com/scrapy/scrapy/issues/6505#issuecomment-2434584223), specifically.
> [Here](https://github.com/scrapy/scrapy/issues/6505#issuecomment-2434584223), specifically.

Understood


diff --git a/instructor/retry.py b/instructor/retry.py
index 205c7843f..853a73a54 100644
--- a/instructor/retry.py
+++ b/instructor/retry.py
@@ -73,7 +73,7 @@ def initialize_usage(mode: Mode) -> CompletionUsage | Any:
     """
     total_usage = CompletionUsage(completion_tokens=0, prompt_tokens=0, total_tokens=0,
         completion_tokens_details = CompletionTokensDetails(audio_tokens=0, reasoning_tokens=0),
-        prompt_token_details = PromptTokensDetails(audio_tokens=0, cached_tokens=0)
+        prompt_tokens_details = PromptTokensDetails(audio_tokens=0, cached_tokens=0)
     )
     if mode in {Mode.ANTHROPIC_TOOLS, Mode.ANTHROPIC_JSON}:
         from anthropic.types import Usage as AnthropicUsage

prompt_tokens_details not being populated correctly in response model
- [ - ] This is actually a bug report.
- [ ] I am not getting good LLM Results
- [ - ] I have tried asking for help in the community on discord or discussions and have not received a response.
- [ - ] I have tried searching the documentation and have not found an answer.

**What Model are you using?**

- [ -] gpt-3.5-turbo
- [ -] gpt-4-turbo
- [- ] gpt-4
- [ ] Other (please specify)

**Describe the bug**
I'm unable to see how many tokens were cached when I try to use client.chat.completions.create_with_completion. completion.usage.prompt_token_details is always completion.usage.prompt_token_details

**To Reproduce**
```python
import instructor
from time import sleep
from pydantic import BaseModel, field_validator
from openai import OpenAI
class FakeClass(BaseModel):
    insight: str
client = instructor.from_openai(OpenAI(api_key=OPENAI_API_KEY))
fake_str = "This is a Test string" * 1000

for x in range(0, 3):
    sleep(1)
    insight, completion = client.chat.completions.create_with_completion(
        model="gpt-4o",
        messages=[
            {
                "role": "system",
                "content": fake_str,
            },
            {"role": "user", "content": "Test User Prompt"},
        ],
        max_retries=1,
        response_model=FakeClass,
    )

    print(completion.usage.prompt_token_details.model_dump())
```

I get this:
```python
{'audio_tokens': 0, 'cached_tokens': 0}
```

**Expected behavior**
I expect to see how many token have been cached. I know OpenAI is caching it since I tried it without wrapping it around Instructor
```python

open_ai_client = OpenAI(api_key=OPENAI_API_KEY)
open_ai_client.chat.completions.create

for x in range(0, 3):
    sleep(1)
    chat_completion = open_ai_client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {
                "role": "system",
                "content": fake_str,
            },
            {"role": "user", "content": "Test User Prompt"},
        ],
    )

    cached_tokens = chat_completion.usage.prompt_tokens_details.cached_tokens
    if cached_tokens == 0:
        print("No cached tokens")
    else:
        print(f"Tokens used: {cached_tokens}")
        print(chat_completion.usage.prompt_tokens_details.model_dump())
```
I get this output
```
Tokens used: 4864
{'audio_tokens': 0, 'cached_tokens': 4864}
````

**Screenshots**
If applicable, add screenshots to help explain your problem.

This is happening due to a typo in https://github.com/instructor-ai/instructor/blob/bc7a7b3265669ca8266722bb92a437a6da64b66c/instructor/retry.py#L76

Its currently:
```python
 total_usage = CompletionUsage(completion_tokens=0, prompt_tokens=0, total_tokens=0,
        completion_tokens_details = CompletionTokensDetails(audio_tokens=0, reasoning_tokens=0),
        prompt_token_details = PromptTokensDetails(audio_tokens=0, cached_tokens=0)
    )
```

Its supposed to be:
```python
total_usage = CompletionUsage(completion_tokens=0, prompt_tokens=0, total_tokens=0,
        completion_tokens_details = CompletionTokensDetails(audio_tokens=0, reasoning_tokens=0),
        prompt_tokens_details = PromptTokensDetails(audio_tokens=0, cached_tokens=0)
    )
```

notice the typo in **prompt_token_details** vs **prompt_tokens_details**
I have tested in my local and it fixes it.

Supporting docs from OpenAI: https://github.com/openai/openai-python/blob/6e1161bc3ed20eef070063ddd5ac52fd9a531e88/src/openai/types/completion_usage.py#L53


diff --git a/prowler/providers/aws/services/rds/rds_instance_no_public_access/rds_instance_no_public_access.py b/prowler/providers/aws/services/rds/rds_instance_no_public_access/rds_instance_no_public_access.py
index ca0e1b3bebe..dffa850693c 100644
--- a/prowler/providers/aws/services/rds/rds_instance_no_public_access/rds_instance_no_public_access.py
+++ b/prowler/providers/aws/services/rds/rds_instance_no_public_access/rds_instance_no_public_access.py
@@ -37,18 +37,21 @@ def execute(self):
                                     ):
                                         report.status_extended = f"RDS Instance {db_instance.id} is set as publicly accessible and security group {security_group.name} ({security_group.id}) has {db_instance.engine} port {db_instance_port} open to the Internet at endpoint {db_instance.endpoint.get('Address')} but is not in a public subnet."
                                         public_sg = True
+                                        if db_instance.subnet_ids:
+                                            for subnet_id in db_instance.subnet_ids:
+                                                if (
+                                                    subnet_id in vpc_client.vpc_subnets
+                                                    and vpc_client.vpc_subnets[
+                                                        subnet_id
+                                                    ].public
+                                                ):
+                                                    report.status = "FAIL"
+                                                    report.status_extended = f"RDS Instance {db_instance.id} is set as publicly accessible and security group {security_group.name} ({security_group.id}) has {db_instance.engine} port {db_instance_port} open to the Internet at endpoint {db_instance.endpoint.get('Address')} in a public subnet {subnet_id}."
+                                                    break
+                                    if public_sg:
                                         break
                             if public_sg:
                                 break
-                        if db_instance.subnet_ids:
-                            for subnet_id in db_instance.subnet_ids:
-                                if (
-                                    subnet_id in vpc_client.vpc_subnets
-                                    and vpc_client.vpc_subnets[subnet_id].public
-                                ):
-                                    report.status = "FAIL"
-                                    report.status_extended = f"RDS Instance {db_instance.id} is set as publicly accessible and security group {security_group.name} ({security_group.id}) has {db_instance.engine} port {db_instance_port} open to the Internet at endpoint {db_instance.endpoint.get('Address')} in a public subnet {subnet_id}."
-                                    break

             findings.append(report)

rds_instance_no_public_access reports incorrect SG
### Steps to Reproduce

1. prowler aws -c rds_instance_no_public_access
2. AWS
3. N/A
4. See finding's `status_detail`

### Expected behavior

The status detail should correctly identify the Security Group(s) that contain the rule(s)

### Actual Result with Screenshots or Logs

Only one group is in the detail, and it may not be the group that actually has the rule(s). Result may change between runs
![rds_check_correct](https://github.com/user-attachments/assets/743e8e8f-cd97-40f1-8e7e-e85f94c1eb1d)
![rds_check_incorrect](https://github.com/user-attachments/assets/fe7dd840-a636-4ff5-b19f-ef8a6eac5bfc)

### How did you install Prowler?

From pip package (pip install prowler)

### Environment Resource

ECS/Locally

### OS used

Amazon Linux 2/WSL

### Prowler version

Prowler 4.5.3 (You are running the latest version, yay!)

### Pip version

pip 22.0.2 from /usr/lib/python3/dist-packages/pip (python 3.10)

### Context

No changes made between scan runs.
`Default` SG is attached but has no rules.

Looks like in code the IF Statement will `break ` on the first instance of port=DB PORT (3306) and source=ANY and will not report on any other Security groups. Its either public or its not. I suspect that the check doesnt matter how many SGs allow access from ANY, its the fact the RDS instance is publicly available flags it as a fail.

Thanks for the ping! @jmanduca-psfy I'll review your issue with the team.
Thanks @garym-krrv for the insight too, that looks strange 🤔


diff --git a/prowler/providers/aws/services/rds/rds_service.py b/prowler/providers/aws/services/rds/rds_service.py
index b4dd5d66ce9..60d24b1cb1b 100644
--- a/prowler/providers/aws/services/rds/rds_service.py
+++ b/prowler/providers/aws/services/rds/rds_service.py
@@ -446,7 +446,7 @@ def _describe_db_event_subscriptions(self, regional_client):
                                     arn=arn,
                                     sns_topic_arn=event["SnsTopicArn"],
                                     status=event["Status"],
-                                    source_type=event["SourceType"],
+                                    source_type=event.get("SourceType", ""),
                                     source_id=event.get("SourceIdsList", []),
                                     event_list=event.get("EventCategoriesList", []),
                                     enabled=event["Enabled"],

rds_service.py error
### Steps to Reproduce

1. Running prowler aws -log-level ERROR
2. AWS
3. Single Account

### Expected behavior

To execute without throwing the error

### Actual Result with Screenshots or Logs

2024-11-28 10:20:18,851 [File: rds_service.py:458] [Module: rds_service] ERROR: eu-west-2 -- KeyError[449]: 'SourceType'

We have an postgress RDS 16.3 instance in eu-west-2 and the script is throwing this error

### How did you install Prowler?

From pip package (pip install prowler)

### Environment Resource

2. ECS Fargate task

### OS used

3. Alpine Linux

### Prowler version

4.6.0

### Pip version

na

### Context

_No response_


diff --git a/prowler/providers/aws/services/ec2/lib/instance.py b/prowler/providers/aws/services/ec2/lib/instance.py
index 150d37db52e..bef05ad7c50 100644
--- a/prowler/providers/aws/services/ec2/lib/instance.py
+++ b/prowler/providers/aws/services/ec2/lib/instance.py
@@ -1,3 +1,4 @@
+from prowler.lib.check.models import Severity
 from prowler.providers.aws.services.ec2.ec2_service import Instance
 from prowler.providers.aws.services.vpc.vpc_service import VpcSubnet

@@ -15,13 +16,13 @@ def get_instance_public_status(
         tuple: The status and severity of the instance status.
     """
     status = f"Instance {instance.id} has {service} exposed to 0.0.0.0/0 but with no public IP address."
-    severity = "medium"
+    severity = Severity.medium

     if instance.public_ip:
         status = f"Instance {instance.id} has {service} exposed to 0.0.0.0/0 on public IP address {instance.public_ip} but it is placed in a private subnet {instance.subnet_id}."
-        severity = "high"
+        severity = Severity.high
         if vpc_subnets[instance.subnet_id].public:
             status = f"Instance {instance.id} has {service} exposed to 0.0.0.0/0 on public IP address {instance.public_ip} in public subnet {instance.subnet_id}."
-            severity = "critical"
+            severity = Severity.critical

     return status, severity

Prowler CSV Output no longer outputs Failed Critical Findings
### Steps to Reproduce

Running prowler 4.5.3
Execute prowler aws against an AWS account.
CSV Outputs PASS critical findings but does output FAIL findings.

Prowler Dashboard does not display any cirtical findings

It was working with 4.3.5 but after I upgraded to 4.5.3 this now fails.

json-ocsf files are fine. These do contain Critical Fail

### Expected behavior

Failed critical findings to be contained in CSV file to use with prowler dashboard

### Actual Result with Screenshots or Logs

prowler aws --output-bucket-no-assume prowler-output-bucket-XXXXXXXXXXXX --output-directory output --slack --role arn:aws:iam::XXXXXXXXXXXX:role/XXXXXXXXXXX --role-session-name XXXXX_XXXXXX

### How did you install Prowler?

From pip package (pip install prowler)

### Environment Resource

2. Fargate Task running python:alpine with python 3.12

### OS used

3. Alpine Linux

### Prowler version

4.5.3

### Pip version

3.12

### Context

_No response_

Hello! @garym-krrv Could you add the `--log-level ERROR` flag to your Prowler execution to see if we can find logs that indicate what the error might be?
Will do.. let me get back to you @pedrooot

Many Thanks


diff --git a/prowler/providers/kubernetes/services/core/core_seccomp_profile_docker_default/core_seccomp_profile_docker_default.py b/prowler/providers/kubernetes/services/core/core_seccomp_profile_docker_default/core_seccomp_profile_docker_default.py
index 71c4a7f28af..20a9c16b2fb 100644
--- a/prowler/providers/kubernetes/services/core/core_seccomp_profile_docker_default/core_seccomp_profile_docker_default.py
+++ b/prowler/providers/kubernetes/services/core/core_seccomp_profile_docker_default/core_seccomp_profile_docker_default.py
@@ -10,18 +10,38 @@ def execute(self) -> Check_Report_Kubernetes:
             report.namespace = pod.namespace
             report.resource_name = pod.name
             report.resource_id = pod.uid
-            if (
+
+            pod_seccomp_correct = (
                 pod.security_context
                 and pod.security_context.seccomp_profile
                 and pod.security_context.seccomp_profile.type == "RuntimeDefault"
-            ):
+            )
+            containers_seccomp_correct = True
+
+            # Check container-level seccomp profile
+            for container in pod.containers.values():
+                if not (
+                    container.security_context
+                    and container.security_context.seccomp_profile
+                    and container.security_context.seccomp_profile.type
+                    == "RuntimeDefault"
+                ):
+                    containers_seccomp_correct = False
+                    break
+
+            # Determine the report status
+            if pod_seccomp_correct or containers_seccomp_correct:
                 report.status = "PASS"
-                report.status_extended = (
-                    f"Pod {pod.name} has docker/default seccomp profile enabled."
-                )
+                report.status_extended = f"Pod {pod.name} and its containers have docker/default seccomp profile enabled."
             else:
                 report.status = "FAIL"
-                report.status_extended = f"Pod {pod.name} does not have docker/default seccomp profile enabled."
+                if not pod_seccomp_correct and not containers_seccomp_correct:
+                    report.status_extended = f"Pod {pod.name} does not have docker/default seccomp profile enabled at both pod and container levels."
+                elif not pod_seccomp_correct:
+                    report.status_extended = f"Pod {pod.name} does not have docker/default seccomp profile enabled at pod level."
+                else:
+                    report.status_extended = f"Pod {pod.name} does not have docker/default seccomp profile enabled at container level."
+
             findings.append(report)

         return findings

CIS 1.8 5.7.2 fails even though seccomp profile is set to runTimeDefault
### Steps to Reproduce

1. Create a Pod with securityContext.seccompProfile.type: RunTimeDefault
2. Run Prowler
3. Prowler will complain on "Pod <pod name> does not have docker/default seccomp profile enabled."

### Expected behavior

CIS 1.8 5.7.2 states "Remediation: Use securityContext to enable the docker/default seccomp profile in your pod definitions. An example is as below: securityContext: seccompProfile: type: RuntimeDefault"

From the Talos documentation (from version 1.2.0 and above): Talos now runs Kubelet with the CRI default Seccomp Profile enabled.
This can be disabled by setting .machine.kubelet.defaultRuntimeSeccompProfileEnabled to false.

I've checked that its set to true on all nodes.

### Actual Result with Screenshots or Logs

pod manifest
```
kubectl -n kube-system get pod kube-apiserver-controlplane1 -ojsonpath='{.spec.containers[*].securityContext}'

{"allowPrivilegeEscalation":false,"capabilities":{"add":["NET_BIND_SERVICE"],"drop":["ALL"]},"seccompProfile":{"type":"RuntimeDefault"}}
```

prowler json
```
"message": "Pod kube-apiserver-controlplane1 does not have docker/default seccomp profile enabled.",
    "status_detail": "Pod kube-apiserver-controlplane1 does not have docker/default seccomp profile enabled.",
```

### How did you install Prowler?

From brew (brew install prowler)

### Environment Resource

Workstation

### OS used

macOS 15.1

### Prowler version

4.5.3

### Pip version

24.3.1

### Context

_No response_

Hey! @misterdohl thanks for the ping. Requirement `5.7.2` is associated with the check `core_seccomp_profile_docker_default`. Based on the information you’ve provided, I can see that a modification will be needed to improve it. I’ll let you know as soon as the change is made!


diff --git a/prowler/providers/aws/services/wafv2/wafv2_service.py b/prowler/providers/aws/services/wafv2/wafv2_service.py
index 4971f1c4223..85feed76f62 100644
--- a/prowler/providers/aws/services/wafv2/wafv2_service.py
+++ b/prowler/providers/aws/services/wafv2/wafv2_service.py
@@ -99,7 +99,7 @@ def _get_logging_configuration(self, acl):
     def _list_resources_for_web_acl(self, acl):
         logger.info("WAFv2 - Describing resources...")
         try:
-            if acl.scope == Scope.REGIONAL or acl.region in self.regional_clients:
+            if acl.scope == Scope.REGIONAL:
                 for resource in self.regional_clients[
                     acl.region
                 ].list_resources_for_web_acl(

Invalid WAF arn
### Steps to Reproduce

Run prowler on Docker against an account with global WAF enabled.

### Expected behavior

To pass

### Actual Result with Screenshots or Logs

2024-11-18 10:09:43,933 [File: wafv2_service.py:122]    [Module: wafv2_service]  ERROR: us-east-1 -- WAFInvalidParameterException[105]: An error occurred (WAFInvalidParameterException) when calling the ListResourcesForWebACL operation: Error reason: The ARN isn't valid. A valid ARN begins with arn: and includes other information separated by colons or slashes., field: RESOURCE_ARN, parameter: arn:aws:wafv2:us-east-1:xxxxx:global/webacl/xxx/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxxx

2024-11-18 10:09:43,934 [File: wafv2_service.py:122]    [Module: wafv2_service]  ERROR: us-east-1 -- WAFInvalidParameterException[105]: An error occurred (WAFInvalidParameterException) when calling the ListResourcesForWebACL operation: Error reason: The ARN isn't valid. A valid ARN begins with arn: and includes other information separated by colons or slashes., field: RESOURCE_ARN, parameter: arn:aws:wafv2:us-east-1:xxxxx:global/webacl/xxx/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxxx

2024-11-18 10:09:43,938 [File: wafv2_service.py:122]    [Module: wafv2_service]  ERROR: us-east-1 -- WAFInvalidParameterException[105]: An error occurred (WAFInvalidParameterException) when calling the ListResourcesForWebACL operation: Error reason: The ARN isn't valid. A valid ARN begins with arn: and includes other information separated by colons or slashes., field: RESOURCE_ARN, parameter: arn:aws:wafv2:us-east-1:xxxxx:global/webacl/xxx/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxxx

### How did you install Prowler?

Docker (docker pull toniblyx/prowler)

### Environment Resource

3. Docker container locally

### OS used

One inside docker

### Prowler version

4.5.3

### Pip version

one inside docker image

### Context

We have three global WAF, and three regional WAF. The three regional works, and the three global throws an malformed arn error.

Thank you very much for using Prowler and for reporting this issue.

We are reviewing it and will provide you with a solution as soon as possible! 🚀


diff --git a/prowler/providers/common/provider.py b/prowler/providers/common/provider.py
index 3681cdcc262..4e86ad42bbf 100644
--- a/prowler/providers/common/provider.py
+++ b/prowler/providers/common/provider.py
@@ -161,54 +161,54 @@ def init_global_provider(arguments: Namespace) -> None:
             if not isinstance(Provider._global, provider_class):
                 if "aws" in provider_class_name.lower():
                     provider_class(
-                        arguments.aws_retries_max_attempts,
-                        arguments.role,
-                        arguments.session_duration,
-                        arguments.external_id,
-                        arguments.role_session_name,
-                        arguments.mfa,
-                        arguments.profile,
-                        set(arguments.region) if arguments.region else None,
-                        arguments.organizations_role,
-                        arguments.scan_unused_services,
-                        arguments.resource_tag,
-                        arguments.resource_arn,
-                        arguments.config_file,
-                        arguments.mutelist_file,
+                        retries_max_attempts=arguments.aws_retries_max_attempts,
+                        role_arn=arguments.role,
+                        session_duration=arguments.session_duration,
+                        external_id=arguments.external_id,
+                        role_session_name=arguments.role_session_name,
+                        mfa=arguments.mfa,
+                        profile=arguments.profile,
+                        regions=set(arguments.region) if arguments.region else None,
+                        organizations_role_arn=arguments.organizations_role,
+                        scan_unused_services=arguments.scan_unused_services,
+                        resource_tags=arguments.resource_tag,
+                        resource_arn=arguments.resource_arn,
+                        config_path=arguments.config_file,
+                        mutelist_path=arguments.mutelist_file,
                         fixer_config=fixer_config,
                     )
                 elif "azure" in provider_class_name.lower():
                     provider_class(
-                        arguments.az_cli_auth,
-                        arguments.sp_env_auth,
-                        arguments.browser_auth,
-                        arguments.managed_identity_auth,
-                        arguments.tenant_id,
-                        arguments.azure_region,
-                        arguments.subscription_id,
-                        arguments.config_file,
-                        arguments.mutelist_file,
+                        az_cli_auth=arguments.az_cli_auth,
+                        sp_env_auth=arguments.sp_env_auth,
+                        browser_auth=arguments.browser_auth,
+                        managed_identity_auth=arguments.managed_identity_auth,
+                        tenant_id=arguments.tenant_id,
+                        region=arguments.azure_region,
+                        subscription_ids=arguments.subscription_id,
+                        config_path=arguments.config_file,
+                        mutelist_path=arguments.mutelist_file,
                         fixer_config=fixer_config,
                     )
                 elif "gcp" in provider_class_name.lower():
                     provider_class(
-                        arguments.organization_id,
-                        arguments.project_id,
-                        arguments.excluded_project_id,
-                        arguments.credentials_file,
-                        arguments.impersonate_service_account,
-                        arguments.list_project_id,
-                        arguments.config_file,
-                        arguments.mutelist_file,
+                        organization_id=arguments.organization_id,
+                        project_ids=arguments.project_id,
+                        excluded_project_ids=arguments.excluded_project_id,
+                        credentials_file=arguments.credentials_file,
+                        impersonate_service_account=arguments.impersonate_service_account,
+                        list_project_ids=arguments.list_project_id,
+                        config_path=arguments.config_file,
+                        mutelist_path=arguments.mutelist_file,
                         fixer_config=fixer_config,
                     )
                 elif "kubernetes" in provider_class_name.lower():
                     provider_class(
-                        arguments.kubeconfig_file,
-                        arguments.context,
-                        arguments.namespace,
-                        arguments.config_file,
-                        arguments.mutelist_file,
+                        kubeconfig_file=arguments.kubeconfig_file,
+                        context=arguments.context,
+                        namespace=arguments.namespace,
+                        config_path=arguments.config_file,
+                        mutelist_path=arguments.mutelist_file,
                         fixer_config=fixer_config,
                     )

AWS MuteList broken
### Steps to Reproduce

**What command are you running:**

```bash
prowler aws -M json-ocsf -f us-west-2 -w mutelist.yaml --ignore-exit-code-3
```

**Cloud provider you are launching:** AWS

**Environment you have:**

* Single account
* Prowler open source
* Issues with:
  * Prowler installed via Pip
  * Prowler Container
* Prowler Version: 4.5.0 or 4.6.0

**Error:**

```
Something went wrong in accessanalyzer_enabled, please use --log-level ERROR

2024-11-06 20:26:48,299 [File: check.py:617] 	[Module: check]	 ERROR: accessanalyzer_enabled -- AttributeError[33]: 'str' object has no attribute 'get'
Something went wrong in appstream_fleet_maximum_session_duration, please use --log-level ERROR

2024-11-06 20:26:49,877 [File: check.py:617] 	[Module: check]	 ERROR: appstream_fleet_maximum_session_duration -- AttributeError[12]: 'str' object has no attribute 'get'
Something went wrong in appstream_fleet_session_disconnect_timeout, please use --log-level ERROR

2024-11-06 20:26:49,880 [File: check.py:617] 	[Module: check]	 ERROR: appstream_fleet_session_disconnect_timeout -- AttributeError[12]: 'str' object has no attribute 'get'
Something went wrong in appstream_fleet_session_idle_disconnect_timeout, please use --log-level ERROR

2024-11-06 20:26:49,884 [File: check.py:617] 	[Module: check]	 ERROR: appstream_fleet_session_idle_disconnect_timeout -- AttributeError[12]: 'str' object has no attribute 'get'
Something went wrong in autoscaling_find_secrets_ec2_launch_configuration, please use --log-level ERROR

2024-11-06 20:26:50,400 [File: check.py:617] 	[Module: check]	 ERROR: autoscaling_find_secrets_ec2_launch_configuration -- AttributeError[16]: 'str' object has no attribute 'get'
Something went wrong in awslambda_function_no_secrets_in_code, please use --log-level ERROR

2024-11-06 20:26:51,828 [File: check.py:617] 	[Module: check]	 ERROR: awslambda_function_no_secrets_in_code -- AttributeError[13]: 'str' object has no attribute 'get'
Something went wrong in awslambda_function_no_secrets_in_variables, please use --log-level ERROR

2024-11-06 20:26:51,832 [File: check.py:617] 	[Module: check]	 ERROR: awslambda_function_no_secrets_in_variables -- AttributeError[11]: 'str' object has no attribute 'get'
```
More errors follow...

### Expected behavior

Prowler will scan the AWS account and mute findings in the mutelist

### Actual Result with Screenshots or Logs

Prowler scans, but the above errors are displayed and no findings are muted.

### How did you install Prowler?

From pip package (pip install prowler)

### Environment Resource

EKS
Local

### OS used

EKS: `toniblyx/prowler:latest` Docker image
Local: macOS 14.7 Python 3.11

### Prowler version

Prowler 4.6.0

### Pip version

24.0

### Context

_No response_

I was able to solve this by updating the arguments to the AWS provider when it is created:
I made this change in the `providers/common/provider.py` file
```diff
10a11
> from prowler.lib.mutelist.models import mutelist_schema
176,177c177,178
<                         arguments.config_file,
<                         arguments.mutelist_file,
---
>                         config_path=arguments.config_file,
>                         mutelist_path=arguments.mutelist_file,
```

The `arguments.mutelist_file` argument was getting passed into the AWS provider as the `config_content` parameter because keyword args were not specified to skip the unused parameters.


diff --git a/vllm/executor/ray_utils.py b/vllm/executor/ray_utils.py
index 4f28efd639084..426aa1b5c728f 100644
--- a/vllm/executor/ray_utils.py
+++ b/vllm/executor/ray_utils.py
@@ -277,10 +277,14 @@ def initialize_ray_cluster(
                 f"Total number of devices: {device_bundles}.")
     else:
         num_devices_in_cluster = ray.cluster_resources().get(device_str, 0)
+        # Log a warning message and delay resource allocation failure response.
+        # Avoid immediate rejection to allow user-initiated placement group
+        # created and wait cluster to be ready
         if parallel_config.world_size > num_devices_in_cluster:
-            raise ValueError(
-                f"The number of required {device_str}s exceeds the total "
-                f"number of available {device_str}s in the placement group.")
+            logger.warning(
+                "The number of required %ss exceeds the total "
+                "number of available %ss in the placement group.", device_str,
+                device_str)
         # Create a new placement group
         placement_group_specs: List[Dict[str, float]] = ([{
             device_str: 1.0

[Feature]: Allow Ray placement group more time to wait for resources to be ready
### 🚀 The feature, motivation and pitch

```
ray start --head --dashboard-host 0.0.0.0 --disable-usage-stats --block  --num-gpus 1

python -m vllm.entrypoints.openai.api_server     --model Qwen/Qwen2.5-1.5B-Instruct     --trust-remote-code     --distributed-executor-backend ray     --tensor-parallel-size 2
```

I am using a ray cluster as the distributed executor backend. However, my cluster sometimes takes time to be ready. there are multiple reasons
- image downloading speed is different, some node has image cache and some does not.
- node short of resources, node level autoscaler will make decision to bring up more nodes but it takes time for new ray node to join and form a cluster

I do not want the engine to early exit.

![image](https://github.com/user-attachments/assets/7a621963-21fb-4f16-9573-71299e951bad)

![image](https://github.com/user-attachments/assets/5b06c173-1465-4675-a5a0-2627a3b8b18d)

### Alternatives

_No response_

### Additional context

_No response_

### Before submitting a new issue...

- [X] Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://docs.vllm.ai/en/latest/), which can answer lots of frequently asked questions.


diff --git a/vllm/entrypoints/openai/serving_completion.py b/vllm/entrypoints/openai/serving_completion.py
index fc1c4908d6650..03bcf26ae7e91 100644
--- a/vllm/entrypoints/openai/serving_completion.py
+++ b/vllm/entrypoints/openai/serving_completion.py
@@ -392,6 +392,12 @@ def request_output_to_completion_response(
             prompt_token_ids = final_res.prompt_token_ids
             assert prompt_token_ids is not None
             prompt_logprobs = final_res.prompt_logprobs
+            if prompt_logprobs:
+                for logprob_dict in prompt_logprobs:
+                    if logprob_dict:
+                        for logprob_values in logprob_dict.values():
+                            if logprob_values.logprob == float('-inf'):
+                                logprob_values.logprob = -9999.0
             prompt_text = final_res.prompt

             token_ids: GenericSequence[int]

[Bug]: Internal Server Error when echo'ing logprobs with sampling
### Your current environment

<details>
<summary>The output of `python collect_env.py`</summary>

```text
Your output of `python collect_env.py` here
```

</details>

### Model Input Dumps

_No response_

### 🐛 Describe the bug

Sending an request that asks for prompt logprobs while also specifying certain sampling parameters can result in an Internal Server Error.  For example:
```
curl -v http://localhost:8000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
      "model": "meta-llama/Llama-3.2-1B",
      "prompt": "correct horse battery staple",
      "echo": 1,
      "logprobs": 1,
      "temperature": 1,
      "top_k": 1
  }'
```

Results in this stack trace:
```
INFO:     ::1:47926 - "POST /v1/completions HTTP/1.1" 500 Internal Server Error
ERROR:    Exception in ASGI application
Traceback (most recent call last):
  File "/opt/vllm/lib64/python3.12/site-packages/uvicorn/protocols/http/httptools_impl.py", line 401, in run_asgi
    result = await app(  # type: ignore[func-returns-value]
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/vllm/lib64/python3.12/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
    return await self.app(scope, receive, send)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/vllm/lib64/python3.12/site-packages/fastapi/applications.py", line 1054, in __call__
    await super().__call__(scope, receive, send)
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/applications.py", line 113, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/middleware/errors.py", line 187, in __call__
    raise exc
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/middleware/errors.py", line 165, in __call__
    await self.app(scope, receive, _send)
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/middleware/cors.py", line 85, in __call__
    await self.app(scope, receive, send)
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
    await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    raise exc
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
    await app(scope, receive, sender)
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/routing.py", line 715, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/routing.py", line 735, in app
    await route.handle(scope, receive, send)
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/routing.py", line 288, in handle
    await self.app(scope, receive, send)
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/routing.py", line 76, in app
    await wrap_app_handling_exceptions(app, request)(scope, receive, send)
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    raise exc
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
    await app(scope, receive, sender)
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/routing.py", line 73, in app
    response = await f(request)
               ^^^^^^^^^^^^^^^^
  File "/opt/vllm/lib64/python3.12/site-packages/fastapi/routing.py", line 301, in app
    raw_response = await run_endpoint_function(
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/vllm/lib64/python3.12/site-packages/fastapi/routing.py", line 212, in run_endpoint_function
    return await dependant.call(**values)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/vllm/lib64/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 337, in create_completion
    return JSONResponse(content=generator.model_dump())
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/responses.py", line 180, in __init__
    super().__init__(content, status_code, headers, media_type, background)
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/responses.py", line 43, in __init__
    self.body = self.render(content)
                ^^^^^^^^^^^^^^^^^^^^
  File "/opt/vllm/lib64/python3.12/site-packages/starlette/responses.py", line 183, in render
    return json.dumps(
           ^^^^^^^^^^^
  File "/usr/lib64/python3.12/json/__init__.py", line 238, in dumps
    **kw).encode(obj)
          ^^^^^^^^^^^
  File "/usr/lib64/python3.12/json/encoder.py", line 200, in encode
    chunks = self.iterencode(o, _one_shot=True)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib64/python3.12/json/encoder.py", line 258, in iterencode
    return _iterencode(o, 0)
           ^^^^^^^^^^^^^^^^^
ValueError: Out of range float values are not JSON compliant: -inf
```

From what I can see this happens because the logprobs on at least one of the prompt tokens is -inf. The `top_k: 1` sampling parameter that makes the probablity of any token other than the top-1 0.

### Before submitting a new issue...

- [X] Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://docs.vllm.ai/en/latest/), which can answer lots of frequently asked questions.

@DarkLight1337 could you please assign me to this one? I'll investigate.


diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py
index 4692762493f00..e8d964a722f60 100644
--- a/vllm/v1/worker/gpu_model_runner.py
+++ b/vllm/v1/worker/gpu_model_runner.py
@@ -260,7 +260,8 @@ def _prepare_inputs(self, scheduler_output: "SchedulerOutput"):
         # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
         # -> [0, 1, M, M + 1, M + 2, M + 3, M + 4, 2 * M, 2 * M + 1, 2 * M + 2]
         # where M is the max_model_len.
-        token_indices = positions_np + req_indices * self.max_model_len
+        token_indices = (positions_np +
+                         req_indices * self.input_batch.token_ids_cpu.shape[1])
         token_indices = torch.from_numpy(token_indices)
         input_ids = torch.empty((total_num_scheduled_tokens, ),
                                 dtype=torch.int32,
@@ -273,9 +274,15 @@ def _prepare_inputs(self, scheduler_output: "SchedulerOutput"):
                            out=input_ids)

         # Calculate the slot mapping.
+        # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
+        # -> [0, 0, K, K, K + 1, K + 1, K + 2, 2 * K, 2 * K, 2 * K + 1]
+        # where K is the max_num_blocks_per_req and the block size is 2.
+        # NOTE(woosuk): We can't simply use `token_indices // block_size` here
+        # because M (max_model_len) is not necessarily divisible by block_size.
         block_numbers = self.input_batch.block_table_cpu_tensor.flatten()[
-            token_indices // self.block_size]
-        block_offsets = token_indices % self.block_size
+            req_indices * self.max_num_blocks_per_req +
+            positions_np // self.block_size]
+        block_offsets = torch.from_numpy(positions_np % self.block_size)
         slot_mapping = torch.empty((total_num_scheduled_tokens, ),
                                    dtype=torch.int32,
                                    device="cpu",

[Bug, V1]: LlaVa outputs wrong results in batch inference with V1 code（V0 code is correct)
### Your current environment

<details>
<summary>The output of `python collect_env.py`</summary>

```text
Warning: Your installation of OpenCV appears to be broken: module 'cv2.dnn' has no attribute 'DictValue'.Please follow the instructions at https://github.com/opencv/opencv-python/issues/884 to correct your environment. The import of cv2 has been skipped.
WARNING 12-04 08:26:48 cuda.py:30] You are using a deprecated `pynvml` package. Please install `nvidia-ml-py` instead, and make sure to uninstall `pynvml`. When both of them are installed, `pynvml` will take precedence and cause errors. See https://pypi.org/project/pynvml for more information.
Collecting environment information...
PyTorch version: 2.5.1+cu124
Is debug build: False
CUDA used to build PyTorch: 12.4
ROCM used to build PyTorch: N/A

OS: Ubuntu 22.04.4 LTS (x86_64)
GCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0
Clang version: Could not collect
CMake version: version 3.29.2
Libc version: glibc-2.35

Python version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)
Python platform: Linux-5.4.0-125-generic-x86_64-with-glibc2.35
Is CUDA available: True
CUDA runtime version: 12.4.131
CUDA_MODULE_LOADING set to: LAZY
GPU models and configuration:
GPU 0: NVIDIA GeForce RTX 4090
GPU 1: NVIDIA GeForce RTX 4090
GPU 2: NVIDIA GeForce RTX 4090
GPU 3: NVIDIA GeForce RTX 4090
GPU 4: NVIDIA GeForce RTX 4090
GPU 5: NVIDIA GeForce RTX 4090
GPU 6: NVIDIA GeForce RTX 4090
GPU 7: NVIDIA GeForce RTX 4090

Nvidia driver version: 535.54.03
cuDNN version: Probably one of the following:
/usr/lib/x86_64-linux-gnu/libcudnn.so.9.1.0
/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.1.0
/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.1.0
/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.1.0
/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.1.0
/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.1.0
/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.1.0
/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.1.0
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: True

CPU:
Architecture:                    x86_64
CPU op-mode(s):                  32-bit, 64-bit
Address sizes:                   52 bits physical, 57 bits virtual
Byte Order:                      Little Endian
CPU(s):                          96
On-line CPU(s) list:             0-95
Vendor ID:                       GenuineIntel
Model name:                      Intel(R) Xeon(R) Gold 5318Y CPU @ 2.10GHz
CPU family:                      6
Model:                           106
Thread(s) per core:              2
Core(s) per socket:              24
Socket(s):                       2
Stepping:                        6
Frequency boost:                 enabled
CPU max MHz:                     2101.0000
CPU min MHz:                     800.0000
BogoMIPS:                        4200.00
Flags:                           fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities
Virtualization:                  VT-x
L1d cache:                       2.3 MiB (48 instances)
L1i cache:                       1.5 MiB (48 instances)
L2 cache:                        60 MiB (48 instances)
L3 cache:                        72 MiB (2 instances)
NUMA node(s):                    2
NUMA node0 CPU(s):               0-23,48-71
NUMA node1 CPU(s):               24-47,72-95
Vulnerability Itlb multihit:     Not affected
Vulnerability L1tf:              Not affected
Vulnerability Mds:               Not affected
Vulnerability Meltdown:          Not affected
Vulnerability Mmio stale data:   Mitigation; Clear CPU buffers; SMT vulnerable
Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp
Vulnerability Spectre v1:        Mitigation; usercopy/swapgs barriers and __user pointer sanitization
Vulnerability Spectre v2:        Mitigation; Enhanced IBRS, IBPB conditional, RSB filling
Vulnerability Srbds:             Not affected
Vulnerability Tsx async abort:   Not affected

Versions of relevant libraries:
[pip3] numpy==1.26.4
[pip3] nvidia-cublas-cu12==12.4.5.8
[pip3] nvidia-cuda-cupti-cu12==12.4.127
[pip3] nvidia-cuda-nvrtc-cu12==12.4.127
[pip3] nvidia-cuda-runtime-cu12==12.4.127
[pip3] nvidia-cudnn-cu12==9.1.0.70
[pip3] nvidia-cudnn-frontend==1.3.0
[pip3] nvidia-cufft-cu12==11.2.1.3
[pip3] nvidia-curand-cu12==10.3.5.147
[pip3] nvidia-cusolver-cu12==11.6.1.9
[pip3] nvidia-cusparse-cu12==12.3.1.170
[pip3] nvidia-dali-cuda120==1.37.1
[pip3] nvidia-ml-py==12.560.30
[pip3] nvidia-nccl-cu12==2.21.5
[pip3] nvidia-nvimgcodec-cu12==0.2.0.7
[pip3] nvidia-nvjitlink-cu12==12.4.127
[pip3] nvidia-nvtx-cu12==12.4.127
[pip3] nvidia-pyindex==1.0.9
[pip3] onnx==1.16.0
[pip3] optree==0.11.0
[pip3] pynvml==11.4.1
[pip3] pytorch-quantization==2.1.2
[pip3] pytorch-triton==3.0.0+989adb9a2
[pip3] pyzmq==26.0.3
[pip3] torch==2.5.1
[pip3] torch-tensorrt==2.4.0a0
[pip3] torchvision==0.20.1
[pip3] transformers==4.46.3
[pip3] triton==3.1.0
[conda] Could not collect
ROCM Version: Could not collect
Neuron SDK Version: N/A
vLLM Version: 0.6.4.post2.dev221+g3bc94cab
vLLM Build Flags:
CUDA Archs: 5.2 6.0 6.1 7.0 7.2 7.5 8.0 8.6 8.7 9.0+PTX; ROCm: Disabled; Neuron: Disabled
GPU Topology:
GPU0    GPU1    GPU2    GPU3    GPU4    GPU5    GPU6    GPU7    NIC0    NIC1    NIC2    NIC3    CPU Affinity    NUMA Affinity   GPU NUMA ID
GPU0     X      PXB     PXB     PXB     SYS     SYS     SYS     SYS     PXB     PXB     SYS     SYS     0-23,48-71      0               N/A
GPU1    PXB      X      PXB     PXB     SYS     SYS     SYS     SYS     PIX     PIX     SYS     SYS     0-23,48-71      0               N/A
GPU2    PXB     PXB      X      PIX     SYS     SYS     SYS     SYS     PXB     PXB     SYS     SYS     0-23,48-71      0               N/A
GPU3    PXB     PXB     PIX      X      SYS     SYS     SYS     SYS     PXB     PXB     SYS     SYS     0-23,48-71      0               N/A
GPU4    SYS     SYS     SYS     SYS      X      NODE    NODE    NODE    SYS     SYS     NODE    NODE    24-47,72-95     1               N/A
GPU5    SYS     SYS     SYS     SYS     NODE     X      PXB     PXB     SYS     SYS     PXB     PXB     24-47,72-95     1               N/A
GPU6    SYS     SYS     SYS     SYS     NODE    PXB      X      PXB     SYS     SYS     PIX     PIX     24-47,72-95     1               N/A
GPU7    SYS     SYS     SYS     SYS     NODE    PXB     PXB      X      SYS     SYS     PXB     PXB     24-47,72-95     1               N/A
NIC0    PXB     PIX     PXB     PXB     SYS     SYS     SYS     SYS      X      PIX     SYS     SYS
NIC1    PXB     PIX     PXB     PXB     SYS     SYS     SYS     SYS     PIX      X      SYS     SYS
NIC2    SYS     SYS     SYS     SYS     NODE    PXB     PIX     PXB     SYS     SYS      X      PIX
NIC3    SYS     SYS     SYS     SYS     NODE    PXB     PIX     PXB     SYS     SYS     PIX      X

Legend:

  X    = Self
  SYS  = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)
  NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node
  PHB  = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)
  PXB  = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)
  PIX  = Connection traversing at most a single PCIe bridge
  NV#  = Connection traversing a bonded set of # NVLinks

NIC Legend:

  NIC0: mlx5_0
  NIC1: mlx5_1
  NIC2: mlx5_2
  NIC3: mlx5_3

NVIDIA_VISIBLE_DEVICES=all
CUBLAS_VERSION=12.4.5.8
NVIDIA_REQUIRE_CUDA=cuda>=9.0
CUDA_CACHE_DISABLE=1
TORCH_CUDA_ARCH_LIST=5.2 6.0 6.1 7.0 7.2 7.5 8.0 8.6 8.7 9.0+PTX
NCCL_VERSION=2.21.5
NVIDIA_DRIVER_CAPABILITIES=compute,utility,video
NVIDIA_PRODUCT_NAME=PyTorch
CUDA_VERSION=12.4.1.003
PYTORCH_VERSION=2.4.0a0+07cecf4
PYTORCH_BUILD_NUMBER=0
CUDNN_VERSION=9.1.0.70
PYTORCH_HOME=/opt/pytorch/pytorch
LD_LIBRARY_PATH=/usr/local/lib/python3.10/dist-packages/cv2/../../lib64:/usr/local/lib/python3.10/dist-packages/torch/lib:/usr/local/lib/python3.10/dist-packages/torch_tensorrt/lib:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
NVIDIA_BUILD_ID=91431255
CUDA_DRIVER_VERSION=550.54.15
PYTORCH_BUILD_VERSION=2.4.0a0+07cecf4
CUDA_HOME=/usr/local/cuda
CUDA_HOME=/usr/local/cuda
CUDA_MODULE_LOADING=LAZY
NVIDIA_REQUIRE_JETPACK_HOST_MOUNTS=
NVIDIA_PYTORCH_VERSION=24.05
TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1
```

</details>

### Model Input Dumps

Version：
0.6.4.post2.dev221+g3bc94cab

Test Image：
[test_img.zip](https://github.com/user-attachments/files/18004889/test_img.zip)

### 🐛 Describe the bug

@WoosukKwon Here is an fatal bug on V1 offline batch inference

```
import os
from vllm import LLM, SamplingParams, envs
from PIL import Image
import time

prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"

fname = 'animal.jpg' # get form the .zip file
image = Image.open(fname).convert('RGB')

envs.VLLM_USE_V1 = True
llm = LLM(model='llava-1.5-7b-hf', trust_remote_code=True, dtype="bfloat16",
          tokenizer_mode="auto", gpu_memory_utilization=0.9, max_model_len=1000, max_num_batched_tokens=1200)

sampling_params = SamplingParams(temperature=0, max_tokens=128)

inputs = {
    "prompt": prompt,
    "multi_modal_data": {
        "image": image
    }
}

outputs = llm.generate([inputs, inputs], sampling_params)
for output in outputs:
    text = output.outputs[0].text
    print(text)
    print()

print("done")
```

The output is

>  The image features a black and white panda bear sitting in a grassy area. The panda is eating grass, and it appears to be enjoying the meal. The scene is set in a lush green environment, with the panda being the main focus of the image.
>
>  ASSA TEXT

That is, though the two inputs are the same, the results are not the same.

Here is my debug info:
(1) If I comment ```envs.VLLM_USE_V1 = True```，I can get the two correct result，so the bug is in V1
(2) If I ajust ```max_model_len``` larger(2 times seqlen)，I can get the two correct result

### Before submitting a new issue...

- [X] Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://docs.vllm.ai/en/latest/), which can answer lots of frequently asked questions.

cc @ywang96 @WoosukKwon
@PYNing Thanks for reporting this bug! I can confirm this is reproducible, and only happens ~~when `max_model_len` is smaller than `max_num_batched_tokens`.~~ when the `max_model_len` is not divisible by `block_size` (default block_size is 16 in V1).

Code to reproduce:
``` python
import os
from vllm import LLM, SamplingParams, envs
from PIL import Image
import time
from vllm.assets.image import ImageAsset

max_model_len = <to-be-specified>

prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
image = ImageAsset("cherry_blossom").pil_image.convert("RGB")

envs.VLLM_USE_V1 = True
llm = LLM(model='llava-hf/llava-1.5-7b-hf', trust_remote_code=True, dtype="bfloat16",
          tokenizer_mode="auto", gpu_memory_utilization=0.9, max_model_len=max_model_len, max_num_batched_tokens=1200)
sampling_params = SamplingParams(temperature=0, max_tokens=128)
inputs = {
    "prompt": prompt,
    "multi_modal_data": {
        "image": image
    }
}
outputs = llm.generate([inputs, inputs], sampling_params)
for output in outputs:
    text = output.outputs[0].text
    print("response:", text)
    print()
```

When setting `max_model_len` to 1199, incorrect responses were generated:
```linux
Response:  The image features a tall tower with a spire, surrounded by a beautiful flowering tree filled with pink flowers. The tower stands tall in the background, while the tree's branches are filled with the vibrant blossoms. The combination of the tower and the flowering tree creates a picturesque scene, showcasing the beauty of nature and architecture.

Response: 100)Follow us on Facebook.
```

When setting `max_model_len` to 700, incorrect responses were generated:
```
Response:  The image features a tall tower with a spire, surrounded by a beautiful flowering tree filled with pink flowers. The tower stands tall in the background, while the tree's branches are filled with the vibrant blossoms. The combination of the tower and the tree creates a picturesque scene, showcasing the beauty of nature and architecture.

Response: ASSISTANT:ASSISTANT:In the image, the color palette is a mix of the primary colors, with the most dominant color being the color of the sky. The image is a beautiful representation of the sky, with the clouds and the sun. The clouds are a mix of the primary colors, with the most dominant color being the color of the sky. The image is a beautiful representation of the sky, with the clouds and the sun. The clouds are a mix of the primary colors, with the
```

When setting `max_model_len` to 1201, incorrect responses were generated:
```linux
Response:  The image features a tall tower with a spire, surrounded by a beautiful blossoming tree filled with pink flowers. The tower stands tall in the background, while the tree's branches are filled with the vibrant pink flowers, creating a picturesque scene. The tower's spire can be seen peeking through the tree's branches, adding a sense of depth to the image.

Response:  yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it should be yes, it
```

When setting `max_model_len` to 800, correct responses were generated:
```linux
Response:  The image features a tall tower with a spire, surrounded by a beautiful cherry blossom tree. The tree is filled with pink flowers, creating a picturesque scene. The tower stands tall in the background, with the blossoming tree in the foreground. The combination of the tower and the tree creates a captivating and serene atmosphere.

Response:  The image features a tall tower with a spire, surrounded by a beautiful cherry blossom tree. The tree is filled with pink flowers, creating a picturesque scene. The tower stands tall in the background, with the blossoming tree in the foreground. The combination of the tower and the tree creates a captivating and serene atmosphere.
```

When setting `max_model_len` to 1200, correct responses were generated:
```linux
Response:  The image features a tall tower with a spire, surrounded by a beautiful cherry blossom tree. The tree is filled with pink flowers, creating a picturesque scene. The tower stands tall in the background, with the blossoming tree in the foreground. The combination of the tower and the tree creates a captivating and serene atmosphere.

Response:  The image features a tall tower with a spire, surrounded by a beautiful cherry blossom tree. The tree is filled with pink flowers, creating a picturesque scene. The tower stands tall in the background, with the blossoming tree in the foreground. The combination of the tower and the tree creates a captivating and serene atmosphere.
```


diff --git a/vllm/engine/metrics.py b/vllm/engine/metrics.py
index 5bfd6a9f4b386..4869557ba9b44 100644
--- a/vllm/engine/metrics.py
+++ b/vllm/engine/metrics.py
@@ -473,13 +473,13 @@ def log(self, stats: Stats) -> None:
             )
             if (stats.cpu_prefix_cache_hit_rate >= 0
                     or stats.gpu_prefix_cache_hit_rate >= 0):
-                logger.info(
+                log_fn(
                     "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
                     stats.gpu_prefix_cache_hit_rate * 100,
                     stats.cpu_prefix_cache_hit_rate * 100,
                 )
             if self.spec_decode_metrics is not None:
-                logger.info(
+                log_fn(
                     self._format_spec_decode_metrics_str(
                         self.spec_decode_metrics))

[Core] Avoid metrics log noise when idle

1a789ae7 [Core] Avoid metrics log noise when idle

commit 1a789ae76695bca801b55c9d4531e5438a3378c4
Author: Russell Bryant <rbryant@redhat.com>
Date:   Thu Sep 26 20:56:41 2024 +0000

    [Core] Avoid metrics log noise when idle

    Prior to this change, there was always a log message every 10 seconds,
    even when the system is idle. The noise when the metrics are all zeros
    doesn't seem very interesting, so this change makes it so the log
    message is only emitted while active. It will log a message whenever
    metrics are non-zero, or when returning to idle from non-idle.
    Repeated messages during idle now go to DEBUG instead of INFO.

    Signed-off-by: Russell Bryant <rbryant@redhat.com>

👋 Hi! Thank you for contributing to the vLLM project.
 Just a reminder: PRs would not trigger full CI run by default. Instead, it would only run `fastcheck` CI which starts running only a small and essential subset of CI tests to quickly catch errors. You can run other CI tests on top of those by going to your `fastcheck` build on Buildkite UI (linked in the PR checks section) and unblock them. If you do not have permission to unblock, ping `simon-mo` or `khluu` to add you in our Buildkite org.

Once the PR is approved and ready to go, your PR reviewer(s) can run CI to test the changes comprehensively before merging.

 To run CI, PR reviewers can do one of these:
- Add `ready` label to the PR
- Enable auto-merge.

🚀
@russellb I think this is a great small QoL improvement. Is this still something get worked on?
> @russellb I think this is a great small QoL improvement. Is this still something get worked on?

It's ready for review when someone has time.
@simon-mo @mgoin Is this something we can merge before the next release?
> I personally like keeping the log as a sort of "heartbeat" to let me know the server is functioning fine even if there is nothing running

I think the functionality of `/health` endpoint should be what indicates this rather than a log message showing idle numbers.
> > I personally like keeping the log as a sort of "heartbeat" to let me know the server is functioning fine even if there is nothing running
>
> I think the functionality of `/health` endpoint should be what indicates this rather than a log message showing idle numbers.

A compromise could be to emit them as DEBUG instead of INFO once the system goes idle.
> A compromise could be to emit them as DEBUG instead of INFO once the system goes idle.

That sounds like a valid solution to me. Should also enable users who would prefer to have the logs enable it. Thoughts @mgoin ?
> > A compromise could be to emit them as DEBUG instead of INFO once the system goes idle.
>
> That sounds like a valid solution to me. Should also enable users who would prefer to have the logs enable it. Thoughts @mgoin ?

I updated the PR to do this.
@mgoin can we go ahead with this approach?
Just to check, do we have other DEBUG logging? Like if I enabled DEBUG would I see lots of other logs and even slow the performance?
> Just to check, do we have other DEBUG logging? Like if I enabled DEBUG would I see lots of other logs and even slow the performance?

It looks like `logger.debug()` is called in 120 places right now.

```
$ git grep 'logger.debug(' | wc -l
120
```

On an idle system, I get all of this every 10 seconds (`vllm serve <model> --tensor-parallel-size 4`):

```
DEBUG 11-19 16:29:44 client.py:186] Waiting for output from MQLLMEngine.
DEBUG 11-19 16:29:44 client.py:186] Waiting for output from MQLLMEngine.
DEBUG 11-19 16:29:44 client.py:186] Waiting for output from MQLLMEngine.
DEBUG 11-19 16:29:44 client.py:186] Waiting for output from MQLLMEngine.
DEBUG 11-19 16:29:44 client.py:186] Waiting for output from MQLLMEngine.
DEBUG 11-19 16:29:44 client.py:186] Waiting for output from MQLLMEngine.
DEBUG 11-19 16:29:44 client.py:186] Waiting for output from MQLLMEngine.
DEBUG 11-19 16:29:44 client.py:186] Waiting for output from MQLLMEngine.
DEBUG 11-19 16:29:44 client.py:186] Waiting for output from MQLLMEngine.
DEBUG 11-19 16:29:44 client.py:186] Waiting for output from MQLLMEngine.
DEBUG 11-19 16:29:51 client.py:165] Heartbeat successful.
INFO 11-19 16:29:51 metrics.py:449] Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Swapped: 0 reqs, Pending: 0 reqs, GPU KV cache usage: 0.0%, CPU KV cache usage: 0.0%.
DEBUG 11-19 16:29:51 client.py:165] Heartbeat successful.
DEBUG 11-19 16:29:51 engine.py:190] Waiting for new requests in engine loop.
```

When I hit the openai API server with a simple generate request, I got zero additional debug messages (just a few INFO messages).
Thanks for the confirmation. From the log it seems not necessary to keep the message even in DEBUG level, because this level already has heartbeat logs. But I'll just leave this comment for reference and won't block the PR for merging.
> Thanks for the confirmation. From the log it seems not necessary to keep the message even in DEBUG level, because this level already has heartbeat logs. But I'll just leave this comment for reference and won't block the PR for merging.

Yeah, that's true. I'm happy with whatever. My main thing was that I think the default production config should have a quiet log when idle.
> My main thing was that I think the default production config should have a quiet log when idle.

I would like to second this. I have several instances running with log shipping to a central place, and it is causing a lot of useless log messages to be collected. If we can reduce at least some of the noise, that would be ideal.