neuralmagic

/

Phi-3-mini-128k-instruct-quantized.w8a8

like 0

llm = LLM(model=model_id, trust_remote_code=True, max_model_len=8196, tensor_parallel_size=number_gpus)

With trust remote code True

File ~/.local/lib/python3.11/site-packages/vllm/config.py:1287, in _get_and_verify_max_len(hf_config, max_model_len, disable_sliding_window, sliding_window_len)
   1280 if disable_sliding_window:
   1281     # TODO(robertgshaw): Find a model that supports rope_scaling
   1282     # with sliding window to see if this case should be allowed.
   1283     raise NotImplementedError(
   1284         "Disabling sliding window is not supported for models "
   1285         "with rope_scaling. Please raise an issue so we can "
   1286         "investigate.")
-> 1287 assert "factor" in rope_scaling
   1288 scaling_factor = rope_scaling["factor"]
   1289 if rope_scaling["type"] == "yarn":

AssertionError: 

Without trust remote code True :

File ~/.local/lib/python3.11/site-packages/transformers/models/phi3/configuration_phi3.py:185, in Phi3Config._rope_scaling_validation(self)
    183 rope_scaling_long_factor = self.rope_scaling.get("long_factor", None)
    184 if rope_scaling_type is None or rope_scaling_type not in ["su", "yarn"]:
--> 185     raise ValueError(f"`rope_scaling`'s type field must be one of ['su', 'yarn'], got {rope_scaling_type}")
    186 if not (
    187     isinstance(rope_scaling_short_factor, list)
    188     and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor)
    189 ):
    190     raise ValueError(
    191         f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}"
    192     )

ValueError: `rope_scaling`'s type field must be one of ['su', 'yarn'], got longrope