Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,41 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|AUDIO|>": 151646,
+  "<|IMAGE|>": 151655,
+  "<|VIDEO|>": 151656,
+  "<|ar|>": 151679,
+  "<|audio_bos|>": 151647,
+  "<|audio_eos|>": 151648,
+  "<|box_end|>": 151649,
+  "<|de|>": 151667,
+  "<|endoftext|>": 151643,
+  "<|en|>": 151665,
+  "<|es|>": 151668,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|fr|>": 151669,
+  "<|hi|>": 151670,
+  "<|id|>": 151676,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|it|>": 151678,
+  "<|jp|>": 151680,
+  "<|kr|>": 151666,
+  "<|nl|>": 151674,
+  "<|pt|>": 151675,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|ru|>": 151677,
+  "<|th|>": 151672,
+  "<|uk|>": 151671,
+  "<|unknown|>": 151681,
+  "<|vision_bos|>": 151652,
+  "<|vision_eos|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vi|>": 151673
+}

chat_template.jinja

@@ -0,0 +1,8 @@
+{% set audio_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
+You are a helpful assistant.<|im_end|>
+{% endif %}<|im_start|>{{ message['role'] }}
+{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
+{% else %}{% for content in message['content'] %}{% if 'audio' in content or 'audio_url' in content or message['type'] == 'audio' %}{% set audio_count.value = audio_count.value + 1 %}Audio {{ audio_count.value }}: <|audio_bos|><|AUDIO|><|audio_eos|>
+{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
+{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "DashengQwen25OmniModelInstruct"
+  ],
+  "audio_encoder": "LemonstoreWrapper",
+  "audio_encoder_args": {
+    "model_name": "audiotransformer_huge.dasheng06b.10s",
+    "pretrained": false,
+    "target_length": 1008
+  },
+  "auto_map": {
+    "AutoConfig": "configuration_midashenglm.MiAudioLLMHFConfig",
+    "AutoModelForCausalLM": "modeling_midashenglm.DashengQwen25OmniModelInstruct"
+  },
+  "freeze": null,
+  "gradient_checkpoint_decoder": false,
+  "lora": null,
+  "model": "DashengQwen25OmniModelInstruct",
+  "model_type": "miaudiollm",
+  "resize_tokenizer": false,
+  "subsample_factor": 5,
+  "text_model": "Qwen/Qwen2.5-Omni-3B",
+  "text_model_args": {},
+  "torch_dtype": "float32",
+  "transformers_version": "4.52.0.dev0",
+  "use_encoderattention_mask": true
+}

configuration_midashenglm.py

@@ -0,0 +1,33 @@
+from typing import Literal
+
+from transformers import PretrainedConfig
+
+
+class MiAudioLLMHFConfig(PretrainedConfig):
+    model_type = "miaudiollm"
+
+    def __init__(
+        self,
+        model: str = "DashengQwen2ModelInstruct",
+        audio_encoder="LemonstoreWrapper",
+        audio_encoder_args=dict(
+            model_name="audiotransformer_base.dasheng.10s", pretrained=True
+        ),
+        text_model="Qwen/Qwen2.5-0.5B-Instruct",
+        text_model_args=dict(),
+        freeze: Literal["audio", "text"] | str | None = None,
+        lora: Literal["encoder", "decoder"] | None = None,
+        subsample_factor: int = 5,
+        use_encoderattention_mask: bool = True,
+        **kwargs,
+    ):
+        self.model = model
+        self.audio_encoder = audio_encoder
+        self.audio_encoder_args = audio_encoder_args
+        self.text_model = text_model
+        self.text_model_args = text_model_args
+        self.freeze = freeze
+        self.lora = lora
+        self.subsample_factor = subsample_factor
+        self.use_encoderattention_mask = use_encoderattention_mask
+        super().__init__(**kwargs)

model.safetensors.index.json

@@ -0,0 +1,843 @@
+{
+  "metadata": {
+    "total_size": 9383791884
+  },
+  "weight_map": {
+    "audio_encoder.model.blocks.0.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.0.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.0.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.0.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.0.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.0.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.0.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.0.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.0.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.0.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.0.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.0.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.1.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.1.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.1.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.1.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.1.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.1.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.1.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.1.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.1.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.1.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.1.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.1.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.10.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.10.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.10.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.10.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.10.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.10.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.10.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.10.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.10.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.10.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.10.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.10.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.11.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.11.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.11.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.11.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.11.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.11.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.11.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.11.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.11.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.11.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.11.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.11.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.12.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.12.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.12.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.12.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.12.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.12.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.12.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.12.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.12.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.12.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.12.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.12.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.13.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.13.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.13.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.13.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.13.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.13.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.13.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.13.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.13.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.13.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.13.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.13.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.14.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.14.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.14.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.14.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.14.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.14.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.14.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.14.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.14.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.14.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.14.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.14.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.15.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.15.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.15.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.15.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.15.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.15.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.15.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.15.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.15.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.15.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.15.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.15.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.16.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.16.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.16.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.16.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.16.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.16.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.16.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.16.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.16.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.16.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.16.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.16.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.17.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.17.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.17.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.17.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.17.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.17.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.17.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.17.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.17.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.17.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.17.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.17.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.18.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.18.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.18.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.18.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.18.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.18.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.18.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.18.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.18.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.18.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.18.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.18.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.19.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.19.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.19.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.19.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.19.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.19.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.19.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.19.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.19.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.19.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.19.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.19.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.2.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.2.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.2.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.2.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.2.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.2.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.2.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.2.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.2.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.2.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.2.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.2.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.20.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.20.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.20.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.20.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.20.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.20.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.20.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.20.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.20.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.20.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.20.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.20.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.21.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.21.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.21.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.21.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.21.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.21.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.21.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.21.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.21.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.21.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.21.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.21.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.22.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.22.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.22.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.22.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.22.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.22.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.22.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.22.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.22.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.22.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.22.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.22.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.23.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.23.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.23.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.23.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.23.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.23.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.23.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.23.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.23.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.23.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.23.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.23.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.24.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.24.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.24.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.24.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.24.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.24.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.24.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.24.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.24.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.24.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.24.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.24.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.25.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.25.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.25.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.25.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.25.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.25.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.25.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.25.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.25.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.25.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.25.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.25.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.26.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.26.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.26.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.26.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.26.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.26.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.26.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.26.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.26.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.26.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.26.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.26.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.27.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.27.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.27.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.27.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.27.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.27.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.27.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.27.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.27.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.27.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.27.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.27.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.28.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.28.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.28.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.28.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.28.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.28.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.28.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.28.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.28.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.28.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.28.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.28.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.29.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.29.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.29.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.29.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.29.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.29.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.29.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.29.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.29.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.29.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.29.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.29.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.3.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.3.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.3.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.3.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.3.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.3.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.3.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.3.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.3.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.3.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.3.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.3.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.30.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.30.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.30.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.30.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.30.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.30.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.30.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.30.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.30.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.30.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.30.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.30.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.31.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.31.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.31.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.31.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.31.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.31.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.31.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.31.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.31.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.31.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.31.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.31.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.4.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.4.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.4.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.4.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.4.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.4.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.4.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.4.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.4.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.4.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.4.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.4.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.5.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.5.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.5.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.5.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.5.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.5.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.5.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.5.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.5.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.5.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.5.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.5.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.6.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.6.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.6.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.6.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.6.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.6.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.6.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.6.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.6.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.6.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.6.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.6.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.7.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.7.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.7.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.7.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.7.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.7.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.7.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.7.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.7.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.7.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.7.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.7.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.8.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.8.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.8.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.8.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.8.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.8.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.8.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.8.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.8.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.8.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.8.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.8.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.9.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.9.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.9.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.9.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.9.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.9.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.9.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.9.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.9.norm1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.9.norm1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.9.norm2.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.blocks.9.norm2.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.freq_pos_embed": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.front_end.0.mel_scale.fb": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.front_end.0.spectrogram.window": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.init_bn.1.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.init_bn.1.num_batches_tracked": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.init_bn.1.running_mean": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.init_bn.1.running_var": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.init_bn.1.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.norm.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.norm.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.patch_embed.proj.bias": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.patch_embed.proj.weight": "model-00001-of-00002.safetensors",
+    "audio_encoder.model.time_pos_embed": "model-00001-of-00002.safetensors",
+    "audio_projector.net.0.bias": "model-00002-of-00002.safetensors",
+    "audio_projector.net.0.weight": "model-00002-of-00002.safetensors",
+    "audio_projector.net.2.bias": "model-00002-of-00002.safetensors",
+    "audio_projector.net.2.weight": "model-00002-of-00002.safetensors",
+    "decoder.lm_head.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.12.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.12.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.12.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.12.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.12.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.12.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.13.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.13.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.13.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.13.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.13.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.13.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.14.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.14.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.14.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.14.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.14.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.14.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.15.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.15.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.15.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.15.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.15.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.15.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.16.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.16.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.16.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.16.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.16.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.16.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.17.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.17.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.17.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.17.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.18.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.18.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.18.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.19.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.19.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.19.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.20.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.20.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.20.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.21.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.21.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.21.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.22.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.22.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.22.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.23.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.23.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.23.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.24.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.24.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.24.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.25.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.25.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.25.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.26.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.26.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.26.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.27.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.27.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.27.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.28.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.28.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.28.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.29.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.29.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.29.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.30.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.30.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.30.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.31.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.31.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.31.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.32.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.32.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.32.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.33.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.33.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.33.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.34.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.34.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.34.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.35.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.35.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.35.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "decoder.model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "decoder.model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "decoder.model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

modeling_midashenglm.py

@@ -0,0 +1,878 @@
+import collections.abc
+from functools import partial
+from typing import Any, Callable, Iterable, Literal, Optional, Tuple, Type, Union
+
+import torch
+import torch.nn as nn
+import torchaudio.transforms as audio_transforms
+from peft import LoraConfig, TaskType, get_peft_model
+from torch import Tensor
+from transformers import PreTrainedModel
+
+from .configuration_midashenglm import MiAudioLLMHFConfig
+
+
+class AudioProjectorSubsample(torch.nn.Module):
+    def __init__(self, in_dim: int, out_dim: int, downsample_rate=5):
+        super().__init__()
+        self.k = downsample_rate
+        self.net = torch.nn.Sequential(
+            torch.nn.Linear(in_dim * self.k, out_dim),
+            torch.nn.GELU(),
+            torch.nn.Linear(out_dim, out_dim),
+        )
+
+    def forward(self, x, mask=None):
+        """
+        inputs is the output of audio encoder.
+        :param x: [B, T, D]
+        :param x_lens: [B, T]
+        :return: [B, T', D']
+        """
+        batch_size, seq_len, dim = x.shape
+        num_frames_to_discard = seq_len % self.k
+        if num_frames_to_discard > 0:
+            x = x[:, :-num_frames_to_discard, :]
+            if mask is not None:
+                mask = mask[:, :-num_frames_to_discard]
+        if mask is None:
+            mask = torch.ones(x.shape[:-1], dtype=torch.long, device=x.device)
+        x = x.reshape(
+            batch_size, -1, self.k * dim
+        )  # rearrange(x, "b (s k) d -> b s (k d)", k=self.k)
+        x = self.net(x)
+        mask = mask.reshape(
+            batch_size, -1, self.k
+        )  # rearrange(mask, "b (s k) -> b s k", k=self.k)
+        mask = mask.any(dim=-1).long()
+        return x, mask
+
+
+# The functions `drop_path` and the module `DropPath` are taken from timm
+def drop_path(
+    x, drop_prob: float = 0.0, training: bool = False, scale_by_keep: bool = True
+):
+    """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
+    This is the same as the DropConnect impl I created for EfficientNet, etc networks, however,
+    the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
+    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for
+    changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use
+    'survival rate' as the argument.
+    """
+    if drop_prob == 0.0 or not training:
+        return x
+    keep_prob = 1 - drop_prob
+    shape = (x.shape[0],) + (1,) * (
+        x.ndim - 1
+    )  # work with diff dim tensors, not just 2D ConvNets
+    random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
+    if keep_prob > 0.0 and scale_by_keep:
+        random_tensor.div_(keep_prob)
+    return x * random_tensor
+
+
+class DropPath(nn.Module):
+    """Drop paths (Stochastic Depth) per sample  (when applied in main path of residual blocks)."""
+
+    def __init__(self, drop_prob: float = 0.0, scale_by_keep: bool = True):
+        super(DropPath, self).__init__()
+        self.drop_prob = drop_prob
+        self.scale_by_keep = scale_by_keep
+
+    def forward(self, x):
+        return drop_path(x, self.drop_prob, self.training, self.scale_by_keep)
+
+    def extra_repr(self):
+        return f"drop_prob={round(self.drop_prob, 3):0.3f}"
+
+
+def to_2tuple(x: Any) -> Tuple[Any, Any]:
+    if isinstance(x, collections.abc.Iterable):
+        return x
+    return (x, x)
+
+
+class AudioPatchEmbed(nn.Module):
+    def __init__(
+        self,
+        input_size: Union[int, Tuple[int, int]] = 64,
+        patch_size: Union[int, Tuple[int, int]] = 16,
+        patch_stride: Union[int, Tuple[int, int]] = 16,
+        in_chans: int = 1,
+        embed_dim: int = 768,
+        norm_layer: Optional[Callable] = None,
+        flatten: bool = False,
+    ):
+        super().__init__()
+        self.input_size = to_2tuple(input_size)
+        self.patch_size = to_2tuple(patch_size)
+        self.patch_stride = to_2tuple(patch_stride)
+        self.grid_size = (
+            self.input_size[0] // self.patch_stride[0],
+            self.input_size[1] // self.patch_stride[1],
+        )
+        self.num_patches = self.grid_size[0] * self.grid_size[1]
+        self.flatten = flatten
+
+        self.proj = nn.Conv2d(
+            in_chans, embed_dim, kernel_size=patch_size, stride=patch_stride
+        )
+        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
+
+    def forward(self, x):
+        x = self.proj(x)
+        if self.flatten:
+            x = torch.permute(
+                torch.flatten(x, 2, 3), (0, 2, 1)
+            )  # rearrange(x, "b c f t -> b (f t) c")
+        x = self.norm(x)
+        return x
+
+
+class LayerScale(nn.Module):
+    def __init__(self, dim, init_values=1e-5, inplace=False):
+        super().__init__()
+        self.inplace = inplace
+        self.gamma = nn.Parameter(init_values * torch.ones(dim))
+
+    def forward(self, x):
+        return x.mul_(self.gamma) if self.inplace else x * self.gamma
+
+
+class Mlp(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Type[torch.nn.Module] = nn.GELU,
+        drop: float = 0.0,
+    ):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+class Attention(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int = 8,
+        qkv_bias: bool = False,
+        attn_drop: float = 0.0,
+        proj_drop: float = 0.0,
+        causal: bool = False,
+    ):
+        super().__init__()
+        assert dim % num_heads == 0, "dim should be divisible by num_heads"
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = head_dim**-0.5
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+        self.causal = causal
+
+    def forward(self, x, mask: Optional[torch.Tensor] = None):
+        B, N, C = x.shape
+        qkv = (
+            self.qkv(x)
+            .reshape(B, N, 3, self.num_heads, C // self.num_heads)
+            .permute(2, 0, 3, 1, 4)
+        )
+        q, k, v = qkv.unbind(0)  # make torchscript happy (cannot use tensor as tuple)
+
+        attn = (q @ k.transpose(-2, -1)) * self.scale
+        # if mask is not None:
+        # # Mask is a tensor of shape [B, T, T]
+        # # Different from self.causal == True, the mask might be something like:
+        # # [False, False, True]
+        # # [False, False, True]
+        # # [True, True, True]
+        # # We use -inf to pad here, since if we would pad by any number, the entries at rows only containing
+        # # [True, True, True] would lead to weights such as: [0.33,0.33,0.33], which is not correct
+        if self.causal:
+            mask_value = -torch.finfo(attn.dtype).max
+            i, j = attn.shape[-2:]
+            mask = torch.ones(i, j, device=q.device, dtype=torch.bool).triu(j - i + 1)
+            attn = attn.masked_fill(mask, mask_value)
+        if mask is not None:
+            # mask value as the lowest possible value in fp32
+            mask_value = torch.finfo(attn.dtype).min
+            # Mask is of shape [1, SRC_LEN]
+            attn_mask = mask[:, None, None, :].expand(B, 1, N, N)
+            # Mask should be of shape
+            # [B,1,Target_len, Source_len]
+            attn = attn.masked_fill(attn_mask, mask_value)
+        attn = attn.softmax(dim=-1)
+        attn = torch.nan_to_num(attn)
+        # Only for the case that a mask with all True entries on a row is passed.
+        # attn = torch.nan_to_num(attn)
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class Block(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int,
+        mlp_ratio: float = 4.0,
+        qkv_bias: bool = False,
+        drop: float = 0.0,
+        attn_drop: float = 0.0,
+        init_values=None,
+        drop_path: float = 0.0,
+        act_layer: Type[torch.nn.Module] = nn.GELU,
+        norm_layer: Type[torch.nn.Module] = nn.LayerNorm,
+        attention_type: Type[torch.nn.Module] = Attention,
+    ):
+        super().__init__()
+        self.norm1 = norm_layer(dim)
+        self.attn = attention_type(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            attn_drop=attn_drop,
+            proj_drop=drop,
+        )
+        self.ls1 = (
+            LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        )
+        self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+        self.norm2 = norm_layer(dim)
+        self.mlp = Mlp(
+            in_features=dim,
+            hidden_features=int(dim * mlp_ratio),
+            act_layer=act_layer,
+            drop=drop,
+        )
+        self.ls2 = (
+            LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        )
+        self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+    # Kwargs usually has a mask parameter that is passed to Attention
+    def forward(self, x, **kwargs):
+        x = x + self.drop_path1(self.ls1(self.attn(self.norm1(x), **kwargs)))
+        x = x + self.drop_path2(self.ls2(self.mlp(self.norm2(x))))
+        return x
+
+
+class RearranceReplace(nn.Module):
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # rearrange(x, "b c f t -> b f c t")
+        # or
+        # rearrange(x, "b f c t -> b c f t")
+        return torch.permute(x, (0, 2, 1, 3))
+
+
+class AudioTransformer(nn.Module):
+    def __init__(
+        self,
+        outputdim: int = 527,
+        patch_size: Union[int, Tuple[int, int]] = 16,
+        patch_stride: Union[int, Tuple[int, int]] = 16,
+        embed_dim: int = 768,
+        depth: int = 12,
+        num_heads: int = 12,
+        mlp_ratio: float = 4.0,
+        qkv_bias: bool = True,
+        drop_rate: float = 0.0,
+        attn_drop_rate: float = 0.0,
+        drop_path_rate: float = 0.0,
+        norm_layer: torch.nn.Module | None = None,
+        act_layer: Type[torch.nn.Module] = nn.GELU,
+        init_values=None,
+        target_length: int = 1012,
+        input_channels: int = 1,
+        pooling: Literal["mean", "token", "dm", "logit", "cat"] | None = "token",
+        time_patch_out: float | None = None,
+        freq_patch_out: float | None = None,
+        block_type: Type[torch.nn.Module] = Block,
+        attention_type: Type[torch.nn.Module] = Attention,
+        eval_avg: Literal["mean", "max", "cat"] = "mean",
+        n_mels: int = 64,
+        n_fft: int = 512,
+        hop_size: int = 160,
+        win_size: int = 512,
+        f_min: float = 0.0,
+        f_max: float = 8000.0,
+        sample_rate: int = 16000,
+        center: bool = True,
+        pad_last: bool = True,
+    ):
+        super().__init__()
+        assert pooling in ("mean", "token", "dm", "logit", "cat", None)
+        self.outputdim = outputdim
+        self.pooling = pooling
+        self.embed_dim = embed_dim
+        self.patch_stride = patch_stride
+        self.patch_size = patch_size
+        self.n_mels = n_mels
+        self.n_fft = n_fft
+        self.hop_size = hop_size
+        self.win_size = win_size
+        self.f_min = f_min
+        self.f_max = f_max
+        self.sample_rate = sample_rate
+        self.center = center
+        self.pad_last = pad_last
+        self.input_channels = input_channels
+        self.eval_avg = eval_avg
+        self.time_patch_out = time_patch_out
+        self.freq_patch_out = freq_patch_out
+
+        self.front_end = nn.Sequential(
+            audio_transforms.MelSpectrogram(
+                f_min=self.f_min,
+                f_max=self.f_max,
+                center=self.center,
+                win_length=self.win_size,
+                hop_length=self.hop_size,
+                sample_rate=self.sample_rate,
+                n_fft=self.n_fft,
+                n_mels=self.n_mels,
+            ),
+            audio_transforms.AmplitudeToDB(top_db=120),
+        )
+
+        self.init_bn = nn.Sequential(
+            # Rearrange("b c f t -> b f c t"),
+            RearranceReplace(),
+            torch.nn.BatchNorm2d(self.n_mels, momentum=0.01),
+            # Rearrange("b f c t -> b c f t"),
+            RearranceReplace(),
+        )
+
+        self.target_length = target_length
+
+        patch_stride = to_2tuple(self.patch_stride)[-1]
+        # Allowed length in number of frames, otherwise the positional embedding will throw an error
+        self.maximal_allowed_length = self.target_length
+
+        self.patch_embed = AudioPatchEmbed(
+            input_size=(self.n_mels, target_length),
+            embed_dim=self.embed_dim,
+            in_chans=self.input_channels,
+            patch_size=self.patch_size,
+            flatten=False,
+            patch_stride=self.patch_stride,
+        )
+
+        if self.pooling == "token":
+            self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+            self.token_pos_embed = nn.Parameter(torch.randn(1, embed_dim) * 0.02)
+
+        self.time_pos_embed = nn.Parameter(
+            torch.randn(1, embed_dim, 1, self.patch_embed.grid_size[1]) * 0.02
+        )
+        self.freq_pos_embed = nn.Parameter(
+            torch.randn(1, embed_dim, self.patch_embed.grid_size[0], 1) * 0.02
+        )
+
+        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
+        act_layer = act_layer or nn.GELU
+        dpr = [
+            x.item() for x in torch.linspace(0, drop_path_rate, depth)
+        ]  # stochastic depth decay rule
+        self.pos_drop = nn.Dropout(p=drop_rate)
+        self.blocks = nn.ModuleList(
+            block_type(
+                dim=embed_dim,
+                num_heads=num_heads,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                init_values=init_values,
+                drop=drop_rate,
+                attn_drop=attn_drop_rate,
+                drop_path=dpr[i],
+                norm_layer=norm_layer,
+                act_layer=act_layer,
+                attention_type=attention_type,
+            )
+            for i in range(depth)
+        )
+        self.norm = norm_layer(embed_dim)
+        if hasattr(self, "cls_token") and self.cls_token is not None:
+            nn.init.normal_(self.cls_token, std=1e-6)
+
+    def forward_features(self, x: torch.Tensor, **kwargs) -> torch.Tensor:
+        t = x.shape[-1]
+        x = x + self.time_pos_embed[:, :, :, :t]
+        x = (
+            x + self.freq_pos_embed[:, :, :, :]
+        )  # Just to support __getitem__ in posembed
+        x = torch.permute(
+            torch.flatten(x, 2, 3), (0, 2, 1)
+        )  # rearrange(x, "b c f t -> b (f t) c")
+        if self.pooling == "token":
+            cls_token = self.cls_token.expand(x.shape[0], -1, -1)
+            cls_token = cls_token + self.token_pos_embed
+            x = torch.cat((cls_token, x), dim=1)
+        x = self.pos_drop(x)
+        for block in self.blocks:
+            x = block(x, **kwargs)
+        x = self.norm(x)
+        return x
+
+    # TODO
+    # ================ 从此行开始，与Dasheng代码严重分歧 ================
+
+    def forward_head(self, x: torch.Tensor, **kwargs) -> torch.Tensor:
+        mask = kwargs.get("mask", None)
+        if self.pooling == "token":
+            x = x[:, 0]
+            return x.sigmoid()
+        elif self.pooling == "mean":
+            if mask is not None:
+                m = (1.0 - mask.float()).unsqueeze(-1)  # 1.0 means is masked
+                x = torch.nan_to_num((x * m).sum(1) / m.sum(1))
+            else:
+                x = x.mean(1)
+            return x.sigmoid()
+        elif self.pooling == "logit":
+            if mask is not None:
+                m = (1.0 - mask.float()).unsqueeze(-1)  # 1.0 means is masked
+                x = torch.nan_to_num((x * m).sum(1) / m.sum(1))
+            else:
+                x = x.mean(1)
+            return x
+        elif self.pooling == "dm":
+            # Unpack using the frequency dimension, which is constant
+            b, _, d = x.shape
+            x = x.reshape(
+                b, -1, self.patch_embed.grid_size[0], d
+            )  # rearrange(x, "b (f t) d -> b f t d")
+            # First poolin frequency, then sigmoid the (B T D) output
+            x = (x.mean(1)).sigmoid()
+            return x.mean(1)
+        elif self.pooling is None:
+            return x
+        else:
+            return x.mean(1)
+
+    def _audiosample_to_mellength(self, lengths: torch.Tensor) -> torch.Tensor:
+        if self.center:
+            lengths = lengths + self.win_size
+        lengths = 1 + ((lengths - self.win_size) / self.hop_size).long()
+        return lengths
+
+    # Calculates the number of patches for a given length in audio-samples
+    # For example : torch.Tensor([16000]) will return 250 for Dasheng
+    def _audiosample_to_patchlength(self, lengths: torch.Tensor) -> torch.Tensor:
+        lengths = self._audiosample_to_mellength(lengths)
+        return self._frames_to_patchlength(lengths)
+
+    # Calcualtes the same as above but for a spectrogram input
+    # i.e., [100] will return 25 for Dasheng
+    def _frames_to_patchlength(self, lengths: torch.Tensor) -> torch.Tensor:
+        patch_stride = to_2tuple(self.patch_stride)
+        patch_size = to_2tuple(self.patch_size)
+        frequency_patch_size = self.n_mels // patch_stride[0]
+        time_patch_size = patch_stride[1]
+        time_window_size = patch_size[1]
+        number_of_tokens = (
+            torch.floor((lengths - time_window_size) / time_patch_size) + 1
+        ) * frequency_patch_size
+        if self.pooling == "token":
+            number_of_tokens += 1
+        return number_of_tokens
+
+    # Note that we use (... t f) -> (f t)  here, meaning that patches are ordered as:
+    # 0 4 -> 0 4 1 5 2 6 3 7
+    # 1 5
+    # 2 6
+    # 3 7
+    # This function does the  (t f) -> (f t) transform
+    def _reshape_mask_to_ft_format(self, mask: torch.Tensor) -> torch.Tensor:
+        n_freq_patches = self.n_mels // to_2tuple(self.patch_stride)[0]
+        mask = (
+            mask.reshape(-1, n_freq_patches)
+            .transpose(-2, -1)
+            .flatten(-2)
+            .reshape_as(mask)
+        )
+        return mask
+
+    def _to_binary_mask(self, lengths: torch.Tensor, max_length: int) -> torch.Tensor:
+        batch_size = len(lengths)
+        lengths = self._audiosample_to_patchlength(lengths)
+        idx = torch.arange(max_length, device=lengths.device)
+        idx = idx.repeat(batch_size).view(batch_size, max_length)
+        mask = (idx >= lengths.unsqueeze(-1)).bool()
+        return mask
+
+    def _to_mask(self, lengths: torch.Tensor, max_length: int) -> torch.Tensor:
+        batch_size = len(lengths)
+        idx = torch.arange(max_length, device=lengths.device)
+        idx = idx.repeat(batch_size).view(batch_size, max_length)
+        mask = (idx >= lengths.unsqueeze(-1)).bool()
+        return mask
+
+    def _create_mask(self, x_length, audio_length_in_spec_frames: int):
+        max_length_in_patches = self._frames_to_patchlength(
+            torch.tensor(audio_length_in_spec_frames)
+        )
+        mask_1d = self._to_binary_mask(x_length, max_length=int(max_length_in_patches))
+        return mask_1d
+
+    def forward(
+        self, x: torch.Tensor, x_length: Optional[torch.Tensor] = None
+    ) -> torch.Tensor:
+        x = self.front_end(x)
+        target_length_in_patches = self.target_length // 4
+        x = x.unsqueeze(1)
+        x = self.init_bn(x)
+
+        x = self.patch_embed(x)
+        t = x.shape[-1]
+
+        input_splits = x.split(target_length_in_patches, dim=-1)
+        mask = None  # Single mask
+        masks = [None for _ in range(len(input_splits))]
+
+        if x_length is not None:
+            assert len(x_length) == len(x), (
+                "batchsizes of input x and x_length need to be same"
+            )
+            assert x_length.ndim == 1, "Lengths are of size (B,)"
+            scaled_lengths = (
+                x_length / (self.hop_size * 4)
+            ).long()  # 40ms for all dasheng models
+            # Note that the mask is in (t f) format, but transformers here use (f t) format
+            mask = self._to_mask(
+                max_length=t,
+                lengths=scaled_lengths,
+            )
+            # Trim mask to only use valid "patches", since x.shape[-1] is based on the possibly padded input
+            masks = mask.split(target_length_in_patches, dim=-1)
+
+        outputs = []
+
+        for split_x, mask in zip(input_splits, masks):
+            forward_kwargs = {}
+            forward_kwargs["mask"] = mask
+            split_x = self.forward_features(split_x, **forward_kwargs)
+            split_x = self.forward_head(split_x, **forward_kwargs)
+            outputs.append(split_x)
+        x = torch.cat(outputs, dim=1)
+        return x
+
+
+class LemonstoreWrapper(nn.Module):
+    def __init__(
+        self,
+        append_cls_token: bool = False,
+        **kwargs,
+    ):
+        super().__init__()
+        self.append_cls_token = (
+            append_cls_token  # Pool all tokens to one as a "cls" token
+        )
+
+        model_default_kwargs = {
+            "audiotransformer_huge.dasheng06b.10s": {
+                "embed_dim": 1280,
+                "depth": 32,
+                "num_heads": 16,
+                "pooling": "mean",
+                "drop_path_rate": 0.0,
+                "outputdim": 527,
+                "patch_size": [64, 4],
+                "patch_stride": [64, 4],
+                "target_length": 1008,
+            }
+        }
+        if "pretrained" in kwargs:
+            del kwargs["pretrained"]
+
+        create_kwargs = model_default_kwargs[kwargs.pop("model_name")]
+        create_kwargs.update(kwargs)
+        create_kwargs.update(
+            pooling=None,
+            eval_avg="cat",
+        )
+
+        self.model = AudioTransformer(**create_kwargs)
+        self.embed_dim = self.model.embed_dim
+
+    def _to_mask(self, lengths: torch.Tensor, max_length: int) -> torch.Tensor:
+        batch_size = len(lengths)
+        idx = torch.arange(max_length, device=lengths.device)
+        idx = idx.repeat(batch_size).view(batch_size, max_length)
+        mask = (idx < lengths.unsqueeze(-1)).long()
+        return mask
+
+    def _create_encoder_attention_mask(
+        self, model_output: torch.Tensor, input_lengths: torch.Tensor
+    ):
+        scaled_lengths = (
+            input_lengths / (self.model.hop_size * 4)
+        ).long()  # 40ms for all dasheng models
+        return self._to_mask(scaled_lengths, max_length=model_output.shape[1])
+
+    def forward(
+        self,
+        input: torch.Tensor,
+        input_length: Optional[torch.Tensor] = None,
+        return_attention_mask: bool = False,
+    ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
+        emb = self.model(input, input_length)
+        # Outputs are added to multiple of 10s, remove the padded items
+        if input_length is not None:
+            input_length = input_length + self.model.n_fft
+            scaled_lengths = (
+                (1 + (input_length - self.model.n_fft) / self.model.hop_size) // 4
+            ).long()  # 40ms for all dasheng models
+            max_length = torch.max(scaled_lengths)
+            emb = emb[:, :max_length, :]
+        if self.append_cls_token:
+            emb = torch.cat([emb.mean(1, keepdims=True), emb], dim=1)
+        if return_attention_mask and input_length is not None:
+            return emb, self._create_encoder_attention_mask(emb, input_length)
+        return emb
+
+
+class DashengQwen25OmniModelInstruct(PreTrainedModel):
+    config_class = MiAudioLLMHFConfig
+
+    def __init__(self, config: MiAudioLLMHFConfig):
+        super().__init__(config)
+
+        audio_encoder = config.audio_encoder
+        audio_encoder_args = config.audio_encoder_args
+        text_model = config.text_model
+        text_model_args = config.text_model_args
+        freeze = config.freeze
+        lora = config.lora
+        subsample_factor = config.subsample_factor
+        use_encoderattention_mask = config.use_encoderattention_mask
+        resize_tokenizer = True
+        lora_r = 8
+        lora_target_modules = ("q_proj", "v_proj")
+        force_fp32 = False
+
+        from transformers.models.qwen2_5_omni import (
+            Qwen2_5OmniProcessor,
+            Qwen2_5OmniThinkerForConditionalGeneration,
+        )
+
+        self.subsample_factor = subsample_factor
+        self.lora = lora
+        self.use_encoderattention_mask = use_encoderattention_mask
+        # Encoder part
+        assert audio_encoder == "LemonstoreWrapper"
+        self.audio_encoder = LemonstoreWrapper(**audio_encoder_args)
+        if lora == "encoder":
+            encoder_peft_config = LoraConfig(
+                target_modules=["q_proj", "v_proj"],
+                inference_mode=False,
+                r=8,
+                lora_alpha=32,
+                lora_dropout=0.1,
+            )
+            self.audio_encoder = get_peft_model(self.audio_encoder, encoder_peft_config)
+
+        # For some reason, torch.cuda.is_bf16_supported() does return True on V100
+        has_bf16support = torch.cuda.get_device_capability(torch.device("cuda"))[0] > 7
+
+        # decoder
+        self.processor = Qwen2_5OmniProcessor.from_pretrained(text_model)
+        self.tokenizer = self.processor.tokenizer
+        self.decoder = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained(
+            text_model,
+            attn_implementation="sdpa",
+            torch_dtype=torch.bfloat16
+            if not force_fp32 and has_bf16support
+            else torch.float32,
+            **text_model_args,
+        )
+        del self.decoder.visual
+        del self.decoder.audio_tower
+        hidden_size_text = self.decoder.model.config.hidden_size
+        # Overwrite default ForCausalLMLoss, now also support reduction
+        special_tokens = [
+            "<|en|>",
+            "<|kr|>",
+            "<|de|>",
+            "<|es|>",
+            "<|fr|>",
+            "<|hi|>",
+            "<|uk|>",
+            "<|th|>",
+            "<|vi|>",
+            "<|nl|>",
+            "<|pt|>",
+            "<|id|>",
+            "<|ru|>",
+            "<|it|>",
+            "<|ar|>",
+            "<|jp|>",
+            "<|unknown|>",
+        ]
+        self.tokenizer.add_special_tokens({"additional_special_tokens": special_tokens})
+        if resize_tokenizer:
+            self.decoder.model.resize_token_embeddings(len(self.tokenizer))
+        if lora == "decoder":
+            peft_config = LoraConfig(
+                target_modules=lora_target_modules,
+                task_type=TaskType.CAUSAL_LM,
+                trainable_token_indices={
+                    "embed_tokens": self.tokenizer.convert_tokens_to_ids(special_tokens)
+                },
+                inference_mode=False,
+                r=lora_r,
+                lora_alpha=32,
+                lora_dropout=0.1,
+            )
+            self.decoder = get_peft_model(self.decoder, peft_config)
+            self.decoder.print_trainable_parameters()
+        if freeze is not None and "text" in freeze:
+            lora_config = LoraConfig(
+                target_modules="dummy-target-modules",
+                trainable_token_indices={
+                    "embed_tokens": self.tokenizer.convert_tokens_to_ids(special_tokens)
+                },
+            )
+            self.decoder = get_peft_model(self.decoder, lora_config)
+            self.decoder.print_trainable_parameters()
+
+        # audio projector
+        self.audio_projector = AudioProjectorSubsample(
+            self.audio_encoder.embed_dim, hidden_size_text, self.subsample_factor
+        )
+
+    def _forward_audio_encoder(self, audios, audio_length: Iterable[int] | None):
+        encoder_out = self.audio_encoder(
+            audios, audio_length, return_attention_mask=self.use_encoderattention_mask
+        )
+        encoder_atts = None
+
+        if self.use_encoderattention_mask:
+            encoder_out, encoder_atts = encoder_out
+
+        # audio projector
+        encoder_out, encoder_atts = self.audio_projector(encoder_out, encoder_atts)
+
+        return encoder_out, encoder_atts
+
+    def _prepare_with_input_ids(
+        self, input_ids: torch.Tensor, audio_embeddings, audio_token_id
+    ):
+        special_mask = input_ids == audio_token_id
+        assert audio_embeddings.shape[1] <= (special_mask.sum(-1)).max(), (
+            "Mask and audio embeddings seem to have different sizes"
+        )
+        input_embeddings = self.decoder.model.embed_tokens(input_ids)
+        audio_embeddings = audio_embeddings.to(input_embeddings.dtype)
+
+        for i in range(len(special_mask)):
+            mask = special_mask[i]
+            number_of_tokens = mask.sum(-1)
+            input_embeddings[i, mask] = audio_embeddings[i, :number_of_tokens]
+        return input_embeddings
+
+    def forward(
+        self,
+        input_ids: Tensor,
+        input_values: Tensor,
+        audio_length: Iterable[int] | None,
+        return_loss: bool = False,
+        attention_mask: Tensor | None = None,
+        audio_token_id: int | None = None,
+    ):
+        input_values = input_values.to(self.device)
+        audio_encoder_hidden_states, _ = self._forward_audio_encoder(
+            input_values, audio_length=audio_length
+        )
+
+        input_ids = input_ids.to(self.device)
+        input_embeds = self._prepare_with_input_ids(
+            input_ids=input_ids,
+            audio_embeddings=audio_encoder_hidden_states,
+            audio_token_id=audio_token_id,
+        )
+        input_mask = attention_mask
+        decoder_targets = torch.nn.functional.pad(input_ids[:, 1:], (0, 1), value=-100)
+
+        decoder_output = self.decoder(
+            input_ids=None,
+            inputs_embeds=input_embeds,
+            attention_mask=input_mask,
+            labels=decoder_targets,
+            return_dict=True,
+        )
+
+        if return_loss:
+            return decoder_output.loss
+        return decoder_output.logits
+
+    def generate(
+        self,
+        input_ids: Tensor,
+        input_values: Tensor,
+        audio_length: Iterable[int] | None,
+        use_nucleus_sampling=False,
+        max_length=1024,
+        top_p=1.0,
+        top_k: int = 50,
+        temperature: float = 1.0,
+        repetition_penalty=1.0,
+        return_text=True,
+        # The following are only used by HF
+        attention_mask: Tensor | None = None,
+        audio_token_id: int | None = None,
+    ):
+        encoder_hidden_states, encoder_atts = self._forward_audio_encoder(
+            input_values, audio_length=audio_length
+        )
+
+        input_ids = input_ids.to(self.device)
+        input_embeds = self._prepare_with_input_ids(
+            input_ids=input_ids,
+            audio_embeddings=encoder_hidden_states,
+            audio_token_id=audio_token_id,
+        )
+        input_mask = attention_mask
+
+        outputs = self.decoder.generate(
+            inputs_embeds=input_embeds,
+            attention_mask=input_mask,
+            do_sample=use_nucleus_sampling,
+            max_new_tokens=max_length,
+            top_p=top_p,
+            top_k=top_k,
+            temperature=temperature,
+            repetition_penalty=repetition_penalty,
+            eos_token_id=[self.tokenizer.pad_token_id, self.tokenizer.eos_token_id],
+            pad_token_id=self.tokenizer.pad_token_id,
+        )
+        if not return_text:
+            return outputs
+        texts = self.tokenizer.batch_decode(
+            outputs,
+            add_special_tokens=False,
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=True,
+        )
+        return texts

preprocessor_config.json

@@ -0,0 +1,13 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_midashenglm.MiAudioLLMProcessor"
+  },
+  "do_normalize": false,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "processor_class": "MiAudioLLMProcessor",
+  "return_attention_mask": false,
+  "sampling_rate": 16000
+}

processing.py

@@ -0,0 +1,277 @@
+from __future__ import annotations
+
+from typing import List
+
+import numpy as np
+import torch
+from transformers import Qwen2Tokenizer, Qwen2TokenizerFast, Wav2Vec2FeatureExtractor
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
+
+
+class MiAudioLLMProcessorKwargs(ProcessingKwargs):
+    _defaults = {
+        "text_kwargs": {
+            "padding": True,
+            "padding_side": "left",
+        },
+        "audio_kwargs": {},
+    }
+
+
+def calculate_mel_frames_dasheng(
+    audio_length_samples: int,
+    n_fft: int = 512,
+    hop_size: int = 160,
+    dasheng_subsampling: int = 4,
+    center=True,
+    model_subsampling: int = 5,
+) -> int:
+    """Calculate the number of Mel-spectrogram frames."""
+    if center:
+        audio_length_samples = audio_length_samples + n_fft
+
+    return (
+        int(1 + ((audio_length_samples - n_fft) / hop_size))
+        // dasheng_subsampling
+        // model_subsampling
+    )
+
+
+class MiAudioLLMProcessor(ProcessorMixin):
+    attributes = ["feature_extractor", "tokenizer"]
+    valid_kwargs = [
+        "chat_template",
+        "audio_token",
+        "audio_bos_token",
+        "audio_eos_token",
+    ]
+    feature_extractor_class = "Wav2Vec2FeatureExtractor"
+    tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
+
+    def __init__(
+        self,
+        feature_extractor: Wav2Vec2FeatureExtractor | None = None,
+        tokenizer: Qwen2Tokenizer | Qwen2TokenizerFast | None = None,
+        model_subsampling: int = 5,
+        chat_template: str | None = None,
+        # TODO 是否可以移除？
+        audio_token: str = "<|AUDIO|>",
+        audio_bos_token: str = "<|audio_bos|>",
+        audio_eos_token: str = "<|audio_eos|>",
+    ):
+        if chat_template is None:
+            chat_template = self.default_chat_template
+        assert tokenizer is not None, "Tokenizer Needs to be passed"
+        self.audio_token = (
+            tokenizer.audio_token if hasattr(tokenizer, "audio_token") else audio_token
+        )
+        self.audio_token_id = tokenizer.convert_tokens_to_ids(self.audio_token)
+        self.audio_bos_token = (
+            tokenizer.audio_bos_token
+            if hasattr(tokenizer, "audio_bos_token")
+            else audio_bos_token
+        )
+        self.audio_eos_token = (
+            tokenizer.audio_eos_token
+            if hasattr(tokenizer, "audio_eos_token")
+            else audio_eos_token
+        )
+        self.model_subsampling = model_subsampling
+        # Fix Normalization
+        if feature_extractor is not None and feature_extractor.do_normalize is True:
+            feature_extractor.do_normalize = False
+        super().__init__(feature_extractor, tokenizer, chat_template=chat_template)
+
+    def __call__(
+        self,
+        text: List[str] | None = None,
+        audio: List[np.ndarray] | List[torch.Tensor] | None = None,
+        **kwargs: Unpack[MiAudioLLMProcessorKwargs],
+    ) -> BatchFeature:
+        if text is None:
+            raise ValueError("You need to specify `text` input to process.")
+        elif isinstance(text, str):
+            text = [text]
+        elif not isinstance(text, list) and not isinstance(text[0], str):
+            raise ValueError(
+                "Invalid input text. Please provide a string, or a list of strings"
+            )
+
+        output_kwargs = self._merge_kwargs(
+            MiAudioLLMProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+
+        if audio is not None:
+            if isinstance(audio[0], torch.Tensor):
+                audio = [sample_.numpy() for sample_ in audio]
+
+            if isinstance(audio[0], torch.Tensor):
+                audio = [sample_.squeeze(0) for sample_ in audio]
+                if not all(x_.ndim == 1 for x_ in audio):
+                    raise ValueError("All samples in a list must be 1D.")
+            if isinstance(audio[0], np.ndarray):
+                if not all(x_.ndim == 1 for x_ in audio):
+                    raise ValueError("All samples in a list must be 1D.")
+            # ensure we have as much audios as audio tokens
+            num_audio_tokens = sum(sample.count(self.audio_token) for sample in text)
+            num_audios = 1 if type(audio) is np.ndarray else len(audio)
+            if num_audio_tokens != num_audios:
+                raise ValueError(
+                    f"Found {num_audio_tokens} {self.audio_token} token{'s' if num_audio_tokens > 1 else ''} in provided text but received {num_audios} audio{'s' if num_audios > 1 else ''}"
+                )
+
+            # Some kwargs should not be changed so we can expand text with audio tokens below
+            output_kwargs["audio_kwargs"]["return_attention_mask"] = True
+            output_kwargs["audio_kwargs"]["padding"] = True
+            output_kwargs["audio_kwargs"]["return_tensors"] = "pt"
+
+            # + Padding
+            audio_inputs = self.feature_extractor(
+                audio, **output_kwargs["audio_kwargs"]
+            )
+
+            # remove attention mask, dasheng uses lengths
+            audio_feature_mask = audio_inputs.pop("attention_mask")
+
+            expanded_text = []
+            audio_lengths = audio_feature_mask.sum(-1).tolist()
+            audio_inputs["audio_length"] = torch.tensor(audio_lengths).long()
+            audio_inputs["audio_token_id"] = (
+                self.audio_token_id
+            )  # Pass to the model such that i knows what is the placeholder id
+
+            for sample in text:
+                replace_str = []
+                while self.audio_token in sample:
+                    audio_length = audio_lengths.pop(0)
+                    num_audio_tokens = calculate_mel_frames_dasheng(
+                        audio_length, model_subsampling=self.model_subsampling
+                    )
+
+                    expanded_audio_token = self.audio_token * num_audio_tokens
+
+                    audio_token_start_idx = sample.find(self.audio_token)
+                    audio_token_end_idx = audio_token_start_idx + len(self.audio_token)
+
+                    has_bos = (
+                        sample[
+                            audio_token_start_idx
+                            - len(self.audio_bos_token) : audio_token_start_idx
+                        ]
+                        == self.audio_bos_token
+                    )
+                    has_eos = (
+                        sample[
+                            audio_token_end_idx : audio_token_end_idx
+                            + len(self.audio_eos_token)
+                        ]
+                        == self.audio_eos_token
+                    )
+
+                    # Check if this audio token is surrounded by bos/eos tokens
+                    if not has_bos and not has_eos:
+                        expanded_audio_token = (
+                            self.audio_bos_token
+                            + expanded_audio_token
+                            + self.audio_eos_token
+                        )
+
+                    replace_str.append(expanded_audio_token)
+                    sample = sample.replace(self.audio_token, "<placeholder>", 1)
+
+                while "<placeholder>" in sample:
+                    sample = sample.replace("<placeholder>", replace_str.pop(0), 1)
+                expanded_text.append(sample)
+            text = expanded_text
+
+        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", "pt")
+        inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
+        if hasattr(self, "_check_special_mm_tokens"):
+            self._check_special_mm_tokens(text, inputs, modalities=["audio"])
+
+        if audio is not None:
+            inputs.update(audio_inputs)
+
+        return BatchFeature(data={**inputs}, tensor_type=return_tensors)
+
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+        the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, **kwargs)
+
+    @property
+    def model_input_names(self):
+        tokenizer_input_names = self.tokenizer.model_input_names
+        feature_extractor_input_names = self.feature_extractor.model_input_names
+        return list(
+            dict.fromkeys(
+                tokenizer_input_names + feature_extractor_input_names + ["audio_length"]
+            )
+        )
+
+    @property
+    # NOTE: we don't have default templates anymore, and the below is kept only because the hub config is not yet updated!
+    def default_chat_template(self):
+        """
+        This default vicuna template formats inputs in the form of a chat history. For each message in the chat history:
+        * the template will output the role of the speaker followed by the content of the message.
+        * content is a list of strings and audios.
+        * If the content element is an audio, the template will output a sequence of <|AUDIO|> tokens
+
+        Example:
+
+        ```python
+        messages = [
+            {'role': 'system', 'content': 'You are a helpful assistant.'},
+            {"role": "user", "content": [
+                {"type": "audio", "audio_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/glass-breaking-151256.mp3"},
+                {"type": "text", "text": "What's that sound?"},
+            ]},
+            {"role": "assistant", "content": "It is the sound of glass shattering."},
+            {"role": "user", "content": [
+                {"type": "audio", "audio_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/f2641_0_throatclearing.wav"},
+                {"type": "text", "text": "How about this one?"},
+            ]},
+        ]
+
+        result = template.render(messages=messages, add_generation_prompt=True)
+        ```
+        """
+        # fmt: off
+        return (
+            "{% set audio_count = namespace(value=0) %}"
+            "{% for message in messages %}"
+                "{% if loop.first and message['role'] != 'system' %}"
+                    "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
+                "{% endif %}"
+                "<|im_start|>{{ message['role'] }}\n"
+                "{% if message['content'] is string %}"
+                    "{{ message['content'] }}<|im_end|>\n"
+                "{% else %}"
+                    "{% for content in message['content'] %}"
+                        "{% if 'audio' in content or 'audio_url' in content or message['type'] == 'audio' %}"
+                            "{% set audio_count.value = audio_count.value + 1 %}"
+                            "Audio {{ audio_count.value }}: <|audio_bos|><|AUDIO|><|audio_eos|>\n"
+                        "{% elif 'text' in content %}"
+                            "{{ content['text'] }}"
+                        "{% endif %}"
+                    "{% endfor %}"
+                    "<|im_end|>\n"
+                "{% endif %}"
+            "{% endfor %}"
+            "{% if add_generation_prompt %}"
+                "<|im_start|>assistant\n"
+            "{% endif %}"
+        )

processing_midashenglm.py

@@ -0,0 +1,277 @@
+from __future__ import annotations
+
+from typing import List
+
+import numpy as np
+import torch
+from transformers import Qwen2Tokenizer, Qwen2TokenizerFast, Wav2Vec2FeatureExtractor
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
+
+
+class MiAudioLLMProcessorKwargs(ProcessingKwargs):
+    _defaults = {
+        "text_kwargs": {
+            "padding": True,
+            "padding_side": "left",
+        },
+        "audio_kwargs": {},
+    }
+
+
+def calculate_mel_frames_dasheng(
+    audio_length_samples: int,
+    n_fft: int = 512,
+    hop_size: int = 160,
+    dasheng_subsampling: int = 4,
+    center=True,
+    model_subsampling: int = 5,
+) -> int:
+    """Calculate the number of Mel-spectrogram frames."""
+    if center:
+        audio_length_samples = audio_length_samples + n_fft
+
+    return (
+        int(1 + ((audio_length_samples - n_fft) / hop_size))
+        // dasheng_subsampling
+        // model_subsampling
+    )
+
+
+class MiAudioLLMProcessor(ProcessorMixin):
+    attributes = ["feature_extractor", "tokenizer"]
+    valid_kwargs = [
+        "chat_template",
+        "audio_token",
+        "audio_bos_token",
+        "audio_eos_token",
+    ]
+    feature_extractor_class = "Wav2Vec2FeatureExtractor"
+    tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
+
+    def __init__(
+        self,
+        feature_extractor: Wav2Vec2FeatureExtractor | None = None,
+        tokenizer: Qwen2Tokenizer | Qwen2TokenizerFast | None = None,
+        model_subsampling: int = 5,
+        chat_template: str | None = None,
+        # TODO 是否可以移除？
+        audio_token: str = "<|AUDIO|>",
+        audio_bos_token: str = "<|audio_bos|>",
+        audio_eos_token: str = "<|audio_eos|>",
+    ):
+        if chat_template is None:
+            chat_template = self.default_chat_template
+        assert tokenizer is not None, "Tokenizer Needs to be passed"
+        self.audio_token = (
+            tokenizer.audio_token if hasattr(tokenizer, "audio_token") else audio_token
+        )
+        self.audio_token_id = tokenizer.convert_tokens_to_ids(self.audio_token)
+        self.audio_bos_token = (
+            tokenizer.audio_bos_token
+            if hasattr(tokenizer, "audio_bos_token")
+            else audio_bos_token
+        )
+        self.audio_eos_token = (
+            tokenizer.audio_eos_token
+            if hasattr(tokenizer, "audio_eos_token")
+            else audio_eos_token
+        )
+        self.model_subsampling = model_subsampling
+        # Fix Normalization
+        if feature_extractor is not None and feature_extractor.do_normalize is True:
+            feature_extractor.do_normalize = False
+        super().__init__(feature_extractor, tokenizer, chat_template=chat_template)
+
+    def __call__(
+        self,
+        text: List[str] | None = None,
+        audio: List[np.ndarray] | List[torch.Tensor] | None = None,
+        **kwargs: Unpack[MiAudioLLMProcessorKwargs],
+    ) -> BatchFeature:
+        if text is None:
+            raise ValueError("You need to specify `text` input to process.")
+        elif isinstance(text, str):
+            text = [text]
+        elif not isinstance(text, list) and not isinstance(text[0], str):
+            raise ValueError(
+                "Invalid input text. Please provide a string, or a list of strings"
+            )
+
+        output_kwargs = self._merge_kwargs(
+            MiAudioLLMProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+
+        if audio is not None:
+            if isinstance(audio[0], torch.Tensor):
+                audio = [sample_.numpy() for sample_ in audio]
+
+            if isinstance(audio[0], torch.Tensor):
+                audio = [sample_.squeeze(0) for sample_ in audio]
+                if not all(x_.ndim == 1 for x_ in audio):
+                    raise ValueError("All samples in a list must be 1D.")
+            if isinstance(audio[0], np.ndarray):
+                if not all(x_.ndim == 1 for x_ in audio):
+                    raise ValueError("All samples in a list must be 1D.")
+            # ensure we have as much audios as audio tokens
+            num_audio_tokens = sum(sample.count(self.audio_token) for sample in text)
+            num_audios = 1 if type(audio) is np.ndarray else len(audio)
+            if num_audio_tokens != num_audios:
+                raise ValueError(
+                    f"Found {num_audio_tokens} {self.audio_token} token{'s' if num_audio_tokens > 1 else ''} in provided text but received {num_audios} audio{'s' if num_audios > 1 else ''}"
+                )
+
+            # Some kwargs should not be changed so we can expand text with audio tokens below
+            output_kwargs["audio_kwargs"]["return_attention_mask"] = True
+            output_kwargs["audio_kwargs"]["padding"] = True
+            output_kwargs["audio_kwargs"]["return_tensors"] = "pt"
+
+            # + Padding
+            audio_inputs = self.feature_extractor(
+                audio, **output_kwargs["audio_kwargs"]
+            )
+
+            # remove attention mask, dasheng uses lengths
+            audio_feature_mask = audio_inputs.pop("attention_mask")
+
+            expanded_text = []
+            audio_lengths = audio_feature_mask.sum(-1).tolist()
+            audio_inputs["audio_length"] = torch.tensor(audio_lengths).long()
+            audio_inputs["audio_token_id"] = (
+                self.audio_token_id
+            )  # Pass to the model such that i knows what is the placeholder id
+
+            for sample in text:
+                replace_str = []
+                while self.audio_token in sample:
+                    audio_length = audio_lengths.pop(0)
+                    num_audio_tokens = calculate_mel_frames_dasheng(
+                        audio_length, model_subsampling=self.model_subsampling
+                    )
+
+                    expanded_audio_token = self.audio_token * num_audio_tokens
+
+                    audio_token_start_idx = sample.find(self.audio_token)
+                    audio_token_end_idx = audio_token_start_idx + len(self.audio_token)
+
+                    has_bos = (
+                        sample[
+                            audio_token_start_idx
+                            - len(self.audio_bos_token) : audio_token_start_idx
+                        ]
+                        == self.audio_bos_token
+                    )
+                    has_eos = (
+                        sample[
+                            audio_token_end_idx : audio_token_end_idx
+                            + len(self.audio_eos_token)
+                        ]
+                        == self.audio_eos_token
+                    )
+
+                    # Check if this audio token is surrounded by bos/eos tokens
+                    if not has_bos and not has_eos:
+                        expanded_audio_token = (
+                            self.audio_bos_token
+                            + expanded_audio_token
+                            + self.audio_eos_token
+                        )
+
+                    replace_str.append(expanded_audio_token)
+                    sample = sample.replace(self.audio_token, "<placeholder>", 1)
+
+                while "<placeholder>" in sample:
+                    sample = sample.replace("<placeholder>", replace_str.pop(0), 1)
+                expanded_text.append(sample)
+            text = expanded_text
+
+        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", "pt")
+        inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
+        if hasattr(self, "_check_special_mm_tokens"):
+            self._check_special_mm_tokens(text, inputs, modalities=["audio"])
+
+        if audio is not None:
+            inputs.update(audio_inputs)
+
+        return BatchFeature(data={**inputs}, tensor_type=return_tensors)
+
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+        the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, **kwargs)
+
+    @property
+    def model_input_names(self):
+        tokenizer_input_names = self.tokenizer.model_input_names
+        feature_extractor_input_names = self.feature_extractor.model_input_names
+        return list(
+            dict.fromkeys(
+                tokenizer_input_names + feature_extractor_input_names + ["audio_length"]
+            )
+        )
+
+    @property
+    # NOTE: we don't have default templates anymore, and the below is kept only because the hub config is not yet updated!
+    def default_chat_template(self):
+        """
+        This default vicuna template formats inputs in the form of a chat history. For each message in the chat history:
+        * the template will output the role of the speaker followed by the content of the message.
+        * content is a list of strings and audios.
+        * If the content element is an audio, the template will output a sequence of <|AUDIO|> tokens
+
+        Example:
+
+        ```python
+        messages = [
+            {'role': 'system', 'content': 'You are a helpful assistant.'},
+            {"role": "user", "content": [
+                {"type": "audio", "audio_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/glass-breaking-151256.mp3"},
+                {"type": "text", "text": "What's that sound?"},
+            ]},
+            {"role": "assistant", "content": "It is the sound of glass shattering."},
+            {"role": "user", "content": [
+                {"type": "audio", "audio_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/f2641_0_throatclearing.wav"},
+                {"type": "text", "text": "How about this one?"},
+            ]},
+        ]
+
+        result = template.render(messages=messages, add_generation_prompt=True)
+        ```
+        """
+        # fmt: off
+        return (
+            "{% set audio_count = namespace(value=0) %}"
+            "{% for message in messages %}"
+                "{% if loop.first and message['role'] != 'system' %}"
+                    "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
+                "{% endif %}"
+                "<|im_start|>{{ message['role'] }}\n"
+                "{% if message['content'] is string %}"
+                    "{{ message['content'] }}<|im_end|>\n"
+                "{% else %}"
+                    "{% for content in message['content'] %}"
+                        "{% if 'audio' in content or 'audio_url' in content or message['type'] == 'audio' %}"
+                            "{% set audio_count.value = audio_count.value + 1 %}"
+                            "Audio {{ audio_count.value }}: <|audio_bos|><|AUDIO|><|audio_eos|>\n"
+                        "{% elif 'text' in content %}"
+                            "{{ content['text'] }}"
+                        "{% endif %}"
+                    "{% endfor %}"
+                    "<|im_end|>\n"
+                "{% endif %}"
+            "{% endfor %}"
+            "{% if add_generation_prompt %}"
+                "<|im_start|>assistant\n"
+            "{% endif %}"
+        )

processor_config.json

@@ -0,0 +1,10 @@
+{
+  "audio_bos_token": "<|audio_bos|>",
+  "audio_eos_token": "<|audio_eos|>",
+  "audio_token": "<|AUDIO|>",
+  "auto_map": {
+    "AutoProcessor": "processing_midashenglm.MiAudioLLMProcessor"
+  },
+  "model_subsampling": 5,
+  "processor_class": "MiAudioLLMProcessor"
+}

special_tokens_map.json

@@ -0,0 +1,144 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|en|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|kr|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|de|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|es|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|fr|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|hi|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|uk|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|th|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|vi|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|nl|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|pt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|ru|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|it|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|ar|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|jp|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|unknown|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "audio_bos_token": "<|audio_bos|>",
+  "audio_eos_token": "<|audio_eos|>",
+  "audio_token": "<|AUDIO|>",
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<|IMAGE|>",
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "video_token": "<|VIDEO|>",
+  "vision_bos_token": "<|vision_bos|>",
+  "vision_eos_token": "<|vision_eos|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c40343a9d670f4fadbe6415ed2cff441055f663e51d813f2315c3368399914d5
+size 11424986

tokenizer_config.json

@@ -0,0 +1,365 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|AUDIO|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|audio_bos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|audio_eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_bos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|IMAGE|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|VIDEO|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<|en|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "<|kr|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<|de|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<|es|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "<|fr|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|hi|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|uk|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|th|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|vi|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<|nl|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<|pt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151676": {
+      "content": "<|id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151677": {
+      "content": "<|ru|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151678": {
+      "content": "<|it|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151679": {
+      "content": "<|ar|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151680": {
+      "content": "<|jp|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151681": {
+      "content": "<|unknown|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|en|>",
+    "<|kr|>",
+    "<|de|>",
+    "<|es|>",
+    "<|fr|>",
+    "<|hi|>",
+    "<|uk|>",
+    "<|th|>",
+    "<|vi|>",
+    "<|nl|>",
+    "<|pt|>",
+    "<|id|>",
+    "<|ru|>",
+    "<|it|>",
+    "<|ar|>",
+    "<|jp|>",
+    "<|unknown|>"
+  ],
+  "audio_bos_token": "<|audio_bos|>",
+  "audio_eos_token": "<|audio_eos|>",
+  "audio_token": "<|AUDIO|>",
+  "auto_map": {
+    "AutoProcessor": "processing_midashenglm.MiAudioLLMProcessor"
+  },
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {
+    "audio_bos_token": "<|audio_bos|>",
+    "audio_eos_token": "<|audio_eos|>",
+    "audio_token": "<|AUDIO|>",
+    "image_token": "<|IMAGE|>",
+    "video_token": "<|VIDEO|>",
+    "vision_bos_token": "<|vision_bos|>",
+    "vision_eos_token": "<|vision_eos|>"
+  },
+  "image_token": "<|IMAGE|>",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MiAudioLLMProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "video_token": "<|VIDEO|>",
+  "vision_bos_token": "<|vision_bos|>",
+  "vision_eos_token": "<|vision_eos|>"
+}