InternVL3-8B latency is too much
InternVL3-8B latency is too much compared to InternVL2-8B
Per frame inferencing for selecting best frames based on the output_scores for a specific prompt is taking around 3 to 4 seconds which is much higher than InternVL2-8B
Are there any parameters that needs to be set explicitly ?
self.model.config.llm_config: Qwen2Config {
"_attn_implementation_autoset": true,
"_name_or_path": "./pretrained/Qwen2.5-32B-Instruct",
"architectures": [
"Qwen2ForCausalLM"
],
"attention_dropout": 0.0,
"bos_token_id": 151643,
"eos_token_id": 151643,
"hidden_act": "silu",
"hidden_size": 3584,
"initializer_range": 0.02,
"intermediate_size": 18944,
"max_position_embeddings": 32768,
"max_window_layers": 70,
"model_type": "qwen2",
"moe_config": null,
"num_attention_heads": 28,
"num_hidden_layers": 28,
"num_key_value_heads": 4,
"rms_norm_eps": 1e-06,
"rope_scaling": {
"factor": 2.0,
"rope_type": "dynamic",
"type": "dynamic"
},
"rope_theta": 1000000.0,
"sliding_window": null,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.48.3",
"use_bfloat16": true,
"use_cache": false,
"use_sliding_window": false,
"vocab_size": 151674
}
self.model.config.vision_config: InternVisionConfig {
"_attn_implementation_autoset": true,
"_name_or_path": "OpenGVLab/InternViT-6B-448px-V1-5",
"architectures": [
"InternVisionModel"
],
"attention_dropout": 0.0,
"auto_map": {
"AutoConfig": "configuration_intern_vit.InternVisionConfig",
"AutoModel": "modeling_intern_vit.InternVisionModel"
},
"capacity_factor": 1.2,
"drop_path_rate": 0.1,
"dropout": 0.0,
"eval_capacity_factor": 1.4,
"hidden_act": "gelu",
"hidden_size": 1024,
"image_size": 448,
"initializer_factor": 0.1,
"initializer_range": 1e-10,
"intermediate_size": 4096,
"laux_allreduce": "all_nodes",
"layer_norm_eps": 1e-06,
"model_type": "intern_vit_6b",
"moe_coeff_ratio": 0.5,
"moe_intermediate_size": 768,
"moe_output_scale": 4.0,
"noisy_gate_policy": "RSample_before",
"norm_type": "layer_norm",
"num_attention_heads": 16,
"num_channels": 3,
"num_experts": 8,
"num_hidden_layers": 24,
"num_routed_experts": 4,
"num_shared_experts": 4,
"patch_size": 14,
"qk_normalization": false,
"qkv_bias": true,
"shared_expert_intermediate_size": 3072,
"torch_dtype": "bfloat16",
"transformers_version": "4.48.3",
"use_bfloat16": true,
"use_flash_attn": true,
"use_moe": false,
"use_residual": true,
"use_rts": false,
"use_weighted_residual": false
}