czczup commited on
Commit
696d810
1 Parent(s): 4a03a6f

Upload folder using huggingface_hub

Browse files
README.md CHANGED
@@ -144,7 +144,7 @@ model = AutoModel.from_pretrained(
144
  path,
145
  torch_dtype=torch.bfloat16,
146
  low_cpu_mem_usage=True,
147
- use_flash_attn=False,
148
  trust_remote_code=True).eval().cuda()
149
  ```
150
 
@@ -159,7 +159,7 @@ model = AutoModel.from_pretrained(
159
  torch_dtype=torch.bfloat16,
160
  load_in_8bit=True,
161
  low_cpu_mem_usage=True,
162
- use_flash_attn=False,
163
  trust_remote_code=True).eval()
164
  ```
165
 
@@ -208,7 +208,7 @@ model = AutoModel.from_pretrained(
208
  path,
209
  torch_dtype=torch.bfloat16,
210
  low_cpu_mem_usage=True,
211
- use_flash_attn=False,
212
  trust_remote_code=True,
213
  device_map=device_map).eval()
214
  ```
@@ -305,7 +305,7 @@ model = AutoModel.from_pretrained(
305
  path,
306
  torch_dtype=torch.bfloat16,
307
  low_cpu_mem_usage=True,
308
- use_flash_attn=False,
309
  trust_remote_code=True).eval().cuda()
310
  tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
311
 
 
144
  path,
145
  torch_dtype=torch.bfloat16,
146
  low_cpu_mem_usage=True,
147
+ use_flash_attn=True,
148
  trust_remote_code=True).eval().cuda()
149
  ```
150
 
 
159
  torch_dtype=torch.bfloat16,
160
  load_in_8bit=True,
161
  low_cpu_mem_usage=True,
162
+ use_flash_attn=True,
163
  trust_remote_code=True).eval()
164
  ```
165
 
 
208
  path,
209
  torch_dtype=torch.bfloat16,
210
  low_cpu_mem_usage=True,
211
+ use_flash_attn=True,
212
  trust_remote_code=True,
213
  device_map=device_map).eval()
214
  ```
 
305
  path,
306
  torch_dtype=torch.bfloat16,
307
  low_cpu_mem_usage=True,
308
+ use_flash_attn=True,
309
  trust_remote_code=True).eval().cuda()
310
  tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
311
 
configuration_intern_vit.py CHANGED
@@ -71,7 +71,7 @@ class InternVisionConfig(PretrainedConfig):
71
  intermediate_size=12800,
72
  qk_normalization=True,
73
  num_hidden_layers=48,
74
- use_flash_attn=False,
75
  hidden_act='gelu',
76
  norm_type='rms_norm',
77
  layer_norm_eps=1e-6,
 
71
  intermediate_size=12800,
72
  qk_normalization=True,
73
  num_hidden_layers=48,
74
+ use_flash_attn=True,
75
  hidden_act='gelu',
76
  norm_type='rms_norm',
77
  layer_norm_eps=1e-6,
modeling_internvl_chat.py CHANGED
@@ -18,7 +18,7 @@ from transformers.utils import ModelOutput, logging
18
 
19
  from .configuration_internvl_chat import InternVLChatConfig
20
  from .conversation import get_conv_template
21
- from .modeling_intern_vit import InternVisionModel
22
  from .modeling_internlm2 import InternLM2ForCausalLM
23
 
24
  logger = logging.get_logger(__name__)
@@ -50,6 +50,7 @@ class InternVLChatModel(PreTrainedModel):
50
  self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio ** 2))
51
  self.downsample_ratio = config.downsample_ratio
52
  self.ps_version = config.ps_version
 
53
  config.vision_config.use_flash_attn = True if use_flash_attn else False
54
  config.llm_config.attn_implementation = 'flash_attention_2' if use_flash_attn else 'eager'
55
 
 
18
 
19
  from .configuration_internvl_chat import InternVLChatConfig
20
  from .conversation import get_conv_template
21
+ from .modeling_intern_vit import InternVisionModel, has_flash_attn
22
  from .modeling_internlm2 import InternLM2ForCausalLM
23
 
24
  logger = logging.get_logger(__name__)
 
50
  self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio ** 2))
51
  self.downsample_ratio = config.downsample_ratio
52
  self.ps_version = config.ps_version
53
+ use_flash_attn = use_flash_attn if has_flash_attn else False
54
  config.vision_config.use_flash_attn = True if use_flash_attn else False
55
  config.llm_config.attn_implementation = 'flash_attention_2' if use_flash_attn else 'eager'
56