update tokenizer

app_dialogue.py

@@ -26,18 +26,26 @@ from transformers import AutoImageProcessor, TextIteratorStreamer
 from transformers import AutoModelForVision2Seq
 
 
+# model_name_or_path = "Salesforce/xgen-mm-phi3-mini-instruct-interleave-r-v1.5"
+# model = AutoModelForVision2Seq.from_pretrained(model_name_or_path, trust_remote_code=True)
+# tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=False, legacy=False)
+# image_processor = AutoImageProcessor.from_pretrained(model_name_or_path, trust_remote_code=True)
+# tokenizer = model.update_special_tokens(tokenizer)
+
 DEVICE = torch.device("cuda")
 MODELS = {
     "xgen-mm-phi3-mini-instruct-interleave-r-v1.5": AutoModelForVision2Seq.from_pretrained(
         "Salesforce/xgen-mm-phi3-mini-instruct-interleave-r-v1.5",
-        torch_dtype=torch.bfloat16,
+        # torch_dtype=torch.bfloat16,
         # _attn_implementation="flash_attention_2",
         trust_remote_code=True
     ).to(DEVICE),
 }
+TOKENIZER = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=False, legacy=False)
 PROCESSOR = AutoImageProcessor.from_pretrained(
     "Salesforce/xgen-mm-phi3-mini-instruct-interleave-r-v1.5", trust_remote_code=True
 )
+TOKENIZER = model.update_special_tokens(TOKENIZER)
 
 SYSTEM_PROMPT = [
     {