Update app.py

app.py

@@ -1,3 +1,63 @@
+from auto_round import AutoRoundConfig ## must import for auto-round format
+import requests
+import torch
+from PIL import Image
+from transformers import AutoProcessor, LlavaForConditionalGeneration
+
+
+quantized_model_path="OPEA/llama-joycaption-alpha-two-hf-llava-int4-sym-inc"
+
+# Load JoyCaption INT4 Model
+processor = AutoProcessor.from_pretrained(quantized_model_path)
+model = LlavaForConditionalGeneration.from_pretrained(
+    quantized_model_path,
+    device_map="auto",
+    revision="bc917a8" ## ##AutoGPTQ format
+)
+model.eval()
+
+image_url = "http://images.cocodataset.org/train2017/000000116003.jpg"
+content = "Write a descriptive caption for this image in a formal tone."
+
+# Preparation for inference
+with torch.no_grad():
+    image = Image.open(requests.get(image_url, stream=True).raw)
+    messages = [
+        {
+            "role": "system",
+            "content": "You are a helpful image captioner.",
+        },
+        {
+            "role": "user",
+            "content": content,
+        },
+    ]
+    prompt = processor.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
+    assert isinstance(prompt, str)
+    inputs = processor(text=[prompt], images=[image], return_tensors="pt").to(model.device)
+    inputs['pixel_values'] = inputs['pixel_values'].to(model.dtype)
+    
+    # Generate the captions
+    generate_ids = model.generate(
+        **inputs,
+        max_new_tokens=50,
+        do_sample=False,
+        suppress_tokens=None,
+        use_cache=True,
+        temperature=0.6,
+        top_k=None,
+        top_p=0.9,
+    )[0]
+    
+    # Trim off the prompt
+    generate_ids = generate_ids[inputs['input_ids'].shape[1]:]
+    
+    # Decode the caption
+    caption = processor.tokenizer.decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
+    caption = caption.strip()
+    print(caption)
+
+
 '''import os
 import re
 from typing import List, Optional, Union