test

app.py

@@ -1,75 +1,96 @@
 import gradio as gr
-from transformers import AutoProcessor, AutoTokenizer
-from qwen_vl_utils import process_vision_info
-from transformers import Qwen2_5_VLForConditionalGeneration
+from transformers import AutoModel, AutoProcessor, AutoTokenizer
 import torch
 from PIL import Image
+import torchvision.transforms as T
+from torchvision.transforms.functional import InterpolationMode
 
-# ImageNet constants (not used in this code, kept for reference)
+# ImageNet constants
 IMAGENET_MEAN = (0.485, 0.456, 0.406)
 IMAGENET_STD = (0.229, 0.224, 0.225)
 
 # Load model and processor
-model_name = "rinkhanh000/Qwen2.5VL-7B_ViMemeCap"
-model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
+model_name = 'rinkhanh000/Vintern-ViMemeCap'
+model = AutoModel.from_pretrained(
     model_name,
     torch_dtype=torch.float32,  # Use float32 for CPU
+    low_cpu_mem_usage=True,
     trust_remote_code=True
-).eval()  # No device_map or cuda
+).eval()  # No .cuda()
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, use_fast=False)
 
-processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
+def build_transform(input_size):
+    MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
+    transform = T.Compose([
+        T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
+        T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
+        T.ToTensor(),
+        T.Normalize(mean=MEAN, std=STD)
+    ])
+    return transform
+
+def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
+    best_ratio_diff = float('inf')
+    best_ratio = (1, 1)
+    area = width * height
+    for ratio in target_ratios:
+        target_aspect_ratio = ratio[0] / ratio[1]
+        ratio_diff = abs(aspect_ratio - target_aspect_ratio)
+        if ratio_diff < best_ratio_diff:
+            best_ratio_diff = ratio_diff
+            best_ratio = ratio
+        elif ratio_diff == best_ratio_diff:
+            if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
+                best_ratio = ratio
+    return best_ratio
+
+def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=False):
+    orig_width, orig_height = image.size
+    aspect_ratio = orig_width / orig_height
+    target_ratios = set(
+        (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1)
+        if i * j <= max_num and i * j >= min_num)
+    target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
+    target_aspect_ratio = find_closest_aspect_ratio(
+        aspect_ratio, target_ratios, orig_width, orig_height, image_size)
+    target_width = image_size * target_aspect_ratio[0]
+    target_height = image_size * target_aspect_ratio[1]
+    blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
+    resized_img = image.resize((target_width, target_height))
+    processed_images = []
+    for i in range(blocks):
+        box = (
+            (i % (target_width // image_size)) * image_size,
+            (i // (target_width // image_size)) * image_size,
+            ((i % (target_width // image_size)) + 1) * image_size,
+            ((i // (target_width // image_size)) + 1) * image_size
+        )
+        split_img = resized_img.crop(box)
+        processed_images.append(split_img)
+    assert len(processed_images) == blocks
+    if use_thumbnail and len(processed_images) != 1:
+        thumbnail_img = image.resize((image_size, image_size))
+        processed_images.append(thumbnail_img)
+    return processed_images
+
+def load_image(image_file, input_size=448, max_num=12):
+    image = Image.open(image_file).convert('RGB')
+    transform = build_transform(input_size=input_size)
+    images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num)
+    pixel_values = [transform(image) for image in images]
+    pixel_values = torch.stack(pixel_values)
+    return pixel_values
 
 # Prediction function
 def predict_from_prompt_and_image(prompt, image):
     if not prompt or not image:
         return {"Error": "Please provide both a prompt and an image"}
     try:
-        messages = [
-            {
-                "role": "user",
-                "content": [
-                    {
-                        "type": "image",
-                        "image": image  # PIL image from Gradio
-                    },
-                    {
-                        "type": "text",
-                        "text": prompt  # User's text input
-                    }
-                ]
-            }
-        ]
-    
-        # Prepare inputs for inference
-        text = processor.apply_chat_template(
-            messages, tokenize=False, add_generation_prompt=True
-        )
-        image_inputs, video_inputs = process_vision_info(messages)
-        inputs = processor(
-            text=[text],
-            images=image_inputs,
-            videos=video_inputs,
-            padding=True,
-            return_tensors="pt"
-        )
-        # No .to("cuda") - keep on CPU
-        
-        # Generate response
-        generation_config = {
-            "max_new_tokens": 512,
-            "do_sample": False,  # Enable beam search
-            "num_beams": 3,     # 3 beams
-            "repetition_penalty": 3.5
-        }
-        generated_ids = model.generate(**inputs, **generation_config)
-        generated_ids_trimmed = [
-            out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
-        ]
-        response = processor.batch_decode(
-            generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
-        )[0]
-        return response
-
+        generation_config = dict(max_new_tokens=512, do_sample=False, num_beams=3, repetition_penalty=3.5)
+        question = prompt.strip()
+        pixel_values = load_image(image, max_num=6).to(torch.float32)  # Use float32 for CPU
+        response = model.chat(tokenizer, pixel_values, question, generation_config)
+        return {response}
     except Exception as e:
         return {"Error": f"Failed to process: {str(e)}"}
 

requirements.txt

@@ -1,6 +1,5 @@
-gradio 
-transformers 
-torch 
-pillow 
-torchvision 
-qwen-vl-utils
+gradio==4.44.0
+transformers==4.44.2
+torch==2.4.1
+Pillow==10.4.0
+torchvision==0.19.1