Create app.py

app.py

@@ -0,0 +1,85 @@
+import gradio as gr
+from gradio import Interface, Textbox, Image, Markdown
+from transformers import AutoModelForCausalLM, AutoTokenizer, CLIPProcessor, CLIPModel
+import torch
+from PIL import Image as PILImage
+
+# Load models and tokenizers
+openelm_270m_instruct = AutoModelForCausalLM.from_pretrained("apple/OpenELM-270M", trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained("NousResearch/Llama-2-7b-hf")
+clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+def generate_text(prompt, num_tokens):
+    tokenized_prompt = tokenizer(prompt, return_tensors="pt")
+    
+    output_ids = openelm_270m_instruct.generate(
+        tokenized_prompt["input_ids"],
+        max_length=int(num_tokens),
+        pad_token_id=0,
+    )
+    
+    output_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)
+    return output_text
+
+def process_image(image):
+    if isinstance(image, str):
+        image = PILImage.open(image).convert("RGB")
+    inputs = clip_processor(images=image, return_tensors="pt")
+    image_features = clip_model.get_image_features(**inputs)
+    return image_features
+
+def generate_multimodal(text_prompt, image, num_tokens):
+    # Process the image
+    image_features = process_image(image)
+    
+    # Combine text prompt with image features (you may need to adjust this based on OpenELM's specific requirements)
+    combined_input = f"{text_prompt} [IMAGE]"
+    
+    # Generate text based on the combined input
+    output = generate_text(combined_input, num_tokens)
+    
+    return output
+
+def greet(text_input, image_input, num_tokens):
+    if image_input is not None:
+        return generate_multimodal(text_input, image_input, num_tokens)
+    else:
+        return generate_text(text_input, num_tokens)
+
+developer_info = """
+This space is developed by Ahmadreza Anaami
+Feel free to set via API key too
+Models used: apple/OpenELM-270M, openai/clip-vit-base-patch32
+"""
+
+iface = gr.Interface(
+    fn=greet,
+    inputs=[
+        Textbox(label="Enter Text Here:", type="text"),
+        Image(label="Upload Image (optional):"),
+        Textbox(label="Number of generated tokens:", type="text")
+    ],
+    outputs=[Textbox(label="Generated answer:")],
+    title="OpenELM-270M Multimodal",
+    description=developer_info,
+    css="""
+        #dev-info {
+            font-size: 0.8rem;
+            color: #888;
+            margin-top: 1rem;
+            text-align: center;
+        }
+        .gr-input text {
+            padding: 10px;
+            border-radius: 5px;
+            font-size: 1rem;
+        }
+        .gr-output.gr-slider label {
+            font-weight: bold;
+        }
+    """
+)
+
+if __name__ == "__main__":
+    iface.launch()