app

back.py

@@ -0,0 +1,84 @@
+import spaces
+import torch
+import re
+import gradio as gr
+from threading import Thread
+from transformers import TextIteratorStreamer, AutoTokenizer, AutoModelForCausalLM
+from PIL import ImageDraw
+from torchvision.transforms.v2 import Resize
+import subprocess
+
+#subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
+
+#subprocess.run('cp -r moondream/torch clients/python/moondream/torch')
+#subprocess.run('pip install moondream[gpu]')
+
+#def load_moondream():
+#    """Load Moondream model and tokenizer."""
+#    model = AutoModelForCausalLM.from_pretrained(
+#        "vikhyatk/moondream2", trust_remote_code=True, device_map={"": "cuda"}
+#    )
+#    tokenizer = AutoTokenizer.from_pretrained("vikhyatk/moondream2")
+#    return model, tokenizer
+
+"""Load Moondream model and tokenizer."""
+moondream = AutoModelForCausalLM.from_pretrained(
+    "vikhyatk/moondream2", trust_remote_code=True, device_map={"": "cuda"}
+)
+tokenizer = AutoTokenizer.from_pretrained("vikhyatk/moondream2")
+
+#model_id = "vikhyatk/moondream2"
+#revision = "2025-01-09"
+#tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision)
+#moondream = AutoModelForCausalLM.from_pretrained(
+#    model_id, trust_remote_code=True, revision=revision,
+#    torch_dtype=torch.bfloat16, device_map={"": "cuda"},
+#)
+
+#moondream.eval()
+
+@spaces.GPU(durtion="150")
+def answer_questions(image_tuples, prompt_text):
+    result = ""
+    Q_and_A = ""
+    prompts = [p.strip() for p in prompt_text.split('?')]
+    image_embeds = [img[0] for img in image_tuples if img[0] is not None]
+    answers = []
+
+    for prompt in prompts:
+        answers.append(moondream.batch_answer(
+            images=[img.convert("RGB") for img in image_embeds],
+            prompts=[prompt] * len(image_embeds),
+            tokenizer=tokenizer
+        ))
+
+    for i, prompt in enumerate(prompts):
+        Q_and_A += f"### Q: {prompt}\n"
+        for j, image_tuple in enumerate(image_tuples):
+            image_name = f"image{j+1}"
+            answer_text = answers[i][j]
+            Q_and_A += f"**{image_name} A:** \n {answer_text} \n"
+
+    result = {'headers': prompts, 'data': answers}
+    #print("result\n{}\n\nQ_and_A\n{}\n\n".format(result, Q_and_A))
+    return Q_and_A, result
+
+with gr.Blocks() as demo:
+    gr.Markdown("# moondream2 unofficial batch processing demo")
+    gr.Markdown("1. Select images\n2. Enter one or more prompts separated by commas. Ex: Describe this image, What is in this image?\n\n")
+    gr.Markdown("**Currently each image will be sent as a batch with the prompts thus asking each prompt on each image**")
+    gr.Markdown("*Running on free CPU space tier currently so results may take a bit to process compared to duplicating space and using GPU space hardware*")
+    gr.Markdown("A tiny vision language model. [moondream2](https://huggingface.co/vikhyatk/moondream2)")
+    with gr.Row():
+        img = gr.Gallery(label="Upload Images", type="pil", preview=True, columns=4)
+    with gr.Row():
+        prompt = gr.Textbox(label="Input Prompts", placeholder="Enter prompts (one prompt for each image provided) separated by question marks. Ex: Describe this image? What is in this image?", lines=8)
+    with gr.Row():
+        submit = gr.Button("Submit")
+    with gr.Row():
+        output = gr.Markdown(label="Questions and Answers", line_breaks=True)
+    with gr.Row():
+        output2 = gr.Dataframe(label="Structured Dataframe", type="array", wrap=True)
+    submit.click(answer_questions, inputs=[img, prompt], outputs=[output, output2])
+
+demo.queue().launch()