app.py

import os
import subprocess
subprocess.run('pip install flash-attn==2.6.3 --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
import random
import spaces
import numpy as np
import torch
from PIL import Image
import gradio as gr
from transformers import AutoModelForCausalLM
from test_img_edit import pipe_img_edit
from test_img_to_txt import pipe_txt_gen
from test_txt_to_img import pipe_t2i


# Constants
MAX_SEED = 10000

hf_token = os.getenv("HF_TOKEN")

HUB_MODEL_ID = "AIDC-AI/Ovis-U1-3B"
model, loading_info = AutoModelForCausalLM.from_pretrained(
    HUB_MODEL_ID, 
    torch_dtype=torch.bfloat16,
    output_loading_info=True,
    token=hf_token,
    trust_remote_code=True
    )
print(f'Loading info of Ovis-U1:\n{loading_info}')

model = model.eval().to("cuda")
model = model.to(torch.bfloat16)

def set_global_seed(seed: int = 42):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

def randomize_seed_fn(seed: int, randomize: bool) -> int:
    return random.randint(0, MAX_SEED) if randomize else seed

@spaces.GPU
def process_txt_to_img(prompt: str, height: int, width: int, steps: int, final_seed: int, guidance_scale: float, progress: gr.Progress = gr.Progress(track_tqdm=True)) -> list[Image.Image]:
    set_global_seed(final_seed)
    images = pipe_t2i(model, prompt, height, width, steps, cfg=guidance_scale, seed=final_seed)
    return images

@spaces.GPU
def process_img_to_txt(prompt: str, img: Image.Image, progress: gr.Progress = gr.Progress(track_tqdm=True)) -> str:
    output_text = pipe_txt_gen(model, img, prompt)
    return output_text

@spaces.GPU
def process_img_txt_to_img(prompt: str, img: Image.Image, steps: int, final_seed: int, txt_cfg: float, img_cfg: float, progress: gr.Progress = gr.Progress(track_tqdm=True)) -> list[Image.Image]:
    set_global_seed(final_seed)
    images = pipe_img_edit(model, img, prompt, steps, txt_cfg, img_cfg, seed=final_seed)
    return images

# Gradio UI
with gr.Blocks(title="Ovis-U1-3B") as demo:
    gr.Markdown('''# Ovis-U1-3B
    ''')

    with gr.Row():
        with gr.Column():
            with gr.Tabs():
                with gr.TabItem("Image + Text → Image"):
                    edit_image_input = gr.Image(label="Input Image", type="pil")
                    with gr.Row():
                        edit_prompt_input = gr.Textbox(
                            label="Prompt",
                            show_label=False,
                            placeholder="Describe the editing instruction...",
                            container=False,
                            lines=1
                        )
                        run_edit_image_btn = gr.Button("Run", scale=0)

                    with gr.Accordion("Advanced Settings", open=False):

                        with gr.Row():

                            edit_img_guidance_slider = gr.Slider(
                                label="Image Guidance Scale",
                                minimum=1.0, maximum=10.0,
                                step=0.1, value=1.5
                            )

                            edit_txt_guidance_slider = gr.Slider(
                                label="Text Guidance Scale",
                                minimum=1.0, maximum=30.0,
                                step=0.5, value=6.0
                            )

                        edit_num_steps_slider = gr.Slider(
                            label='Steps',
                            minimum=40, maximum=100, 
                            value=50, step=1
                        )
                        edit_seed_slider = gr.Slider(
                            label="Seed",
                            minimum=0, maximum=int(MAX_SEED),
                            step=1, value=42
                        )
                        edit_randomize_checkbox = gr.Checkbox(
                            label="Randomize seed", value=False
                        )

                    img_edit_examples_data = [
                        ["imgs/train.png", "Modify this image in a Ghibli style. "],
                        ["imgs/chair.png", "Transfer the image into a faceted low-poly 3-D render style."],
                        ["imgs/car.png", "Replace the tiny house on wheels in the image with a vintage car."],
                    ]
                    gr.Examples(
                        examples=img_edit_examples_data,
                        inputs=[edit_image_input, edit_prompt_input],
                        cache_examples=False, 
                        label="Image Editing Examples"
                    )

                with gr.TabItem("Text → Image"):
                    with gr.Row():
                        prompt_gen_input = gr.Textbox(
                            label="Prompt",
                            show_label=False,
                            placeholder="Describe the image you want...",
                            container=False,
                            lines=1
                        )
                        run_image_gen_btn = gr.Button("Run", scale=0)

                    with gr.Accordion("Advanced Settings", open=False):
                        with gr.Row():
                            height_slider = gr.Slider(
                                label='height', 
                                minimum=256, maximum=1536, 
                                value=1024, step=32
                            )
                            width_slider = gr.Slider(
                                label='width', 
                                minimum=256, maximum=1536, 
                                value=1024, step=32
                            )

                        guidance_slider = gr.Slider(
                            label="Guidance Scale",
                            minimum=1.0, maximum=30.0,
                            step=0.5, value=5.0
                        )
                        
                        num_steps_slider = gr.Slider(
                            label='Steps',
                            minimum=40, maximum=100, 
                            value=50, step=1
                        )
                        seed_slider = gr.Slider(
                            label="Seed",
                            minimum=0, maximum=int(MAX_SEED),
                            step=1, value=42
                        )
                        randomize_checkbox = gr.Checkbox(
                            label="Randomize seed", value=False
                        )

                    text_gen_examples_data = [
                        ["A breathtaking fairy with teal wings sits gracefully on a lotus flower in a serene pond, exuding elegance."],
                        ["A winter mountain landscape at deep night with snowy terrain and colorful flowers, under beautiful clouds and no people, portrayed as an anime background illustration with intricate detail and sharp focus."],
                        ["A photo of a pug wearing a cowboy hat and bandana, sitting on a hay bale."]
                    ]
                    gr.Examples(
                        examples=text_gen_examples_data,
                        inputs=[prompt_gen_input],
                        cache_examples=False, 
                        label="Image Generation Examples"
                    )

                with gr.TabItem("Image → Text"):
                    image_understand_input = gr.Image(label="Input Image", type="pil")
                    with gr.Row():
                        prompt_understand_input = gr.Textbox(
                            label="Prompt",
                            show_label=False,
                            placeholder="Describe the question about image...",
                            container=False,
                            lines=1
                        )
                        run_image_understand_btn = gr.Button("Run", scale=0)

                    image_understanding_examples_data = [
                        ["imgs/table.webp", "In what scenario does this picture take place?"],
                        ["imgs/count.png", "How many broccoli are there in the picture?"],
                        ["imgs/foot.webp", "Where is this picture located?"],
                    ]
                    gr.Examples(
                        examples=image_understanding_examples_data,
                        inputs=[image_understand_input, prompt_understand_input],
                        cache_examples=False, 
                        label="Image Understanding Examples"
                    )
            
            clean_btn  = gr.Button("Clear All Inputs/Outputs")

        with gr.Column():
            output_gallery = gr.Gallery(label="Generated Images", columns=2, visible=True) # Default to visible, content will control
            output_text    = gr.Textbox(label="Generated Text", visible=False, lines=5, interactive=False)

    @spaces.GPU
    def run_img_txt_to_img_tab(prompt, img, steps, seed, txt_cfg, img_cfg, progress=gr.Progress(track_tqdm=True)):
        if img is None:
            return (
                gr.update(value=[], visible=False),
                gr.update(value="Please upload an image for editing.", visible=True)
            )
        # Seed is already finalized by the randomize_seed_fn in the click chain
        imgs = process_img_txt_to_img(prompt, img, steps, seed, txt_cfg, img_cfg, progress=progress)
        return (
            gr.update(value=imgs, visible=True),
            gr.update(value="", visible=False)
        )

    @spaces.GPU
    def run_txt_to_img_tab(prompt, height, width, steps, seed, guidance, progress=gr.Progress(track_tqdm=True)):
        # Seed is already finalized by the randomize_seed_fn in the click chain
        imgs = process_txt_to_img(prompt, height, width, steps, seed, guidance, progress=progress)
        return (
            gr.update(value=imgs, visible=True),
            gr.update(value="", visible=False)
        )

    @spaces.GPU
    def run_img_to_txt_tab(img, prompt, progress=gr.Progress(track_tqdm=True)):
        if img is None:
            return (
                gr.update(value=[], visible=False),
                gr.update(value="Please upload an image for understanding.", visible=True)
            )
        txt = process_img_to_txt(prompt, img, progress=progress)
        return (
            gr.update(value=[], visible=False),
            gr.update(value=txt, visible=True)
        )

    def clean_all_fn():
        return (
            # Tab 1 inputs
            gr.update(value=None),
            gr.update(value=""),
            gr.update(value=1.5),
            gr.update(value=6.0),
            gr.update(value=50),
            gr.update(value=42),
            gr.update(value=False),
            # Tab 2 inputs
            gr.update(value=""),  # prompt_gen_input
            gr.update(value=1024),
            gr.update(value=1024),
            gr.update(value=5.0),
            gr.update(value=50),
            gr.update(value=42),  # seed_slider
            gr.update(value=False), # randomize_checkbox
            # Tab 3 inputs
            gr.update(value=None), # image_understand_input
            gr.update(value=""),  # prompt_understand_input
            # Outputs
            gr.update(value=[], visible=True), # output_gallery (reset and keep visible for next gen)
            gr.update(value="", visible=False) # output_text (reset and hide)
        )

    # Event listeners for Image + Text -> Image
    edit_inputs = [edit_prompt_input, edit_image_input, edit_num_steps_slider, edit_seed_slider, edit_txt_guidance_slider, edit_img_guidance_slider]
    
    run_edit_image_btn.click(
        fn=randomize_seed_fn,
        inputs=[edit_seed_slider, edit_randomize_checkbox],
        outputs=[edit_seed_slider]
    ).then(
        fn=run_img_txt_to_img_tab,
        inputs=edit_inputs, 
        outputs=[output_gallery, output_text]
    )

    edit_prompt_input.submit(
        fn=randomize_seed_fn,
        inputs=[edit_seed_slider, edit_randomize_checkbox],
        outputs=[edit_seed_slider]
    ).then(
        fn=run_img_txt_to_img_tab,
        inputs=edit_inputs,
        outputs=[output_gallery, output_text]
    )

    # Event listeners for Text -> Image
    gen_inputs = [prompt_gen_input, height_slider, width_slider, num_steps_slider, seed_slider, guidance_slider]
    
    run_image_gen_btn.click(
        fn=randomize_seed_fn,
        inputs=[seed_slider, randomize_checkbox],
        outputs=[seed_slider]
    ).then(
        fn=run_txt_to_img_tab,
        inputs=gen_inputs, 
        outputs=[output_gallery, output_text]
    )

    prompt_gen_input.submit(
        fn=randomize_seed_fn,
        inputs=[seed_slider, randomize_checkbox],
        outputs=[seed_slider]
    ).then(
        fn=run_txt_to_img_tab,
        inputs=gen_inputs,
        outputs=[output_gallery, output_text]
    )

    # Event listeners for Image -> Text
    understand_inputs = [image_understand_input, prompt_understand_input]

    run_image_understand_btn.click(
        fn=run_img_to_txt_tab,
        inputs=understand_inputs,
        outputs=[output_gallery, output_text]
    )

    prompt_understand_input.submit(
        fn=run_img_to_txt_tab,
        inputs=understand_inputs,
        outputs=[output_gallery, output_text]
    )

    clean_btn.click(
        fn=clean_all_fn,
        inputs=[],
        outputs=[
            edit_image_input, edit_prompt_input, edit_img_guidance_slider, edit_txt_guidance_slider,
            edit_num_steps_slider, edit_seed_slider, edit_randomize_checkbox,
            prompt_gen_input, height_slider, width_slider, guidance_slider, num_steps_slider, seed_slider, randomize_checkbox, 
            image_understand_input, prompt_understand_input,
            output_gallery, output_text
        ]
    )

if __name__ == "__main__":
    demo.launch(share=True)