app.py

import streamlit as st
import requests
import zipfile
import io
import os
import sys
import torch
import numpy as np
import matplotlib.pyplot as plt
import cv2
from PIL import Image
from diffusers import StableDiffusionInpaintPipeline, EulerDiscreteScheduler
import copy

# ========== Download SAM Repo ==========
def download_sam_repo():
    repo_url = "https://github.com/facebookresearch/segment-anything/archive/refs/heads/main.zip"
    repo_dir = "segment-anything-main"
    if not os.path.exists(repo_dir):
        st.info("🔽 Downloading Segment Anything repo...")
        response = requests.get(repo_url)
        if response.status_code == 200:
            with zipfile.ZipFile(io.BytesIO(response.content)) as zip_ref:
                zip_ref.extractall(".")
            st.success("✅ Segment Anything repo downloaded!")
        else:
            st.error(f"❌ Failed to download repo: {response.status_code}")

download_sam_repo()
sys.path.append(os.path.abspath("segment-anything-main"))

from segment_anything import sam_model_registry, SamAutomaticMaskGenerator

# ========== Download SAM Model Checkpoint ==========
def download_file(url, output_path):
    if not os.path.exists(output_path):
        st.info(f"🔽 Downloading {os.path.basename(output_path)}...")
        response = requests.get(url, stream=True)
        with open(output_path, 'wb') as f:
            for chunk in response.iter_content(chunk_size=8192):
                if chunk:
                    f.write(chunk)
        st.success(f"✅ Downloaded {os.path.basename(output_path)}")

sam_url = "https://huggingface.co/camenduru/segment_anything/resolve/main/sam_vit_h_4b8939.pth"
download_file(sam_url, "sam_vit_h_4b8939.pth")

# ========== Load SAM Model ==========
@st.cache_resource
def load_sam_model():
    sam = sam_model_registry["vit_h"](checkpoint="sam_vit_h_4b8939.pth")
    sam.to("cuda")
    mask_generator = SamAutomaticMaskGenerator(
        model=sam,
        points_per_side=32,
        pred_iou_thresh=0.99,
        stability_score_thresh=0.92,
        crop_n_layers=1,
        crop_n_points_downscale_factor=2,
        min_mask_region_area=100,
    )
    return mask_generator

# ========== Load SD Model ==========
@st.cache_resource
def load_sd_pipeline():
    model_dir = 'stabilityai/stable-diffusion-2-inpainting'
    scheduler = EulerDiscreteScheduler.from_pretrained(model_dir, subfolder='scheduler')
    pipe = StableDiffusionInpaintPipeline.from_pretrained(
        model_dir,
        scheduler=scheduler,
        torch_dtype=torch.float16,
        revision="fp16"
    ).to("cuda")
    pipe.enable_xformers_memory_efficient_attention()
    return pipe

# ========== Display masks ==========
def show_masks(image, masks):
    fig, ax = plt.subplots(figsize=(10, 10))
    ax.imshow(image)
    for i, mask in enumerate(masks):
        m = mask['segmentation']
        color = np.random.random((1, 3)).tolist()[0]
        img = np.ones((m.shape[0], m.shape[1], 3))
        for j in range(3):
            img[:, :, j] = color[j]
        ax.imshow(np.dstack((img, m * 0.35)))
        contours, _ = cv2.findContours(m.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
        if contours:
            cnt = contours[0]
            M = cv2.moments(cnt)
            if M["m00"] != 0:
                cx = int(M["m10"] / M["m00"])
                cy = int(M["m01"] / M["m00"])
                ax.text(cx, cy, str(i), color='white', fontsize=16, ha='center', va='center', fontweight='bold')
    ax.axis('off')
    st.pyplot(fig)

# ========== Image Grid ==========
def create_image_grid(original_image, images, names, rows, columns):
    images = copy.copy(images)
    names = copy.copy(names)

    images.insert(0, original_image)
    names.insert(0, "Original")

    fig, axes = plt.subplots(rows, columns, figsize=(15, 15))
    for idx, (img, name) in enumerate(zip(images, names)):
        row, col = divmod(idx, columns)
        axes[row, col].imshow(img)
        axes[row, col].set_title(name)
        axes[row, col].axis('off')
    for idx in range(len(images), rows * columns):
        row, col = divmod(idx, columns)
        axes[row, col].axis('off')
    plt.tight_layout()
    st.pyplot(fig)

# ========== Streamlit UI ==========
st.title("🎨 Segment & Inpaint with Streamlit")

uploaded_file = st.file_uploader("Upload an Image", type=['png', 'jpg', 'jpeg'])

if uploaded_file:
    source_image = Image.open(uploaded_file).convert("RGB").resize((512, 512))
    st.image(source_image, caption="Uploaded Image", use_column_width=True)

    mask_gen = load_sam_model()
    masks = mask_gen.generate(np.asarray(source_image))
    st.write(f"Number of Segments Found: {len(masks)}")

    show_masks(source_image, masks)

    mask_idx = st.number_input(f"Select Mask Index (0 to {len(masks)-1})", min_value=0, max_value=len(masks)-1, value=0)
    prompt = st.text_input("Enter Inpainting Prompt", "a skirt full of text")
    generate = st.button("Generate Inpainting")

    if generate:
        segmentation_mask = masks[mask_idx]['segmentation']
        stable_mask = Image.fromarray(segmentation_mask * 255).convert("RGB")

        pipe = load_sd_pipeline()
        generator = torch.Generator(device="cuda").manual_seed(77)

        images = []
        for i in range(4):
            result = pipe(
                prompt=prompt,
                guidance_scale=7.5,
                num_inference_steps=50,
                generator=generator,
                image=source_image,
                mask_image=stable_mask
            ).images[0]
            images.append(result)

        create_image_grid(source_image, images, [prompt]*4, 2, 3)