|
import gradio as gr |
|
import torch |
|
from PIL import Image |
|
|
|
|
|
import numpy as np |
|
import pandas as pd |
|
import math |
|
|
|
import os |
|
|
|
from api_helper import encode_numpy_array, decode_numpy_array, preprocess_image |
|
|
|
|
|
|
|
clip_retrieval_service_url = "https://knn.laion.ai/knn-service" |
|
|
|
|
|
clip_model="ViT-L/14" |
|
clip_image_size = 224 |
|
clip_model_id ="laion5B-L-14" |
|
|
|
|
|
|
|
max_tabs = 10 |
|
input_images = [None for i in range(max_tabs)] |
|
input_prompts = [None for i in range(max_tabs)] |
|
embedding_plots = [None for i in range(max_tabs)] |
|
embedding_powers = [1. for i in range(max_tabs)] |
|
|
|
embedding_base64s = [None for i in range(max_tabs)] |
|
|
|
|
|
debug_print_on = False |
|
|
|
def debug_print(*args, **kwargs): |
|
if debug_print_on: |
|
print(*args, **kwargs) |
|
|
|
|
|
|
|
def image_to_embedding(input_im): |
|
|
|
input_im = Image.fromarray(input_im) |
|
prepro = preprocess(input_im).unsqueeze(0).to(device) |
|
with torch.no_grad(): |
|
image_embeddings = model.encode_image(prepro) |
|
image_embeddings /= image_embeddings.norm(dim=-1, keepdim=True) |
|
image_embeddings_np = image_embeddings.cpu().to(torch.float32).detach().numpy() |
|
return image_embeddings_np |
|
|
|
def prompt_to_embedding(prompt): |
|
|
|
text = tokenizer([prompt]).to(device) |
|
with torch.no_grad(): |
|
prompt_embededdings = model.encode_text(text) |
|
prompt_embededdings /= prompt_embededdings.norm(dim=-1, keepdim=True) |
|
prompt_embededdings_np = prompt_embededdings.cpu().to(torch.float32).detach().numpy() |
|
return prompt_embededdings_np |
|
|
|
def embedding_to_image(embeddings): |
|
|
|
size = math.ceil(math.sqrt(embeddings.shape[0])) |
|
image_embeddings_square = np.pad(embeddings, (0, size**2 - embeddings.shape[0]), 'constant') |
|
image_embeddings_square.resize(size,size) |
|
embedding_image = Image.fromarray(image_embeddings_square, mode="L") |
|
return embedding_image |
|
|
|
def embedding_to_base64(embeddings): |
|
|
|
import base64 |
|
|
|
embeddings = embeddings.astype(np.float32) |
|
embeddings_b64 = base64.urlsafe_b64encode(embeddings).decode() |
|
return embeddings_b64 |
|
|
|
def base64_to_embedding(embeddings_b64): |
|
|
|
import base64 |
|
embeddings = base64.urlsafe_b64decode(embeddings_b64) |
|
embeddings = np.frombuffer(embeddings, dtype=np.float32) |
|
|
|
return embeddings |
|
|
|
def is_prompt_embeddings(prompt): |
|
if prompt is None or prompt == "": |
|
return False |
|
try: |
|
embedding = base64_to_embedding(prompt) |
|
return True |
|
except Exception as e: |
|
return False |
|
|
|
def is_prompt_in_example_embeddings(prompt): |
|
keys = example_embeddings.keys() |
|
if prompt is None or prompt == "": |
|
return False |
|
if prompt in keys: |
|
return True |
|
return False |
|
|
|
def safe_url(url): |
|
import urllib.parse |
|
url = urllib.parse.quote(url, safe=':/') |
|
|
|
if url.count('.jpg') > 0: |
|
url = url.split('.jpg')[0] + '.jpg' |
|
return url |
|
|
|
def main( |
|
|
|
embeddings, |
|
n_samples=4, |
|
): |
|
debug_print("main") |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
embeddings = base64_to_embedding(embeddings) |
|
|
|
embeddings = embeddings.tolist() |
|
results = clip_retrieval_client.query(embedding_input=embeddings) |
|
images = [] |
|
for result in results: |
|
if len(images) >= n_samples: |
|
break |
|
url = safe_url(result["url"]) |
|
similarty = float("{:.4f}".format(result["similarity"])) |
|
title = str(similarty) + ' ' + result["caption"] |
|
|
|
|
|
|
|
|
|
|
|
|
|
import requests |
|
from io import BytesIO |
|
try: |
|
response = requests.get(url) |
|
if not response.ok: |
|
continue |
|
bytes = BytesIO(response.content) |
|
image = Image.open(bytes) |
|
if image.mode != 'RGB': |
|
image = image.convert('RGB') |
|
|
|
|
|
|
|
|
|
images.append((image, title)) |
|
except Exception as e: |
|
print(e) |
|
return images |
|
|
|
def on_image_load_update_embeddings(image_data): |
|
debug_print("on_image_load_update_embeddings") |
|
|
|
if image_data is None: |
|
|
|
|
|
|
|
|
|
return '' |
|
embeddings = image_to_embedding(image_data) |
|
embeddings_b64 = embedding_to_base64(embeddings) |
|
|
|
return embeddings_b64 |
|
|
|
def on_image_as_payload_update_embeddings(payload): |
|
debug_print("on_image_as_payload_update_embeddings") |
|
if payload is None or payload == "": |
|
return '' |
|
image_data = decode_numpy_array(payload) |
|
embeddings = image_to_embedding(image_data) |
|
embeddings_b64 = embedding_to_base64(embeddings) |
|
|
|
return embeddings_b64 |
|
|
|
def on_prompt_change_update_embeddings(prompt): |
|
debug_print("on_prompt_change_update_embeddings") |
|
|
|
if prompt is None or prompt == "": |
|
embeddings = prompt_to_embedding('') |
|
embeddings_b64 = embedding_to_base64(embeddings) |
|
return gr.Text.update(embedding_to_base64(embeddings)) |
|
embeddings = prompt_to_embedding(prompt) |
|
embeddings_b64 = embedding_to_base64(embeddings) |
|
return embeddings_b64 |
|
|
|
def update_average_embeddings(embedding_base64s_state, embedding_powers): |
|
debug_print("update_average_embeddings") |
|
final_embedding = None |
|
num_embeddings = 0 |
|
for i, embedding_base64 in enumerate(embedding_base64s_state): |
|
if embedding_base64 is None or embedding_base64 == "": |
|
continue |
|
embedding = base64_to_embedding(embedding_base64) |
|
embedding = embedding * embedding_powers[i] |
|
if final_embedding is None: |
|
final_embedding = embedding |
|
else: |
|
final_embedding = final_embedding + embedding |
|
num_embeddings += 1 |
|
if final_embedding is None: |
|
|
|
|
|
|
|
return '' |
|
|
|
|
|
|
|
|
|
|
|
final_embedding /= np.linalg.norm(final_embedding) |
|
|
|
embeddings_b64 = embedding_to_base64(final_embedding) |
|
return embeddings_b64 |
|
|
|
def on_power_change_update_average_embeddings(embedding_base64s_state, embedding_power_state, power, idx): |
|
debug_print("on_power_change_update_average_embeddings") |
|
embedding_power_state[idx] = power |
|
embeddings_b64 = update_average_embeddings(embedding_base64s_state, embedding_power_state) |
|
return embeddings_b64 |
|
|
|
def on_embeddings_changed_update_average_embeddings(embedding_base64s_state, embedding_power_state, embedding_base64, idx): |
|
debug_print("on_embeddings_changed_update_average_embeddings") |
|
embedding_base64s_state[idx] = embedding_base64 if embedding_base64 != '' else None |
|
embeddings_b64 = update_average_embeddings(embedding_base64s_state, embedding_power_state) |
|
return embeddings_b64 |
|
|
|
def on_embeddings_changed_update_plot(embeddings_b64): |
|
debug_print("on_embeddings_changed_update_plot") |
|
|
|
if embeddings_b64 is None or embeddings_b64 == "": |
|
data = pd.DataFrame({ |
|
'embedding': [], |
|
'index': []}) |
|
update = gr.LinePlot.update(data, |
|
x="index", |
|
y="embedding", |
|
|
|
title="Embeddings", |
|
|
|
|
|
tooltip=['index', 'embedding'], |
|
|
|
|
|
width=0) |
|
return update |
|
|
|
embeddings = base64_to_embedding(embeddings_b64) |
|
data = pd.DataFrame({ |
|
'embedding': embeddings, |
|
'index': [n for n in range(len(embeddings))]}) |
|
return gr.LinePlot.update(data, |
|
x="index", |
|
y="embedding", |
|
|
|
title="Embeddings", |
|
|
|
|
|
tooltip=['index', 'embedding'], |
|
|
|
|
|
width=embeddings.shape[0]) |
|
|
|
def on_example_image_click_set_image(input_image, image_url): |
|
debug_print("on_example_image_click_set_image") |
|
input_image.value = image_url |
|
|
|
|
|
device = "cuda:0" if torch.cuda.is_available() else "cpu" |
|
|
|
from clip_retrieval.load_clip import load_clip, get_tokenizer |
|
from clip_retrieval.clip_client import ClipClient, Modality |
|
model, preprocess = load_clip(clip_model, use_jit=True, device=device) |
|
tokenizer = get_tokenizer(clip_model) |
|
clip_retrieval_client = ClipClient( |
|
url=clip_retrieval_service_url, |
|
indice_name=clip_model_id, |
|
use_safety_model = False, |
|
use_violence_detector = False, |
|
|
|
) |
|
|
|
|
|
|
|
examples = [ |
|
|
|
|
|
|
|
["pup1.jpg", "", "Pup no teacup.jpg"], |
|
] |
|
|
|
image_folder ="images" |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
test_images_urls = [ |
|
"https://www.mdig.com.br/imagens/bichos/caes_miniatura_chicara_07.jpg", |
|
"https://i.pinimg.com/236x/04/ac/0e/04ac0e05964b75c9db59de94c571339e.jpg", |
|
"https://i.pinimg.com/236x/a1/55/9e/a1559e56ae5fb6e4d19c43c6396b5940--teacup-yorkie-yorkie-puppy.jpg", |
|
"https://i.pinimg.com/236x/11/c4/51/11c4518febb0869bd2b2d391d0753f44.jpg", |
|
] |
|
|
|
example_embeddings = { |
|
"#Black & White embeddings": "F0kxPHAqE7t3DoY79djWOwA6Cb2hjK88EkuIvXdEgTzS2yY93WXvOsKffL08qjU9oGVJvZtXD7wiQ-u7QTLhvGRqozpSFqo8fCMaOy42NDyyXCC9ls69Olk_A7zJ6Ik97AwLOyNjCryYr4W8kREmPfIOPb0xrde7137Fu3Jr5bwGKGU90T-lvI1pMT1ftz-9qy3vPMTnRDzx97C8fRWjPGbQU71d6f26ASZyPdg3Qrx-saS9FaAbu83DK732Ry-9WQ7HPPPiwTzY4gS97gc1PGXmRrzsZUS9kQwmPKDZvzw4F4a9zElPPQjmdj2Lqak9SHFXPJmPvDwRLPU7YvqHPV7OYDx7K-q8wfdIveWEXT1kYE289sfOOwH6YbuID129kMivu1uvCb35jGi9shisPNsXz7zb0xk99u_ivE68QL2ibjo8jCAmvPIz5Lyv9kU8rUIKPbCciD23RQG9P48tPfEpsLwbZkE8wtjHvHqvB72k_Au818IRO2pLHz2U2yq9M1_hvTG2FD05si275m4rvL85ojtDSCo9xWclvTOEgLrMt449xggSvFQzT72wtOw7mtgLvRe-N732MlW8bGIWvMi_m7p1XSA91Sq2PHDDxrxcMC29RPoJPVeIzTyCC448PVequz8rLz2Rsnk6yayDvVjAJjuJxqe9ivRpvVEKjTwwxoM9OKOtPUjlhT20uYO8ynh2PYLVZjwREFS8wWWIPLZGrz2oNxy7GR4gvaENaDzBxJu8ZdYGtw7B7rxG2sC7mP0QPYkKXbzbh4E7HrsqvJ64P73AL_Y8I6L-vB59jjyVE8O7jshZOw19JLwQD1A9NG3xu00JHr08iBC9WANYvBe-N73tpoS6dOZaPT1Btzy9kcI8MzNRvNH7hDuMnwK8o88MvZWU5rwDUhk9sBEPPGi_DL0BY8u8-cKPvGqIjbw90_c8y19CvVYT8bxLLxU920igvX_DgTyAzh-8q2lZOy-UmbxXnlU8or9cPJruuj3HJHy9a5-ZPUJ717w9frq88QG2vIwxQz2dIam8ET5Uu2ftYzsxRxI9nDwcPDQz5jxtrf079M5rPAVIrryU_U87fslzPSTkQrxpCK08VY1NvdsXTzuwsz48rvvavIsTY705S2G936llvTsk0bwQhEG6zoYovD0TXjoseXq9bt45PfQJqDzYp_I8z3WLvGTflLwWLui78YjDvAx4Trw2-yO9oXunvADWCj1wlQQ8NmF-PG1uszzCusW8jujMO3W637yShqy7O4loPHI4ybw4yr271thYPB9hAj0PwUQ8L0fRPLkzlD091oY88PvBOjv4wDyBMh69eUsePfOb6DyI4Zq8TxH-u0uCi702EZc8kqoQvQQkAbwz9AY841FFvebosTs2z708FpwnPUqOPb0zlQi9KNeWvBbeALwy70U91KD_PHPxSTwi8wO8aroMPYpCdb1QLDI9VgwAvL-JKz1I1Za8EshhO-EUbD2cpQW9UEeQvA8vBD0KvYq6ZJ6DvWZsVb1UdT07gHVPPUM-VLxbYZM8SoXXPFP6iDwC-AU9_5p4vIuPKD0d7x26nrBSvWaCxjn2aCY9iaLDvD8yYbzQPzo9BihlPaxUar2IeMa7zJniOsRbiD0zigS9P-TqvJMO7bsuR3u8KS_WvEL1XT3vWoG7OmGqvBsJdb3_vlE5js_2vP-wF70sbJq8XqWJPOOOM71wkJk9WJRqvA7XNzqVV-O8OjBuPW5_O71u-QK92baeOovezDw-NPS8G-dkvRx_UD1xzYe94dxoPM0_-TzqrSE8DUWhPLz6qzwr4TW9yOPIO5FmYzyM_oC9fj8gvWtdljuotQg8lwlDPPPRubw4Bxc8f6y5u_x2trxXH2Q81m2TPadef7yfNwc9AlIuvKNWXr2A2c28s6e1uspCsrzBHtk66N90PeUS-TqID109vydpu8vDVb1ZknC97JqRu18mLT1hIZg8CNmBt43v1DsezNw73krSPH_eo7yAPlA99-k0PNBamLw16jA90kSlu-pTIz0DPLs8sdQLvMP3Mz3_zCS6UOMmvX4mFr3OhtK7tjA8PT6wJL2DTy69bE17vCFNVr1LJVS8zIuouVlRzjy34jA9CV-QPD5j3DwrYu68CD4uvZAP57zhOQs8krApvOvEV707mAC7giNdPYjSbjxfdLg8gZacu3ritLz6NPI7KDTBvEEhxDzihwE8pX0vPWS4GTtZfk28YPBwPQMaFr3Jqxs8t5bsuw3moj0q2q07AuRuPDSLnbw1-zi961XVuq_qKL1ofAW8CwGBvVScoz3uDny9px98PA35cTxwaYk6jwUJPQUyZbx2mVM9SMqnPdPMejxuREA9dK0UPe4VCLrn6vg8LBTOO8c2v7yL1wW9PUWJPIqrybzN3h67_-sSvVaMHz2DKno8uDCSPN3ZiL2B5JK9-CsOPNdBrbzEIPY7gCjdvEAJCjv3LX-8NjupvTIxijw55-I7CgjIvOx1Hj33TUg81is6vLs8Gj1K7VA8nyetPXnXAbgJ1US92c7tPNAYP7zzeVi9eaVGvaT8izv9_pi92S3sPCywZD3ihPK8pJ2NPZe0BT3EYSG9ahv8uic4k7wNc-M8bX9QvTUsirzUYaC8LCVrPS5Y7jyqsnk4UYbaPLF8P70_x7C80qB_u5jBjDzQqCO9vqn8vCUcMb3Z6TY9ZLRcPYLJHz3ZPJg7OMW9O23YBr3YeZu89uKCPTdrqjuwB-O8ZL82vJ7ClbvHOq-76d4xvITNT7z2x6Q8NQAkvatTUb2PhlY8NCe0OzOLR71EliA8-W6APBeomjstTFG99YfouysDNT2R3b28y7K4vLt2-bvKf6C9bf6XvH8EBrzZMak8BNpuPLLJhz1G6gU9UouwPE-QRT1lKKA8bjaGO9JQGDwpjtQ7yi_BvLv_ATyE1Ju99_8nPRh1gr1Vyyk6Q_WzPc60fz2-XoC9nsTxvJfFDT2zS9m8kx-2u3mlxjyfqf68Tje9PcK_G7v_F467sdXOvIzZdr3LONw8ngu2vF8Rgr35KP87TzCEvQNZS7xGrkU976MhPTOLsr2woYg7OGAmvRPIzLxLtaM7ofWDPXREq7wSwsi7T7dVPEiO5zwaOtu8OudNvHa5nLxYLpC9h445OydP3rsAwMG7rV9ZvWnBfb0Gwoo8ZwPXvPa72ztudDc9l4gfPJTxsjxCioM90JIwPaFs5rxs46q8yGKQux4injyRLZ281PZivejwfLwwZGE8NTinvDfsdzwSkMk8hMtmPSTkrbyigau8L4kVvJntIrrwqTo8dNSPvctfQruCT0M8lm1Wu6mpPD2Mmpc971YEOY4sGT2Be1M9rewmPfCeS7yGn9Y8KXYaPVbF0DvtdQm9r2bhvPwGGz1jNOe8KQ0xPbEPOz38nIO8PEQaPKFsZj30PNW9Be4FvRVfzj2HUvq7aJiRPB-lNz1GIze87PZrPKefPz3g_ni8VfOPvYRFbb1j-7W8_Z0xvVLvrrz8kZQ9_2dLPbAC4zxthKa8fzwJvff_p7waPzG9FV_jvLKlDr3ED249QDy3PD1SabxxX0i9M0hAvVL1Rz0cGCG9coYuvY--bj1sKhO9d_jmPHowQL0Bm_g863DIPUM-VD0OVhQ8I5H2PLwD_rxb5zY7MJqdOy6wOrxhIS09yOU7PT74ljwRi_M9WYDLvAtKIb01Yf68WfwQPf2YsTv0-vu8bp_qPHKDrL1z1KS8Xf_bPDCQXD1oIZE9lGFjuzaN5LzWIXk8nJsvPSgdtTyqEEq9286EOyZm6rtF8DM95ugxugOlDz115YI86aumO388CT1SPRC8MtM5vB6liDwqxT06Q5agvJ-TAz0CikY9MxbBvP2_QTwxhsc8NLeDOzql9LxJxmq8MSWCPfVtkTzuo3W78-gSvenIvrxOofe8zr5APQBnCL3z4us7cKEhvQ4lWL0FTwK9cMMcPHpNZT3G1hK9XOKhvZF2nz1mRdq8dgeou_fTQTz43968VndaPNdZ_DsoxLo8n0gPvTcp0Tq17Js85EnDPWiyZ733Fsk8AFwuPGuUv7sYHcs8GGtBvZK7dju76bi9KHsavWsqEzwk0xA8", |
|
"#Fire embeddings": "VsgiPPjckjuCP-q8fs25PC1DT7z-1li7_bAkvPkCrDx4W9E7V9sYvf3_D72Oyau8vXGFPVCSFD00qhO9-mSEPGQ8kT1_pN67qvXxvDwZDb1nLnO8TbvSO8Gn0j3OdUc9dNaqPP2wJL170AQ9CX-0PJu1jTycyBW9JAVvvAtWPjyF1pe8o4awO12vBrvkJ0O8EtsBPUszvbzduJO8zghmPeIBezyBaKE8STaIPBUnkDq7D1q8swJCPUR1kLy9qmU7itcaPZ2flr2fOlg7gcpKvQh__DzNAEM8xqSJPVNTb7s7aMA7cJ3sPK8ujDx4W1G8-RVhvcra-jxRuD-9RTkAPdr0dDx9a7Q7Y1q9ux2WLT0QBJO8DWkHPUE8pby9Ioi8kFydPJX0Xbzwpme8YeVuvL0iiD0q2Ti9oE1OPGXPlL0qswS805g7vWb1pD1ivPi8oOskvAm7BT323P685LIju0N1WL0WYLq80sEWPShRx7zTSVC83qLkvF-_1bwZ1RG9833EvIRlVr0jo2m8xxmpvGXirjyLTDo9hRa-PKAnGj2tVwK9aGogvay5xjxvZB69MoSnPHSHmzxcc1k79hhQvfIufbyH7Zo9E2NNvRq_0Dz0o_i8Tx2iPJb0lTypgPY8g90cPV7VljuyyZc7gXvfPLjW97x5W6097CGdvJhT9Lk7yum8AJsbPP90izw3uj49qUeEvQE277xqQaE70UybPIw2-Tq1O2y8yD9dPD0_wbxZY6485v7DvN3LP7vsg8Y8OZE_PehzGzwNj1-86kouPW2Ni70FM9w7wDIzvHUibzwqO_Q7c4djvc2eh73TIwq9icSkvA7x0rzb9Ky9Lpp4vaSs0rwUdkM6WLK9PFhQOL2vZ348TUYPveAqVrw3HOi87JbyPPGmXj06foG6E1CPPMgD-rx7gT29kgpMPau5fLxaTe271DaSvUjUpr0v6a28hLRBvS-tbjznpHy8rbm0uic-UTwCwZm8UOEavCZnmrz9Ek69cU65vcUGabznc0g8oZwDPWVX8ry3YVi9MPwRPZqilzu_H-E8m7WNOs-INLy-5pI85ZzivLewQ7wlVLY9WnaAPOMBMz1j-JM8MF47Pfe2gjuxjWq5LWEqPRohejwumni84ow3vBoOvLzHyqu8Z7mvvB1t9jzE83I83LglvQRv4zxAx448KRUuvI3UGT0Ojxe9IJM-O-fCzjwKVv29DMtUPX6R6LwDIwQ9gwNRPaOG1LzCSsA8f20AvbUoLj1SyzU9o3MNPJd8cz0mBQO8A0kvPVhQFDx-axC9Ts7IPEtkxDyxjeq6O7fPPI1Jbz3VDS49UQcrOsPg_LwpUSM8VwFxve7P5rwOQCw61uTbu1WMdbxV22A8IM-PPKsxHz1U8SG9cmGvvD1Kgjx3cRI9WMVXvPigwbxR9JA9zoj8vJJGlDzWlXC9N7q-vFGlAT0Ywps6XoYrvD946zy3_668ZZNnu4OhSz3WqHi9dl4cuE1ZKb3fj4I82s4KvILKJr3hjFu8w35TPZDker20d2E9DS22PRBmmLxmaug8D_EKPRUnEDsbcB2911n7u0PEwzyl-yu9lWl9vFFWljzP_Xc7i5slvU5_yzzJtGo80RBKvcKUFLxQkgs8jXImvDUyRLxjWr07RxCcPfPfybw_eGu8HZaJu9VvPDzCIwK8XjecPC9LRT56bkc9tWSRvFD0Rr2QgtE77x7SvPbUkzyikR48vVv6PPSQujuS96i9G10Du50wKj0UdsO5NIFcvGxnjbtLRnu9ZvVIvDI1vLyDjo28URppPCegVj1oLk-9u5oEPWzcrLspApQ9W4mavJEzHrufOti7RHWQPF9warw1vZI8vaplugiojzxjHuw8LSXrPH0v9bwwmoM9Dt6CPPIufTwHu_G8XK8qve6psrrvHtI60l8Rvdwt6byZj7M8VVMDPI3nxboqd7M84gFXu4-rI71DE688wEXxPIxfsLycKq07rWrJvAowJb1mG_28M-YRvYUWtTxq1Da95v7DvJHRKj0VJxC7QU_sPEdyxbyw8hY9xUIWPaYhPDtVtay7faepvBg3Tb1Ai2E8Kuz2vNOrebygnF09X13QvEN1xjy4dM67XE0cvE67Lr0yhCe9rbmiPZvuf7yT2YW8_E5DvR8TXr1QVt46iLGuPJqXTb3Z7DY9LunjPNkw2Lw3uj499ZCEPbLJl7vefAw9TWznvJ8nPr0OLZI8Ff7Yva0IoLsEmIg9x__juwaoe7vmmgY97vgdPUIAuTwOQCw79-_0O_K5ubxN96M8GkqNPY2YbL0bcJ08y3gtPK0IoDucAfY8uLAfPbDceb0c0iK9ysdgOQgKFbtkCwq-NqfIvGD7Aj3cuEk8z8QFO7CjKzwNyzA9aslsPdRv8ryQq4i8e7LEPLaKD7v7KI-9fJSzPO6WmLy27Dg8TNGTvOSyIz0arLa9zRNdPW7G_TxUUzm958JOPbrWCz3L2kS9tU50PZ3ut7yenwS9rQggvH_1-LxCTxK8OeAqPSvsUr3Ijja9Tn9LPYixrjuD3Qq9HzyVPGgbNbzyV7S8UQerOj5l9bymITw8H4BIu3g1Hb3KePU8F9VQvZvIp7yN50W8poNlvB5alL3M7cy8IUSLO0HaH70NLba8PI70vGq2QLwRjMy8YF0aPZmPIbzVvqe9MHF5PcjdobuWuM08rQggvaMRiL1Tokg97-JcvMgsjb0-8A09bNysPbrpJTxQQyC9e-NUvF8OZTz_Xve85CdDPEXB-DyPb9s8XdVePVRAjbx3SFu8lM4pvHTWhjwgk746a2dDvF2vBjw00Gs9akEhPE3kCT2PmJK8D2bgPIS0Qb2xRY-8-jdbPQuloLvyzFO9uRKKPHaEdD12Xhy8Je0Uve-TcT1lMRo9PlI3PafSCLhy1nK8qLzrPA0E_zzkJ0M8eUiTPf1hubxkqYS9IhvUPPeNyzz7O006wmvdOxX-2L0EDcy8zcRxvTp-gT0f4nE7u3HfPB3lBj2Pq6w8FHZDvZlANj0OQKw7z3WavIVlMj1B2qi7jP0GvMBF8btDOWM8w-B8PX4cpb2tkP08YqnwPHLW8jzPJgs9-mSEvbf_rrzN7QS845-tvP3_j72M_Ya7ZkQ0vczajryGKTS9RMSNvS-Hnz3aQ_I8IIAkPbV3vTx8Mgq9z4i0vP_pTr1UZmU8lDDTOwl_ND1zdKU8G3AdPUlJIr0aIfo77x7Su1Zmi7wmK7e90oVFvSMuSj0WwuO85WvJPKa_ErwarD88mFN0OkJPpLrB0Am8hviavMhl_7xIIxK98sxTPKZwJzz8YW89lGwAvFIaIbx2QLi8sY3qvMTzcj37O006BPoxvVrFjzwPU6K81tFBPGzcrLqmg-W87ZY8vObYj7wEvjy9cZ0ku48Nsjtmpl09Z71gvcOnCrzPxAW9e24RPWUeAL3WIAm9YHDYPCTflrrcuEk8Q2KaPS-tyrze3hG936JAvGhqoDy_gWa9v1uyOw6PFz2QglG86DfuPEX9yTzazhw9sFSuPCEI3ru4_wq9QrHNPDwZDTz5UTK9DS02vYLw2jy_qh09v1syPNP6Ur0HMwu9HVpcvVgU57wdNJY8uvy_PO6AezxBPK68vHG7vEHahLvnr7S9DAcCvdWCerz9Es67lB25vO-T8bsUFJq84oyTvV8O5TsCIzG7gcrKPH5rkLvQrsQ8KaCOvCXJ1bsmBQO9XP4nPesle71QMJi8qm0Uu8YZTb3nwjw9C2lzvOSyI7xXPcI8sHp0PZZDk7xvUQS9-cZRPZCC0TvoERY9mrVVPOvS-Twez_u72jA0vSo74rwOQCy7nrIMvYcA4juuzKq8C0MJPXRLbj2yjU88FMUuvSv_bD3_1pC9yfAXva_yOj2tCKC769JePRF5DjzIixi9gBl-Pe-8KD0G5Ci9Cc7DPKxEJz1grJc9La4KvaERWb1nppU7rd_oPDdFn7yatag8HSnDvIxfDD0VTcQ83_EHPeBmJ709LCe99qOMvApDP72cAfa8YCHbPGpBITpXsuG8qfiGvKNzBD2QXAI9o5luvXGKijzaQ_K8R5h5PVJpjLug_mI9ZVfyPC_WAT0AI3m9", |
|
} |
|
|
|
tabbed_examples = { |
|
"Pups": { |
|
"Pup1": "pup1.jpg", |
|
"Prompt": "Teacup Yorkies", |
|
"Pup2": "pup2.jpg", |
|
"Pup3": "pup3.jpg", |
|
"Pup4": "pup4.jpeg", |
|
"Pup5": "pup5.jpg", |
|
}, |
|
"Prototypes": { |
|
"Black & White": "#Black & White Embedding Prototype", |
|
"Fire": "#Fire Embedding Prototype", |
|
}, |
|
"Portraits": { |
|
"Snoop": "Snoop Dogg.jpg", |
|
"Snoop Prompt": "Snoop Dogg", |
|
"Ray": "Ray-Liotta-Goodfellas.jpg", |
|
"Ray Prompt": "Ray Liotta, Goodfellas", |
|
"Anya": "Anya Taylor-Joy 003.jpg", |
|
"Anya Prompt": "Anya Taylor-Joy, The Queen's Gambit", |
|
"Billie": "billie eilish 004.jpeg", |
|
"Billie Prompt": "Billie Eilish, blonde hair", |
|
"Lizzo": "Lizzo 001.jpeg", |
|
"Lizzo Prompt": "Lizzo,", |
|
"Donkey": "Donkey.jpg", |
|
"Donkey Prompt": "Donkey, from Shrek", |
|
}, |
|
"Transforms": { |
|
"ColorWheel001": "ColorWheel001.jpg", |
|
"ColorWheel001 BW": "ColorWheel001 BW.jpg", |
|
"ColorWheel002": "ColorWheel002.jpg", |
|
"ColorWheel002 BW": "ColorWheel002 BW.jpg", |
|
}, |
|
"CoCo": { |
|
"452650": "452650.jpeg", |
|
"Prompt 1": "a college dorm with a desk and bunk beds", |
|
"371739": "371739.jpeg", |
|
"Prompt 2": "a large banana is placed before a stuffed monkey.", |
|
"557922": "557922.jpeg", |
|
"Prompt 3": "a person sitting on a bench using a cell phone", |
|
"540554": "540554.jpeg", |
|
"Prompt 4": "two trains are coming down the tracks, a steam engine and a modern train.", |
|
}, |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
tile_size = 110 |
|
image_examples_tile_size = 50 |
|
|
|
with gr.Blocks() as demo: |
|
with gr.Row(): |
|
with gr.Column(scale=5): |
|
gr.Markdown( |
|
""" |
|
# Soho-Clip Embeddings Explorer |
|
|
|
A tool for exploring CLIP embedding space. |
|
|
|
**Example #1** - removing the Teacup from the image |
|
|
|
* Add the image Pups->Pup1 on Input tab 1 |
|
* Add the text prompt "Teacup." on Input tab 2 |
|
* Make the Input 2 embeddings negative by setting the power to -1 |
|
* Click the "Search Embedding Space" to see the results |
|
|
|
""") |
|
with gr.Column(scale=2, min_width=(tile_size+20)*3): |
|
with gr.Row(): |
|
with gr.Column(scale=1, min_width=tile_size): |
|
gr.Markdown("#### Pup in cup:") |
|
with gr.Column(scale=1, min_width=tile_size): |
|
gr.Markdown("#### - 'Teacup'") |
|
with gr.Column(scale=1, min_width=tile_size): |
|
gr.Markdown("#### = Pup") |
|
for example in examples: |
|
with gr.Row(): |
|
for example in example: |
|
with gr.Column(scale=1, min_width=tile_size): |
|
if len(example): |
|
local_path = os.path.join(image_folder, example) |
|
gr.Image( |
|
value = local_path, shape=(tile_size,tile_size), |
|
show_label=False, interactive=False) \ |
|
.style(height=tile_size, width=tile_size) |
|
|
|
with gr.Row(): |
|
for i in range(max_tabs): |
|
with gr.Tab(f"Input {i+1}"): |
|
with gr.Row(): |
|
with gr.Column(scale=1, min_width=240): |
|
input_images[i] = gr.Image(label="Image Prompt", show_label=True) |
|
with gr.Column(scale=3, min_width=600): |
|
embedding_plots[i] = gr.LinePlot(show_label=False).style(container=False) |
|
|
|
with gr.Row(): |
|
with gr.Column(scale=2, min_width=240): |
|
input_prompts[i] = gr.Textbox(label="Text Prompt", show_label=True, max_lines=4) |
|
with gr.Column(scale=3, min_width=600): |
|
with gr.Row(): |
|
|
|
|
|
embedding_powers[i] = gr.Slider(minimum=-3, maximum=3, value=1, label="Power", show_label=True, interactive=True) |
|
with gr.Row(): |
|
with gr.Accordion(f"Embeddings (base64)", open=False): |
|
embedding_base64s[i] = gr.Textbox(show_label=False, live=True) |
|
for idx, (tab_title, examples) in enumerate(tabbed_examples.items()): |
|
with gr.Tab(tab_title): |
|
with gr.Row(): |
|
for idx, (title, example) in enumerate(examples.items()): |
|
if example.endswith(".jpg") or example.endswith(".jpeg"): |
|
|
|
local_path = os.path.join(image_folder, example) |
|
with gr.Column(scale=1, min_width=image_examples_tile_size): |
|
gr.Examples( |
|
examples=[local_path], |
|
inputs=input_images[i], |
|
label=title, |
|
) |
|
else: |
|
|
|
with gr.Column(scale=1, min_width=image_examples_tile_size*2): |
|
gr.Examples( |
|
examples=[example], |
|
inputs=input_prompts[i], |
|
label=title, |
|
) |
|
|
|
with gr.Row(): |
|
average_embedding_plot = gr.LinePlot(show_label=True, label="Average Embeddings (base64)").style(container=False) |
|
with gr.Row(): |
|
with gr.Accordion(f"Avergage embeddings in base 64", open=False): |
|
average_embedding_base64 = gr.Textbox(show_label=False) |
|
with gr.Row(): |
|
with gr.Column(scale=1, min_width=200): |
|
n_samples = gr.Slider(1, 16, value=4, step=1, label="Number images") |
|
with gr.Column(scale=3, min_width=200): |
|
submit = gr.Button("Search embedding space") |
|
|
|
|
|
|
|
output = gr.Gallery(label="Closest images in Laion 5b using kNN", show_label=True)\ |
|
.style(grid=[4,4], height="auto") |
|
submit.click(main, inputs= [average_embedding_base64, n_samples], outputs=output) |
|
|
|
|
|
embedding_base64s_state = gr.State(value=[None for i in range(max_tabs)]) |
|
embedding_power_state = gr.State(value=[1. for i in range(max_tabs)]) |
|
|
|
def on_image_load(input_image, idx_state, embedding_base64s_state, embedding_power_state): |
|
debug_print("on_image_load") |
|
embeddings_b64 = on_image_load_update_embeddings(input_image) |
|
new_plot = on_embeddings_changed_update_plot(embeddings_b64) |
|
average_embeddings_b64 = on_embeddings_changed_update_average_embeddings(embedding_base64s_state, embedding_power_state, embeddings_b64, idx_state) |
|
new_average_plot = on_embeddings_changed_update_plot(average_embeddings_b64) |
|
return embeddings_b64, new_plot, average_embeddings_b64, new_average_plot |
|
|
|
def on_prompt_change(prompt, idx_state, embedding_base64s_state, embedding_power_state): |
|
debug_print("on_prompt_change") |
|
if is_prompt_embeddings(prompt): |
|
embeddings_b64 = prompt |
|
elif is_prompt_in_example_embeddings(prompt): |
|
embeddings_b64 = example_embeddings[prompt] |
|
else: |
|
embeddings_b64 = on_prompt_change_update_embeddings(prompt) |
|
new_plot = on_embeddings_changed_update_plot(embeddings_b64) |
|
average_embeddings_b64 = on_embeddings_changed_update_average_embeddings(embedding_base64s_state, embedding_power_state, embeddings_b64, idx_state) |
|
new_average_plot = on_embeddings_changed_update_plot(average_embeddings_b64) |
|
return embeddings_b64, new_plot, average_embeddings_b64, new_average_plot |
|
|
|
def on_power_change(power, idx_state, embedding_base64s_state, embedding_power_state): |
|
debug_print("on_power_change") |
|
average_embeddings_b64 = on_power_change_update_average_embeddings(embedding_base64s_state, embedding_power_state, power, idx_state) |
|
new_average_plot = on_embeddings_changed_update_plot(average_embeddings_b64) |
|
return average_embeddings_b64, new_average_plot |
|
|
|
for i in range(max_tabs): |
|
idx_state = gr.State(value=i) |
|
input_images[i].change(on_image_load, |
|
[input_images[i], idx_state, embedding_base64s_state, embedding_power_state], |
|
[embedding_base64s[i], embedding_plots[i], average_embedding_base64, average_embedding_plot]) |
|
input_prompts[i].change(on_prompt_change, |
|
[input_prompts[i], idx_state, embedding_base64s_state, embedding_power_state], |
|
[embedding_base64s[i], embedding_plots[i], average_embedding_base64, average_embedding_plot]) |
|
embedding_powers[i].change(on_power_change, |
|
[embedding_powers[i], idx_state, embedding_base64s_state, embedding_power_state], |
|
[average_embedding_base64, average_embedding_plot]) |
|
|
|
with gr.Row(): |
|
gr.Markdown( |
|
""" |
|
My interest is to use CLIP for image/video understanding (see [CLIP_visual-spatial-reasoning](https://github.com/Sohojoe/CLIP_visual-spatial-reasoning).) |
|
|
|
**Example #2** - adding black & white embedding prototype |
|
|
|
* Add the image Pups->Pup4 on Input tab 1 |
|
* Add Prototypes->Black&White on Input tab 2 |
|
* Set Input 2 embeddings power to 1.3 |
|
* Click the "Search Embedding Space" to see the results |
|
* Note: You may need to play with the power with different source images |
|
|
|
### Initial Features |
|
|
|
- Combine up to 10 Images and/or text inputs to create an average embedding space. |
|
- Search the laion 5b images via a kNN search |
|
|
|
### Known limitations |
|
|
|
- I'm getting formatting bugs when running on Huggingface (vs my Mac Book). This is impacting: |
|
- The galary |
|
- The Embeddings Tab |
|
|
|
### Acknowledgements |
|
|
|
- I heavily build on [clip-retrieval](https://rom1504.github.io/clip-retrieval/) and use their API. Please [cite](https://github.com/rom1504/clip-retrieval#citation) the authors if you use this work. |
|
- [CLIP](https://openai.com/blog/clip/) |
|
- [Stable Diffusion](https://github.com/CompVis/stable-diffusion) |
|
|
|
""") |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
with gr.Row(): |
|
gr.Markdown( |
|
""" |
|
### API |
|
|
|
UI elements to mock out the API |
|
""") |
|
with gr.Row(): |
|
_input = gr.Textbox(label="Input") |
|
with gr.Accordion(f"Embeddings (base64)", open=False): |
|
_output = gr.Textbox(value="", lines=2, label="Output") |
|
_btn = gr.Button(value="Submit") |
|
_btn.click(on_prompt_change_update_embeddings, inputs=_input, outputs=[_output], api_name="text_to_embeddings") |
|
|
|
with gr.Row(): |
|
_input = gr.Image(label="Image Prompt", show_label=True) |
|
with gr.Accordion(f"Embeddings (base64)", open=False): |
|
_output = gr.Textbox(value="", lines=2, label="Output") |
|
_btn = gr.Button(value="Submit") |
|
_btn.click(on_image_load_update_embeddings, inputs=_input, outputs=[_output], api_name="image_to_embeddings") |
|
with gr.Row(): |
|
def _on_image_load(input_image): |
|
debug_print("_on_image_load") |
|
|
|
if input_image.shape[0] > clip_image_size or input_image.shape[1] > clip_image_size: |
|
input_image = preprocess_image(input_image, clip_image_size) |
|
payload = encode_numpy_array(input_image) |
|
return payload |
|
|
|
_input = gr.Image(label="Image Prompt", show_label=True) |
|
with gr.Accordion(f"Image (base64)", open=False): |
|
_input_as_texct = gr.Textbox(value="", lines=2, label="Output") |
|
_input.change(_on_image_load, inputs=_input, outputs=[_input_as_texct]) |
|
with gr.Accordion(f"Embeddings (base64)", open=False): |
|
_output = gr.Textbox(value="", lines=2, label="Output") |
|
_btn = gr.Button(value="Submit") |
|
_btn.click(on_image_as_payload_update_embeddings, inputs=_input_as_texct, outputs=[_output], api_name="image_as_payload_to_embeddings") |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
demo.launch(debug=True) |
