File Rename

lrn_vector_embeddings.py

@@ -0,0 +1,109 @@
+import json
+import os
+import numpy as np
+from numpy.linalg import norm
+import cv2
+from io import StringIO, BytesIO
+from umap import UMAP
+from sklearn.preprocessing import MinMaxScaler
+import pandas as pd
+from tqdm import tqdm
+import base64
+from transformers import BridgeTowerProcessor, BridgeTowerForContrastiveLearning, BridgeTowerForImageAndTextRetrieval, BridgeTowerForMaskedLM
+import requests
+from PIL import Image
+import torch
+
+url1='http://farm3.staticflickr.com/2519/4126738647_cc436c111b_z.jpg'
+cap1='A motorcycle sits parked across from a herd of livestock'
+
+url2='http://farm3.staticflickr.com/2046/2003879022_1b4b466d1d_z.jpg'
+cap2='Motorcycle on platform to be worked on in garage'
+
+url3='https://i.natgeofe.com/n/548467d8-c5f1-4551-9f58-6817a8d2c45e/NationalGeographic_2572187_3x2.jpg'
+cap3='a cat laying down stretched out near a laptop'
+
+img1 = {
+  'flickr_url': url1,
+  'caption': cap1,
+  'image_path' : './shared_data/motorcycle_1.jpg'
+}
+
+img2 = {
+    'flickr_url': url2,
+    'caption': cap2,
+    'image_path' : './shared_data/motorcycle_2.jpg'
+}
+
+img3 = {
+    'flickr_url' : url3,
+    'caption': cap3,
+    'image_path' : './shared_data/cat_1.jpg'
+}
+
+def bt_embeddings_from_local(text, image):
+
+    model = BridgeTowerForContrastiveLearning.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc")
+    processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc")
+
+    processed_inputs  = processor(image, text, padding=True, return_tensors="pt")
+
+    #inputs  = processor(prompt, base64_image, padding=True, return_tensors="pt")
+    outputs = model(**processed_inputs)
+
+    cross_modal_embeddings = outputs.cross_embeds
+    text_embeddings = outputs.text_embeds
+    image_embeddings = outputs.image_embeds
+    return {
+        'cross_modal_embeddings': cross_modal_embeddings,
+        'text_embeddings': text_embeddings,
+        'image_embeddings': image_embeddings
+    }
+    
+
+def bt_scores_with_image_and_text_retrieval():
+    url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+    image = Image.open(requests.get(url, stream=True).raw)
+    texts = ["An image of two cats chilling on a couch", "A football player scoring a goal"]
+
+    processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-gaudi")
+    model = BridgeTowerForImageAndTextRetrieval.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-gaudi")
+
+# forward pass
+    scores = dict()
+    for text in texts:
+        # prepare inputs
+        encoding = processor(image, text, return_tensors="pt") 
+        outputs = model(**encoding)
+        scores[text] = outputs.logits[0,1].item()
+    return scores
+
+
+def bt_with_masked_input():
+    url = "http://images.cocodataset.org/val2017/000000360943.jpg"
+    image = Image.open(requests.get(url, stream=True).raw).convert("RGB")
+    text = "a <mask> looking out of the window"
+
+
+    processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-gaudi")
+    model = BridgeTowerForMaskedLM.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-gaudi")
+
+    # prepare inputs
+    encoding = processor(image, text, return_tensors="pt")
+
+    # forward pass
+    outputs = model(**encoding)
+
+    token_ids = outputs.logits.argmax(dim=-1).squeeze(0).tolist()
+    if isinstance(token_ids, list):
+        results = processor.tokenizer.decode(token_ids)
+    else:
+        results = processor.tokenizer.decode([token_ids])
+
+    print(results)
+    return results
+#res = bt_embeddingsl()
+#print((res['text_embeddings']))
+for img in [img1, img2, img3]:
+    embeddings = bt_embeddings_from_local(img['caption'], Image.open(img['image_path']))
+    print(embeddings['cross_modal_embeddings'][0].shape)

s1_lrn_gradio.py

@@ -0,0 +1,37 @@
+import gradio as gr
+
+def greet(name, intensity): #Number of inputs should match the number of input components
+    return "Hello, " + name + "!" * int(intensity)
+
+
+basicDemo = gr.Interface(
+    fn=greet,
+    inputs=["text", "slider"],
+    outputs=["text"],
+)
+
+
+with gr.Blocks() as blockDemo:
+    gr.Markdown("Enter your Name and Intensity.")
+    with gr.Row():
+        inp1 = gr.Textbox(placeholder="What is your name?")
+        inp2 = gr.Slider(minimum=1, maximum=100)
+        out = gr.Textbox()
+    btn = gr.Button("Run")
+    btn.click(fn=greet, inputs=[inp1,inp2], outputs=out)
+
+
+def random_response(messages, history):
+    return "I am a bot. I don't understand human language. I can only say Hello. 🤖"
+
+with gr.Blocks() as chatInterfaceDemo:
+        with gr.Row():
+            with gr.Column(scale=4):
+               gr.Video(height=512, width=512, elem_id="video", interactive=False )
+            with gr.Column(scale=7):
+                gr.ChatInterface(
+                    fn=random_response, 
+                    type="messages"                )
+
+chatInterfaceDemo.launch(share=False)  # Share your demo with just 1 extra parameter 🚀
+

s2_download_data.py

@@ -0,0 +1,67 @@
+import requests
+from PIL import Image
+from IPython.display import display
+import huggingface_hub
+from huggingface_hub import list_datasets
+from huggingface_hub import HfApi
+
+# You can use your own uploaded images and captions. 
+# You will be responsible for the legal use of images that 
+#  you are going to use.
+
+url1='http://farm3.staticflickr.com/2519/4126738647_cc436c111b_z.jpg'
+cap1='A motorcycle sits parked across from a herd of livestock'
+
+url2='http://farm3.staticflickr.com/2046/2003879022_1b4b466d1d_z.jpg'
+cap2='Motorcycle on platform to be worked on in garage'
+
+url3='https://i.natgeofe.com/n/548467d8-c5f1-4551-9f58-6817a8d2c45e/NationalGeographic_2572187_3x2.jpg'
+cap3='a cat laying down stretched out near a laptop'
+
+img1 = {
+  'flickr_url': url1,
+  'caption': cap1,
+  'image_path' : './shared_data/motorcycle_1.jpg'
+}
+
+img2 = {
+    'flickr_url': url2,
+    'caption': cap2,
+    'image_path' : './shared_data/motorcycle_2.jpg'
+}
+
+img3 = {
+    'flickr_url' : url3,
+    'caption': cap3,
+    'image_path' : './shared_data/cat_1.jpg'
+}
+
+def download_images():
+    # download images
+    imgs = [img1, img2, img3]
+    for img in imgs:
+        data = requests.get(img['flickr_url']).content
+        with open(img['image_path'], 'wb') as f:
+            f.write(data)
+
+    for img in [img1, img2, img3]:
+        image = Image.open(img['image_path'])
+        caption = img['caption']
+        display(image)
+        print(caption)
+        
+def load_data_from_huggingface(hf_dataset_name):
+        
+    api = HfApi()
+
+    #list models from huggingface
+
+    #models = list(api.list_models())
+
+    #list datasets from huggingface
+
+    #datasets = list(api.list_datasets())
+
+
+    return api.list_datasets(search=hf_dataset_name)
+   

s3_data_to_vector_embedding.py

@@ -0,0 +1,62 @@
+from numpy.linalg import norm
+from transformers import BridgeTowerProcessor, BridgeTowerForContrastiveLearning
+import torch
+from PIL import Image
+
+
+url1='http://farm3.staticflickr.com/2519/4126738647_cc436c111b_z.jpg'
+cap1='A motorcycle sits parked across from a herd of livestock'
+
+url2='http://farm3.staticflickr.com/2046/2003879022_1b4b466d1d_z.jpg'
+cap2='Motorcycle on platform to be worked on in garage'
+
+url3='https://i.natgeofe.com/n/548467d8-c5f1-4551-9f58-6817a8d2c45e/NationalGeographic_2572187_3x2.jpg'
+cap3='a cat laying down stretched out near a laptop'
+
+img1 = {
+  'flickr_url': url1,
+  'caption': cap1,
+  'image_path' : './shared_data/motorcycle_1.jpg',
+  'tensor_path' : './shared_data/motorcycle_1'
+}
+
+img2 = {
+    'flickr_url': url2,
+    'caption': cap2,
+    'image_path' : './shared_data/motorcycle_2.jpg',
+    'tensor_path' : './shared_data/motorcycle_2'
+}
+
+img3 = {
+    'flickr_url' : url3,
+    'caption': cap3,
+    'image_path' : './shared_data/cat_1.jpg',
+    'tensor_path' : './shared_data/cat_1'
+}
+
+def bt_embeddings_from_local(text, image):
+
+    model = BridgeTowerForContrastiveLearning.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc")
+    processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc")
+
+    processed_inputs  = processor(image, text, padding=True, return_tensors="pt")
+
+    outputs = model(**processed_inputs)
+
+    cross_modal_embeddings = outputs.cross_embeds
+    text_embeddings = outputs.text_embeds
+    image_embeddings = outputs.image_embeds
+    return {
+        'cross_modal_embeddings': cross_modal_embeddings,
+        'text_embeddings': text_embeddings,
+        'image_embeddings': image_embeddings
+    }
+
+def save_embeddings():
+    for img in [img1, img2, img3]:
+        embedding = bt_embeddings_from_local(img['caption'], Image.open(img['image_path']))
+        print(embedding['cross_modal_embeddings'][0].shape) #<class 'torch.Tensor'>
+        torch.save(embedding['cross_modal_embeddings'][0], img['tensor_path'] + '.pt')
+    
+save_embeddings()
+

s4_calculate_distance.py

@@ -0,0 +1,83 @@
+import numpy as np
+from numpy.linalg import norm
+import torch
+from IPython.display import display
+import cv2
+
+url1='http://farm3.staticflickr.com/2519/4126738647_cc436c111b_z.jpg'
+cap1='A motorcycle sits parked across from a herd of livestock'
+
+url2='http://farm3.staticflickr.com/2046/2003879022_1b4b466d1d_z.jpg'
+cap2='Motorcycle on platform to be worked on in garage'
+
+url3='https://i.natgeofe.com/n/548467d8-c5f1-4551-9f58-6817a8d2c45e/NationalGeographic_2572187_3x2.jpg'
+cap3='a cat laying down stretched out near a laptop'
+
+img1 = {
+  'flickr_url': url1,
+  'caption': cap1,
+  'image_path' : './shared_data/motorcycle_1.jpg',
+  'tensor_path' : './shared_data/motorcycle_1'
+}
+
+img2 = {
+    'flickr_url': url2,
+    'caption': cap2,
+    'image_path' : './shared_data/motorcycle_2.jpg',
+    'tensor_path' : './shared_data/motorcycle_2'
+}
+
+img3 = {
+    'flickr_url' : url3,
+    'caption': cap3,
+    'image_path' : './shared_data/cat_1.jpg',
+    'tensor_path' : './shared_data/cat_1'
+}
+
+def load_tensor(path):
+    return torch.load(path)
+
+def load_embeddings():
+    ex1_embed = load_tensor(img1['tensor_path'] + '.pt')
+    ex2_embed = load_tensor(img2['tensor_path'] + '.pt')
+    ex3_embed = load_tensor(img3['tensor_path'] + '.pt')
+    return ex1_embed.data.numpy(), ex2_embed.data.numpy(), ex3_embed.data.numpy()
+
+def cosine_similarity(vec1, vec2):
+    similarity = np.dot(vec1,vec2)/(norm(vec1)*norm(vec2))
+    return similarity
+
+def calculate_cosine_distance():
+    ex1_embed, ex2_embed, ex3_embed = load_embeddings()
+    similarity1 = cosine_similarity(ex1_embed, ex2_embed)
+    similarity2 = cosine_similarity(ex1_embed, ex3_embed)
+    similarity3 = cosine_similarity(ex2_embed, ex3_embed)
+    return [similarity1, similarity2, similarity3]
+
+def calcuate_euclidean_distance():
+    ex1_embed, ex2_embed, ex3_embed = load_embeddings()
+    distance1 = cv2.norm(ex1_embed,ex2_embed, cv2.NORM_L2)
+    distance2 = cv2.norm(ex1_embed,ex3_embed, cv2.NORM_L2)
+    distance3 = cv2.norm(ex2_embed,ex3_embed, cv2.NORM_L2)
+    return [distance1, distance2, distance3]
+
+def show_cosine_distance():
+    distances = calculate_cosine_distance()
+    print("Cosine similarity between ex1_embeded and ex2_embeded is:")
+    display(distances[0])
+    print("Cosine similarity between ex1_embeded and ex3_embeded is:")
+    display(distances[1])
+    print("Cosine similarity between ex2_embeded and ex2_embeded is:")
+    display(distances[2])
+
+def show_euclidean_distance():
+    distances = calcuate_euclidean_distance()
+    print("Euclidean distance between ex1_embeded and ex2_embeded is:")
+    display(distances[0])
+    print("Euclidean distance between ex1_embeded and ex3_embeded is:")
+    display(distances[1])
+    print("Euclidean distance between ex2_embeded and ex2_embeded is:")
+    display(distances[2])
+
+show_cosine_distance()
+show_euclidean_distance()

s5-how-to-umap.py

@@ -0,0 +1,126 @@
+from IPython.display import display
+from umap import UMAP
+from sklearn.preprocessing import MinMaxScaler
+import pandas as pd
+from tqdm import tqdm
+import matplotlib.pyplot as plt
+import seaborn as sns
+from s2_download_data import load_data_from_huggingface
+from utils import prepare_dataset_for_umap_visualization as data_prep
+from s3_data_to_vector_embedding import bt_embeddings_from_local
+import random
+
+# prompt templates
+templates = [
+    'a picture of {}',
+    'an image of {}',
+    'a nice {}',
+    'a beautiful {}',
+]
+# function helps to prepare list image-text pairs from the first [test_size] data
+def data_prep(hf_dataset_name, templates=templates, test_size=1000):
+    # load Huggingface dataset (download if needed)
+    
+    #dataset = load_dataset(hf_dataset_name, trust_remote_code=True)
+    dataset = load_data_from_huggingface(hf_dataset_name)
+    # split dataset with specific test_size
+    train_test_dataset = dataset['train'].train_test_split(test_size=test_size)
+    # get the test dataset
+    test_dataset = train_test_dataset['test']
+    img_txt_pairs = []
+    for i in range(len(test_dataset)):
+        img_txt_pairs.append({
+            'caption' : templates[random.randint(0, len(templates)-1)],
+            'pil_img' : test_dataset[i]['image']
+        })
+    return img_txt_pairs
+    
+# prepare image_text pairs 
+
+# for the first 50 data of Huggingface dataset 
+#  "yashikota/cat-image-dataset"
+cat_img_txt_pairs = data_prep("yashikota/cat-image-dataset", 
+                             "cat", test_size=50)
+
+# for the first 50 data of Huggingface dataset 
+#  "tanganke/stanford_cars"
+car_img_txt_pairs = data_prep("tanganke/stanford_cars", 
+                             "car", test_size=50)
+
+# display an example of a cat image-text pair data
+display(cat_img_txt_pairs[0]['caption'])
+display(cat_img_txt_pairs[0]['pil_img'])
+
+# display an example of a car image-text pair data
+display(car_img_txt_pairs[0]['caption'])
+display(car_img_txt_pairs[0]['pil_img'])
+
+# compute BridgeTower embeddings for cat image-text pairs
+def load_cat_and_car_embeddings():
+    
+    def load_embeddings(img_txt_pair):
+        pil_img = img_txt_pair['pil_img']
+        caption = img_txt_pair['caption']
+        return bt_embeddings_from_local(caption, pil_img)
+    
+    cat_embeddings = []
+    for img_txt_pair in tqdm(
+                            cat_img_txt_pairs, 
+                            total=len(cat_img_txt_pairs)
+                        ):
+        pil_img = img_txt_pair['pil_img']
+        caption = img_txt_pair['caption']
+        embedding =load_embeddings(caption, pil_img)
+        cat_embeddings.append(embedding)
+
+    # compute BridgeTower embeddings for car image-text pairs
+    car_embeddings = []
+    for img_txt_pair in tqdm(
+                            car_img_txt_pairs, 
+                            total=len(car_img_txt_pairs)
+                        ):
+        pil_img = img_txt_pair['pil_img']
+        caption = img_txt_pair['caption']
+        embedding = load_embeddings(caption, pil_img)
+        car_embeddings.append(embedding)
+    return cat_embeddings, car_embeddings
+                        
+
+# function transforms high-dimension vectors to 2D vectors using UMAP
+def dimensionality_reduction(embed_arr, label):
+    X_scaled = MinMaxScaler().fit_transform(embed_arr)
+    print(X_scaled)
+    mapper = UMAP(n_components=2, metric="cosine").fit(X_scaled)
+    df_emb = pd.DataFrame(mapper.embedding_, columns=["X", "Y"])
+    df_emb["label"] = label
+    print(df_emb)
+    return df_emb
+
+def show_umap_visualization():
+    def reduce_dimensions():
+        cat_embeddings, car_embeddings = load_cat_and_car_embeddings()
+        # stacking embeddings of cat and car examples into one numpy array
+        all_embeddings = np.concatenate([cat_embeddings, car_embeddings])
+
+        # prepare labels for the 3 examples
+        labels = ['cat'] * len(cat_embeddings) + ['car'] * len(car_embeddings)
+
+        # compute dimensionality reduction for the 3 examples
+        reduced_dim_emb = dimensionality_reduction(all_embeddings, labels)
+        return reduced_dim_emb
+
+    reduced_dim_emb = reduce_dimensions()
+    # Plot the centroids against the cluster
+    fig, ax = plt.subplots(figsize=(8,6)) # Set figsize
+
+    sns.set_style("whitegrid", {'axes.grid' : False})
+    sns.scatterplot(data=reduced_dim_emb, 
+                    x=reduced_dim_emb['X'], 
+                    y=reduced_dim_emb['Y'], 
+                    hue='label', 
+                    palette='bright')
+    sns.move_legend(ax, "upper left", bbox_to_anchor=(1, 1))
+    plt.title('Scatter plot of images of cats and cars using UMAP')
+    plt.xlabel('X')
+    plt.ylabel('Y')
+    plt.show()