text_plain_output_1.png image_output_1.png
from PIL import Image

from collections import Counter

from torch.nn.utils.rnn import pad_sequence

from torch.utils.data import DataLoader,Dataset

from torch.utils.data import DataLoader,Dataset

import matplotlib.image as mpimg

import matplotlib.pyplot as plt

import os

import pandas as pd

import spacy

import torch

import torch

import pandas as pd
caption_file = data_location + '/captions.txt'
df = pd.read_csv(caption_file)

import matplotlib.pyplot as plt
import matplotlib.image as mpimg
data_idx = 56
image_path = data_location + '/Images/' + df.iloc[data_idx, 0]
img = mpimg.imread(image_path)

spacy_eng = spacy.load('en')
text = 'This is a good place to find a city'
[token.text.lower() for token in spacy_eng.tokenizer(text)]

class Vocabulary:
    def __init__(self, freq_threshold):
        self.itos = {0: '<PAD>', 1: '<SOS>', 2: '<EOS>', 3: '<UNK>'}
        self.stoi = {v: k for k, v in self.itos.items()}
        self.freq_threshold = freq_threshold
    def __len__(self):
        return len(self.itos)
    @staticmethod
    def tokenize(text):
        return [token.text.lower() for token in spacy_eng.tokenizer(text)]
    def build_vocab(self, sentence_list):
        frequencies = Counter()
        idx = 4
        for sentence in sentence_list:
            for word in self.tokenize(sentence):
                frequencies[word] += 1
                if frequencies[word] == self.freq_threshold:
                    self.stoi[word] = idx
                    self.itos[idx] = word
                    idx += 1
    def numericalize(self, text):
        """ For each word in the text corresponding index token for that word form the vocab built as list """
        tokenized_text = self.tokenize(text)
        return [self.stoi[token] if token in self.stoi else self.stoi['<UNK>'] for token in tokenized_text]

class FlickrDataset(Dataset):
    """
    FlickrDataset
    """
    def __init__(self, root_dir, captions_file, transform=None, freq_threshold=5):
        self.root_dir = root_dir
        self.df = pd.read_csv(caption_file)
        self.transform = transform
        self.imgs = self.df['image']
        self.captions = self.df['caption']
        self.vocab = Vocabulary(freq_threshold)
        self.vocab.build_vocab(self.captions.tolist())
    def __len__(self):
        return len(self.df)
    def __getitem__(self, idx):
        caption = self.captions[idx]
        img_name = self.imgs[idx]
        img_location = os.path.join(self.root_dir, img_name)
        img = Image.open(img_location).convert('RGB')
        if self.transform is not None:
            img = self.transform(img)
        caption_vec = []
        caption_vec += [self.vocab.stoi['<SOS>']]
        caption_vec += self.vocab.numericalize(caption)
        caption_vec += [self.vocab.stoi['<EOS>']]
        return (img, torch.tensor(caption_vec))

class CapsCollate:
    """
    Collate to apply the padding to the captions with dataloader
    """
    def __init__(self, pad_idx, batch_first=False):
        self.pad_idx = pad_idx
        self.batch_first = batch_first
    def __call__(self, batch):
        imgs = [item[0].unsqueeze(0) for item in batch]
        imgs = torch.cat(imgs, dim=0)
        targets = [item[1] for item in batch]
        targets = pad_sequence(targets, batch_first=self.batch_first, padding_value=self.pad_idx)
        return (imgs, targets)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
device