text_plain_output_1.png
from pprint import pprint

from tqdm import tqdm

from transformers import AutoTokenizer

import operator

import pandas as pd

train_df = pd.read_parquet(paths.DIFUSSION_DB_META)
train_df.rename(columns={'Prompt': 'prompt'}, inplace=True)
train_df['prompt'] = train_df['prompt'].astype(str)
train_df['prompt'] = train_df['prompt'].apply(lambda x: x.lower())

def build_vocab(sentences, verbose=True):
    """
    Builds a vocabulary dictionary where keys are the unique words in our sentences and 
    the values are the word counts.
    :param sentences: list of list of words.
    :return: dictionary of words and their count.
    """
    vocab = {}
    for sentence in tqdm(sentences, disable=not verbose):
        for word in sentence:
            try:
                vocab[word] += 1
            except KeyError:
                vocab[word] = 1
    return vocab
def division(n, d):
    """Avoid zero division"""
    return n / d if d else -1
def check_coverage(vocab, embeddings_index):
    """
    :param vocab: a python dictionary with all the words in our dataframe as keys and their count as value.
    :param embeddings_index: a dict-like object where its keys are words and the values are index or the corresponding word's embedding.
    """
    a = {}
    oov = {}
    k = 0
    i = 0
    for word in tqdm(vocab):
        try:
            a[word] = embeddings_index[word]
            k += vocab[word]
        except:
            oov[word] = vocab[word]
            i += vocab[word]
            pass
    sorted_x = sorted(oov.items(), key=operator.itemgetter(1))[::-1]
    return sorted_x
def count_sentence_match(sentences, embeddings_index):
    """
    :param sentences: list of list of words
    :return: dictionary of words and their count
    """
    sentences_matches = []
    for sentence in tqdm(sentences):
        match = 0
        no_match = 0
        for word in sentence:
            try:
                embeddings_index[word]
                match += 1
            except KeyError:
                no_match += 1
        sentences_matches.append(division(match, match + no_match))
    return sentences_matches
def count_word_exists(sentences, vocab_dict):
    """
    :param sentences: list of list of words
    :return: dictionary of words and values
    """
    sentences_matches = []
    word_appeared = []
    for sentence in tqdm(sentences):
        exist = 0
        word_list = []
        for word in sentence:
            try:
                vocab_dict[word]
                exist += 1
                word_list.append(word)
            except KeyError:
                pass
        sentences_matches.append(exist)
        word_appeared.append(word_list)
    return (sentences_matches, word_appeared)
def check_intersection(vocab_input, vocab_tokenizer):
    vocab_input = list(vocab_input.keys())
    vocab_tokenizer = list(vocab_tokenizer.keys())
    intersection = list(set(vocab_input) & set(vocab_tokenizer))
    input_percentage = len(intersection) / len(vocab_input)
    tokenizer_percentage = len(intersection) / len(vocab_tokenizer)

tqdm.pandas()
sentences = train_df['prompt'].progress_apply(lambda x: x.split()).values
vocab_input = build_vocab(sentences)

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
vocab_tokenizer = tokenizer.get_vocab()

oov = check_coverage(vocab_input, vocab_tokenizer)
sentences_matches = count_sentence_match(sentences_df, vocab_tokenizer)
check_intersection(vocab_input, vocab_tokenizer)
train_df['match'] = sentences_matches