text_plain_output_1.png
from gensim.models import KeyedVectors

import nltk

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

FILE_PATH = '../input/fasttext-wikinews/wiki-news-300d-1M.vec'

keyed_vec = KeyedVectors.load_word2vec_format(FILE_PATH)

word_vec = keyed_vec.get_vector('foobar')

keras_embedding = keyed_vec.get_keras_embedding()
keras_embedding.get_config()

def mean_fasttext(arr, embedding_dim=300):
    """
    Create the average of the fasttext embeddings from each word in a document. 
    Very slow function, needs to be optimized for larger datasets
    """
    mean_vectors = []
    for document in arr:
        tokens = nltk.tokenize.word_tokenize(document)
        vectors = [keyed_vec.get_vector(token) for token in tokens if token in keyed_vec.vocab]
        if vectors:
            mean_vec = np.vstack(vectors).mean(axis=0)
            mean_vectors.append(mean_vec)
        else:
            mean_vectors.append(np.zeros(embedding_dim))
    embedding = np.vstack(mean_vectors)
    return embedding

data_sample = pd.read_csv('../input/quora-insincere-questions-classification/train.csv', nrows=6000)
train_sample = data_sample[:5000]
test_sample = data_sample[5000:]

X_train = mean_fasttext(train_sample['question_text'].values)
X_test = mean_fasttext(test_sample['question_text'].values)
y_train = train_sample['target'].values
y_test = test_sample['target'].values
print(X_train.shape)
print(y_train.shape)