text_html_output_1.png application_vnd.jupyter.stderr_output_1.png
import numpy as np

import pandas as pd

import tensorflow as tf

import tensorflow_hub as hub

TRAIN_INPUT = 'twitgen_train_201906011956.csv'
VALID_INPUT = 'twitgen_valid_201906011956.csv'
TEST_INPUT = 'twitgen_test_201906011956.csv'
EMBEDDING_DIM = 512
MAXLEN = 50

basepath = '/kaggle/input/'

df_train = pd.read_csv(basepath + TRAIN_INPUT, index_col=['id', 'time'], parse_dates=['time'])
df_valid = pd.read_csv(basepath + VALID_INPUT, index_col=['id', 'time'], parse_dates=['time'])
df_test = pd.read_csv(basepath + TEST_INPUT, index_col=['id', 'time'], parse_dates=['time'])

def prepare_data(df):
    text = df['text'].tolist()
    text = [' '.join(t.split()[0:MAXLEN]) for t in text]
    text = np.array(text, dtype=object)
    label = df['male'].tolist()
    return (text, label)
train_text, y_train = prepare_data(df_train)
valid_text, y_valid = prepare_data(df_valid)
test_text, y_test = prepare_data(df_test)

module_url = 'https://tfhub.dev/google/universal-sentence-encoder-large/3'
def get_embeddings(text):
    with tf.Graph().as_default():
        embed = hub.Module(module_url)
        embeddings = embed(text)
        with tf.Session() as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.tables_initializer())
            return sess.run(embeddings)
chunk = int(len(train_text) / 2)
X_train = np.concatenate([get_embeddings(train_text[:chunk]), get_embeddings(train_text[chunk:])])
X_valid = get_embeddings(valid_text)
X_test = get_embeddings(test_text)
(X_train.shape, X_valid.shape, X_test.shape)