text_plain_output_2.png application_vnd.jupyter.stderr_output_1.png
import gensim

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

df = pd.read_csv('../input/train.csv')

df.shape

def read_questions(row, column_name):
    return gensim.utils.simple_preprocess(str(row[column_name]).encode('utf-8'))
documents = []
for index, row in df.iterrows():
    documents.append(read_questions(row, 'question1'))
    if row['is_duplicate'] == 0:
        documents.append(read_questions(row, 'question2'))

print("List of lists. Let's confirm: ", type(documents), ' of ', type(documents[0]))