text_plain_output_1.png
from nltk.stem import WordNetLemmatizer

from nltk.stem.porter import PorterStemmer

import nltk

import numpy as np

import pandas as pd

import re

import tensorflow as tf

import re
import numpy as np
import pandas as pd
import nltk
from nltk.stem import WordNetLemmatizer
from nltk.stem.porter import PorterStemmer
import tensorflow as tf
data = pd.read_csv('../input/TechCrunch.csv', sep=',', error_bad_lines=False, encoding='ISO-8859-1')
data['title'] = data['title'].map(lambda x: re.sub('[^\\x00-\\x7F]+', ' ', x))
data['url'] = data['url'].map(lambda x: re.sub('[^\\x00-\\x7F]+', ' ', x))
def combineProperNouns(a):
    y = 0
    while y <= len(a) - 2:
        if a[y][0].isupper() == True and a[y + 1][0].isupper() == True:
            a[y] = str(a[y]) + '+' + str(a[y + 1])
            a[y + 1:] = a[y + 2:]
        else:
            y = y + 1
    return a
def recreateDataWithCombinedProperNouns(data):
    tempData = []
    for x in data.split('.'):
        tempPhrase = []
        for y in x.split(','):
            z = y.split(' ')
            z = [a for a in z if len(a) > 0]
            tempPhrase.append(' '.join(combineProperNouns(z)))
        tempData.append(','.join(tempPhrase))
    data = '.'.join(tempData)
    return data
def removeDotsFromAcronyms(data):
    counter = 0
    while counter < len(data) - 2:
        if data[counter] == '.' and data[counter + 2] == '.':
            data = data[:counter] + str(data[counter + 1]) + ' ' + data[counter + 3:]
            counter = counter + 1
        elif data[counter] == '.' and data[counter - 1].isupper() == True:
            data = data[:counter] + data[counter + 1:]
        else:
            counter = counter + 1
    return data
def stemAndLemmatize(data, columnNames):
    wordnet_lemmatizer = WordNetLemmatizer()
    porter_stemmer = PorterStemmer()
    for columnName in columnNames:
        data[columnName] = data[columnName].map(lambda x: ' '.join([porter_stemmer.stem(y) for y in x.split(' ')]))
        data[columnName] = data[columnName].map(lambda x: ' '.join([wordnet_lemmatizer.lemmatize(y) for y in x.split(' ')]))
    return data
data['newTitle'] = data['title'].map(lambda x: recreateDataWithCombinedProperNouns(x))
data = stemAndLemmatize(data, ['title'])
data['newTitle'] = data['newTitle'].map(lambda x: ' '.join([y for y in x.split(' ') if nltk.pos_tag(y.split())[0][1] not in ['DT', 'IN', 'PDT', 'TO']]))
data['newTitle'] = data['newTitle'].map(lambda x: ' '.join([y for y in x.split(' ') if len(y) > 1]))

tagList = ['NNS', 'NNP', 'NNPS', 'VB', 'VBD', 'VBG']
data['newTitle'] = data['newTitle'].map(lambda x: ' '.join([y for y in x.split(' ') if nltk.pos_tag(y.split())[0][1] in tagList]))
wordList = set([y for x in data['newTitle'].values for y in x.split(' ')])
print('The number of words are {}'.format(len(wordList)))
vocab_size = len(wordList)
word2int = {}
int2word = {}
for i, word in enumerate(wordList):
    word2int[word] = i
    int2word[word] = i
words = []
WINDOW_SIZE = 2
for sentence in data['newTitle'].values:
    newSentence = sentence.split(' ')
    for word_index, word in enumerate(newSentence):
        for nb_word in newSentence[max(word_index - WINDOW_SIZE, 0):min(word_index + WINDOW_SIZE, len(newSentence)) + 1]:
            if nb_word != word:
                words.append([word, nb_word])
def to_one_hot(data_point_index, vocab_size):
    temp = np.zeros(vocab_size)
    temp[data_point_index] = 1
    return temp
x_train = []
y_train = []
for data_word in words:
    x_train.append(to_one_hot(word2int[data_word[0]], vocab_size))
    y_train.append(to_one_hot(word2int[data_word[1]], vocab_size))
x_train = np.asarray(x_train)
y_train = np.asarray(y_train)
x = tf.placeholder(tf.float32, shape=(None, vocab_size))
y_label = tf.placeholder(tf.float32, shape=(None, vocab_size))
EMBEDDING_DIM = 5
W1 = tf.Variable(tf.random_normal([vocab_size, EMBEDDING_DIM]))
b1 = tf.Variable(tf.random_normal([EMBEDDING_DIM]))
hidden_representation = tf.add(tf.matmul(x, W1), b1)
W2 = tf.Variable(tf.random_normal([EMBEDDING_DIM, vocab_size]))
b2 = tf.Variable(tf.random_normal([vocab_size]))
prediction = tf.nn.softmax(tf.add(tf.matmul(hidden_representation, W2), b2))