text_plain_output_1.png
from nltk.tokenize import WhitespaceTokenizer

from subprocess import check_output

import numpy as np # linear algebra

import operator

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import numpy as np
import pandas as pd
import tensorflow as tf
from sklearn.model_selection import train_test_split
from subprocess import check_output
import operator
import nltk

df = pd.read_csv('../input/spam.csv', encoding='latin-1')

df.drop(['Unnamed: 2', 'Unnamed: 3', 'Unnamed: 4'], axis=1, inplace=True)

df = df.rename(columns={'v1': 'class', 'v2': 'text'})

from nltk.tokenize import WhitespaceTokenizer
tokeniser = WhitespaceTokenizer()
def tokenize(sentence):
    return tokeniser.tokenize(sentence)

num_top_words = 1000
all_words = {}
def build_words(string_in):
    for w in tokenize(string_in):
        all_words[w] = all_words.get(w, 0) + 1
for x in df['text']:
    build_words(x)
sorted_words = sorted(all_words.items(), key=operator.itemgetter(1), reverse=True)
sorted_words = list(map(lambda x: x[0], sorted_words))
sorted_words = sorted_words[:num_top_words]
words_by_emails = []
def count_words_per_email(text):
    row = np.zeros(len(sorted_words))
    for word in tokenize(text):
        try:
            row[sorted_words.index(word)] = row[sorted_words.index(word)] + 1
        except ValueError:
            pass
    return row
X_rows = []
for _row in df['text']:
    X_rows.append(count_words_per_email(_row))
X_rows = np.array(X_rows)
print(X_rows.shape)