text_plain_output_1.png
from bs4 import BeautifulSoup

from keras.preprocessing import text, sequence

from nltk.corpus import stopwords

from sklearn.linear_model import LogisticRegression

from string import punctuation

import matplotlib.pyplot as plt

import pandas as pd

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import seaborn as sns

train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')
test = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')
submission = pd.read_csv('../input/quora-insincere-questions-classification/sample_submission.csv')

x = train.target.value_counts()

train.rename(columns={'question_text': 'text'}, inplace=True)

train.isnull().sum()

train.drop('qid', inplace=True, axis=1)

test.rename(columns={'question_text': 'text'}, inplace=True)

test.isnull().sum()

test.drop('qid', inplace=True, axis=1)

stop = set(stopwords.words('english'))
punctuation = list(string.punctuation)
stop.update(punctuation)

def strip_html(text):
    soup = BeautifulSoup(text, 'html.parser')
    return soup.get_text()
def remove_between_square_brackets(text):
    return re.sub('\\[[^]]*\\]', '', text)
def remove_urls(text):
    return re.sub('http\\S+', '', text)
def remove_hash(text):
    text = ' '.join((word.strip() for word in re.split('#|_', text)))
    return text
def remove_stopwords(text):
    final_text = []
    for i in text.split():
        if i.strip().lower() not in stop:
            final_text.append(i.strip())
    return ' '.join(final_text)
def denoise_text(text):
    text = strip_html(text)
    text = remove_between_square_brackets(text)
    text = remove_urls(text)
    text = remove_hash(text)
    text = remove_stopwords(text)
    return text

max_features = 20000
maxlen = 100

tokenizer = text.Tokenizer(num_words=max_features)
tokenizer.fit_on_texts(train.text)
tokenized_train = tokenizer.texts_to_sequences(train.text)
X = sequence.pad_sequences(tokenized_train, maxlen=maxlen)

tokenized_test = tokenizer.texts_to_sequences(test.text)
sub_test = sequence.pad_sequences(tokenized_test, maxlen=maxlen)

sub_test.shape

from sklearn.linear_model import LogisticRegression
logit = LogisticRegression(max_iter=1000)
logit.fit(X_train, y_train)

score = logit.score(X_train, y_train)

y = logit.predict(X_test)

final = logit.predict(sub_test)

final.shape