text_plain_output_1.png
from nltk.corpus import stopwords

from nltk.corpus import wordnet

from nltk.stem import WordNetLemmatizer

import nltk

import pandas as pd

import re

df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')

df.shape

df.isnull().sum()

df.target.value_counts()

df.keyword.value_counts()

df.drop('location', axis=1, inplace=True)
df.dropna(inplace=True)

df.shape

df.isnull().sum()

df.reset_index(drop=True, inplace=True)

nltk.download('punkt')
nltk.download('stopwords')
nltk.download('averaged_perceptron_tagger')
nltk.download('wordnet')

def Lower(text):
    return text.lower()

def Tokenisation(text):
    return nltk.word_tokenize(text)

Stpwrd_List = stopwords.words('english')

def StopWordsAlphaText(tokenized_text):
    filtred_text = []
    for word in tokenized_text:
        word = word.strip('!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~')
        val = re.search('^[a-zA-Z][a-zA-Z0-9]*$', word)
        if word not in Stpwrd_List and val is not None:
            filtred_text.append(word)
    return filtred_text

tag_dict = {'J': wordnet.ADJ, 'N': wordnet.NOUN, 'V': wordnet.VERB, 'R': wordnet.ADV}

def get_wordnet_pos(word):
    tag = nltk.pos_tag([word])[0][1][0].upper()
    return tag_dict.get(tag, wordnet.NOUN)

lemmatizer = WordNetLemmatizer()

def Lemmetizer(tokens):
    lemmetized_text = []
    for word in tokens:
        word = lemmatizer.lemmatize(word, get_wordnet_pos(word))
        lemmetized_text.append(word)
    return lemmetized_text

df.text = df.text.apply(Lower)

df.text = df.text.apply(Tokenisation)

df.text = df.text.apply(StopWordsAlphaText)

df.text = df.text.apply(Lemmetizer)

df.head()