text_html_output_1.png
from nltk.corpus import stopwords

from nltk.stem.porter import PorterStemmer

from sklearn.feature_extraction.text import TfidfVectorizer

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')

stop = stopwords.words('english')
stemmer = PorterStemmer()
def clean_text(text):
    processed_text = re.sub('(@\\[A-Za-z0-9]+)|([^A-Za-z \\t])|(\\w+:\\/\\/\\S+)|^rt|http.+?', '', text)
    processed_text = processed_text.lower()
    processed_text = [word for word in processed_text.split() if word not in stop]
    processed_text = ' '.join([stemmer.stem(word) for word in processed_text])
    return processed_text
def preprocessing(df):
    df.keyword.fillna('', inplace=True)
    df.location.fillna('', inplace=True)
    df.text = df.keyword + df.location + df.text
    df.text = df.text.apply(lambda text: clean_text(text))
    df.drop(columns=['keyword', 'location'], inplace=True)
    return df

train_df = preprocessing(train_df)
test_df = preprocessing(test_df)

X_train = train_df['text']
Y_train = train_df['target']
X_test = test_df['text']

X_all = pd.concat([X_train, X_test])

tfidf_vectorizer = TfidfVectorizer(max_features=15000)
tfidf_vectorizer.fit(X_all)
X_train = tfidf_vectorizer.transform(X_train)
X_test = tfidf_vectorizer.transform(X_test)

X_train = X_train.toarray()
X_test = X_test.toarray()

test_pred = xgb.predict(X_test)
submission = pd.DataFrame({'id': test_df['id'], 'target': test_pred})
submission.to_csv('submission.csv', index=False)