text_plain_output_1.png
from nltk.corpus import stopwords

import pandas as pd

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

variant = pd.read_csv('/kaggle/input/msk-redefining-cancer-treatment/training_variants.zip')

text_data = pd.read_csv('/kaggle/input/msk-redefining-cancer-treatment/training_text.zip', sep='\\|\\|', engine='python', names=['ID', 'TEXT'], skiprows=1)

from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))

import re
def cleaning(text, index, column):
    if type(text) is not int:
        string = ''
        text = re.sub('[^a-zA-Z0-9\n]', ' ', text)
        text = re.sub('\\s+', ' ', text)
        text = text.lower()
        for word in text.split():
            if word not in stop_words:
                string += word + ' '
        text_data[column][index] = string

for index, row in text_data.iterrows():
    if type(row['TEXT']) is str:
        cleaning(row['TEXT'], index, 'TEXT')

result = pd.merge(variant, text_data, on='ID', how='left')

result[result.isnull().any(axis=1)]

result.loc[result['TEXT'].isnull(), 'TEXT'] = result['Gene'] + '' + result['Variation']

y_true = result['Class'].values
result.Gene = result.Gene.str.replace('\\s+', '_')
result.Variation = result.Variation.str.replace('\\s+', '_')

x = result.drop('Class', axis=1)
y = result.iloc[:, 3:4]

unique_gene = result['Gene'].value_counts()
unique_gene.head(20)