text_html_output_1.png
from simpletransformers.classification import ClassificationModel, ClassificationArgs

from sklearn.model_selection import train_test_split

import pandas as pd

df = pd.read_csv('../input/misoimprovedta/ta-misogyny-train (3).csv', header=None, sep='\t')
df_eval = pd.read_csv('../input/misoimprovedta/ta-misogyny-dev (2).csv', header=None, sep='\t')
df_test = pd.read_csv('../input/misoimprovedta/ta-misogyny-test (2).csv', header=None)

def create_labels(sentence):
    splits = sentence.split('\t')
    return splits[0]
def change_sentence(sentence):
    splits = sentence.split('\t')
    return splits[1]
df_eval.rename(columns={0: 'Labels', 1: 'Text'}, inplace=True)
df_eval = df_eval[['Text', 'Labels']]
df.rename(columns={0: 'Labels', 1: 'Text'}, inplace=True)
df = df[['Text', 'Labels']]
df_test.rename(columns={0: 'Text'}, inplace=True)

from sklearn.model_selection import train_test_split
X_test, X_dev, y_test, y_dev = train_test_split(df_eval['Text'], df_eval['Labels'], random_state=0)
df_test_ = pd.concat([X_test, y_test], axis=1)
df_dev = pd.concat([X_dev, y_dev], axis=1)
df_dev

model_args = ClassificationArgs()

model_args.overwrite_output_dir = True
model_args.eval_batch_size = 8
model_args.train_batch_size = 8
model_args.learning_rate = 4e-05

model = ClassificationModel('bert', 'google/muril-base-cased', num_labels=9, args=model_args, tokenizer_type='bert', tokenizer_name='google/muril-base-cased')

predictions, raw_outputs = model.predict(df_test_['Text'].to_list())