text_plain_output_2.png application_vnd.jupyter.stderr_output_1.png
from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.model_selection import train_test_split, GridSearchCV

from spacy.util import compounding, minibatch

import pandas as pd

import random

import re

import spacy

try:
    df_train = pd.read_csv('data/train.csv')
    df_test = pd.read_csv('data/test.csv')
except:
    df_train = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
    df_test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')

df_train.fillna('', inplace=True)
df_test.fillna('', inplace=True)

tf = TfidfVectorizer(ngram_range=(1, 1))
X_train, X_val, y_train, y_val = train_test_split(tf.fit_transform(df_train['text']), df_train['target'])
df_train['is_training'] = [1 if x in y_train.index else 0 for x in df_train.index]

import spacy
from spacy.util import compounding, minibatch
train_data = []
for idx, row in df_train[df_train['sentiment'] != 'neutral'].iterrows():
    text = row['text']
    selected_text = row['selected_text']
    if selected_text in text:
        entities = []
        try:
            for match in re.finditer(re.escape(selected_text), text):
                start_char = match.start()
                end_char = match.end()
                entity_label = row['sentiment']
                entities.append((start_char, end_char, entity_label))
        except Exception as e:
            raise e
        train_data.append((text, {'entities': entities}))

def spacy_train_custom(train_data, epochs=10):
    sample_print = 2
    nlp = spacy.blank('en')
    if 'ner' not in nlp.pipe_names:
        ner = nlp.create_pipe('ner')
        nlp.add_pipe(ner, last=True)
    else:
        ner = nlp.get_pipe('ner')
    for _, annotations in train_data:
        for ent in annotations.get('entities'):
            ner.add_label(ent[2])
    pipe_exceptions = ['ner', 'trf_wordpiecer', 'trf_tok2vec']
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe not in pipe_exceptions]
    with nlp.disable_pipes(*other_pipes):
        nlp.begin_training()
        n_iterations = epochs
        for itn in range(n_iterations):
            random.shuffle(train_data)
            losses = {}
            batches = minibatch(train_data, size=compounding(4.0, 32.0, 1.001))
            for batch_no, batch in enumerate(batches):
                texts, annotations = zip(*batch)
                nlp.update(texts, annotations, drop=0.3, losses=losses)
    for text, _ in train_data[:sample_print]:
        doc = nlp(text)
    return nlp

nlp = spacy_train_custom(train_data[0:1000], epochs=20)