text_plain_output_1.png
from spacy.tokens import DocBin

from spacy.util import filter_spans

from tqdm import tqdm

import fr_core_news_sm

import json

import os

import os

import pandas as pd

import pandas as pd

import re

import re

import re
def trim_entity_spans(data: list) -> list:
    """Removes leading and trailing white spaces from entity spans.
    Args:
        data (list): The data to be cleaned in spaCy JSON format.
    Returns:
        list: The cleaned data.
    """
    invalid_span_tokens = re.compile('\\s')
    cleaned_data = []
    for text, annotations in data:
        entities = annotations['labels']
        valid_entities = []
        for start, end, label in entities:
            valid_start = start
            valid_end = end
            while valid_start < len(text) and invalid_span_tokens.match(text[valid_start]):
                valid_start += 1
            while valid_end > 1 and invalid_span_tokens.match(text[valid_end - 1]):
                valid_end -= 1
            valid_entities.append([valid_start, valid_end, label])
        cleaned_data.append([text, {'labels': valid_entities}])
    return cleaned_data

def starts_with_punctuation(s):
    if re.match('^\\W', s):
        return True
    else:
        return False
def ends_with_punctuation(s):
    if re.search('\\W$', s):
        return True
    else:
        return False
def dp(train_data):
    list_train_data = []
    for i, ele in enumerate(train_data):
        list_train_data.append((train_data[i]['id'], train_data[i]['text'], {'labels': train_data[i]['label']}))
    ent_prob = []
    for i, ele in enumerate(list_train_data):
        ents = []
        for j, elem in enumerate(list_train_data[i][2]['labels']):
            start = elem[0]
            end = elem[1]
            ent = list_train_data[i][1][start:end]
            if starts_with_punctuation(ent) and ends_with_punctuation(ent):
                ents.append((list_train_data[i][0], ent, start, end, 'YES', 'YES'))
            elif starts_with_punctuation(ent) and (not ends_with_punctuation(ent)):
                ents.append((list_train_data[i][0], ent, start, end, 'YES', 'NO'))
            elif not starts_with_punctuation(ent) and ends_with_punctuation(ent):
                ents.append((list_train_data[i][0], ent, start, end, 'NO', 'YES'))
            else:
                ents.append((list_train_data[i][0], ent, start, end, 'NO', 'NO'))
        for i, ent in enumerate(ents):
            if ent[4] == 'YES' or ent[5] == 'YES':
                ent_prob.append(ent)
    train_data_df = pd.DataFrame(train_data)
    ent_prob_df = pd.DataFrame(ent_prob, columns=['id', 'text', 'start', 'end', 'punBeg', 'punEnd'])
    merged = pd.merge(train_data_df, ent_prob_df, on='id')
    common_index = merged.index
    train_data_df = train_data_df.drop(index=common_index)
    for index, row in merged.iterrows():
        if row['punBeg'] == 'YES':
            for item in row['label']:
                if int(item[0]) == int(row['start']):
                    item[0] = item[0] + 1
        if row['punEnd'] == 'YES':
            for item in row['label']:
                if int(item[1]) == int(row['end']):
                    item[1] = item[1] - 1
    for index, row in merged.iterrows():
        new_row = {'id': row['id'], 'text': row['text_x'], 'label': row['label'], 'Comments': row['Comments']}
        train_data_df = train_data_df.append(new_row, ignore_index=True)
    dict_list = train_data_df.to_dict(orient='records')
    list_train_data = []
    for i, ele in enumerate(dict_list):
        list_train_data.append((dict_list[i]['text'], {'labels': dict_list[i]['label']}))
    list_train_data = []
    for i, ele in enumerate(dict_list):
        list_train_data.append((dict_list[i]['text'], {'labels': dict_list[i]['label']}))
    for i, ele in enumerate(list_train_data):
        entities = []
        for label in list_train_data[i][1]['labels']:
            tuple_ = (label[0], label[1], label[2])
            entities.append(tuple_)
            list_train_data[i][1]['labels'] = entities
    return list_train_data

import pandas as pd
import json
import os
os.chdir('/kaggle/input/d/fatimahabib1/niort-sentences')
with open('annotations-niort-sentence-level.jsonl', 'r', encoding='utf-8') as f:
    s = f.read()
    data = json.loads(s)

train_data = data['annotations']

list_train_data = []
for i, ele in enumerate(train_data):
    list_train_data.append((train_data[i]['text'], {'labels': train_data[i]['label']}))

training_list = trim_entity_spans(list_train_data)

ent_prob = []
ents = []
for i, ele in enumerate(training_list):
    for j, elem in enumerate(training_list[i][1]['labels']):
        start = elem[0]
        end = elem[1]
        ent = training_list[i][0][start:end]
        num_spaces_beginning = len(ent) - len(ent.lstrip())
        num_spaces_end = len(ent) - len(ent.rstrip())
        ents.append((training_list[i][0], ent, start, end, num_spaces_beginning, num_spaces_end))
    for i, ent in enumerate(ents):
        if ent[1] != ent[1].strip():
            ent_prob.append(ent)

nlp = fr_core_news_sm.load()
db = DocBin()
for text, annot in tqdm(training_list):
    doc = nlp.make_doc(text)
    ents = []
    for start, end, label in annot['labels']:
        span = doc.char_span(start, end, label=label, alignment_mode='contract')
        if span is None:
            print('Skipping entity')
        else:
            ents.append(span)
    pat_orig = len(ents)
    filtered = filter_spans(ents)
    pat_filt = len(filtered)
    doc.ents = ents
    doc.ents = ents
    db.add(doc)
db.to_disk('/kaggle/working/train.spacy')