text_html_output_1.png
from spacy.tokens import DocBin

from spacy.util import filter_spans

from tqdm import tqdm

import fr_core_news_sm

import json

import os

import os

import pandas as pd

import pandas as pd

import re

import re

import spacy

spacy.require_gpu()

import re
def trim_entity_spans(data: list) -> list:
    """Removes leading and trailing white spaces from entity spans.
    Args:
        data (list): The data to be cleaned in spaCy JSON format.
    Returns:
        list: The cleaned data.
    """
    invalid_span_tokens = re.compile('\\s')
    cleaned_data = []
    for text, annotations in data:
        entities = annotations['labels']
        valid_entities = []
        for start, end, label in entities:
            valid_start = start
            valid_end = end
            while valid_start < len(text) and invalid_span_tokens.match(text[valid_start]):
                valid_start += 1
            while valid_end > 1 and invalid_span_tokens.match(text[valid_end - 1]):
                valid_end -= 1
            valid_entities.append([valid_start, valid_end, label])
        cleaned_data.append([text, {'labels': valid_entities}])
    return cleaned_data

def starts_with_punctuation(s):
    if re.match('^\\W', s):
        return True
    else:
        return False
def ends_with_punctuation(s):
    if re.search('\\W$', s):
        return True
    else:
        return False
def dp(train_data):
    list_train_data = []
    for i, ele in enumerate(train_data):
        list_train_data.append((train_data[i]['id'], train_data[i]['text'], {'labels': train_data[i]['label']}))
    ent_prob = []
    for i, ele in enumerate(list_train_data):
        ents = []
        for j, elem in enumerate(list_train_data[i][2]['labels']):
            start = elem[0]
            end = elem[1]
            ent = list_train_data[i][1][start:end]
            if starts_with_punctuation(ent) and ends_with_punctuation(ent):
                ents.append((list_train_data[i][0], ent, start, end, 'YES', 'YES'))
            elif starts_with_punctuation(ent) and (not ends_with_punctuation(ent)):
                ents.append((list_train_data[i][0], ent, start, end, 'YES', 'NO'))
            elif not starts_with_punctuation(ent) and ends_with_punctuation(ent):
                ents.append((list_train_data[i][0], ent, start, end, 'NO', 'YES'))
            else:
                ents.append((list_train_data[i][0], ent, start, end, 'NO', 'NO'))
        for i, ent in enumerate(ents):
            if ent[4] == 'YES' or ent[5] == 'YES':
                ent_prob.append(ent)
    train_data_df = pd.DataFrame(train_data)
    ent_prob_df = pd.DataFrame(ent_prob, columns=['id', 'text', 'start', 'end', 'punBeg', 'punEnd'])
    merged = pd.merge(train_data_df, ent_prob_df, on='id')
    common_index = merged.index
    train_data_df = train_data_df.drop(index=common_index)
    for index, row in merged.iterrows():
        if row['punBeg'] == 'YES':
            for item in row['label']:
                if int(item[0]) == int(row['start']):
                    item[0] = item[0] + 1
        if row['punEnd'] == 'YES':
            for item in row['label']:
                if int(item[1]) == int(row['end']):
                    item[1] = item[1] - 1
    for index, row in merged.iterrows():
        new_row = {'id': row['id'], 'text': row['text_x'], 'label': row['label'], 'Comments': row['Comments']}
        train_data_df = train_data_df.append(new_row, ignore_index=True)
    dict_list = train_data_df.to_dict(orient='records')
    list_train_data = []
    for i, ele in enumerate(dict_list):
        list_train_data.append((dict_list[i]['text'], {'labels': dict_list[i]['label']}))
    list_train_data = []
    for i, ele in enumerate(dict_list):
        list_train_data.append((dict_list[i]['text'], {'labels': dict_list[i]['label']}))
    for i, ele in enumerate(list_train_data):
        entities = []
        for label in list_train_data[i][1]['labels']:
            tuple_ = (label[0], label[1], label[2])
            entities.append(tuple_)
            list_train_data[i][1]['labels'] = entities
    return list_train_data

import pandas as pd
import json
import os
os.chdir('/kaggle/input/d/fatimahabib1/niort-sentences')
with open('annotations-niort-sentence-level.jsonl', 'r', encoding='utf-8') as f:
    s = f.read()
    data = json.loads(s)

train_data = data['annotations']

list_train_data = []
for i, ele in enumerate(train_data):
    list_train_data.append((train_data[i]['text'], {'labels': train_data[i]['label']}))

training_list = trim_entity_spans(list_train_data)

ent_prob = []
ents = []
for i, ele in enumerate(training_list):
    for j, elem in enumerate(training_list[i][1]['labels']):
        start = elem[0]
        end = elem[1]
        ent = training_list[i][0][start:end]
        num_spaces_beginning = len(ent) - len(ent.lstrip())
        num_spaces_end = len(ent) - len(ent.rstrip())
        ents.append((training_list[i][0], ent, start, end, num_spaces_beginning, num_spaces_end))
    for i, ent in enumerate(ents):
        if ent[1] != ent[1].strip():
            ent_prob.append(ent)

nlp = fr_core_news_sm.load()
db = DocBin()
for text, annot in tqdm(training_list):
    doc = nlp.make_doc(text)
    ents = []
    for start, end, label in annot['labels']:
        span = doc.char_span(start, end, label=label, alignment_mode='contract')
        ents.append(span)
    pat_orig = len(ents)
    filtered = filter_spans(ents)
    pat_filt = len(filtered)
    doc.ents = ents
    doc.ents = ents
    db.add(doc)
db.to_disk('/kaggle/working/train.spacy')

best_nlp = spacy.load('/kaggle/working/model-best')

colors = {'INFO': '#F67DE3', 'SUB': '#7DF6D9'}
options = {'colors': colors}
doc = best_nlp('pour les constructions de moins de 90 m² de surface de plancher : 30 m² de surface de plancher supplémentaire par rapport à la surface de plancher existante à la date d’approbation du PLU. ')

doc = best_nlp('La  peinture  sur  les  murs  en  pierre  en  taille  ou  en  moellon  est  interdite.  L’emploi  à  nu  des matériaux destinés à être enduits est strictement interdit. ')

test_sent = 'Les  règles  applicables  aux  constructions  non  prévues  ci-dessus  sont  celles  auxquelles  elles sont le plus directement assimilables. Lorsqu’un projet comporte plusieurs destinations, les places de stationnement se calculent au prorata de la surface de plancher de chaque destination de construction. En  cas  de  changement  de  destination,  il  ne  sera  exigé  que  les  places  de  stationnement correspondant au différentiel entre les deux destinations.  L’ensemble  des  dispositions  prévues  dans  cet  article  ne  s’applique  pas  aux  demandes d’extension ou de surélévation apportées aux immeubles de logements existants sans création de  logement  supplémentaire.  Ces  dispositions  ne  s’appliquent  pas  non  plus  dans  le  cas  de création  de  surface  de  plancher  liée  à  une  annexe  (habitation)  sans  création  de  logement supplémentaire. '
doc = best_nlp(test_sent)
spacy.displacy.render(doc, style='ent', options=options, jupyter=True)