Update spaCy pipeline

README.md

@@ -26,8 +26,8 @@ model-index:
 | **Name** | `fr_trf_nrp` |
 | **Version** | `0.0.0` |
 | **spaCy** | `>=3.8.3,<3.9.0` |
-| **Default Pipeline** | `ner_transformer`, `ner`, `merge_entities`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
-| **Components** | `ner_transformer`, `ner`, `merge_entities`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
+| **Default Pipeline** | `ner_transformer`, `ner`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
+| **Components** | `ner_transformer`, `ner`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
 | **Vectors** | 0 keys, 0 unique vectors (0 dimensions) |
 | **Sources** | n/a |
 | **License** | n/a |

base_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae82900483b15cbd017ba63a5a7e8833ffa9d295fe764b168d33281a2bcd746c
+oid sha256:bee41def552f5762e57377fc1a6e91f6bc085801a1d67d74a51387d947181fbd
 size 443537828

config.cfg

@@ -10,7 +10,7 @@ seed = 17
 
 [nlp]
 lang = "fr"
-pipeline = ["ner_transformer","ner","merge_entities","base_transformer","morphologizer","tagger","parser","trainable_lemmatizer"]
+pipeline = ["ner_transformer","ner","base_transformer","morphologizer","tagger","parser","trainable_lemmatizer"]
 batch_size = 512
 disabled = []
 before_creation = null
@@ -43,9 +43,6 @@ use_fast = true
 
 [components.base_transformer.model.transformer_config]
 
-[components.merge_entities]
-factory = "merge_entities"
-
 [components.morphologizer]
 factory = "morphologizer"
 extend = false

fr_trf_nrp-any-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21aa8991ffd5c4f6327d8c41b4deef58c781fb3c5575f8ae7169e89f6a477854
-size 822747526
+oid sha256:39f5ef22d177266bdc2c9d17caa66688d4f5158d39761527a0e54ca3430ede3e
+size 822748847

meta.json

@@ -567,7 +567,6 @@
   "pipeline":[
     "ner_transformer",
     "ner",
-    "merge_entities",
     "base_transformer",
     "morphologizer",
     "tagger",
@@ -577,7 +576,6 @@
   "components":[
     "ner_transformer",
     "ner",
-    "merge_entities",
     "base_transformer",
     "morphologizer",
     "tagger",

ner_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec954b90f0e7d6ddb6e3473315aa58d75ce25f4afb0340776ec337a7817ea614
+oid sha256:160182b4b82beb281481ab70990e790227fc111d80961ae73b375f48b1e924a5
 size 440759145

use_custom_tokenizer.py

@@ -1,13 +1,12 @@
 from spacy.util import registry
-
-from commercial_registry_ner.spacy.custom_tokenizer.custom_tokenizer import (
-    custom_tokenizer,
-)
-
+from spacy.tokenizer import Tokenizer
+import pathlib
 
 @registry.tokenizers("customize_tokenizer")
 def make_customize_tokenizer():
     def customize_tokenizer(nlp):
-        return custom_tokenizer(nlp)
+        tokenizer = Tokenizer(nlp.vocab)
+        script_dir = pathlib.Path(__file__).parent.resolve()
+        return tokenizer.from_disk(script_dir / "tokenizer")
 
     return customize_tokenizer