model improved

maker.py

@@ -100,7 +100,9 @@ class UDEmbedsDataset(object):
         emb=torch.stack(m)
     return{"inputs_embeds":emb[ids[:8192],:],"labels":[self.label2id[p] for p in upos[:8192]]}
 from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DefaultDataCollator,TrainingArguments,Trainer
+from tokenizers.pre_tokenizers import Sequence,Punctuation
 tkz=AutoTokenizer.from_pretrained(src)
+tkz.backend_tokenizer.pre_tokenizer=Sequence([Punctuation(),tkz.backend_tokenizer.pre_tokenizer])
 trainDS=UDEmbedsDataset("train.conllu",tkz)
 devDS=UDEmbedsDataset("dev.conllu",tkz)
 testDS=UDEmbedsDataset("test.conllu",tkz)

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:830c264f28fba9953e623b9f3dd35e58d30bd6469e297dfaf1d2220e037e9ce4
+oid sha256:622ac1217a6e379a516f12900f773f83fd05c5a2c2cf9b87b3f848a6d7810162
 size 663202290

tokenizer.json

@@ -1052,10 +1052,19 @@
     "type": "NFC"
   },
   "pre_tokenizer": {
-    "type": "ByteLevel",
-    "add_prefix_space": false,
-    "trim_offsets": true,
-    "use_regex": true
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "Punctuation",
+        "behavior": "Isolated"
+      },
+      {
+        "type": "ByteLevel",
+        "add_prefix_space": false,
+        "trim_offsets": true,
+        "use_regex": true
+      }
+    ]
   },
   "post_processor": {
     "type": "TemplateProcessing",