KoichiYasuoka
/

modernbert-japanese-130m-ud-embeds

Token Classification

dependency-parsing

Model card Files Files and versions Community

KoichiYasuoka commited on Feb 13

Commit

dc42f39

·

1 Parent(s): 223b87d

model improved

Files changed (2) hide show

maker.py +1 -1
pytorch_model.bin +1 -1

maker.py CHANGED Viewed

@@ -99,7 +99,7 @@ from tokenizers import Regex
 from copy import deepcopy
 otk=AutoTokenizer.from_pretrained(src)
 ntk=deepcopy(otk)
-ntk.backend_tokenizer.pre_tokenizer=Sequence([Split("[ぁ-ん]","isolated"),otk.backend_tokenizer.pre_tokenizer])
 trainDS=UDEmbedsDataset("train.conllu",ntk,otk)
 devDS=UDEmbedsDataset("dev.conllu",ntk,otk)
 testDS=UDEmbedsDataset("test.conllu",ntk,otk)

 from copy import deepcopy
 otk=AutoTokenizer.from_pretrained(src)
 ntk=deepcopy(otk)
+ntk.backend_tokenizer.pre_tokenizer=Sequence([Split(Regex("[ぁ-ん]"),"isolated"),otk.backend_tokenizer.pre_tokenizer])
 trainDS=UDEmbedsDataset("train.conllu",ntk,otk)
 devDS=UDEmbedsDataset("dev.conllu",ntk,otk)
 testDS=UDEmbedsDataset("test.conllu",ntk,otk)

pytorch_model.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b74f493a41ee73d8b1dce645987d4daebc5a31f40b8737179b92fa50e1b3a5b
 size 530122154

 version https://git-lfs.github.com/spec/v1
+oid sha256:51561774a9700240973c5c468cfb1a2ac08291e38c5683a74819427ef236904b
 size 530122154