model improved

README.md

@@ -48,20 +48,22 @@ class TransformersUD(object):
       AutoModelForTokenClassification,AutoConfig,TokenClassificationPipeline)
     self.tokenizer=AutoTokenizer.from_pretrained(bert)
     self.model=AutoModelForQuestionAnswering.from_pretrained(bert)
-    d=os.path.join(bert,"tagger")
-    if os.path.isdir(d):
-      m=AutoModelForTokenClassification.from_pretrained(d)
+    x=AutoModelForTokenClassification.from_pretrained
+    if os.path.isdir(bert):
+      d,t=x(os.path.join(bert,"deprel")),x(os.path.join(bert,"tagger"))
     else:
       from transformers.file_utils import hf_bucket_url
-      c=AutoConfig.from_pretrained(hf_bucket_url(bert,"tagger/config.json"))
-      m=AutoModelForTokenClassification.from_pretrained(
-        hf_bucket_url(bert,"tagger/pytorch_model.bin"),config=c)
-    self.tagger=TokenClassificationPipeline(model=m,tokenizer=self.tokenizer,
+      c=AutoConfig.from_pretrained(hf_bucket_url(bert,"deprel/config.json"))
+      d=x(hf_bucket_url(bert,"deprel/pytorch_model.bin"),config=c)
+      s=AutoConfig.from_pretrained(hf_bucket_url(bert,"tagger/config.json"))
+      t=x(hf_bucket_url(bert,"tagger/pytorch_model.bin"),config=s)
+    self.deprel=TokenClassificationPipeline(model=d,tokenizer=self.tokenizer,
       aggregation_strategy="simple")
+    self.tagger=TokenClassificationPipeline(model=t,tokenizer=self.tokenizer)
   def __call__(self,text):
     import numpy,torch,ufal.chu_liu_edmonds
-    y=self.tagger(text)
-    w=[(t["start"],t["end"],t["entity_group"].split("|")) for t in y]
+    w=[(t["start"],t["end"],t["entity_group"]) for t in self.deprel(text)]
+    z={t["start"]:t["entity"][2:].split("|") for t in self.tagger(text)}
     r=[text[s:e] for s,e,p in w]
     v=self.tokenizer(r,add_special_tokens=False)["input_ids"]
     m=numpy.full((len(v)+1,len(v)+1),numpy.nan)
@@ -77,8 +79,8 @@ class TransformersUD(object):
     h=ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0]
     u="# text = "+text.replace("\n"," ")+"\n"
     for i,(s,e,p) in enumerate(w,1):
-      u+="\t".join([str(i),r[i-1],"_",p[0],"_","|".join(p[1:-1]),str(h[i]),
-        p[-1],"_","_" if i<len(w) and w[i][0]<e else "SpaceAfter=No"])+"\n"
+      u+="\t".join([str(i),r[i-1],"_",z[s][0],"_","|".join(z[s][1:]),str(h[i])
+        ,p,"_","_" if i<len(w) and w[i][0]<e else "SpaceAfter=No"])+"\n"
     return u
 
 nlp=TransformersUD("KoichiYasuoka/deberta-base-japanese-aozora-ud-head")

tagger/config.json

@@ -10,348 +10,86 @@
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
   "id2label": {
-    "0": "B-ADJ|_|acl",
-    "1": "B-ADJ|_|advcl",
-    "2": "B-ADJ|_|amod",
-    "3": "B-ADJ|_|ccomp",
-    "4": "B-ADJ|_|csubj",
-    "5": "B-ADJ|_|dep",
-    "6": "B-ADJ|_|dislocated",
-    "7": "B-ADJ|_|nmod",
-    "8": "B-ADJ|_|nsubj",
-    "9": "B-ADJ|_|obj",
-    "10": "B-ADJ|_|obl",
-    "11": "B-ADJ|_|root",
-    "12": "B-ADP|_|case",
-    "13": "B-ADP|_|fixed",
-    "14": "B-ADV|_|advcl",
-    "15": "B-ADV|_|advmod",
-    "16": "B-ADV|_|dep",
-    "17": "B-ADV|_|obj",
-    "18": "B-ADV|_|root",
-    "19": "B-AUX|Polarity=Neg|aux",
-    "20": "B-AUX|_|aux",
-    "21": "B-AUX|_|cop",
-    "22": "B-AUX|_|fixed",
-    "23": "B-AUX|_|root",
-    "24": "B-CCONJ|_|cc",
-    "25": "B-DET|_|det",
-    "26": "B-INTJ|_|discourse",
-    "27": "B-INTJ|_|root",
-    "28": "B-NOUN|Polarity=Neg|obl",
-    "29": "B-NOUN|Polarity=Neg|root",
-    "30": "B-NOUN|_|acl",
-    "31": "B-NOUN|_|advcl",
-    "32": "B-NOUN|_|ccomp",
-    "33": "B-NOUN|_|compound",
-    "34": "B-NOUN|_|csubj",
-    "35": "B-NOUN|_|dislocated",
-    "36": "B-NOUN|_|nmod",
-    "37": "B-NOUN|_|nsubj",
-    "38": "B-NOUN|_|obj",
-    "39": "B-NOUN|_|obl",
-    "40": "B-NOUN|_|root",
-    "41": "B-NUM|_|advcl",
-    "42": "B-NUM|_|compound",
-    "43": "B-NUM|_|dislocated",
-    "44": "B-NUM|_|nmod",
-    "45": "B-NUM|_|nsubj",
-    "46": "B-NUM|_|nummod",
-    "47": "B-NUM|_|obj",
-    "48": "B-NUM|_|obl",
-    "49": "B-NUM|_|root",
-    "50": "B-PART|_|mark",
-    "51": "B-PRON|_|acl",
-    "52": "B-PRON|_|advcl",
-    "53": "B-PRON|_|dislocated",
-    "54": "B-PRON|_|nmod",
-    "55": "B-PRON|_|nsubj",
-    "56": "B-PRON|_|obj",
-    "57": "B-PRON|_|obl",
-    "58": "B-PRON|_|root",
-    "59": "B-PROPN|_|acl",
-    "60": "B-PROPN|_|advcl",
-    "61": "B-PROPN|_|compound",
-    "62": "B-PROPN|_|dislocated",
-    "63": "B-PROPN|_|nmod",
-    "64": "B-PROPN|_|nsubj",
-    "65": "B-PROPN|_|obj",
-    "66": "B-PROPN|_|obl",
-    "67": "B-PROPN|_|root",
-    "68": "B-PUNCT|_|punct",
-    "69": "B-SCONJ|_|mark",
-    "70": "B-SYM|_|compound",
-    "71": "B-SYM|_|dep",
-    "72": "B-SYM|_|nmod",
-    "73": "B-SYM|_|obl",
-    "74": "B-VERB|_|acl",
-    "75": "B-VERB|_|advcl",
-    "76": "B-VERB|_|ccomp",
-    "77": "B-VERB|_|compound",
-    "78": "B-VERB|_|csubj",
-    "79": "B-VERB|_|dislocated",
-    "80": "B-VERB|_|nmod",
-    "81": "B-VERB|_|obj",
-    "82": "B-VERB|_|obl",
-    "83": "B-VERB|_|root",
-    "84": "B-X|_|dep",
-    "85": "B-X|_|nmod",
-    "86": "I-ADJ|_|acl",
-    "87": "I-ADJ|_|advcl",
-    "88": "I-ADJ|_|amod",
-    "89": "I-ADJ|_|ccomp",
-    "90": "I-ADJ|_|csubj",
-    "91": "I-ADJ|_|dep",
-    "92": "I-ADJ|_|dislocated",
-    "93": "I-ADJ|_|nmod",
-    "94": "I-ADJ|_|nsubj",
-    "95": "I-ADJ|_|obj",
-    "96": "I-ADJ|_|obl",
-    "97": "I-ADJ|_|root",
-    "98": "I-ADP|_|case",
-    "99": "I-ADP|_|fixed",
-    "100": "I-ADV|_|advcl",
-    "101": "I-ADV|_|advmod",
-    "102": "I-ADV|_|dep",
-    "103": "I-ADV|_|obj",
-    "104": "I-ADV|_|root",
-    "105": "I-AUX|Polarity=Neg|aux",
-    "106": "I-AUX|_|aux",
-    "107": "I-AUX|_|cop",
-    "108": "I-AUX|_|fixed",
-    "109": "I-AUX|_|root",
-    "110": "I-CCONJ|_|cc",
-    "111": "I-DET|_|det",
-    "112": "I-INTJ|_|discourse",
-    "113": "I-INTJ|_|root",
-    "114": "I-NOUN|Polarity=Neg|obl",
-    "115": "I-NOUN|Polarity=Neg|root",
-    "116": "I-NOUN|_|acl",
-    "117": "I-NOUN|_|advcl",
-    "118": "I-NOUN|_|ccomp",
-    "119": "I-NOUN|_|compound",
-    "120": "I-NOUN|_|csubj",
-    "121": "I-NOUN|_|dislocated",
-    "122": "I-NOUN|_|nmod",
-    "123": "I-NOUN|_|nsubj",
-    "124": "I-NOUN|_|obj",
-    "125": "I-NOUN|_|obl",
-    "126": "I-NOUN|_|root",
-    "127": "I-NUM|_|advcl",
-    "128": "I-NUM|_|compound",
-    "129": "I-NUM|_|dislocated",
-    "130": "I-NUM|_|nmod",
-    "131": "I-NUM|_|nsubj",
-    "132": "I-NUM|_|nummod",
-    "133": "I-NUM|_|obj",
-    "134": "I-NUM|_|obl",
-    "135": "I-NUM|_|root",
-    "136": "I-PART|_|mark",
-    "137": "I-PRON|_|acl",
-    "138": "I-PRON|_|advcl",
-    "139": "I-PRON|_|dislocated",
-    "140": "I-PRON|_|nmod",
-    "141": "I-PRON|_|nsubj",
-    "142": "I-PRON|_|obj",
-    "143": "I-PRON|_|obl",
-    "144": "I-PRON|_|root",
-    "145": "I-PROPN|_|acl",
-    "146": "I-PROPN|_|advcl",
-    "147": "I-PROPN|_|compound",
-    "148": "I-PROPN|_|dislocated",
-    "149": "I-PROPN|_|nmod",
-    "150": "I-PROPN|_|nsubj",
-    "151": "I-PROPN|_|obj",
-    "152": "I-PROPN|_|obl",
-    "153": "I-PROPN|_|root",
-    "154": "I-PUNCT|_|punct",
-    "155": "I-SCONJ|_|mark",
-    "156": "I-SYM|_|dep",
-    "157": "I-SYM|_|nmod",
-    "158": "I-VERB|_|acl",
-    "159": "I-VERB|_|advcl",
-    "160": "I-VERB|_|ccomp",
-    "161": "I-VERB|_|compound",
-    "162": "I-VERB|_|csubj",
-    "163": "I-VERB|_|dislocated",
-    "164": "I-VERB|_|nmod",
-    "165": "I-VERB|_|obj",
-    "166": "I-VERB|_|obl",
-    "167": "I-VERB|_|root",
-    "168": "I-X|_|nmod"
+    "0": "B-ADJ|_",
+    "1": "B-ADP|_",
+    "2": "B-ADV|_",
+    "3": "B-AUX|Polarity=Neg",
+    "4": "B-AUX|_",
+    "5": "B-CCONJ|_",
+    "6": "B-DET|_",
+    "7": "B-INTJ|_",
+    "8": "B-NOUN|Polarity=Neg",
+    "9": "B-NOUN|_",
+    "10": "B-NUM|_",
+    "11": "B-PART|_",
+    "12": "B-PRON|_",
+    "13": "B-PROPN|_",
+    "14": "B-PUNCT|_",
+    "15": "B-SCONJ|_",
+    "16": "B-SYM|_",
+    "17": "B-VERB|_",
+    "18": "B-X|_",
+    "19": "I-ADJ|_",
+    "20": "I-ADP|_",
+    "21": "I-ADV|_",
+    "22": "I-AUX|Polarity=Neg",
+    "23": "I-AUX|_",
+    "24": "I-CCONJ|_",
+    "25": "I-DET|_",
+    "26": "I-INTJ|_",
+    "27": "I-NOUN|Polarity=Neg",
+    "28": "I-NOUN|_",
+    "29": "I-NUM|_",
+    "30": "I-PART|_",
+    "31": "I-PRON|_",
+    "32": "I-PROPN|_",
+    "33": "I-PUNCT|_",
+    "34": "I-SCONJ|_",
+    "35": "I-SYM|_",
+    "36": "I-VERB|_",
+    "37": "I-X|_"
   },
   "initializer_range": 0.02,
   "intermediate_size": 3072,
   "label2id": {
-    "B-ADJ|_|acl": 0,
-    "B-ADJ|_|advcl": 1,
-    "B-ADJ|_|amod": 2,
-    "B-ADJ|_|ccomp": 3,
-    "B-ADJ|_|csubj": 4,
-    "B-ADJ|_|dep": 5,
-    "B-ADJ|_|dislocated": 6,
-    "B-ADJ|_|nmod": 7,
-    "B-ADJ|_|nsubj": 8,
-    "B-ADJ|_|obj": 9,
-    "B-ADJ|_|obl": 10,
-    "B-ADJ|_|root": 11,
-    "B-ADP|_|case": 12,
-    "B-ADP|_|fixed": 13,
-    "B-ADV|_|advcl": 14,
-    "B-ADV|_|advmod": 15,
-    "B-ADV|_|dep": 16,
-    "B-ADV|_|obj": 17,
-    "B-ADV|_|root": 18,
-    "B-AUX|Polarity=Neg|aux": 19,
-    "B-AUX|_|aux": 20,
-    "B-AUX|_|cop": 21,
-    "B-AUX|_|fixed": 22,
-    "B-AUX|_|root": 23,
-    "B-CCONJ|_|cc": 24,
-    "B-DET|_|det": 25,
-    "B-INTJ|_|discourse": 26,
-    "B-INTJ|_|root": 27,
-    "B-NOUN|Polarity=Neg|obl": 28,
-    "B-NOUN|Polarity=Neg|root": 29,
-    "B-NOUN|_|acl": 30,
-    "B-NOUN|_|advcl": 31,
-    "B-NOUN|_|ccomp": 32,
-    "B-NOUN|_|compound": 33,
-    "B-NOUN|_|csubj": 34,
-    "B-NOUN|_|dislocated": 35,
-    "B-NOUN|_|nmod": 36,
-    "B-NOUN|_|nsubj": 37,
-    "B-NOUN|_|obj": 38,
-    "B-NOUN|_|obl": 39,
-    "B-NOUN|_|root": 40,
-    "B-NUM|_|advcl": 41,
-    "B-NUM|_|compound": 42,
-    "B-NUM|_|dislocated": 43,
-    "B-NUM|_|nmod": 44,
-    "B-NUM|_|nsubj": 45,
-    "B-NUM|_|nummod": 46,
-    "B-NUM|_|obj": 47,
-    "B-NUM|_|obl": 48,
-    "B-NUM|_|root": 49,
-    "B-PART|_|mark": 50,
-    "B-PRON|_|acl": 51,
-    "B-PRON|_|advcl": 52,
-    "B-PRON|_|dislocated": 53,
-    "B-PRON|_|nmod": 54,
-    "B-PRON|_|nsubj": 55,
-    "B-PRON|_|obj": 56,
-    "B-PRON|_|obl": 57,
-    "B-PRON|_|root": 58,
-    "B-PROPN|_|acl": 59,
-    "B-PROPN|_|advcl": 60,
-    "B-PROPN|_|compound": 61,
-    "B-PROPN|_|dislocated": 62,
-    "B-PROPN|_|nmod": 63,
-    "B-PROPN|_|nsubj": 64,
-    "B-PROPN|_|obj": 65,
-    "B-PROPN|_|obl": 66,
-    "B-PROPN|_|root": 67,
-    "B-PUNCT|_|punct": 68,
-    "B-SCONJ|_|mark": 69,
-    "B-SYM|_|compound": 70,
-    "B-SYM|_|dep": 71,
-    "B-SYM|_|nmod": 72,
-    "B-SYM|_|obl": 73,
-    "B-VERB|_|acl": 74,
-    "B-VERB|_|advcl": 75,
-    "B-VERB|_|ccomp": 76,
-    "B-VERB|_|compound": 77,
-    "B-VERB|_|csubj": 78,
-    "B-VERB|_|dislocated": 79,
-    "B-VERB|_|nmod": 80,
-    "B-VERB|_|obj": 81,
-    "B-VERB|_|obl": 82,
-    "B-VERB|_|root": 83,
-    "B-X|_|dep": 84,
-    "B-X|_|nmod": 85,
-    "I-ADJ|_|acl": 86,
-    "I-ADJ|_|advcl": 87,
-    "I-ADJ|_|amod": 88,
-    "I-ADJ|_|ccomp": 89,
-    "I-ADJ|_|csubj": 90,
-    "I-ADJ|_|dep": 91,
-    "I-ADJ|_|dislocated": 92,
-    "I-ADJ|_|nmod": 93,
-    "I-ADJ|_|nsubj": 94,
-    "I-ADJ|_|obj": 95,
-    "I-ADJ|_|obl": 96,
-    "I-ADJ|_|root": 97,
-    "I-ADP|_|case": 98,
-    "I-ADP|_|fixed": 99,
-    "I-ADV|_|advcl": 100,
-    "I-ADV|_|advmod": 101,
-    "I-ADV|_|dep": 102,
-    "I-ADV|_|obj": 103,
-    "I-ADV|_|root": 104,
-    "I-AUX|Polarity=Neg|aux": 105,
-    "I-AUX|_|aux": 106,
-    "I-AUX|_|cop": 107,
-    "I-AUX|_|fixed": 108,
-    "I-AUX|_|root": 109,
-    "I-CCONJ|_|cc": 110,
-    "I-DET|_|det": 111,
-    "I-INTJ|_|discourse": 112,
-    "I-INTJ|_|root": 113,
-    "I-NOUN|Polarity=Neg|obl": 114,
-    "I-NOUN|Polarity=Neg|root": 115,
-    "I-NOUN|_|acl": 116,
-    "I-NOUN|_|advcl": 117,
-    "I-NOUN|_|ccomp": 118,
-    "I-NOUN|_|compound": 119,
-    "I-NOUN|_|csubj": 120,
-    "I-NOUN|_|dislocated": 121,
-    "I-NOUN|_|nmod": 122,
-    "I-NOUN|_|nsubj": 123,
-    "I-NOUN|_|obj": 124,
-    "I-NOUN|_|obl": 125,
-    "I-NOUN|_|root": 126,
-    "I-NUM|_|advcl": 127,
-    "I-NUM|_|compound": 128,
-    "I-NUM|_|dislocated": 129,
-    "I-NUM|_|nmod": 130,
-    "I-NUM|_|nsubj": 131,
-    "I-NUM|_|nummod": 132,
-    "I-NUM|_|obj": 133,
-    "I-NUM|_|obl": 134,
-    "I-NUM|_|root": 135,
-    "I-PART|_|mark": 136,
-    "I-PRON|_|acl": 137,
-    "I-PRON|_|advcl": 138,
-    "I-PRON|_|dislocated": 139,
-    "I-PRON|_|nmod": 140,
-    "I-PRON|_|nsubj": 141,
-    "I-PRON|_|obj": 142,
-    "I-PRON|_|obl": 143,
-    "I-PRON|_|root": 144,
-    "I-PROPN|_|acl": 145,
-    "I-PROPN|_|advcl": 146,
-    "I-PROPN|_|compound": 147,
-    "I-PROPN|_|dislocated": 148,
-    "I-PROPN|_|nmod": 149,
-    "I-PROPN|_|nsubj": 150,
-    "I-PROPN|_|obj": 151,
-    "I-PROPN|_|obl": 152,
-    "I-PROPN|_|root": 153,
-    "I-PUNCT|_|punct": 154,
-    "I-SCONJ|_|mark": 155,
-    "I-SYM|_|dep": 156,
-    "I-SYM|_|nmod": 157,
-    "I-VERB|_|acl": 158,
-    "I-VERB|_|advcl": 159,
-    "I-VERB|_|ccomp": 160,
-    "I-VERB|_|compound": 161,
-    "I-VERB|_|csubj": 162,
-    "I-VERB|_|dislocated": 163,
-    "I-VERB|_|nmod": 164,
-    "I-VERB|_|obj": 165,
-    "I-VERB|_|obl": 166,
-    "I-VERB|_|root": 167,
-    "I-X|_|nmod": 168
+    "B-ADJ|_": 0,
+    "B-ADP|_": 1,
+    "B-ADV|_": 2,
+    "B-AUX|Polarity=Neg": 3,
+    "B-AUX|_": 4,
+    "B-CCONJ|_": 5,
+    "B-DET|_": 6,
+    "B-INTJ|_": 7,
+    "B-NOUN|Polarity=Neg": 8,
+    "B-NOUN|_": 9,
+    "B-NUM|_": 10,
+    "B-PART|_": 11,
+    "B-PRON|_": 12,
+    "B-PROPN|_": 13,
+    "B-PUNCT|_": 14,
+    "B-SCONJ|_": 15,
+    "B-SYM|_": 16,
+    "B-VERB|_": 17,
+    "B-X|_": 18,
+    "I-ADJ|_": 19,
+    "I-ADP|_": 20,
+    "I-ADV|_": 21,
+    "I-AUX|Polarity=Neg": 22,
+    "I-AUX|_": 23,
+    "I-CCONJ|_": 24,
+    "I-DET|_": 25,
+    "I-INTJ|_": 26,
+    "I-NOUN|Polarity=Neg": 27,
+    "I-NOUN|_": 28,
+    "I-NUM|_": 29,
+    "I-PART|_": 30,
+    "I-PRON|_": 31,
+    "I-PROPN|_": 32,
+    "I-PUNCT|_": 33,
+    "I-SCONJ|_": 34,
+    "I-SYM|_": 35,
+    "I-VERB|_": 36,
+    "I-X|_": 37
   },
   "layer_norm_eps": 1e-07,
   "max_position_embeddings": 512,

tagger/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c261f7caed9b512d349d60b7def09f3eb26ef1075432140ca3d00dfa73ef9389
-size 440691763
+oid sha256:e837d28f6b426a84a65c0dd3881b30ffabfb28f3f5f5235472bffc074457b4d5
+size 440288755