File size: 5,280 Bytes
8afc5a1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 |
#! /usr/bin/python3
import os
src="LiquidAI/LFM2-1.2B"
tgt="KoichiYasuoka/lfm2-1.2b-japanese-ud-embeds"
url="https://github.com/UniversalDependencies/UD_Japanese-GSDLUW"
d=os.path.basename(url)
os.system(f"test -d {d} || git clone --depth=1 {url}")
os.system(f"for F in train dev test ; do cp {d}/*-$F.conllu $F.conllu ; done")
class UDEmbedsDataset(object):
def __init__(self,conllu,tokenizer,oldtokenizer=None,embeddings=None):
self.conllu=open(conllu,"r",encoding="utf-8")
self.tokenizer=tokenizer
self.oldtokenizer=oldtokenizer if oldtokenizer else tokenizer
self.embeddings=embeddings
self.seeks=[0]
label=set(["SYM","SYM.","SYM|_"])
dep=set()
s=self.conllu.readline()
while s!="":
if s=="\n":
self.seeks.append(self.conllu.tell())
else:
w=s.split("\t")
if len(w)==10:
if w[0].isdecimal():
p=w[3]
q="" if w[5]=="_" else "|"+w[5]
d=("|" if w[6]=="0" else "|l-" if int(w[0])<int(w[6]) else "|r-")+w[7]
for k in [p,p+".","B-"+p,"B-"+p+".","I-"+p,"I-"+p+".",p+q+"|_",p+q+d]:
label.add(k)
s=self.conllu.readline()
self.label2id={l:i for i,l in enumerate(sorted(label))}
def __call__(*args):
lid={l:i for i,l in enumerate(sorted(set(sum([list(t.label2id) for t in args],[]))))}
for t in args:
t.label2id=lid
return lid
def __del__(self):
self.conllu.close()
__len__=lambda self:len(self.seeks)-1
def __getitem__(self,i):
import torch,numpy
self.conllu.seek(self.seeks[i])
c,t,s=[],[""],False
while t[0]!="\n":
t=self.conllu.readline().split("\t")
if len(t)==10 and t[0].isdecimal():
if s:
t[1]=" "+t[1]
c.append(t)
s=t[9].find("SpaceAfter=No")<0
x=[True if t[6]=="0" or int(t[6])>j or sum([1 if int(c[i][6])==j+1 else 0 for i in range(j+1,len(c))])>0 else False for j,t in enumerate(c)]
v=self.tokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"]
ids,upos=[self.tokenizer.bos_token_id],["SYM."]
for i,(j,k) in enumerate(zip(v,c)):
if j==[]:
j=[self.tokenizer.unk_token_id]
p=k[3] if x[i] else k[3]+"."
ids+=j
upos+=[p] if len(j)==1 else ["B-"+p]+["I-"+p]*(len(j)-1)
x=[True if t[6]=="0" or int(t[6])>j or sum([1 if int(c[i][6])==j+1 else 0 for i in range(j+1,len(c))])>0 else False for j,t in enumerate(c)]
if len(x)<62:
x=[True]*len(x)
else:
w=sum([len(x)-i+1 if b else 0 for i,b in enumerate(x)])+len(ids)+1
for i in range(len(x)):
if x[i]==False and w+len(x)-i<2000:
x[i]=True
w+=len(x)-i+1
v=self.oldtokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"]
p=[t[3] if t[5]=="_" else t[3]+"|"+t[5] for i,t in enumerate(c)]
d=[t[7] if t[6]=="0" else "l-"+t[7] if int(t[0])<int(t[6]) else "r-"+t[7] for t in c]
idx=[-1]
upos.append("SYM|_")
for i in range(len(x)):
if x[i]:
idx.append(i)
upos.append(p[i]+"|"+d[i] if c[i][6]=="0" else p[i]+"|_")
for j in range(i+1,len(x)):
idx.append(j)
upos.append(p[j]+"|"+d[j] if int(c[j][6])==i+1 else p[i]+"|"+d[i] if int(c[i][6])==j+1 else p[j]+"|_")
idx.append(-1)
upos.append("SYM|_")
with torch.no_grad():
m=[]
for j in v:
if j==[]:
j=[self.tokenizer.unk_token_id]
m.append(self.embeddings[j,:].sum(axis=0))
m.append(self.embeddings[self.tokenizer.eos_token_id])
emb=torch.from_numpy(numpy.array(m))
return{"inputs_embeds":torch.vstack((torch.from_numpy(self.embeddings[ids,:]),emb[idx,:])),"labels":[self.label2id[p] for p in upos]}
from transformers import AutoTokenizer,AutoConfig,Lfm2PreTrainedModel,DefaultDataCollator,TrainingArguments,Trainer
from transformers.modeling_layers import GenericForTokenClassification
class Lfm2ForTokenClassification(GenericForTokenClassification,Lfm2PreTrainedModel):
pass
from tokenizers.pre_tokenizers import Sequence,Split,Whitespace,Punctuation
from tokenizers import Regex
from copy import deepcopy
otk=AutoTokenizer.from_pretrained(src,unk_token="<|endoftext|>")
otk.model_input_names=["input_ids","attention_mask"]
ntk=deepcopy(otk)
ntk.backend_tokenizer.pre_tokenizer=Sequence([Split(Regex("[ぁ-ん]"),"isolated"),Whitespace(),Punctuation(),otk.backend_tokenizer.pre_tokenizer])
trainDS=UDEmbedsDataset("train.conllu",ntk,otk)
devDS=UDEmbedsDataset("dev.conllu",ntk,otk)
testDS=UDEmbedsDataset("test.conllu",ntk,otk)
lid=trainDS(devDS,testDS)
cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True,trust_remote_code=True)
mdl=Lfm2ForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True,trust_remote_code=True)
trainDS.embeddings=mdl.get_input_embeddings().weight.cpu().detach().numpy()
arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=1,dataloader_pin_memory=False,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False)
trn=Trainer(args=arg,data_collator=DefaultDataCollator(),model=mdl,train_dataset=trainDS)
trn.train()
trn.save_model(tgt)
otk.save_pretrained(tgt)
|