Upload 14 files

README.md

@@ -0,0 +1,4 @@
+---
+language:
+- en
+---

added_tokens.json

@@ -0,0 +1 @@
+{}

bert_ner_model_loader.py

@@ -0,0 +1,191 @@
+"""BERT NER Inference."""
+
+from __future__ import absolute_import, division, print_function
+
+import json
+import os
+
+import torch
+import torch.nn.functional as F
+from torch.nn import CrossEntropyLoss
+from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
+from torch.utils.data.distributed import DistributedSampler
+from tqdm import tqdm, trange
+from nltk import word_tokenize
+# from transformers import (BertConfig, BertForTokenClassification,
+                                #   BertTokenizer)
+from pytorch_transformers import (BertForTokenClassification, BertTokenizer)
+
+
+class BertNer(BertForTokenClassification):
+
+    def forward(self, input_ids, token_type_ids=None, attention_mask=None, valid_ids=None):
+        sequence_output = self.bert(input_ids, token_type_ids, attention_mask, head_mask=None)[0]
+        batch_size,max_len,feat_dim = sequence_output.shape
+        valid_output = torch.zeros(batch_size,max_len,feat_dim,dtype=torch.float32,device='cpu')
+        # valid_output = torch.zeros(batch_size,max_len,feat_dim,dtype=torch.float32,device='cuda' if torch.cuda.is_available() else 'cpu')
+        for i in range(batch_size):
+            jj = -1
+            for j in range(max_len):
+                    if valid_ids[i][j].item() == 1:
+                        jj += 1
+                        valid_output[i][jj] = sequence_output[i][j]
+        sequence_output = self.dropout(valid_output)
+        logits = self.classifier(sequence_output)
+        return logits
+
+class Ner:
+
+    def __init__(self,model_dir: str):
+        self.model , self.tokenizer, self.model_config = self.load_model(model_dir)
+        self.label_map = self.model_config["label_map"]
+        self.max_seq_length = self.model_config["max_seq_length"]
+        self.label_map = {int(k):v for k,v in self.label_map.items()}
+        self.device = "cpu"
+        # self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.model = self.model.to(self.device)
+        self.model.eval()
+
+    def load_model(self, model_dir: str, model_config: str = "model_config.json"):
+        model_config = os.path.join(model_dir,model_config)
+        model_config = json.load(open(model_config))
+        model = BertNer.from_pretrained(model_dir)
+        tokenizer = BertTokenizer.from_pretrained(model_dir, do_lower_case=model_config["do_lower"])
+        return model, tokenizer, model_config
+
+    def tokenize(self, text: str):
+        """ tokenize input"""
+        words = word_tokenize(text)
+        tokens = []
+        valid_positions = []
+        for i,word in enumerate(words):
+            token = self.tokenizer.tokenize(word)
+            tokens.extend(token)
+            for i in range(len(token)):
+                if i == 0:
+                    valid_positions.append(1)
+                else:
+                    valid_positions.append(0)
+        # print("valid positions from text o/p:=>", valid_positions)
+        return tokens, valid_positions
+
+    def preprocess(self, text: str):
+        """ preprocess """
+        tokens, valid_positions = self.tokenize(text)
+        ## insert "[CLS]"
+        tokens.insert(0,"[CLS]")
+        valid_positions.insert(0,1)
+        ## insert "[SEP]"
+        tokens.append("[SEP]")
+        valid_positions.append(1)
+        segment_ids = []
+        for i in range(len(tokens)):
+            segment_ids.append(0)
+        input_ids = self.tokenizer.convert_tokens_to_ids(tokens)
+        # print("input ids with berttokenizer:=>", input_ids)
+        input_mask = [1] * len(input_ids)
+        while len(input_ids) < self.max_seq_length:
+            input_ids.append(0)
+            input_mask.append(0)
+            segment_ids.append(0)
+            valid_positions.append(0)
+        return input_ids,input_mask,segment_ids,valid_positions
+
+    def predict_entity(self, B_lab, I_lab, words, labels, entity_list):
+        temp=[]
+        entity=[]
+
+        for word, (label, confidence), B_l, I_l in zip(words, labels, B_lab, I_lab):
+
+            if ((label==B_l) or (label==I_l)) and label!='O':
+                if label==B_l:
+                    entity.append(temp)
+                    temp=[]
+                    temp.append(label)
+                    
+                temp.append(word)
+
+        entity.append(temp)
+        # print(entity)
+
+        entity_name_label = []
+        for entity_name in entity[1:]:
+            for ent_key, ent_value in entity_list.items():
+                if (ent_key==entity_name[0]):
+                    # entity_name_label.append(' '.join(entity_name[1:]) + ": " + ent_value)
+                    entity_name_label.append([' '.join(entity_name[1:]), ent_value])
+        
+        return entity_name_label
+
+    def predict(self, text: str):
+        input_ids,input_mask,segment_ids,valid_ids = self.preprocess(text)
+        # print("valid ids:=>", segment_ids)
+        input_ids = torch.tensor([input_ids],dtype=torch.long,device=self.device)
+        input_mask = torch.tensor([input_mask],dtype=torch.long,device=self.device)
+        segment_ids = torch.tensor([segment_ids],dtype=torch.long,device=self.device)
+        valid_ids = torch.tensor([valid_ids],dtype=torch.long,device=self.device)
+
+        with torch.no_grad():
+            logits = self.model(input_ids, segment_ids, input_mask,valid_ids)
+        # print("logit values:=>", logits)
+        logits = F.softmax(logits,dim=2)
+        # print("logit values:=>", logits[0])
+        logits_label = torch.argmax(logits,dim=2)
+        logits_label = logits_label.detach().cpu().numpy().tolist()[0]
+        # print("logits label value list:=>", logits_label)
+
+        logits_confidence = [values[label].item() for values,label in zip(logits[0],logits_label)]
+
+        logits = []
+        pos = 0
+        for index,mask in enumerate(valid_ids[0]):
+            if index == 0:
+                continue
+            if mask == 1:
+                logits.append((logits_label[index-pos],logits_confidence[index-pos]))
+            else:
+                pos += 1
+        logits.pop()
+        labels = [(self.label_map[label],confidence) for label,confidence in logits]
+        words = word_tokenize(text)
+
+        entity_list = {'B-PER':'Person',
+                    'B-FAC':'Facility',
+                    'B-LOC':'Location',
+                    'B-ORG':'Organization',
+                    'B-ART':'Work Of Art',
+                    'B-EVENT':'Event',
+                    'B-DATE':'Date-Time Entity',
+                    'B-TIME':'Date-Time Entity',
+                    'B-LAW':'Law Terms',
+                    'B-PRODUCT':'Product',
+                    'B-PERCENT':'Percentage',
+                    'B-MONEY':'Currency',
+                    'B-LANGUAGE':'Langauge',
+                    'B-NORP':'Nationality / Religion / Political group',
+                    'B-QUANTITY':'Quantity',
+                    'B-ORDINAL':'Ordinal Number',
+                    'B-CARDINAL':'Cardinal Number'}
+        
+        B_labels=[]
+        I_labels=[]
+        for label, confidence in labels:
+            if (label[:1]=='B'):
+                B_labels.append(label)
+                I_labels.append('O')
+            elif (label[:1]=='I'):
+                I_labels.append(label)
+                B_labels.append('O')
+            else:
+                B_labels.append('O')
+                I_labels.append('O')
+
+        assert len(labels) == len(words) == len(I_labels) == len(B_labels)
+
+        output = self.predict_entity(B_labels, I_labels, words, labels, entity_list)
+        print(output)
+
+        # output = [{"word":word,"tag":label,"confidence":confidence} for word,(label,confidence) in zip(words,labels)]
+        return output
+
+

biobert_utils.py

@@ -0,0 +1,160 @@
+"""BERT NER Inference."""
+
+import json
+import os
+import torch
+import torch.nn.functional as F
+from nltk import word_tokenize
+from pytorch_transformers import (BertForTokenClassification, BertTokenizer)
+
+
+class BertNer(BertForTokenClassification):
+
+    def forward(self, input_ids, token_type_ids=None, attention_mask=None, valid_ids=None):
+        sequence_output = self.bert(input_ids, token_type_ids, attention_mask, head_mask=None)[0]
+        batch_size,max_len,feat_dim = sequence_output.shape
+        # valid_output = torch.zeros(batch_size,max_len,feat_dim,dtype=torch.float32,device='cuda' if torch.cuda.is_available() else 'cpu')
+        valid_output = torch.zeros(batch_size,max_len,feat_dim,dtype=torch.float32,device='cpu')
+        for i in range(batch_size):
+            jj = -1
+            for j in range(max_len):
+                    if valid_ids[i][j].item() == 1:
+                        jj += 1
+                        valid_output[i][jj] = sequence_output[i][j]
+        sequence_output = self.dropout(valid_output)
+        logits = self.classifier(sequence_output)
+        return logits
+
+class BIOBERT_Ner:
+
+    def __init__(self,model_dir: str):
+        self.model , self.tokenizer, self.model_config = self.load_model(model_dir)
+        self.label_map = self.model_config["label_map"]
+        self.max_seq_length = self.model_config["max_seq_length"]
+        self.label_map = {int(k):v for k,v in self.label_map.items()}
+        self.device = "cpu"
+        # self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.model = self.model.to(self.device)
+        self.model.eval()
+
+    def load_model(self, model_dir: str, model_config: str = "model_config.json"):
+        model_config = os.path.join(model_dir,model_config)
+        model_config = json.load(open(model_config))
+        model = BertNer.from_pretrained(model_dir)
+        tokenizer = BertTokenizer.from_pretrained(model_dir, do_lower_case=model_config["do_lower"])
+        return model, tokenizer, model_config
+
+    def tokenize(self, text: str):
+        """ tokenize input"""
+        words = word_tokenize(text)
+        tokens = []
+        valid_positions = []
+        for i,word in enumerate(words):
+            token = self.tokenizer.tokenize(word)
+            tokens.extend(token)
+            for i in range(len(token)):
+                if i == 0:
+                    valid_positions.append(1)
+                else:
+                    valid_positions.append(0)
+        return tokens, valid_positions
+
+    def preprocess(self, text: str):
+        """ preprocess """
+
+        tokens, valid_positions = self.tokenize(text)
+
+        ## insert "[CLS]"
+        tokens.insert(0,"[CLS]")
+
+        valid_positions.insert(0,1)
+
+        ## insert "[SEP]"
+        tokens.append("[SEP]")
+
+        valid_positions.append(1)
+        segment_ids = []
+        for i in range(len(tokens)):
+            segment_ids.append(0)
+        input_ids = self.tokenizer.convert_tokens_to_ids(tokens)
+        input_mask = [1] * len(input_ids)
+        while len(input_ids) < self.max_seq_length:
+            input_ids.append(0)
+            input_mask.append(0)
+            segment_ids.append(0)
+            valid_positions.append(0)
+        return input_ids,input_mask,segment_ids,valid_positions
+
+    def predict_entity(self, B_lab, I_lab, words, labels, entity_list):
+        temp=[]
+        entity=[]
+
+        for word, label, B_l, I_l in zip(words, labels, B_lab, I_lab):
+
+            if ((label==B_l) or (label==I_l)) and label!='O':
+                if label==B_l:
+                    entity.append(temp)
+                    temp=[]
+                    temp.append(label)
+                    
+                temp.append(word)
+
+        entity.append(temp)
+
+        entity_name_label = []
+        for entity_name in entity[1:]:
+            for ent_key, ent_value in entity_list.items():
+                if (ent_key==entity_name[0]):
+                    entity_name_label.append([' '.join(entity_name[1:]), ent_value])
+        
+        return entity_name_label
+
+    def predict(self, text: str):
+        print("text:", text)
+        input_ids,input_mask,segment_ids,valid_ids = self.preprocess(text)
+        input_ids = torch.tensor([input_ids],dtype=torch.long,device=self.device)
+        input_mask = torch.tensor([input_mask],dtype=torch.long,device=self.device)
+        segment_ids = torch.tensor([segment_ids],dtype=torch.long,device=self.device)
+        valid_ids = torch.tensor([valid_ids],dtype=torch.long,device=self.device)
+
+        with torch.no_grad():
+            logits = self.model(input_ids, segment_ids, input_mask,valid_ids)
+        logits = F.softmax(logits,dim=2)
+        logits_label = torch.argmax(logits,dim=2)
+        logits_label = logits_label.detach().cpu().numpy().tolist()[0]
+
+        logits = []
+        pos = 0
+        for index,mask in enumerate(valid_ids[0]):
+            if index == 0:
+                continue
+            if mask == 1:
+                logits.append((logits_label[index-pos]))
+            else:
+                pos += 1
+        logits.pop()
+        labels = [(self.label_map[label]) for label in logits]
+        words = word_tokenize(text)
+
+        entity_list = {'B-ANATOMY':'Anatomy', 'B-GENE':'Gene', 'B-CHEMICAL':'Chemical', 'B-DISEASE':'Disease', 'B-PROTEIN':'Protein', 'B-ORGANISM':'Organism', 'B-CANCER':'Cancer', 'B-ORGAN':'Organ', 'B-CELL':'Cell', 'B-TISSUE':'Tissue', 'B-PATHOLOGY_TERM':'Pathlogy', 'B-COMPLEX':'Complex', 'B-TAXON':'Taxon'}
+        
+        B_labels=[]
+        I_labels=[]
+        for label in labels:
+            if (label[:1]=='B'):
+                B_labels.append(label)
+                I_labels.append('O')
+            elif (label[:1]=='I'):
+                I_labels.append(label)
+                B_labels.append('O')
+            else:
+                B_labels.append('O')
+                I_labels.append('O')
+
+        assert len(labels) == len(words) == len(I_labels) == len(B_labels)
+
+        output = self.predict_entity(B_labels, I_labels, words, labels, entity_list)
+
+        return output
+
+

config.json

@@ -0,0 +1,34 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "finetuning_task": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "is_decoder": false,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "num_labels": 27,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "output_past": true,
+  "pruned_heads": {},
+  "torchscript": false,
+  "type_vocab_size": 2,
+  "use_bfloat16": false,
+  "vocab_size": 28996
+}

eval_results.txt

@@ -0,0 +1,4 @@
+f1 = 0.6010022731969414
+loss = 0.24734244643932496
+precision = 0.6018625506596533
+recall = 0.6001444515141614

model_config.json

@@ -0,0 +1,38 @@
+{
+    "bert_model": "biobert-base-cased",
+    "do_lower": false,
+    "max_seq_length": 128,
+    "num_labels": 29,
+    "label_map": 
+        {
+            "0"  :"O",
+            "1"  :"B-ANATOMY",           
+            "2"  :"I-ANATOMY",                           
+            "3"  :"B-GENE",
+            "4"  :"I-GENE",
+            "5"  :"B-CHEMICAL",      
+            "6"  :"I-CHEMICAL",      
+            "7"  :"B-DISEASE",       
+            "8"  :"I-DISEASE",
+            "9"  :"B-PROTEIN",
+            "10" :"I-PROTEIN",
+            "11" :"B-ORGANISM",
+            "12" :"I-ORGANISM",
+            "13" :"B-CANCER",
+            "14" :"I-CANCER",
+            "15" :"B-ORGAN",
+            "16" :"I-ORGAN",
+            "17" :"B-CELL",
+            "18" :"I-CELL",
+            "19" :"B-TISSUE",
+            "20" :"I-TISSUE",
+            "21": "B-PATHOLOGY_TERM",
+            "22": "I-PATHOLOGY_TERM",
+            "23": "B-COMPLEX",
+            "24": "I-COMPLEX",
+            "25": "B-TAXON",
+            "26": "I-TAXON",
+            "27" :"[CLS]",
+            "28" :"[SEP]"
+            }
+    }

packages.txt

@@ -0,0 +1 @@
+git lfs update --force

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:412399c4d81a36efcc63d3c6eebb37d9a442576b0e637eac08fd45d830b02efa
+size 433372007

requirements.txt

@@ -0,0 +1,37 @@
+boto3==1.14.56
+botocore==1.17.56
+certifi==2020.6.20
+chardet==3.0.4
+click==7.1.2
+docutils==0.15.2
+fire==0.3.1
+Flask==1.1.2
+Flask-Cors==3.0.9
+geoip2==3.0.0
+idna==2.8
+itsdangerous==1.1.0
+Jinja2==2.11.2
+jmespath==0.10.0
+joblib==0.16.0
+MarkupSafe==1.1.1
+maxminddb==1.5.2
+nltk==3.4.5
+numpy==1.23.5
+Pillow==7.2.0
+python-dateutil==2.8.2
+pytorch-transformers==1.2.0
+pytz==2020.1
+regex==2020.7.14
+requests==2.22.0
+s3transfer==0.3.3
+sacremoses==0.0.43
+sentencepiece==0.1.91
+six==1.15.0
+termcolor==1.1.0
+torch==1.4.0
+torchvision==0.5.0
+tqdm==4.48.2
+transformers==2.1.1
+urllib3==1.25.6
+uWSGI==2.0.19.1
+Werkzeug==1.0.1

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "init_inputs": []}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8d04d6c200da456bab414d4ce7ba1a7473ac55f2c50f2a14f43782b55cbb225
+size 1208