<repo_name>34153320/Domain_language_model_I<file_sep>/README.md
# Domain_language_model_I

Language models training on large scale dataset (for instance, BERT, XLnet, and GPT2) work for general purpose. However, in our daily application the language models are frequently used in specific domain. To avoid large scale retraining, and also transfer the pre-trained learned LM, here a strategy is proposed to obtain domain language model based on GPT2. 
<file_sep>/generate.py
import tensorflow as tf
import Transformer
from domain_transfer import create_domain_dict

"""
   Written by <NAME>. Part of the codes are borrowed from openai/GPT-2.
"""
domain_dict = ["I", "You",  "My",  "They",  "It",  "Am", "Are", "Need", "Feel", "Is",  "Hungry",  
               "Help", "Tired", "Not", "How", "Okay", "Very", "Thirsty", "Comfortable", "Right",
               "Please", "Hope", "Clean", "Glasses", "Nurse", "Closer", "Bring", "What", "Where", 
               "Tell", "That", "Going", "Music", "Like", "Outside", "Do", "Have", "Faith", 
               "Success", "Coming", "Good", "Bad", "Here", "Family", "Hello", "Goodbye", 
               "Computer", "Yes", "Up", "No"]

def top_k_logits(logits, k):
    if k == 0:
        # no truncation
        return logits

    def _top_k():
        values, _ = tf.nn.top_k(logits, k=k)
        min_values = values[:, -1, tf.newaxis]
        return tf.where(
            logits < min_values,
            tf.ones_like(logits, dtype=logits.dtype) * -1e10,
            logits,
        )
    return tf.cond(
       tf.equal(k, 0),
       lambda: logits,
       lambda: _top_k(),
    )

def sample_sequence(*, hparams, length, reduced_dict_index, 
                    start_token=None, batch_size=None, context=None, 
                    temperature=1, top_k=0):
    if start_token is None:
        assert context is not None, 'Specify exactly one of start_token and context!'
    else:
        assert context is None, 'Specify exactly one of start_token and context!'
        context = tf.fill([batch_size, 1], start_token)
      
    # mask dict is too large
    # dict_mask = np.zeros((hparams.n_vocab, hparams.n_vocab))
   
    def step(hparams, tokens, past=None):
        lm_output = Transformer.model(hparams=hparams, X=tokens, past=past, reuse=tf.AUTO_REUSE)

        logits = lm_output['logits'][:, :, :hparams.n_vocab] # keep output dimension the same
        # tf.gather collect the corresponding logits 
        logits = tf.gather(logits, reduced_dict_index)
#         logits = logits * dict_mask
        presents = lm_output['present']
        presents.set_shape(Transformer.past_shape(hparams=hparams, batch_size=batch_size))
        return {
            'logits': logits,
            'presents': presents,
        }

    """
       For domain transfer, the general langauage model maintains the structure. 
       Change the final softmax to adapte to specific domain knowledge.
    """
    with tf.name_scope('sample_sequence'):
        def body(past, prev, output):
            next_outputs = step(hparams, prev, past=past)
            logits = next_outputs['logits'][:, -1, :]  / tf.to_float(temperature)
            logits = top_k_logits(logits, k=top_k)
            samples = tf.multinomial(logits, num_samples=1, output_dtype=tf.int32)
            return [
                next_outputs['presents'] if past is None else tf.concat([past, next_outputs['presents']], axis=-2),
                samples,
                tf.concat([output, samples], axis=1)
            ]

        past, prev, output = body(None, context, context)
      
        def cond(*args):
            return True

        memo_, curre_, tokens = tf.while_loop(
            cond=cond, body=body,
            maximum_iterations=length - 1,
            loop_vars=[
                past,
                prev,
                output
            ],
            shape_invariants=[
                tf.TensorShape(Transformer.past_shape(hparams=hparams, batch_size=batch_size)),
                tf.TensorShape([batch_size, None]),
                tf.TensorShape([batch_size, None]),
            ],
            back_prop=False,
        )

        # tokens: outputs, curre_: current word choice, memo_: the previous word sequence.
        return tokens, curre_, memo_  

   
# reduce_dict = create_domain_dict(encoder_path, domain_dict)

<file_sep>/Domain_transfer.py
"""
  Written by <NAME>. The code is used for domain transfer.
  From larget scale general vocabulary to domain specific vocabulary. 
"""

import tensorflow as tf
from functools import lru_cache
import json
import re

@lru_cache()
def bytes_to_unicode():
    """
       Returns list of utf-8 byte and corresponding list of unicode strings
    """
    bs = list(range(ord("!"), ord("~")+1)) + list(range(ord("¡"), ord("¬")+1)) + \
         list(range(ord("®"), ord("ÿ")+1))
    cs = bs[:]
    n  = 0
    for b in range(2**8):
        if b not in bs:
           bs.append(b)
           cs.append(2**8+n)
           n += 1
    cs = [chr(n) for n in cs]
    
    return dict(zip(bs, cs))
 
def get_pairs(word):
    """Return set of symbol pairs in a word.
       Word is represented as tuple of symbols (symbols being variable-length strings).
    """
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs

def byte_decoder():
     """
        Reversion of bytes_to_unicode
     """
     return {v:k for k, v in bytes_to_unicode().items()}
    
class Encoder:
    def __init__(self, encoder, bpe_dict=None): # 
        # bpe_ranks : reduced version from bpe_merges
        self.encoder = encoder
        self.decoder = {v:k for k, v in self.encoder.items()}
        
        self.byte_encoder = bytes_to_unicode()
        self.byte_decoder = byte_decoder()
        
        if bpe_dict is not None:
           self.bpe_pair = dict(zip(bpe_dict, range(len(bpe_dict))))
        
        self.cache = {}
        
        self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
    
    def bpe(self, token):
        if token in self.cache:
             return self.cache[token]
        word  = tuple(token)
        pairs = get_pairs(word)
        
        if not pairs:
            return token
        while True:
              bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf')))
              if bigram not in self.bpe_ranks:
                 break
              first, second = bigram
              new_word = []
              i = 0
              while i < len(word):
                   try:
                       j = word.index(first, i)
                       new_word.extend(word[i:j])
                       i = j
                   except:
                       new_word.extend(word[i:])
                       break
                   
                   if word[i] == first and i < len(word)-1 and word[i+1]==second:
                      new_word.append(first+second)
                      i += 2
                   else:
                      new_word.append(word[i])
                      i += 1
              new_word = tuple(new_word)
              word  = new_word
              if len(word) == 1:
                 break
              else:
                 pairs = get_pairs(word)
        word = ' '.join(word)
        self.cache[token] = word
        
        return word
                     
    def encode(self, text):
        if bpe_dict is not None:
           bpe_tokens = []
           for token in re.findall(self.pat, text):
               token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
               bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
           return bpe_tokens
        else: 
           nbpe_tokens = []
           if " " in text:
              tokens = []
              text_  = text.split()
              tokens.append(text_[0])
              for sub_str in text_[1:]:
                  tokens.append(" " + sub_str)
              nbpe_tokens.extend(self.encoder[bpe_token] for bpe_token in tokens)
           else:
              nbpe_tokens.append(self.encoder[text])
           return nbpe_tokens
        
            
    def decode(self, tokens, dict_index):
    """
       Input digit tokens index the word sequences. tokens: int
       decoder is the dictionary using index as key and word as the value.
    """
     #  byte_encoder = bytes_to_unicode()
       text = ''.join(self.decoder[token] for token in tokens)
     # text = ''.join([self.decoder[dict_index[token]] for token in tokens])     
       text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace")
       return text 
      
def global_domain(global_dict, domain_voc):
    """
       Translating domain vocabulary by global dict index.
    """
    

def create_encoder(model_path, domain_dict):
    
    with open(model_path + 'encoder.json', 'r') as read_file:
         complete_encoder = json.load(read_file)
    with open(model_path + 'vocab.bpe', 'r', encoding="utf-8") as read_file:
         bpe_data = read_file.read()
    bpe_dict = [tuple(str_.split()) for str_ in bpe_data.split('\n')[1:-1]]
    
    # get domain_encoder
    reduce_encoder = {}
    byte_decoder = byte_decoder()
    for sub_text in domain_dict:
       for keys, index_seq in complete_encoder.items():
           keys = bytearray([byte_decoder[c] for c in keys]).decode('utf-8', errors='replace')
           if sub_text == keys or sub_text.lower()==keys or \
              (" " + sub_text==keys) or (" " + sub_text.lower()==keys):
              # dict {k:v}
              reduced_encoder[keys] = index_seq
              
     
     # get domain bpe: excluding out-vocabulary gram
    bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split('\n')[1:-1]]
     # by searching the bpe_merges to find provided dict, reversely get reduced bpe
    
    self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
    
    return Encoder(
         encoder    = reduced_encoder,  # using the reduced encoder dictionary
         bpe_ranks = bpe_ranks, 
         )
    
  
<file_sep>/context_sampling.py
# sampling sentences based on providing contexts

import fire
import json
import os
import numpy as np
import tensorflow as tf
import Transformer, generate, Domain_transfer

def pretrain_model(
    model_name="pretrained",
    seed = 1234, 
    nsamples=1,
    batch_size=1,
    length = 10,
    temperature=1, 
    top_k = 9,
    models_dir='models',
):
    model_dir = os.path.expanduser(os.path.expandvars(models_dir))
    if batch_size is None:
        batch_size = 1
    assert nsamples % batch_size == 0
    
    


