Hi, I’m applying pytorch to a character-based language model. I have to redefine the loss function according to the customized model. The consequence is the optimization becomes extremely slow, especially in the multiple embedded for loops here. I wonder if there’s any way to convert the NLL() and coemitNumerators() here to the built-in NLLloss function in pytorch? I look forward to hearing any thoughts, and love to share the full code if anyone is interested!
def coemitNumerators(word, ix_inventory, automata):
    states = [0] * len(automata)

    numerators = {}
    
    for pos, someSeg in enumerate(word):
        for seg in ix_inventory:
            numerators[pos, seg] = torch.tensor(1.) 
    # In each pos of a word, set the transition prob of a phoneme to 1.

    for somePos, someSeg in enumerate(word):
        for i, M in enumerate(automata):
            for seg in ix_inventory:
                numerators[somePos, seg] = numerators[somePos, seg] * M.tProb[(states[i], seg)]
            states[i] = M.D[(states[i], someSeg.item())]
    return numerators

def NLL(wlist, automata, inventory):
    nll = 0
    for w in wlist:
        numerators = coemitNumerators(w, inventory, automata)
        for realpos, realseg in enumerate(w):
            nll = nll - torch.log(numerators[realpos, realseg.item()] / sum(numerators[realpos, seg] for seg in inventory))
    return nll

def train_lm(dataset, dev, params, automata, ix_inventory, phone2ix, ix2phone, prev_perplexity = 1e10):

    variables = []
    for automaton in automata:
        variables += automaton.tProb.values()
    optimizer = torch.optim.Adam(variables, params['learning_rate'])
    num_examples, seq_len = dataset.size()
    batches = [
        (start, start + params['batch_size'])
        for start in range(0, num_examples, params['batch_size'])
    ]

    for epoch in range(params['epochs']):
        ep_loss = 0.
        start_time = time.time()
        random.shuffle(batches)
        for b_idx, (start, end) in enumerate(batches):
            loss = NLL(dataset[start:end], automata, ix_inventory)
            optimizer.zero_grad()
            
            loss.backward()
            optimizer.step()
            with torch.no_grad():
                for v in variables:
                    v.clamp_(0, 1)
            ep_loss += loss.detach()