is there a chance they are cause by not zeroing the grad parameters
The script that I used for training is:
def paramsforepoch(epoch):
    p = dict()
    regimes = [[1, 18, 1e-3, 5e-4],
               [19, 29, 5e-3, 5e-4],
               [30, 43, 1e-3, 0],
               [44, 52, 5e-4, 0],
               [53, 1e8, 1e-4, 0]]
    for i, row in enumerate(regimes):
        if epoch >= row[0] and epoch <= row[1]:
            p['learning_rate'] = row[2]
            p['weight_decay'] = row[3]
    return p

avg_loss = list()
fig1, ax1 = plt.subplots()
fig2, ax2 = plt.subplots()
# train the model
# TODO: Compute training accuracy and test accuracy
# TODO: train it on some data and see if it overfits.
# TODO: train the data on final model

# create a temporary optimizer
optimizer = optim.SGD(net.parameters(), lr=args.learning_rate, momentum=args.momentum, weight_decay=0.0005)

def adjustlrwd(p):
    for param_group in optimizer.state_dict()['param_groups']:
        param_group['lr'] = p['learning_rate']
        param_group['weight_decay'] = p['weight_decay']

# train the network
def train(epoch):

    # set the optimizer for this epoch
    if epoch > 0 or epoch > 18 or epoch > 29 or epoch > 43 or epoch > 52:
        p = paramsforepoch(epoch)
        print("Configuring optimizer with lr={:.3f} and weight_decay={:.3f}".format(p['learning_rate'], p['weight_decay']))
        adjustlrwd(p)
    ###########################################################################

    global avg_loss
    correct = 0
    net.train()
    for b_idx, (data, targets) in enumerate(train_loader):
        # trying to overfit a small data
        if b_idx == 100:
            break

        if args.cuda:
            data.cuda(), targets.cuda()
        # convert the data and targets into Variable and cuda form
        data, targets = Variable(data), Variable(targets)

        # train the network
        optimizer.zero_grad()
        scores = net.forward(data)
        loss = F.nll_loss(scores, targets)

        # compute the accuracy
        pred = scores.data.max(1)[1] # get the index of the max log-probability
        correct += pred.eq(targets.data).cpu().sum()

        avg_loss.append(loss.data[0])
        loss.backward()
        optimizer.step()

        if b_idx % args.log_schedule == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, (b_idx+1) * len(data), len(train_loader.dataset),
                100. * (b_idx+1) / 100, loss.data[0]))
            # also plot the loss, it should go down exponentially at some point
            ax1.plot(avg_loss)
            fig1.savefig("Squeezenet_loss.jpg")

    # now that the epoch is completed plot the accuracy
    accuracy = correct / 6400.0
    print("training accuracy ({:.2f}%)".format(100*accuracy))
    ax2.plot(100*accuracy)
    fig2.savefig("Training-test-acc.jpg")