I have made my own dataloader. I am trying to train a network to solve jigsaw puzzle. The training loss remains constant and the gradients are very small. What could be the possible reason?
Also volatile gpu util is 0 most of the time.
This is how i am doing my training:
steps = args.iter_start
train_data = DataLoader(video_path=train_path, batch_size=args.batch, frame_size=args.img_shape,
classes=args.classes, jigsaw_frames=args.jigsaw_frame)
total_combinations = train_data.get_total_combinations()
batches_per_epoch = total_combinations // args.batch
for epoch in range(args.epochs):
    train_data.frame_counter = 0
    train_data.video_counter = 0

    
    lr = adjust_learning_rate(optimizer, epoch, init_lr=args.lr, step=20, decay=0.1)


    for batch_count in range(batches_per_epoch):

        images, labels, original = train_data.get_next_batch(batch_size=args.batch)
        images, labels = Variable(images).cuda(), Variable(labels).cuda()

        # Forward + Backward + Optimize
        outputs = net(images)

        prec1, prec5 = compute_accuracy(outputs.cuda().data, labels.cuda().data, topk=(1, 5))


        acc = prec1.item()
        loss = criterion(outputs, labels)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if steps % 20 == 0:
            print(
                ('[Epoch: %2d/%2d], [Steps: %5d], [LR %.5f], [Loss: % 1.3f], [Accuracy % 2.2f%%]' % (
                    epoch + 1, args.epochs, steps,
                    lr, loss, acc)))

        
        steps += 1

Network is as follow:
class Network(nn.Module):
def __init__(self, classes=1000):
    super(Network, self).__init__()

    self.conv = nn.Sequential()
    self.conv.add_module('conv1_s1', nn.Conv2d(3, 96, kernel_size=11, stride=4, padding=0))
    self.conv.add_module('relu1_s1', nn.ReLU(inplace=True))
    self.conv.add_module('pool1_s1', nn.MaxPool2d(kernel_size=3, stride=2))
    self.conv.add_module('lrn1_s1', LRN(local_size=5, alpha=0.0001, beta=0.75))

    self.conv.add_module('conv2_s1', nn.Conv2d(96, 256, kernel_size=5, padding=2, groups=2))
    self.conv.add_module('relu2_s1', nn.ReLU(inplace=True))
    self.conv.add_module('pool2_s1', nn.MaxPool2d(kernel_size=3, stride=2))
    self.conv.add_module('lrn2_s1', LRN(local_size=5, alpha=0.0001, beta=0.75))

    self.conv.add_module('conv3_s1', nn.Conv2d(256, 384, kernel_size=3, padding=1))
    self.conv.add_module('relu3_s1', nn.ReLU(inplace=True))

    self.conv.add_module('conv4_s1', nn.Conv2d(384, 384, kernel_size=3, padding=1, groups=2))
    self.conv.add_module('relu4_s1', nn.ReLU(inplace=True))

    self.conv.add_module('conv5_s1', nn.Conv2d(384, 256, kernel_size=3, padding=1, groups=2))
    self.conv.add_module('relu5_s1', nn.ReLU(inplace=True))
    self.conv.add_module('pool5_s1', nn.MaxPool2d(kernel_size=3, stride=2))

    self.fc6 = nn.Sequential()
    self.fc6.add_module('fc6_s1', nn.Linear(256 * 6 * 6, 4096))
    self.fc6.add_module('relu6_s1', nn.ReLU(inplace=True))
    self.fc6.add_module('drop6_s1', nn.Dropout(p=0.5))

    self.fc7 = nn.Sequential()
    self.fc7.add_module('fc7_s1', nn.Linear(4096, 4096))
    self.fc7.add_module('relu7_s1', nn.ReLU(inplace=True))
    self.fc7.add_module('drop7_s1', nn.Dropout(p=0.5))

    self.fc8 = nn.Sequential()
    self.fc8.add_module('fc8', nn.Linear(9 * 4096, 4096))
    self.fc8.add_module('relu8', nn.ReLU(inplace=True))
    self.fc8.add_module('drop8', nn.Dropout(p=0.5))

    self.classifier = nn.Sequential()
    self.classifier.add_module('fc9', nn.Linear(4096, classes))
    self.apply(weights_init)


def forward(self, x):

    B, T, C, H, W = x.size()
    x = x.transpose(0, 1)

    x_list = []
    for i in range(9):
        z = self.conv(x[i])

        z = self.fc6(z.view(B, -1))
        z = self.fc7(z)
        z = z.view([B, 1, -1])
        x_list.append(z)

    x = cat(x_list, 1)
    x = self.fc8(x.view(B, -1))
    x = self.classifier(x)

    return x

def weights_init(model):
if type(model) in [nn.Conv2d, nn.Linear]:
nn.init.xavier_normal(model.weight.data)
nn.init.constant(model.bias.data, 0.1)