This is htop image. The Mem was increased until it is full.
image1794×356 112 KB
This is my GPU usage image. I’m using only 1 GPU because I tried increasing the number of gpu but the performance didn’t improve and memory was increasing too.
image1006×248 104 KB
The label and out’s tensor size is (64,1).
And the batch size is 64, num_workers is 32.
However, even though I delete the labels/outs and run training, the memory continues to increase.
Also, I checked the consumed memory with some code. please check this.
image1338×1006 414 KB
   def _train(self, batch_iter, num_iter, num_batches, val_gen):
        start_time = time.time()
        self._model.train()

        losses = 0
        num_corrects = 0
        num_total = 0
        #labels = []
        #outs = []

        data_time = AverageMeter()
        batch_time = AverageMeter()

        pid = os.getpid()
        prev_mem = 0
        end = time.time()
        for i, batch in enumerate(batch_iter):
            i += 1
            data_time.update(time.time() - end)

            label, out, pred, train_loss = self._forward(batch)
            if i==1:
                print(label.size(), ' ', out.size())

            #train_loss = self._get_loss(label, out)
            losses += train_loss.item()

            #compute gradient and do Noam step
            self._opt.step(train_loss)

            #measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

            num_corrects += (pred == label).sum().item()
            num_total += len(label)

            if i%1000 == 0:
                cur_mem = (int(open('/proc/%s/statm'%pid, 'r').read().split()[1])+0.0)/256
                add_mem = cur_mem - prev_mem
                prev_mem = cur_mem
                print(" train iterations : {}, added mem: {}M".format(i, add_mem))

                #labels.extend(label.squeeze(-1).data.cpu().numpy())
                #outs.extend(out.squeeze(-1).data.cpu().numpy())

                acc = num_corrects / num_total
                #auc = roc_auc_score(labels, outs)
                losses = losses / 1000
                training_time = time.time() - start_time

                print('correct: {}, total: {}'.format(num_corrects, num_total))
                #print('[Train]     time: {}, loss: {}, acc: {}, auc: {}'.format(training_time, loss, acc, auc))
                print('[Train] [{0}/{1}]\t'
                      'time {batch_time.avg:.3f}\t'
                      'data time {data_time.avg:.3f}\t'
                      'loss {loss:.4f}\t'
                      'acc {acc:.4f}\t'
                      'training time {training_time:.4f}\t'.format(
                       i, num_iter, acc=acc, batch_time=batch_time, data_time=data_time, loss=losses, training_time=training_time
                       ))
                losses = 0

            if i % 100000 == 0:
                self._test('Validation', val_gen, i)
                print('Current best weight: {}.pt, best acc: {}'.format(self.max_step, self.max_acc))

                cur_weight = self._model.state_dict()
                torch.save(cur_weight, '{}{}.pt'.format(self._weight_path, i))

                mylist = []
                #mylist.append(self.max_auc)
                mylist.append(self.max_acc)
                mylist.append(i)

                with open('output.csv', 'a', newline='') as file:
                    writer = csv.writer(file)
                    writer.writerow(mylist)
                print('--------------------------------------------------------')