Hi @ptrblck, my GPU GeForce GTX 1060 at Windows 10 is also at 0-1% utilization, but memory is constantly high. I have tried a lot of fixes from different forums but nothing.
Here’s my code.
Any ideas?

crop_size = (100, 100)

transforms = tv.transforms.Compose([
                                    #tv.transforms.transforms.CenterCrop(crop_size),
                                    #tv.transforms.transforms.RandomAffine(5),
                                    # tv.transforms.transforms.RandomHorizontalFlip(),
                                    # tv.transforms.transforms.RandomVerticalFlip(),
                                    tv.transforms.transforms.Resize(crop_size),
                                    #tv.transforms.transforms.RandomRotation(20),
                                    #tv.transforms.transforms.RandomCrop(crop_size),
                                    tv.transforms.transforms.ToTensor(),
                                   ])

batch_size = 128

train_folders = tv.datasets.ImageFolder('./train', transform=transforms)
train_loader = pt.utils.data.DataLoader(train_folders, 
                                        batch_size=batch_size, 
                                        shuffle=True, 
                                        #num_workers=2, 
                                        pin_memory=True)
print('Batches info: {}'.format(next(iter(train_loader))[0].shape))
print('Expected # of batches for epoch {}'.format(int(len(train_folders)/batch_size)))

valid_folders = tv.datasets.ImageFolder('./valid', transform=transforms)
valid_loader = pt.utils.data.DataLoader(valid_folders, 
                                        batch_size=batch_size, 
                                        shuffle=True, 
                                        #num_workers=2, 
                                        pin_memory=True)

test_folders = tv.datasets.ImageFolder('./test', transform=transforms)
test_loader = pt.utils.data.DataLoader(test_folders, 
                                       batch_size=batch_size, 
                                       shuffle=True,
                                       #num_workers=2, 
                                       pin_memory=True)

input_shape = next(iter(train_loader))[0].shape # for connecting convolutional outputs to linear

# CONV model

pt.cuda.is_available(), pt.cuda.current_device(), pt.cuda.get_device_name()
device = 'cuda' if pt.cuda.is_available() else 'cpu'
#device = 'cpu'

class Convolve2D(nn.Module):
    def __init__(self, input_shape, in_channels, out_channels, kernel_size, maxp_kernel_size, output_dim):
        super().__init__()
        
        self.conv = nn.Sequential(nn.Conv2d(in_channels, out_channels, kernel_size),
                                  nn.Dropout(p=0.26),
                                  nn.SELU(),
                                  nn.AvgPool2d(kernel_size=maxp_kernel_size, stride=1),
                                  
                                  nn.BatchNorm2d(num_features=out_channels),
        
                                  nn.Conv2d(out_channels, out_channels, kernel_size),
                                  nn.Dropout(p=0.26),
                                  nn.SELU(),
                                  nn.AvgPool2d(kernel_size=maxp_kernel_size, stride=1),
                                  
                                  nn.BatchNorm2d(num_features=out_channels),
                                  
                                  nn.Conv2d(out_channels, out_channels, kernel_size),
                                  nn.Dropout(p=0.26),
                                  nn.SELU(),
                                  nn.AvgPool2d(kernel_size=maxp_kernel_size, stride=1),                                  
                                  
                                  nn.BatchNorm2d(num_features=out_channels),
                                  
                                  nn.Conv2d(out_channels, out_channels, kernel_size),
                                  nn.Dropout(p=0.26),
                                  nn.SELU(),
                                  nn.AvgPool2d(kernel_size=maxp_kernel_size, stride=1),                                  
                                  
                                  nn.BatchNorm2d(num_features=out_channels),

                                  nn.Conv2d(out_channels, out_channels, kernel_size),
                                  nn.Dropout(p=0.26),
                                  nn.SELU(),
                                  nn.AvgPool2d(kernel_size=maxp_kernel_size, stride=1),  
                                  
                                  nn.BatchNorm2d(num_features=out_channels),

                                  nn.Conv2d(out_channels, out_channels, kernel_size),
                                  nn.Dropout(p=0.26),
                                  nn.SELU(),
                                  nn.AvgPool2d(kernel_size=maxp_kernel_size),
                                  
                                  nn.BatchNorm2d(num_features=out_channels),

                                  nn.Conv2d(out_channels, out_channels, kernel_size),
                                  nn.Dropout(p=0.26),
                                  nn.SELU(),
                                  nn.AvgPool2d(kernel_size=maxp_kernel_size),
                                                                    
                                  nn.Flatten())
        
        self.fc = nn.Sequential(nn.Linear(in_features=self.conv(pt.zeros(*input_shape)).shape[1], 
                                          out_features=output_dim),
                                nn.LogSoftmax(1))
        
    def forward(self, x):
        #print('x.shape', x.shape)
        conv = self.conv(x)
        out = self.fc(conv)
        #print('out.shape', out.shape)
        return out
    
def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

input_shape = next(iter(train_loader))[0].shape # for connecting convolutional outputs to linear
conv2d_hp = dict(input_shape=input_shape, 
                   in_channels=3, 
                   out_channels=5, 
                   kernel_size=3,
                   maxp_kernel_size=3,
                   output_dim=12)
conv2d = Convolve2D(**conv2d_hp).to(device)
criterion = nn.NLLLoss(reduction='none') # with loss.mean => the behind the scenes of reduction='mean'
LR = 0.005
optimizer = pt.optim.Adam(conv2d.parameters(), lr=LR)
# print(conv2d)
print('Number of parameters: {:,}'.format(count_parameters(conv2d)))


for epoch in range(3):
    conv2d.train()
    running_loss = []
    #samples = 0
    for i, (image, label) in enumerate(train_loader):
        image = image.to(device, non_blocking=True)
        label = label.to(device, non_blocking=True)
        
        out = conv2d(image)
        loss = criterion(out, label)
        
        conv2d.zero_grad()
        loss.mean().backward()
        optimizer.step()
        
        running_loss.append(loss.mean().item())
        #samples += len(image)
        print('Epoch {}, Step: {}, Loss: {:.4f}'.format(epoch+1, i+1, loss.mean().item()))
    print('Epoch {}, Loss: {:.4f}'.format(epoch+1, np.mean(running_loss)))
    with pt.no_grad():
        conv2d.eval()
        correct = 0
        total = 0
        for image, label in test_loader:
            image = image.to(device)
            label = label.to(device)
            out = conv2d(image)
            #top_p, top_class = out.topk(1, dim=1)
            #equals = top_class == label.view(*top_class.shape)
            #accuracy = pt.mean(equals.type(pt.float))
            predicted = pt.max(out.data, 1)
            total += label.size(0)
            correct += (predicted[1] == label).sum().item()
        print('Test Accuracy on {} images: {:.2f} %'.format(total, 100 * correct / total))
print('Saving model...')
pt.save(conv2d.state_dict(), 'conv2d_checkpoint.pth')
print('Model saved!')```