Hello all,
I am training an auto encoder on tabular data that has 272 features that are mostly sparse one-hot encodings. I am experiencing incredibly slow training. In fact it appears to possibly be training faster on CPU than on GPU. The time to train one epoch with a batch size of 5 is 5 minutes.
I’m hoping someone can provide some insight into why this is happening. Thank you in advance.
My set-up is a very basic fully connected network:
class FCNetwork(nn.Module):
    """Fully Connected Network Class"""
    def __init__(self, n_input, layers, n_output, act=('relu', nn.ReLU())):
        """
        :param n_input: Integer. Size of input vector.
        :param layers: Tuple. containing the desired hidden layer architecture.
        :param n_output: Size of output vector.
        :param act: Tuple ('name', act_func). The first element should be a string
            describing the activation function. The second element is the activation
            function itself. Default is ``('ReLU', nn.ReLU())``.
        """
        super().__init__()
        self.input = nn.Linear(n_input, layers[0])
        self.hidden = self.init_hidden(layers, activation=act)
        self.output = nn.Linear(layers[-1], n_output)
        
    def init_hidden(self, layers, activation, dropout=0.0):

        n_layers = len(layers)
        modules = OrderedDict()
        a_name = activation[0]
            
        modules[f'{a_name}_in'] = activation[1]
        
        for i in range(n_layers - 1):
            modules[f'fc{i}'] = nn.Linear(layers[i], layers[i + 1])
            modules[f'{a_name}{i}'] = activation[1]
            modules[f'drop{i}'] = nn.Dropout(p=dropout)
            
        modules[f'{a_name}_out'] = activation[1]
        
        return nn.Sequential(modules)
            
    def forward(self, x):
        x = x.float()
        x = self.input(x)
        x = self.hidden(x)
        return self.output(x)
    

And this is my training method:
def train_encoder(model, trainload, epochs, criterion=nn.MSELoss(), optimizer=optim.Adam, lr=1e-4, testload=None):
    """
    Train auto-encoder reconstruction for given number of epochs.

    :param trainload: a DataLoader object containing training variables and targets used for training.
    :param testload: Optional. a DataLoader containing the validation set. If included, both training and
        validation loss will be tracked and can be plotted using model.plot_loss().
    :param epochs: Number of times the network will view the entire data set
    :param optimizer: Learning method. Default optim.Adam
    :param lr: Learning Rate. Default 0.003
    :param criterion: Loss function
    :return:
    """
    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    print(f'Training model on {device}')
    model = model.to(device)
    opt = optimizer(model.parameters(), lr)

    train_loss = []
    valid_loss = []

    for e in tqdm(range(epochs)):
        running_tl = 0
        running_vl = 0
        for x in trainload:
            x = x.to(device).float()
            opt.zero_grad()
            loss = criterion(model(x), x)
            loss.backward()
            opt.step()
            running_tl += loss.item()

        if testload is not None:
            model.eval()
            with torch.no_grad():
                for x in testload:
                    x = x.to(device).float()
                    loss = criterion(model(x), x)
                    running_vl += loss.item()
                valid_loss.append(running_vl / len(testload))
            model.train()
            
        train_loss.append(running_tl / len(trainload))
    
    return train_loss, valid_loss