Thanks, here it is
def get_valid_accuracy(valid_X, valid_y, model, batch_size=32):
    """
    Computes validation accuracy given a validation set.
    """
    model.eval()
    num_batches = valid_X.size(0) // batch_size
    all_preds = []
    for k in range(num_batches + 1):
        start_idx, end_idx = k * batch_size, (k + 1) * batch_size
        batch_X = valid_X[start_idx:end_idx, :]

        probs = model(batch_X).flatten()
        preds = torch.where(probs >= 0.5, 1, 0)
        all_preds += preds.tolist()

    all_preds = np.array(all_preds)
    valid_y = np.array(valid_y.cpu())
    acc = np.mean(all_preds == valid_y)
    return acc

I just tried adding a call to model.train() every time after get_valid_accuracy runs in the loop but still getting the same issue.