Hello,
I’m working on a project where I have mulitplications Sparse x dense, where the sparse matrix is fixed (same case than the post : Autograd for sparse matmul: getting either cuda memory leak or ‘buffers have already been freed’ error )
I have used the hack from this post to be able to compute the backward.
However, I tried running a test to see how much time I would gain, and the outcome is pretty disappointing.
Inputs : Identity (5484*5484) -  which is a very sparse matrix -,  random vector (5484, 256)
looping 100 times;
Sparse : 1.2 sec
dense : 1.6sec
Is it normal ? It would mean than sparse representation is mostly useful for storage.
Here is the entire code :
import torch
import torch.nn as nn
from torch.autograd import Variable
import time
class LeftMMFixed(torch.autograd.Function):
    """
    Implementation of matrix multiplication of a Sparse Variable with a Dense Variable, returning a Dense one.
    This is added because there's no autograd for sparse yet. No gradient computed on the sparse weights.
    """

    def __init__(self):
        super(LeftMMFixed, self).__init__()
        self.sparse_weights = None

    def forward(self, sparse_weights, x):
        if self.sparse_weights is None:
            self.sparse_weights = sparse_weights
        return torch.mm(self.sparse_weights, x)

    def backward(self, grad_output):
        sparse_weights, = self.sparse_weights
        return None, torch.mm(sparse_weights.t(), grad_output)

class FixedSparseLinMod(nn.Module):
    """
    A module that takes a sparse matrix and does the left matrix multiplication."""
    def __init__(self, sparse_mat):
        super(FixedSparseLinMod, self).__init__()
        inds, vals, dims = sparse_mat
        i = torch.LongTensor(inds)
        v = torch.FloatTensor(vals)
        s = torch.Size(dims)
        self.sparse_mat = nn.Parameter(torch.sparse.FloatTensor(i, v, s).cuda(), requires_grad=False)
        self.mm = LeftMMFixed()

    def forward(self, x):
        return self.mm(self.sparse_mat, x.t()).t()

if __name__=="__main__":
    N_sensor=5484
    N=100

    indices = [  list(range(N_sensor)), list(range(N_sensor)) ]
    values = [1]*N_sensor
    size = (N_sensor, N_sensor)

    adj_dense=Variable(torch.eye(N_sensor)).cuda()
    adj_sbmm = FixedSparseLinMod([indices, values, size])

    x = Variable(torch.randn(256, N_sensor)).cuda()
    y = x.t()

    torch.cuda.synchronize()
    t1=time.time()

    for i in range(N):
        out = adj_sbmm(x)

    torch.cuda.synchronize()
    t2=time.time()

    for i in range(N):
        out = torch.matmul(adj_dense, y)

    torch.cuda.synchronize()
    t3=time.time()

    print("sparse :{}".format(t2-t1))
    print("dense : {}".format(t3-t2))