When i run your code in each device i m getting:
cpu --> 0.00017951
cuda:0 --> 0.000260612
this looks confusing. I was expecting results similar to yours.
My hyperparameters:
batch_size = 100 (batching graphs in DGL) --> dgl.batch()
input_feat_dim = 156
hidden_dim1= 32
hidden_dim2= 16
This is the complete model by using Graph Convolutions if it helps further. Sorry sending you the codes partially. And thanks alot for taking time for this issue.
class GraphConvolution(nn.Module):
def __init__(self, in_features, out_features, dropout=0., act=F.relu):
    super(GraphConvolution, self).__init__()
    self.in_features = in_features
    self.out_features = out_features
    self.dropout = dropout
    self.act = act
    self.weight = glorot_init(in_features, out_features)
    self.reset_parameters()

def reset_parameters(self):
    torch.nn.init.xavier_uniform_(self.weight)

def forward(self, input, adj):
    input = F.dropout(input, self.dropout, self.training)
    support = torch.mm(input, self.weight)
    output = torch.spmm(adj, support)
    output = self.act(output)
    return output

def __repr__(self):
    return self.__class__.__name__ + ' (' \
           + str(self.in_features) + ' -> ' \
           + str(self.out_features) + ')'

class GVAE(nn.Module):
def __init__(self, input_feat_dim, hidden_dim1, hidden_dim2, emb_matrix, dropout):
    super(GVAE, self).__init__()

    self.gc1 = GraphConvolution(input_feat_dim, hidden_dim1, dropout, act=F.relu)
    # Import node features
    self.gc1.weight.data = torch.Tensor(emb_weights)
    self.gc1.weight.requires_grad = True
    self.gc2 = GraphConvolution(hidden_dim1, hidden_dim2, dropout, act=lambda x: x)
    self.gc3 = GraphConvolution(hidden_dim1, hidden_dim2, dropout, act=lambda x: x)
    self.decoder = InnerProductDecoder(dropout, act=lambda x: x)

def encode(self, x, adj):
    # x will be our embedding vectors
    hidden1 = self.gc1(x, adj)
    return self.gc2(hidden1, adj), self.gc3(hidden1, adj)

def reparameterize(self, mu, logvar):
    if self.training:
        std = torch.exp(logvar)
        eps = torch.randn_like(std)
        return eps.mul(std).add_(mu)
    else:
        return mu

def forward(self, x, adj):
    mu, logvar = self.encode(x, adj)
    z = self.reparameterize(mu, logvar)
    decoded = self.decoder(z)

    return decoded, mu, logvar

class InnerProductDecoder(nn.Module):
“”“Decoder for using inner product for prediction.”""
def __init__(self, dropout, act=torch.sigmoid):
    super(InnerProductDecoder, self).__init__()
    self.dropout = dropout
    self.act = act

def forward(self, z):
    z = F.dropout(z, self.dropout, training=self.training)
    adj = self.act(torch.mm(z, z.t()))
    return adj