If I understand correctly…
Try this, you should build a small example to test its correctness.
import torch

u = torch.randn(3, 1, 40, 64)
v = torch.randn(1, 3, 40, 64)
_u = u.reshape(3, 1, -1)
_v = v.reshape(1, 3, -1)
_result = torch.bmm(_u.permute(2, 0, 1), _v.permute(2, 0, 1))
result = _result.permute(1, 2, 0).reshape(3, 3, 40, 64)