Thank you for your reply. it seems that the conditioning itself will not help much with multi-modality but it is rather about the specific structure of the implemented VAE. So I used MNIST dataset instead of the previous dataset. But when all the dimensions of decoder and encoder part were K I got this error:
<ipython-input-96-ac87136b4d41> in train(epoch)
     13         data, cond = data.to(device), one_hot(cond, cond_dim).to(device)
     14         optimizer.zero_grad()
---> 15         recon_batch, mu, logvar = cvae(data, cond)
     16      #   print("mu", mu)
     17      #   print("logvar", logvar)

~\anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
   1049         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1050                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1051             return forward_call(*input, **kwargs)
   1052         # Do not call functions when jit is used
   1053         full_backward_hooks, non_full_backward_hooks = [], []

<ipython-input-88-258e170a951c> in forward(self, x, c)
     42 
     43     def forward(self, x, c):
---> 44         mu, logvar = self.encode(x.view(-1, 2), c)
     45         z = self.reparameterize(mu, logvar)
     46         return self.decode(z, c), mu, logvar

<ipython-input-88-258e170a951c> in encode(self, x, c)
     14 
     15     def encode(self, x, c):
---> 16         concat_input = torch.cat([x, c], 1)
     17         h1 = F.selu(self.fc1(F.selu(self.fc0(concat_input))))
     18         return self.fc21(h1), self.fc22(h1)

RuntimeError: torch.cat(): Sizes of tensors must match except in dimension 1. Got 39200 and 100 in dimension 0 (The offending index is 1)

and it just works when the dimensions were  similar to the following code:
def __init__(self, c_dim):
        super(CVAE, self).__init__()
        # encoder part
        self.fc0 = nn.Linear(784 + c_dim, 512)
        self.fc1 = nn.Linear(512, 256)
        self.fc21 = nn.Linear(256, 2)
        self.fc22 = nn.Linear(256, 2)
        # decoder part
        self.fc3 = nn.Linear(2 + c_dim, 256)
        self.fc4 = nn.Linear(256, 512)
        self.fc5 = nn.Linear(512, 784)