Hey @ptrblck bro, I came up with an idea where we can initialize the weights and biases in different groups with the same value and update them during training with the same value as well.
class Model(nn.Module):
    def __init__(self):
       # some other codes

        # initialize the weights
        np.random.seed(123)

        a = np.random.randn(self.num_filter_maps,self.embed_size,kernel_size)
        a = torch.from_numpy(a).type(torch.FloatTensor)
        self.cnn_weight = nn.Parameter(a)
        
        b = np.random.randn(64)
        b = torch.from_numpy(b).type(torch.FloatTensor)
        self.cnn_bias = nn.Parameter(b)

        #linear output
        self.fc = nn.Linear(num_filter_maps, label_space)
        xavier_uniform_(self.fc.weight)


    def forward(self,x):
        # pdb.set_trace()
        batch_size = x.shape[0]
        max_len = x.shape[1]
        with torch.no_grad():
            lengths = torch.count_nonzero(x,dim=-1).cpu()
        with torch.no_grad():
            conv = nn.Conv1d(50*self.embed_size,
                            self.num_filter_maps*50,
                            kernel_size=self.kernel_size,
                            padding=int(self.kernel_size//2))
        conv.weight.data = self.cnn_weight.repeat(50,1,1).data.clone()
        conv.bias.data = self.cnn_bias.repeat(50).data.clone()

        return something

and then during training we only update the self.cnn_weights and self.cnn_bias instead of the whole CNN layer.