You can either force the weights of the non-existent connexions to equal zero, or do (I think it’s what you are trying) two different fully connected layers. the second solution seems more elegant.
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(2,1)
        self.fc2 = nn.Linear(2,1)

    def forward(self, x):
        splitted = torch.split(x,1,1)
        x1 = torch.cat(splitted[:2],1)
        x2 = torch.cat(splitted[1:],1)
        x1 = F.relu(self.fc1(x1))
        x2 = F.relu(self.fc1(x2))
        return torch.cat([x1,x2],1)