You have to use any linear transformation. Resnet (which has residual connections) has linear transformations to handle that.

  
      github.com
  
  
    pytorch/vision/blob/038105ff3a18f42903903d2a7714f2e083932a6a/torchvision/models/resnet.py#L133-L139 15

def _make_layer(self, block, planes, blocks, stride=1):
    downsample = None
    if stride != 1 or self.inplanes != planes * block.expansion:
        downsample = nn.Sequential(
            conv1x1(self.inplanes, planes * block.expansion, stride),
            nn.BatchNorm2d(planes * block.expansion),
        )