The code will output the same result as the reptrained resnet50 during evaluation (and using the same self.fc at the end for the sake of debugging).
Code snippet:
class PABN(nn.Module):
    def __init__(self):
        super(PABN, self).__init__()
        num_classes = 576  
      
        self.backbone = list(models.resnet50(pretrained=True).children())[:-3]  # removes conv 5, avgpool, fully con
        self.backbone = nn.Sequential(*self.backbone) 
    
        self.branches = nn.ModuleList()
        [self.branches.append(self._get_shared_branch_backbone()) for _ in range(1)]
    
        self.fcs = nn.ModuleList()
        [self.fcs.append(list(models.resnet50(pretrained=True).children())[-1]) for _ in range(1)]
    
    def _get_shared_branch_backbone(self):
        net = list(models.resnet50(pretrained=True).children())[7:-1]  # Gets conv5 + AAP2
        return nn.Sequential(*net)
    
    def forward(self, images, extra_label):
        batch_size=images.size()[0]
        # forward shared backbone:
        x_backbone_out = self.backbone(images)
    
        # forward of separate branch based on extra_label label
        branch_outs = []
        for inp, e_l in zip(x_backbone_out, extra_label):
            e_l=e_l.item() # to get an integer for indexing
            inp_exp=torch.unsqueeze(inp, 0)  # add the batch dim
            x = self.branches[e_l](inp_exp)
            x = x.squeeze(3).squeeze(2)
            branch_outs.append(x)
        branch_outs = torch.cat(branch_outs, 0)
    
        #if not self.training:  # so IF TESTING-eval, return the feature vector:
        #    return branch_outs
    
        branch_outs_fc = []
        for inp, e_l in  zip(branch_outs, extra_label):  # use the fully connected layers
            e_l=e_l.item()
            inp_exp = torch.unsqueeze(inp, 0)  # add the batch dim
            branch_outs_fc.append(self.fcs[e_l](inp_exp))
        branch_outs_fc = torch.cat(branch_outs_fc, 0)
        return branch_outs_fc


model = PABN().eval()
x = torch.randn(2, 3, 224, 224)
output = model(x, torch.tensor([0, 0]))

model_reference = models.resnet50(pretrained=True).eval()
output_reference = model_reference(x)

print((output - output_reference).abs().max())
> tensor(1.1921e-06., grad_fn=<MaxBackward1>)

However, during training, you are creating loops over the batch dimension, thus feeding one sample to the branches at a time.
Since the branches use batchnorm layers, feeding a single sample will most likely result in bad estimates, which should explain your divergence.