That looks indeed a bit weird.
Unfortunately, I cannot reproduce it with this small code snippet:
class MyModel(nn.Module):
    def __init__(self):
        super(MyModel, self).__init__()
        self.bn = nn.BatchNorm2d(3)
        
    def _load_param(self, old_state_dict):
        state_dict = copy.deepcopy(self.state_dict()) 
        for key in state_dict:
            old_key = key.replace('bn.', '')
            state_dict[key] = old_state_dict[old_key]
        
        self.load_state_dict(state_dict)

bn_old = nn.BatchNorm2d(3)
for _ in range(100):
    out = bn_old(torch.randn(8, 3, 24, 24) * 5 + 7)

model = MyModel()
print(model.state_dict())

model._load_param(bn_old.state_dict())
print(model.state_dict())
print(bn_old.state_dict())

Could you check, what might be missing in my code and post a code snippet to reproduce this issue?