Thank your for the reply. Your understanding is correct. Do you mean that it is because of the pseudo randomness?
But I add code below before the fine tuning to confirm the randomness will not affect the result.
# Set the seed value all over the place to make this reproducible.
def setup_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)  # if you are using multi-GPU.
    np.random.seed(seed)  # Numpy module.
    random.seed(seed)  # Python random module.
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True  

setup_seed(42)