Both models have the same, default decoder_start_token_id, and I when I manually override this to any arbitrary token <a>, the beam output looks something like <a><s>rek likes to play football. </s>.
EDIT: The below code has decoder skip_special_tokens=True, but this flag is false for above examples.
    def test_step(self, batch, batch_id):
        """Generate a summary and write it to a file while calculating rouge"""

        # We store the file id to help with consistent results across several GPUs
        SAMsumIDs = batch["SAMsumID"]

        # generate y_preds given seed_ids, placing kwargs in dict is handy for debugging
        if self.sampling == "beam":
            generate_kwargs = {"input_ids": batch["source_ids"],
                               "early_stopping": self.generation_early_stopping,
                               "no_repeat_ngram_size": self.no_repeat_ngrams,
                               "max_length": self.tokenizer.max_len,
                               "top_k": 50,
                               "num_beams": self.beam_size}

        elif self.sampling == "nucleus":
            generate_kwargs = {"input_ids": batch["source_ids"],
                               "max_length": self.tokenizer.max_len,
                               "top_k": 50,
                               "do_sample": True,
                               "top_p": 0.95}

        y_preds = self.model.generate(**generate_kwargs)
        # save generated hypotheses to hypo dest with the same name as the original file, but with a 0 prepended
        for file_id, hypo_token_ids in zip(SAMsumIDs, y_preds):
            prefix = "source_0" if self.anti else "hypo_"
            filename = f"{self.hypo_dest}/{prefix}{file_id}.txt"
            with open(filename, "w") as handle:
                hypo = self.tokenizer.decode(hypo_token_ids, skip_special_tokens=True)
                handle.write(f"{hypo}\n")

        return {"hypo_dialogues": None}