Deep Learning, Sora, Computer Vision, Transformers, Attention.
torch.cat([torch.sin(pe_out), torch.cos(pe_out)], dim=-1) pe_out.shape #torch.Size([432, 768]) (out[0] + pe_out.unsqueeze(0)).shape #torch.Size([1, 432, 768]) class PositionalEncoding(nn.Module): def __init__(self, embed_size, patch_size): super().__init__() self.embed_size = embed_size