Pytorch, Optimization, Deep Learning, Artificial Intelligence, Tensorboard.
MyPatchDropout(PatchDropout): def forward(self, x): prefix_tokens = x[:, :self.num_prefix_tokens] x = x[:, self.num_prefix_tokens:] B = x.shape[0] L = x.shape[1] num_keep = max(1, int(L * (1. - self.prob))) keep_indices = torch.argsort(torch.randn(B, L, device=x.device), dim=-1)[:, :num_keep] # The