CLIP output doesn't match the official weight
### System Info

I am using transformer 4.20.1. 

I found that the official CLIP model outputs don't match the hugging face one. 


### Who can help?

@NielsRogge @stevhliu 

### Information

- [X] The official example scripts
- [ ] My own modified scripts

### Tasks

- [X] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [ ] My own task or dataset (give details below)

### Reproduction


To install clip, run `pip install git+https://github.com/openai/CLIP.git`

```python
import torch
import clip
from PIL import Image
import requests


device = "cpu"
model, preprocess = clip.load("ViT-B/32", device=device)

url = "http://images.cocodataset.org/val2017/000000039769.jpg"
image = Image.open(requests.get(url, stream=True).raw)
image = preprocess(image).unsqueeze(0).to(device)
text = clip.tokenize(["a photo of a cat", "a photo of a dog"]).to(device)

with torch.no_grad():
    clip_image_features = model.encode_image(image)
    clip_text_features = model.encode_text(text)
    
    logits_per_image, logits_per_text = model(image, text)
    probs = logits_per_image.softmax(dim=-1).cpu().numpy()

print("CLIP Label probs:", probs)  # prints: [[0.9927937  0.00421068 0.00299572]]

from transformers import CLIPProcessor, CLIPModel
model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")

image = Image.open(requests.get(url, stream=True).raw)
inputs = processor(text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True)
outputs = model(**inputs)
logits_per_image = outputs.logits_per_image 
probs = logits_per_image.softmax(dim=1) 

print("HF Label probs:", probs)  # prints: [[0.9927937  0.00421068 0.00299572]]

print(clip_text_features.shape) 
print(outputs.text_model_output.pooler_output.shape)
print((clip_text_features - outputs.text_model_output.pooler_output).abs().max())
assert torch.allclose(clip_text_features, outputs.text_model_output.pooler_output, atol=1e-5)

```


### Expected behavior

The feature and probability should be the same. 
Hi @xvjiarui, actually the output between original and hf is the same, but you compare with the wrong tensor. Firstly, we can compare the code between the original and hf (here we focus on the text features part).

```
# Original CLIP
def encode_image(self, image):
    return self.visual(image.type(self.dtype))

def encode_text(self, text):
    x = self.token_embedding(text).type(self.dtype)  # [batch_size, n_ctx, d_model]

    x = x + self.positional_embedding.type(self.dtype)
    x = x.permute(1, 0, 2)  # NLD -> LND
    x = self.transformer(x)
    x = x.permute(1, 0, 2)  # LND -> NLD
    x = self.ln_final(x).type(self.dtype)

    # x.shape = [batch_size, n_ctx, transformer.width]
    # take features from the eot embedding (eot_token is the highest number in each sequence)
    x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection

    return x

def forward(self, image, text):
    image_features = self.encode_image(image)
    text_features = self.encode_text(text)

    # normalized features
    image_features = image_features / image_features.norm(dim=1, keepdim=True)
    text_features = text_features / text_features.norm(dim=1, keepdim=True)

    # cosine similarity as logits
    logit_scale = self.logit_scale.exp()
    logits_per_image = logit_scale * image_features @ text_features.t()
    logits_per_text = logits_per_image.t()

    # shape = [global_batch_size, global_batch_size]
    return logits_per_image, logits_per_text
```
```
# HF CLIP
def forward(
    self,
    input_ids: Optional[torch.LongTensor] = None,
    pixel_values: Optional[torch.FloatTensor] = None,
    attention_mask: Optional[torch.Tensor] = None,
    position_ids: Optional[torch.LongTensor] = None,
    return_loss: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple, CLIPOutput]:
    # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    vision_outputs = self.vision_model(
        pixel_values=pixel_values,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    text_outputs = self.text_model(
        input_ids=input_ids,
        attention_mask=attention_mask,
        position_ids=position_ids,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    image_embeds = vision_outputs[1]
    image_embeds = self.visual_projection(image_embeds)

    text_embeds = text_outputs[1]
    text_embeds = self.text_projection(text_embeds)

    # normalized features
    image_embeds = image_embeds / image_embeds.norm(p=2, dim=-1, keepdim=True)
    text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True)

    # cosine similarity as logits
    logit_scale = self.logit_scale.exp()
    logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * logit_scale
    logits_per_image = logits_per_text.T

    loss = None
    if return_loss:
        loss = clip_loss(logits_per_text)

    if not return_dict:
        output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)
        return ((loss,) + output) if loss is not None else output

    return CLIPOutput(
        loss=loss,
        logits_per_image=logits_per_image,
        logits_per_text=logits_per_text,
        text_embeds=text_embeds,
        image_embeds=image_embeds,
        text_model_output=text_outputs,
        vision_model_output=vision_outputs,
    )
```
you may found out that the output of original clip `encode_text` method should equal to `text_embeds = self.text_projection(text_embeds)` in the hf repo. You can modify your script like this one to check it.
```
import torch
import clip
from PIL import Image
import requests
import numpy as np

device = "cpu"
model, preprocess = clip.load("ViT-B/32", device=device)

url = "http://images.cocodataset.org/val2017/000000039769.jpg"
image = Image.open(requests.get(url, stream=True).raw)
image = preprocess(image).unsqueeze(0).to(device)
text = clip.tokenize(["a photo of a cat", "a photo of a dog"]).to(device)

with torch.no_grad():
    clip_image_features = model.encode_image(image)
    clip_text_features = model.encode_text(text)

    logits_per_image, logits_per_text = model(image, text)
    org_probs = logits_per_image.softmax(dim=-1).cpu().numpy()

print("CLIP Label probs:", org_probs)  # prints: [[0.9927937  0.00421068 0.00299572]]

from transformers import CLIPProcessor, CLIPModel

model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")

image = Image.open(requests.get(url, stream=True).raw)
inputs = processor(
    text=["a photo of a cat", "a photo of a dog"],
    images=image,
    return_tensors="pt",
    padding=True,
)
outputs = model(**inputs)
logits_per_image = outputs.logits_per_image
hf_probs = logits_per_image.softmax(dim=1).detach().cpu().numpy()

print("HF Label probs:", hf_probs)  # prints: [[0.9927937  0.00421068 0.00299572]]

print(np.allclose(org_probs, hf_probs))
print(
    torch.allclose(
        clip_text_features,
        model.text_projection(outputs.text_model_output.pooler_output).detach(),
        atol=1e-5,
    )
)

```<|||||>Hi @aRyBernAlTEglOTRO thanks for your reply. 

I see. That makes sense.