I am doing the following to finetune CLIPMode further on my own dataset.
But with no luck.
Please advise.
Model load
from transformers import CLIPProcessor, CLIPModel, CLIPConfig
config = CLIPConfig.from_pretrained("openai/clip-vit-base-patch32")
model = CLIPModel(config)
processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")

Data loader
import torch
from torch.utils.data import Dataset
from PIL import Image

class CLIPDataset(Dataset):
    def __init__(self, root_dir, df,processor, max_target_length=32):
        self.root_dir = root_dir
        self.df = df
        self.processor = processor
        self.max_target_length = max_target_length

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        file_name = self.df['image'][idx]
        image = Image.open(self.root_dir + "/" + file_name).convert("RGB")
        text = self.df['title'][idx]

        pixel_values = self.processor.feature_extractor(image, return_tensors="pt").pixel_values
        labels = self.processor.tokenizer(text, 
                                padding="max_length", 
                                max_length=self.max_target_length,
                                truncation=True).input_ids
        labels = [label if label != self.processor.tokenizer.pad_token_id else -100 for label in labels]
        return {"input_ids":torch.tensor(labels), "pixel_values":pixel_values.squeeze()}

Using default data_collator with Trainer
trainer = Seq2SeqTrainer(
    model,
    args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    data_collator=default_data_collator,
    tokenizer=processor.feature_extractor
)