Image Audio Alingment Train OpenClip With Features

.gitattributes

@@ -420,3 +420,6 @@ Vaani/Img_Audio_Alignment/checkpoints/csip/probs/probs_epoch_85_loss_4.1363.png
 Vaani/Img_Audio_Alignment/checkpoints/csip/probs/probs_epoch_8_loss_4.1522.png filter=lfs diff=lfs merge=lfs -text
 Vaani/Img_Audio_Alignment/checkpoints/csip/probs/probs_epoch_96_loss_4.1361.png filter=lfs diff=lfs merge=lfs -text
 Vaani/Img_Audio_Alignment/checkpoints/csip/probs/probs_epoch_9_loss_4.1521.png filter=lfs diff=lfs merge=lfs -text
+Vaani/Img_Audio_Alignment/available_img_audios_TRAIN3.csv filter=lfs diff=lfs merge=lfs -text
+Vaani/Img_Audio_Alignment/dog[[:space:]]and[[:space:]]cat.png filter=lfs diff=lfs merge=lfs -text
+Vaani/SDFT/SD2_1_Audio/dog[[:space:]]and[[:space:]]cat.png filter=lfs diff=lfs merge=lfs -text

Vaani/.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "editor.fontFamily": "JetBrains Mono Light",
+    "svg.preview.background": "black"
+}

Vaani/Img_Audio_Alignment/_2.1.1_Train_OpenCLIP.py

@@ -5989,7 +5989,8 @@ def load_checkpoint(
 
 # /home/IITB/ai-at-ieor/23m1521/.conda/envs/openclip2/lib/python3.11/site-packages/open_clip/factory.py
 HF_HUB_PREFIX = 'hf-hub:'
-_MODEL_CONFIG_PATHS = [Path(__file__).parent / f"model_configs/"]
+# _MODEL_CONFIG_PATHS = [Path(__file__).parent / f"model_configs/"]
+_MODEL_CONFIG_PATHS = [Path("/home/IITB/ai-at-ieor/23m1521/ashish/MTP/Vaani/Img_Audio_Alignment/model_configs")]
 _MODEL_CONFIGS = {}  # directory (model_name: config) of model architecture configs
 
 import json
@@ -6952,15 +6953,24 @@ class VaaniImageAudioDataset(torch.utils.data.Dataset):
 
     def __getitem__(self, idx):
         return {
-            'image_path': open_clip_imgaug(Image.open(self.image_paths[idx])), 
+            'image_path': self.image_paths[idx], 
             'audio_path': self.audio_paths[idx]
         }
 
 
 def collate_fn(batch):
-    image_tensor = [item['image_path'] for item in batch]
-    audio_tensor = CLAPAudioProcessor([item['audio_path'] for item in batch], resample=True)
-    return {'image_tensor': torch.stack(image_tensor), 'audio_tensor': audio_tensor}
+    image_tensor = [open_clip_imgaug(Image.open(item['image_path'])) for item in batch]
+    image_paths = [item['image_path'] for item in batch]
+    
+    audio_paths = [item['audio_path'] for item in batch]
+    audio_tensor = CLAPAudioProcessor(audio_paths, resample=True)
+    
+    return {
+        'image_tensor': torch.stack(image_tensor),
+        'image_paths': image_paths,
+        'audio_tensor': audio_tensor,
+        'audio_paths': audio_paths
+    }
 
 
 
@@ -6996,8 +7006,10 @@ test_dataloader = torch.utils.data.DataLoader(
 )
 
 batch = next(iter(train_dataloader))
-image_tensor_batch = batch['image_tensor']
-audio_tensor_batch = batch['audio_tensor']
+image_tensor_batch = batch['image_tensor'].to(device=device)
+audio_tensor_batch = batch['audio_tensor'].to(device=device)
+image_paths_batch = batch['image_paths']
+audio_paths_batch = batch['audio_paths']
 print("Image batch shape:", image_tensor_batch.shape) # [BATCH_SIZE, 3, 224, 224]
 print("Audio batch shape:", audio_tensor_batch.shape) # [BATCH_SIZE, 1, 44100]
 
@@ -7063,8 +7075,8 @@ def load_checkpoint(checkpoint_dir, model, optimizer, scheduler):
     path = os.path.join(checkpoint_dir, best_ckpt)
     checkpoint = torch.load(path)
     model.load_state_dict(checkpoint['model_state'])
-    optimizer.load_state_dict(checkpoint['optimizer_state'])
-    scheduler.load_state_dict(checkpoint['scheduler_state'])
+    # optimizer.load_state_dict(checkpoint['optimizer_state'])
+    # scheduler.load_state_dict(checkpoint['scheduler_state'])
     start_epoch = checkpoint['epoch']
     best_loss = checkpoint['best_loss']
     print(f"Resumed training from epoch {start_epoch+1} with best loss {best_loss:.4f}")
@@ -7187,7 +7199,8 @@ def train_model(model, train_loader, test_loader,
         writer.add_scalar("Similarity/Test", avg_test_sim, epoch + 1)
         writer.add_scalar("Learning Rate", current_lr, epoch + 1)
 
-        print(f"Epoch {epoch+1} | Loss: ({avg_train_loss:.4f}, {avg_test_loss:.4f}) |"
+        print(f"\n\n |"
+              f"Epoch {epoch+1} | Loss: ({avg_train_loss:.4f}, {avg_test_loss:.4f}, {best_loss:.4f}) |"
               f"LR: {current_lr:.2e} |"
               f"Similarity: ({avg_train_sim:.4f}, {avg_test_sim:.4f})"
               f"| I-Loss: ({avg_train_i_loss:.4f}, {avg_test_i_loss:.4f})"
@@ -7224,16 +7237,21 @@ def train_model(model, train_loader, test_loader,
 
 
 model_name = "csip_model_openClip_CLAP"
+epochs = 500
+
 learning_rate = 1e-5
-epochs = 100
 optimizer = torch.optim.AdamW(csip_model.parameters(), lr=learning_rate)
 scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs, eta_min=1e-10)
 
-subprocess.run([
-    "rm", 
-    "-rf", 
-    f"/home/IITB/ai-at-ieor/23m1521/ashish/MTP/Vaani/Img_Audio_Alignment/{model_name}",
-])
+# learning_rate = 1e-3
+# optimizer = torch.optim.AdamW(csip_model.parameters(), lr=learning_rate)
+# scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=2, eta_min=1e-5)
+
+# subprocess.run([
+#     "rm", 
+#     "-rf", 
+#     f"/home/IITB/ai-at-ieor/23m1521/ashish/MTP/Vaani/Img_Audio_Alignment/{model_name}",
+# ])
 
 train_model(
     model=csip_model,

Vaani/Img_Audio_Alignment/available_img_audios_TRAIN3.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:719e5fef279658370c6f754a34c8bc1ba1af4fe259c46b53002b3932a1889e89
+size 17466633

Vaani/Img_Audio_Alignment/csip_model_openClip_CLAP/checkpoints/csip/csip_best_epoch_539.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b03f512670a8be47816cdb9132322ce8eddc5fe76c289ffd95dbd887a85bfddd
+size 2691054590

Vaani/Img_Audio_Alignment/csip_model_openClip_CLAP/checkpoints/csip/csip_best_epoch_567.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd8f88bdc9e15be4d9af9d1378a8019072cf78a566dd6c78d2e113c96603a4c2
+size 2691054590

Vaani/Img_Audio_Alignment/csip_model_openClip_CLAP/checkpoints/csip/csip_epoch_572_best_epoch_-1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b88cd9caf9c0049c9c5c97c120df6bfbfda1aa5d27aa23e0d06816503816226b
+size 2691066446

Vaani/Img_Audio_Alignment/csip_model_openClip_CLAP/checkpoints/csip/csip_epoch_573_best_epoch_-1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2560edf996a13c7874a2e3fa461d0342b82aa4824f0e48f960e4c6ef487cc03d
+size 2691066446