The issue is now fixed. Turns out the matplotlib patches function to plot the bounding box takes the top-left coordinates of the bounding box along with the width and height. I’ve been passing in mid-point coordinates of the box when I really should have been passing in the top-left coordinates (even-though the width and height were being passed correctly!).
I’ve made the correction, along with a little rectangle to show labels, and now the plots look like this.

plot_fixed800×800 143 KB

And here’s the fixed code:
import torch
import matplotlib.pyplot as plt
import matplotlib.patches as patches
from torch.utils.data import Dataset, DataLoader
from PIL import Image
from torchvision import transforms
import os
import pandas as pd

class VOCDataset(Dataset):
    """Some Information about VOCDataset"""
    def __init__(self, annotations_path, img_dir, label_dir, transforms, reshape_height, reshape_width):
        self.annotations = pd.read_csv(annotations_path)
        self.img_dir = img_dir
        self.label_dir = label_dir
        self.transforms = transforms
        self.new_height = reshape_height
        self.new_width = reshape_width
        super(VOCDataset, self).__init__()

    def __getitem__(self, index):
        img_file_name = self.annotations.iloc[index][0];
        img_file_path = os.path.join(self.img_dir, img_file_name)
        img = Image.open(img_file_path)
        tensor_img = self.transforms(img)

        label_file_name = self.annotations.iloc[index][1];
        label_file_path = os.path.join(self.label_dir, label_file_name)
        label = list()
        with open(label_file_path, "r") as f:
            line = f.readlines()[0]
            label = [int(float(i)) if int(float(i)) == float(i) else float(i) for i in line.split(" ")]
        return tensor_img, torch.tensor(label)

    def __len__(self):
        return len(self.annotations)

def visualize_imgs(imgs, label, rows, cols):
    VOC_CLASSES = ["aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", 
                    "dog","horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"]
    imgs = imgs.detach().numpy().transpose(0, 2, 3, 1)
    label = label.detach().numpy()
    fig = plt.figure(figsize=(8, 8))
    for i in range(imgs.shape[0]):
        ax = fig.add_subplot(rows, cols, i + 1, xticks=[], yticks=[])

        # to plot the bounding box
        label[i][1] -= (label[i][3] / 2)
        label[i][2] -= (label[i][4] / 2)
        top, left, width, height = [448*i for i in label[i][1:]]
        rect = patches.Rectangle((top, left), width, height, edgecolor='r', facecolor='none', linewidth=2.0)
        ax.add_patch(rect)

        # to plot a little rectangle on top of the bounding box
        LABEL_RECT_HEIGHT = 30
        label_rect = patches.Rectangle((top, left - LABEL_RECT_HEIGHT), width, LABEL_RECT_HEIGHT, facecolor='red', linewidth=2.0)
        ax.add_patch(label_rect)

        ax.text(top, left, VOC_CLASSES[int(label[i][0])], horizontalalignment='left', verticalalignment='bottom', color='white', weight='bold')
        ax.imshow(imgs[i], origin='upper')
    plt.show()

if __name__ == '__main__':

    IMG_DIR = "G:/dl_projects/yolov3_aladdin/PASCAL_VOC/images"
    LABEL_DIR = "G:/dl_projects/yolov3_aladdin/PASCAL_VOC/labels"
    BATCH_SIZE = 4
    NUM_WORKERS = 1
    PIN_MEMORY = False
    TRANSFORMS = transforms.Compose([
        transforms.Resize((448, 448)),
        transforms.ToTensor(),
    ])

    train_dataset = VOCDataset(
        "G:/dl_projects/yolov3_aladdin/PASCAL_VOC/100examples.csv",
        img_dir=IMG_DIR,
        label_dir=LABEL_DIR,
        transforms=TRANSFORMS,
        reshape_height=448,
        reshape_width=448
    )

    train_loader = DataLoader(
        dataset=train_dataset,
        batch_size=BATCH_SIZE,
        num_workers=NUM_WORKERS,
        pin_memory=PIN_MEMORY,
        shuffle=True,
        drop_last=True,
    )

    it = iter(train_loader)
    imgs, labels = next(it)
    visualize_imgs(imgs, labels, rows=2, cols=2)