Hi, @ptrblck Thank you for the reply.
Here is why I use a different batch size:

In Caffe, they go through each example pair. For each image, they get the augmentations for the current frame which is based on the motion-model

In Caffe, for each image pair with augmentation, there are 11 images. They fill a list of images where it gets filled to 50, rest they keep it for the next batch.
Augmented images: (generated from PyTorch code, ignore the green boxes)

0013790×2040 2.17 MB
 .
My intention was to take advantage of num_workers in PyTorch data loaders, where I feed 4 examples and then augment them in collate_fn, wherein total I get 44 images for each iteration.
The PyTorch code is below:
import sys
from pathlib import Path

import numpy as np
import torch
from loguru import logger
from torch.utils.data import DataLoader, Dataset
from tqdm import tqdm

try:
    from goturn.dataloaders.alov import AlovDataset
    from goturn.dataloaders.imagenet import ImageNetDataset
    from goturn.dataloaders.sampler import sample_generator
    from goturn.helper.image_io import resize
except ImportError:
    logger.error('Please run $source settings.sh from root directory')
    sys.exit(1)


class GoturnDataloader(Dataset):
    """Docstring for goturnDataloader. """

    def __init__(self, imagenet_path, alov_path, mean_file=None, isTrain=True,
                 val_ratio=0.005, width=227, height=227,
                 dbg=False):
        """Dataloader initialization for goturn tracker """

        # ALOV
        img_dir = Path(imagenet_path).joinpath('images')
        ann_dir = Path(imagenet_path).joinpath('gt')
        self._imagenetD = ImageNetDataset(str(img_dir), str(ann_dir),
                                          isTrain=isTrain,
                                          val_ratio=val_ratio)

        # Imagenet
        img_dir = Path(alov_path).joinpath('images')
        ann_dir = Path(alov_path).joinpath('gt')
        self._alovD = AlovDataset(str(img_dir), str(ann_dir),
                                  isTrain=isTrain, val_ratio=val_ratio)

        # sample generator
        self._sample_gen = sample_generator(5, 15, -0.4, 0.4, dbg=dbg)
        self._kGenExPerImage = 10

        self._images = []
        self._targets = []
        self._bboxes = []

        self._width = width
        self._height = height
        if mean_file:
            self._mean = np.load(mean_file)
        else:
            self._mean = np.array([104, 117, 123])

        self._minDataLen = min(len(self._imagenetD), len(self._alovD))
        
        self._maxDataLen = max(len(self._imagenetD), len(self._alovD))

    def __len__(self):
        ''' length of the total dataset, is max of one of the dataset '''
        return self._maxDataLen

    def __getitem__(self, idx):
        """Get the current idx data
        @idx: Current index for the data
        """
        if self._minDataLen == len(self._imagenetD):
            imagenet_pair = self._imagenetD[idx % self._minDataLen]
            alov_pair = self._alovD[idx]
        else:
            imagenet_pair = self._imagenetD[idx]
            alov_pair = self._alovD[idx % self._minDataLen]

        return imagenet_pair, alov_pair

    def collate(self, batch):
        ''' Custom data collation for alov and imagenet
        @batch: batch of data
        '''
        self._images = []
        self._targets = []
        self._bboxes = []

        for i, batch_i in enumerate(batch):
            for i, (img_prev, bbox_prev, img_cur, bbox_cur) in enumerate(batch_i):
                self._sample_gen.reset(bbox_cur, bbox_prev, img_cur,
                                       img_prev)
                self.__make_training_samples()

        for i, (im, tar, bbox) in enumerate(zip(self._images,
                                                self._targets,
                                                self._bboxes)):

            im = resize(im, (self._width, self._height)) - self._mean
            # im = im / 255.
            self._images[i] = np.transpose(im, axes=(2, 0, 1))
            tar = resize(tar, (self._width, self._height)) - self._mean
            # tar = tar / 255.
            self._targets[i] = np.transpose(tar, axes=(2, 0, 1))
            self._bboxes[i] = np.array([bbox.x1, bbox.y1, bbox.x2,
                                        bbox.y2])

        images = torch.from_numpy(np.stack(self._images))
        targets = torch.from_numpy(np.stack(self._targets))
        bboxes = torch.from_numpy(np.stack(self._bboxes))

        return images, targets, bboxes

    def __make_training_samples(self):
        """
        1. First decide the current search region, which is
        kContextFactor(=2) * current bounding box.
        2. Crop the valid search region and copy to the new padded image
        3. Recenter the actual bounding box of the object to the new
        padded image
        4. Scale the bounding box for regression
        """

        sample_gen = self._sample_gen
        images = self._images
        targets = self._targets
        bboxes = self._bboxes

        image, target, bbox_gt_scaled = sample_gen.make_true_sample()

        images.append(image)
        targets.append(target)
        bboxes.append(bbox_gt_scaled)

        # Generate more number of examples
        images, targets, bbox_gt_scaled = sample_gen.make_training_samples(self._kGenExPerImage, images, targets, bboxes)


if __name__ == "__main__":
    imagenet_path = '/media/nthere/datasets/ISLVRC2014_Det/'
    alov_path = '/media/nthere/datasets/ALOV/'
    objGoturn = GoturnDataloader(imagenet_path, alov_path, dbg=False)

    dataloader = DataLoader(objGoturn, batch_size=2, shuffle=True,
                            num_workers=0, collate_fn=objGoturn.collate)

    for i, *data in tqdm(enumerate(dataloader), desc='Loading imagenet/alov', total=len(objGoturn), unit='files'):
        pass


i will be happy to share more information.