here is My personal mapper for augmentation
def mapper(dataset_dict):
    dataset_dict = copy.deepcopy(dataset_dict)  # it will be modified by code below
    image = utils.read_image(dataset_dict["file_name"], format="BGR")    

    transform_list = [
                     #T.Resize((800,800)),
                     T.ResizeShortestEdge(short_edge_length=(640, 672, 704, 736, 768, 800), max_size=1333, sample_style='choice')
                     ,T.RandomApply(T.RandomFlip(prob=0.5, horizontal=True, vertical=False),prob=0.3)
                     ,T.RandomApply(T.RandomRotation((-10,10)),prob=0.4)
                     ,T.RandomApply(T.RandomSaturation(0.8, 1.2),prob=0.3)
                     ,T.RandomApply(T.RandomBrightness(0.8, 1.2),prob=0.2)
                     ,T.RandomApply(T.RandomContrast(0.6, 1.3),prob=0.2)
                     ,T.RandomApply(T.RandomLighting(0.7),prob=0.4)

                      ]
    image, transforms = T.apply_transform_gens(transform_list, image)
    dataset_dict["image"] = torch.as_tensor(image.transpose(2, 0, 1).astype("float32"))

    annos = [
        utils.transform_instance_annotations(obj, transforms, image.shape[:2])
        for obj in dataset_dict.pop("annotations")
        if obj.get("iscrowd", 0) == 0
    ]

    instances = utils.annotations_to_instances(annos, image.shape[:2])
    dataset_dict["instances"] = instances
    return dataset_dict

Here so far no problem
but when I add the
                     #,T.RandomApply(T.RandomCrop('relative_range', (0.4, 0.6)),prob=0.05)
to the augmentation loss become diverged and show Nan or infinity number.
and here is my configuration for detectron2
import os
from detectron2.data import DatasetMapper, build_detection_train_loader   # the default mapper
import detectron2.data.transforms as T
## Adding the new CascadeDropoutROIHeads to the config file

cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("Misc/cascade_mask_rcnn_R_50_FPN_3x.yaml"))
cfg.DATASETS.TRAIN = ("kogas_train",)
cfg.DATASETS.TEST = ("kogas_test",)
cfg.TEST.EVAL_PERIOD = 200

cfg.DATALOADER.NUM_WORKERS = 4
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("Misc/cascade_mask_rcnn_R_50_FPN_3x.yaml")  # Let training initialize from model zoo
cfg.SOLVER.IMS_PER_BATCH = 12#6

cfg.SOLVER.MAX_ITER = 15000    # Need to train longer
cfg.MODEL.RESNETS.DEPTH = 50 #RESNET 50,101
cfg.SOLVER.CHECKPOINT_PERIOD = 200

#cfg.MODEL.ROI_HEADS.NAME= "CascadeROIHeads"

cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4  # 4 classes
cfg.MODEL.MASK_ON = False
cfg.OUTPUT_DIR = save_path

cfg.SOLVER.LR_SCHEDULER_NAME = 'WarmupCosineLR'
cfg.SOLVER.BASE_LS = 0.0001

os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)    
dataloader = build_detection_train_loader(cfg, mapper=mapper2)
trainer = CocoTrainer(cfg)
trainer.resume_or_load(resume=False)