import os import glob import sys from os.path import join ''' Note: Anywhere empty boxes means [] and not [[]] ''' def remove_true_positives(gts, preds): def true_positive(gt, pred): # If center of pred is inside the gt, it is a true positive c_pred = ((pred[0]+pred[2])/2., (pred[1]+pred[3])/2.) if (c_pred[0] >= gt[0] and c_pred[0] <= gt[2] and c_pred[1] >= gt[1] and c_pred[1] <= gt[3]): return True return False tps = 0 fns = 0 for gt in gts: # First check if any true positive exists # If more than one exists, do not include it in next set of preds add_tp = False new_preds = [] for pred in preds: if true_positive(gt, pred): add_tp = True else: new_preds.append(pred) preds = new_preds if add_tp: tps += 1 else: fns += 1 return preds, tps, fns def calc_metric_single(gts, preds, threshold,): ''' Returns fp, tp, tn, fn ''' preds = list(filter(lambda x: x[0] >= threshold, preds)) preds = [pred[1:] for pred in preds] # Remove the scores if len(gts) == 0: return len(preds), 0, 1 if len(preds) == 0 else 0, 0 preds, tps, fns = remove_true_positives(gts, preds) # All remaining will have to fps fps = len(preds) return fps, tps, 0, fns def calc_metrics_at_thresh(im_dict, threshold): ''' Returns fp, tp, tn, fn ''' fps, tps, tns, fns = 0, 0, 0, 0 for key in im_dict: fp,tp,tn,fn = calc_metric_single(im_dict[key]['gt'], im_dict[key]['preds'], threshold) fps+=fp tps+=tp tns+=tn fns+=fn return fps, tps, tns, fns from joblib import Parallel, delayed def calc_metrics(inp): im_dict, tr = inp out = dict() for t in tr: fp, tp, tn, fn = calc_metrics_at_thresh(im_dict, t) out[t] = [fp, tp, tn, fn] return out def calc_froc_from_dict(im_dict, fps_req = [0.025,0.05,0.1,0.15,0.2,0.3], save_to = None): num_images = len(im_dict) gap = 0.005 n = int(1/gap) thresholds = [i * gap for i in range(n)] fps = [0 for _ in range(n)] tps = [0 for _ in range(n)] tns = [0 for _ in range(n)] fns = [0 for _ in range(n)] for i,t in enumerate(thresholds): fps[i], tps[i], tns[i], fns[i] = calc_metrics_at_thresh(im_dict, t) # Now calculate the sensitivities senses = [] for t,f in zip(tps, fns): try: senses.append(t/(t+f)) except: senses.append(0.) if save_to is not None: f = open(save_to, 'w') for fp,s in zip(fps, senses): f.write(f'{fp/num_images} {s}\n') f.close() senses_req = [] for fp_req in fps_req: for i,f in enumerate(fps): if f/num_images < fp_req: if fp_req == 0.1: print(fps[i], tps[i], tns[i], fns[i]) prec = tps[i]/(tps[i] + fps[i]) recall = tps[i]/(tps[i] + fns[i]) f1 = 2*prec*recall/(prec+recall) spec = tns[i]/ (tns[i] + fps[i]) print(f'Specificity: {spec}') print(f'Precision: {prec}') print(f'Recall: {recall}') print(f'F1: {f1}') senses_req.append(senses[i-1]) break return senses_req, fps_req def file_to_bbox(file_name): try: content = open(file_name, 'r').readlines() st = 0 if len(content) == 0: # Empty File Should Return [] return [] if content[0].split()[0].isalpha(): st = 1 return [[float(x) for x in line.split()[st:]] for line in content] except FileNotFoundError: print(f'No Corresponding Box Found for file {file_name}, using [] as preds') return [] except Exception as e: print('Some Error',e) return [] def generate_image_dict(preds_folder_name='preds_42', root_fol='/home/pranjal/densebreeast_datasets/AIIMS_C1', mal_path=None, ben_path=None, gt_path=None, mal_img_path = None, ben_img_path = None ): mal_path = join(root_fol, mal_path) if mal_path else join( root_fol, 'mal', preds_folder_name) ben_path = join(root_fol, ben_path) if ben_path else join( root_fol, 'ben', preds_folder_name) mal_img_path = join(root_fol, mal_img_path) if mal_img_path else join( root_fol, 'mal', 'images') ben_img_path = join(root_fol, ben_img_path) if ben_img_path else join( root_fol, 'ben', 'images') gt_path = join(root_fol, gt_path) if gt_path else join( root_fol, 'mal', 'gt') ''' image_dict structure: 'image_name(without txt/png)' : {'gt' : [[...]], 'preds' : [[]]} ''' image_dict = dict() # GT Might be sightly different from images, therefore we will index gts based on # the images folder instead. for file in os.listdir(mal_img_path): if not file.endswith('.png'): continue file = file[:-4] + '.txt' file = join(gt_path, file) key = os.path.split(file)[-1][:-4] image_dict[key] = dict() image_dict[key]['gt'] = file_to_bbox(file) image_dict[key]['preds'] = [] for file in glob.glob(join(mal_path, '*.txt')): key = os.path.split(file)[-1][:-4] assert key in image_dict image_dict[key]['preds'] = file_to_bbox(file) for file in os.listdir(ben_img_path): if not file.endswith('.png'): continue file = file[:-4] + '.txt' file = join(ben_path, file) key = os.path.split(file)[-1][:-4] if key == 'Calc-Test_P_00353_LEFT_CC' or key == 'Calc-Training_P_00600_LEFT_CC': # Corrupt Files in Dataset continue if key in image_dict: print(key) # assert key not in image_dict if key in image_dict: print(f'Unexpected Error. {key} exists in multiple splits') continue image_dict[key] = dict() image_dict[key]['preds'] = file_to_bbox(file) image_dict[key]['gt'] = [] return image_dict def pretty_print_fps(senses,fps): for s,f in zip(senses,fps): print(f'Sensitivty at {f}: {s}') def get_froc_points(preds_image_folder, root_fol, fps_req = [0.025,0.05,0.1,0.15,0.2,0.3], save_to = None): im_dict = generate_image_dict(preds_image_folder, root_fol = root_fol) # print(im_dict) print(len(im_dict)) senses, fps = calc_froc_from_dict(im_dict, fps_req, save_to = save_to) return senses, fps if __name__ == '__main__': seed = '42' if len(sys.argv)== 1 else sys.argv[1] root_fol = '../bilateral_new/MammoDatasets/AIIMS_highres_reliable/test_2' if len(sys.argv) <= 2: save_to = None else: save_to = sys.argv[2] senses, fps = get_froc_points(f'preds_{seed}',root_fol, save_to = save_to) pretty_print_fps(senses, fps)