text_plain_output_2.png text_plain_output_1.png
from glob import glob

from itertools import chain

from keras.preprocessing.image import ImageDataGenerator

from random import sample

import numpy as np # linear algebra

import os

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import sklearn.model_selection as skl

all_xray_df = pd.read_csv('/kaggle/input/data/Data_Entry_2017.csv')
all_image_paths = {os.path.basename(x): x for x in glob(os.path.join('/kaggle/input/data', 'images*', '*', '*.png'))}
all_xray_df['path'] = all_xray_df['Image Index'].map(all_image_paths.get)
all_xray_df.sample(3)

all_labels = np.unique(list(chain(*all_xray_df['Finding Labels'].map(lambda x: x.split('|')).tolist())))
all_labels = [x for x in all_labels if len(x) > 0]
for c_label in all_labels:
    if len(c_label) > 1:
        all_xray_df[c_label] = all_xray_df['Finding Labels'].map(lambda finding: 1.0 if c_label in finding else 0)
all_xray_df['pneumonia_class'] = all_xray_df['Pneumonia']
all_xray_df.sample(3)

def create_splits(df, test_size, column_name):
    train_df, valid_df = skl.train_test_split(df, test_size=test_size, stratify=df[column_name])
    p_inds = train_df[train_df[column_name] == 1].index.tolist()
    np_inds = train_df[train_df[column_name] == 0].index.tolist()
    np_sample = sample(np_inds, len(p_inds))
    train_df = train_df.loc[p_inds + np_sample]
    p_inds = valid_df[valid_df[column_name] == 1].index.tolist()
    np_inds = valid_df[valid_df[column_name] == 0].index.tolist()
    np_sample = sample(np_inds, 4 * len(p_inds))
    valid_df = valid_df.loc[p_inds + np_sample]
    return (train_df, valid_df)
train_df, valid_df = create_splits(all_xray_df, 0.2, 'pneumonia_class')

def my_image_augmentation():
    my_idg = ImageDataGenerator(rescale=1.0 / 255.0, horizontal_flip=True, vertical_flip=False, height_shift_range=0.1, width_shift_range=0.1, rotation_range=20, shear_range=0.1, zoom_range=0.1)
    return my_idg
def make_train_gen(train_df, img_size, batch_size):
    idg = my_image_augmentation()
    train_gen = idg.flow_from_dataframe(dataframe=train_df, directory=None, x_col='path', y_col='pneumonia_class', class_mode='raw', target_size=img_size, batch_size=batch_size)
    return train_gen
def make_val_gen(valid_df, img_size, batch_size):
    val_idg = ImageDataGenerator(rescale=1.0 / 255.0)
    val_gen = val_idg.flow_from_dataframe(dataframe=valid_df, directory=None, x_col='path', y_col='pneumonia_class', class_mode='raw', target_size=img_size, batch_size=batch_size)
    return val_gen
batch_size = 64
img_size = (224, 224)
train_gen = make_train_gen(train_df, img_size, batch_size)
val_gen = make_val_gen(valid_df, img_size, batch_size)