text_html_output_1.png
from sklearn.preprocessing import MultiLabelBinarizer

from tensorflow.keras.utils import load_img

import cv2

import hashlib

import matplotlib.pyplot as plt

import numpy as np

import os

import pandas as pd

import random

def load_image(filename):
    image = load_img('../input/plant-pathology-2021-fgvc8/train_images/' + filename)
def load_random_image(filenames):
    sample = random.choice(filenames)
    image = load_img('../input/plant-pathology-2021-fgvc8/train_images/' + sample)
def load_augmented_random_image(filenames):
    sample = random.choice(filenames)
    image = load_img('/kaggle/working/' + sample)
def load_image_for_augmentation(image_path):
    image = cv2.imread(image_path)
    if image.shape[-1] == 1:
        image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
    image = np.array(image)
    return image
def format_path_gcs(st):
    return GCS_DS_PATH + '/train_images/' + st
def format_resized_image_path_gcs(st):
    return GCS_DS_PATH + '/img_sz_384/' + st
def format_tpu_path(st):
    return '/kaggle/input/resized-plant2021' + '/img_sz_384/' + st

# Defining mem usage reduction function to try to make dataframes lighter

def reduce_mem_usage(df, verbose=True):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    start_mem = df.memory_usage().sum() / 1024**2
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)

    end_mem = df.memory_usage().sum() / 1024**2
    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))

    return df

train_df = pd.read_csv('../input/plant-pathology-2021-fgvc8/train.csv')
IMAGE_PATH = '../input/plant-pathology-2021-fgvc8/test-images/'
train_df = reduce_mem_usage(train_df)

RESIZED_IMAGE_PATH = '../input/resized-plant2021/img_sz_384/'

train_df.labels.value_counts()

initial_length = len(train_df)
hashes = {}
duplicates = []
originals = []
for index, row in train_df.iterrows():
    filename = row['image']
    with open(os.path.join(RESIZED_IMAGE_PATH, filename), 'rb') as f:
        hash = hashlib.md5(f.read()).hexdigest()
    if hash in hashes:
        duplicates.append(filename)
        originals.append(hashes[hash])
        train_df.drop(index, inplace=True)
    else:
        hashes[hash] = filename

train_df.labels.value_counts()

labels = train_df['labels'].tolist()
unique_labels = set()
for label in labels:
    unique_labels.update(label.split())

common_labels = [label[0] for label in pd.Series(labels).str.split(expand=True).stack().value_counts()[:6].items()]
mlb = MultiLabelBinarizer(classes=common_labels)
label_matrix = mlb.fit_transform(train_df['labels'].str.split())
label_df = pd.DataFrame(label_matrix, columns=common_labels)
train_df.reset_index(drop=True, inplace=True)
label_df.reset_index(drop=True, inplace=True)
new_df = pd.concat([train_df, label_df], axis=1)
train_df = new_df.drop('labels', axis=1)
train_df.head()