text_plain_output_1.png image_output_1.png
from itertools import chain

from sklearn.model_selection import train_test_split

from tqdm import tqdm

import numpy as np

import numpy as np

import os

import pandas as pd

import pandas as pd

DATAFRAME_PATH = '/kaggle/input/productcategorization2/image_taxonomy_and_description.csv'
IMAGE_DIRECTORY_PATH = '/kaggle/input/productcategorization2/new_images/new_images'

images_list = [int(iter.split('.')[0]) for iter in os.listdir(IMAGE_DIRECTORY_PATH)]
df = pd.read_csv(DATAFRAME_PATH)
df1 = df.copy()
df1 = df1[df1['product_id'].isin(images_list)]
df1.reset_index(drop=True, inplace=True)
df1.drop({'long_description', 'short_description'}, axis=1, inplace=True)
unique_labels_having_one_cat = sorted(list(set(chain.from_iterable(df['taxonomy'].apply(lambda x: [iter for iter in x.split('|') if iter.count('>') == 0]).to_list()))))
unique_labels_having_two_cat = sorted(list(set(chain.from_iterable(df['taxonomy'].apply(lambda x: [iter for iter in x.split('|') if iter.count('>') == 1]).to_list()))))
unique_labels_having_three_cat = sorted(list(set(chain.from_iterable(df['taxonomy'].apply(lambda x: [iter for iter in x.split('|') if iter.count('>') == 2]).to_list()))))

df1.shape

df1['three_label_taxonomy'] = df.apply(lambda row: [iter for iter in row['taxonomy'].split('|') if iter.count('>') == 2], axis=1)

for iter in unique_labels_having_three_cat:
    df1[iter] = 0
for idx, row in df1.iterrows():
    cats = row['three_label_taxonomy']
    for it in cats:
        df1.loc[idx, it] += 1

df1['powerlabel'] = 0
for idx, row in tqdm(df1.iterrows()):
    count = 0
    for i, iter in enumerate(unique_labels_having_three_cat):
        count += np.ceil(1.01 ** (203 - i - 1)) * row[iter]
    df1.loc[idx, 'powerlabel'] = count

max = int(df1['powerlabel'].max())

powercount = {}
powerlabels = np.unique(df1['powerlabel'])
for p in powerlabels:
    powercount[p] = np.count_nonzero(df1['powerlabel'] == p)

train_inds, val_inds = train_test_split(np.array(list(range(df1.shape[0]))), test_size=0.2, random_state=0)

train_df = df1.loc[train_inds, :].reset_index(drop=True)
train_df.drop({'taxonomy', 'three_label_taxonomy'}, inplace=True, axis=1)
val_df = df1.loc[val_inds, :].reset_index(drop=True)
val_df.drop({'taxonomy', 'three_label_taxonomy'}, inplace=True, axis=1)

train_df[train_df['Footwear>Kids>Sandals'] == 1]