text_plain_output_1.png image_output_1.png
from sklearn.model_selection import train_test_split

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

class CustomImageList(ImageList):
    def open(self, fn):
        img = fn.reshape(28, 28)
        img = np.stack((img,) * 3, axis=-1)
        return Image(pil2tensor(img, dtype=np.float32))
    @classmethod
    def from_csv_custom(cls, path: PathOrStr, csv_name: str, imgIdx: int=1, header: str='infer', **kwargs) -> 'ItemList':
        df = pd.read_csv(Path(path) / csv_name, header=header)
        res = super().from_df(df, path=path, cols=0, **kwargs)
        res.items = df.iloc[:, imgIdx:].apply(lambda x: x.values / 255.0, axis=1).values
        return res
    @classmethod
    def from_df_custom(cls, path: PathOrStr, df: DataFrame, imgIdx: int=1, header: str='infer', **kwargs) -> 'ItemList':
        res = super().from_df(df, path=path, cols=0, **kwargs)
        res.items = df.iloc[:, imgIdx:].apply(lambda x: x.values / 255.0, axis=1).values
        return res

test = CustomImageList.from_csv_custom(path=path, csv_name='test.csv', imgIdx=0)

data = CustomImageList.from_csv_custom(path=path, csv_name='train.csv', imgIdx=1).split_by_rand_pct(0.2).label_from_df(cols='label').add_test(test, label=0).transform(get_transforms(do_flip=False)).databunch(bs=128, num_workers=0).normalize(imagenet_stats)

learn = cnn_learner(data, models.resnet18, metrics=[accuracy], model_dir='/kaggle/working/models')

learn.lr_find()

learn.fit_one_cycle(4, max_lr=0.01)

learn.unfreeze()
learn.lr_find()

learn.fit_one_cycle(10, max_lr=slice(1e-06, 0.0001))

predictions, *_ = learn.get_preds(DatasetType.Test)
labels = np.argmax(predictions, 1)
submission_df = pd.DataFrame({'ImageId': list(range(1, len(labels) + 1)), 'Label': labels})
submission_df.to_csv(f'submission_orig.csv', index=False)

train_df = pd.read_csv(path + '/train.csv')
from sklearn.model_selection import train_test_split
train_df, val_df = train_test_split(train_df, test_size=0.2)

proportions = pd.DataFrame({0: [0.5], 1: [0.05], 2: [0.1], 3: [0.03], 4: [0.03], 5: [0.03], 6: [0.03], 7: [0.5], 8: [0.5], 9: [0.5]})

imbalanced_train_df = train_df.groupby('label').apply(lambda x: x.sample(frac=proportions[x.name]))

imbalanced_train_df['label'].hist(figsize=(10, 5))