text_plain_output_1.png
from ast import literal_eval

from pathlib import Path

from tqdm.notebook import tqdm

import joblib

import json

import numpy as np

import pandas as pd

import pytorch_lightning as pl

SEED = 42
ROOT_DIR = '../input'
MEL_PATHS = sorted(Path(ROOT_DIR).glob('birdclef-2022-melspectrogram-compute/rich_train_metadata.csv'))
TRAIN_LABEL_PATHS = sorted(Path(ROOT_DIR).glob('birdclef-2022-melspectrogram-compute/LABEL_IDS.json'))
N_CLASSES = 21
SR = 32000
DURATION = 7
MAX_READ_SAMPLES = 5
USE_FOLD = 0
TRAIN_BATCH_SIZE = 32
TRAIN_NUM_WORKERS = 2
VAL_BATCH_SIZE = 32
VAL_NUM_WORKERS = 2
EPOCHS = 15

with open(Path(ROOT_DIR) / 'birdclef-2022/scored_birds.json') as json_file:
    scored_birds = json.load(json_file)

def get_df(mel_paths=MEL_PATHS, train_label_paths=TRAIN_LABEL_PATHS):
    df = None
    LABEL_IDS = {}
    for file_path in mel_paths:
        temp = pd.read_csv(str(file_path), index_col=0)
        temp['impath'] = temp.apply(lambda row: file_path.parent / 'audio_images/{}.npy'.format(row.filename), axis=1)
        df = temp if df is None else df.append(temp)
        df['secondary_labels'] = df['secondary_labels'].apply(literal_eval)
    for file_path in train_label_paths:
        with open(str(file_path)) as f:
            LABEL_IDS.update(json.load(f))
    return (LABEL_IDS, df)

def load_data(df):
    def load_row(row):
        return (row.filename, np.load(str(row.impath))[:MAX_READ_SAMPLES])
    pool = joblib.Parallel(4)
    mapper = joblib.delayed(load_row)
    tasks = [mapper(row) for row in df.itertuples(False)]
    res = pool(tqdm(tasks))
    res = dict(res)
    return res

pl.seed_everything(SEED)
LABEL_IDS, df = get_df()
audio_image_store = load_data(df)
len(audio_image_store)