text_plain_output_1.png
from tqdm import tqdm

import numpy as np # linear algebra

import os

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import pickle

base_dir = '/kaggle/input/coast-data'
coasts = pd.read_csv(os.path.join(base_dir, 'CoastTrain_imagery_details.csv'))
coasts.shape

coasts.name.value_counts()

coasts.iloc[0]

class_set = set()
class_index = []
coasts_df = coasts
for i in range(coasts.shape[0]):
    if coasts.loc[i, 'name'].startswith('Ortho'):
        class_index.append(i)
    '\n    if (coasts.loc[i, "name"].startswith("Ortho") or coasts.loc[i, "name"].startswith("NAIP")) and not coasts.loc[i, "name"].startswith("Orthophoto_8_001"):# or coasts.loc[i, "name"].startswith("Quad"):\n        class_index.append(i)\n    '
coasts_df = coasts.iloc[class_index]
coasts = coasts_df
'\ncoasts.set_index(range(coasts.shape[0]))\nprint(coasts_df.shape)\ncoasts_df["classes_array"].value_counts()\n'

coasts.shape

first_coast = coasts.iloc[3]
directory = first_coast['name'].split('.')[0]
filename = first_coast['images'].split('/')[-1].split('.')[-2]
data = np.load(os.path.join(base_dir, directory, filename + '.npz'))

lst = data.files

np.argmax(data['label'], axis=2)

label = data['label']
new_label = np.argmax(label, axis=2)

def get_mapping(classes):
    mapping = dict()
    if classes == "'water', 'herbaceous vegetation', 'woody vegetation', 'non-vegetated-dry', 'non-vegetated-wet', 'surf', 'developed'":
        mapping = {0: 0, 1: 4, 2: 4, 3: 3, 4: 3, 5: 1, 6: 5}
    elif classes == "'water', 'sand', 'gravel', 'cobble_boulder', 'vegetated', 'coastal_defense', 'buildings', 'pavement_road', 'vehicles', 'people', 'other_anthro', 'nodata'":
        mapping = {0: 0, 1: 2, 2: 2, 3: 3, 4: 4, 5: 5, 6: 5, 7: 5, 8: 5, 9: 5, 10: 5, 11: 6}
    elif classes == "'water', 'sand', 'gravel', 'cobble_boulder', 'vegetated', 'development', 'nodata', 'unusual'":
        mapping = {0: 0, 1: 2, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 6}
    elif classes == "'water', 'surf', 'bare_ground', 'vegetated_ground', 'development', 'no_data'":
        mapping = {0: 0, 1: 1, 2: 3, 3: 4, 4: 5, 5: 6}
    elif classes == "'water', 'surf', 'bare_ground', 'vegtated_ground', 'development', 'nodata'":
        mapping = {0: 0, 1: 1, 2: 3, 3: 4, 4: 5, 5: 6}
    elif classes == "'water', 'surf', 'sand', 'dev'":
        mapping = {0: 0, 1: 1, 2: 2, 3: 5}
    elif classes == "'water', 'whitewater', 'mud_silt', 'sand', 'gravel_shell', 'cobble_boulder', 'bedrock', 'ice_snow', 'vegetated', 'development', 'other'":
        mapping = {0: 0, 1: 4, 2: 3, 3: 2, 4: 2, 5: 3, 6: 3, 7: 3, 8: 4, 9: 5, 10: 6}
    elif classes == "'water', 'whitewater', 'sediment', 'other_bare_natural_terrain', 'marsh_vegetation', 'terrestrial_vegetation', 'agricultural', 'development', 'nodata', 'unusual', 'unknown'":
        mapping = {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 4, 6: 4, 7: 5, 8: 6, 9: 6, 10: 6}
    elif classes == "'water', 'whitewater', 'sediment', 'other_natural_terrain', 'vegetated_surface', 'agricultural', 'development', 'cloud', 'nodata', 'unusual', 'unknown'":
        mapping = {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 4, 6: 5, 7: 6, 8: 6, 9: 6, 10: 6}
    elif classes == "'water', 'whitewater', 'sediment', 'sediment_plume', 'other_natural_terrain', 'vegetated_surface', 'agricultural', 'development', 'cloud', 'nodata', 'unusual', 'unknown'":
        mapping = {0: 0, 1: 1, 2: 2, 3: 2, 4: 3, 5: 4, 6: 4, 7: 5, 8: 6, 9: 6, 10: 6, 11: 6}
    elif classes == "'water', 'whitewater', 'sediment', 'terrestrial_vegetation', 'marsh_vegetation', 'development', 'unusual', 'unknown', 'nodata'":
        mapping = {0: 0, 1: 1, 2: 2, 3: 4, 4: 4, 5: 5, 6: 6, 7: 6, 8: 6}
    return mapping

import pickle
classes_list = ['water', 'whitewater', 'sediment', 'other_natural_terrain', 'vegetation', 'development', 'unknown']
from tqdm import tqdm
count = 0
for i in tqdm(range(coasts.shape[0])):
    coast = coasts.iloc[i]
    classes_array = coast['classes_array']
    directory = coast['name'].split('.')[0]
    dir_names = coast['images'].split('/')[-1].split('.')[:-1]
    filename = ''
    for s in dir_names:
        filename += s + '.'
    filename = filename + 'npz'
    if os.path.exists(os.path.join(base_dir, directory, filename)):
        data = np.load(os.path.join(base_dir, directory, filename))
    elif os.path.exists(os.path.join(base_dir, directory, filename[:-10] + '.npz')):
        data = np.load(os.path.join(base_dir, directory, filename[:-10] + '.npz'))
    else:
        print(directory)
        print(filename)
        continue
    try:
        image = data['orig_image'][:, :, [0, 1, 2]]
    except:
        image = data['image'][:, :, [0, 1, 2]]
    if not (image.shape[0] == 1024 and image.shape[1] == 1024) and (not (image.shape[0] == 2048 and image.shape[1] == 2048)):
        continue
    count += 1
    label = data['label']
    if len(label.shape) != 3:
        print(directory)
        print(filename)
    label = np.argmax(label, axis=2)
    mapping = get_mapping(classes_array)
    normalized_label = label
    for x in range(label.shape[0]):
        for y in range(label.shape[1]):
            normalized_label[x, y] = mapping[normalized_label[x, y]]
    class_count = {c: 0 for c in classes_list}
    for class_index in range(len(classes_list)):
        class_count[classes_list[class_index]] = int(np.count_nonzero(normalized_label == class_index))
    assert int(sum(list(class_count.values()))) == label.shape[0] * label.shape[1]
    with open('coast_images/image_' + str(count) + '.npy', 'wb') as f:
        np.save(f, image)
    with open('coast_labels/image_' + str(count) + '_label.npy', 'wb') as fl:
        np.save(fl, normalized_label)
    with open('coast_info/image_' + str(count) + '_info.pkl', 'wb') as fi:
        pickle.dump(class_count, fi)
print(count)