text_plain_output_3.png text_plain_output_2.png text_plain_output_1.png
import gc

import numpy as np # linear algebra

import pandas as pd

import pycountry

import numpy as np
import pandas as pd
import gc
train = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/train_data.csv')
df = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/test_data.csv')
def agg_functions(df1):
    features = ['c1', 'deviceId', 'unitDisplayType', 'bundleId']
    global train
    for x in features:
        t = train.groupby(x)['winBid'].agg(['std', 'max', 'min', 'mean', 'median', 'var', 'sem'])
        t.columns = [x + c + '_bidWin' for c in t.columns]
        t = t.astype({c: np.float32 for c in t.columns})
        t.reset_index(inplace=True)
        df1 = df1.merge(t, on=x, how='left')
        gc.collect()
    return df1
df = agg_functions(df)
train = agg_functions(train)
s = train.deviceIdstd_bidWin.mean()
df['deviceIdstd_bidWin'].fillna(s, inplace=True)
s = train.deviceIdmin_bidWin.mean()
df['deviceIdmin_bidWin'].fillna(s, inplace=True)
s = train.deviceIdmax_bidWin.mean()
df['deviceIdmax_bidWin'].fillna(s, inplace=True)
s = train.deviceIdmean_bidWin.mean()
df['deviceIdmean_bidWin'].fillna(s, inplace=True)
s = train.deviceIdmedian_bidWin.mean()
df['deviceIdmedian_bidWin'].fillna(s, inplace=True)
s = train.deviceIdvar_bidWin.mean()
df['deviceIdvar_bidWin'].fillna(s, inplace=True)
s = train.deviceIdsem_bidWin.mean()
df['deviceIdsem_bidWin'].fillna(s, inplace=True)
del train
gc.collect()
cols = ['unitDisplayTypemin_bidWin']
df.drop(cols, axis=1, inplace=True)

df['size'] = df['size'].apply(lambda x: x.split('x'))
df['size'] = df['size'].apply(lambda x: int(x[0]) * int(x[1]))
df['size'] = df['size'] * 0.0001
df.mediationProviderVersion = df.mediationProviderVersion.apply(lambda x: x.replace('-beta', '.') if 'beta' in x else x + '.0')
def ver_convert_num(ver):
    ver = ver.str.split('.', expand=True)
    digit = [2, 1, 1, 2]
    for i, x in enumerate(ver.columns):
        if int(ver[x].str.len().min()) == digit[i]:
            continue
        ver[x] = ver[x].apply(lambda x: x if len(x) == digit[i] else (digit[i] - len(x)) * '0' + x)
    ver = ver.apply(lambda x: int(''.join(x)), axis=1)
    return ver
df.mediationProviderVersion = ver_convert_num(df.mediationProviderVersion)
rank = {102100: 0, 103100: 1, 103200: 2, 103400: 3, 103500: 4, 103600: 5, 103700: 6, 110000: 7, 110003: 8, 110004: 9, 110005: 10, 110007: 11, 110101: 12, 110106: 13, 110110: 14, 111000: 15, 111100: 16, 111102: 17, 111103: 18, 111104: 19, 111200: 20, 111305: 21, 112003: 22, 112100: 23, 113100: 24, 113200: 25, 113201: 26, 113300: 27, 114000: 28, 114100: 29, 114200: 30, 114201: 31, 114300: 32, 114304: 33, 114400: 34}
unranked = []
for x in df.mediationProviderVersion.unique():
    if x not in rank.keys():
        unranked.append(x)
t = list(rank.keys())
tem = [i for i in unranked if i < t[0]]
def interver(tem, border, index):
    global rank
    tem.sort()
    if len(tem) > 0:
        tem.append(border)
        for i, c in enumerate(tem):
            if c == border:
                continue
            rank[c] = index - (i + 1) / len(tem)
interver(tem, t[0], 0)
for x in range(0, len(t), 2):
    if x + 1 == len(t):
        continue
    tem = [i for i in unranked if i < t[x + 1] and i > t[x]]
    interver(tem, t[x + 1], x + 1)
    tem = [i for i in unranked if i < t[x] and i > t[x - 1]]
    interver(tem, t[x], x)
tem = [i for i in unranked if i > t[-1]]
tem.sort()
if len(tem) > 0:
    h = rank[t[-1]] + 1
    for i, c in enumerate(tem):
        if i + 1 == len(tem):
            rank[c] = h
            continue
        rank[c] = h - (i + 1) / len(tem)
df.mediationProviderVersion = df.mediationProviderVersion.apply(lambda x: rank[x])
df['os'] = df.osAndVersion.apply(lambda x: x.split('-')[0])
df['ver'] = df.osAndVersion.apply(lambda x: x.split('-')[1].split('.')[0])
os_rank = [{4: 0, 5: 1, 6: 2, 7: 3, 8: 4, 9: 5, 10: 6, 11: 7}, {10: 0, 11: 1, 12: 2, 13: 3, 14: 4, 15: 5, 16: 6}]
threshold = [3, 1]
for i, c in enumerate(df.os.unique()):
    rank = os_rank[i]
    for x in df.loc[df.os == c, 'ver'].unique():
        if int(x) not in rank.keys():
            g = rank.keys()
            x = int(x)
            rank[x] = min(g) - x if x < min(g) else rank[max(g)] + (x - max(g))
    df.loc[df.os == c, 'ver'] = df.loc[df.os == c, 'ver'].apply(lambda y: rank[int(y)] + threshold[i])
del df['os']
df['ver'] = df['ver'].astype(np.float32)
import gc
gc.collect()
df

import pycountry
countries = {}
for country in pycountry.countries:
    countries[country.name] = country.alpha_2
country = pd.read_csv('../input/country-name-hdi/csvData.csv')
s = country.hdi2019.median()
country['countryCode'] = country.country.apply(lambda x: countries.get(x, '0_0'))
country.loc[country.countryCode == '0_0', 'hdi2019'] = s
country.drop(['pop2022', 'country'], axis=1, inplace=True)
country.drop_duplicates(subset='countryCode', keep='first', inplace=True)
country.set_index('countryCode', inplace=True)
country = country.loc[:, 'hdi2019'].to_dict()
df.countryCode.fillna('0_0', inplace=True)
df['countryCode'] = df['countryCode'].apply(lambda x: country.get(x, s))
df['countryCode'] = df['countryCode'].astype(np.float32)
df

from tqdm import tqdm
def preprocess(df1):
    unique_dict = {'unitDisplayType': ['rewarded', 'interstitial', 'banner'], 'bundleId': ['com.tilegarden.match3', 'com.loop.match3d', 'com.tintash.nailsalon', '1502447854', 'se.ace.fishinc', 'com.kamilbilge.ropesavior3d', '1523081624', 'com.AppIdeas.LevelUpRunner', '1529614832', '1586795332', 'dovi.coalmining.inc', '1542256628', '1436213906', 'com.YayySAL.DodgeAgent', 'com.volt.dresstoimpress', '1579489488', '1582745578', '1569586264'], 'connectionType': ['3G', 'WIFI', 'UNKNOWN', '0_0'], 'c1': ['cb2', '7d3', '8bd', 'ad3', '59b', 'fdc', '7ca', '7b8', '9b5', 'dfa', '1ba', '2c1', 'ce2', '3db', '76e', '313', 'f8d', 'c2e', '82a', 'b22', 'f0f', '3dc', 'c17', '403', '707', '94f', 'cb3', '87b', '7be', 'd69', '00e', 'ea0', '397', '443', '466', '16a', '1c3', 'f99', 'f17', '05b', '2f6', '8f8', '49e', 'a65', '064', 'da9', 'd79', 'ab5', 'fd5', '541'], 'c3': ['6b', '79', '4b', '4e'], 'c2': ['6.0', '3.0', '9.0', '7.0', '8.0', '5.0', '2.0', '1.0', '4.0']}
    def unique_one_hot(df2, name):
        df2[name] = df2[name].astype(str)
        df2[name].fillna('0_0', inplace=True)
        df2[name] = df2[name].str.replace('nan', '0_0')
        unique = unique_dict[name]
        dummies = {x: y for x, y in zip(unique, pd.get_dummies(unique, dtype=np.uint8).values)}
        one_hot = pd.DataFrame(columns=[f'{name}_{i}' for i, x in enumerate(unique)])
        one_hot[[f'{name}_{i}' for i, x in enumerate(unique)]] = np.array([dummies[x] for x in df2[name]])
        one_hot = one_hot.astype({f'{name}_{i}': np.uint8 for i, x in enumerate(unique)})
        df2 = df2.join(one_hot)
        return df2
    categorical = ['unitDisplayType', 'connectionType', 'bundleId', 'c3', 'c1']
    for cat in categorical:
        df1 = unique_one_hot(df1, cat)
    df1.drop(categorical, axis=1, inplace=True)
    gc.collect()
    return df1
df = preprocess(df)

df