text_plain_output_1.png
from pandas.io.json import json_normalize

from tqdm import tqdm

from tqdm.notebook import tqdm

import json_lines

import pandas as pd

import random

data0 = []
with open('../input/youtubes-channels-dataset/YouTubeDataset_withChannelElapsed.json', 'rb') as f:
    for i, item in enumerate(json_lines.reader(f)):
        if i < 10000:
            data0 += [item]

users0 = json_normalize(data0[0][0])
users0

for i, item in tqdm(enumerate(data0[0])):
    if 0 < i and i < 10000:
        usersi = json_normalize(item)
        users0 = pd.concat([users0, usersi])

N = list(range(10000))
data1 = users0.copy()
data1['index0'] = N

data1 = data1.set_index('index0', drop=True)
data1

data2 = data1.drop(['channelId', 'videoId', 'videoPublished'], axis=1)
data2

data2 = data2.astype(float)

target = ['subscriberCount']
dataY = data2[target[0]]
dataX = data2.drop(target, axis=1)

n = len(dataX)
random.seed(2021)
random.shuffle(N)

trainX = dataX.loc[N[0:n // 4 * 3]]
trainY = dataY.loc[N[0:n // 4 * 3]]
testX = dataX.loc[N[n // 4 * 3:]]
testY = dataY.loc[N[n // 4 * 3:]]

df_columns = list(dataX.columns)

def create_numeric_feature(input_df):
    use_columns = df_columns
    return input_df[use_columns].copy()

from tqdm import tqdm
def to_feature(input_df):
    processors = [create_numeric_feature]
    out_df = pd.DataFrame()
    for func in tqdm(processors, total=len(processors)):
        with Timer(prefix='create' + func.__name__ + ' '):
            _df = func(input_df)
        assert len(_df) == len(input_df), func.__name__
        out_df = pd.concat([out_df, _df], axis=1)
    return out_df

train_feat_df = to_feature(trainX)
test_feat_df = to_feature(testX)