text_plain_output_1.png
from fastai.tabular.all import df_shrink

from time import sleep

import gc

import os

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import numpy as np
import pandas as pd
import os

files_static = [f'/kaggle/input/cccscicandmal2020/StaticAnalysis/{f}' for f in os.listdir('/kaggle/input/cccscicandmal2020/StaticAnalysis') if f.endswith('.csv') and (not f.endswith('Riskware.csv'))]

for f in files_static:
    print(f)
    df_static = pd.read_csv(f, sep=',', encoding='utf-8')
    df_static = df_shrink(df_static)
    nans = df_static.isna().sum().sort_index(ascending=False)
    if nans.iloc[0] > 0:
        print(f'Found N/A values in any of the columns of {f}, DROPPING')
        df_static = df_static.dropna(axis=0)
    label = f.split('/')[-1].split('.')[0]
    if 'Ben' in label:
        df_static['Label'] = 'Benign'
    else:
        df_static['Label'] = label
    dupli = df_static.duplicated().sum()
    if dupli > 0:
        print(f, dupli, 'fully duplicate rows to remove')
    df_static.drop_duplicates(inplace=True)
    df_static.reset_index(inplace=True, drop=True)
    df_static.columns = [f'F{i}' for i in range(9504)] + ['Label']
    print(df_static.Label.value_counts())
    df_static.to_parquet(f'static-{label}.parquet')
    df_static.drop(df_static.index[:], inplace=True)
    del df_static, nans, dupli
    sleep(5)
    gc.collect()