text_plain_output_1.png
import numpy as np # linear algebra

import os

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import numpy as np
import pandas as pd
import os
files = []
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        fp = os.path.join(dirname, filename)
        files.append(fp)

dfs = [pd.read_csv(f, sep=',', encoding='utf-8') for f in files]
[df.shape for df in dfs]

for i, df in enumerate(dfs):
    dfs[i] = df.drop(columns=['SourceIP', 'DestinationIP', 'SourcePort', 'DestinationPort', 'TimeStamp'])

for df in dfs:
    df.replace([np.inf, -np.inf], np.nan, inplace=True)
    df.dropna(inplace=True)
[df.shape for df in dfs]

for df in dfs:
    print(df.duplicated().sum(), 'fully duplicate rows to remove')
    df.drop_duplicates(inplace=True)
    df.reset_index(inplace=True, drop=True)
[df.shape for df in dfs]