text_html_output_1.png
import json

import numpy as np

import pandas as pd

pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
pd.set_option('display.width', None)
pd.set_option('display.max_colwidth', None)

f = open('../input/automatic-ticket-classification-data/complaints-2021-05-14_08_16.json')
data = json.load(f)
df = pd.json_normalize(data)

df.shape

list(df.columns)

df.isna().sum(0)

df.columns = [re.sub('^_', '', col) for col in df.columns]
list(df.columns)

df.columns = [re.sub('^\\bsource\\b\\.', '', col) for col in df.columns]
list(df.columns)

df['complaint_what_happened'].replace('', np.nan, inplace=True)
df.dropna(subset=['complaint_what_happened'], inplace=True)
len(df[df['complaint_what_happened'] == ''])

df.shape

def clean_text(text):
    """This function 
        - makes the given text lowercase
        - removes text in square brackets
        - removes punctuation and 
        - removes words containing numbers.
    :param text: text to be cleaned
    :return: cleaned text
    """
    text = text.lower()
    text = re.sub('\\[.*?\\]', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\\w*\\d\\w*', '', text)
    return text

df_clean = pd.DataFrame(df['complaint_what_happened'].apply(lambda x: clean_text(x)))
df_clean.head()