text_html_output_1.png
from sklearn.feature_extraction.text import CountVectorizer

from sklearn.model_selection import train_test_split

import lightgbm as lgbm

import numpy as np

import pandas as pd

train = pd.read_csv('../input/train.csv')
test = pd.read_csv('../input/test.csv')

train_ = train['comment_text']
test_ = test['comment_text']
alldata = pd.concat([train_, test_], axis=0)
alldata = pd.DataFrame(alldata)

alldata.comment_text.fillna('blllllllllllllllllllllllaaaaaaaaaaaaaaaahhhhhhhh...!!!', inplace=True)

countvec = CountVectorizer(max_features=1500, ngram_range=(1, 2))

countvecdata = countvec.fit_transform(alldata['comment_text'])

countvec_df = pd.DataFrame(countvecdata.todense())

countvec_df_train = countvecdata[:len(train_)]
countvec_df_test = countvecdata[len(train_):]

countvec_df_train_ = countvec_df_train.astype('float32')
countvec_df_test_ = countvec_df_test.astype('float32')

col = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
preds = np.zeros((test.shape[0], len(col)))

params = {'objective': 'binary', 'learning_rate': 0.02, 'num_leaves': 76, 'feature_fraction': 0.64, 'bagging_fraction': 0.8, 'bagging_freq': 1, 'boosting_type': 'gbdt', 'metric': 'binary_logloss'}

for i, j in enumerate(col):
    print('fitting column : ' + j)
    X_train, X_valid, Y_train, Y_valid = train_test_split(countvec_df_train_, train[j], random_state=7, test_size=0.33)
    d_train = lgbm.Dataset(X_train, Y_train)
    d_valid = lgbm.Dataset(X_valid, Y_valid)
    bst = lgbm.train(params, d_train, 5000, valid_sets=[d_valid], verbose_eval=50, early_stopping_rounds=100)
    print('predicting for :' + j)
    preds[:, i] = bst.predict(countvec_df_test_)
print('Fininshed Training')