text_plain_output_1.png
from sklearn.preprocessing import LabelEncoder

from tqdm import tqdm

import nltk

import os

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import train_test_split
import lightgbm as lgb
import nltk
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
import joblib
from spacy.util import minibatch, compounding
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer
from nltk.tokenize import sent_tokenize
from nltk.sentiment import SentimentIntensityAnalyzer
import multiprocessing
from tqdm import tqdm
from sklearn.model_selection import KFold
le = LabelEncoder()
import os

df = pd.read_csv('/kaggle/input/WELFakeAdditional/final (1).csv')

def get_pos_counts(text):
    tokens = nltk.word_tokenize(text)
    pos_tags = nltk.pos_tag(tokens)
    pos_counts = {'NN': 0, 'VB': 0, 'JJ': 0}
    for word, tag in pos_tags:
        if tag.startswith('NN'):
            pos_counts['NN'] += 1
        elif tag.startswith('VB'):
            pos_counts['VB'] += 1
        elif tag.startswith('JJ'):
            pos_counts['JJ'] += 1
    return (pos_counts['NN'], pos_counts['VB'], pos_counts['JJ'])

df = pd.read_csv('/kaggle/input/WELFakeAdditional/final (1).csv')
df.drop('sentiment', axis=1, inplace=True)
df.drop('subject', axis=1, inplace=True)
df.drop('date', axis=1, inplace=True)
df.drop('subject.1', axis=1, inplace=True)
df.drop('date.1', axis=1, inplace=True)
df.dropna(inplace=True)
tqdm.pandas(desc='Processing title column')
df[['title_nouns', 'title_verbs', 'title_adjectives']] = df['title'].progress_apply(get_pos_counts).apply(pd.Series)
tqdm.pandas(desc='Processing text column')
df[['text_nouns', 'text_verbs', 'text_adjectives']] = df['text'].progress_apply(get_pos_counts).apply(pd.Series)
df.head()