text_plain_output_1.png
from bs4 import BeautifulSoup

from html import unescape

import csv

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

import csv
import gc
from pathlib import Path
columns = ['tweetcreatedts', 'extractedts', 'userid', 'tweetid', 'text', 'language']
dataframe_collection = []
csvfile = '/kaggle/input/ukraine-russian-crisis-twitter-dataset-1-2-m-rows/UkraineCombinedTweetsDeduped_MAR14.csv.gzip'
df = pd.read_csv(csvfile, compression='gzip', index_col=0, encoding='utf-8', quoting=csv.QUOTE_ALL)
df = df[columns]

df = df[df['language'] == 'en']
df.reset_index(drop=True, inplace=True)
df.shape

import re
from bs4 import BeautifulSoup
from html import unescape
def remove_urls(x):
    cleaned_string = re.sub('(https|http)?:\\/\\/(\\w|\\.|\\/|\\?|\\=|\\&|\\%)*\\b', '', str(x), flags=re.MULTILINE)
    return cleaned_string

def unescape_stuff(x):
    soup = BeautifulSoup(unescape(x), 'lxml')
    return soup.text

def deEmojify(x):
    regrex_pattern = re.compile(pattern='[😀-🙏🌀-🗿🚀-\U0001f6ff\U0001f1e0-🇿]+', flags=re.UNICODE)
    return regrex_pattern.sub('', x)

def remove_symbols(x):
    cleaned_string = re.sub('[^a-zA-Z0-9]+', ' ', x)
    return cleaned_string

def unify_whitespaces(x):
    cleaned_string = re.sub(' +', ' ', x)
    return cleaned_string

import swifter
df['text'] = df['text'].swifter.apply(remove_urls)
df['text'] = df['text'].swifter.apply(unescape_stuff)
df['text'] = df['text'].swifter.apply(deEmojify)
df['text'] = df['text'].swifter.apply(remove_symbols)
df['text'] = df['text'].swifter.apply(unify_whitespaces)