text_plain_output_1.png
from langdetect import detect

import pandas as pd

npis_csv = '/kaggle/input/covid19-challenges/npi_canada.csv'
raw_data = pd.read_csv(npis_csv, encoding='ISO-8859-1')
df = raw_data.dropna(how='any', subset=['start_date', 'region', 'intervention_category'])
df['region'] = df['region'].replace('Newfoundland', 'Newfoundland and Labrador')
num_rows_removed = len(raw_data) - len(df)
regions = list(set(df.region.values))
num_cats = list(set(df.intervention_category.values))
num_interventions = len(num_cats)
df['start_date'] = pd.to_datetime(df['start_date'], format='%Y-%m-%d')
earliest_start_date = df['start_date'].min()
latest_start_date = df['start_date'].max()
num_days = latest_start_date - earliest_start_date

merged_tweets_csv = '/kaggle/input/npi-twitterverse-april-30/tweets_to_intervention_category.source_urls.tsv'
colnames = ['npi_record_id', 'intervention_category', 'oxford_government_response_category', 'source_url', 'id', 'conversation_id', 'created_at', 'date', 'time', 'timezone', 'user_id', 'username', 'name', 'place', 'tweet', 'mentions', 'urls', 'photos', 'replies_count', 'retweets_count', 'likes_count', 'hashtags', 'cashtags', 'link', 'retweet', 'quote_url', 'video', 'near', 'geo', 'source', 'user_rt_id', 'user_rt', 'retweet_id', 'reply_to', 'retweet_date', 'translate', 'trans_src', 'trans_dest']
tweets_df = pd.read_csv(merged_tweets_csv, encoding='utf-8', error_bad_lines=False, engine='python', names=colnames)
tweets_df = tweets_df.dropna(how='any', subset=['npi_record_id', 'intervention_category', 'tweet'])
data = []
for index, row in tweets_df.iterrows():
    tweet = row['tweet'].strip()
    if tweet != '':
        language = ''
        try:
            language = detect(tweet)
        except:
            language = 'error'
        if language == 'en':
            data.append([row['intervention_category'], tweet])
tweets_df_en = pd.DataFrame(data, columns=['intervention_category', 'tweet'])
print('Number of non-english tweets = {}'.format(len(tweets_df) - len(tweets_df_en)))
print('Number of tweets collected = {}'.format(len(tweets_df_en)))