text_html_output_1.png
from plotly.subplots import make_subplots

import pandas as pd

import plotly.express as px

import plotly.graph_objects as go

data_2017 = pd.read_csv('../input/kaggle-survey-2017/multipleChoiceResponses.csv', encoding='ISO-8859-1', low_memory=False)
data_2018 = pd.read_csv('../input/kaggle-survey-2018/multipleChoiceResponses.csv', low_memory=False)
data_2019 = pd.read_csv('../input/kaggle-survey-2019/multiple_choice_responses.csv', low_memory=False)
data_2020 = pd.read_csv('../input/kaggle-survey-2020/kaggle_survey_2020_responses.csv', low_memory=False)

def consolidate_country_2017_to_2020(x):
    if (x == 'United States') | (x == 'United States of America'):
        return 'USA'
    elif (x == 'United Kingdom of Great Britain and Northern Ireland') | (x == 'United Kingdom'):
        return 'UK'
    elif x == "People 's Republic of China":
        return 'China'
    elif x == 'Iran, Islamic Republic of...':
        return 'Iran'
    else:
        return x

def consolidate_salary_2017(x):
    try:
        x = int(x.replace(',', ''))
        if x < 30000:
            return '0-30k'
        if x < 50000:
            return '30-50k'
        elif x < 100000:
            return '50-100k'
        elif x < 150000:
            return '100-150k'
        elif x < 200000:
            return '150-200k'
        elif x < 250000:
            return '200-250k'
        elif x < 500000:
            return '250-500k'
        else:
            return '$500k+'
    except:
        return 'No compensation disclosed'
def consolidate_salary_2018(x):
    if (x == '0-10,000') | (x == '10-20,000') | (x == '20-30,000'):
        return '0-30k'
    elif (x == '30-40,000') | (x == '40-50,000'):
        return '30-50k'
    elif (x == '50-60,000') | (x == '60-70,000') | (x == '70-80,000') | (x == '80-90,000') | (x == '90-100,000'):
        return '50-100k'
    elif (x == '100-125,000') | (x == '125-150,000'):
        return '100-150k'
    elif x == '150-200,000':
        return '150-200k'
    elif x == '200-250,000':
        return '200-250k'
    elif (x == '250-300,000') | (x == '300-400,000') | (x == '400-500,000'):
        return '250-500k'
    elif x == '500,000+':
        return '$500k+'
    else:
        return 'No compensation disclosed'
def consolidate_salary_2019_2020(x):
    if (x == '$0-999') | (x == '1,000-1,999') | (x == '2,000-2,999') | (x == '3,000-3,999') | (x == '4,000-4,999') | (x == '5,000-7,499') | (x == '7,500-9,999') | (x == '10,000-14,999') | (x == '15,000-19,999') | (x == '20,000-24,999') | (x == '25,000-29,999'):
        return '0-30k'
    elif (x == '30,000-39,999') | (x == '40,000-49,999'):
        return '30-50k'
    elif (x == '50,000-59,999  ') | (x == '60,000-69,999') | (x == '70,000-79,999') | (x == '80,000-89,999') | (x == '90,000-99,999'):
        return '50-100k'
    elif (x == '100,000-124,999') | (x == '125,000-149,999'):
        return '100-150k'
    elif x == '150,000-199,999':
        return '150-200k'
    elif x == '200,000-249,999':
        return '200-250k'
    elif (x == '250,000-299,999') | (x == '300,000-500,000'):
        return '250-500k'
    elif x == '> $500,000':
        return '$500k+'
    else:
        return 'No compensation disclosed'

def consolidate_sex_2017_to_2020(x):
    if (x == 'Prefer not to say ') | (x == 'Prefer to self-describe') | (x == 'Nonbinary') | (x == 'A different identity') | (x == 'Prefer not to say') | (x == 'Non-binary, genderqueer, or gender non-conforming'):
        return 'Other'
    elif x == 'Man':
        return 'Male'
    elif x == 'Woman':
        return 'Female'
    else:
        return x

def consolidate_job_titles_2017_to_2020(x):
    if (x == 'Data Scientist') | (x == 'Machine Learning Engineer') | (x == 'Predictive Modeler'):
        return 'Data science/ML engineers'
    elif (x == 'Software Engineer') | (x == 'Software Developer/Software Engineer') | (x == 'Programmer') | (x == 'Developer Advocate') | (x == 'Computer Scientist'):
        return 'Software developers'
    elif (x == 'Data Analyst') | (x == 'Business Analyst') | (x == 'Data Miner') | (x == 'Marketing Analyst') | (x == 'Data Journalist'):
        return 'Data/business analysts'
    elif (x == 'DBA/Database Engineer') | (x == 'Data Engineer') | (x == 'Data Miner') | (x == 'Marketing Analyst') | (x == 'Data Journalist'):
        return 'Data engineers'
    elif (x == 'Student') | (x == 'Researcher'):
        return 'Students'
    else:
        return 'Other job fields'

def consolidate_age_2017(x):
    if x < 25:
        return 'Under 25'
    elif x < 35:
        return '25-35'
    elif x < 45:
        return '35-45'
    elif x < 60:
        return '45-60'
    elif x < 70:
        return '60-69'
    else:
        return '70+'
def consolidate_age_2018_to_2020(x):
    if (x == '18-21') | (x == '22-24'):
        return 'Under 25'
    elif (x == '25-29') | (x == '30-34'):
        return '25-35'
    elif (x == '35-39') | (x == '40-44'):
        return '35-45'
    elif (x == '45-49') | (x == '50-54') | (x == '55-59'):
        return '45-60'
    else:
        return x

def consolidate_vizmodules_2018_to_2020(x):
    if x == 'MatplotlibSeaborn':
        return 'Matplotlib & Seaborn'
    elif (x == 'Plotly / Plotly Express') | (x == 'Plotly'):
        return 'Plotly Alone'
    elif (x == 'MatplotlibPlotly / Plotly Express') | (x == 'MatplotlibPlotly'):
        return 'Matplotlib & Plotly'
    elif (x == 'MatplotlibSeabornPlotly / Plotly ExpressBokeh') | (x == 'MatplotlibPlotly / Plotly ExpressBokehSeaborn') | (x == 'MatplotlibPlotlyBokehSeaborn'):
        return 'All 4 modules'
    elif x == 'Bokeh':
        return 'Bokeh Alone'
    elif x == 'MatplotlibBokeh':
        return 'Matplotlib & Bokeh'
    elif x == 'Seaborn':
        return 'Seaborn Alone'
    elif x == 'Matplotlib':
        return 'Matplotlib Alone'
    elif x == 'Shiny':
        return 'Shiny Alone'
    elif (x == 'Ggplot / ggplot2') | (x == 'ggplot2'):
        return 'Ggplot Alone'
    elif (x == 'Ggplot / ggplot2Shiny') | (x == 'ggplot2Shiny'):
        return 'Both Modules'
    else:
        return x

def consolidate_mlmodules_2018_to_2020(x):
    if (x == 'Scikit-learn') | (x == 'Scikit-Learn') | (x == 'Scikit-LearnrandomForest') | (x == 'Scikit-learnRandomForest') | (x == 'Scikit-learnCaret') | (x == 'randomForest') | (x == 'Caret') | (x == 'Scikit-learnRandomForestCaret') | (x == 'CaretrandomForest') | (x == 'RandomForest') | (x == 'Scikit-LearnCaret') | (x == 'RandomForestCaret') | (x == 'Scikit-LearnCaretrandomForest'):
        return 'Classic ML modules'
    elif (x == 'TensorFlow') | (x == 'TensorFlowKeras') | (x == 'PyTorch') | (x == 'TensorFlowKerasPyTorch') | (x == 'Keras') | (x == 'TensorFlowPyTorch') | (x == 'KerasPyTorch') | (x == 'TensorFlowKerasPyTorchFast.ai') | (x == 'TensorFlowKerasPyTorchFastai') | (x == 'PyTorchFast.ai') | (x == 'TensorFlowKerasFastai') | (x == 'Fast.ai') | (x == 'TensorFlowKerasFast.ai') | (x == 'KerasPyTorchFast.ai') | (x == 'TensorFlowPyTorchFast.ai') | (x == 'Fastai') | (x == 'PyTorchFastai') | (x == 'KerasFast.ai') | (x == 'TensorFlowPyTorchFastai') | (x == 'TensorFlowFast.ai ') | (x == 'KerasPyTorchFastai') | (x == 'TensorFlowFastai') | (x == 'KerasFastai') | (x == 'TensorFlowFast.ai') | (x == 'Xgboost') | (x == 'XgboostLightGBM') | (x == 'Xgboostlightgbm') | (x == 'Xgboostlightgbmcatboost') | (x == 'XgboostLightGBMCatBoost') | (x == 'LightGBM') | (x == 'Xgboostcatboost') | (x == 'XgboostCatBoost') | (x == 'lightgbm') | (x == 'CatBoost') | (x == 'catboost') | (x == 'LightGBMCatBoost') | (x == 'lightgbmcatboost') | (x == 'TensorFlowKerasXgboost') | (x == 'TensorFlowKerasPyTorchXgboost') | (x == 'TensorFlowKerasXgboostLightGBM') | (x == 'TensorFlowXgboost') | (x == 'KerasXgboost') | (x == 'TensorFlowKerasPyTorchXgboostLightGBM') | (x == 'TensorFlowKerasXgboostlightgbm') | (x == 'PyTorchXgboost') | (x == 'TensorFlowKerasXgboostlightgbmcatboost') | (x == 'KerasXgboostLightGBM') | (x == 'KerasPyTorchXgboost') | (x == 'KerasPyTorchXgboostLightGBM') | (x == 'PyTorchLightGBM') | (x == 'TensorFlowKerasPyTorchFast.aiXgboostLightGBM ') | (x == 'TensorFlowKerasPyTorchXgboostLightGBMCatBoost') | (x == 'PyTorchXgboostLightGBM') | (x == 'TensorFlowKerasXgboostLightGBMCatBoost') | (x == 'TensorFlowKerasPyTorchXgboostlightgbm') | (x == 'TensorFlowPyTorchXgboost') | (x == 'TensorFlowKerasPyTorchXgboostlightgbmcatboost'):
        return 'Deep Learning & Boosting modules'
    else:
        return x

# Merge all survey data from all years into one dataframe and unify responses as much as possible
consolidated_data =  pd.DataFrame()
year_2017 =  pd.DataFrame()
year_2018 =  pd.DataFrame()
year_2019 =  pd.DataFrame()
year_2020 =  pd.DataFrame()

#'17
year_2017['Salary'] = data_2017.CompensationAmount.apply(lambda x: consolidate_salary_2017(x))
year_2017['Country'] = data_2017.Country.apply(lambda x: consolidate_country_2017_to_2020(x))
year_2017['Age'] = data_2017.Age.drop(0).apply(lambda x: consolidate_age_2017(x))
year_2017['Job_field'] = data_2017.CurrentJobTitleSelect.drop(0).apply(lambda x: consolidate_job_titles_2017_to_2020(x))
year_2017['Sex'] = data_2017.GenderSelect.drop(0).apply(lambda x: consolidate_sex_2017_to_2020(x))
year_2017['Year'] = 2017

#'18
year_2018['Salary'] = data_2018.Q9.apply(lambda x: consolidate_salary_2018(x))
year_2018['Country'] = data_2018.Q3.drop(0).apply(lambda x: consolidate_country_2017_to_2020(x))
year_2018['Age'] = data_2018.Q2.drop(0).replace({'70-79' : '70+', '80+' : '70+'})
year_2018['Age'] = year_2018.Age.apply(lambda x: consolidate_age_2018_to_2020(x))
year_2018['Job_field'] = data_2018.Q6.drop(0).apply(lambda x: consolidate_job_titles_2017_to_2020(x))
year_2018['Sex'] = data_2018.Q1.drop(0).apply(lambda x: consolidate_sex_2017_to_2020(x))
year_2018['Visual_modules'] = (data_2018['Q21_Part_1'].fillna('').drop(0).str.strip(' ') + data_2018['Q21_Part_2'].fillna('').drop(0).str.strip(' ') + data_2018['Q21_Part_3'].fillna('').drop(0).str.strip(' ') + data_2018['Q21_Part_4'].fillna('').drop(0).str.strip(' ') 
                                  + data_2018['Q21_Part_5'].fillna('').drop(0).str.strip(' ') + data_2018['Q21_Part_6'].fillna('').drop(0).str.strip(' ') + data_2018['Q21_Part_7'].fillna('').drop(0).str.strip(' ') + data_2018['Q21_Part_8'].fillna('').drop(0).str.strip(' ') 
                                  + data_2018['Q21_Part_9'].fillna('').drop(0).str.strip(' ') + data_2018['Q21_Part_10'].fillna('').drop(0).str.strip(' ') + data_2018['Q21_Part_11'].fillna('').drop(0).str.strip(' ') + data_2018['Q21_Part_12'].fillna('').drop(0).str.strip(' ') 
                                  + data_2018['Q21_Part_9'].fillna('').drop(0).str.strip(' '))
year_2018['Visual_modules'] = year_2018.Visual_modules.apply(lambda x: consolidate_vizmodules_2018_to_2020(x))
year_2018['ML_modules'] = (data_2018['Q19_Part_1'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_2'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_3'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_4'].fillna('').drop(0).str.strip(' ')
                                 + data_2018['Q19_Part_5'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_6'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_7'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_8'].fillna('').drop(0).str.strip(' ')
                                 + data_2018['Q19_Part_9'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_10'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_11'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_12'].fillna('').drop(0).str.strip(' ')
                                 + data_2018['Q19_Part_13'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_14'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_15'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_16'].fillna('').drop(0).str.strip(' ')
                                 + data_2018['Q19_Part_17'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_18'].fillna('').drop(0).str.strip(' ') + data_2018['Q19_Part_19'].fillna('').drop(0).str.strip(' '))
year_2018['ML_modules'] = year_2018.ML_modules.apply(lambda x: consolidate_mlmodules_2018_to_2020(x))
year_2018['Year'] = 2018 

#'19
year_2019['Salary'] = data_2019.Q10.apply(lambda x: consolidate_salary_2019_2020(x))
year_2019['Country'] = data_2019.Q3.drop(0).apply(lambda x: consolidate_country_2017_to_2020(x))
year_2019['Age'] = data_2019.Q1.drop(0)
year_2019['Age'] = year_2019.Age.apply(lambda x: consolidate_age_2018_to_2020(x))
year_2019['Job_field'] = data_2019.Q5.drop(0).apply(lambda x: consolidate_job_titles_2017_to_2020(x))
year_2019['Sex'] = data_2019.Q2.drop(0).apply(lambda x: consolidate_sex_2017_to_2020(x))
year_2019['Visual_modules'] = (data_2019['Q20_Part_1'].fillna('').drop(0).str.strip(' ') + data_2019['Q20_Part_2'].fillna('').drop(0).str.strip(' ') + data_2019['Q20_Part_3'].fillna('').drop(0).str.strip(' ') + data_2019['Q20_Part_4'].fillna('').drop(0).str.strip(' ') 
                                  + data_2019['Q20_Part_5'].fillna('').drop(0).str.strip(' ') + data_2019['Q20_Part_6'].fillna('').drop(0).str.strip(' ') + data_2019['Q20_Part_7'].fillna('').drop(0).str.strip(' ') + data_2019['Q20_Part_8'].fillna('').drop(0).str.strip(' ') 
                                  + data_2019['Q20_Part_9'].fillna('').drop(0).str.strip(' ') + data_2019['Q20_Part_10'].fillna('').drop(0).str.strip(' ') + data_2019['Q20_Part_11'].fillna('').drop(0).str.strip(' ') + data_2019['Q20_Part_12'].fillna('').drop(0).str.strip(' '))
year_2019['Visual_modules'] = year_2019.Visual_modules.apply(lambda x: consolidate_vizmodules_2018_to_2020(x))
year_2019['ML_modules'] = (data_2019['Q28_Part_1'].fillna('').drop(0).str.strip(' ') + data_2019['Q28_Part_2'].fillna('').drop(0).str.strip(' ') + data_2019['Q28_Part_3'].fillna('').drop(0).str.strip(' ') + data_2019['Q28_Part_4'].fillna('').drop(0).str.strip(' ')
                                 + data_2019['Q28_Part_5'].fillna('').drop(0).str.strip(' ') + data_2019['Q28_Part_6'].fillna('').drop(0).str.strip(' ') + data_2019['Q28_Part_7'].fillna('').drop(0).str.strip(' ') + data_2019['Q28_Part_8'].fillna('').drop(0).str.strip(' ')
                                 + data_2019['Q28_Part_9'].fillna('').drop(0).str.strip(' ') + data_2019['Q28_Part_10'].fillna('').drop(0).str.strip(' ') + data_2019['Q28_Part_11'].fillna('').drop(0).str.strip(' ') + data_2019['Q28_Part_12'].fillna('').drop(0).str.strip(' '))
year_2019['ML_modules'] = year_2019.ML_modules.apply(lambda x: consolidate_mlmodules_2018_to_2020(x))
year_2019['Year'] = 2019 

#'20 
year_2020['Salary'] = data_2020.Q24.apply(lambda x: consolidate_salary_2019_2020(x))
year_2020['Country'] = data_2020.Q3.drop(0).apply(lambda x: consolidate_country_2017_to_2020(x))
year_2020['Age'] = data_2020.Q1.drop(0)
year_2020['Age'] = year_2020.Age.apply(lambda x: consolidate_age_2018_to_2020(x))
year_2020['Job_field'] = data_2020.Q5.drop(0).apply(lambda x: consolidate_job_titles_2017_to_2020(x))
year_2020['Sex'] = data_2020.Q2.drop(0).apply(lambda x: consolidate_sex_2017_to_2020(x))
year_2020['Visual_modules'] = (data_2020['Q14_Part_1'].fillna('').drop(0).str.strip(' ') + data_2020['Q14_Part_2'].fillna('').drop(0).str.strip(' ') + data_2020['Q14_Part_3'].fillna('').drop(0).str.strip(' ') + data_2020['Q14_Part_4'].fillna('').drop(0).str.strip(' ')
                              + data_2020['Q14_Part_5'].fillna('').drop(0).str.strip(' ') + data_2020['Q14_Part_6'].fillna('').drop(0).str.strip(' ') + data_2020['Q14_Part_7'].fillna('').drop(0).str.strip(' ') + data_2020['Q14_Part_8'].fillna('').drop(0).str.strip(' ')
                              + data_2020['Q14_Part_9'].fillna('').drop(0).str.strip(' ') + data_2020['Q14_Part_10'].fillna('').drop(0).str.strip(' ') + data_2020['Q14_Part_11'].fillna('').drop(0).str.strip(' ') + data_2020['Q14_OTHER'].fillna('').drop(0).str.strip(' '))
year_2020['Visual_modules'] = year_2020.Visual_modules.apply(lambda x: consolidate_vizmodules_2018_to_2020(x))
year_2020['ML_modules'] = (data_2020['Q16_Part_1'].fillna('').drop(0).str.strip(' ') + data_2020['Q16_Part_2'].fillna('').drop(0).str.strip(' ') + data_2020['Q16_Part_3'].fillna('').drop(0).str.strip(' ') + data_2020['Q16_Part_4'].fillna('').drop(0).str.strip(' ')
                                 + data_2020['Q16_Part_5'].fillna('').drop(0).str.strip(' ') + data_2020['Q16_Part_6'].fillna('').drop(0).str.strip(' ') + data_2020['Q16_Part_7'].fillna('').drop(0).str.strip(' ') + data_2020['Q16_Part_8'].fillna('').drop(0).str.strip(' ') 
                                 + data_2020['Q16_Part_9'].fillna('').drop(0).str.strip(' ') + data_2020['Q16_Part_10'].fillna('').drop(0).str.strip(' ') + data_2020['Q16_Part_11'].fillna('').drop(0).str.strip(' ') + data_2020['Q16_Part_12'].fillna('').drop(0).str.strip(' ') 
                                 + data_2020['Q16_Part_13'].fillna('').drop(0).str.strip(' ') + data_2020['Q16_Part_14'].fillna('').drop(0).str.strip(' ') + data_2020['Q16_Part_15'].fillna('').drop(0).str.strip(' ') + data_2020['Q16_OTHER'].fillna('').drop(0).str.strip(' '))
year_2020['ML_modules'] = year_2020.ML_modules.apply(lambda x: consolidate_mlmodules_2018_to_2020(x))
year_2020['Year'] = 2020

#consolidating the dataframe
consolidated_data = year_2017
consolidated_data = consolidated_data.append(year_2018)
consolidated_data = consolidated_data.append(year_2019)
consolidated_data = consolidated_data.append(year_2020)

consolidated_data = consolidated_data.reset_index(drop=True)

#making subplots for top 10 countries on total number of respondents

sub_2017 = year_2017['Country'].value_counts().sort_values(ascending = False).head(10).reset_index().round(1)
sub_2018 = year_2018['Country'].value_counts().sort_values(ascending = False).head(10).reset_index().round(1)
sub_2019 = year_2019['Country'].value_counts().sort_values(ascending = False).head(10).reset_index().round(1)
sub_2020 = year_2020['Country'].value_counts().sort_values(ascending = False).head(10).reset_index().round(1)

fig = make_subplots(
    rows=2, cols=2,
    subplot_titles=("2017", "2018", "2019", "2020"))

fig.add_trace(go.Bar(x=sub_2017['index'], y=sub_2017['Country']),
              row=1, col=1)
              
fig.add_trace(go.Bar(x=sub_2018['index'], y=sub_2018['Country']),
              row=1, col=2)

fig.add_trace(go.Bar(x=sub_2019['index'], y=sub_2019['Country']),
              row=2, col=1)

fig.add_trace(go.Bar(x=sub_2020['index'], y=sub_2020['Country']),
              row=2, col=2)

fig.update_layout(
                  title_text="Top 10 countries 2017-20 (total number of respondents)", title_x=0.5)

fig.update_layout(showlegend=False)
fig.show()

animate_countries = consolidated_data.groupby(['Year', 'Country']).size().reset_index()
animate_countries['Percentage'] = consolidated_data.groupby(['Year', 'Country']).size().groupby(level=0).apply(lambda x: 100 * x / float(x.sum())).values.round(2)
animate_countries.columns = ['Year', 'Country', 'Counts', 'Percentage']
animate_countries = animate_countries.loc[(animate_countries['Country'] == 'USA') | (animate_countries['Country'] == 'India')]
fig = px.bar(animate_countries, x='Country', y='Percentage', color='Country', animation_frame='Year', range_y=[0, 50], text=animate_countries['Percentage'].apply(lambda x: '{0:1.2f}%'.format(x)))
fig.layout.updatemenus[0].buttons[0].args[1]['frame']['duration'] = 2200
fig.update_layout(title_text='India vs USA (2017-2020)', title_x=0.5)
fig.update_yaxes(title='Portion of Respondents (%)')
fig.update_xaxes(title='')
fig.update_traces(marker_line_color='rgb(31, 27, 25)', marker_line_width=1.2, opacity=1)
fig.update_layout(showlegend=False)
fig.show()