<file_sep>import requests as rq
import pandas as pd

## Dados sobre quedas de meteoritos ##
jsondata = rq.get('https://data.nasa.gov/resource/y77d-th95.json').json()
print(jsondata)
df = pd.DataFrame(jsondata)

df.mass[1]
df.mass = pd.to_numeric(df.mass)
df.mass[1]

type(df.mass[0])<file_sep>#import os
#os.chdir('C:/Users/Victor/data_science_projects/metallica_lyrics/codes')
import pandas as pd
import unicodecsv 
import stringAux as straux

#############################################
### Part 1 - Creating a metallica subset ####
#############################################
with open('../datasets/songdata.csv', 'rb') as f:
    reader = unicodecsv.DictReader(f)
    lyrics_list = list(reader)
    
lyrics_df = pd.DataFrame(lyrics_list)
lyrics_df = lyrics_df.drop(columns = 'link') # this column won't be necessary
# the following function returns a dataframe provided that the artist is metallica
metallica_df = lyrics_df.loc[lyrics_df['artist'] == 'Metallica']

###########################################
### Part 2 - Storing all words in a set ###
###########################################
metallica_list = list(metallica_df['text'])
all_words = straux.generate_string_list(metallica_list)

###########################################
### Part 3 - Removing clutter from data ###
###########################################
all_words = straux.remove_verses(all_words)
all_words = straux.remove_chorus(all_words)
all_words = straux.clean_strings(all_words)
all_words = straux.remove_empty_elements(all_words)

###########################################
### Part 4 - Creating cleaned dataframe ###
###########################################
words_df = pd.DataFrame(all_words).rename(columns = {0: 'word'})
words_df = words_df['word'].value_counts()
most_frequent_words = words_df.iloc[0:50]

##############################################
### Part 5 - Plotting and configuring plot ###
##############################################
words_plot = most_frequent_words.plot.barh(x='Index', y='word', 
                                           sort_columns=True, figsize=(12,8), 
                                           title='50 most frequent words in examined Metallica\'s lyrics')
words_plot.set(xlabel='Count', ylabel='Words')
words_plot.invert_yaxis()

# The following for loop is to put the value aside each bar for clearer visualization 
# https://stackoverflow.com/questions/23591254/python-pandas-matplotlib-annotating-labels-above-bar-chart-columns
# http://robertmitchellv.com/blog-bar-chart-annotations-pandas-mpl.html
for i in words_plot.patches:
    # the +0.6 in the y position is to attempt to centralize it (it's +0.6 because the y axis is inverted)
    words_plot.annotate(str(i.get_width()), xy=(i.get_width(), i.get_y()+0.6), fontsize=8)

#######################################
### Part 6 - Exporting plot as pdf ###
#######################################
fig = words_plot.get_figure()
fig.savefig('../plot.pdf')<file_sep># -*- coding: utf-8 -*-
import pandas as pd

mat = pd.read_csv('data_science_projects/bootstrap sampling/student-mat.csv')
#por = pd.read_csv('data_science_projects/bootstrap sampling/student-por.csv')
mat = mat.drop(columns=['school', 'address', 'reason', 'nursery', 'famsize', 
                        'Pstatus', 'Mjob', 'Fjob', 'traveltime', 'schoolsup',
                        'famsup', 'guardian', 'paid', 'activities', 'higher',
                        'internet', 'romantic', 'famrel', 'freetime', 'goout',
                        'health'])

bs_sample = pd.DataFrame()
N = 1000 # tamanho da minha amostra bs
for i in range(0, N):
    bs_sample = bs_sample.append(mat.sample(replace=True))

#mat.describe()
#bs_sample.describe()
mat.mean()
bs_sample.mean()<file_sep># data_science_projects
Repository with some data science projects that I'm currently working on for learning purposes. 
<file_sep># -*- coding: utf-8 -*-
import plotly.graph_objs as go
import plotly.offline

tracebr = go.Bar(
    x=['1980', '2017'],
    y=[0.235, 2.056],
    name='Brasil',
    marker=dict(
        color='rgb(40,200,40)'),
)
traceus = go.Bar(
    x=['1980', '2017'],
    y=[2.768, 19.390],
    name='EUA',
    marker=dict(
        color='rgb(40,40,200)'),
)

tracech = go.Bar(
    x=['1980', '2017'],
    y=[0.189, 12.2400],
    name='China',
    marker=dict(
        color='rgb(200,40,40)'),
)

data = [tracebr, traceus, tracech]
layout = go.Layout(
    barmode='group',
    title='PIB em trilhões (1980 x 2017)'
)

fig = go.Figure(data=data, layout=layout)
plotly.offline.plot(fig, filename='bar.html')
<file_sep># -*- coding: utf-8 -*-
"""
Created on Sat Dec  1 16:20:58 2018

@author: Victor
"""
# This function the elements of a list on another list
def append_elements_on_a_list(list, updated_list):
    for element in list:
        updated_list.append(element.lower()) # All words must be preferably lowercase
    return updated_list

# This function receives a list of strings and returns ALL words (lowercased)
# from across all strings in the list of strings and returns all of these 
# words in a single list
def generate_string_list(list_of_strings):
    string_list = []
    for string in list_of_strings:
        # I need this temporary list to store the substrings from the current
        # string in the iteration
        temp_list = string.split()
        string_list = append_elements_on_a_list(temp_list, string_list)
    return string_list

def clean_strings(list):
    i = 0
    for string in list:
        list[i] = string.strip(',"!.?()\'[]1234567890-:;')
        i += 1
    return list

def remove_chorus(list):
    while('[chorus]' in list):
        list.remove('[chorus]')
    return list

def remove_verses(list):
    i = 0
    for string in list:
        if string.startswith('[v'):
            del list[i]
        elif string.startswith('(v'):
            del list[i]
        i += 1
    return list

def remove_empty_elements(list):
    i = 0
    for string in list:
        if not string:
            del list[i]
        i += 1
    return list<file_sep># -*- coding: utf-8 -*-
import pandas as pd
suicide_rates = pd.read_csv('data_science_projects/brazil_suicides_plot/master.csv')
br_rates = suicide_rates[suicide_rates['country']=='Brazil'][suicide_rates['sex'] 
                        == 'male'][suicide_rates['age']=='15-24 years']

br_rates.year = pd.to_datetime(br_rates.year, format='%Y')
br_rates.plot(x='year', y='suicides/100k pop')<file_sep># -*- coding: utf-8 -*-
import plotly.figure_factory as ff
import pandas as pd
import plotly.offline


df = pd.read_csv('C:/Users/Victor/data_science_projects/plotly/band_members.csv')

colors = dict(Vocals = 'rgb(190, 30, 30)',
              Keyboards = 'rgb(30, 190, 30)',
              Drums = 'rgb(30, 30, 190)',
              Bass = 'rgb(190, 30, 190)', 
              Guitar = 'rgb(190, 190, 30)')


df.Start = pd.to_datetime(df.Start, format='%Y')
df.Finish = pd.to_datetime(df.Finish, format='%Y')

df = df.rename(columns={"Name": "Task"})

fig = ff.create_gantt(df, colors=colors, index_col='Role', show_colorbar=True,
                      bar_width=0.4, showgrid_x=True, showgrid_y=True, 
                      title='Membros Blind Guardian (1984-2019)')

plotly.offline.plot(fig, filename='gantt.html')
<file_sep>import pandas as pd
from pandasql import sqldf as psql
import sqlite3 as lite

## USANDO UM PANDAS DF COMO UMA TABLE ##
data = {'funcionario': ['Severino', 'Benedito', 'Aparecida', 'Marlene', 'Rosa',
                        'Sebastião', 'Valentina', '<NAME>', 'Rita'], 
     'salario': [900, 1100, 840, 1250, 980, 1400, 925, 760, 1900]}
Salaries = pd.DataFrame(data=data)

psql("SELECT * FROM Salaries").head()


## AGORA CRIANDO UMA TABELA E EXPORTANDO PARA UM PANDAS DF
con = lite.connect('test.db')
cur = con.cursor()
cur.execute("CREATE TABLE music(album TEXT, artista TEXT, lançamento TEXT)")
cur.execute("INSERT INTO music VALUES('Walls of Jericho', 'Helloween', '1985')")
cur.execute("INSERT INTO music VALUES('Visions', 'Stratovarius', '1997')")
cur.execute("INSERT INTO music VALUES('Once', 'Nightwish', '2004')")

albums_df = pd.read_sql_query("SELECT * FROM music", con)<file_sep># -*- coding: utf-8 -*-
import xarray as xr
#Além do xarray, é necessário instalar netcdf4

# International Comprehensive Ocean-Atmosphere Data Set da década de 1700
# from https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.nodc:ICOADS-netCDF
ds = xr.open_dataset('../nc_file/ICOADS_R3.0.0_1700-09.nc')
df = ds.to_dataframe()