# encoding=utf8
import sys

reload(sys)
sys.setdefaultencoding('utf8')



import requests
import json


import time
from random import randint
import pandas as pd
import os.path
import sys
import codecs
import numpy as np
import re
import urllib2

print sys.stdout.encoding


class GoogleTrendCrawler:

    def __init__(self, path, startyear):
        self.startyear = startyear

        if not os.path.exists(path):
            os.mkdir(path)
        self.logfilename = path+"log-fails.txt"

        self.html_base = u"http://www.google.com/trends/fetchComponent?q="

        self.query_type = u"&cid=TIMESERIES_GRAPH_0&export=3"
        self.path = path
        self.headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}



    def create_sample(self, N, allpeople, path):
        # sample 100 people
        allpeople.reset_index(inplace=True)
        people = allpeople.ix[np.random.random_integers(0, len(allpeople), N)]
        people.to_csv(path+"selected_people.csv")
        return people

    def create_stratified_sample(self, N, allpeople, path):
        women = allpeople[allpeople["gender"] == "female"]
        women.reset_index(inplace=True)
        men = allpeople[allpeople["gender"] == "male"]
        men.reset_index(inplace=True)
        print "women and men shape: "
        print women.shape
        print men.shape
        women = women.ix[np.random.random_integers(0, len(women), N)]
        #women.dropnan(inplace=True, how=any)
        men = men.ix[np.random.random_integers(0, len(men), N)]