import sys, os, requests, random, string, json, locale, gc, socket, subprocess
from time import time as tm, sleep as slp
from itertools import cycle

import redis
from pyquery import PyQuery as pq_

from cities import countries
from configs import configs
from parse_profile_details import indeed_resumes_details
from db_access import db_insert_hash, get_distincts, id_exists

from user_agents import user_agents



data_dir = '../../data/'
locale.setlocale( locale.LC_ALL, 'en_US.UTF-8' ) 
skillset = configs['skills_file'] #--change this to change the filename

class indeed_resumes(object):
	def __init__(self, country_code, master, index):
		self.country_code = country_code
		self.master = master
		self.index = int(index)
		self.keywords = open(skillset+'.json', 'rb')
		self.init_url = 'http://www.indeed.com/resumes?q=%s&co='+self.country_code+'&start=%d&limit=%d'
		self.fixed_test_url = 'http://www.indeed.com/resumes?q=excel&co='+self.country_code
		self.url_ = 'http://www.indeed.com/resumes%s'
		self.user_agents_cycle = cycle(user_agents)
		self.max_recursion_depth = 7
		self.time_all = []

	def init_redis(self):
		if not self.r_host.get('all_count'):
			self.r_host.set('all_count', 0)
		return

	def save_to_disk(self, data, unique_id):
		if not id_exists(unique_id):
			directory = '../../data/resumes/%s' % unique_id[0:2]
			if not os.path.exists(directory):
				os.makedirs(directory)
			filename = '%s/%s.json' % (directory, unique_id)
			f = open(filename, 'wb')
			f.write(json.dumps(data))
			f.close()
		return

	def resource_collection(self, keyword_index, keyword, sort, rest_kewords=False):
		start_time = tm()
		n_all = 0
		n_profiles = {}
		keyword = '%s' % keyword.replace('/', ' ')
		keyword = keyword.strip('\n')
		init_url = self.init_url % (keyword.replace(' ', '+'), 0, 50)
		
		filtering_urls, result_count  = self.get_filter_urls(init_url, 0)
		
		if result_count >= 1000:
			counter = 10
		else:
			counter = int(max(float(result_count)/100, 1))
		
		for route in filtering_urls:
			url_ = self.url_ % pq_(route).children('a').attr('href')
			for i in range(counter):
				if i == 0:
					beg = i
					end = i+100
				else:
					beg = end
					end = end+100
				postfix = '&start=%d&limit=%d&radius=100&%s&co=%s' % (beg, end, sort, self.country_code)	
				print url_+postfix	
				data = self.get_resource(url_+postfix, 0)

				for each in data:
					item = pq_(each)
					unique_id = item.attr('id')
					city_ = item('.location').text()
					n_profiles[unique_id] = city_
					profile_data = indeed_resumes_details(unique_id).resource_collection()
					self.save_to_disk(profile_data, unique_id)
					n_all += 1

			db_success = False
			while not db_success:
				try:
					db_insert_hash(n_profiles, self.country_code)
					db_success = True
				except:
					print 'db locked..will wait for 5 secs and try again..'
					slp(5)
					pass
			print 'inserted %d records to db.. %s, %d' % (len(n_profiles), keyword, keyword_index)	
			n_profiles = {}
			slp(0) #--sleeping for 2 secs for every filter for not making calls too fast and get blocked quickly
			gc.collect()
		gc.collect()
		current_time = tm()
		self.time_all.append((keyword, n_all, current_time - start_time))
		print 'current time passed..%d secs for one round of %s (%d)' % (int(current_time - begin_time), keyword, keyword_index)
		return
	

	def get_filter_urls(self, init_url, counter):
		if counter >= self.max_recursion_depth:
			print 'max recursion depth achieved in the get_filter_urls'
			#slp(300)
			return ([], 0)
		
		filtering_urls = []
		resp = None
		while not resp:
			try:
				user_agent = self.user_agents_cycle.next()
				resp = requests.get(init_url, headers = {'user_agent': user_agent})
			except Exception, e:
				print str(e), '###'
				slp(10)
				pass
		if resp.status_code == 200 or len(self.get_static_resource(self.fixed_test_url)):
			filtering_urls = pq_(resp.text)
			count =  filtering_urls('#search_header #rezsearch #search_table #result_count').text().split(' ')[0].replace(',', '')
			filtering_urls = filtering_urls('.refinement')
			if count.isdigit():
				count = int(count)
			else:
				count = 0
			return (filtering_urls, count)
		else:
			slp(1)
			return self.get_filter_urls(init_url, counter+1)

	def get_resource(self, url_, counter):
		if counter >= self.max_recursion_depth:
			print 'max recursion depth achieved in the get_resource'
			#slp(300)
			return []
		data = []
		resp = None
		while not resp:
			try:
				user_agent = self.user_agents_cycle.next()
				resp = requests.get(url_, headers = {'user_agent': user_agent})
			except Exception, e:
				print str(e), '@@@'
				slp(10)
				pass
		if resp.status_code == 200 or len(self.get_static_resource(self.fixed_test_url)):
			data = pq_(resp.text)
			data = data('#results').children()
			return data
		else:
			slp(1)
			return self.get_resource(url_, counter+1)

	def get_static_resource(self, url):
		data = []
		resp = None
		try:
			while not resp:
				try:
					user_agent = self.user_agents_cycle.next()
					resp = requests.get(url, headers = {'user_agent': user_agent})
				except Exception, e:
					print str(e), '!!!'
					slp(5)
					pass
			if resp.status_code == 200:
				data = pq_(resp.text)
				data = data('#results').children()