

from . import helpers

import pandas as pd
import numpy as np
from pandas import DataFrame, Series

from pandas_datareader import data
from datetime import datetime, timedelta


import re
import os
import requests
import time






class StockInfo():
	def __init__(self, StockNumber):
		if isinstance(StockNumber, str) is False:
			print('StockNumber must be string')
			self.__StockNumber = '2330.TW'
			
		else:
			self.__StockNumber = StockNumber+'.TW'
	
	def get_StockNumber(self):
		return self.__StockNumber
		
	def fetch_StockPrice(self, StartTime, EndTime):
# 		self.__StockPrice = data.DataReader(self.__StockNumber, 
# 											'yahoo',StartTime, EndTime)
		self.__StockPrice = data.DataReader(self.__StockNumber, 
											'yahoo',StartTime, EndTime)
		
	def get_StockPrice(self):
		return self.__StockPrice
	
	def fetch_StockActions(self, StartTime, EndTime):
		self.__StockActions = data.DataReader(self.__StockNumber,
											'yahoo-actions',StartTime, EndTime)
	def get_StockActions(self):
		return self.__StockActions
		
		
class Crawler():
	def __init__(self, prefix='data'):
		if not os.path.isdir(prefix):
			os.mkdir(prefix)
		self.prefix = prefix
		# pass
	
	def get_tse_one_day(self, spec_date):
		date_str = '{0}{1:02d}{2:02d}'.format(spec_date.year, spec_date.month, spec_date.day)
		url = 'http://www.twse.com.tw/exchangeReport/MI_INDEX'

		query_params = {
			'date': date_str,
			'response': 'json',
			'type': 'ALL',
			'_': str(round(time.time() * 1000) - 500)
		}

		# Get json data
		page = requests.get(url, params=query_params)

		if not page.ok:
			logging.error("Can not get TSE data at {}".format(date_str))

		content = page.json()
		# print(content)
		# key = 'Nodata'
		isoffday = True
		for key in content.keys():
			if isinstance(content[key], list):
				if len(content[key][0]) == 16:
					isoffday = False
					break
		if isoffday:
			print('No data at this day %4d/%02d/%02d'% 
					(spec_date.year,spec_date.month, spec_date.day))
			return -1

		# For compatible with original data
		# date_str_mingguo = '{0}/{1:02d}/{2:02d}'.format(spec_date.year - 1911,\
		# 	spec_date.month, spec_date.day)

		data_df = DataFrame(data=content[key], 
							columns=['code','name','volume','transaction','turnover',
									 'open','high','low','close','UD','difference',
									 'last_buy', 'last_buy_volume',
									 'last_sell','last_sell_volume','PE_ratio'])

		data_df = data_df.applymap(lambda x: re.sub(",","",x))# clear comma
		data_df.replace({'UD':{'<p style= color:red>+</p>':'+',
							   '<p style= color:green>-</p>':'-'}},
						inplace=True)

		return data_df
	
	def get_otc_one_day(self, spec_date):
		date_str = '{0}/{1:02d}/{2:02d}'.format(spec_date.year-1911, spec_date.month, spec_date.day)
		
		ttime = str(int(time.time()*100))
		url = 'http://www.tpex.org.tw/web/stock/aftertrading/daily_close_quotes/stk_quote_result.php?l=zh-tw&d={}&_={}'.format(date_str, ttime)
		page = requests.get(url)

		if not page.ok:
			logging.error("Can not get OTC data at {}".format(date_str))
		# print(page.content)
		content = page.json()

		# print(content)
		# key = 'Nodata'
		if (len(content['aaData']) + len(content['mmData'])) == 0:
			print('No data at this day ' + date_str)
			return -1
		data_df = DataFrame(data=content['aaData'] + content['mmData'], 
							columns=['code','name','close','difference','open',
									 'high','low','avg','volume','turnover',
									 'transaction','last_buy',
									 'last_sell','NumOfShare','NextRefPrice',
									 'NextUpperPrice', 'NextLowerPrice'])

		data_df = data_df.applymap(lambda x: re.sub(",","",x))# clear comma
		
		return data_df

	def check_all_tse_data(self):
		Filelist = os.listdir(self.prefix)
		if 'offday.xlsx' in Filelist:
			offday_ser = pd.read_excel(self.prefix + '/offday.xlsx')
			offday_ser = offday_ser['date'].copy()
		else:
			offday_ser = Series(name='date', data='First')

		offday_update = False
		lastday_update = False

		Now = datetime.now()
		Nowdate = datetime(Now.year, Now.month, Now.day)

		if 'lastday.txt' in Filelist:
			with open(self.prefix + '/lastday.txt', 'r') as f:
				read_data = f.read()
				f.close()
				Startdate = datetime(int(read_data[0:4]), 
									int(read_data[4:6]), 
									int(read_data[6:8]))
		else:
			#Start from 2004(093)/02/11
			Startdate = datetime(2004, 2, 11)
		
		datediff = timedelta(days=1)
		
		while Startdate <= Nowdate:
			date_str = '{0}{1:02d}{2:02d}'.\
					format(Startdate.year-1911,Startdate.month, Startdate.day)
			print('Read ' + date_str)
			if ('%s.xlsx' %(date_str)) not in Filelist:# not in FileList
				if (offday_ser != date_str).all():# not a offday
					lastday_update = True
					data_df = self.get_tse_one_day(Startdate) # collect data
					if isinstance(data_df, DataFrame):# success
						data_df.to_excel('{0}/{1}.xlsx'.format(self.prefix,date_str))# save data
					else:# is an offday, update offday series
						offday_ser.set_value( len(offday_ser), date_str)
						offday_update = True
						print(date_str + 'is an offday')
				else:
					print(date_str + ' is known as an offday')
			else:
				print(date_str + ' is in FileList')
			Startdate = Startdate + datediff

		if offday_update:
			offday_ser.to_excel(self.prefix + '/offday.xlsx')

		if lastday_update:
			with open(self.prefix + '/lastday.txt', 'w') as f:
				# Nowdate += timedelta(days=-1)
				date_str = '{0}{1:02d}{2:02d}'.\
					format(Nowdate.year,Nowdate.month, Nowdate.day)
				f.write(date_str)
				f.close()

	def check_all_otc_data(self):
		Filelist = os.listdir(self.prefix)
		if 'offdayOTC.xlsx' in Filelist:
			offday_ser = pd.read_excel(self.prefix + '/offdayOTC.xlsx')
			offday_ser = offday_ser['date'].copy()
		else:
			offday_ser = Series(name='date', data='First')

		offday_update = False
		lastday_update = False

		Now = datetime.now()
		Nowdate = datetime(Now.year, Now.month, Now.day)

		if 'lastdayOTC.txt' in Filelist:
			with open(self.prefix + '/lastdayOTC.txt', 'r') as f:
				read_data = f.read()
				f.close()
				Startdate = datetime(int(read_data[0:4]), 
									int(read_data[4:6]), 
									int(read_data[6:8]))
		else:
			#Start from 2007(096)/04/23
			Startdate = datetime(2007, 4, 23)
			# Startdate = datetime(2008, 2, 28)
		
		datediff = timedelta(days=1)
		
		while Startdate <= Nowdate:
			date_str = '{0}{1:02d}{2:02d}'.\
					format(Startdate.year-1911,Startdate.month, Startdate.day)
			print('Read ' + date_str + ' OTC')
			if ('%sOTC.xlsx' %(date_str)) not in Filelist:# not in FileList
				if (offday_ser != date_str).all():# not a offday
					lastday_update = True
					
					time.sleep(np.random.random())

					data_df = self.get_otc_one_day(Startdate) # collect data
					if isinstance(data_df, DataFrame):# success
						data_df.to_excel('{0}/{1}OTC.xlsx'.format(self.prefix,date_str))# save data
					else:# is an offday, update offday series
						offday_ser.set_value( len(offday_ser), date_str)
						offday_update = True
						print(date_str + 'is an offday')
				else:
					print(date_str + ' is known as an offday')
			else:
				print(date_str + ' is in FileList')
			Startdate = Startdate + datediff

		if offday_update:
			offday_ser.to_excel(self.prefix + '/offdayOTC.xlsx')

		if lastday_update:
			with open(self.prefix + '/lastdayOTC.txt', 'w') as f:
				# Nowdate += timedelta(days=-1)
				date_str = '{0}{1:02d}{2:02d}'.\
					format(Nowdate.year,Nowdate.month, Nowdate.day)
				f.write(date_str)
				f.close()











	

