import sys
import gzip
import time
import numpy
import json
import time
import logging
import operator
import config
import csv
import StringIO
import subprocess
import HTMLParser

#from py2neo import Graph, Path, Node, Relationship,authenticate
from scipy import stats

from django.shortcuts import render_to_response
from django.shortcuts import get_object_or_404, render
from browser.forms import (CreateSemSet,ComSearchSets,CreateSearchSet,CreatePubSet)
from django.http import HttpResponseRedirect, HttpResponse
from django.core.urlresolvers import reverse
from django.template import RequestContext
#from browser.medline_parser import *
from browser.tasks import *
from browser.models import SearchSet,Compare,Overlap,Filters
from django.template.defaulttags import register
#from django.template.context_processors import csrf
from django.shortcuts import redirect
from django.core.exceptions import ObjectDoesNotExist
from math import exp
from django_datatables_view.base_datatable_view import BaseDatatableView
from collections import defaultdict
from django.views.decorators.csrf import csrf_exempt
from django.core.cache import cache
from django.views.decorators.cache import cache_page
from django.core.serializers.json import DjangoJSONEncoder
from django.core import serializers
from sets import Set
from settings import DATA_FOLDER

#neo4j
from neo4j.v1 import GraphDatabase,basic_auth
auth_token = basic_auth(config.user, config.password)
driver = GraphDatabase.driver("bolt://"+config.server+":"+config.port,auth=auth_token)
#logger.debug(config.server)

#===============GoogleAuth Start
from django.core.urlresolvers import reverse
from django.contrib import messages
from django.views.generic.base import View
from social_auth.backends import AuthFailed
from social_auth.views import complete
from django.contrib.auth.decorators import login_required
from django.conf import settings

#rest API
from rest_framework import viewsets
from browser.serializers import SearchSetSerializer

#logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p',level=logging.WARNING)
#logger = logging.getLogger(__name__)
#logging.basicConfig(filename='run.log',level=logging.DEBUG)

class SearchSetViewSet(viewsets.ModelViewSet):
	#queryset = SearchSet.objects.all()
	serializer_class = SearchSetSerializer

	def get_queryset(self):
		user = self.request.user
		logger.debug('user.id = '+str(user.id))
		return SearchSet.objects.filter(user_id=str(user.id))
		#return SearchSet.objects.all()

class AuthComplete(View):
    def get(self, request, *args, **kwargs):
		logging.warning('error')
		backend = kwargs.pop('backend')
		try:
			return complete(request, backend, *args, **kwargs)
		except AuthFailed:
			logging.warning('error')
			messages.error(request, "Your Google Apps domain isn't authorized for this app")
			return HttpResponseRedirect(reverse('gauth_login'))

class LoginError(View):
    def get(self, request, *args, **kwargs):
        return HttpResponse(status=401)

#===============GoogleAuth End

@register.filter
def get_item(dictionary, key):
    return dictionary.get(key)
@register.filter
def mysplit(value, sep = "."):
    parts = value.split(sep)
    return (parts)

tmpDir=settings.MEDIA_ROOT

def people(request):
	#example query
	sem="match (s:SearchSet)--(p:Pubmed)--(st:SDB_triple)--(si:SDB_item) where s.name = 'tom gaunt_2' return count(distinct(p)) as c,si order by c;"
	mesh="match (s:SearchSet)--(p:Pubmed)--(m:Mesh) where s.name = 'tom gaunt_2' return count(distinct(p)) as c,m order by c;"


def ajax_graph_metrics(request):
	session = driver.session()
	logger.debug('getting graph metrics')
	uAll=SearchSet.objects.order_by().values_list('user_id',flat=True).exclude(user_id='2').distinct().count()
	if request.is_ajax():
		# get data for graph
		# only want search sets that I (user 2) haven't created
		#gCom = "match (s:SearchSet) where not s.name =~ '.*_2' return count(s) as s union match (s:Pubmed) return count(s) as s union match (s:Mesh) return count(s) as s union match (s:SDB_triple) return count(s) as s union match (s:SDB_item) return count(s) as s;"
		gCom = "match (s:Pubmed) return count(s) as s union match (s:Mesh) return count(s) as s union match (s:SDB_triple) return count(s) as s union match (s:SDB_item) return count(s) as s;"
		logger.debug(gCom)
		#data = [int(uAll)]
		data = []
		for res in session.run(gCom):
			data.append(res[0])
		metrics = data
		logger.debug(data)

		# get user and article set over time
		# select user_id,job_start from browser_searchset where user_id != 2 and job_status = 'Complete';
		logger.debug("getting time data...")
		s = SearchSet.objects.filter(job_status='Complete').exclude(user_id='2')
		tData = []
		aDic = {}
		for i in s:
			u = i.user_id
			t = i.job_start.split(" ")[0].split("-")[0:2]
			t = "-".join(t)
			if t in aDic:
				aDic[t].append(u)
			else:
				aDic[t] = [u]
		c = Compare.objects.filter(job_status='View results').exclude(user_id='2')
		cDic = {}
		for i in c:
			#id = i.id
			id = i.job_name
			t = i.job_start.split(" ")[0].split("-")[0:2]
			t = "-".join(t)
			if t in cDic:
				cDic[t].append(id)
			else:
				cDic[t] = [id]

		cats = []
		uCounts = []
		aCounts = []
		cCounts = []
		cCountOld = []
		oldCount = []
		for a in sorted(aDic):
			cats.append(a)
			#logger.debug(a)
			if a in aDic:
				uCount = len(list(set(aDic[a] + oldCount)))
				uCounts.append(uCount)

				aCount = len(aDic[a] + oldCount)
				aCounts.append(aCount)
				oldCount = aDic[a] + oldCount
			else:
				uCounts.append(0)
				aCounts.append(0)
			if a in cDic:
				cCount = len(list(set(cDic[a] + cCountOld)))
				cCounts.append(cCount)
				cCountOld = cDic[a] + cCountOld
			else:
				cCounts.append(0)
		lastTop=24
		uCounts = uCounts[-lastTop:len(uCounts)]
		aCounts = aCounts[-lastTop:len(aCounts)]
		cCounts = cCounts[-lastTop:len(cCounts)]
		cats = cats[-lastTop:len(cats)]
		#logger.debug(uCounts)

	else:
		data = 'fail'
		logger.debug('not ajax request')
	mimetype = 'application/json'
	session.close()
	return HttpResponse(json.dumps({'metrics':metrics,'uCounts':uCounts,'aCounts':aCounts,'cCounts':cCounts,'cats':cats}), mimetype)


def ajax_test(request):
	object = ''
	f = tmpDir+'/new.txt'
	logger.debug('looking for file ',f)
	if os.path.isfile(f):
		object = "file exists"
		logger.debug('object exists')
	return HttpResponse(object)

def issn_to_name(iList):
	logger.debug('Running issn_to_name'+str(iList))
	#check for null entries
	if 'null' in iList:
		iList.remove('null')
	iString = ",".join(iList)
	start=time.time()
	print "\n### Getting ids ###"
	url="http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?"
	#params = {'term': '0140-6736+OR+0022-2275'}
	params = {'term': iString}

	r = requests.post(url)

	# GET with params in URL
	r = requests.get(url, params=params)

	#create random file name
	n = 10
	ran=''.join(["%s" % randint(0, 9) for num in range(0, n)])
	rSplit = r.text.split("<")

	iDic = {}
	iName = []
	for i in rSplit:
		l = re.match(r'To>(.*?)$', i)
		if l:
			m = l.group(1).replace('[Journal]','').replace('"','').strip().encode("ascii")
			iName.append(m)

	for i in range(0,len(iList)):
		iDic[iList[i]]=iName[i]
	logger.debug(iDic)
	return iDic

def pubmed_id_details(pList):
	logger.debub('Getting pubmed info')

def pmid_to_info(pList):
	#logger.debug('Running pmid_to_info'+str(pList))
	iString = ",".join(pList)
	start=time.time()
	print "\n### Getting ids ###"
	url="http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?"
	#params = {'term': '0140-6736+OR+0022-2275'}
	params = {'db':'pubmed','id': iString}

	# GET with params in URL
	r = requests.get(url, params=params)

	#print r.text
	rSplit = r.text.split("<")

	ptDic = {}
	pjDic = {}
	t = jt = 'n/a'
	for i in rSplit:
		#print "i",i
		#check pubmed id
		pmid_match = re.match(r'Item Name="pubmed" Type="String">(.*?)$', i)
		if pmid_match:
			pmid = pmid_match.group(1)
			#print pmid

		#get title
		t_match = re.match(r'Item Name="Title" Type="String">(.*?)$', i)
		if t_match:
			t = t_match.group(1)
			#print t

		#get jorunal name
		jt_match = re.match(r'Item Name="FullJournalName" Type="String">(.*?)$', i)
		if jt_match:
			jt =  jt_match.group(1)
			#print jt

		entry_match = re.match(r'/DocSum>', i)
		if entry_match:
			#print "\n"
			ptDic[pmid]=t
			pjDic[pmid]=jt
			jt='n/a'
			t='n/a'

	#print pDic
	return [ptDic,pjDic]

def about(request):
	context = {'nbar': 'about'}
	return render_to_response('about.html', context, context_instance=RequestContext(request))

def citation(request):
        context = {'nbar': 'citation'}
        return render_to_response('citation.html', context, context_instance=RequestContext(request))

def help(request):
	context = {'nbar': 'help'}
	return render_to_response('help.html', context, context_instance=RequestContext(request))

def dt_test_page(request):
	return render_to_response('dt_test_page.html')

def contact(request):
	context = {'nbar': 'contact'}
	return render_to_response('contact.html', context, context_instance=RequestContext(request))

def get_semmed_items(request):
	session = driver.session()
	if request.is_ajax():
		q = request.GET.get('term', '').split(',')[-1].strip()
		logger.debug('q = '+q)
		#get data for autocomplete
		gCom = "match (sem:SDB_item) where sem.name =~ '(?i)"+q+".*' return sem.name;"
		logger.debug(gCom)
		sList = []
		for res in session.run(gCom):
			v=res[0].encode("ascii")
			json_data = {}
			json_data['id']=v
			json_data['label']=v
			json_data['value']=v
			sList.append(json_data)
		sList_json = json.dumps(sList)
		logger.debug(len(sList))
	else:
		data = 'fail'
		logger.debug('not ajax request')
	mimetype = 'application/json'
	session.close()
	return HttpResponse(sList_json, mimetype)

def index(request):
	userInfo = "UserID:"+str(request.user.id)+" - "
	logger.debug(userInfo+"In index")

	form1 = CreateSearchSet()
	form_sem = CreateSemSet()
	form2 = ComSearchSets()
	form_pub = CreatePubSet()
	if request.method == 'POST':
		# create a form instance and populate it with data from the request:
		if request.POST['formType'] == "ss":
			if request.user.is_authenticated():
				form1 = CreateSearchSet(request.POST, request.FILES)
				#print "f = ",request.FILES
				# check whether it's valid:
				if form1.is_valid():
					# process the data in form.cleaned_data as required
					ss_file = request.FILES['ss_file']
					#save to file
					fileStore = tmpDir+'abstracts/'+str(ss_file)

					id=form1.cleaned_data['job_name'].strip()
					#remove special characters
					id = re.sub('[^A-Za-z0-9 _-]+', '', id)
					desc=form1.cleaned_data['ss_desc'].strip()
					searchParams=[id,str(request.user.id)]

					#add job and user data to sqlite db
					q = SearchSet(user_id=str(request.user.id), job_name=id, job_start=time.strftime("%Y-%m-%d %H:%M:%S"),job_status='Pending',ss_desc=desc,pTotal=0,ss_file=ss_file,job_progress=0)
					q.save()

					#run job in background
					#j = db_citations.delay(searchParams,fileStore)
					j = pmid_process.delay(searchParams,fileStore)

					SearchSet.objects.filter(user_id=str(request.user.id),job_name=id).update(job_id=j)

					# redirect to a new URL:
					return HttpResponseRedirect('jobs/')
			else:
				logger.debug(userInfo+"User authentication problem")
				return HttpResponseRedirect('/')
		if request.POST['formType'] == "ss_sem":
			if request.user.is_authenticated():
				form_sem = CreateSemSet(request.POST)
				#print "f = ",request.FILES
				# check whether it's valid:
				if form_sem.is_valid():
					# process the data in form.cleaned_data as required
					#add to graph db
					id=form_sem.cleaned_data['job_name'].strip()
					#remove special characters
					id = re.sub('[^A-Za-z0-9 _-]+', '', id)
					desc=form_sem.cleaned_data['ss_desc'].strip()
					sem_location = request.POST["ss_sem"]
					logger.debug('job_name = '+id)
					logger.debug('desc = '+desc)
					logger.debug('sem location = '+sem_location)
					searchParams=[id,str(request.user.id), sem_location,desc]

					#add job and user data to sqlite db
					descWithSem = sem_location+": "+desc
					q = SearchSet(user_id=str(request.user.id), job_name=id, job_start=time.strftime("%Y-%m-%d %H:%M:%S"),job_status='Pending',ss_desc=descWithSem,pTotal=0,ss_file='',job_progress=0)
					q.save()

					#run job in background
					logger.debug(userInfo+"Running db_sem")
					j = db_sem.delay(searchParams)

					SearchSet.objects.filter(user_id=str(request.user.id),job_name=id).update(job_id=j)

					# redirect to a new URL:
					return HttpResponseRedirect('jobs/')
			else:
				logger.debug(userInfo+"User authentication problem")
				return HttpResponseRedirect('/')
		if request.POST['formType'] == "ss_pub":
			if request.user.is_authenticated():
				form_pub = CreatePubSet(request.POST)
				#print "f = ",request.FILES
				# check whether it's valid:
				if form_pub.is_valid():
					# process the data in form.cleaned_data as required
					#add to graph db
					id=form_pub.cleaned_data['job_name'].strip()
					#remove special characters
					id = re.sub('[^A-Za-z0-9 _-]+', '', id)
					desc=form_pub.cleaned_data['ss_desc'].strip()
					logger.debug('job_name = '+id)
					logger.debug('desc = '+desc)
					searchParams=[id,str(request.user.id),desc]


					q = SearchSet(user_id=str(request.user.id), job_name=id, job_start=time.strftime("%Y-%m-%d %H:%M:%S"),job_status='Pending',ss_desc=desc,pTotal=0,ss_file='',job_progress=0)
					q.save()

					#run job in background
					logger.debug(userInfo+"Running pub_sem")
					j = pub_sem.delay(searchParams)

					SearchSet.objects.filter(user_id=str(request.user.id),job_name=id).update(job_id=j)

					# redirect to a new URL:
					return HttpResponseRedirect('jobs/')
			else:
				logger.debug(userInfo+"User authentication problem")
				return HttpResponseRedirect('/')
		if request.POST['formType'] == "com":
			logger.debug(userInfo+"Comparing search sets")
			form2 = ComSearchSets(request.POST)
			if form2.is_valid():
				ass=form2.cleaned_data['a']
				bss=form2.cleaned_data['b']
				comType = form2.cleaned_data['comType']

				#get year range data
				yearRange = request.POST["yearRange"]
				logger.debug('yearRange:'+yearRange)
				#add one year to upper bound to make it inclusive
				year2 = int(yearRange.split("-")[1].strip())
				yearRange = yearRange.split("-")[0].strip()+" - "+str(year2)
				logger.debug('yearRange corrected:'+yearRange)


				#check if analysing one or two search sets
				if len(ass)>1 and len(bss)==0:
					logger.debug(userInfo+"analysing single search set")
					logger.debug("ss1 - "+str(ass))
					s1=SearchSet.objects.get(job_name=ass,user_id=str(request.user.id))
					jobName = str(s1.id)
					for c in comType:
						print "c = ",c
						try:
							jCheck = Compare.objects.get(job_name=jobName,year_range=yearRange,user_id=str(request.user.id),job_type=c)
							logger.debug(userInfo+"job_status = "+str(jCheck.job_status))
							#delete entry if not complete and resubmitted
							if jCheck.job_progress != 100:
								logger.debug(userInfo+"Deleting job: "+str(jCheck.job_name))
								jCheck.delete()
								jCheck = False
						except ObjectDoesNotExist:
							jCheck = False
						if jCheck==False:
							jobDesc = str(s1.job_name)
							q = Compare(user_id=str(request.user.id), year_range=yearRange, job_desc=jobDesc, job_name=jobName, job_start=time.strftime("%Y-%m-%d %H:%M:%S"), job_status='Pending',job_type=c,job_progress=0)
							q.save()
							j=single_ss_Wrapper.delay(str(request.user.id),s1.id,c,yearRange)
						else:
							logger.debug(userInfo+"Search set comparison already run")
				elif len(ass)>1 and len(bss)>1:
					logger.debug(userInfo+"analysing two search sets")
					logger.debug("ss1 - "+str(ass))
					logger.debug("ss2 - "+str(bss))

					#get ids for search sets
					s1=SearchSet.objects.get(job_name=ass,user_id=str(request.user.id))
					s2=SearchSet.objects.get(job_name=bss,user_id=str(request.user.id))

					#include year2 to deal with year filtering option
					jobName = str(s1.id)+"_"+str(s2.id)+"_"+str(year2)
					for jobType in comType:
						logger.debug("jobType = "+jobType)
						try:
							jCheck = Compare.objects.get(job_name=jobName,year_range=yearRange,user_id=str(request.user.id),job_type=jobType)
							logger.debug(userInfo+"job_status = "+str(jCheck.job_status))
							#removed this section as allows same job to run if already running.
							#delete entry if not complete and resubmitted
							#if jCheck.job_progress != 100:
							#	logger.debug(userInfo+"Deleting job: "+str(jCheck.job_name))
							#	jCheck.delete()
							#	jCheck = False
						except ObjectDoesNotExist:
							jCheck = False
						if jCheck==False:
							jobDesc = str(s1.job_name)+" : "+str(s2.job_name)
							q = Compare(user_id=str(request.user.id), job_desc=jobDesc, year_range=yearRange, job_name=jobName, job_start=time.strftime("%Y-%m-%d %H:%M:%S"), job_status='Pending',job_type=jobType,job_progress=0)
							q.save()
							#j=comWrapper.delay(str(request.user.id),s1.id,s2.id,jobType,yearRange)
							j=comWrapper.delay(q.id)
						else:
							logger.debug(userInfo+"Search set comparison already run")
				return HttpResponseRedirect('jobs/')
	else:
		form1 = CreateSearchSet()
		form_sem = CreateSemSet()
		form2 = ComSearchSets()
		form_pub = CreatePubSet()

	#get search set data for table
	j=SearchSet.objects.filter(user_id=str(request.user.id),job_status='Complete')

	#get example data for table
	exampleData=[]
	eCheck={}
	e=Compare.objects.filter(user_id='None',job_status='View results')
	for i in e:
		eName = i.job_name
		c1,c2=i.job_desc.split(':')
		if eName in eCheck:
			eCheck[eName].append(i.job_type+':'+str(i.id))
		else:
			eCheck[eName]=[c1,c2,i.job_start,i.job_type+':'+str(i.id)]
	#sort the methods
	for e in eCheck:
		eCheck[e][3:6] = sorted(eCheck[e][3:6])
		#exampleData[i.job_desc]=[c1,c2,i.job_start]
	logger.debug(eCheck)
	context = {'s': j, 'exampleData':eCheck, 'form1': form1, 'form2': form2, 'form_sem':form_sem, 'form_pub':form_pub, 'nbar': 'home'}
	return render_to_response('index.html', context, context_instance=RequestContext(request))

@cache_page(None)
def articleDetails(request,num):
	session = driver.session()
	userInfo = "UserID:"+str(request.user.id)+" - "
	logger.debug(userInfo+"In article details")
	resID=num
	logger.debug(userInfo+str(resID))
	q = SearchSet.objects.get(id=resID)

	gCom = "match (s:SearchSet)<-[r:INCLUDES]->(p:Pubmed) where s.name = '"+q.job_name+"_"+q.user_id+"' return p.dp,p.issn;"
	logger.debug(userInfo+"gCom:"+gCom)
	years = set()
	yearCounts = defaultdict(dict)
	for res in session.run(gCom):
		if type(res[0]) != type(None):
			#logger.debug(res)
			y = res[0].split(" ")[0]
			j = res[1]
			if type(y) != type(None) and type(j) != type(None) and y != '':
				y = int(y)
				j = j.encode("ascii")
				years.add(y)
				if y in yearCounts:
					if j in yearCounts[y]:
						yearCounts[y][j]+=1
					else:
						yearCounts[y][j]=1
				else:
					yearCounts[y][j]=1
	#logger.debug(years)
	article_data=[]
	if len(years)>0:
		years = range(min(years),max(years)+1)
		logger.debug(years)
		#logger.debug(len(yearCounts))
		#'1995': {'1040-872X': 1, '0090-3493': 2
		jTotals = {}
		for i in yearCounts:
			#logger.debug('i = '+str(i))
			for j in yearCounts[i]:
				if j in jTotals:
					jTotals[j] = jTotals[j]+1
				else:
					jTotals[j]=1
				jTotals[j]
				#logger.debug(str(j)+":"+str(yearCounts[i][j]))
		numTopJ = 10
		topJs = dict(sorted(jTotals.items(), key=operator.itemgetter(1),reverse=True)[0:numTopJ])
		#logger.debug(topJs)

		#create top counts
		topCounts = defaultdict(dict)
		for i in years:
			topCounts[i]['Other']=0
			for j in topJs:
				if i in yearCounts:
					if j in yearCounts[i]:
						topCounts[i][j] = yearCounts[i][j]
					else:
						topCounts[i][j] = 0
				else:
					topCounts[i][j] = 0
		#logger.debug(topCounts)

		#add counts not in the top set as 'Other'
		for i in yearCounts:
			for j in yearCounts[i]:
				if j not in topCounts[i]:
					topCounts[int(i)]['Other'] += yearCounts[i][j]
		#logger.debug(topCounts)

		#convert ISSN to name
		iList = []
		for i in topJs:
			iList.append(i)
		iName = issn_to_name(iList)

		topJs['Other']=0
		for t in topJs:
			if t in iName:
				a = {'name':iName[t],'data':[]}
			else:
				a = {'name':t,'data':[]}
			for i in topCounts:
				a['data'].append(topCounts[i][t])
			article_data.append(a)
		#logger.debug(article_data)




	context = {'years':years,'aData':json.dumps(article_data),'ss':q.job_name,'nbar': 'results'}
	session.close()
	return render_to_response('articles.html', context, context_instance=RequestContext(request))


#@login_required
def jobs(request):
	userInfo = "UserID:"+str(request.user.id)+" - "
	logger.debug(userInfo+"In jobs")
	context = {'nbar': 'results'}
	return render_to_response('jobs.html', context, context_instance=RequestContext(request))

#@login_required
@cache_page(None)
def results(request,num):
	userInfo = "UserID:"+str(request.user.id)+" - "
	logger.debug(userInfo+"In results")