#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
	go through user cleaned monthly tweets (MO_tweets.dat) and create user monthly CVs (MO_CV.json)
	need access to MongoDB for individual word CVs. Both, input and output are .json txt files.
"""
import sys, io, json, os
import pymongo
from pymongo import MongoClient
from nltk.stem.snowball import SnowballStemmer
from collections import defaultdict, OrderedDict
import math
import codecs
import logging
import traceback
import multiprocessing
import functools
from itertools import repeat
import numpy as np

# used for our Wiki SR database querying. We saved CVs for stemmed words for better accuracy.
STEMMER = SnowballStemmer("english", ignore_stopwords=True )

# where we work
IN_DIR = "../../../DATA/General/"

# Mongo collections
CV = ""
AID7s = ""

# this global dictionary will change for each month MO
USR_TWEETS = defaultdict(int)

# .json input and output
F_IN_NAME = "_tweets.dat"
F_OUT_CV_NAME = "_CV.json"

# I put this here in order to cache the word CVs
# so to reduce the number of queries to MongoDB
words_CVs_lst = defaultdict(int)
#############################################################################################
# take the right collection = TF-IDF based concept vectors (CV) based on Wiki
#############################################################################################
def set_global_conn_params(client = MongoClient(), dbs="test", CV_collection="CV_stemmed_pruned",
	aid_collection="AID"):
	global CV, AID7s
	# connect to Mongo db test
	db = client[dbs]
	CV = db[CV_collection]
	AID7s = db[aid_collection]

#############################################################################################
# read in monthly user tweets from the right file (one per each month, 
# created by ""monthly_user_tweets_clean_2_json.py)
#############################################################################################
def read_in_monthly_user_tweets(MO):

	f = codecs.open(str(MO) + F_IN_NAME, "r")

	for line in f:
		line7s = json.loads(line)
		user = line7s["_id"]
		tweets_text = line7s["txt"]
		count = line7s["count"]
		USR_TWEETS[user] = defaultdict(int)
		USR_TWEETS[user]["text"] = tweets_text
		USR_TWEETS[user]["count"] = count

	f.close()

def stem_word(token):
	return STEMMER.stem( token ) 

# given two vectors as dictionaries with *ANY* sets of keys
# return their cosine similarity *vectors may be of ANY dim*
# cosine sim (v1,v2) = v1.v2 / ||v1|| ||v2||
def cosine_2_vectors(v1, v2):
	# numerator for the cosine formula 
	SR_num = 0.0
	# two denominator terms in the formula
	v1_sq_sum = 0.0
	v2_sq_sum = 0.0
	keys_1 = set(v1.keys())
	#print len(keys_1)
	keys_2 = set(v2.keys())
	#print len(keys_2)
	# separate the different keys and common keys
	different2_keys = keys_2 - keys_1
	different1_keys = keys_1 - keys_2
	common_keys = keys_1 & keys_2
	#print len(common_keys), common_keys
	# for common keys, we calculate formula as is
	# SR = v1.v2 / ||v1|| ||v2||
	for term in common_keys:
		v1_fq = v1[term]
		v2_fq = v2[term]
		SR_num += v1_fq * v2_fq
		v1_sq_sum += v1_fq * v1_fq
		v2_sq_sum += v2_fq * v2_fq
	# for different keys, we just take resepective non-zero 
	# dict terms for calculating denominator (nominator is zero)
	for term in different1_keys:
		v1_fq = v1[term]
		v1_sq_sum += v1_fq * v1_fq
	for term in different2_keys:
		v2_fq = v2[term]
		v2_sq_sum += v2_fq * v2_fq
	# sum all in denominator and sqrt in the end
	SR_den = math.sqrt(v1_sq_sum*v2_sq_sum)
	try:
		SR = SR_num/SR_den