#!/usr/bin/env python
# -*- coding: utf-8 -*-
__author__ = 'priska'

"""
Prepare sensim data for clustering. 

Depending on the clustering algorithm you want to use, choose one of the matrix 
format options and MatrixMarket header if needed. Possible options so far are 
--matrix=cw for Chinese Whispers [1], 
--matrix=sim for a similarity matrix compatible with scikit-learn [2]
--matrix=sim --mm=True for a similarity matrix compatible with fastcluster's hclust.R [3]
--matrix=feat for a feature matrix compatible with scikit-learn [2]

Required file(s): ctx_lmi.gz or sensim_lmi_l200.gz

where ctx_lmi.gz can be produced with the sensim pipeline[4] and sensim_lmi_l200.gz
is the output of the JoBimText pipeline[5].

[1] http://wortschatz.informatik.uni-leipzig.de/~cbiemann/software/CW.html
[2] http://scikit-learn.org/stable/modules/clustering.html#clustering
[3] http://cran.r-project.org/web/packages/fastcluster/fastcluster.pdf
[4] https://github.com/policecar/sensim
[5] http://maggie.lt.informatik.tu-darmstadt.de/jobimtext/

Usage:	
		python clustering_utils.py --prune=10
"""

from config import *

try:
	import IPython
	from IPython import embed
except ImportError:
	pass

import re
import os

import gzip
from optparse import OptionParser
import logging

# set up logger
logging.basicConfig( level=logging.INFO,
					 format='%(asctime)s %(levelname)s %(message)s' )
log = logging.getLogger(__name__)

def make_pair_dict( sim_file, pair_dict_file, prune_factor ):
	"""
	Make a dictionary that maps a unique numeric id to each pair.

	Input  format: l_pair \t r_pair \t similarity
	Output format: id \t pair
	"""

	log.info( "Generating id2pair dict" )

	d_pairs 	= {}
	nid 		= 0
	num_lines 	= 0 
	num_entries = 0

	with gzip.open( sim_file, 'r' ) as f:

		current_left_pair = None
		l = 0

		for line in f:
			
			num_lines += 1
			if num_lines % 1000000 == 0:
				log.info( "%d lines later" % num_lines )

			# preparatory
			items = line.split('\t')
			l_pair = items[0]
			r_pair = items[1]
			sim = float( items[2][:-1] )

			# prune to use only X most similar pairs
			if l_pair == current_left_pair:
				l += 1
			else:
				l = 0
				current_left_pair = l_pair

			# filter pairs with special characters
			m_l = re.search( r"[^a-zA-Z:@]", l_pair )
			m_r = re.search( r"[^a-zA-Z:@]", r_pair )

			# prune entries with similarity < prune factor or either of the above
			# re l, keep only the 50 most similar pairs per pair
			if sim < prune_factor or m_l or m_r or l > 50 or ( l_pair == r_pair ):
				continue
			else:
				num_entries += 1

			# add left pair to dict
			if not d_pairs.get( l_pair ):