
import os
import sys
import itertools
import operator
import nltk
import numpy as np
import matplotlib.pyplot as plt

from nltk.util import ngrams

from collections import Counter

from spell_checker import SpellChecker
from sklearn.decomposition import TruncatedSVD
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import Normalizer

sys.path.insert(0, '/Users/diakite_w/Documents/Dev/ExperimentationsACA/FrenchLefffLemmatizer')
from FrenchLefffLemmatizer import FrenchLefffLemmatizer



def extract_ngrams(documents, n):
    '''
            Return list of n-grams
    '''
    chained_documents = list(itertools.chain.from_iterable(documents))
    return Counter(ngrams(chained_documents, n))


def tokenize(text):

    fll = FrenchLefffLemmatizer()
    splck = SpellChecker()

    contracted_pronouns = ["l'", "m'", "n'", "d'", "c'", "j'", "qu'", "s'"]
    dictionnary = []
    stopwords = [w.rstrip() for w in open('stopwords-fr.txt')]

    # Put everything to lower case
    text = text.lower()

    # Tokenize text
    tokens = nltk.tokenize.word_tokenize(text)
    print('Nombre de tokens dans le texte :', len(tokens))

    #tokens = [splck.correct(t) if t not in dictionnary else t for t in tokens]

    # Remove contacted pronous from tokens
    tokens = [t[2:] if t[:2] in contracted_pronouns else t for t in tokens]

    tokens = [t for t in tokens if len(t) > 2]

    tokens = [t for t in tokens if t not in stopwords]

    tokens = [fll.lemmatize(t) for t in tokens]

    print('Nombre de tokens apres traitement :', len(tokens), '\n')
    return tokens


def tokens_to_vec(tokens):
	vec = np.zeros(len(word_index_map))
	for token in tokens:
		idx = word_index_map[token]
		vec[idx] = 1
	return vec


def read_txt(textfile):
	with open(textfile, 'r') as f:
	    text = f.read()
	    text = text.replace('\n', ' ')
	    text = text.replace('- ', '')
	    text = text.replace('.', '')
	    text = text.replace('-', '')
	    text = text.replace("‘l'", 'ï')
	return text


def get_all_doc(directory):
	'''
		Read all txt documents and append them in string
	'''
	documents = [] 
	counter = 1
	for filename in os.listdir(directory):
		if filename.endswith('.txt'):
			print('\n[...] Reading document', counter)
			filename = 'data/' + filename
			documents.append(read_txt(filename))
			counter += 1
	return documents


documents = get_all_doc('data/')
all_tokens = [tokenize(doc) for doc in documents]
vocabulary = list(set(itertools.chain.from_iterable(all_tokens)))
print ('\nVocab size:', len(vocabulary))


# Computing n-grams
bigrams = extract_ngrams(all_tokens, 2)
trigrams = extract_ngrams(all_tokens, 3)
[print(t) for t in trigrams.most_common(5)]
print('\n')
[print(t) for t in bigrams.most_common(10)]


'''
# Key: word - value: index
word_index_map = {j: i for i, j in enumerate(vocabulary)}

# Key: index - value: word
index_word_map = sorted(word_index_map.items(), key=operator.itemgetter(1))
index_word_map = [t[0] for t in index_word_map]

N = len(documents)
D = len(word_index_map)
X = np.zeros((D,N))
i = 0
for tokens in all_tokens:
	X[:,i] = tokens_to_vec(tokens)
	i += 1
print(X.shape)

svd = TruncatedSVD()
Z = svd.fit_transform(X)
print('Z shape', Z.shape)

plt.scatter(Z[:,0], Z[:,1])
print('D:', D)
for i in range(D):
	plt.annotate(s=index_word_map[i], xy=(Z[i,0], Z[i,1]))
plt.show()
'''







	








