# -*- coding: utf-8 -*-
#! /usr/bin/env python3
#from HindiTokenizer import Tokenizer
from gensim import corpora , models
import gensim
import sys
import os
import re
import math
import codecs
from operator import itemgetter
reload(sys)
sys.setdefaultencoding('utf8')
path='/home/priyansh/Downloads/ltrc/1055'
doc_set=[]
stopwords=[] #contain all stopwords
with codecs.open("/home/priyansh/Downloads/ltrc/extractor/clustering/stopwords.txt",'r','utf-8') as fl:
    for line in fl:
        for word in line.split():
            stopwords.append(word)
fl.close()

''' Lightweight Hindi stemmer
Copyright © 2010 Luís Gomes <luismsgomes@gmail.com>.

Implementation of algorithm described in

    A Lightweight Stemmer for Hindi
    Ananthakrishnan Ramanathan and Durgesh D Rao
    http://computing.open.ac.uk/Sites/EACLSouthAsia/Papers/p6-Ramanathan.pdf

    @conference{ramanathan2003lightweight,
      title={{A lightweight stemmer for Hindi}},
      author={Ramanathan, A. and Rao, D.},
      booktitle={Workshop on Computational Linguistics for South-Asian Languages, EACL},
      year={2003}
    }

Ported from HindiStemmer.java, part of of Lucene.
'''

suffixes = {
    1: ["ो", "े", "ू", "ु", "ी", "ि", "ा"],
    2: ["कर", "ाओ", "िए", "ाई", "ाए", "ने", "नी", "ना", "ते", "ीं", "ती", "ता", "ाँ", "ां", "ों", "ें"],
    3: ["ाकर", "ाइए", "ाईं", "ाया", "ेगी", "ेगा", "ोगी", "ोगे", "ाने", "ाना", "ाते", "ाती", "ाता", "तीं", "ाओं", "ाएं", "ुओं", "ुएं", "ुआं"],
    4: ["ाएगी", "ाएगा", "ाओगी", "ाओगे", "एंगी", "ेंगी", "एंगे", "ेंगे", "ूंगी", "ूंगा", "ातीं", "नाओं", "नाएं", "ताओं", "ताएं", "ियाँ", "ियों", "ियां"],
    5: ["ाएंगी", "ाएंगे", "ाऊंगी", "ाऊंगा", "ाइयाँ", "ाइयों", "ाइयां"],
}

def hi_stem(word):
    for L in 5, 4, 3, 2, 1:
        if len(word) > L + 1:
            for suf in suffixes[L]:
                if word.endswith(suf):
                    return word[:-L]
    return word
doc_name=[]
for filename in os.listdir(path):
    doc_name.append(filename)
    infilename=os.path.join(path,filename)
    doc_set.append(infilename)
texts=[]
for doc in doc_set:
    #print doc
    stopped_tokens=[]
    tokens=[]
    with codecs.open(doc,'r','utf-8') as fl:
        for line in fl:
            temp=[]
            temp = line.split()
            for word in temp:
                tokens.append(word)
    fl.close()
    for i in tokens:
        if i not in stopwords:
            stopped_tokens.append(i)

    stemmed_tokens=[]
    for token in stopped_tokens:
        stemmed_token = hi_stem(token)
        stemmed_tokens.append(stemmed_token)


    texts.append(stemmed_tokens)
    #break
#print texts
dictionary = corpora.Dictionary(texts)
corpus = [dictionary.doc2bow(text) for text in texts]
ldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics=5, id2word = dictionary, passes=10)
#print ldamodel.print_topics(num_topics=5,num_words=5)
doc_topics=[]
for doc_vector in corpus:
    doc_topics.append(ldamodel[doc_vector])

for i in range(0,len(doc_topics)):
    doc_topics[i] = sorted(doc_topics[i],key=itemgetter(1),reverse=True)


for i in doc_topics:
    print i

doc_topics_words=[]
for i in range(0,len(doc_topics)):
    temp=[]
    for j in range(0,len(doc_topics[i])):
        temp.append(ldamodel.print_topic(doc_topics[i][j][0]))
    temp3=[]
    for x in temp:
        temp1=[]
        word=[]
        prob=[]
        temp1 = x.split("+")
        for y in temp1:
            prob.append(y.split("*")[0].split('"')[0])
            word.append(y.split("*")[1].split('"')[1])
        topic_words=[]
        for k in range(0,len(word)):
            temp_tup=[]
            temp_tup.append(word[k])
            temp_tup.append(prob[k])
            topic_words.append(tuple(temp_tup))
        #temp3=[]
        temp3.append(topic_words)

    doc_topics_words.append(temp3)
#print ldamodel[corpus[0]]
#print len(doc_topics_words)
#print len(doc_topics)
for i in doc_topics_words:
    print i

result=[]
result =  ldamodel.print_topics(num_topics=5)
#print ldamodel.print_topics(num_topics=2, num_words=4)
#print result
temp2=[]
for i in result:
    string = i[1]
    #print string
    temp=[]
    temp1=[]
    temp = string.split("+")
    for j in temp:
        temp1.append(j.split("*")[1].split('"')[1])

    temp2.append(temp1)

fd = codecs.open("output.txt",'w','utf-8')
count=0
for i in temp2:
    string=''
    string+="Topic"+" "+str(count)+":"+" "
    for j in i:
        string+=j+" "

    fd.write(string+"\n")
    count+=1


fd.close()
