# -*- coding: utf-8 -*-

import json, csv, traceback
import parselogic

class decoder:

    # MAY WANT TO INPUT SOME OF THESE ARGUMENTS W/IN A DICTIONARY?
    def __init__ (self, keywords, dirIn, dirOut,
                  hiMem, mode, lcase, emoji, logging,
                  yesterday_dict, today_dict, out_extension='.tsv'):             #Update parser.csv to set extension
        self.keywords = []
        for kw in keywords:
            self.keywords.append(kw)
        self.dirOut = dirOut
        self.mode = mode
        self.lcase = lcase
        self.logging = logging
        
        self.n_tweets = 0
        self.n_matches = 0
        self.n_warnings = 0
        self.n_errors = 0

        self.out_extension=out_extension
        
        #Dictionaries to keep track of duplicate tweet IDs over all parsed files - logging
        self.yesterday_dict = yesterday_dict
        self.today_dict = today_dict

        self.emojis = {}
        if emoji is not None:
            with open(emoji, 'r') as emoji_file:
                for line in emoji_file:
                    unic = line.split(',')[0].lower()
                    trans = line.split(',')[1]
                    self.emojis[unic] = trans

    # This cleans the JSON format and seperates into individual tweet data
    # It will return a list of tweets
    def fixjson(self, dirIn, fileName, hiMem, *kargs):

        # This is always a work in progress... 
        def lineparse(self, dirIn, fileName):
            count = 0
            tweet = ''


            def jsonline(line, tweet, status): #UPDATE THIS?

                #if tweet[-1:] in ['[',']','{','}','(',')']:
                #    end = False
                #elif line[:1] in ['[',']','{','}','(',')']:
                #    end = False
                #else:
                #    end = True

                if status == 'first':
                    tweet = '{'
                    status = 'working'

                elif status == 'more':
                    tweet = '{'+line.strip()
                    status = 'working'
                    
                elif line.strip() == '}{':
                    tweet = tweet + '}'
                    status = 'more'

                #elif line.strip() == '}' and end:
                #    tweet = tweet + '}'
                #    status = 'done'
                                        
                else:
                    tweet = tweet+line.strip()
                    
                return [tweet, status]


            with open(dirIn+fileName, 'r', encoding='utf-8') as f:
                count = 0
                #errors = 0
                j = ['','first']
                for line in f.readlines():
                    tweet = j[0]
                    status = j[1]
                    #j = jsonline(line, tweet, status) ### 
                    
                    if j[1] in ['more', 'done']:
                        tweet = j[0]
                        try:
                            dic = json.loads(tweet, strict=False)
                            parsed_text = decoder.parseText(self, dic) # Parse the text so that it can be examined
                            parsed_quote = decoder.parseQuote(self, dic) # Parse the quote so that it can be examined
                            kwtext = parsed_text+' '+parsed_quote
                            self.n_tweets += 1
                            if decoder.checkForKWs(self, kwtext) == True: # Means that a keyword was found in the tweet
                                try: 
                                    tweetid = ('\''+dic['id_str'])
                                except:
                                    try: tweetid = ('\''+str(dic['id'])) # t_id
                                    except: tweetid = '\''
                                if tweetid in self.yesterday_dict:
                                    self.yesterday_dict[tweetid] += 1
                                elif tweetid in self.today_dict:
                                    self.today_dict[tweetid] += 1
                                else:
                                    self.today_dict[tweetid] = 1
                                    decoder.writeToCSV(self, dic, parsed_text, parsed_quote, fileName, count)
                                    count += 1
                                #if j[1] == 'done':
                                #    f.close()
                                #    print('DONE\n')
                                #    return None

                        #except ValueError:
                        #    self.n_errors += 1
                        #    j = ['','more'] ### THIS NEEDS UPDATING?
                        #    pass
                        
                        except:
                            ### These can be un-commented if you are trying to diagnose errors
                            ### FUTURE: incorporate error logging here
                            #print(j[0])
                            #print('\n Count: '+str(count)+
                            #      '\nErrors: '+str(errors)+
                            #      '\nLength: '+str(len(tweet))+
                            #      '\n')
                            #if len(tweet) < 100: # This catches Twitter API hiccups
                            #    print(tweet)
                            #print(tweet) 
                            #traceback.print_exc() # This is the most helpful info to print
                            self.n_errors += 1
                            j = ['','more'] ### 
                            #break ### Break here when diagnosing errors
                    j = jsonline(line, tweet, status) # The last tweet will be ignored.

                f.close()
                return None
            

        # hiMem is preffered, when working memory permits
        if hiMem:
            self.n_tweets = 0
            count = 0
            with open(dirIn+fileName, 'r', encoding='utf-8') as f: 
                data = f.read().replace('}{','},{')
                data = r'{"tweet": ['+data+']}' #creates a list of tweets to be read
                try: 
                    fixed = json.loads(data, strict=False)
                    f.close() #closes the file 
                    for data in fixed['tweet']: # This grabs each tweet one by one
                        self.n_tweets += 1
                        parsed_text = decoder.parseText(self, data) # parse the text so that it can be examined
                        parsed_quote = decoder.parseQuote(self, data) # parse the text so that it can be examined
                        kwtext = parsed_text+' '+parsed_quote
                        #decoder._tweets_checked += 1 #increment the number of tweets checked
                        if decoder.checkForKWs(self, kwtext) == True: #means that a keyword was found in the tweet
                            #decoder._tweet_count += 1    #increment the count on the number of tweets printed
                            try: 
                                tweetid = ('\''+data['id_str'])
                            except:
                                try: tweetid = ('\''+str(data['id'])) # t_id
                                except: tweetid = '\''
                            if tweetid in self.yesterday_dict:
                                self.yesterday_dict[tweetid] += 1
                            elif tweetid in self.today_dict:
                                self.today_dict[tweetid] += 1
                            else:
                                self.today_dict[tweetid] = 1
                                decoder.writeToCSV(self, data, parsed_text, parsed_quote, fileName, count)
                                count += 1
                    
                except: 
                    traceback.print_exc()
                    self.n_tweets = 0
                    self.today_dict = {}
                    print('\n'+fileName+' : HiMem failed! Trying with LoMem...') 
                    f.close()
                    try:
                        decoder.fixjson(self, dirIn, fileName, hiMem=False)
                    except:
                        print('\n'+fileName+' : LoMem failed! See exception.') 
                        traceback.print_exc()
                        pass
                    return False

        else:
            lineparse(self, dirIn, fileName)