import os, sys, re, json, feedparser, urllib, webbrowser, lxml.html
from utils.charset import to_unicode
from utils.cleanser import sweep
from classifier import get_words, FisherClassifier
from pprint import pprint

class _Entry(object):
    
    def __init__(self, entry):
        self.entry = entry
    
    def get_title(self):
        return self.entry['title']
    
    def get_summary(self):
        return self.entry['summary']
    
    def get_content(self):
        # merge text content
        text = '%s\n%s' % (self.get_title().encode('utf-8'), self.get_summary().encode('utf-8'))
        text = sweep(to_unicode(text))
        # remove html tags
        content = lxml.html.fromstring(text).text_content()
        return content
    
    def get_link(self):
        return self.entry['feedburner_origlink'].encode('utf-8')

def read(filename):
    return open(filename, 'r').read()

def train(classifier, feed):
    links = []
    categories = {}
    for l in read('training_data.txt').split('\n'):
        if not l: break
        link, cat = l.split('\t')
        links.append(link)
        categories[link] = cat
    
    for e in feedparser.parse(feed)['entries']:
        e = _Entry(e)
        # if you have already backuped, skip entry
        if e.get_link() in links and categories[e.get_link()]:
            continue
        content = e.get_content()
        # get category
        guess = classifier.classify(content)
        if guess == None:
            guess = 'Unknown'
        print '-----'
        print 'Title: ' + e.get_title()
        print 'Link: ' + e.get_link()
        print 'Content: ' + content[:500] + '\n'
        print 'Guess: ' + guess + '\n'
        # whether open feed with browser
        answer = raw_input('Open link (y/n): ')
        if answer == 'y':
            webbrowser.open(e.get_link())
        # decide the category of that page
        cat = raw_input('Enter category: ')
        if cat != '':
            classifier.train(content, cat)
        # backup history
        f = open('training_data.txt', 'a')
        f.write('%s\t%s\n' % (e.get_link(), cat))
        f.close()

def main():
    feed = 'http://feeds.feedburner.com/creativeapplicationsnet'
    classifier = FisherClassifier(get_words)
    train(classifier, feed)

if __name__ == '__main__': main()