# -*- coding: UTF-8 -*-
#!/usr/bin/env python

#------------------------------------------------------------------------------
# Name:         Analyseur (or Analizer in English)
# Purpose:      Analyzes the offers published on GeoRezo, extracts and formats
#               interesting informations: contracts types, date, etc.
#
# Authors:      pvernier (https://github.com/pvernier)
#               & Guts (https://github.com/Guts)
#
# Python:       3.4.x
# Created:      01/05/2014
# Updated:      15/01/2017
#
# Licence:      GPL 3
#------------------------------------------------------------------------------

###############################################################################
########### Libraries #############
###################################

# Standard library
from itertools import zip_longest
import logging
from os import path, environ
import sys
from datetime import datetime
import re
import sys
import threading # multi threads handling
from xml.etree import ElementTree as ET
from xml.sax.saxutils import escape  # '<' -> '&lt;'

# 3rd party modules
import arrow
from bs4 import BeautifulSoup
import html
import nltk
from nltk.corpus import stopwords
import pytz

# Django project
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.db import IntegrityError
from django.utils import timezone
from jobs.models import Contract, GeorezoRSS, Offer, Source, Status,\
                        Place, PlaceVariations, \
                        Technology, TechnologyVariations, \
                        JobPosition, JobPositionVariations

###############################################################################
############ Globals ##############
###################################

logger = logging.getLogger("ElPaso")
paris_tz = pytz.timezone("Europe/Paris")

###############################################################################
############ Functions ############
###################################

# Print iterations progress
def printProgress (iteration, total, prefix = '', suffix = '', decimals = 1,
                   barLength = 100, fill = '█'):
    """
    Call in a loop to create terminal progress bar
    @params:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        barLength   - Optional  : character length of bar (Int)
    """
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    filledLength = int(barLength * iteration // total)
    bar = fill * filledLength + '-' * (barLength - filledLength)
    sys.stdout.write('\r%s |%s| %s%s %s' % (prefix, bar, percent, '%', suffix)),
    if iteration == total:
        sys.stdout.write('\n')
    sys.stdout.flush()

###############################################################################
############ Classes ##############
###################################


class Analizer():
    """
    analyze of last offers published on GeoRezo and stored in the main table.
    """
    def __init__(self, li_offers_ids, opt_contracts=1, opt_places=1,
                 opt_technos=1, opt_skills=1, opt_words=1, offer_source="GEOREZO_RSS"):
        """
        manage the connection cursor and task related

        li_offers_ids = IDs list of offers to process
        opt_contracts = option to parse contracts types
        opt_places = option to parse contracts places
        opt_technos = option to parse contracts technologies
        opt_skills = option to parse contracts jobs label
        opt_words = option to parse contracts words
        """
        # a little log message to know where we are
        offers_count = len(li_offers_ids)
        logger.info("Launching analisis on {} offers.".format(offers_count))

        # progress bar ony if debug mode is disabled
        if not settings.DEBUG:
            i = 0   # start
            printProgress(i, total=offers_count,
                          prefix = 'Progress:', suffix = 'Complete', barLength = 50)
        else:
            pass

        for offer_id in li_offers_ids:
            self.offer_id = offer_id
            # chekcs if offer has already been added
            if Offer.objects.filter(id_rss = offer_id).exists():
                logger.error("Offer RSS_ID already exists in DB: {}"\
                             .format(offer_id))
                # Update Progress Bar only if debug mode is disabled
                if not settings.DEBUG:
                    i += 1
                    printProgress(i, offers_count, prefix = 'Progress:',
                                  suffix = 'Complete', barLength = 50)
                else:
                    pass
                continue
            else:
                pass
            # get raw offer from georezo_rss table
            raw_offer = GeorezoRSS.objects.get(id_rss=offer_id)
            # clean title
            clean_title = self.remove_tags(raw_offer.title)
            # determine contract type
            contract_type = self.parse_contract_type(clean_title)
            place = self.parse_place(clean_title)
            # clean content
            clean_content =  self.remove_tags(raw_offer.content)
            # content cleaning and nltk tokenizing
            content_words = self.parse_words(raw_offer.content)
            content_title = self.parse_words(raw_offer.title)
            technos = self.parse_technology(content_words)
            jobs_labels = self.parse_jobs_positions(content_title)
            # determine offer week number YYYYWW
            week_number = self.define_week_number(raw_offer.pub_date)
            # add new offer
            clean_offer = Offer(id_rss=offer_id,
                                title=clean_title,
                                content=clean_content,
                                pub_date=raw_offer.pub_date,
                                contract=Contract.objects.get(abbrv=contract_type),
                                week=week_number,
                                source=Source.objects.get(name=offer_source),
                                status=Status.objects.get(name="ND"),
                                place=Place.objects.get(name=place)
                                )
            try:
                clean_offer.save()
            except IntegrityError as err_msg:
                logger.error("Offer RSS_ID ({}) already exists in DB: {}"\
                             .format(offer_id, err_msg))
                # Update Progress Bar only if debug mode is disabled
                if not settings.DEBUG:
                    i += 1
                    printProgress(i, offers_count, prefix = 'Progress:',
                                  suffix = 'Complete', barLength = 50)
                else:
                    pass
                continue
            # associate ManyToMany relationships
            clean_offer.technologies.set(technos)
            clean_offer.jobs_positions.set(jobs_labels)
            logger.info("Offer analyzed and inserted jobs.offer: {}"
                        .format(offer_id))
            # Update Progress Bar only if debug mode is disabled
            if not settings.DEBUG:
                i += 1
                printProgress(i, offers_count, prefix = 'Progress:',
                              suffix = 'Complete', barLength = 50)
            else:
                pass


    def parse_contract_type(self, offer_raw_title):
        """
        Extraction of types of contracts: CDI, CDD, mission, volontariat, etc.
        In theory, the offer's title is formatted to contain the type between []

        li_id = list of offers'IDs to process
        db_cursor = connection cursor to the DB where to store extracted data
        """
        # clean the title: excluding text out of brackets
        try:
            contract = offer_raw_title.split("[")[1].split("]")[0]
        except IndexError:
            logger.warning("Title bad formatted. Offer RSS ID: {}"
                           .format(self.offer_id))
            contract = offer_raw_title.split(']')[0].lstrip('[')

        logger.info("Contract extracted from title: {}".format(contract))

        contract = contract.lower()
        # print(contract)
        # depending on the type found, inserting into the DB
        if contract[0:3] == 'cdi'\
            or "cne" in contract:
            logging.info("Found contract type: CDI")
            return "CDI"
        elif contract[0:3] == 'cdd':
            logging.info("Found contract type: CDD")
            return "CDD"
        elif "fpt" in contract\
            or "f.p.t" in contract\
            or "fp" in contract\
            or "fpe" in contract\
            or "ftp" in contract\
            or "vacat" in contract:
            logging.info("Found contract type: FPE/FPT")
            return "FP E/T"
        elif "stage" in contract:
            return "Stage"
        elif "appr" in contract\
            or "alternance" in contract\
            or contract == "altern"\
            or "alternant" in contract:
            logging.info("Found contract type: ALTERNANCE")
            return "Alternance"
        elif "vi" in contract \
            or "volontariat" in contract\
            or "vcat" in contract\
            or "vci" in contract\
            or "vsc" in contract\
            or "vsi" in contract:
            logging.info("Found contract type: VOLONTARIAT")
            return "Volontariat"
        elif "these" in contract or "thèse" in contract:
            return "Thèse"
        elif "postdoctorat" in contract\
            or "postdoc" in contract\
            or "post-doc" in contract\
            or "post-doctorat" in contract:
            logging.info("Found contract type: POSTDOCTORAT")
            return "Postdoctorat"
        elif "mission" in contract \
            or "interim" in contract\
            or "intérim" in contract:
            logging.info("Found contract type: INTERIM")
            return "Intérim"
        elif "benevolat" in contract\
            or "bénévolat" in contract\
            or "cae" in contract:
            logging.info("Found contract type: AUTRES")
            return "Autres"
        else:
            pass

        logging.info("Contract type not determined so value = ND")
        # end of function
        return "ND"

    def parse_place(self, offer_raw_title):
        """
        Extraction of types of contracts: CDI, CDD, mission, volontariat, etc.
        In theory, the offer's title is formatted to contain the place between ()
        """
        dpt_code = re.findall("(2[AB]|[0-9]+)", offer_raw_title)
        if len(dpt_code) == 1:
            logger.info("French department code extracted from title: {}".format(dpt_code))
            if Place.objects.filter(code=dpt_code[0]).exists():
                place_name = Place.objects.get(code=dpt_code[0]).name
                return Place.objects.get(name=place_name)
            else:
                pass
        elif not len(dpt_code):
            logger.warning("Not department code found in title. Triying to find a place anyway...")
            try:
                title = offer_raw_title.split("[")[1].split("]")[1]
            except IndexError:
                logger.warning("Title bad formatted. Offer RSS ID: {}"
                               .format(self.offer_id))
                title = offer_raw_title.split(']')[0].lstrip('[')

        elif len(dpt_code) > 1:
            logger.error("More than code found in title: {}".format(";".join(dpt_code)))

        return "ND"

    def parse_words(self, offer_raw_content):
        """
        Extraction of words mentioned into the offers. The goal is to perform
        a semantic analysis.
        It's based on NLTK: http://www.nltk.org
        """
        # get list of common French words to filter
        stop_fr = set(stopwords.words('french'))   # add specific French

        # custom list
        li_stop_custom = ('(', ')', '...', '.', ':', ';', '/', 'nbsp', '&', '#',
                          ',', '-', ':', 'http', 'img', 'br', 'amp', '<', '>',
                          '%', 'border', '*', 'border=', 'les', 'leurs', '&',
                          '#', '-', '+', ':', '.', ';', 'à', 'où', 'des', ',',
                          'nbsp', 'De', 'Des', 'et', 'en', '(', ')', 'pour',
                          'plus', 'sein', 'sous', 'Les', 'auprès', 'etc',
                          'the', 'for', 'ème', 'via', 'Vos', 'dès', 'plein',
                          'tel', 'etc.', 'etc..', 'Ces', 'tél', 'cela', 'ceci',
                          'cet')

        contenu = BeautifulSoup(offer_raw_content, "html.parser").get_text()
        contenu = self.remove_tags(contenu)
        # contenu = self.clean_xml(contenu)
        contenu_tokenized = nltk.word_tokenize(contenu)
        # print(len(contenu_tokenized))

        # stop words filter
        for mot in contenu_tokenized:
            if mot in stop_fr or mot in li_stop_custom:
                contenu_tokenized = list(filter((mot).__ne__, contenu_tokenized))

        logger.info("Words parsed: {}".format(len(contenu_tokenized)))
            
        # print(len(contenu_tokenized))
        return contenu_tokenized

    def parse_technology(self, offer_content_tokenized):
        """TO DOCUMENT
        """
        technos_matched = []
        for word in offer_content_tokenized:
            if TechnologyVariations.objects.filter(label=word.lower()).exists():
                techno_name = TechnologyVariations.objects.get(label=word.lower()).name
                technos_matched.append(Technology.objects.get(name=techno_name))
            else:
                continue
        logger.info("Technologies identified: {}".format(technos_matched))
        return technos_matched

    def parse_jobs_positions(self, offer_content_tokenized):
        """TO DOCUMENT
        """
        jobs_positions_matched = []
        for word in offer_content_tokenized:
            if JobPositionVariations.objects.filter(label=word.lower()).exists():
                job_label = JobPositionVariations.objects.get(label=word.lower()).name
                jobs_positions_matched.append(JobPosition.objects.get(name=job_label))
            else:
                continue
        logger.info("Jobs positions identified: {}"\
                    .format(jobs_positions_matched))
        return jobs_positions_matched

    def define_week_number(self, offer_raw_datetime):
        """
        Extracts year and week number from offer publication datetime
        for more convenience in graphical representation.
        """
        return "{}{}".format(arrow.get(offer_raw_datetime).isocalendar()[0],
                             str(arrow.get(offer_raw_datetime).isocalendar()[1]).zfill(2)
                             )

    # ------------ UTILITIES -------------------------------------------------

    def remove_tags(self, html_text):
        """
        very basic cleaner for HTML markups
        """
        html_text = html.unescape(html_text)
        try:
            text = ' '.join(ET.fromstring(html_text).itertext())
        except:
            TAG_RE = re.compile(r'<[^>]+>')
            return TAG_RE.sub(' ', html_text)
        # end of function
        return text.lower()

    def remove_accents(self, input_str, substitute=u""):
        """Clean string from special characters.

        source: http://stackoverflow.com/a/5843560
        """
        return unicode(substitute).join(char for char in input_str if char.isalnum())

    def clean_xml(self, invalid_xml, mode="soft", substitute="_"):
        """Clean string of XML invalid characters.

        source: http://stackoverflow.com/a/13322581/2556577
        """
        # assumptions:
        #   doc = *( start_tag / end_tag / text )
        #   start_tag = '<' name *attr [ '/' ] '>'
        #   end_tag = '<' '/' name '>'
        ws = r'[ \t\r\n]*'  # allow ws between any token
        name = '[a-zA-Z]+'  # note: expand if necessary but the stricter the better
        attr = '{name} {ws} = {ws} "[^"]*"'  # note: fragile against missing '"'; no "'"
        start_tag = '< {ws} {name} {ws} (?:{attr} {ws})* /? {ws} >'
        end_tag = '{ws}'.join(['<', '/', '{name}', '>'])
        tag = '{start_tag} | {end_tag}'

        assert '{{' not in tag
        while '{' in tag:   # unwrap definitions
            tag = tag.format(**vars())

        tag_regex = re.compile('(%s)' % tag, flags=re.VERBOSE)

        # escape &, <, > in the text
        iters = [iter(tag_regex.split(invalid_xml))] * 2
        pairs = zip_longest(*iters, fillvalue='')  # iterate 2 items at a time

        # get the clean version
        clean_version = ''.join(escape(text) + tag for text, tag in pairs)
        if mode == "strict":
            clean_version = re.sub(r"<.*?>", substitute, clean_version)
        else:
            pass
        return clean_version


###############################################################################
###### Stand alone program ########
###################################

if __name__ == '__main__':
    u"""
    standalone execution.
    """
    print('Stand-alone execution')
    # DB connection settings
    db = path.abspath(r"../../elpaso.sqlite")
    conn = sqlite3.connect(db)
    c = conn.cursor()
    # fetching the ID list
    c.execute("SELECT * FROM georezo")
    liste_input = [i[0] for i in c.fetchall()]
    #liste_input = c.fetchall()
    print(liste_input)
    Analizer(liste_input, db)

    # closing
    conn.close()
