"""Module responsible for loading trained models and guessing virus names.

Overall, the idea is to guess a name for a virus from the names that Antiviruses
give it.

When using the Guesser class, all Exceptions are wrapped in a
NameGeneratorException.

Usage:
In [1]: from name_generator import Guesser
In [2]: g = Guesser()
In [3]: g.guess_everything({"F-Prot": "W32/AddInstall.A",
   ...:                     "Comodo": "Application.Win32.InstalleRex.KG"})
Out[3]:
{'compiler': 'unknown',
 '_type': 'Application',
 'group': 'unknown',
 'ident': 'A',
 'family': 'AddInstall',
 'platform': 'Win32',
 'language': 'unknown'}

All labels are guessed using CRFSUITE conditional random fields.
For example, we would have two family labels in the example above:
"AddInstall" and "InstalleRex".

The following strategies are used to pick among the labeled antivirus names:
    - Family is guessed using TFIDF ratios for families across all documents.

    - Group and Identity are guessed by most commonly occurring groups and
      identities within AVs that guessed the picked family or guessed close to
      a picked family.  This is because the labels for group and identity only
      make sense within the confines of a specific family.

    - Platform is guessed using heuristics.

    - language, compiler, and _type are those that occur most often in the
      labeled set.

"""

import itertools
import Levenshtein
import logging
import numpy as np
import re
import scipy
import ujson as json

from collections import Counter
from collections import defaultdict
from pycrfsuite import Tagger
from sklearn.feature_extraction.text import TfidfVectorizer

CRF_MODEL_PATH = "all_train.model"
TANIMOTO_MAPPING_PATH = "tanimoto_mapping.json"
VOCAB_PATH = "tfidf_vocab.json"
IDF_WEIGHTS_PATH = "tfidf_idf.json"

REGEX_NONWORD = re.compile("\W")
REGEX_NONWORD_SAVED = re.compile("(\W)")
UNKNOWN = 'unknown'
FAMILY = 'family'
PLATFORM = 'platform'
GROUP = 'group'
IDENT = 'ident'
GENERIC_FAMILIES = set(['Generic', 'Gen', 'GENERIC', 'Genetic'])
OTHER_GUESSABLE_TAGS = ['language', 'compiler', '_type']

AVS = set(['TotalDefense',
           'ViRobot',
           'TheHacker',
           'ClamAV',
           'CAT-QuickHeal',
           'Antiy-AVL',
           'Baidu-International',
           'Agnitum',
           'Bkav',
           'nProtect',
           'Jiangmin',
           'Commtouch',
           'F-Prot',
           'Microsoft',
           'SUPERAntiSpyware',
           'Ad-Aware',
           'Symantec',
           'AhnLab-V3',
           'Rising',
           'NANO-Antivirus',
           'Norman',
           'Ikarus',
           'Kingsoft',
           'K7AntiVirus',
           'Panda',
           'VBA32',
           'Emsisoft',
           'Fortinet',
           'F-Secure',
           'Malwarebytes',
           'MicroWorld-eScan',
           'BitDefender',
           'Avast',
           'Kaspersky',
           'DrWeb',
           'Sophos',
           'Comodo',
           'GData',
           'ESET-NOD32',
           'AVG',
           'AntiVir',
           'VIPRE',
           'McAfee'])

TEMPLATES = (
    (('w', -2), ),
    (('w', -1), ),
    (('w',  0), ),
    (('w',  1), ),
    (('w',  2), ),
    (('w', -1), ('w',  0)),
    (('w',  0), ('w',  1)),
    (('pos', -2), ),
    (('pos', -1), ),
    (('pos',  0), ),
    (('pos',  1), ),
    (('pos',  2), ),
    (('pos', -2), ('pos', -1)),
    (('pos', -1), ('pos',  0)),
    (('pos',  0), ('pos',  1)),
    (('pos',  1), ('pos',  2)),
    (('pos', -2), ('pos', -1), ('pos',  0)),
    (('pos', -1), ('pos',  0), ('pos',  1)),
    (('pos',  0), ('pos',  1), ('pos',  2)),
    (('av', 0), ),
    )


class NameGeneratorException(Exception):
    pass


class FamilyPostproc(object):
    def families_to_canonical(self, families):
        """Convert list of family lists to post list of postprocessed lists.
        :param list families: list of lists of families.
        :rtype: list.

        """
        all_output_families = []
        for cluster in families:
            inner_cluster = []
            for fam in cluster:
                inner_cluster.append(self.synonimous_mapping.get(fam, fam))
            all_output_families.append(inner_cluster)
        return all_output_families


class TanimotoPostproc(FamilyPostproc):
    def __init__(self, save_file=False):
        self.synonimous_mapping = json.load(open(save_file))


class EditDistancePostproc(FamilyPostproc):
    def __init__(self, similarity_func=Levenshtein.jaro_winkler, threshold=0.9):
        self.similarity_func = similarity_func
        self.threshold = threshold

    def mapping_from_one_list(self, names_list):
        """Convert list of families to list of postprocessed families.

        Uses edit distance to replace similar names with longer names
        that are < `self.threshold` edit distance away.

        :param list names_list: list of families.
        :rtype: list.

        """
        all_clusters = []
        names_list_uniq = list(set(names_list))
        indices = set(range(len(names_list_uniq)))
        while indices:
            current_idx = indices.pop()
            current_w = names_list_uniq[current_idx]
            current_cluster = [current_w]
            idxes_to_discard = set()

            for idx in indices:
                comparison_w = names_list_uniq[idx]
                if comparison_w == current_w: