#!/usr/bin/env /opt/ccp4/ccp4-7.0/bin/ccp4-python
"""
Short document classifier

Extract two types of features:
* bag-of-words (BOW) - ngrams, with skip-grams
* text attributes

Combine them - options are:
* combine all into a single set of features so that (e.g. sentence length) would be on a 
par with one of the ngrams. The issue here is how to combine features that are on different 
scales. Options: 
** I've tried scaling the attributes so that they fit onto the same scale as BOW after TFDIF and then combine into a single vector
** Use a FeatureUnion to combine them - possibly with weights.
* create separate models for the BOW and attribute and then combine the models with an ensemble method.

Grid search.
I should probably try a grid search to determine the optimum parameters for feature selection and learning (e.g. parameters to TFDIF).
I should maybe even include a grid search to trial the different ways of combining the features?

Thoughts:
Should look at how the different text attribute features perform, as I probably don't need all of them => look at
selecting the best features. 


"""
import cPickle
import sys
from time import time

# External module imports
import numpy as np
import pandas as pd

from sklearn.pipeline import FeatureUnion, Pipeline
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.svm import SVC
from sklearn import metrics
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import SGDClassifier

# Our imports
from skipgram_vectorizer import SkipgramVectorizer
from sklearn_util import TextFeatures, get_dataset, size_mb

count_vectorizer = Pipeline([ ('counts', SkipgramVectorizer(ngram_range=(1,2),
                                                            #stop_words='english',
                                                            stop_words=None,
                                                            ngram_skip=3,
                                                            #tokenizer=tokenizer,
                                                            max_features=None) ),
                             ('tfidf',TfidfTransformer() ),
                             #('best', TruncatedSVD(n_components=1000)),
                             ])

feature_vectorizer = Pipeline([ ('extract_features', TextFeatures()),
                                ('vect',DictVectorizer(sparse=False)),
                                #('scale',StandardScaler(with_mean=True)),
                                ('scale',MinMaxScaler()),
                                #('norm', Normalizer())
                                #('best', TruncatedSVD(n_components=3)),
                             ])

vectorizer = FeatureUnion([("countvec", count_vectorizer), ("features", feature_vectorizer)],
#                             # weight components in FeatureUnion - could use for fitting
#                             transformer_weights={
#                                                  'countvec': 0.8,
#                                                  'features': 0.2,
#                                                  }
                          )
#vectorizer = count_vectorizer
#vectorizer = feature_vectorizer

# Extract data from the file
#data, target, dataframe = get_dataset(filename='../When I am criticised.xlsx')
with open('data.pkl') as f: data, target, dataframe = cPickle.load(f)
#data_ref, target_ref, _ = get_dataset(filename='../WhenCriticisedRef.xlsx')
with open('data_ref.pkl') as f: data_ref, target_ref = cPickle.load(f)

#data, target = data[:2], target[:2]

# Split into training and testing sets - either deterministically or randomly
if False:
    data_train, data_test, target_train, target_test = train_test_split(data, target, test_size=0.2)
else:
    # Combine ref and collected data
    data = np.hstack((data, data_ref))
    target = np.hstack((target, target_ref))
    cut = int(float(len(data))*0.8)
    data_train, data_test, target_train, target_test = data[:cut], data[cut:], target[:cut], target[cut:]
    #data_train, data_test, target_train, target_test = data_ref, data[cut:], target_ref, target[cut:]

print("SIZES ",map(len,(data_train, data_test, target_train, target_test)))
data_train_size_mb = size_mb(data_train)
data_test_size_mb = size_mb(data_test)
print('data loaded - train: {0}Mb test: {1}Mb'.format(data_train_size_mb, data_test_size_mb))


#with open('text_clf.pkl') as f: text_clf = cPickle.load(f)
# print("DIR ",dir(text_clf))
# print("GOT ",text_clf.predict(data_test[0]))
# print("GOT ",text_clf.predict_proba(data_test[0]))
# sys.exit()


if True:
    # Create the pipeline
    text_clf = Pipeline([('vect', vectorizer),
                         #('clf', MultinomialNB()),
                         ('clf', RandomForestClassifier(n_estimators=100)),
                         #('clf', LinearSVC()),
                         #('clf', SGDClassifier()),
                         #('clf', SVC(probability=True)), # probability slows method but allows retrievel of probabilities
                         ])
    
    t0 = time()
    text_clf.fit(data_train, target_train)
    train_time = time() - t0
    print("train time: %0.3fs" % train_time)
    
    step = 'vect'
    step = 'best'
    #print("FEATURES ",text_clf.named_steps['vect'].named_steps[step].feature_names_)
    #print("FEATURES ",text_clf.named_steps['vect'].named_steps[step].components_)
    #print("VOCAB ",text_clf.named_steps['vect'].named_steps[step].vocabulary_)
    
    t0 = time()
    pred = text_clf.predict(data_test)
    test_time = time() - t0
    print("test time:  %0.3fs" % test_time)
    
    
else:
    print("Extracting features from the training data")
    t0 = time()
    # Fit the vectorizers and generate the vectors
    vectors_train = vectorizer.fit_transform(data_train)
    #vectors_train = add_features(vectors_train, data_train)
    #for v in vectorizer.vocabulary_: print("GOT ",v)
    #print("VOCAB ",len(vectorizer.vocabulary_))
    print("FEATURES ",vectorizer.named_steps['vect'].feature_names_)
    print("VECTORS ",vectors_train)

    from sklearn.feature_selection import SelectKBest
    from sklearn.feature_selection import chi2
    SK = SelectKBest(chi2, k=3)
    X_new = SK.fit_transform(vectors_train, target_train)
    
    print("GOT ",X_new,dir(SK))
    print("GOT ",SK.get_params())
    print("GOT3 ",SK._get_param_names())
    print("GOT2  ",SK.scores_)

    sys.exit()