#!/usr/bin/env python -W ignore::UndefinedMetricWarning

"""
This script runs the bootstrap kfold validation experiments as used in
the publication.

Usage:
  validation.py [--interpro] [--pfam] [--mf] [--cc] [--bp]
             [--induce] [--chain] [--verbose] [--binary] [--top=T]
             [--model=M] [--n_jobs=J] [--n_splits=S] [--n_iterations=I]
             [--h_iterations=H] [--directory=DIR] [--output_folder=OUT]
  validation.py -h | --help

Options:
  -h --help     Show this screen.
  --interpro    Use interpro domains in features.
  --pfam        Use Pfam domains in features.
  --mf          Use Molecular Function Gene Ontology in features.
  --cc          Use Cellular Compartment Gene Ontology in features.
  --bp          Use Biological Process Gene Ontology in features.
  --binary      Use binary feature encoding instead of ternary.
  --induce      Use ULCA inducer over Gene Ontology.
  --chain       Use Classifier chains to learn label dependencies.
  --verbose     Print intermediate output for debugging.
  --model=M         A binary classifier from Scikit-Learn implementing fit,
                    predict and predict_proba [default: LogisticRegression]
  --n_jobs=J        Number of processes to run in parallel [default: 1]
  --n_splits=S      Number of cross-validation splits [default: 5]
  --h_iterations=H  Number of hyperparameter tuning
                    iterations per fold [default: 30]
  --n_iterations=I  Number of bootstrap iterations [default: 5]
  --directory=DIR   Output directory [default: ./results/]
  --output_folder=OUT  Output directory [default: None]
"""

import json
import logging
import pandas as pd
import numpy as np
import scipy as sp
import warnings
import joblib

from itertools import product
from operator import itemgetter
from collections import Counter
from datetime import datetime