text:
from src.ui import *
from src.general import *
from src.files import *

from src.features import *
from src.dataset import *
from src.evaluation import *

import numpy
import csv
import argparse
import textwrap
import timeit
from sklearn.metrics import confusion_matrix

from sklearn.externals import joblib

from sklearn import mixture
from sklearn import preprocessing as pp
import xgboost as xgb

__version_info__ = ('1', '0', '0')
__version__ = '.'.join(__version_info__)

final_result = {}

def main(argv):

    tot_start = timeit.default_timer()

    numpy.random.seed(123456)  # let's make randomization predictable

    parser = argparse.ArgumentParser(
        prefix_chars='-+',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent('''\
            DCASE 2016
            Task 1: Acoustic Scene Classification
            Baseline system
            ---------------------------------------------
                Tampere University of Technology / Audio Research Group
                Author:  Toni Heittola ( toni.heittola@tut.fi )

            System description
                This is an baseline implementation for D-CASE 2016 challenge acoustic scene classification task.
                Features: MFCC (static+delta+acceleration)
                Classifier: GMM

        '''))

    # Setup argument handling
    parser.add_argument("-development", help="Use the system in the development mode", action='store_true',
                        default=False, dest='development')
    parser.add_argument("-challenge", help="Use the system in the challenge mode", action='store_true',
                        default=False, dest='challenge')

    parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__)
    args = parser.parse_args()

    # Load parameters from config file
    parameter_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                  os.path.splitext(os.path.basename(__file__))[0]+'.yaml')
    params = load_parameters(parameter_file)
    params = process_parameters(params)
    make_folders(params)

    title("DCASE 2016::Acoustic Scene Classification / Baseline System")

    # Check if mode is defined
    if not (args.development or args.challenge):
        args.development = True
        args.challenge = False

    dataset_evaluation_mode = 'folds'
    if args.development and not args.challenge:
        print "Running system in development mode"
        dataset_evaluation_mode = 'folds'
    elif not args.development and args.challenge:
        print "Running system in challenge mode"
        dataset_evaluation_mode = 'full'

    # Get dataset container class
    dataset = eval(params['general']['development_dataset'])(data_path=params['path']['data'])
    plot_name = params['classifier']['method']

    # Fetch data over internet and setup the data
    # ==================================================
    if params['flow']['initialize']:
        dataset.fetch()

    # Extract features for all audio files in the dataset
    # ==================================================
    if params['flow']['extract_features']:
        section_header('Feature extraction')

        # Collect files in train sets
        files = []
        for fold in dataset.folds(mode=dataset_evaluation_mode):
            for item_id, item in enumerate(dataset.train(fold)):
                if item['file'] not in files:
                    files.append(item['file'])
            for item_id, item in enumerate(dataset.test(fold)):
                if item['file'] not in files:
                    files.append(item['file'])
        files = sorted(files)

        # Go through files and make sure all features are extracted
        do_feature_extraction(files=files,
                              dataset=dataset,
                              feature_path=params['path']['features'],
                              params=params['features'],
                              overwrite=params['general']['overwrite'])

        foot()

    # Prepare feature normalizers
    # ==================================================
    if params['flow']['feature_normalizer']:
        section_header('Feature normalizer')

        do_feature_normalization(dataset=dataset,
                                 feature_normalizer_path=params['path']['feature_normalizers'],
                                 feature_path=params['path']['features'],
                                 dataset_evaluation_mode=dataset_evaluation_mode,
                                 overwrite=params['general']['overwrite'])

        foot()

    train_start = 0.0
    train_end = 0.0
    test_start = 0.0
    test_end = 0.0
    # System training
    # ==================================================
    if params['flow']['train_system']:
        section_header('System training')

	train_start = timeit.default_timer()
        do_system_training(dataset=dataset,
                           model_path=params['path']['models'],
                           feature_normalizer_path=params['path']['feature_normalizers'],
                           feature_path=params['path']['features'],
                           classifier_params=params['classifier']['parameters'],
                           classifier_method=params['classifier']['method'],
                           dataset_evaluation_mode=dataset_evaluation_mode,
                           overwrite=params['general']['overwrite']
                           )
	train_end = timeit.default_timer()

        foot()

    # System evaluation in development mode
    if args.development and not args.challenge:
        # System testing
        # ==================================================
        if params['flow']['test_system']:
            section_header('System testing')
	    test_start = timeit.default_timer()

            do_system_testing(dataset=dataset,
                              feature_path=params['path']['features'],
                              result_path=params['path']['results'],
                              model_path=params['path']['models'],
                              feature_params=params['features'],
                              dataset_evaluation_mode=dataset_evaluation_mode,
                              classifier_method=params['classifier']['method'],
                              overwrite=params['general']['overwrite']
                              )

 	    test_end = timeit.default_timer()

            foot()

        # System evaluation
        # ==================================================
        if params['flow']['evaluate_system']:
            section_header('System evaluation')

            do_system_evaluation(dataset=dataset,
                                 dataset_evaluation_mode=dataset_evaluation_mode,
                                 result_path=params['path']['results'])

            foot()

    # System evaluation with challenge data
    elif not args.development and args.challenge:
        # Fetch data over internet and setup the data
        challenge_dataset = eval(params['general']['challenge_dataset'])()

        if params['flow']['initialize']:
            challenge_dataset.fetch()

        # System testing
        if params['flow']['test_system']:
            section_header('System testing with challenge data')

            do_system_testing(dataset=challenge_dataset,
                              feature_path=params['path']['features'],
                              result_path=params['path']['challenge_results'],
                              model_path=params['path']['models'],
                              feature_params=params['features'],
                              dataset_evaluation_mode=dataset_evaluation_mode,
                              classifier_method=params['classifier']['method'],
                              overwrite=True
                              )

            foot()

            print " "
            print "Your results for the challenge data are stored at ["+params['path']['challenge_results']+"]"
            print " "

    tot_end = timeit.default_timer()
    print " "
    print "Train Time : " + str(train_end-train_start)
    print " "
    print " "
    print "Test Time : " + str(test_end-test_start)
    print " "
    print " "
    print "Total Time : " + str(tot_end-tot_start)
    print " "
    final_result['train_time'] = train_end-train_start
    final_result['test_time'] = test_end-test_start

    final_result['tot_time'] = tot_end-tot_start
    joblib.dump(final_result, 'result' + plot_name + '.pkl')

    return 0

def process_parameters(params):
    """Parameter post-processing.

    Parameters
    ----------
    params : dict
        parameters in dict

    Returns
    -------
    params : dict
        processed parameters

    """

    # Convert feature extraction window and hop sizes seconds to samples
    params['features']['mfcc']['win_length'] = int(params['features']['win_length_seconds'] * params['features']['fs'])
    params['features']['mfcc']['hop_length'] = int(params['features']['hop_length_seconds'] * params['features']['fs'])

    # Copy parameters for current classifier method
    params['classifier']['parameters'] = params['classifier_parameters'][params['classifier']['method']]

    # Hash
    params['features']['hash'] = get_parameter_hash(params['features'])
    params['classifier']['hash'] = get_parameter_hash(params['classifier'])

    # Paths
    params['path']['data'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), params['path']['data'])
    params['path']['base'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), params['path']['base'])

    # Features
    params['path']['features_'] = params['path']['features']
    params['path']['features'] = os.path.join(params['path']['base'],
                                              params['path']['features'],
                                              params['features']['hash'])

    # Feature normalizers
    params['path']['feature_normalizers_'] = params['path']['feature_normalizers']
    params['path']['feature_normalizers'] = os.path.join(params['path']['base'],
                                                         params['path']['feature_normalizers'],
                                                         params['features']['hash'])

    # Models
    params['path']['models_'] = params['path']['models']
    params['path']['models'] = os.path.join(params['path']['base'],
                                            params['path']['models'],
                                            params['features']['hash'], params['classifier']['hash'])
    # Results
    params['path']['results_'] = params['path']['results']
    params['path']['results'] = os.path.join(params['path']['base'],
                                             params['path']['results'],
                                             params['features']['hash'], params['classifier']['hash'])

    return params

def make_folders(params, parameter_filename='parameters.yaml'):
    """Create all needed folders, and saves parameters in yaml-file for easier manual browsing of data.

    Parameters
    ----------
    params : dict
        parameters in dict

    parameter_filename : str
        filename to save parameters used to generate the folder name

    Returns
    -------
    nothing

    """

    # Check that target path exists, create if not
    check_path(params['path']['features'])
    check_path(params['path']['feature_normalizers'])
    check_path(params['path']['models'])
    check_path(params['path']['results'])

    # Save parameters into folders to help manual browsing of files.

    # Features
    feature_parameter_filename = os.path.join(params['path']['features'], parameter_filename)
    if not os.path.isfile(feature_parameter_filename):
        save_parameters(feature_parameter_filename, params['features'])

    # Feature normalizers
    feature_normalizer_parameter_filename = os.path.join(params['path']['feature_normalizers'], parameter_filename)
    if not os.path.isfile(feature_normalizer_parameter_filename):
        save_parameters(feature_normalizer_parameter_filename, params['features'])

    # Models
    model_features_parameter_filename = os.path.join(params['path']['base'],
                                                     params['path']['models_'],
                                                     params['features']['hash'],
                                                     parameter_filename)
    if not os.path.isfile(model_features_parameter_filename):
        save_parameters(model_features_parameter_filename, params['features'])

    model_models_parameter_filename = os.path.join(params['path']['base'],
                                                   params['path']['models_'],
                                                   params['features']['hash'],
                                                   params['classifier']['hash'],
                                                   parameter_filename)
    if not os.path.isfile(model_models_parameter_filename):
        save_parameters(model_models_parameter_filename, params['classifier'])

    # Results
    # Save parameters into folders to help manual browsing of files.
    result_features_parameter_filename = os.path.join(params['path']['base'],
                                                      params['path']['results_'],
                                                      params['features']['hash'],
                                                      parameter_filename)
    if not os.path.isfile(result_features_parameter_filename):
        save_parameters(result_features_parameter_filename, params['features'])

    result_models_parameter_filename = os.path.join(params['path']['base'],
                                                    params['path']['results_'],
                                                    params['features']['hash'],
                                                    params['classifier']['hash'],
                                                    parameter_filename)
    if not os.path.isfile(result_models_parameter_filename):
        save_parameters(result_models_parameter_filename, params['classifier'])

def get_feature_filename(audio_file, path, extension='cpickle'):
    """Get feature filename

    Parameters
    ----------
    audio_file : str
        audio file name from which the features are extracted

    path :  str
        feature path

    extension : str
        file extension
        (Default value='cpickle')

    Returns
    -------
    feature_filename : str
        full feature filename

    """

    audio_filename = os.path.split(audio_file)[1]
    return os.path.join(path, os.path.splitext(audio_filename)[0] + '.' + extension)

def get_feature_normalizer_filename(fold, path, extension='cpickle'):
    """Get normalizer filename

    Parameters
    ----------
    fold : int >= 0
        evaluation fold number

    path :  str
        normalizer path

    extension : str
        file extension
        (Default value='cpickle')

    Returns
    -------
    normalizer_filename : str
        full normalizer filename

    """

    return os.path.join(path, 'scale_fold' + str(fold) + '.' + extension)

def get_model_filename(fold, path, extension='cpickle'):
    """Get model filename

    Parameters
    ----------
    fold : int >= 0
        evaluation fold number

    path :  str
        model path

    extension : str
        file extension
        (Default value='cpickle')

    Returns
    -------
    model_filename : str
        full model filename

    """

    return os.path.join(path, 'model_fold' + str(fold) + '.' + extension)

def get_result_filename(fold, path, extension='txt'):
    """Get result filename

    Parameters
    ----------
    fold : int >= 0
        evaluation fold number

    path :  str
        result path

    extension : str
        file extension
        (Default value='cpickle')

    Returns
    -------
    result_filename : str
        full result filename

    """

    if fold == 0:
        return os.path.join(path, 'results.' + extension)
    else:
        return os.path.join(path, 'results_fold' + str(fold) + '.' + extension)

def do_feature_extraction(files, dataset, feature_path, params, overwrite=False):
    """Feature extraction

    Parameters
    ----------
    files : list
        file list

    dataset : class
        dataset class

    feature_path : str
        path where the features are saved

    params : dict
        parameter dict

    overwrite : bool
        overwrite existing feature files
        (Default value=False)

    Returns
    -------
    nothing

    Raises
    -------
    IOError
        Audio file not found.

    """

    # Check that target path exists, create if not
    check_path(feature_path)

    for file_id, audio_filename in enumerate(files):
        # Get feature filename
        current_feature_file = get_feature_filename(audio_file=os.path.split(audio_filename)[1], path=feature_path)

        progress(title_text='Extracting',
                 percentage=(float(file_id) / len(files)),
                 note=os.path.split(audio_filename)[1])

        if not os.path.isfile(current_feature_file) or overwrite:
            # Load audio data
            if os.path.isfile(dataset.relative_to_absolute_path(audio_filename)):
                y, fs = load_audio(filename=dataset.relative_to_absolute_path(audio_filename), mono=True, fs=params['fs'])
            else:
                raise IOError("Audio file not found [%s]" % audio_filename)

            # Extract features
            feature_data = feature_extraction(y=y,
                                              fs=fs,
                                              include_mfcc0=params['include_mfcc0'],
                                              include_delta=params['include_delta'],
                                              include_acceleration=params['include_acceleration'],
                                              mfcc_params=params['mfcc'],
                                              delta_params=params['mfcc_delta'],
                                              acceleration_params=params['mfcc_acceleration'])
            # Save
            save_data(current_feature_file, feature_data)

def do_feature_normalization(dataset, feature_normalizer_path, feature_path, dataset_evaluation_mode='folds', overwrite=False):
    """Feature normalization

    Calculated normalization factors for each evaluation fold based on the training material available.

    Parameters
    ----------
    dataset : class
        dataset class

    feature_normalizer_path : str
        path where the feature normalizers are saved.

    feature_path : str
        path where the features are saved.

    dataset_evaluation_mode : str ['folds', 'full']
        evaluation mode, 'full' all material available is considered to belong to one fold.
        (Default value='folds')

    overwrite : bool
        overwrite existing normalizers
        (Default value=False)

    Returns
    -------
    nothing

    Raises
    -------
    IOError
        Feature file not found.

    """

    # Check that target path exists, create if not
    check_path(feature_normalizer_path)

    for fold in dataset.folds(mode=dataset_evaluation_mode):
        current_normalizer_file = get_feature_normalizer_filename(fold=fold, path=feature_normalizer_path)

        if not os.path.isfile(current_normalizer_file) or overwrite:
            # Initialize statistics
            file_count = len(dataset.train(fold))
            normalizer = FeatureNormalizer()

            for item_id, item in enumerate(dataset.train(fold)):
                progress(title_text='Collecting data',
                         fold=fold,
                         percentage=(float(item_id) / file_count),
                         note=os.path.split(item['file'])[1])
                # Load features
                if os.path.isfile(get_feature_filename(audio_file=item['file'], path=feature_path)):
                    feature_data = load_data(get_feature_filename(audio_file=item['file'], path=feature_path))['stat']
                else:
                    raise IOError("Feature file not found [%s]" % (item['file']))

                # Accumulate statistics
                normalizer.accumulate(feature_data)

            # Calculate normalization factors
            normalizer.finalize()

            # Save
            save_data(current_normalizer_file, normalizer)

def do_system_training(dataset, model_path, feature_normalizer_path, feature_path, classifier_params,
                       dataset_evaluation_mode='folds', classifier_method='xgboost', overwrite=False):
    """System training

    model container format:

    {
        'normalizer': normalizer class
        'models' :
            {
                'office' : mixture.GMM class
                'home' : mixture.GMM class
                ...
            }
    }

    Parameters
    ----------
    dataset : class
        dataset class

    model_path : str
        path where the models are saved.

    feature_normalizer_path : str
        path where the feature normalizers are saved.

    feature_path : str
        path where the features are saved.

    classifier_params : dict
        parameter dict

    dataset_evaluation_mode : str ['folds', 'full']
        evaluation mode, 'full' all material available is considered to belong to one fold.
        (Default value='folds')

    classifier_method : str ['xgboost']
        classifier method, currently only GMM supported
        (Default value='xgboost')

    overwrite : bool
        overwrite existing models
        (Default value=False)

    Returns
    -------
    nothing

    Raises
    -------
    ValueError
        classifier_method is unknown.

    IOError
        Feature normalizer not found.
        Feature file not found.

    """

    if classifier_method != 'xgboost':
        raise ValueError("Unknown classifier method ["+classifier_method+"]")

    # Check that target path exists, create if not
    check_path(model_path)

    for fold in dataset.folds(mode=dataset_evaluation_mode):
        current_model_file = get_model_filename(fold=fold, path=model_path)
        if not os.path.isfile(current_model_file) or overwrite:
            # Load normalizer
            feature_normalizer_filename = get_feature_normalizer_filename(fold=fold, path=feature_normalizer_path)
            if os.path.isfile(feature_normalizer_filename):
                normalizer = load_data(feature_normalizer_filename)
            else:
                raise IOError("Feature normalizer not found [%s]" % feature_normalizer_filename)

            # Initialize model container
            model_container = {'normalizer': normalizer, 'models': {}}

            # Collect training examples
            file_count = len(dataset.train(fold))
            data = {}
            for item_id, item in enumerate(dataset.train(fold)):
                progress(title_text='Collecting data',
                         fold=fold,
                         percentage=(float(item_id) / file_count),
                         note=os.path.split(item['file'])[1])

                # Load features
                feature_filename = get_feature_filename(audio_file=item['file'], path=feature_path)
                if os.path.isfile(feature_filename):
                    feature_data = load_data(feature_filename)['feat']
                else:
                    raise IOError("Features not found [%s]" % (item['file']))

                # Scale features
                feature_data = model_container['normalizer'].normalize(feature_data)

                # Store features per class label
                if item['scene_label'] not in data:
                    data[item['scene_label']] = feature_data
                else:
                    data[item['scene_label']] = numpy.vstack((data[item['scene_label']], feature_data))
	    le = pp.LabelEncoder()

            tot_data = {}

            # Train models for each class
            for label in data:
                progress(title_text='Train models',
                         fold=fold,
                         note=label)
                if classifier_method == 'xgboost':
                    if 'x' not in tot_data:
                        tot_data['x'] = data[label]
                        tot_data['y'] = numpy.repeat(label,len(data[label]), axis=0)
                    else:
                        tot_data['x'] = numpy.vstack((tot_data['x'], data[label]))
                        tot_data['y'] = numpy.hstack((tot_data['y'], numpy.repeat(label, len(data[label]), axis=0)))
		else:
                    raise ValueError("Unknown classifier method ["+classifier_method+"]")
	    print tot_data['y']
	    tot_data['y'] = le.fit_transform(tot_data['y'])

	    DT = xgb.DMatrix(tot_data['x'], tot_data['y'])
	    #params = {'max_depth': 30, 'num_class': 10, 'objective' : 'multi:softmax'}
            #params = **classifier_params
	    print classifier_params
	    model_container['models'] = xgb.train(classifier_params, DT, 20)
	    # Save models
            save_data(current_model_file, model_container)

def do_system_testing(dataset, result_path, feature_path, model_path, feature_params,
                      dataset_evaluation_mode='folds', classifier_method='xgboost', overwrite=False):
    """System testing.

    If extracted features are not found from disk, they are extracted but not saved.

    Parameters
    ----------
    dataset : class
        dataset class

    result_path : str
        path where the results are saved.

    feature_path : str
        path where the features are saved.

    model_path : str
        path where the models are saved.

    feature_params : dict
        parameter dict

    dataset_evaluation_mode : str ['folds', 'full']
        evaluation mode, 'full' all material available is considered to belong to one fold.
        (Default value='folds')

    classifier_method : str ['xgboost']
        classifier method, currently only GMM supported
        (Default value='xgboost')

    overwrite : bool
        overwrite existing models
        (Default value=False)

    Returns
    -------
    nothing

    Raises
    -------
    ValueError
        classifier_method is unknown.

    IOError
        Model file not found.
        Audio file not found.

    """

    if classifier_method != 'xgboost':
        raise ValueError("Unknown classifier method ["+classifier_method+"]")

    # Check that target path exists, create if not
    check_path(result_path)

    for fold in dataset.folds(mode=dataset_evaluation_mode):
        current_result_file = get_result_filename(fold=fold, path=result_path)
        if not os.path.isfile(current_result_file) or overwrite:
            results = []

            # Load class model container
            model_filename = get_model_filename(fold=fold, path=model_path)
            if os.path.isfile(model_filename):
                model_container = load_data(model_filename)
            else:
                raise IOError("Model file not found [%s]" % model_filename)

            file_count = len(dataset.test(fold))
            for file_id, item in enumerate(dataset.test(fold)):
                progress(title_text='Testing',
                         fold=fold,
                         percentage=(float(file_id) / file_count),
                         note=os.path.split(item['file'])[1])

                # Load features
                feature_filename = get_feature_filename(audio_file=item['file'], path=feature_path)

                if os.path.isfile(feature_filename):
                    feature_data = load_data(feature_filename)['feat']
                else:
                    # Load audio
                    if os.path.isfile(dataset.relative_to_absolute_path(item['file'])):
                        y, fs = load_audio(filename=dataset.relative_to_absolute_path(item['file']), mono=True, fs=feature_params['fs'])
                    else:
                        raise IOError("Audio file not found [%s]" % (item['file']))

                    feature_data = feature_extraction(y=y,
                                                      fs=fs,
                                                      include_mfcc0=feature_params['include_mfcc0'],
                                                      include_delta=feature_params['include_delta'],
                                                      include_acceleration=feature_params['include_acceleration'],
                                                      mfcc_params=feature_params['mfcc'],
                                                      delta_params=feature_params['mfcc_delta'],
                                                      acceleration_params=feature_params['mfcc_acceleration'],
                                                      statistics=False)['feat']

                # Normalize features
                feature_data = model_container['normalizer'].normalize(feature_data)

                # Do classification for the block
                if classifier_method == 'xgboost':
		    #feature_data = xgb.DMatrix(feature_data, numpy.repeat(item['scene_label'], len(feature_data), axis=0))
	            feature_data = xgb.DMatrix(feature_data)
		    current_result = dataset.scene_labels[do_classification_xgboost(feature_data, model_container)]
		else:
                    raise ValueError("Unknown classifier method ["+classifier_method+"]")

                # Store the result
                results.append((dataset.absolute_to_relative(item['file']), current_result))

            # Save testing results
            with open(current_result_file, 'wt') as f:
                writer = csv.writer(f, delimiter='\t')
                for result_item in results:
                    writer.writerow(result_item)

def do_classification_gmm(feature_data, model_container):
    """GMM classification for give feature matrix

    model container format:

    {
        'normalizer': normalizer class
        'models' :
            {
                'office' : mixture.GMM class
                'home' : mixture.GMM class
                ...
            }
    }

    Parameters
    ----------
    feature_data : numpy.ndarray [shape=(t, feature vector length)]
        feature matrix

    model_container : dict
        model container

    Returns
    -------
    result : str
        classification result as scene label

    """

    # Initialize log-likelihood matrix to -inf
    logls = numpy.empty(len(model_container['models']))
    logls.fill(-numpy.inf)

    for label_id, label in enumerate(model_container['models']):
        logls[label_id] = numpy.sum(model_container['models'][label].score(feature_data))

    classification_result_id = numpy.argmax(logls)
    return model_container['models'].keys()[classification_result_id]

def do_classification_xgboost(feature_data, model_container):
    """xgboost classification for give feature matrix

    model container format:

    {
        'normalizer': normalizer class
        'models' :
            {
                'office' : mixture.GMM class
                'home' : mixture.GMM class
                ...
            }
    }

    Parameters
    ----------
    feature_data : numpy.ndarray [shape=(t, feature vector length)]
        feature matrix

    model_container : dict
        model container

    Returns
    -------
    result : str
        classification result as scene label

    """

    # Initialize log-likelihood matrix to -inf
    logls = numpy.empty(10)
    logls.fill(-numpy.inf)

    #for label_id, label in enumerate(model_container['models']):
    #    logls[label_id] = numpy.sum(model_container['models'][label].score(feature_data))
    #logls = numpy.sum(model_container['models'].predict_log_proba(feature_data),0)
    logls = numpy.sum(numpy.log(model_container['models'].predict(feature_data)),0)

    classification_result_id = numpy.argmax(logls)
    return classification_result_id

def do_system_evaluation(dataset, result_path, dataset_evaluation_mode='folds'):
    """System evaluation. Testing outputs are collected and evaluated. Evaluation results are printed.

    Parameters
    ----------
    dataset : class
        dataset class

    result_path : str
        path where the results are saved.

    dataset_evaluation_mode : str ['folds', 'full']
        evaluation mode, 'full' all material available is considered to belong to one fold.
        (Default value='folds')

    Returns
    -------
    nothing

    Raises
    -------
    IOError
        Result file not found

    """

    dcase2016_scene_metric = DCASE2016_SceneClassification_Metrics(class_list=dataset.scene_labels)
    tot_cm = numpy.zeros((dataset.scene_label_count, dataset.scene_label_count))
    results_fold = []
    for fold in dataset.folds(mode=dataset_evaluation_mode):
        dcase2016_scene_metric_fold = DCASE2016_SceneClassification_Metrics(class_list=dataset.scene_labels)
        results = []
        result_filename = get_result_filename(fold=fold, path=result_path)

        if os.path.isfile(result_filename):
            with open(result_filename, 'rt') as f:
                for row in csv.reader(f, delimiter='\t'):
                    results.append(row)
        else:
            raise IOError("Result file not found [%s]" % result_filename)

        y_true = []
        y_pred = []
        for result in results:
            y_true.append(dataset.file_meta(result[0])[0]['scene_label'])
            y_pred.append(result[1])
        dcase2016_scene_metric.evaluate(system_output=y_pred, annotated_ground_truth=y_true)
        dcase2016_scene_metric_fold.evaluate(system_output=y_pred, annotated_ground_truth=y_true)
        results_fold.append(dcase2016_scene_metric_fold.results())
    	tot_cm += confusion_matrix(y_true, y_pred)

    print tot_cm
    #plot_cm(tot_cm, dataset.scene_labels,name=plot_name)
    #joblib.dump(tot_cm, plot_name + '.pkl')
    final_result['tot_cm'] = tot_cm
    final_result['tot_cm_acc'] = numpy.sum(numpy.diag(tot_cm))/numpy.sum(tot_cm)

    results = dcase2016_scene_metric.results()

    print "  File-wise evaluation, over %d folds" % dataset.fold_count
    fold_labels = ''
    separator = '     =====================+======+======+==========+  +'
    if dataset.fold_count > 1:
        for fold in dataset.folds(mode=dataset_evaluation_mode):
            fold_labels += " {:8s} |".format('Fold'+str(fold))
            separator += "==========+"
    print "     {:20s} | {:4s} : {:4s} | {:8s} |  |".format('Scene label', 'Nref', 'Nsys', 'Accuracy')+fold_labels
    print separator
    for label_id, label in enumerate(sorted(results['class_wise_accuracy'])):
        fold_values = ''
        if dataset.fold_count > 1:
            for fold in dataset.folds(mode=dataset_evaluation_mode):
                fold_values += " {:5.1f} %  |".format(results_fold[fold-1]['class_wise_accuracy'][label] * 100)
        print "     {:20s} | {:4d} : {:4d} | {:5.1f} %  |  |".format(label,
                                                                     results['class_wise_data'][label]['Nref'],
                                                                     results['class_wise_data'][label]['Nsys'],
                                                                     results['class_wise_accuracy'][label] * 100)+fold_values
    print separator
    fold_values = ''
    if dataset.fold_count > 1:
        for fold in dataset.folds(mode=dataset_evaluation_mode):
            fold_values += " {:5.1f} %  |".format(results_fold[fold-1]['overall_accuracy'] * 100)

    print "     {:20s} | {:4d} : {:4d} | {:5.1f} %  |  |".format('Overall accuracy',
                                                                 results['Nref'],
                                                                 results['Nsys'],
                                                                 results['overall_accuracy'] * 100)+fold_values

    final_result['result'] = results

if __name__ == "__main__":
    try:
        sys.exit(main(sys.argv))
    except (ValueError, IOError) as e:
        sys.exit(e)


text:
package org.apache.activemq.usecases;

import javax.jms.Destination;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageProducer;

import org.apache.activemq.test.JmsTopicSendReceiveWithTwoConnectionsTest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 *
 */
public class PublishOnTopicConsumedMessageTest extends JmsTopicSendReceiveWithTwoConnectionsTest {

   private static final Logger LOG = LoggerFactory.getLogger(PublishOnTopicConsumedMessageTest.class);

   private MessageProducer replyProducer;

   public synchronized void onMessage(Message message) {

      // lets resend the message somewhere else
      try {
         Message msgCopy = (Message) ((org.apache.activemq.command.Message) message).copy();
         replyProducer.send(msgCopy);

         // log.info("Sending reply: " + message);
         super.onMessage(message);
      }
      catch (JMSException e) {
         LOG.info("Failed to send message: " + e);
         e.printStackTrace();
      }
   }

   protected void setUp() throws Exception {
      super.setUp();

      Destination replyDestination = null;

      if (topic) {
         replyDestination = receiveSession.createTopic("REPLY." + getSubject());
      }
      else {
         replyDestination = receiveSession.createQueue("REPLY." + getSubject());
      }

      replyProducer = receiveSession.createProducer(replyDestination);
      LOG.info("Created replyProducer: " + replyProducer);

   }
}

redpajama_set_name: RedPajamaGithub


text:
package org.apache.nifi.toolkit.cli.impl.command.nifi.pg;

import org.apache.commons.cli.MissingOptionException;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.toolkit.cli.api.Context;
import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient;
import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient;
import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException;
import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupBox;
import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient;
import org.apache.nifi.toolkit.cli.impl.command.CommandOption;
import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand;
import org.apache.nifi.toolkit.cli.impl.result.StringResult;
import org.apache.nifi.web.api.dto.PositionDTO;
import org.apache.nifi.web.api.dto.ProcessGroupDTO;
import org.apache.nifi.web.api.dto.VersionControlInformationDTO;
import org.apache.nifi.web.api.entity.ProcessGroupEntity;
import org.apache.nifi.web.api.entity.RegistryClientEntity;
import org.apache.nifi.web.api.entity.RegistryClientsEntity;

import java.io.IOException;
import java.util.Properties;
import java.util.Set;

/**
 * Command for importing a flow to NiFi from NiFi Registry.
 */
public class PGImport extends AbstractNiFiCommand<StringResult> {

    public PGImport() {
        super("pg-import", StringResult.class);
    }

    @Override
    public String getDescription() {
        return "Creates a new process group by importing a versioned flow from a registry. If no process group id is " +
                "specified, then the created process group will be placed in the root group. If only one registry client " +
                "exists in NiFi, then it does not need to be specified and will be automatically selected. The x and y " +
                "coordinates for the position of the imported process group may be optionally specified. If left blank, " +
                "the position will automatically be selected to avoid overlapping with existing process groups.";
    }

    @Override
    protected void doInitialize(Context context) {
        addOption(CommandOption.PG_ID.createOption());
        addOption(CommandOption.REGISTRY_CLIENT_ID.createOption());
        addOption(CommandOption.BUCKET_ID.createOption());
        addOption(CommandOption.FLOW_ID.createOption());
        addOption(CommandOption.FLOW_VERSION.createOption());
        addOption(CommandOption.POS_X.createOption());
        addOption(CommandOption.POS_Y.createOption());
    }

    @Override
    public StringResult doExecute(final NiFiClient client, final Properties properties)
            throws NiFiClientException, IOException, MissingOptionException {

        final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID);
        final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID);
        final Integer flowVersion = getRequiredIntArg(properties, CommandOption.FLOW_VERSION);

        final String posXStr = getArg(properties, CommandOption.POS_X);
        final String posYStr = getArg(properties, CommandOption.POS_Y);

        final boolean posXExists = StringUtils.isNotBlank(posXStr);
        final boolean posYExists = StringUtils.isNotBlank(posYStr);

        if ((posXExists && !posYExists)) {
            throw new IllegalArgumentException("Missing Y position - Please specify both X and Y, or specify neither");
        }

        if ((posYExists && !posXExists)) {
            throw new IllegalArgumentException("Missing X position - Please specify both X and Y, or specify neither");
        }

        // if a registry client is specified use it, otherwise see if there is only one available and use that,
        // if more than one is available then throw an exception because we don't know which one to use
        String registryId = getArg(properties, CommandOption.REGISTRY_CLIENT_ID);
        if (StringUtils.isBlank(registryId)) {
            final RegistryClientsEntity registries = client.getControllerClient().getRegistryClients();

            final Set<RegistryClientEntity> entities = registries.getRegistries();
            if (entities == null || entities.isEmpty()) {
                throw new NiFiClientException("No registry clients available");
            }

            if (entities.size() == 1) {
                registryId = entities.stream().findFirst().get().getId();
            } else {
                throw new MissingOptionException(CommandOption.REGISTRY_CLIENT_ID.getLongName()
                        + " must be provided when there is more than one available");
            }
        }

        // get the optional id of the parent PG, otherwise fallback to the root group
        String parentPgId = getArg(properties, CommandOption.PG_ID);
        if (StringUtils.isBlank(parentPgId)) {
            final FlowClient flowClient = client.getFlowClient();
            parentPgId = flowClient.getRootGroupId();
        }

        final VersionControlInformationDTO versionControlInfo = new VersionControlInformationDTO();
        versionControlInfo.setRegistryId(registryId);
        versionControlInfo.setBucketId(bucketId);
        versionControlInfo.setFlowId(flowId);
        versionControlInfo.setVersion(flowVersion);

        final PositionDTO posDto = new PositionDTO();
        if (posXExists && posYExists) {
            posDto.setX(Double.parseDouble(posXStr));
            posDto.setY(Double.parseDouble(posYStr));
        } else {
            final ProcessGroupBox pgBox = client.getFlowClient().getSuggestedProcessGroupCoordinates(parentPgId);
            posDto.setX(Integer.valueOf(pgBox.getX()).doubleValue());
            posDto.setY(Integer.valueOf(pgBox.getY()).doubleValue());
        }

        final ProcessGroupDTO pgDto = new ProcessGroupDTO();
        pgDto.setVersionControlInformation(versionControlInfo);
        pgDto.setPosition(posDto);

        final ProcessGroupEntity pgEntity = new ProcessGroupEntity();
        pgEntity.setComponent(pgDto);
        pgEntity.setRevision(getInitialRevisionDTO());

        final ProcessGroupClient pgClient = client.getProcessGroupClient();
        final ProcessGroupEntity createdEntity = pgClient.createProcessGroup(parentPgId, pgEntity);
        return new StringResult(createdEntity.getId(), getContext().isInteractive());
    }

}


text:
package memorysystem;

public interface GenericPooledLinkedListInterface<T> {

	int compare(T arg0, T arg1);
	void copy(T src, T dest);

}


text:
<?php

/**
 * @namespace
 */
namespace Zend\Server;

/**
 * \Zend\Server\Cache: cache server definitions
 *
 * @uses       \Zend\Server\Definition
 * @category   Zend
 * @package    Zend_Server
 * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
 * @license    http://framework.zend.com/license/new-bsd     New BSD License
 */
class Cache
{
    /**
     * @var array Methods to skip when caching server
     */
    protected static $_skipMethods = array();

    /**
     * Cache a file containing the dispatch list.
     *
     * Serializes the server definition stores the information
     * in $filename.
     *
     * Returns false on any error (typically, inability to write to file), true
     * on success.
     *
     * @param  string $filename
     * @param  \Zend\Server\Server $server
     * @return bool
     */
    public static function save($filename, Server $server)
    {
        if (!is_string($filename)
            || (!file_exists($filename) && !is_writable(dirname($filename))))
        {
            return false;
        }

        $methods = $server->getFunctions();

        if ($methods instanceof Definition) {
            $definition = new Definition();
            foreach ($methods as $method) {
                if (in_array($method->getName(), self::$_skipMethods)) {
                    continue;
                }
                $definition->addMethod($method);
            }
            $methods = $definition;
        }

        if (0 === @file_put_contents($filename, serialize($methods))) {
            return false;
        }

        return true;
    }

    /**
     * Load server definition from a file
     *
     * Unserializes a stored server definition from $filename. Returns false if
     * it fails in any way, true on success.
     *
     * Useful to prevent needing to build the server definition on each
     * request. Sample usage:
     *
     * <code>
     * if (!Zend\Server\Cache::get($filename, $server)) {
     *     require_once 'Some/Service/ServiceClass.php';
     *     require_once 'Another/Service/ServiceClass.php';
     *
     *     // Attach Some\Service\ServiceClass with namespace 'some'
     *     $server->attach('Some\Service\ServiceClass', 'some');
     *
     *     // Attach Another\Service\ServiceClass with namespace 'another'
     *     $server->attach('Another\Service\ServiceClass', 'another');
     *
     *     Zend\Server\Cache::save($filename, $server);
     * }
     *
     * $response = $server->handle();
     * echo $response;
     * </code>
     *
     * @param  string $filename
     * @param  \Zend\Server\Server $server
     * @return bool
     */
    public static function get($filename, Server $server)
    {
        if (!is_string($filename)
            || !file_exists($filename)
            || !is_readable($filename))
        {
            return false;
        }

        if (false === ($dispatch = @file_get_contents($filename))) {
            return false;
        }

        if (false === ($dispatchArray = @unserialize($dispatch))) {
            return false;
        }

        $server->loadFunctions($dispatchArray);

        return true;
    }

    /**
     * Remove a cache file
     *
     * @param  string $filename
     * @return boolean
     */
    public static function delete($filename)
    {
        if (is_string($filename) && file_exists($filename)) {
            unlink($filename);
            return true;
        }

        return false;
    }
}


text:
import json
from chargebee.model import Model
from chargebee import request
from chargebee import APIError

class ImpactedItem(Model):
    class Download(Model):
      fields = ["download_url", "valid_till", "mime_type"]
      pass

    fields = ["count", "download", "items"]


text:
package com.comcast.xfinity.sirius.api.impl.state

import akka.actor.{Props, Actor}
import com.comcast.xfinity.sirius.api.impl._
import com.comcast.xfinity.sirius.api.{SiriusResult, RequestHandler}
import akka.event.Logging
import com.comcast.xfinity.sirius.api.impl.SiriusRequest

object SiriusStateActor {

  /**
   * Create Props for a SiriusStateActor.
   *
   * @param requestHandler the RequestHandler containing the callbacks for manipulating this instance's state
   * @return  Props for creating this actor, which can then be further configured
   *         (e.g. calling `.withDispatcher()` on it)
   */
  def props(requestHandler: RequestHandler): Props = {
    Props(classOf[SiriusStateActor], requestHandler)
  }
}
/**
 * Actor responsible for a applying changes to our in memory state.  It does this by
 * delegating to the passed in {@link RequestHandler}.  The result of calling the
 * associated method on the passed in RequestHandler is sent back to the sender.
 *
 * In the future we may want to not respond to Put and Delete messages...
 *
 * @param requestHandler the request handler containing callbacks for manipulating state
 */
class SiriusStateActor(requestHandler: RequestHandler) extends Actor {

  val logger = Logging(context.system, "Sirius")

  def receive = {
    case req: SiriusRequest =>
      // XXX: With the way things work now, we probably shouldn't
      //      be responding to Puts and Deletes...  These are
      //      responded to when an order has been decided on
      sender ! processSiriusRequestSafely(req)
  }

  private def processSiriusRequestSafely(req: SiriusRequest): SiriusResult = {
    try {
      req match {
        case Get(key) => requestHandler.handleGet(key)
        case Delete(key) => requestHandler.handleDelete(key)
        case Put(key, body) => requestHandler.handlePut(key, body)
      }
    } catch {
      case e: RuntimeException =>
        logger.warning("Unhandled exception in handling {}: {}", req, e)
        SiriusResult.error(e)
    }
  }
}


text:
Skills

                            {% for skill in site.skills %}

                                            {{skill.name}}
                                            {{skill.percent}}%

                            {% endfor %}


text:
package io.sarl.lang.scoping.extensions.numbers.arithmetic;

import org.eclipse.xtext.xbase.lib.Inline;
import org.eclipse.xtext.xbase.lib.Pure;

/** Provide static operators for numbers of type {@code Number}.
 *
 * @author $Author: sgalland$
 * @version $FullVersion$
 * @mavengroupid $GroupId$
 * @mavenartifactid $ArtifactId$
 * @since 0.7
 * @see "https://github.com/eclipse/xtext-extras/issues/186"
 */
@SuppressWarnings("checkstyle:methodcount")
public final class NumberArithmeticExtensions {

	private NumberArithmeticExtensions() {
		//
	}

	/** The unary {@code minus} operator. This is the equivalent to
	 * the Java's {@code -} function. This function is not null-safe.
	 *
	 * @param number a number.
	 * @return {@code -number}
	 */
	@Pure
	@Inline(value = "(-($1.doubleValue()))", constantExpression = true)
	public static double operator_minus(Number number) {
		return -number.doubleValue();
	}

	/** The binary {@code minus} operator. This is the equivalent to
	 * the Java {@code -} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left-right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() - $2)", constantExpression = true)
	public static double operator_minus(Number left, long right) {
		return left.doubleValue() - right;
	}

	/** The binary {@code minus} operator. This is the equivalent to
	 * the Java {@code -} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left-right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() - $2)", constantExpression = true)
	public static double operator_minus(Number left, byte right) {
		return left.doubleValue() - right;
	}

	/** The binary {@code minus} operator. This is the equivalent to
	 * the Java {@code -} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left-right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() - $2)", constantExpression = true)
	public static double operator_minus(Number left, int right) {
		return left.doubleValue() - right;
	}

	/** The binary {@code minus} operator. This is the equivalent to
	 * the Java {@code -} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left-right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() - $2)", constantExpression = true)
	public static double operator_minus(Number left, short right) {
		return left.doubleValue() - right;
	}

	/** The binary {@code minus} operator. This is the equivalent to
	 * the Java {@code -} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left-right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() - $2)", constantExpression = true)
	public static double operator_minus(Number left, double right) {
		return left.doubleValue() - right;
	}

	/** The binary {@code minus} operator. This is the equivalent to
	 * the Java {@code -} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left-right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() - $2)", constantExpression = true)
	public static double operator_minus(Number left, float right) {
		return left.doubleValue() - right;
	}

	/** The binary {@code minus} operator. This is the equivalent to
	 * the Java {@code -} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left-right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() - $2.doubleValue())", constantExpression = true)
	public static double operator_minus(Number left, Number right) {
		return left.doubleValue() - right.doubleValue();
	}

	/** The binary {@code plus} operator. This is the equivalent to
	 * the Java {@code +} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left+right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() + $2)", constantExpression = true)
	public static double operator_plus(Number left, long right) {
		return left.doubleValue() + right;
	}

	/** The binary {@code plus} operator. This is the equivalent to
	 * the Java {@code +} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left+right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() + $2)", constantExpression = true)
	public static double operator_plus(Number left, byte right) {
		return left.doubleValue() + right;
	}

	/** The binary {@code plus} operator. This is the equivalent to
	 * the Java {@code +} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left+right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() + $2)", constantExpression = true)
	public static double operator_plus(Number left, int right) {
		return left.doubleValue() + right;
	}

	/** The binary {@code plus} operator. This is the equivalent to
	 * the Java {@code +} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left+right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() + $2)", constantExpression = true)
	public static double operator_plus(Number left, short right) {
		return left.doubleValue() + right;
	}

	/** The binary {@code plus} operator. This is the equivalent to
	 * the Java {@code +} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left+right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() + $2)", constantExpression = true)
	public static double operator_plus(Number left, double right) {
		return left.doubleValue() + right;
	}

	/** The binary {@code plus} operator. This is the equivalent to
	 * the Java {@code +} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left+right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() + $2)", constantExpression = true)
	public static double operator_plus(Number left, float right) {
		return left.doubleValue() + right;
	}

	/** The binary {@code plus} operator. This is the equivalent to
	 * the Java {@code +} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left+right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() + $2.doubleValue())", constantExpression = true)
	public static double operator_plus(Number left, Number right) {
		return left.doubleValue() + right.doubleValue();
	}

	/** The binary {@code power} operator. This is the equivalent to
	 * the Java's {@code Math.pow()} function. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code Math::pow(left, right)}
	 */
	@Pure
	@Inline(value = "$3.pow($1.doubleValue(), $2)", imported = Math.class)
	public static double operator_power(Number left, long right) {
		return Math.pow(left.doubleValue(), right);
	}

	/** The binary {@code power} operator. This is the equivalent to
	 * the Java's {@code Math.pow()} function. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code Math::pow(left, right)}
	 */
	@Pure
	@Inline(value = "$3.pow($1.doubleValue(), $2)", imported = Math.class)
	public static double operator_power(Number left, byte right) {
		return Math.pow(left.doubleValue(), right);
	}

	/** The binary {@code power} operator. This is the equivalent to
	 * the Java's {@code Math.pow()} function. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code Math::pow(left, right)}
	 */
	@Pure
	@Inline(value = "$3.pow($1.doubleValue(), $2)", imported = Math.class)
	public static double operator_power(Number left, int right) {
		return Math.pow(left.doubleValue(), right);
	}

	/** The binary {@code power} operator. This is the equivalent to
	 * the Java's {@code Math.pow()} function.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code Math::pow(left, right)}
	 */
	@Pure
	@Inline(value = "$3.pow($1.doubleValue(), $2)", imported = Math.class)
	public static double operator_power(Number left, short right) {
		return Math.pow(left.doubleValue(), right);
	}

	/** The binary {@code power} operator. This is the equivalent to
	 * the Java's {@code Math.pow()} function. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code Math::pow(left, right)}
	 */
	@Pure
	@Inline(value = "$3.pow($1.doubleValue(), $2)", imported = Math.class)
	public static double operator_power(Number left, double right) {
		return Math.pow(left.doubleValue(), right);
	}

	/** The binary {@code power} operator. This is the equivalent to
	 * the Java's {@code Math.pow()} function. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code Math::pow(left, right)}
	 */
	@Pure
	@Inline(value = "$3.pow($1.doubleValue(), $2)", imported = Math.class)
	public static double operator_power(Number left, float right) {
		return Math.pow(left.doubleValue(), right);
	}

	/** The binary {@code power} operator. This is the equivalent to
	 * the Java's {@code Math.pow()} function.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code Math::pow(left, right)}
	 */
	@Pure
	@Inline(value = "$3.pow($1.doubleValue(), $2.doubleValue())", imported = Math.class)
	public static double operator_power(Number left, Number right) {
		return Math.pow(left.doubleValue(), right.doubleValue());
	}

	/** The binary {@code divide} operator. This is the equivalent to
	 * the Java {@code /} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left/right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() / $2)", constantExpression = true)
	public static double operator_divide(Number left, long right) {
		return left.doubleValue() / right;
	}

	/** The binary {@code divide} operator. This is the equivalent to
	 * the Java {@code /} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left/right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() / $2)", constantExpression = true)
	public static double operator_divide(Number left, byte right) {
		return left.doubleValue() / right;
	}

	/** The binary {@code divide} operator. This is the equivalent to
	 * the Java {@code /} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left/right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() / $2)", constantExpression = true)
	public static double operator_divide(Number left, int right) {
		return left.doubleValue() / right;
	}

	/** The binary {@code divide} operator. This is the equivalent to
	 * the Java {@code /} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left/right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() / $2)", constantExpression = true)
	public static double operator_divide(Number left, short right) {
		return left.doubleValue() / right;
	}

	/** The binary {@code divide} operator. This is the equivalent to
	 * the Java {@code /} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left/right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() / $2)", constantExpression = true)
	public static double operator_divide(Number left, double right) {
		return left.doubleValue() / right;
	}

	/** The binary {@code divide} operator. This is the equivalent to
	 * the Java {@code /} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left/right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() / $2)", constantExpression = true)
	public static double operator_divide(Number left, float right) {
		return left.doubleValue() / right;
	}

	/** The binary {@code divide} operator. This is the equivalent to
	 * the Java {@code /} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left/right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() / $2.doubleValue())", constantExpression = true)
	public static double operator_divide(Number left, Number right) {
		return left.doubleValue() / right.doubleValue();
	}

	/** The binary {@code multiply} operator. This is the equivalent to
	 * the Java {@code *} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left*right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() * $2)", constantExpression = true)
	public static double operator_multiply(Number left, long right) {
		return left.doubleValue() * right;
	}

	/** The binary {@code multiply} operator. This is the equivalent to
	 * the Java {@code *} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left*right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() * $2)", constantExpression = true)
	public static double operator_multiply(Number left, byte right) {
		return left.doubleValue() * right;
	}

	/** The binary {@code multiply} operator. This is the equivalent to
	 * the Java {@code *} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left*right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() * $2)", constantExpression = true)
	public static double operator_multiply(Number left, int right) {
		return left.doubleValue() * right;
	}

	/** The binary {@code multiply} operator. This is the equivalent to
	 * the Java {@code *} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left*right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() * $2)", constantExpression = true)
	public static double operator_multiply(Number left, short right) {
		return left.doubleValue() * right;
	}

	/** The binary {@code multiply} operator. This is the equivalent to
	 * the Java {@code *} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left*right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() * $2)", constantExpression = true)
	public static double operator_multiply(Number left, double right) {
		return left.doubleValue() * right;
	}

	/** The binary {@code multiply} operator. This is the equivalent to
	 * the Java {@code *} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left*right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() * $2)", constantExpression = true)
	public static double operator_multiply(Number left, float right) {
		return left.doubleValue() * right;
	}

	/** The binary {@code multiply} operator. This is the equivalent to
	 * the Java {@code *} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left*right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() * $2.doubleValue())", constantExpression = true)
	public static double operator_multiply(Number left, Number right) {
		return left.doubleValue() * right.doubleValue();
	}

	/** The binary {@code modulo} operator. This is the equivalent to
	 * the Java {@code %} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left%right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() % $2)", constantExpression = true)
	public static double operator_modulo(Number left, long right) {
		return left.doubleValue() % right;
	}

	/** The binary {@code modulo} operator. This is the equivalent to
	 * the Java {@code %} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left%right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() % $2)", constantExpression = true)
	public static double operator_modulo(Number left, byte right) {
		return left.doubleValue() % right;
	}

	/** The binary {@code modulo} operator. This is the equivalent to
	 * the Java {@code %} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left%right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() % $2)", constantExpression = true)
	public static double operator_modulo(Number left, int right) {
		return left.doubleValue() % right;
	}

	/** The binary {@code modulo} operator. This is the equivalent to
	 * the Java {@code %} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left%right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() % $2)", constantExpression = true)
	public static double operator_modulo(Number left, short right) {
		return left.doubleValue() % right;
	}

	/** The binary {@code modulo} operator. This is the equivalent to
	 * the Java {@code %} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left%right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() % $2)", constantExpression = true)
	public static double operator_modulo(Number left, double right) {
		return left.doubleValue() % right;
	}

	/** The binary {@code modulo} operator. This is the equivalent to
	 * the Java {@code %} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left%right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() % $2)", constantExpression = true)
	public static double operator_modulo(Number left, float right) {
		return left.doubleValue() % right;
	}

	/** The binary {@code modulo} operator. This is the equivalent to
	 * the Java {@code %} operator. This function is not null-safe.
	 *
	 * @param left a number.
	 * @param right a number.
	 * @return {@code left%right}
	 */
	@Pure
	@Inline(value = "($1.doubleValue() % $2.doubleValue())", constantExpression = true)
	public static double operator_modulo(Number left, Number right) {
		return left.doubleValue() % right.doubleValue();
	}

}


text:
function emptyToSparse (str) {
  return (!str || !str.length)
    ? undefined
    : str
}

// Validator to ensure that a property exists
function validatePresenceOf (value) {
  return value && value.length
}

// Create an object out of a nested path
function nestedPath (obj, path, val) {
  if (typeof obj !== 'object') {
    return obj
  }
  var keys = path.split('.')
  if (keys.length > 1) {
    path = keys.shift()
    return nestedPath(obj[path], keys.join('.'), val)
  }
  if (val !== undefined) {
    obj[path] = val
  }
  return obj[path]
}

// Convert a document or an array of documents to objects
function dataToObjects (data) {
  if (!data) return null
  if (data instanceof Array) {
    return data.map(function(doc) {
      return doc.toObject()
    })
  }
  return data.toObject()
}

// Execute if function or return object
function objectOrFunction(obj) {
  if (toString.call(obj) == '[object Function]') {
    return obj()
  }
  return obj
}

// Destroy a database and disconnect
function dropDatabase (db, fn) {
  db.connection.db.executeDbCommand({
    dropDatabase: 1
  }, function(err, result) {
    db.disconnect()
    fn && fn(err, result)
  })
}

// Drop all collections in a database
function dropCollections (db, each, fn) {
  if (each && !fn) { fn = each; each = null }
  var conn = db.connection
    , cols = Object.keys(conn.collections)

  cols.forEach(function (name, k) {
    console.log('name: ', name, k)
    conn.collections[name].drop(function (err) {
      if (err == 'ns not found') {
        err = null
      }
      each && each(err, name)
    })
  })
  fn && fn(null)
}

// Exports
module.exports = {
  emptyToSparse: emptyToSparse
, validatePresenceOf: validatePresenceOf
, nestedPath: nestedPath
, dataToObjects: dataToObjects
, objectOrFunction: objectOrFunction
, dropDatabase: dropDatabase
, dropCollections: dropCollections
}


text:
local import = require("import")
local env = {import=import}

local _ENV = import(
env, -- import inside this environment
{ "math", -- from the module
	"min", -- 'math.min' imported as 'min'
	{"pi", "PI"} -- 'math.pi' imported as 'PI'
},
{ "_G",
	"print",
})
print( min(PI, 3, 10) == 3 and "ok" or "error" )


text:
using Microsoft.Graph;

// **NOTE** This file was generated by a tool and any changes will be overwritten.

namespace Microsoft.OneDrive.Sdk
{
    /// <summary>
    /// The type ItemChildrenCollectionPage.
    /// </summary>
    public partial class ItemChildrenCollectionPage : CollectionPage<Item>, IItemChildrenCollectionPage
    {
        /// <summary>
        /// Gets the next page <see cref="IItemChildrenCollectionRequest"/> instance.
        /// </summary>
        public IItemChildrenCollectionRequest NextPageRequest { get; private set; }

        /// <summary>
        /// Initializes the NextPageRequest property.
        /// </summary>
        public void InitializeNextPageRequest(IBaseClient client, string nextPageLinkString)
        {
            if (!string.IsNullOrEmpty(nextPageLinkString))
            {
                this.NextPageRequest = new ItemChildrenCollectionRequest(
                    nextPageLinkString,
                    client,
                    null);
            }
        }
    }
}


text:
package sangria.schema

import sangria.execution.ValueCoercionHelper
import sangria.renderer.SchemaRenderer

import language.existentials

object SchemaComparator {
  def compare(oldSchema: Schema[_, _], newSchema: Schema[_, _]): Vector[SchemaChange] =
    findChanges(oldSchema, newSchema)

  private def findChanges(oldSchema: Schema[_, _], newSchema: Schema[_, _]) = {
    val oldTypes = oldSchema.availableTypeNames.toSet
    val newTypes = newSchema.availableTypeNames.toSet

    val removed = oldTypes.diff(newTypes).toVector.map(name ⇒
      SchemaChange.TypeRemoved(oldSchema.types(name)._2))

    val added = newTypes.diff(oldTypes).toVector.map(name ⇒
      SchemaChange.TypeAdded(newSchema.types(name)._2))

    val changed = oldTypes.intersect(newTypes).flatMap { name ⇒
      val oldType = oldSchema.types(name)._2
      val newType = newSchema.types(name)._2

      if (oldType.getClass == newType.getClass)
        findChangesInTypes(oldType, newType)
      else
        Vector(SchemaChange.TypeKindChanged(newType, oldType))
    }

    removed ++ added ++ changed ++
      findChangesInSchema(oldSchema, newSchema) ++
      findInDirectives(oldSchema, newSchema)
  }

  private def findInDirectives(oldSchema: Schema[_, _], newSchema: Schema[_, _]) = {
    val oldDirs = oldSchema.directives.map(_.name).toSet
    val newDirs = newSchema.directives.map(_.name).toSet

    val removed = oldDirs.diff(newDirs).toVector.map(name ⇒
      SchemaChange.DirectiveRemoved(oldSchema.directivesByName(name)))

    val added = newDirs.diff(oldDirs).toVector.map(name ⇒
      SchemaChange.DirectiveAdded(newSchema.directivesByName(name)))

    val changed = oldDirs.intersect(newDirs).flatMap { name ⇒
      val oldDir = oldSchema.directivesByName(name)
      val newDir = newSchema.directivesByName(name)

      findDescriptionChanged(oldDir, newDir, SchemaChange.DirectiveDescriptionChanged(newDir, _, _)) ++
        findInDirective(oldDir, newDir)
    }

    removed ++ added ++ changed
  }

  private def findInDirective(oldDir: Directive, newDir: Directive): Vector[SchemaChange] = {
    val locationChanges = findInDirectiveLocations(oldDir, newDir)
    val fieldChanges = findInArgs(oldDir.arguments, newDir.arguments,
      added = SchemaChange.DirectiveArgumentAdded(newDir, _, _),
      removed = SchemaChange.DirectiveArgumentRemoved(oldDir, _),
      description = SchemaChange.DirectiveArgumentDescriptionChanged(newDir, _, _, _),
      default = SchemaChange.DirectiveArgumentDefaultChanged(newDir, _, _, _),
      typeChange = SchemaChange.DirectiveArgumentTypeChanged(newDir, _, _, _, _))

    locationChanges ++ fieldChanges
  }

  private def findInDirectiveLocations(oldDir: Directive, newDir: Directive): Vector[SchemaChange] = {
    val oldLocs = oldDir.locations
    val newLocs = newDir.locations

    val removed = oldLocs.diff(newLocs).toVector.map(loc ⇒
      SchemaChange.DirectiveLocationRemoved(oldDir, loc))

    val added = newLocs.diff(oldLocs).toVector.map(loc ⇒
      SchemaChange.DirectiveLocationAdded(newDir, loc))

    removed ++ added
  }

  def findChangesInSchema(oldSchema: Schema[_, _], newSchema: Schema[_, _]): Vector[SchemaChange] = {
    val withQuery =
      if (oldSchema.query.name != newSchema.query.name)
        Vector(SchemaChange.SchemaQueryTypeChanged(oldSchema.query, newSchema.query))
      else
        Vector.empty

    val withMutation =
      if (oldSchema.mutation.map(_.name) != newSchema.mutation.map(_.name))
        withQuery :+ SchemaChange.SchemaMutationTypeChanged(oldSchema.mutation, newSchema.mutation)
      else
        withQuery

    if (oldSchema.subscription.map(_.name) != newSchema.subscription.map(_.name))
      withMutation :+ SchemaChange.SchemaSubscriptionTypeChanged(oldSchema.subscription, newSchema.subscription)
    else
      withMutation
  }

  def findChangesInTypes(oldType: Type with Named, newType: Type with Named): Vector[SchemaChange] = {
    val typeChanges = (oldType, newType) match {
      case (o: EnumType[_], n: EnumType[_]) ⇒ findInEnumTypes(o, n)
      case (o: UnionType[_], n: UnionType[_]) ⇒ findInUnionTypes(o, n)
      case (o: InputObjectType[_], n: InputObjectType[_]) ⇒ findInInputObjectTypes(o, n)
      case (o: ObjectType[_, _], n: ObjectType[_, _]) ⇒ findInObjectTypes(o, n)
      case (o: InterfaceType[_, _], n: InterfaceType[_, _]) ⇒ findInInterfaceTypes(o, n)
      case _ ⇒ Vector.empty
    }

    typeChanges ++ findDescriptionChanged(oldType, newType, SchemaChange.TypeDescriptionChanged(newType, _, _))
  }

  private def findInUnionTypes(oldType: UnionType[_], newType: UnionType[_]): Vector[SchemaChange] = {
    val oldTypes = oldType.types.map(_.name).toSet
    val newTypes = newType.types.map(_.name).toSet

    val removed = oldTypes.diff(newTypes).toVector.map(name ⇒
      SchemaChange.UnionMemberRemoved(oldType, oldType.types.find(_.name == name).get))

    val added = newTypes.diff(oldTypes).toVector.map(name ⇒
      SchemaChange.UnionMemberAdded(newType, newType.types.find(_.name == name).get))

    removed ++ added
  }

  private def findInEnumTypes(oldType: EnumType[_], newType: EnumType[_]): Vector[SchemaChange] = {
    val oldValues = oldType.values.map(_.name).toSet
    val newValues = newType.values.map(_.name).toSet

    val removed = oldValues.diff(newValues).toVector.map(name ⇒
      SchemaChange.EnumValueRemoved(oldType, oldType.byName(name)))

    val added = newValues.diff(oldValues).toVector.map(name ⇒
      SchemaChange.EnumValueAdded(newType, newType.byName(name)))

    val changed = oldValues.intersect(newValues).flatMap { name ⇒
      val oldValue = oldType.byName(name)
      val newValue = newType.byName(name)

      findDescriptionChanged(oldValue, newValue, SchemaChange.EnumValueDescriptionChanged(newType, newValue, _, _)) ++
        findDeprecationChanged(oldValue, newValue, SchemaChange.EnumValueDeprecated(newType, newValue, _, _))
    }

    removed ++ added ++ changed
  }

  private def findInInputObjectTypes(oldType: InputObjectType[_], newType: InputObjectType[_]): Vector[SchemaChange] = {
    val oldFields = oldType.fields.map(_.name).toSet
    val newFields = newType.fields.map(_.name).toSet

    val removed = oldFields.diff(newFields).toVector.map(name ⇒
      SchemaChange.InputFieldRemoved(oldType, oldType.fieldsByName(name)))

    val added = newFields.diff(oldFields).toVector.map { name ⇒
      val field = newType.fieldsByName(name)

      SchemaChange.InputFieldAdded(newType, field, !isOptional(field))
    }

    val changed = oldFields.intersect(newFields).flatMap { name ⇒
      val oldField = oldType.fieldsByName(name)
      val newField = newType.fieldsByName(name)

      findDescriptionChanged(oldField, newField, SchemaChange.InputFieldDescriptionChanged(newType, newField, _, _)) ++
        findInInputFields(oldType, newType, oldField, newField)
    }

    removed ++ added ++ changed
  }

  private def findInObjectTypes(oldType: ObjectType[_, _], newType: ObjectType[_, _]): Vector[SchemaChange] = {
    val interfaceChanges = findInObjectTypeInterfaces(oldType, newType)
    val fieldChanges = findInObjectTypeFields(oldType, newType)

    interfaceChanges ++ fieldChanges
  }

  private def findInInterfaceTypes(oldType: InterfaceType[_, _], newType: InterfaceType[_, _]): Vector[SchemaChange] =
    findInObjectTypeFields(oldType, newType)

  private def findInObjectTypeInterfaces(oldType: ObjectType[_, _], newType: ObjectType[_, _]): Vector[SchemaChange] = {
    val oldInts = oldType.allInterfaces.map(_.name).toSet
    val newInts = newType.allInterfaces.map(_.name).toSet

    val removed = oldInts.diff(newInts).toVector.map(name ⇒
      SchemaChange.ObjectTypeInterfaceRemoved(oldType, oldType.interfaces.find(_.name == name).get))

    val added = newInts.diff(oldInts).toVector.map(name ⇒
      SchemaChange.ObjectTypeInterfaceAdded(newType, newType.interfaces.find(_.name == name).get))

    removed ++ added
  }

  private def findInObjectTypeFields(oldType: ObjectLikeType[_, _], newType: ObjectLikeType[_, _]): Vector[SchemaChange] = {
    val oldFields = oldType.fields.map(_.name).toSet
    val newFields = newType.fields.map(_.name).toSet

    val removed = oldFields.diff(newFields).toVector.map(name ⇒
      SchemaChange.FieldRemoved(oldType, oldType.fieldsByName(name).head))

    val added = newFields.diff(oldFields).toVector.map(name ⇒
      SchemaChange.FieldAdded(newType, newType.fieldsByName(name).head))

    val changed = oldFields.intersect(newFields).flatMap { name ⇒
      val oldField = oldType.fieldsByName(name).head
      val newField = newType.fieldsByName(name).head

      findDescriptionChanged(oldField, newField, SchemaChange.FieldDescriptionChanged(newType, newField, _, _)) ++
        findDeprecationChanged(oldField, newField, SchemaChange.FieldDeprecationChanged(newType, newField, _, _)) ++
        findInFields(oldType, newType, oldField, newField)
    }

    removed ++ added ++ changed
  }

  private def findInFields(oldType: ObjectLikeType[_, _], newType: ObjectLikeType[_, _], oldField: Field[_, _], newField: Field[_, _]): Vector[SchemaChange] = {
    val oldFieldType = SchemaRenderer.renderTypeName(oldField.fieldType)
    val newFieldType = SchemaRenderer.renderTypeName(newField.fieldType)

    val typeChanges =
      if (oldFieldType != newFieldType)
        Vector(SchemaChange.FieldTypeChanged(newType, newField, SchemaRenderer.renderTypeName(nonContainer(oldField)) != newFieldType,
          oldField.fieldType, newField.fieldType))
      else
        Vector.empty

    val argChanges = findInArgs(oldField.arguments, newField.arguments,
      added = SchemaChange.ObjectTypeArgumentAdded(newType, newField, _, _),
      removed = SchemaChange.ObjectTypeArgumentRemoved(oldType, oldField, _),
      description = SchemaChange.ObjectTypeArgumentDescriptionChanged(newType, newField, _, _, _),
      default = SchemaChange.ObjectTypeArgumentDefaultChanged(newType, newField, _, _, _),
      typeChange = SchemaChange.ObjectTypeArgumentTypeChanged(newType, newField, _, _, _, _))

    typeChanges ++ argChanges
  }

  private def findInArgs(
    oldArgs: List[Argument[_]],
    newArgs: List[Argument[_]],
    added: (Argument[_], Boolean) ⇒ SchemaChange,
    removed: Argument[_] ⇒ SchemaChange,
    description: (Argument[_], Option[String], Option[String]) ⇒ SchemaChange,
    default: (Argument[_], Option[String], Option[String]) ⇒ SchemaChange,
    typeChange: (Argument[_], Boolean, InputType[_], InputType[_]) ⇒ SchemaChange
  ): Vector[SchemaChange] = {
    val oldA = oldArgs.map(_.name).toSet
    val newA = newArgs.map(_.name).toSet

    val remove = oldA.diff(newA).toVector.map(name ⇒
      removed(oldArgs.find(_.name == name).get))

    val add = newA.diff(oldA).toVector.map { name ⇒
      val arg = newArgs.find(_.name == name).get

      added(arg, !isOptional(arg))
    }

    val changed = oldA.intersect(newA).flatMap { name ⇒
      val oldArg = oldArgs.find(_.name == name).get
      val newArg = newArgs.find(_.name == name).get

      findDescriptionChanged(oldArg, newArg, description(newArg, _, _)) ++
        findInArg(oldArg, newArg, default(newArg, _, _), typeChange(newArg, _, _, _))
    }

    remove ++ add ++ changed
  }

  private def findInArg(
    oldArg: Argument[_],
    newArg: Argument[_],
    default: (Option[String], Option[String]) ⇒ SchemaChange,
    typeChange: (Boolean, InputType[_], InputType[_]) ⇒ SchemaChange
  ): Vector[SchemaChange] = {
    val oldDefault = oldArg.defaultValue.flatMap(dv ⇒ DefaultValueRenderer.renderInputValueCompact(dv, oldArg.argumentType, coercionHelper))
    val newDefault = newArg.defaultValue.flatMap(dv ⇒ DefaultValueRenderer.renderInputValueCompact(dv, newArg.argumentType, coercionHelper))

    val withDefault =
      if (oldDefault != newDefault)
        Vector(default(oldDefault, newDefault))
      else
        Vector.empty

    val oldArgType = SchemaRenderer.renderTypeName(oldArg.argumentType)
    val newArgType = SchemaRenderer.renderTypeName(newArg.argumentType)

    val withType =
      if (oldArgType != newArgType)
        withDefault :+ typeChange(SchemaRenderer.renderTypeName(nonContainer(newArg)) != oldArgType, oldArg.argumentType, newArg.argumentType)
      else
        withDefault

    withType
  }

  private def findInInputFields(oldType: InputObjectType[_], newType: InputObjectType[_], oldField: InputField[_], newField: InputField[_]): Vector[SchemaChange] = {
    val oldDefault = oldField.defaultValue.flatMap(dv ⇒ DefaultValueRenderer.renderInputValueCompact(dv, oldField.fieldType, coercionHelper))
    val newDefault = newField.defaultValue.flatMap(dv ⇒ DefaultValueRenderer.renderInputValueCompact(dv, newField.fieldType, coercionHelper))

    val withDefault =
      if (oldDefault != newDefault)
        Vector(SchemaChange.InputFieldDefaultChanged(newType, newField, oldDefault, newDefault))
      else
        Vector.empty

    val oldFieldType = SchemaRenderer.renderTypeName(oldField.fieldType)
    val newFieldType = SchemaRenderer.renderTypeName(newField.fieldType)

    val withType =
      if (oldFieldType != newFieldType)
        withDefault :+ SchemaChange.InputFieldTypeChanged(newType, newField, SchemaRenderer.renderTypeName(nonContainer(newField)) != oldFieldType,
          oldField.fieldType, newField.fieldType)
      else
        withDefault

    withType
  }

  private def isOptional(field: InputField[_]) = field.fieldType match {
    case _: OptionInputType[_] ⇒ true
    case _ ⇒ false
  }

  private def isOptional(argument: Argument[_]) = argument.argumentType match {
    case _: OptionInputType[_] ⇒ true
    case _ ⇒ false
  }

  private def nonContainer(field: InputField[_]) = field.fieldType match {
    case OptionInputType(ofType) ⇒ ofType
    case tpe ⇒ tpe
  }

  private def nonContainer(field: Field[_, _]) = field.fieldType match {
    case OptionType(ofType) ⇒ ofType
    case tpe ⇒ tpe
  }

  private def nonContainer(argument: Argument[_]) = argument.argumentType match {
    case OptionInputType(ofType) ⇒ ofType
    case tpe ⇒ tpe
  }

  private def findDescriptionChanged(o: Named, n: Named, fn: (Option[String], Option[String]) ⇒ SchemaChange): Vector[SchemaChange] =
    if (o.description != n.description) Vector(fn(o.description, n.description))
    else Vector.empty

  private def findDeprecationChanged(o: HasDeprecation, n: HasDeprecation, fn: (Option[String], Option[String]) ⇒ SchemaChange): Vector[SchemaChange] =
    if (o.deprecationReason != n.deprecationReason) Vector(fn(o.deprecationReason, n.deprecationReason))
    else Vector.empty

  private val coercionHelper = new ValueCoercionHelper[Any]
}

sealed trait SchemaChange {
  def breakingChange: Boolean
  def description: String
}

object SchemaChange {
  sealed trait TypeChange {
    def tpe: Type with Named
  }

  sealed trait DeprecationChange extends TypeChange {
    def oldDeprecationReason: Option[String]
    def newDeprecationReason: Option[String]
  }

  sealed trait DescriptionChange extends TypeChange {
    def oldDescription: Option[String]
    def newDescription: Option[String]
  }

  sealed abstract class AbstractChange(val description: String, val breakingChange: Boolean) extends SchemaChange

  // Breaking changes

  case class TypeRemoved(tpe: Type with Named) extends AbstractChange(s"`${tpe.name}` type was removed", true) with TypeChange

  case class DirectiveRemoved(directive: Directive) extends AbstractChange(s"`${directive.name}` directive was removed", true)

  case class TypeKindChanged(tpe: Type with Named, oldTpe: Type with Named)
    extends AbstractChange({
      val oldKind = kind(oldTpe)
      val newKind = kind(tpe)

      s"`${tpe.name}` changed from ${article(oldKind)} $oldKind type to ${article(newKind)} $newKind type"
    }, true) with TypeChange

  case class EnumValueRemoved(tpe: EnumType[_], value: EnumValue[_])
    extends AbstractChange(s"Enum value `${value.name}` was removed from enum `${tpe.name}`", true) with TypeChange

  case class UnionMemberRemoved(tpe: UnionType[_], member: ObjectType[_, _])
    extends AbstractChange(s"`${member.name}` type was removed from union `${tpe.name}`", true) with TypeChange

  case class InputFieldRemoved(tpe: InputObjectType[_], field: InputField[_])
    extends AbstractChange(s"Input field `${field.name}` was removed from `${tpe.name}` type", true) with TypeChange

  case class ObjectTypeArgumentRemoved(tpe: ObjectLikeType[_, _], field: Field[_, _], argument: Argument[_])
    extends AbstractChange(s"Argument `${argument.name}` was removed from `${tpe.name}.${field.name}` field", true) with TypeChange

  case class DirectiveArgumentRemoved(directive: Directive, argument: Argument[_])
    extends AbstractChange(s"Argument `${argument.name}` was removed from `${directive.name}` directive", true)

  case class SchemaQueryTypeChanged(oldType: ObjectType[_, _], newType: ObjectType[_, _])
    extends AbstractChange(s"Schema query type changed from `${oldType.name}` to `${newType.name}` type", true)

  case class FieldRemoved(tpe: ObjectLikeType[_, _], field: Field[_, _])
    extends AbstractChange(s"Field `${field.name}` was removed from `${tpe.name}` type", true) with TypeChange

  case class DirectiveLocationRemoved(directive: Directive, location: DirectiveLocation.Value)
    extends AbstractChange(s"`$location` directive location removed from `${directive.name}` directive", true)

  case class ObjectTypeInterfaceRemoved(tpe: ObjectType[_, _], interface: InterfaceType[_, _])
    extends AbstractChange(s"`${tpe.name}` object type no longer implements `${interface.name}` interface", true) with TypeChange

  // Non-breaking changes

  case class TypeAdded(tpe: Type with Named)
    extends AbstractChange(s"`${tpe.name}` type was added", false) with TypeChange

  case class DirectiveAdded(directive: Directive) extends AbstractChange(s"`${directive.name}` directive was added", false)

  case class TypeDescriptionChanged(tpe: Type with Named, oldDescription: Option[String], newDescription: Option[String])
    extends AbstractChange(s"`${tpe.name}` type description is changed", false) with DescriptionChange

  case class EnumValueAdded(tpe: EnumType[_], value: EnumValue[_])
    extends AbstractChange(s"Enum value `${value.name}` was added to enum `${tpe.name}`", false) with TypeChange

  case class EnumValueDescriptionChanged(tpe: EnumType[_], value: EnumValue[_], oldDescription: Option[String], newDescription: Option[String])
    extends AbstractChange(s"`${tpe.name}.${value.name}` description changed", false) with DescriptionChange

  case class EnumValueDeprecated(tpe: EnumType[_], value: EnumValue[_], oldDeprecationReason: Option[String], newDeprecationReason: Option[String])
    extends AbstractChange(s"Enum value `${value.name}` was deprecated in enum `${tpe.name}`", false) with DeprecationChange

  case class UnionMemberAdded(tpe: UnionType[_], member: ObjectType[_, _])
    extends AbstractChange(s"`${member.name}` type was added to union `${tpe.name}`", false) with TypeChange

  case class InputFieldDescriptionChanged(tpe: InputObjectType[_], field: InputField[_], oldDescription: Option[String], newDescription: Option[String])
    extends AbstractChange(s"`${tpe.name}.${field.name}` description is changed", false) with DescriptionChange

  case class DirectiveDescriptionChanged(directive: Directive, oldDescription: Option[String], newDescription: Option[String])
    extends AbstractChange(s"`${directive.name}` directive description is changed", false)

  case class FieldDescriptionChanged(tpe: ObjectLikeType[_, _], field: Field[_, _], oldDescription: Option[String], newDescription: Option[String])
    extends AbstractChange(s"`${tpe.name}.${field.name}` description is changed", false) with DescriptionChange

  case class ObjectTypeArgumentDescriptionChanged(tpe: ObjectLikeType[_, _], field: Field[_, _], argument: Argument[_], oldDescription: Option[String], newDescription: Option[String])
    extends AbstractChange(s"`${tpe.name}.${field.name}(${argument.name})` description is changed", false) with DescriptionChange

  case class DirectiveArgumentDescriptionChanged(directive: Directive, argument: Argument[_], oldDescription: Option[String], newDescription: Option[String])
    extends AbstractChange(s"`${directive.name}(${argument.name})` description is changed", false)

  case class FieldDeprecationChanged(tpe: ObjectLikeType[_, _], field: Field[_, _], oldDeprecationReason: Option[String], newDeprecationReason: Option[String])
    extends AbstractChange(s"Field `${field.name}` was deprecated in `${tpe.name}` type", false) with DeprecationChange

  case class InputFieldDefaultChanged(tpe: InputObjectType[_], field: InputField[_], oldDefault: Option[String], newDefault: Option[String])
    extends AbstractChange(s"`${tpe.name}.${field.name}` default value changed from ${oldDefault.fold("none")("`" + _ + "`")} to ${newDefault.fold("none")("`" + _ + "`")}", false) with TypeChange

  case class ObjectTypeArgumentDefaultChanged(tpe: ObjectLikeType[_, _], field: Field[_, _], argument: Argument[_], oldDefault: Option[String], newDefault: Option[String])
    extends AbstractChange(s"`${tpe.name}.${field.name}(${argument.name})` default value changed from ${oldDefault.fold("none")("`" + _ + "`")} to ${newDefault.fold("none")("`" + _ + "`")}", false) with TypeChange

  case class DirectiveArgumentDefaultChanged(directive: Directive, argument: Argument[_], oldDefault: Option[String], newDefault: Option[String])
    extends AbstractChange(s"`${directive.name}(${argument.name})` default value changed from ${oldDefault.fold("none")("`" + _ + "`")} to ${newDefault.fold("none")("`" + _ + "`")}", false)

  case class ObjectTypeInterfaceAdded(tpe: ObjectType[_, _], interface: InterfaceType[_, _])
    extends AbstractChange(s"`${tpe.name}` object type now implements `${interface.name}` interface", false) with TypeChange

  case class FieldAdded(tpe: ObjectLikeType[_, _], field: Field[_, _])
    extends AbstractChange(s"Field `${field.name}` was added to `${tpe.name}` type", false) with TypeChange

  case class DirectiveLocationAdded(directive: Directive, location: DirectiveLocation.Value)
    extends AbstractChange(s"`$location` directive location added to `${directive.name}` directive", false)

  // May be a breaking change

  case class InputFieldAdded(tpe: InputObjectType[_], field: InputField[_], breaking: Boolean)
    extends AbstractChange(s"Input field `${field.name}` was added to `${tpe.name}` type", breaking) with TypeChange

  case class ObjectTypeArgumentAdded(tpe: ObjectLikeType[_, _], field: Field[_, _], argument: Argument[_], breaking: Boolean)
    extends AbstractChange(s"Argument `${argument.name}` was added to `${tpe.name}.${field.name}` field", breaking) with TypeChange

  case class DirectiveArgumentAdded(directive: Directive, argument: Argument[_], breaking: Boolean)
    extends AbstractChange(s"Argument `${argument.name}` was added to `${directive.name}` directive", breaking)

  case class InputFieldTypeChanged(tpe: InputObjectType[_], field: InputField[_], breaking: Boolean, oldFiledType: InputType[_], newFieldType: InputType[_])
    extends AbstractChange(s"`${tpe.name}.${field.name}` input field type changed from `${SchemaRenderer.renderTypeName(oldFiledType)}` to `${SchemaRenderer.renderTypeName(newFieldType)}`", breaking) with TypeChange

  case class ObjectTypeArgumentTypeChanged(tpe: ObjectLikeType[_, _], field: Field[_, _], argument: Argument[_], breaking: Boolean, oldFiledType: InputType[_], newFieldType: InputType[_])
    extends AbstractChange(s"`${tpe.name}.${field.name}(${argument.name})` type changed from `${SchemaRenderer.renderTypeName(oldFiledType)}` to `${SchemaRenderer.renderTypeName(newFieldType)}`", breaking) with TypeChange

  case class DirectiveArgumentTypeChanged(directive: Directive, argument: Argument[_], breaking: Boolean, oldFiledType: InputType[_], newFieldType: InputType[_])
    extends AbstractChange(s"`${directive.name}(${argument.name})` type changed from `${SchemaRenderer.renderTypeName(oldFiledType)}` to `${SchemaRenderer.renderTypeName(newFieldType)}`", breaking)

  case class FieldTypeChanged(tpe: ObjectLikeType[_, _], field: Field[_, _], breaking: Boolean, oldFiledType: OutputType[_], newFieldType: OutputType[_])
    extends AbstractChange(s"`${tpe.name}.${field.name}` field type changed from `${SchemaRenderer.renderTypeName(oldFiledType)}` to `${SchemaRenderer.renderTypeName(newFieldType)}`", breaking) with TypeChange

  case class SchemaMutationTypeChanged(oldType: Option[ObjectType[_, _]], newType: Option[ObjectType[_, _]])
    extends AbstractChange(s"Schema mutation type changed from ${oldType.fold("none")(t ⇒ "`" + t.name + "`")} to ${newType.fold("none")(t ⇒ "`" + t.name + "`")} type", oldType.nonEmpty)

  case class SchemaSubscriptionTypeChanged(oldType: Option[ObjectType[_, _]], newType: Option[ObjectType[_, _]])
    extends AbstractChange(s"Schema subscription type changed from ${oldType.fold("none")(t ⇒ "`" + t.name + "`")} to ${newType.fold("none")(t ⇒ "`" + t.name + "`")} type", oldType.nonEmpty)

  private val AnArticleLetters = Set('a', 'e', 'i', 'o')

  private def kind(tpe: Type) = tpe match {
    case _: ObjectType[_, _] ⇒ "Object"
    case _: InterfaceType[_, _] ⇒ "Interface"
    case _: ScalarType[_] ⇒ "Scalar"
    case _: UnionType[_] ⇒ "Union"
    case _: EnumType[_] ⇒ "Enum"
    case _: InputObjectType[_] ⇒ "InputObject"
    case t ⇒ throw new IllegalStateException(s"Unsupported type kind: $t")
  }

  private def article(word: String) =
    if (AnArticleLetters contains word.head.toLower) "an"
    else "a"
}


text:
from .local import LocalShell

from logging import getLogger
log = getLogger(__name__)

__all__ = ('RemoteShell', 'SecureShell', 'GlobusSecureShell')

class RemoteShell(LocalShell):

    def __init__(self, rsh='rsh', rcp='rcp', hostname='localhost', username=None, **kwargs):
        super(RemoteShell, self).__init__(**kwargs)
        self.rsh = rsh
        self.rcp = rcp
        self.hostname = hostname
        self.username = username
        self.sessions = {}

    def execute(self, cmd, persist=False, timeout=60):
        # TODO: implement persistence
        if self.username is None:
            fullcmd = '%s %s %s' % (self.rsh, self.hostname, cmd)
        else:
            fullcmd = '%s -l %s %s %s' % (self.rsh, self.username, self.hostname, cmd)
        return super(RemoteShell, self).execute(fullcmd, persist, timeout)

class SecureShell(RemoteShell):
    SSH_NEW_KEY_STRING = 'Are you sure you want to continue connecting'

    def __init__(self, rsh='ssh', rcp='scp', **kwargs):
        rsh += ' -oStrictHostKeyChecking=yes -oConnectTimeout=60'
        rcp += ' -oStrictHostKeyChecking=yes -oConnectTimeout=60'
        super(SecureShell, self).__init__(rsh=rsh, rcp=rcp, **kwargs)

class GlobusSecureShell(SecureShell):

    def __init__(self, rsh='gsissh', rcp='gsiscp', **kwargs):
        super(GlobusSecureShell, self).__init__(rsh=rsh, rcp=rcp, **kwargs)