import sys
sys.path.append('../projects/')

import numpy as np
import pandas as pd
import scipy
from scipy import stats
from scipy.special import factorial
from scipy.stats import binom_test, wilcoxon

import os
import pickle
from datetime import datetime
import tabulate
import wandb
from collections import namedtuple, defaultdict, OrderedDict
import json
from ipypb import ipb

# from metalearning import cnnmlp

API = wandb.Api()
MAX_HISTORY_SAMPLES = 4000

DATASET_CORESET_SIZE = 22500
ACCURACY_THRESHOLD = 0.95
TASK_ACC_COLS = [f'Test Accuracy, Query #{i}' for i in range(1, 11)]

QUERY_NAMES = ['blue', 'brown', 'cyan', 'gray', 'green',
               'orange', 'pink', 'purple', 'red', 'yellow',
               'cone', 'cube', 'cylinder', 'dodecahedron', 'ellipsoid',
               'octahedron', 'pyramid', 'rectangle', 'sphere', 'torus',
               'chain_mail', 'marble', 'maze', 'metal', 'metal_weave',
               'polka', 'rubber', 'rug', 'tiles', 'wood_plank']

COLOR = 'color'
COLOR_INDEX = 0
SHAPE = 'shape'
SHAPE_INDEX = 1
TEXTURE = 'texture'
TEXTURE_INDEX = 2
COMBINED = 'combined'
COMBINED_INDEX = 3
DIMENSION_NAMES = [COLOR, SHAPE, TEXTURE]

RESULT_SET_FIELDS = ['name', 'mean', 'std']
ANALYSIS_SET_FIELDS = ['examples', 'log_examples', 'accuracies', 'accuracy_drops',
                       'first_task_accuracies', 'new_task_accuracies', 'accuracy_counts',
                       'examples_by_task']
CONDITION_ANALYSES_FIELDS = DIMENSION_NAMES + [COMBINED]
TOTAL_CURVE_FIELDS = ['raw', 'mean', 'std', 'sem']

ResultSet = namedtuple('ResultSet', RESULT_SET_FIELDS)
AnalysisSet = namedtuple('AnalysisSet', ANALYSIS_SET_FIELDS)
ConditionAnalysesSet = namedtuple('ConditionAnalysesSet', CONDITION_ANALYSES_FIELDS)
TotalCurveResults = namedtuple('TotalCurveResults', TOTAL_CURVE_FIELDS)
TaskConditionalWeights = namedtuple('TaskConditionalWeights', ('additive', 'multiplicative'))

NAMED_TUPLE_CLASSES = (ResultSet, AnalysisSet, ConditionAnalysesSet, TotalCurveResults, TaskConditionalWeights)
for NamedTupleClass in NAMED_TUPLE_CLASSES:
    NamedTupleClass.__new__.__defaults__ = (None,) * len(NamedTupleClass._fields)

RESULT_SET_NAMES = ('Examples to criterion', 'Log examples to criterion',
                    'New task accuracy', 'New task accuracy delta',
                    'First task accuracy by epoch', 'New task accuracy by epoch')
ANALYSIS_FIELDS_TO_NAMES = {field: name for (name, field) in
                            zip(ANALYSIS_SET_FIELDS, RESULT_SET_NAMES)}
ANALYSIS_NAMES_TO_FIELDS = {name: field for (name, field) in
                            zip(ANALYSIS_SET_FIELDS, RESULT_SET_NAMES)}

CACHE_PATH = './analyses_caches/meta_learning_analyses_cache.pickle'
BACKUP_CACHE_PATH = './analyses_caches/meta_learning_analyses_cache_{date}.pickle'

def refresh_cache(new_values_dict=None, cache_path=CACHE_PATH):
    if new_values_dict is None:
        new_values_dict = {}

    if os.path.exists(cache_path):
        with open(cache_path, 'rb') as cache_file:
            cache = pickle.load(cache_file)

    else:
        cache = {}

    cache.update(new_values_dict)

    if os.path.exists(cache_path):
        os.rename(CACHE_PATH, BACKUP_CACHE_PATH.format(date=datetime.now().strftime('%Y-%m-%d_%H-%M-%S')))

    with open(cache_path, 'wb') as cache_file:
        pickle.dump(cache, cache_file)

    return cache

def examples_per_epoch(task, latest_task):
    if task == latest_task:
        return 22500

    return 22500 // (latest_task - 1)

def parse_run_results(current_run_id=None, current_run=None, samples=MAX_HISTORY_SAMPLES):
    if current_run_id is None and current_run is None:
        print('Must provide either a current run or its id')
        return

    if current_run is None:
        current_run = API.run(f'meta-learning-scaling/sequential-benchmark-baseline/{current_run_id}')

    current_df = current_run.history(pandas=True, samples=samples)

    examples_to_criterion = np.empty((10, 10))
    examples_to_criterion.fill(np.nan)
    absolute_accuracy = np.empty((9, 9))
    absolute_accuracy.fill(np.nan)
    accuracy_drop = np.empty((9, 9))
    accuracy_drop.fill(np.nan)

    first_task_accuracy_by_epoch = np.empty((10, samples))
    first_task_accuracy_by_epoch.fill(np.nan)
    new_task_accuracy_by_epoch = np.empty((10, samples))
    new_task_accuracy_by_epoch.fill(np.nan)

    first_row_blank = int(np.isnan(current_df['Test Accuracy'][0]))

    first_task_finished = current_df['Test Accuracy, Query #2'].first_valid_index() - first_row_blank
    examples_to_criterion[0, 0] = first_task_finished * examples_per_epoch(1, 1)
    absolute_accuracy[0, 0] = current_df['Test Accuracy, Query #1'][first_task_finished + 1]
    accuracy_drop[0, 0] = current_df['Test Accuracy, Query #1'][first_task_finished] - absolute_accuracy[0, 0]

    first_task_accuracy_by_epoch[0, 0:first_task_finished] = current_df['Test Accuracy, Query #1'][first_row_blank:first_task_finished + 1]
    new_task_accuracy_by_epoch[0, 0:first_task_finished] = current_df['Test Accuracy, Query #1'][first_row_blank:first_task_finished + 1]

    for current_task in range(2, 11):
        current_task_start = current_df[f'Test Accuracy, Query #{current_task}'].first_valid_index()

        if current_task == 10:
            current_task_end = current_df.shape[0]
        else:
            current_task_end = current_df[f'Test Accuracy, Query #{current_task + 1}'].first_valid_index()

        current_task_subset = current_df[TASK_ACC_COLS][current_task_start:current_task_end] > 0.95

        for task in range(1, current_task + 1):
            number_times_learned = current_task - task + 1
            number_total_tasks = current_task

            examples_to_criterion[number_times_learned - 1, number_total_tasks - 1] = examples_per_epoch(task, current_task) * \
                (current_task_subset[f'Test Accuracy, Query #{task}'].idxmax() - current_task_start + 1)

            if current_task < 10:
                absolute_accuracy[number_times_learned - 1, number_total_tasks - 1] = \
                    current_df[f'Test Accuracy, Query #{task}'][current_task_end]
                accuracy_drop[number_times_learned - 1, number_total_tasks - 1] = \
                    current_df[f'Test Accuracy, Query #{task}'][current_task_end - 1] - \
                    absolute_accuracy[number_times_learned - 1, number_total_tasks - 1]

        first_task_accuracy_by_epoch[current_task - 1, 0:current_task_end - current_task_start] = current_df['Test Accuracy, Query #1'][current_task_start:current_task_end]
        new_task_accuracy_by_epoch[current_task - 1, 0:current_task_end - current_task_start] = current_df[f'Test Accuracy, Query #{current_task}'][current_task_start:current_task_end]

    return examples_to_criterion, absolute_accuracy, accuracy_drop, first_task_accuracy_by_epoch, new_task_accuracy_by_epoch

def parse_run_results_with_new_task_accuracy_and_equal_size(current_run_id=None, current_run=None, samples=MAX_HISTORY_SAMPLES):
    if current_run_id is None and current_run is None:
        print('Must provide either a current run or its id')
        return

    if current_run is None:
        current_run = API.run(f'meta-learning-scaling/sequential-benchmark-baseline/{current_run_id}')

    current_df = current_run.history(pandas=True, samples=samples)

    examples_to_criterion = np.empty((10, 10))
    examples_to_criterion.fill(np.nan)
    absolute_accuracy = np.empty((10, 10))
    absolute_accuracy.fill(np.nan)
    absolute_accuracy_equal_size = np.empty((10, 10))
    absolute_accuracy_equal_size.fill(np.nan)

    first_task_accuracy_by_epoch = np.empty((10, samples))
    first_task_accuracy_by_epoch.fill(np.nan)
    new_task_accuracy_by_epoch = np.empty((10, samples))
    new_task_accuracy_by_epoch.fill(np.nan)

    first_row_blank = int(np.isnan(current_df['Test Accuracy'][0]))

    # import pdb; pdb.set_trace()

    first_task_finished = current_df['Test Accuracy, Query #2'].first_valid_index() - first_row_blank
    examples_to_criterion[0, 0] = first_task_finished * examples_per_epoch(1, 1)
    absolute_accuracy[0, 0] = current_df['Test Accuracy, Query #1'][first_row_blank] # index 0 is all NaNs, presumably the initalization
    absolute_accuracy_equal_size[0, 0] = current_df['Test Accuracy, Query #1'][first_row_blank]

    first_task_accuracy_by_epoch[0, 0:first_task_finished] = current_df['Test Accuracy, Query #1'][first_row_blank:first_task_finished]
    new_task_accuracy_by_epoch[0, 0:first_task_finished] = current_df['Test Accuracy, Query #1'][first_row_blank:first_task_finished]

    for current_task in range(2, 11):
        current_task_start = current_df[f'Test Accuracy, Query #{current_task}'].first_valid_index()

        if current_task == 10:
            current_task_end = current_df.shape[0]
        else:
            current_task_end = current_df[f'Test Accuracy, Query #{current_task + 1}'].first_valid_index()

        current_task_subset = current_df[TASK_ACC_COLS][current_task_start:current_task_end] > 0.95

        for task in range(1, current_task + 1):
            number_times_learned = current_task - task + 1
            number_total_tasks = current_task

            examples_to_criterion[number_times_learned - 1, number_total_tasks - 1] = examples_per_epoch(task, current_task) * \
                (current_task_subset[f'Test Accuracy, Query #{task}'].idxmax() - current_task_start + 1)

            absolute_accuracy[number_times_learned - 1, number_total_tasks - 1] = \
                current_df[f'Test Accuracy, Query #{task}'][current_task_start]

            if task == current_task:
                equal_size_epochs = current_task_start
            else:
                equal_size_epochs = min(current_task_start + current_task - 1, current_task_end - 1)

            absolute_accuracy_equal_size[number_times_learned - 1, number_total_tasks - 1] = \
                current_df[f'Test Accuracy, Query #{task}'][equal_size_epochs]

        first_task_accuracy_by_epoch[current_task - 1, 0:current_task_end - current_task_start] = current_df['Test Accuracy, Query #1'][current_task_start:current_task_end]
        new_task_accuracy_by_epoch[current_task - 1, 0:current_task_end - current_task_start] = current_df[f'Test Accuracy, Query #{current_task}'][current_task_start:current_task_end]

    return examples_to_criterion, absolute_accuracy, absolute_accuracy_equal_size, first_task_accuracy_by_epoch, new_task_accuracy_by_epoch

def parse_forgetting_results(current_run_id=None, current_run=None, samples=MAX_HISTORY_SAMPLES,
                             max_samples_per_curve=31):
    if current_run_id is None and current_run is None:
        print('Must provide either a current run or its id')
        return

    if current_run is None:
        current_run = API.run(f'meta-learning-scaling/sequential-benchmark-forgetting-experiment-revisited/{current_run_id}')

    print(f'Starting to parse run {current_run.name}')
    raw_df = current_run.history(pandas=True, samples=samples)

    forgetting_trjectories = np.empty((10, 10, max_samples_per_curve))
    forgetting_trjectories.fill(np.nan)

    if 'step_resumed_from' in current_run.config:
        step_resumed_from = current_run.config['step_resumed_from']
        post_resume_step = raw_df[900:1100]['_timestamp'].diff().idxmax()
        current_df = raw_df.drop(range(step_resumed_from + 1, post_resume_step + 1), axis=0, inplace=False)
        current_df.reset_index(inplace=True)

    else:
        current_df = raw_df

    for new_task in range(2, 11):
        forgetting_start = current_df[f'Test Accuracy, Query #{new_task}'].first_valid_index() - 1

        if new_task == 10:
            forgetting_end = len(current_df) - 1
        else:
            forgetting_end = current_df[f'Test Accuracy, Query #{new_task + 1}'].first_valid_index() - 1

        forgetting_len = min(forgetting_end - forgetting_start, max_samples_per_curve)

        for forgetting_task in range(1, new_task):
            number_times_learned = new_task - forgetting_task
            number_total_tasks = new_task
            trajectory = np.array(current_df[f'Test Accuracy, Query #{forgetting_task}'][forgetting_start:forgetting_start + forgetting_len], dtype=np.float64)

            if np.any(np.isnan(trajectory)):
                print(f'Found nans for new task {new_task} and forgetting task {forgetting_task}')
                print(forgetting_start, forgetting_end)
                print(trajectory)

            forgetting_trjectories[number_times_learned - 1, number_total_tasks - 1, :forgetting_len] = trajectory

    return forgetting_trjectories

def parse_total_task_training_curve(current_run_id=None, current_run=None, samples=MAX_HISTORY_SAMPLES):
    if current_run_id is None and current_run is None:
        print('Must provide either a current run or its id')
        return

    if current_run is None:
        current_run = API.run(f'meta-learning-scaling/sequential-benchmark-baseline/{current_run_id}')

    current_df = current_run.history(pandas=True, samples=samples)
    total_task_curve = defaultdict(dict)
    task_total_trials = defaultdict(lambda: 0)

    first_row_blank = int(np.isnan(current_df['Test Accuracy'][0]))
    first_task_finished = current_df['Test Accuracy, Query #2'].first_valid_index() - first_row_blank
    episode_trials = examples_per_epoch(1, 1)

    for i, acc in enumerate(current_df['Test Accuracy, Query #1'][first_row_blank:first_task_finished]):
        total_task_curve[1][(i + 1) * episode_trials] = acc

    task_total_trials[1] = (first_task_finished - first_row_blank) * episode_trials

    for current_task in range(2, 11):
        current_task_start = current_df[f'Test Accuracy, Query #{current_task}'].first_valid_index()

        if current_task == 10:
            current_task_end = current_df.shape[0]
        else:
            current_task_end = current_df[f'Test Accuracy, Query #{current_task + 1}'].first_valid_index()

        current_task_length = current_task_end - current_task_start

        for task in range(1, current_task + 1):
            episode_trials = examples_per_epoch(task, current_task)

            for i, acc in enumerate(current_df[f'Test Accuracy, Query #{task}'][current_task_start:current_task_end]):
                total_task_curve[task][task_total_trials[task] + (i + 1) * episode_trials] = acc

            task_total_trials[task] = task_total_trials[task] + (current_task_length * episode_trials)

    # print([(t, len(total_task_curve[t].keys())) for t in range(1, 11)])

    return total_task_curve

def parse_simultaneous_training(current_run_id=None, current_run=None, samples=MAX_HISTORY_SAMPLES):
    if current_run_id is None and current_run is None:
        print('Must provide either a current run or its id')
        return

    if current_run is None:
        current_run = API.run(f'meta-learning-scaling/sequential-benchmark-baseline/{current_run_id}')

    current_df = current_run.history(pandas=True, samples=samples)
    simultaneous_task_accuracies = current_df['Test Per-Query Accuracy (list)']
    first_row_blank = int(np.isnan(simultaneous_task_accuracies[0]))
    return np.array(list(simultaneous_task_accuracies[first_row_blank:]))

def process_multiple_runs_simultaneous_training(runs, debug=False, ignore_runs=None, samples=MAX_HISTORY_SAMPLES):
    simultaneous_training_results = []

    for i, run in enumerate(runs):
        if i > 0 and i % 10 == 0:
            print(run.name, i)
        else:
            print(run.name)

        if ignore_runs is not None and run.name in ignore_runs:
            continue

        simultaneous_training_results.append(parse_simultaneous_training(current_run=run, samples=samples))

    max_length = max([res.shape[0] for res in simultaneous_training_results])
    for i in range(len(simultaneous_training_results)):
        curr_length = simultaneous_training_results[i].shape[0]
        simultaneous_training_results[i] = np.pad(simultaneous_training_results[i], ((0, max_length - curr_length), (0, 0)),
                                                  'edge') #'constant', constant_values=np.nan)

    stacked_results = np.stack(simultaneous_training_results, axis=2)

    all_results_mean = np.nanmean(stacked_results, axis=(1, 2))
    all_results_std = np.nanstd(stacked_results, axis=(1, 2))
    all_results_sem = all_results_std / np.sqrt(np.prod(stacked_results.shape[1:]))

    per_task_results_mean = np.nanmean(stacked_results, axis=(2))
    per_task_results_std = np.nanstd(stacked_results, axis=(2))
    per_task_results_sem = per_task_results_std / np.sqrt(stacked_results.shape[2])

    return stacked_results, all_results_mean, all_results_std, all_results_sem, per_task_results_mean, per_task_results_std, per_task_results_sem

PRINT_HEADERS = ['###'] + [str(x) for x in range(1, 11)]

def pretty_print_results(results, **kwargs):
    result_rows = [[str(i + 1)] + list(results[i]) for i in range(len(results))]
    tab_args = dict(tablefmt='fancy_grid')
    tab_args.update(kwargs)
    print(tabulate.tabulate(result_rows, PRINT_HEADERS, **tab_args))

DEFAULT_PROJECT_PATH = 'meta-learning-scaling/sequential-benchmark-baseline'

def load_runs(max_rep_id, project_path=DEFAULT_PROJECT_PATH, split_runs_by_dimension=True,
              valid_run_ids=None, debug=False):
    runs = API.runs(project_path)

    results = ConditionAnalysesSet([], [], [], [])

    for run in runs:
        config = json.loads(run.json_config)
        run_id = config['dataset_random_seed']['value']

        if isinstance(run_id, dict):
            run_id = run_id['value']

        run_id = int(run_id)

        if debug: print(run.name, run_id)

        # run_name = run.description.split('\n')[0]
        # run_id = int(run_name[run_name.rfind('-') + 1:])

        if valid_run_ids is None or run_id in valid_run_ids:
            dimension = (run_id // 1000) - 1
            rep = run_id % 1000
            if rep < max_rep_id:
                # combined / all runs
                results[COMBINED_INDEX].append(run)
                # by-dimension
                if split_runs_by_dimension:
                        results[dimension].append(run)

    return results

def query_modulated_runs_by_dimension(max_rep_id):
    runs = API.runs('meta-learning-scaling/sequential-benchmark-task-modulated')

    results = {level: ConditionAnalysesSet([], [], [], []) for level in range(1, 5)}

    for run in runs:
        level, run_id = [int(x) for x in run.description.split('\n')[0][-6:].split('-')]
        dimension = (run_id // 1000) - 1
        rep = run_id % 1000
        if rep < max_rep_id:
            results[level][dimension].append(run)
            # combined / all runs
            results[level][3].append(run)

    return results

def epoch_to_trials(name, episode):
    if episode == 1 and 'power' not in name.lower():
        return 22500

    return 45000

def epochs_to_task_completions(runs, name=None, ignore_runs=None, samples=500, ipb_desc=None):
    if name is None:
        name = 'epochs_to_completion'

    results_per_run = []
    last_tasks_per_run = []

    for run in ipb(runs, desc=ipb_desc):
#         if i > 0 and i % 10 == 0:
#             print(run.name, i)
#         else:
#             print(run.name)

        if ignore_runs is not None and run.name in ignore_runs:
            continue

        df = run.history(pandas=True, samples=samples)
        first_row_blank = int(np.isnan(df['Test Accuracy'][0]))
        first_task_finished = df['Test Accuracy, Query #2'].first_valid_index() - first_row_blank

        task_finishes = [first_task_finished]
        last_tasks = [1]

        for current_task in range(2, 11):
            current_task_start = df[f'Test Accuracy, Query #{current_task}'].first_valid_index()

            if current_task == 10:
                current_task_end = df.shape[0]
            else:
                current_task_end = df[f'Test Accuracy, Query #{current_task + 1}'].first_valid_index()

            task_finishes.append(current_task_end)

            sub_df = df[[f'Test Accuracy, Query #{i + 1}' for i in range(current_task)]]
            min_column_name = sub_df.iloc[current_task_end - 2].idxmin()
            last_task_to_finish = int(min_column_name[min_column_name.find('#') + 1:])
            last_tasks.append(last_task_to_finish)

        results_per_run.append(task_finishes)
        last_tasks_per_run.append(last_tasks)

    results_per_run = np.array(results_per_run)
    trials_per_epoch = np.array([epoch_to_trials(name, e) for e in np.arange(1, 11)])
    trials_per_run = results_per_run * trials_per_epoch

    return ResultSet(name=name, mean=np.nanmean(results_per_run, axis=0), std=np.nanstd(results_per_run, axis=0)),\
            ResultSet(name=name, mean=np.nanmean(np.log(results_per_run), axis=0), std=np.nanstd(np.log(results_per_run), axis=0)),\
            ResultSet(name=name, mean=np.nanmean(trials_per_run, axis=0), std=np.nanstd(trials_per_run, axis=0)),\
            ResultSet(name=name, mean=np.nanmean(np.log(trials_per_run), axis=0), std=np.nanstd(np.log(trials_per_run), axis=0)),\
            last_tasks_per_run

def kth_diag_indices(a, k):
    rows, cols = np.diag_indices_from(a)
    if k < 0:
        return rows[-k:], cols[:k]
    elif k > 0:
        return rows[:-k], cols[k:]
    else:
        return rows, cols

def process_multiple_runs(runs, debug=False, ignore_runs=None, samples=MAX_HISTORY_SAMPLES, parse_func=parse_run_results):
    examples = []
    log_examples = []
    abs_accuracies = []
    accuracy_drops = []
    first_task_accuracies = []
    new_task_accuracies = []

    examples_by_task = np.zeros((30, 10, 10))
    log_examples_by_task = np.zeros((30, 10, 10))
    counts_by_task = np.zeros((30, 10, 10))

    for i, run in enumerate(runs):
        if i > 0 and i % 10 == 0:
            print(run.id, run.name, i)
        else:
            print(run.id, run.name)

        if ignore_runs is not None and run.name in ignore_runs:
            continue

        examples_to_criterion, absolute_accuracy, accuracy_drop, first_task_acc, new_task_acc = parse_func(current_run=run, samples=samples)
        # print(examples_to_criterion)
        # print(np.log(examples_to_criterion))
        examples.append(examples_to_criterion)
        log_examples.append(np.log(examples_to_criterion))
        abs_accuracies.append(absolute_accuracy)
        accuracy_drops.append(accuracy_drop)
        first_task_accuracies.append(first_task_acc)
        new_task_accuracies.append(new_task_acc)

        for index, task in enumerate(run.config['query_order']):
            task_examples = np.diag(examples_to_criterion, index)
            diag_r, diag_c = kth_diag_indices(examples_to_criterion, index)
            # examples_by_task[task,:10 - index] += task_examples
            # counts_by_task[task,:10 - index] += 1
            examples_by_task[task, diag_r, diag_c] += task_examples
            log_examples_by_task[task, diag_r, diag_c] += np.log(task_examples)
            counts_by_task[task, diag_r, diag_c] += 1

    # Removing all extraneous nans
    print('Removing extraneous nans')
    first_task_accuracies = np.array(first_task_accuracies)
    new_task_accuracies = np.array(new_task_accuracies)

    max_first_nan_idx = np.max(np.argmax(np.isnan(first_task_accuracies), axis=2))
    print('Max first nan index:', max_first_nan_idx)

    first_task_accuracies = first_task_accuracies[:, :, :max_first_nan_idx]
    new_task_accuracies = new_task_accuracies[:, :, :max_first_nan_idx]
    accuracy_counts = np.count_nonzero(~np.isnan(new_task_accuracies), axis=0)

    output = {}
    for result_set, name, field in zip((examples, log_examples, abs_accuracies, accuracy_drops,
                                        first_task_accuracies, new_task_accuracies),
                                       RESULT_SET_NAMES,
                                       ANALYSIS_SET_FIELDS):
        print(name, field)
        output[field] = ResultSet(name=name,
                                  mean=np.nanmean(result_set, axis=0),
                                  std=np.nanstd(result_set, axis=0))

    # eliminate zeros to allow dividing later
    accuracy_counts[accuracy_counts == 0] = 1
    output['accuracy_counts'] = accuracy_counts

    # to avoid division by zero
    counts_by_task[counts_by_task == 0] = 1
    average_examples_by_task = np.divide(examples_by_task, counts_by_task)
    average_log_examples_by_task = np.divide(log_examples_by_task, counts_by_task)
    output['examples_by_task'] = dict(raw=average_examples_by_task, log=average_log_examples_by_task)

    analysis = AnalysisSet(**output)

    if debug:
        return analysis, examples

    return analysis

def process_multiple_runs_total_task_training_curves(runs, debug=False, ignore_runs=None, samples=MAX_HISTORY_SAMPLES):
    aggregate_total_task_curve = defaultdict(lambda: defaultdict(list))

    for i, run in enumerate(runs):
        if i > 0 and i % 10 == 0:
            print(run.name, i)
        else:
            print(run.name)

        if ignore_runs is not None and run.name in ignore_runs:
            continue

        total_task_curve = parse_total_task_training_curve(current_run=run, samples=samples)
        for task in total_task_curve:
            for num_trials, acc in total_task_curve[task].items():
                aggregate_total_task_curve[task][num_trials].append(acc)

    aggregate_total_task_curve_raw = defaultdict(OrderedDict)
    aggregate_total_task_curve_mean = defaultdict(OrderedDict)
    aggregate_total_task_curve_std = defaultdict(OrderedDict)
    aggregate_total_task_curve_sem = defaultdict(OrderedDict)

    for task in range(1, 11):
        for num_trials in sorted(aggregate_total_task_curve[task]):
            accs = aggregate_total_task_curve[task][num_trials]

            aggregate_total_task_curve_raw[task][num_trials] = accs
            aggregate_total_task_curve_mean[task][num_trials] = np.nanmean(accs)

            std = np.nanstd(accs)
            if np.isnan(std):
                print(f'For accs {accs}, std is nan')

            aggregate_total_task_curve_std[task][num_trials] = std
            aggregate_total_task_curve_sem[task][num_trials] = std / (len(accs) ** 0.5)

    return aggregate_total_task_curve_raw, aggregate_total_task_curve_mean, aggregate_total_task_curve_std, aggregate_total_task_curve_sem

def sign_test(values):
    n = len(values)
    row_faster_results = np.empty((n, n))
    row_faster_results.fill(np.nan)

    col_faster_results = np.empty((n, n))
    col_faster_results.fill(np.nan)

    wilcoxon_statistics = np.empty((n, n))
    wilcoxon_statistics.fill(np.nan)

    wilcoxon_p_values = np.empty((n, n))
    wilcoxon_p_values.fill(np.nan)

    for row in range(n):
        for col in range(row + 1, n):
            row_means = values[row].mean
            row_means = row_means[~np.isnan(row_means)]
            col_means = values[col].mean
            col_means = col_means[~np.isnan(col_means)]

            # Run both explicit comparisons, to account for the possibility of ties
            row_faster = np.sum(row_means <col_means )
            row_faster_results[row, col] = row_faster

            col_faster = np.sum(row_means > col_means)
            col_faster_results[row, col] = col_faster

            s, p = wilcoxon(row_means, col_means)
            wilcoxon_statistics[row, col] = s
            wilcoxon_p_values[row, col] = p

    return row_faster_results, col_faster_results, wilcoxon_statistics, wilcoxon_p_values

MODULATION_LEVELS = ['\\thead[cl]{None}'] + [f'\\thead[cl]{{ {i} }}' for i in range(1, 5)]
SIGN_TEST_PRINT_HEADERS = ['\\thead[cl]{Modulation level}'] + [f'\\thead[cl]{{ {i} }}' for i in range(1, 5)]

def pretty_print_sign_test_results(row_faster_results, col_faster_results,
                                   wilcoxon_statistics=None, wilcoxon_p_values=None,
                                   higher_better=False, **kwargs):
    n = row_faster_results.shape[0]
    print_results = [[''] * n for _ in range(n)]
    if wilcoxon_statistics is not None:
        wilcoxon_results = [[''] * n for _ in range(n)]

    for row in range(n):
        for col in range(row + 1, n):
            row_val = int(row_faster_results[row, col])
            col_val = int(col_faster_results[row, col])
            if higher_better:
                row_val, col_val = col_val, row_val

            result = col_val # max(col_val, row_val)
            n_binom = col_val + row_val
            p = binom_test(max(col_val, row_val), n_binom)

            print_results[row][col] = f'\\makecell[cl]{{ {result} $(n={n_binom})$ \\\\ $p={p:.4f}{(p < 0.05 and p > 0.01) and "^{*}" or ""}{p < 0.01 and "^{**}" or ""}$}}'

            if wilcoxon_statistics is not None:
                wilcoxon_results[row][col] = f'\n{wilcoxon_statistics[row, col]:.4f}, {wilcoxon_p_values[row, col]:.4f}'

    tab_args = dict(tablefmt='fancy_grid')
    tab_args.update(kwargs)

    print_results = [[MODULATION_LEVELS[i]] + row[1:] for i, row in enumerate(print_results)]
    print(tabulate.tabulate(print_results[:-1], SIGN_TEST_PRINT_HEADERS, **tab_args))

    if wilcoxon_statistics is not None:
        wilcoxon_results = [[MODULATION_LEVELS[i]] + row[1:] for i, row in enumerate(wilcoxon_results)]
        print(tabulate.tabulate(wilcoxon_results[:-1], SIGN_TEST_PRINT_HEADERS, **tab_args))

def sign_test_with_sem(values, sample_sizes):
    n = len(values)
    row_faster_results = np.empty((n, n))
    row_faster_results.fill(np.nan)

    col_faster_results = np.empty((n, n))
    col_faster_results.fill(np.nan)

    wilcoxon_statistics = np.empty((n, n))
    wilcoxon_statistics.fill(np.nan)

    wilcoxon_p_values = np.empty((n, n))
    wilcoxon_p_values.fill(np.nan)

    for row in range(n):
        for col in range(row, n):
            row_means = values[row].mean
            row_means = row_means[~np.isnan(row_means)]
            row_sems = values[row].std / (sample_sizes[row] ** 0.5)
            row_sems = row_sems[~np.isnan(row_sems)]

            col_means = values[col].mean
            col_means = col_means[~np.isnan(col_means)]
            col_sems = values[col].std / (sample_sizes[col] ** 0.5)
            col_sems = col_sems[~np.isnan(col_sems)]

            # Run both explicit comparisons, to account for the possibility of ties
            row_faster_results[row, col] = np.sum(row_means + row_sems < col_means - col_sems)
            col_faster_results[row, col] = np.sum(row_means - row_sems > col_means + col_sems)

            signs = row_means < col_means
            signs[signs == 0] = -1

            shifted_row_means = row_means + np.multiply(row_sems, signs)
            shifted_col_means = col_means + np.multiply(col_sems, -1 * signs)
            s, p = wilcoxon(shifted_row_means, shifted_col_means)
            wilcoxon_statistics[row, col] = s
            wilcoxon_p_values[row, col] = p

    return row_faster_results, col_faster_results, wilcoxon_statistics, wilcoxon_p_values

DEFAULT_LEARNING_RATE = 5e-4
DEFAULT_WEIGHT_DECAY = 1e-4

def create_task_conditional_model(additive=True, multiplicative=True, checkpoint_path=None, name=None):
    mod_level = list(range(4))

    model = cnnmlp.TaskConditionalCNNMLP(
        mod_level=mod_level,
        multiplicative_mod=multiplicative,
        additive_mod=additive,
        query_length=30,
        conv_filter_sizes=(16, 32, 48, 64),
        conv_output_size=4480,
        mlp_layer_sizes=(512, 512, 512, 512),
        lr=DEFAULT_LEARNING_RATE,
        weight_decay=DEFAULT_WEIGHT_DECAY,
        use_lr_scheduler=False,
        conv_dropout=False,
        mlp_dropout=False,
        name=name)

    if checkpoint_path is not None:
        model.load_state(checkpoint_path)

    return model

def parse_task_conditional_weights(runs, additive=True, multiplicative=True,
                                   layers=range(4), download_root='/tmp', ignore_runs=None):
    weights_by_run_id = {}

    for i, run in enumerate(runs):
        if ignore_runs is not None and run.name in ignore_runs:
            continue

        j = i + 1
        if j % 10 == 0:
            print(run.name, j)
        else:
            print(run.name)

        dimension = run.config['benchmark_dimension']

        last_checkpoint_file = run.file(f'{run.name.replace("[0, 1, 2, 3]-", "")}-query-9.pth')
        last_checkpoint_file.download(replace=True, root=download_root)
        model = create_task_conditional_model(additive=additive, multiplicative=multiplicative,
                                             checkpoint_path=os.path.join(download_root, last_checkpoint_file.name))

        additive_weights = None
        multiplicative_weights = None

        if additive:
            additive_weights = [model.conv.additive_mod_layers[f'additive-{i}'].weight.detach().cpu().numpy()[:, dimension * 10:(dimension + 1) * 10]
                                for i in layers]

        if multiplicative:
            multiplicative_weights = [model.conv.multiplicative_mod_layers[f'multiplicative-{i}'].weight.detach().cpu().numpy()[:, dimension * 10:(dimension + 1) * 10]
                                      for i in layers]

        weights_by_run_id[run.config['latin_square_index']] = TaskConditionalWeights(additive=additive_weights, multiplicative=multiplicative_weights)

    return weights_by_run_id


"""
각 자리가 숫자(0~9)로만 이루어진 문자열 S가 주어졌을 때, 왼쪽부터 오른쪽으로 하나씩 확인하며 'x', '+' 연산자를 넣어 결과적으로 만들어질 수 있는 가장 큰 수를 구하는 프로그램을 작성하라.
단, 일반적인 계산 방식과 달리 모든 연산은 왼쪽에서부터 순서대로 이루어진다고 가정한다.
1 <= S <= 20
"""
# 내가 짠 코드 <- 숫자가 0과 1이라면 더하는 것이 무조건 크다는 사실을 간과, +한 결과값이랑 x한 결과값이랑 매번 비교함...
s = input()

for i in range(len(s) - 1):
  if i == 0:
    re1 = int(s[i]) + int(s[i + 1])
    re2 = int(s[i]) * int(s[i + 1])
  else:
    re1 = result + int(s[i + 1])
    re2 = result * int(s[i + 1])
  result = max(re1, re2)

print(result)

###################################################

data = input()
result = int(data[0])

for i in range(1, len(data)):
  num = int(data[i])
  if num <= 1 or result <= 1: # 숫자과 0과 1이라면 +, 아니라면 x
    result += num
  else:
    result *= num

print(result)


# -*- coding: utf-8 -*-
"""
Created on Sun Feb 18 21:49:05 2018

@author: jon_c
"""

import turtle
wn = turtle.Screen()
wn.bgcolor("lightgreen")
wn.title("Welcome")

wn.textinput("Name", "Name of player?")
alex = turtle.Turtle()
alex.color("hotpink")
alex.pensize(5)
alex.forward(50)
alex.left(90)
alex.forward(3)

james = turtle.Turtle()
james.left(90)
james.forward(100)

tess = turtle.Turtle()
tess.back(100)

wn.exitonclick()

#1. Modify this program so that before it creates the window, it prompts the user to enter
#the desired background color. It should store the user’s responses in a variable, and
#modify the color of the window according to the user’s wishes. (Hint: you can find a list
#of permitted color names at http://www.tcl.tk/man/tcl8.4/TkCmd/colors.htm. It includes
#some quite unusual ones, like “peach puff” and “HotPink”.)
#2. Do similar changes to allow the user, at runtime, to set tess‘ color.
#3. Do the same for the width of tess‘ pen. Hint: your dialog with the user will return a
#string, but tess‘ pensize method expects its argument to be an int. So you’ll need to
#convert the string to an int before you pass it to pensize.

for f in ["Joe","Zoe","Brad","Angelina","Zuki","Thandi","Paris"]:
    invite = "Hi " + f + ". Please come to my party on Saturday!"
    print(invite)

for i in range(3, 6):
    test = 123
    print (test)

import turtle
wn = turtle.Screen()
wn.bgcolor("lightgreen")
wn.title("Welcome")

wn.textinput("Name", "Name of player?")
alex = turtle.Turtle()
alex.color("hotpink")
for c in ["black", "red", "purple", "blue"]:
    alex.color(c)
    alex.forward(50)
    alex.left(90)

wn.exitonclick()

import turtle
wn = turtle.Screen()
wn.bgcolor("lightgreen")
alex = turtle.Turtle()
alex.shape("turtle")
alex.color("blue", "yellow")

#alex.penup()
size = 20
colors = ["black", "red", "yellow", "blue"]
for colors in range (30):
    alex.stamp()
    size = size + 3
    alex.forward(size)
    alex.right(24)

wn.mainloop()
#yo, how to make the turtle change color with each
#step?

def hello():
    print ("Hello World!")

hello()

def name():
    print ('jon')

name()

def add_numbers(x, y, z):
    a = x + y
    b = x + z
    c = y + z
    print (a, b, c)

add_numbers(1, 2, 3)

def names():
    name = str(input("Enter your name:"))
    if set ("aeiou").intersection(name.lower()):
        print ("Your name has a vowel")
    else:
        print ("Your name does not have a vowel")

    for letter in name:
        print (letter)

names()

import turtle

def draw_square (t, sz):
    for i in range(4):
        t.forward(sz)
        t.left(90)

wn = turtle.Screen()
wn.bgcolor("lightgreen")
wn.title("Alex meets a function")

alex = turtle.Turtle
draw_square (alex, 50)
wn.mainloop()

import turtle

def draw_multicolor_square(t, sz):
    for i in ["red", "purple", "hotpink", "blue"]:
        t.color(i)
        t.forward(sz)
        t.left(90)

wn = turtle.Screen() # Set up the window and its attributes
wn.bgcolor("lightgreen")

tess = turtle.Turtle() # Create tess and set some attributes
tess.pensize(3)

size = 20 # Size of the smallest square
for i in range(15):
    draw_multicolor_square(tess, size)
    size = size + 10 # Increase the size for next time
    tess.forward(10) # Move tess along a little
    tess.right(18) # and give her some turn

wn.mainloop()

def final_amt(p, r, n, t):
    a = p * (1 + (r/n))**(n*t)
    return a

toInvest = float(input("How much do you want to invest?"))
fnl = final_amt(toInvest, 0.08, 12, 5)
print ("At the end of the period you'll have", fnl)

def next_function(a):
    a = 8
    return a
a = 20
print ( next_function (a))

unit = 100
currency = "JPY"
rate = 1.183

money_changer = "we sell {0:6d} {1:>10s} at {2:4.3f} SGD".format(unit, currency, rate)
print(money_changer)
#above is to force a return value with a certain number of decimals or spaces

exam_questions/final/final_exam_resources/john_checkoff_n_1D/Week 2 (1D)/Week 2.py


import sys
sys.stdin = open('input.txt')

dx = [-1, 0, 1, 0]
dy = [0, 1, 0, -1]

def bfs(x, y):
    que = []
    que.append((x, y))
    data[x][y] = 2
    while que:
        x, y = que.pop(0)
        for d in range(4):
            nx = x + dx[d]
            ny = y + dy[d]
            if 0 <= nx < M and 0 <= ny < N and data[nx][ny] == 1:
                que.append((nx, ny))
                data[nx][ny] = 2

T = int(input())
for _ in range(T):
    M, N, K = map(int, input().split())

    data = [[0]*N for _ in range(M)]

    for _ in range(K):
        x, y = map(int, input().split())
        data[x][y] = 1

    ans = 0
    for i in range(M):
        for j in range(N):
            if data[i][j] == 1:
                bfs(i, j)
                ans += 1

    print(ans)


#! /usr/bin/env python3

# Number of integer partitions of n into distinct parts with at least one neighborless part.

from labmath import *       # Available via pip (https://pypi.org/project/labmath/)

print("0 0")

for n in range(1, 101):
    total = 0
    for party in partgen(n):
        partyset = set(party)
        if len(party) != len(partyset): continue
        if any(((x-1) not in partyset) and ((x+1) not in partyset) for x in partyset): total += 1
    print(n, total)


import numpy as np
import matplotlib.pyplot as plt

gamma = np.array([1,2,3,4,5])
fmu=np.array([0.396, 0.469, 0.492, 0.499, 0.502])
ftau = np.array([0.531, 0.584, 0.601, 0.606, 0.608])
fe = np.ones_like(fmu)

ftot = fmu+ftau+fe

fe = fe/ftot
fmu = fmu/ftot
fatu = ftau/ftot

print(fe)
print(fmu)
print(ftau)

fig = plt.figure(1)
plt.scatter(gamma, fe, label="e")
plt.scatter(gamma, fmu, label="mu")
plt.scatter(gamma, ftau, label="tau")
plt.legend(loc="center right")
plt.xlabel(r"\gamma")
plt.ylabel("Ratio")
plt.show()


import torch
import torch.nn as nn
import torch.nn.functional as func
from policies.generic_net import GenericNet
from torch.distributions import Categorical
import numpy as np
from numpy.random import choice
import random
import math

def make_det_vec(values):
    """
    Transform the output vector of a Bernoulli policy into a vector of deterministic choices
    :param values: the Bernoulli policy output vector (turned into a numpy array)
    :return: the vector of binary choices
    """
    choices = []
    #### MODIF : transform in array of ints
    if len(values.shape) ==2 :
        for v in values:
            choices.append(np.argmax(v).data.numpy().astype(int))
    else:
        choices.append(int(np.argmax(values)))
    ####
    return choices

class DiscretePolicy(GenericNet):
    """
    A policy whose probabilistic output is a boolean value for each state
    """
    def __init__(self, l1, l2, l3, l4, learning_rate):
        super(DiscretePolicy, self).__init__()
        self.relu = nn.ReLU()
        self.fc1 = nn.Linear(l1, l2)
        # self.fc1.weight.data.uniform_(-1.0, 1.0)
        self.fc2 = nn.Linear(l2, l3)
        # self.fc2.weight.data.uniform_(-1.0, 1.0)
        self.fc3 = nn.Linear(l3, l4)  # Prob of Left
        # self.fc3.weight.data.uniform_(-1.0, 1.0)
        self.optimizer = torch.optim.Adam(self.parameters(), lr=learning_rate)

    def forward(self, state):
        """
         Compute the pytorch tensors resulting from sending a state or vector of states through the policy network
         The obtained tensors can be used to obtain an action by calling select_action
         :param state: the input state(s)
         :return: the resulting pytorch tensor (here the probability of giving 0 or 1 as output)
         """
        state = torch.from_numpy(state).float()
        state = self.relu(self.fc1(state))
        state = self.relu(self.fc2(state))
        #### MODIF : removed sigmoid because we use 3 digits for MountainCar now.
        action = self.fc3(state)
        ####
        return action

    def select_action(self, state, deterministic=False):
        """
        Compute an action or vector of actions given a state or vector of states
        :param state: the input state(s)
        :param deterministic: whether the policy should be considered deterministic or not
        :return: the resulting action(s)
        """
        with torch.no_grad():
            probs = self.forward(state)
            if deterministic:
                return make_det_vec(probs)
            else:
                #### MODIF : we use Categorical in place of Bernoulli for Mountain Car now.
                action = Categorical(logits = probs).sample().data.numpy().astype(int)
                #### MODIF
            return action

    def train_pg(self, state, action, reward):
        """
        Train the policy using a policy gradient approach
        :param state: the input state(s)
        :param action: the input action(s)
        :param reward: the resulting reward
        :return: the loss applied to train the policy
        """
        action = torch.FloatTensor(action)
        reward = torch.FloatTensor(reward)
        probs = self.forward(state)
        #### MODIF : we use Categorical with digits for Mountain Car now.
        m = Categorical(logits = probs)
        ####
        loss = -m.log_prob(action) * reward  # Negative score function x reward
        #print("loss : "+str(loss))
        self.update(loss)
        return loss

    def train_regress(self, state, action):
        """
          Train the policy to perform the same action(s) in the same state(s) using regression
          :param state: the input state(s)
          :param action: the input action(s)
          :return: the loss applied to train the policy
        """
        action = torch.FloatTensor(action)
        proposed_action = self.forward(state)
        loss = func.cross_entropy(proposed_action, action.flatten().long())
        self.update(loss)
        return loss


import datetime
import socket
import sys, os
import time

inmotionautomation_home_dirname = os.environ['INMOTIONAUTOMATION_HOME']
sys.path.append(inmotionautomation_home_dirname + "/lib/common")
sys.path.append(inmotionautomation_home_dirname + "/lib/site-packages")
sys.path.append(inmotionautomation_home_dirname + "/testsuite/inmotion/Feature/GPS")
sys.path.append(inmotionautomation_home_dirname + "/testsuite/inmotion/Feature/GPS/gps_lib")

import proxy_airlink

listener = "192.168.100.3"
address = "208.81.123.36"
port = 15415
def recvall(sock):
    sock.settimeout(0.1)
    data = ""
    buf = ""
    t = time.time()

    while ((time.time() - t) < 1):
        try:
            buf, addr = sock.recvfrom(1024)
            t = time.time()
            data += buf
        except:
            pass
    return data

proxy_host = proxy_airlink.ProxyAirlink(listener)
remote_proxy_connect = proxy_host.connect()
if not remote_proxy_connect:
    sys.exit("Could not connect Listener via Proxy!")
else:
    print("Listener Is Online.")

sock = remote_proxy_connect.modules.socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((address, port))

print(datetime.datetime.now())

t = time.time()
while ((time.time() - t) < 200):

    buf = recvall(sock)

    print(datetime.datetime.now())

    if buf:
        print(buf)
        t = time.time()
    # time.sleep(1)

sock.close()

print("END")


import datetime
import logging
import os
import sys
from typing import Optional, Sequence, NamedTuple

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.layers import BatchNormalization, Dense, Input

NUM_INPUTS = 96
NUM_OUTPUTS = 1
VAL_SPLIT = 0.1  # Percentage
BATCH_SIZE = 64
EPOCHS = 1000
EXPERIMENTS_NUM = 30
PATIENCE = 10  # Early stoping
LEARNING_RATE = 1e-3
BATCH_NORM_LAYER = False
CLIP_PREDS = True
CLIP_INPUTS = False

class Loss(NamedTuple):
    mse: float
    mae: float
    smape: float

class ForecastResult(NamedTuple):
    targets: np.array
    preds: np.array
    loss: Loss

def make_dataset(df: pd.DataFrame, shuffle: bool = True) -> tf.data.Dataset:
    dataset = tf.data.Dataset.from_tensor_slices(df.to_numpy())
    if shuffle:
        dataset = dataset.shuffle(buffer_size=1000)
    dataset = dataset.map(lambda window: (window[:NUM_INPUTS], window[NUM_INPUTS:]))
    dataset = dataset.batch(batch_size=BATCH_SIZE, drop_remainder=False).prefetch(3)

    return dataset

def get_compiled_model(
    input_shape: Sequence[int],
    num_outputs: int,
    hidden: Sequence[int] = (100, 100, 100),
    activation: str = "relu",
    lr: float = 1e-3,
    batch_norm: bool = False,
) -> keras.Model:
    model = tf.keras.Sequential()
    model.add(Input(shape=input_shape))
    if batch_norm:
        model.add(BatchNormalization())
    for h_units in hidden:
        model.add(Dense(h_units, activation="relu"))
    model.add(Dense(num_outputs))

    model.compile(
        optimizer=keras.optimizers.Adam(lr=lr),
        loss=keras.losses.mse,
        metrics=[keras.metrics.mse],
    )

    return model

def mse(a: np.array, b: np.array) -> float:
    return np.mean(np.square(a - b))

def mae(a: np.array, b: np.array) -> float:
    return np.mean(np.abs(a - b))

def smape(a: np.array, b: np.array) -> float:
    num = np.abs(b - a)
    den = np.abs(a) + np.abs(b)
    sum = np.where(den == 0, 0, num / den)
    return sum.mean()

def scale_dataframe(
    df: pd.DataFrame, cols: Sequence[str] = None, new_max: int = 1, new_min: int = 0
) -> None:
    cols = cols or df.columns
    for col in cols:
        col_max = df[col].max()
        col_min = df[col].min()
        df[col] = new_min + ((df[col] - col_min) * (new_max - new_min)) / (
            col_max - col_min
        )

        assert df[col].min() == new_min
        assert df[col].max() == new_max

def read_csv(path: str, scale: bool = False, **kwargs) -> pd.DataFrame:
    logging.info(f"Reading {path} . . .")
    drop_cols = [f"y{i}" for i in range(NUM_OUTPUTS, 48)]  # drop unnecessary outputs
    drop_cols.append("Unnamed: 0")
    df = pd.read_csv(path)
    df.drop(columns=drop_cols, inplace=True)

    if scale:
        logging.info(f"Scaling {path} . . .")
        input_cols = [f"x{i}" for i in range(96)]
        scale_dataframe(df, input_cols, **kwargs)

    return df

def create_log_dir(path: str, dated: bool = False, fmt: str = "%Y-%m-%d_%H-%M") -> str:
    if dated:
        today = datetime.datetime.now().strftime(fmt)
        path += "-" + today
    os.makedirs(path, exist_ok=True)
    logging.info(f"Folder created {path}")

    return path

def experiment(
    data_model: str,
    scale: bool = False,
    seed: Optional[int] = 42,
    save_figs: bool = False,
    batch_norm_layer: bool = False,
    clip_preds: bool = False,
    clip_inputs: bool = False,
):
    if seed:
        np.random.seed(seed)

    print(f"Running model: {data_model}, scale: {scale}")
    path_fig = create_log_dir(os.path.join("figs", data_model), dated=True)
    path = "data{sep}amarillo_{{model}}_{{dataset}}.csv".format(sep=os.path.sep)

    # mix input differences with real output
    if data_model == "diffmix":
        df_train_norm = read_csv(
            path.format(model="norm", dataset="train"), scale=scale
        )
        df_test_norm = read_csv(path.format(model="norm", dataset="test"), scale=scale)
        df_train_diff = read_csv(
            path.format(model="diff", dataset="train"), scale=scale
        )
        df_test_diff = read_csv(path.format(model="diff", dataset="test"), scale=scale)
        cols = df_train_norm.columns
        x_inputs = [col for col in cols if "x" in col]
        df_train = df_train_diff[x_inputs].copy()
        df_train["y0"] = df_train_norm["y0"].values
        df_test = df_test_diff[x_inputs].copy()
        df_test["y0"] = df_test_norm["y0"].values
        np.array_equal(df_train[x_inputs].values, df_train_diff[x_inputs].values)
        np.array_equal(df_test[x_inputs].values, df_test_diff[x_inputs].values)
        np.array_equal(df_train["y0"].values, df_train_norm["y0"].values)
        np.array_equal(df_test["y0"].values, df_test_norm["y0"].values)
    else:
        df_train = read_csv(path.format(model=data_model, dataset="train"), scale=scale)
        df_test = read_csv(path.format(model=data_model, dataset="test"), scale=scale)

    if clip_inputs and not scale:
        print("Clipping inputs . . .")
        df_train[df_train < 0] = 0
        df_test[df_test < 0] = 0

    # Validation
    len_val = int(VAL_SPLIT * len(df_train))
    val_ind = df_train.sample(len_val).index.values
    df_val = df_train.iloc[val_ind]
    df_train.drop(labels=val_ind, inplace=True)

    # Make datasets
    ds_train = make_dataset(df_train)
    ds_val = make_dataset(df_val)
    ds_test = make_dataset(df_test, shuffle=False)

    model = get_compiled_model(
        input_shape=(NUM_INPUTS,),
        hidden=(100, 100, 100,),
        num_outputs=NUM_OUTPUTS,
        lr=LEARNING_RATE,
        batch_norm=batch_norm_layer,
    )
    print("Training model . . .")
    model.fit(
        ds_train,
        epochs=EPOCHS,
        validation_data=ds_val,
        callbacks=[
            keras.callbacks.EarlyStopping(
                monitor="val_loss", patience=PATIENCE, restore_best_weights=True
            )
        ],
    )

    if data_model != "diff":
        targets = df_test["y0"].values
        preds = model.predict(ds_test).T[0]
    # Calculate predictions on original dataset
    else:
        df_sky = read_csv(path.format(model="sky", dataset="test"))
        df_norm = read_csv(path.format(model="norm", dataset="test"))
        preds_temp = model.predict(ds_test).T[0]
        sky = df_sky["y0"].values
        targets = df_norm["y0"].values
        preds = sky - preds_temp

    if clip_preds:
        preds[preds < 0] = 0

    mse_loss = mse(targets, preds)
    mae_loss = mae(targets, preds)
    smape_loss = smape(targets, preds)
    losses = Loss(mse_loss, mae_loss, smape_loss)
    print(losses)

    if save_figs:
        for index, (target, pred) in enumerate(
            zip(targets.reshape(1, 10, 144)[0], preds.reshape(1, 10, 144)[0],)
        ):
            plt.clf()
            plt.plot(target[:48], "b", label="Targets")
            plt.plot(pred[:48], "r", label="Forecast")
            # plt.ylim(-100, 1100)
            plt.legend()
            plt.title(f"Irradiance level {index+1}")
            temp_path = os.path.join(path_fig, f"fig_level_{index+1}")
            plt.savefig(temp_path)
            logging.debug(f"Fig saved to {temp_path}")

        plt.close()

    return ForecastResult(targets, preds, losses)

logging.basicConfig(level=logging.INFO)
exps = {
    "norm_unscaled_exp": {
        "config": {"data_model": "norm", "scale": False,},
        "results": {},
    },
    "diff_unscaled_exp": {
        "config": {"data_model": "diff", "scale": False,},
        "results": {},
    },
    # "norm_scaled_exp": {
    #     "config": {"data_model": "norm", "scale": True,},
    #     "results": {},
    # },
    # "diff_scaled_exp": {
    #     "config": {"data_model": "diff", "scale": True,},
    #     "results": {},
    # },
    # "diffmix_unscaled_exp": {
    #     "config": {"data_model": "diffmix", "scale": False,},
    #     "results": {},
    # },
    # "diffmix_scaled_exp": {
    #     "config": {"data_model": "diffmix", "scale": True,},
    #     "results": {},
    # },
}

for exp_name, exp_dict in exps.items():
    mean_mse = 0
    mean_mae = 0
    mean_smape = 0
    min_mse = np.inf
    max_mse = -np.inf
    exp_min = 0
    exp_max = 0

    # Run experiments
    for i in range(EXPERIMENTS_NUM):
        mdl = exp_dict["config"]["data_model"]
        scl = exp_dict["config"]["scale"]
        current_exp = experiment(
            mdl,
            scale=scl,
            seed=None,
            save_figs=True,
            batch_norm_layer=BATCH_NORM_LAYER,
            clip_preds=CLIP_PREDS,
            clip_inputs=CLIP_INPUTS,
        )
        exps[exp_name]["results"][str(i)] = current_exp

        mean_mse += current_exp.loss.mse
        mean_mae += current_exp.loss.mae
        mean_smape += current_exp.loss.smape

        if current_exp.loss.mse < min_mse:
            min_mse = current_exp.loss.mse
            exp_min = i

        if current_exp.loss.mse > max_mse:
            max_mse = current_exp.loss.mse
            exp_max = i

    # Compute mean errors across experiments
    exps[exp_name]["metrics"] = {
        "mse": mean_mse / EXPERIMENTS_NUM,
        "mae": mean_mae / EXPERIMENTS_NUM,
        "smape": mean_smape / EXPERIMENTS_NUM,
        "min_mse": min_mse,
        "max_mse": max_mse,
        "exp_min": exp_min,
        "exp_max": exp_max,
    }

# Mean preds
for exp_name in exps.keys():
    all_preds = []
    for fcast in exps[exp_name]["results"].values():
        all_preds.append(fcast.preds)
    mean_preds = np.mean(all_preds, axis=0)
    exps[exp_name]["results"]["mean_pred"] = mean_preds
    targets = exps[exp_name]["results"]["0"].targets

    for index, (target, pred) in enumerate(
        zip(targets.reshape(1, 10, 144)[0], mean_preds.reshape(1, 10, 144)[0],)
    ):
        plt.clf()
        plt.plot(target[:48], "b", label="Targets")
        plt.plot(pred[:48], "r", label="Forecast")
        plt.legend()
        plt.title(f"Irradiance level {index+1}")
        dir_path = os.path.join("figs", "means", exp_name)
        os.makedirs(dir_path, exist_ok=True)
        temp_path = os.path.join(dir_path, f"fig_level_{index+1}")
        plt.savefig(temp_path)
        logging.debug(f"Fig saved to {temp_path}")

    plt.close()

for exp_name, exp_dict in exps.items():
    print(f'{exp_name}: {exp_dict["metrics"]}')


import logging
import inspect

def custom_logger():
    loggername = inspect.stack()[1][3]
    logger = logging.getLogger(__name__)
    fileHandler = logging.FileHandler("logfile.log", 'w', encoding="UTF-8")
    formatter = logging.Formatter("%(asctime)s: %(levelname)s %(name)s: %(message)s")
    fileHandler.setFormatter(formatter)
    logger.addHandler(fileHandler)
    logger.setLevel(logging.DEBUG)
    return logger


from bisect import bisect_left, bisect_right
from datetime import date, datetime, timedelta
from heapq import merge
from itertools import chain, count, cycle
import re

__all__ = ['cron', 'periodic']

__author__ = 'Brandon Carpenter <brandon.carpenter@pnnl.gov>'
__copyright__ = 'Copyright (c) 2016, Battelle Memorial Institute'
__license__ = 'Apache 2.0'

_range_re = re.compile(r'^(.{0}(?=\*)|\w*(?=-)|\w+(?!\*))'
                       r'(?:[*-]?((?<=\*).{0}|(?<=-)\w*|(?=/).{0}))?'
                       r'(?:/(\d+))?$', re.I)

def _split_range(string):
    '''Generator to split cron ranges into (start, end, skip) tuples.

    Takes a string of the folloing forms:

        [start][-[end]][/skip][,...]
        *[/skip]

    where * is equivalent to first-last and yeilds the 3-tuple with each
    iteration.
    '''
    for rng in string.split(','):
        if not rng:
            continue
        match = _range_re.match(rng)
        if match:
            yield match.groups()
        else:
            raise ValueError('bad range expresion: {}'.format(rng))

def _convert_item(item, default, translate=None):
    '''Convert item to an integer.

    Return None if item is None or default if item is the empty string.
    Otherwise, try to convert item using the int() builtin.  If int()
    fails, use the translate function, if given, or re-raise ValueError.
    '''
    if item is None:
        return
    if not item:
        return default
    try:
        return int(item)
    except ValueError:
        if not translate:
            raise
        return translate(item)

def _convert_range(rng, minimum, maximum, translate=None):
    '''Convert range 3-tuples to integer lists.

    If rng evaluates to False, just return int.  Otherwise, convert the
    range values to integers using the minimum as the default for the
    start value, maximum as the default for the end value and translate
    to convert string values.
    '''
    if not rng:
        return rng
    first, last, skip = rng
    first = _convert_item(first, minimum, lambda x: translate(x, 0))
    last = _convert_item(last, maximum, lambda x: translate(x, 1))
    if last is None:
        return [first]
    skip = _convert_item(skip, 1)
    if skip is None:
        return list(range(first, last + 1))
    return list(range(first, last + 1, skip))

def _coallesce_ranges(fieldname, ranges, minimum, maximum, translate=None):
    '''Combine multiple ranges into a single sorted list of values.'''
    if not ranges:
        return
    result = set()
    for rng in ranges:
        rng = _convert_range(rng, minimum, maximum, translate)
        if not rng:
            continue
        if rng[0] < minimum:
            raise ValueError(
                    '{} value of {} is below the minimum of {}'.format(
                    fieldname, rng[0], minimum))
        if rng[-1] > maximum:
            raise ValueError(
                    '{} value of {} is above the maximum of {}'.format(
                    fieldname, rng[-1], maximum))
        result |= set(rng)
    return tuple(sorted(result)) or None

def _translate_month(month, pos):
    '''Translate month names to integers.'''
    try:
        return ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug',
                'sep', 'oct', 'nov', 'dec'].index(month[:3].lower()) + 1
    except ValueError:
        raise ValueError('invalid month name: {}'.format(month))

def _translate_weekday(weekday, pos):
    '''Translate weekday names to integers.'''
    try:
        index = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'].index(
                 weekday[:3].lower()) + 1
    except ValueError:
        raise ValueError('invalid day name: {}'.format(weekday))
    # If Sunday is is in the start position, return it as 0
    if index == 7 and not pos:
        index = 0
    return index

def parse_cron_string(cron_string):
    fields = cron_string.split()
    if len(fields) > 5:
        raise ValueError('too many fields in cron string')
    elif len(fields) < 5:
        raise ValueError('too few fields in cron string')
    minute, hour, day, month, weekday = [
            None if field == '*' else _split_range(field) for field in fields]
    return (_coallesce_ranges('minute', minute, 0, 59),
            _coallesce_ranges('hour', hour, 0, 23),
            _coallesce_ranges('day', day, 1, 31),
            _coallesce_ranges('month', month, 1, 12, _translate_month),
            _coallesce_ranges('weekday', weekday, 0, 7, _translate_weekday))

def _start_stop(start, stop):
    # Default start date/time to current time
    if start is None:
        start = datetime.now()
    elif isinstance(start, (int, float)):
        start = datetime.fromtimestamp(start)
    elif isinstance(start, timedelta):
        start = datetime.now() + start
    if isinstance(stop, (int, float)):
        stop = datetime.fromtimestamp(stop)
    elif isinstance(stop, timedelta):
        stop = start + stop
    return start, stop

def cron(cron_string, start=None, stop=None, second=0):
    '''Return a schedule generator from a cron-style string.

    cron_string is a cron-style time expression consisting of five
    whitespace-separated fields explained in further detail below.
    start and stop are used to bound the schedule and can be None,
    datetime.datetime or datetime.timedelta objects or numeric values,
    such as is returned by time.time(). If start is None, the current
    time is used. If it is a timedelta, it will be added to the current
    time. If stop is None, cron will generate values infinitely. If it
    is a timedelta, the end time is the start time plus stop. Each
    iteration yields a datetime.datetime object. Since the smallest cron
    unit is a minute, second may be passed in to offset the time within
    the minute.

    The following description of the cron fields is taken from the
    crontab(5) man page (with slight modifications).

    The time and date fields are:

           field          allowed values
           -----          --------------
           minute         0-59
           hour           0-23
           day of month   1-31
           month          1-12 (or names, see below)
           day of week    0-7 (0 or 7 is Sunday, or use names)

    A field may contain an asterisk (*), which always stands for
    "first-last".

    Ranges of numbers are allowed.  Ranges are two numbers separated
    with a hyphen.  The specified range is inclusive.  For example, 8-11
    for an 'hours' entry specifies execution at hours 8, 9, 10, and 11.
    If the range start or end value is left off, the first or last value
    will be used.  For example, -8 for an 'hours' entry is equivalent to
    0-8, 20- for a 'days of month' entry is equivalent to 20-31, and -
    for a 'months' entry is equivalent to 1-12.

    Lists are allowed.  A list is a set of numbers (or ranges) separated
    by commas.  Examples: "1,2,5,9", "0-4,8-12".

    Step values can be used in conjunction with ranges.  Following a
    range with "/<number>" specifies skips of the number's value through
    the range.  For example, "0-23/2" can be used in the 'hours' field
    to specify every other hour.  Step values are also permitted after
    an asterisk, "*/2" in the 'hours' field is equivalent to "0-23/2".

    Names can also be used for the 'month' and 'day of week' fields.
    Use at least the first three letters of the particular day or month
    (case does not matter).

    Note: The day can be specified in the following two fields: 'day of
    month', and 'day of week'.  If both fields are restricted (i.e., do
    not contain the "*" character), then both are used to compute
    date/time values.  For example, "30 4 1,15 * 5" is interpreted as
    "4:30 am on the 1st and 15th of each month, plus every Friday."
    '''

    if not 0 <= second < 60:
        raise ValueError('second is out of the range [0, 59]')
    minutes, hours, days, months, weekdays = parse_cron_string(cron_string)
    # Convert 0-Sunday to 7-Sunday to match datetime.isoweekday()
    if weekdays and weekdays[0] == 0:
        weekdays = weekdays[1:] + (() if weekdays[-1] == 7 else (7,))
    # Check that there are some valid month/day combinations.
    if months and days and not weekdays:
        unsafe = set([(2, 30), (2, 31), (4, 31), (6, 31), (9, 31), (11, 31)])
        combos = set([(m, d) for m in months for d in days])
        if not combos - unsafe:
            raise ValueError('given months and days produce only '
                             'impossible combinations')

    start, stop = _start_stop(start, stop)

    # Default fields to full range of values
    months = months or list(range(1, 13))
    hours = hours or list(range(0, 24))
    minutes = minutes or list(range(0, 60))

    def _weekdays(year, month, day=1):
        '''Iterate over all the days in weekdays for the given year and
        month starting with day.
        '''
        dt = date(year, month, day)
        weekday = dt.isoweekday()
        i = bisect_left(weekdays, weekday)
        dt += timedelta(weekdays[i] - weekday if i < len(weekdays)
                        else weekdays[0] + 7 - weekday)
        day, weekday = dt.day, dt.isoweekday()
        for next in chain(weekdays[i + 1:], cycle(weekdays)):
            if day > 31:
                break
            yield day
            day, weekday = day + (next - weekday if next > weekday
                                  else next + 7 - weekday), next

    # Handle special case when the start month and day are in the set
    if start.month in months:
        if (not (days or weekdays) or days and start.day in days or
                weekdays and start.isoweekday() in weekdays):
            if start.hour in hours:
                for minute in minutes[bisect_right(minutes, start.minute):]:
                    yield datetime(start.year, start.month, start.day, start.hour, minute, second)
            for hour in hours[bisect_right(hours, start.hour):]:
                for minute in minutes:
                    yield datetime(start.year, start.month, start.day, hour, minute, second)
        first_month = [(start.year, start.month, start.day + 1)]
    else:
        first_month = []

    # Iterate over all values until stop is hit
    for year, month, first_day in chain(first_month,
            ((start.year, m, 1) for m in months[bisect_right(months, start.month):]),
            ((y, m, 1) for y in count(start.year + 1) for m in months)):
        try:
            if days:
                _days = days[bisect_left(days, first_day):]
                if weekdays:
                    _days = merge(_days, _weekdays(year, month, first_day))
            elif weekdays:
                _days = _weekdays(year, month, first_day)
            else:
                _days = list(range(first_day, 32))
            for day in _days:
                for hour in hours:
                    for minute in minutes:
                        dt = datetime(year, month, day, hour, minute, second)
                        if stop and dt > stop:
                            return
                        yield dt
        except ValueError:
            pass

def periodic(period, start=None, stop=None):
    """Generate periodic datetime objects.

    Yields datetime objects increasing by the given period, which
    can be of type int, long, float, or datetime.timedelta.
    start and stop are used to bound the schedule and can be None,
    datetime.datetime or datetime.timedelta objects or numeric values,
    such as is returned by time.time(). If start is None, the current
    time is used. If it is a timedelta, it will be added to the current
    time. If stop is None, cron will generate values infinitely. If it
    is a timedelta, the end time is the start time plus stop. Each
    iteration yields a datetime.datetime object.
    """
    if not isinstance(period, timedelta):
        period = timedelta(seconds=period)
    dt, stop = _start_stop(start, stop)
    while stop is None or dt < stop:
        yield dt
        dt += period


import os,sys
import pygame
from pygame.locals import *

print('Warning, fonts disabled') if not pygame.font else print('Fonts enabled')
print('Warning, sound disabled') if not pygame.mixer else print ('Sound enabled')

def LoadImage(name, colourkey=None):
    '''
    LoadImage(name, colourkey=None)
    returns the image and the rectangle object created by pygame with the image's quality, when a filename and optional alpha channel key is entered.
    '''
    filename=os.path.join('planning/pygame/imagedata',name)
    try: #tries to load the file and sends an error if the file cannot be opened in image format
        image= pygame.image.load(filename)
    except pygame.error as message:
        print('Cannot load image:',name)
        raise SystemExit(message)
    image = image.convert() #converts the image file into a new surface object
    if colourkey is not None:
        if colourkey == -1: #if colourkey is set to -1, it will set itself to the colour at the top left of the image
            colourkey = image.get_at((0,0))
        image.set_colorkey(colourkey, RLEACCEL)
    return image, image.get_rect()

def LoadSound(name):
    '''
    LoadSound(name)
    returns the sound in a usable format for pygame
    '''
    class NoneSound:
        '''
        NoneSound
        a spoof of pygame Sound where the .play function only passes. To be used when mixer is not present.
        '''
        def play(self):
            pass
    if not pygame.mixer: #checks import of mixer
        return NoneSound()
    fullname = os.path.join('planning/pygame/imagedata',name)
    try: #tries to load the file and sends an error if the file cannot be opened in sound format
        sound = pygame.mixer.Sound(fullname)
    except pygame.error as message:
        print('Cannot load sound: ', fullname)
        raise SystemExit(mesage)
    return sound

class PlayerFist(pygame.sprite.Sprite):
    '''
    PlayerFist
    A pygame sprite that follows mouse movement on screen and follows it.
    Methods:
    Punch(target)
    '''
    def __init__(self):
        pygame.sprite.Sprite.__init__(self)
        self.image, self.rect = LoadImage('fist.bmp',-1)
        self.punching=False
    def update(self):
        '''
        update()
        moves the fist based on mouse position.
        '''
        pos = pygame.mouse.get_pos()
        self.rect.midtop=pos
        if self.punching:
            self.rect.move_ip(5,10)
    def Punch(self,target):
        '''
        Punch(target)
        returns true if the first collides with a target.
        '''
        if not self.punching:
            self.punching=True
            hitbox=self.rect.inflate(-5,-5)
            return hitbox.colliderect(target.rect)
    def Unpunch(self):
        '''
        Unpunch
        pulls the fist back to position
        '''
        self.punching=False

class Chimp(pygame.sprite.Sprite):
    """
    Chimp
    moves a pygame sprite across the screen. it can spin the monkey when it is punched.
    """
    def __init__(self):
        pygame.sprite.Sprite.__init__(self)  # call Sprite intializer
        self.image, self.rect =LoadImage('chimp.bmp', -1)
        screen = pygame.display.get_surface() #gets the size of the display canvas as a viable area.
        self.area = screen.get_rect()
        self.rect.topleft = 10, 10
        self.move = 9
        self.dizzy = False
    def update(self):
        """
        update()
        walk or spin, depending on the monkeys state.
        """
        if self.dizzy:
            self._Spin()
        else:
            self._Walk()
    def _Walk(self): #_method suggests a protected method only to be used by the chimp class.
        """
        _walk()
        move the monkey across the screen, and turn at the ends.
        """
        newpos = self.rect.move((self.move, 0))
        if not self.area.contains(newpos):
            if self.rect.left < self.area.left or \
                    self.rect.right > self.area.right:
                self.move = -self.move
                newpos = self.rect.move((self.move, 0))
                self.image = pygame.transform.flip(self.image, 1, 0)
            self.rect = newpos
    def _Spin(self):
        """
        _spin()
        spin the monkey image.
        """
        center = self.rect.center
        self.dizzy += 12
        if self.dizzy >= 360:
            self.dizzy = 0
            self.image = self.original
        else:
            rotate = pygame.transform.rotate
            self.image = rotate(self.original, self.dizzy)
        self.rect = self.image.get_rect(center=center)
    def IsPunched(self):
        """
        punched()
        this will cause the monkey to start spinning
        """
        if not self.dizzy:
            self.dizzy = 1
            self.original = self.image

def main():
    pygame.init()
    screen=pygame.display.set_mode((468,60)) #get display size.
    pygame.display.set_caption('Monkey Fever')
    pygame.mouse.set_visible(0)

    background=pygame.Surface(screen.get_size()) #creates a surface based on screen size.
    background=background.convert()
    background.fill((250,250,250))

    if pygame.font: #creates a font object as the title, renders it into the surface object.
        font=pygame.font.Font(None,36)
        text=font.render('Pummel The Chimp, And Win $$$',1,(10,10,10))
        textpos=text.get_rect(centerx=background.get_width()/2)
        background.blit(text,textpos) #blit is the final push (paste) of an object onto a surface.

    screen.blit(background, (0,0)) #creates the background on the overarching screen surface.
    pygame.display.flip() #flip updates a display that is idle.

    whiffSound=LoadSound('whiff.wav')
    punchSound=LoadSound('punch.wav')
    chimp=Chimp()
    fist=PlayerFist()
    allsprites=pygame.sprite.RenderPlain((fist,chimp)) #sprite group RenderPlain() draws all the sprites it contains into the surface.
    clock=pygame.time.Clock() #clock helps track time.

    while True:
        clock.tick(60) #setting a maximum framerate for the game.

        for event in pygame.event.get(): #handling every possible input by calling each subroutine respective to the player inputs.
            if event.type==QUIT:
                return
            elif event.type==KEYDOWN and event.key==K_ESCAPE:
                return
            elif event.type==MOUSEBUTTONDOWN:
                if fist.Punch(chimp):
                    punchSound.play()
                    chimp.IsPunched()
                else:
                    whiffSound.play()
            elif event.type==MOUSEBUTTONUP:
                fist.Unpunch()

        allsprites.update()

        screen.blit(background, (0,0))
        allsprites.draw(screen)
        pygame.display.flip()

if __name__=='__main__':
    main()


import csv
file1 = open('login.csv')
csvreader = csv.reader(file1)
header = []
header = next(csvreader)
rows=[]
for row in csvreader:
    rows.append(row)
import  os
def Advisior():
  import csv
  import os
  book = open('bookings.csv')
  csvreader1 = csv.reader(book)
  header = []
  header = next(csvreader1)
  booking=[]
  for row in csvreader1:
          booking.append(row)
  av = open('aval.csv')
  csvreader2 = csv.reader(av)
  header = []
  header = next(csvreader2)
  ava=[]
  for row in csvreader2:
          ava.append(row)
  print("<------Login successfull as Advisior--------->")
  print("1.check boookings  :: ")
  print("2.Booking approvel  :: ")
  print("3.Check avaliablety :: ")
  a=int(input())
  if a==1:
    print("Location      TYPE       Date        UserID")
    for i in range(len(booking)):
      print(booking[i][0],"    ",booking[i][1],"   ",booking[i][2],"    ",booking[i][3])
    Advisior()
  elif a==2:
    print("SNo    ","Location      TYPE       Date        UserID")
    for i in range(len(booking)):
      print(i+1,"    ", booking[i][0],"    ",booking[i][1],"   ",booking[i][2],"    ",booking[i][3])
    a=(int(input("select booking ::")))-1
    for i in range(len(ava)):
      if booking[a][0]==ava[i][0] and ava[i][2]==str(0) and ava[i][3]==booking[a][1].lower():
        print("Booking successfull")
        ava[i][2]=1

        Advisior()
  elif a==3:
    print("Location     cab number     condition")
    for i in range(len(ava)):
      print(ava[i][0],"     ",ava[i][1],"     ",ava[i][2])
    Advisior()
  else:
    print("Wrong input")
  file="aval.csv"
  if(os.path.exists(file) and os.path.isfile(file)):
    os.remove(file)
  fields=["Location","cab number","condition","type"]
  with open("aval.csv", 'w') as csvfile:
    csvwriter = csv.writer(csvfile)
    csvwriter.writerow(fields)
    csvwriter.writerows(ava)
  file="bookings.csv"
  if(os.path.exists(file) and os.path.isfile(file)):
    os.remove(file)
  fields=["Locations","Type","Date","UserID","Status"]
  with open("bookings.csv", 'w') as csvfile:
    csvwriter = csv.writer(csvfile)
    csvwriter.writerow(fields)
    csvwriter.writerows(ava)
def admin():
  import csv,os
  av = open('aval.csv')
  csvreader2 = csv.reader(av)
  header = []
  header = next(csvreader2)
  ava=[]
  for row in csvreader2:
      ava.append(row)
  print("<----------------login successfull as a admin------------>")
  print("1. Check summary of the van::")
  print("2. add new van::")
  a=int(input())
  if a==1:
    print("Location     cab number     condition    type")
    for i in range(len(ava)):
      print(ava[i][0],"     ",ava[i][1],"     ",ava[i][2],"     ",ava[i][3])
    admin()
  elif a==2:
    l=input("Enter the locations ::")
    n=input("Enter the cab number::")
    c=input("enter the condition(in 1 or 0)::")
    t=input("Enter the type ::")
    k=[]
    k.append(l)
    k.append(n)
    k.append(c)
    k.append(t)
    ava.append(k)
    file="aval.csv"
    if(os.path.exists(file) and os.path.isfile(file)):
      os.remove(file)
    fields=["Location","cab number","condition","type"]
    with open("aval.csv", 'w') as csvfile:
      csvwriter = csv.writer(csvfile)
      csvwriter.writerow(fields)
      csvwriter.writerows(ava)
    admin()
  else:
    print("Wrog input logout successfully")
def customer(e):
  print("<----Login successfull as a Customer--->")
  print("select your choice::")
  print("1.Request to advisior to Book the van")
  print("2.See the summary")
  print("for logout press any key::")
  a=int(input())
  k=[]
  r=[]
  l=0
  t=0
  ti=0
  vn=0
  s="pending"
  if a==1:
    l=str(input("ENter the location:"))
    t=str(input("Enter the van size small/medium/large::"))
    ti=str(input("enter tha date::"))

    r.append([l,t,ti,e,s])
    print("Request added successfuly",r)
    customer(e)
  elif a==2:
    print("location name:",l)
    print("van number",vn)
    print("van type:",t)
    print("data:",ti)
    print("Status:",s)
    customer(e)
  else:
    print("Thanks for services")
  with open("bookings.csv", 'w') as csvfile:
    csvwriter = csv.writer(csvfile)
    csvwriter.writerows(r)

def main():
  filename = "bookings.csv"
  fields=["Locations","Type","Date","UserID","Status"]
  with open("bookings.csv", 'w') as csvfile:
    # creating a csv writer object
    csvwriter = csv.writer(csvfile)

    # writing the fields
    csvwriter.writerow(fields)
  print("<----------Please login --------->")
  u=int(input("Enter the type of user 1 (customer) 2(Advisior) 3 (Admin)::-"))
  if u==1 or u==2 or u==3:
    pass
  else:
    print("wrong credintial login again")
    main()
  e=input("Enter the UserID::-")
  p=input("password::-")
  for i in range(len(rows)):
    if rows[i][0]==e:

      a=i
      print(a)
      if rows[a][0]==e and rows[a][1]==p and rows[a][2]==str(u):
        if rows[a][2]=="1":
          customer(e)
        elif rows[a][2]=="2":
          Advisior()
        elif rows[a][2]=="3":
          admin()
      else:
        print("wrong credintial login again")
        main()
    else:
      pass
main()


from typing import List
from collections import defaultdict, deque

class Solution:
    def maxChunksToSorted(self, arr: List[int]) -> int:

        A = sorted(arr)
        hm = defaultdict(deque)
        for i, e in enumerate(A):
            hm[e].append(i)

        proxy = []
        for e in arr:
            proxy.append(hm[e].popleft())

        ret = 0
        cur_max_idx = 0
        for i, e in enumerate(proxy):
            cur_max_idx = max(cur_max_idx, e)
            if cur_max_idx == i:
                ret += 1

        return ret

data_files/768 Max Chunks To Make Sorted II.py


import numpy as np;
import solveCatenary;
import matplotlib.pyplot as plt;

# plotting function
def drawCatenary(fighandle,idx,xsol,z0,z1,z2,r0,addcircle,color):
    xsolaug=np.array([np.concatenate((z0.flatten(),xsol[idx,:],z1.flatten()))]).T;

    xout=xsolaug[0::2];
    yout=xsolaug[1::2];

    plt.figure(fighandle)
    if addcircle==True:
        varphi=np.linspace(0,np.pi*2.0,500);
        xcircle=z2[0]+np.cos(varphi)*r0;
        ycircle=z2[1]+np.sin(varphi)*r0;
        plt.plot(xcircle,ycircle,'k')

    colorstring=color+'.-';
    plt.plot(xout,yout,colorstring)
    plt.axis('equal')

plt.close('all')

N=20; # N is number of links, N+1 number of joints including z0,z1;
      # N needs to be even

z0=np.array([[0,0]]).T;         # left fixation
z1=np.array([[1,0]]).T;         # right fixation
z2=np.array([[0.5,-.8]]).T;     # center of ball
r0=0.5;                         # radius of ball

np.random.seed(27111989);       # set seed
x0=np.random.normal(size=((N-1)*2,1))*.05;          # random initial y values
x0[0:-1:2]=np.array([np.arange(1./N,1.,1./N)]).T;   # equidistant x values

# parameters for solver
TOL_KKT=1e-6;
TOL_const=1e-6;
MAXITER=25000;
MAXITER_PROX=1000;
ABSTOL_PROX=1e-12;
RELTOL_PROX=0;
omega=1;
T=2.0/(9.81*N); # cg~9.81*N
alpha=0.8/T;

# compute local solutions to the NLP
xsol,fsol,lambdasol,niter,prox_iter,g_ineq_tot,g_eq_tot,n_act_constr=solveCatenary.solveCatenary(N,x0,T,alpha,\
                                                                                TOL_const,omega,TOL_KKT,\
                                                                                MAXITER,MAXITER_PROX,ABSTOL_PROX,RELTOL_PROX,z0,z1,z2,r0);

print("optimization terminated in "+str(niter)+" iterations.")

err_norm_g_ineq =np.zeros(shape=(np.shape(g_ineq_tot)[0],1))
for j in range(np.shape(g_ineq_tot)[0]):
    err_norm_g_ineq[j]=np.sqrt(np.sum(g_ineq_tot[j,g_ineq_tot[j,:]<=0]**2));

err_norm_g_eq = np.sqrt(np.sum(g_eq_tot**2,axis=1));

drawCatenary(1,0,xsol,z0,z1,z2,r0,False,'k')
drawCatenary(1,100,xsol,z0,z1,z2,r0,False,'b')
drawCatenary(1,1000,xsol,z0,z1,z2,r0,False,'g')
drawCatenary(1,-1,xsol,z0,z1,z2,r0,True,'r')
plt.xlabel('$x$')
plt.ylabel('$y$')
plt.legend(['initial','k=100','k=1000','constraint','final'])

plt.figure()
plt.subplot(211)
plt.loglog(err_norm_g_eq)
plt.xlabel('iterations')
plt.ylabel('$|h(x_k)|$')

plt.subplot(212)
plt.plot(fsol,'k')
plt.xlabel('iterations')
plt.ylabel('cost')

#tikzplotlib.save("catenary_2.tikz", axis_width='\\figurewidth', axis_height ='\\figureheight')


#!/usr/bin/env python
import os
import sys

if sys.version_info < (3, 8):
    print("Error: dbt does not support this version of Python.")
    print("Please upgrade to Python 3.8 or higher.")
    sys.exit(1)

from setuptools import setup

try:
    from setuptools import find_namespace_packages
except ImportError:
    # the user has a downlevel version of setuptools.
    print("Error: dbt requires setuptools v40.1.0 or higher.")
    print('Please upgrade setuptools with "pip install --upgrade setuptools" ' "and try again")
    sys.exit(1)

package_name = "dbt-tests-adapter"
package_version = "1.8.0a1"
description = """The dbt adapter tests for adapter plugins"""

this_directory = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(this_directory, "README.md")) as f:
    long_description = f.read()

setup(
    name=package_name,
    version=package_version,
    description=description,
    long_description=long_description,
    long_description_content_type="text/markdown",
    author="dbt Labs",
    author_email="info@dbtlabs.com",
    url="https://github.com/dbt-labs/dbt-core/tree/main/tests/adapter",
    packages=find_namespace_packages(include=["dbt", "dbt.*"]),
    install_requires=[
        "dbt-core=={}".format(package_version),
        "pytest>=7.0.0",
    ],
    zip_safe=False,
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: Microsoft :: Windows",
        "Operating System :: MacOS :: MacOS X",
        "Operating System :: POSIX :: Linux",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Programming Language :: Python :: 3.10",
        "Programming Language :: Python :: 3.11",
    ],
    python_requires=">=3.8",
)


import numpy as np
import cv2
from PIL import Image, ImageDraw, ImageFont
import PIL
import matplotlib.pyplot as plt
import os
import shutil
from numpy.core.records import array
from numpy.core.shape_base import block
import time

from rows import img2rows

def img_y_shadow(img_b):
    (h, w) = img_b.shape

    a=[0 for z in range(0,h)]

    for i in range(0,h):
        for j in range(0,w):
            if img_b[i,j] == 255:
                a[i]+=1

    return a

def img_show_array(a):
    plt.imshow(a)
    plt.show()

def show_shadow(arr, direction= 'x'):

    a_max = max(arr)
    if direction == 'x':
        a_shadow = np.zeros((a_max, len(arr)), dtype=int)
        for i in range(0, len(arr)):
            if arr[i]==0:
                continue

            for j in range(0, arr[i]):
                a_shadow[j][i] = 255
    elif direction == 'y':
        a_shadow = np.zeros((len(arr), a_max), dtype=int)
        for i in range(0, len(arr)):
            if arr[i]==0:
                continue

            for j in range(0, arr[i]):
                a_shadow[i][j] = 255

    img_show_array(a_shadow)

img_question = 'question.png'
img = cv2.imread(img_question, 0)
thresh = 200

ret, img_b = cv2.threshold(img, thresh, 255, cv2.THRESH_BINARY_INV)

# plt.imshow(img_b)
# plt.show()

img_y_shadow_a = img_y_shadow(img_b)
show_shadow(img_y_shadow_a, 'y')

(img_h, img_w) = img.shape
row_mark_boxs = img2rows(img_y_shadow_a, img_w, img_h)
print(row_mark_boxs)

def cut_img(img, mark_boxs):
    img_items = []

    for i in range(0, len(mark_boxs)):
        img_org = img.copy()
        box = mark_boxs[i]

        img_item = img_org[box[1]:box[3], box[0]:box[2]]
        img_items.append(img_item)

    return img_items

def save_imgs(dir_name, imgs):
    if os.path.exists(dir_name):
        shutil.rmtree(dir_name)

    if not os.path.exists(dir_name):
        os.makedirs(dir_name)

    img_paths = []

    for i in range(0, len(imgs)):
        file_path = dir_name + '/part_'+ str(i)+ '.jpg'
        cv2.imwrite(file_path, imgs[i])
        img_paths.append(file_path)

    return img_paths

row_imgs = cut_img(img, row_mark_boxs)
imgs = save_imgs('rows', row_imgs)
print(imgs)

kernel = np.ones((3, 3), np.uint80)
row_img_b = cv2.dilate(img_b, kernel, iterations=6)