"""A module with functions to aid in data analysis using the PUDL database.

    TODO: Delete old useless cruft.
"""

import logging

# Useful high-level external modules.
import numpy as np
import pandas as pd
import sqlalchemy as sa

# Our own code...
import pudl
import pudl.constants as pc
import pudl.helpers

logger = logging.getLogger(__name__)

def simple_select(table_name, pudl_engine):
    """
    Generate a DataFrame of the specified table.

    Generates a DataFrame of the specified table, including EIA plant IDs where
    they are available.

    Args:
        table_name (str): A PUDL table name
        pudl_engine (str): A PUDL database connection

    Returns:
        table: A DataFrame of the specified table.
    """
    # Pull in the table
    tbl = pudl.output.pudltabl.get_table_meta(pudl_engine)[table_name]
    # Creates a sql Select object
    select = sa.sql.select([tbl, ])
    # Converts sql object to pandas dataframe

    table = pd.read_sql(select, pudl_engine)

    # If table includes plant_id_eia, get the PUDL Plant ID

    if 'plant_id_eia' in table.columns:
        # Shorthand for readability... pt = PUDL Tables
        pt = pudl.output.pudltabl.get_table_meta(pudl_engine)

        # Pull in plants_eia which connects EIA & PUDL plant IDs
        plants_eia_tbl = pt['plants_eia']
        plants_eia_select = sa.sql.select([
            plants_eia_tbl.c.plant_id_eia,
            plants_eia_tbl.c.plant_id_pudl,
        ])
        plants_eia = pd.read_sql(plants_eia_select, pudl_engine)
        out_df = pd.merge(table, plants_eia, how='left', on='plant_id_eia')
        out_df.plant_id_pudl = out_df.plant_id_pudl.astype(int)
        table = out_df
    else:
        table = table

    return table

def simple_ferc1_plant_ids(pudl_engine):
    """Generates a list of all PUDL plant IDs which map to a single FERC plant.

    Args:
        pudl_engine (str): A PUDL database connection.

    Returns:
        ferc1_simple_plant_ids: A DataFrame containing all the PUDL plant IDs
        that map to a single FERC plant.
    """
    ferc1_plant_ids = pd.read_sql('''SELECT plant_id_pudl FROM plants_ferc''',
                                  pudl_engine)
    ferc1_simple_plant_ids = ferc1_plant_ids.drop_duplicates('plant_id_pudl',
                                                             keep=False)
    return ferc1_simple_plant_ids

def simple_eia_plant_ids(pudl_engine):
    """Generates a list of all PUDL plant IDs that map to a single EIA plant.

    Args:
        pudl_engine (str): A PUDL database connection.

    Returns:
        eia_simple_plant_ids: A DataFrame containing all the PUDL plant IDs that
        map to a single EIA plant.
    """
    eia_plant_ids = pd.read_sql('''SELECT plant_id_pudl FROM plants_eia''',
                                pudl_engine)
    eia_simple_plant_ids = eia_plant_ids.drop_duplicates('plant_id_pudl',
                                                         keep=False)
    return eia_simple_plant_ids

def simple_pudl_plant_ids(pudl_engine):
    """Retrieves all PUDL plant IDs that map to single EIA and single FERC
    plant ID.

    Args:
        pudl_engine (str): A PUDL database connection.

    Returns:
        pudl_simple: A DataFrame containing all the PUDL plant IDs that map to a
        single EIA and single FERC plant ID.
    """
    ferc1_simple = simple_ferc1_plant_ids(pudl_engine)
    eia_simple = simple_eia_plant_ids(pudl_engine)
    pudl_simple = np.intersect1d(ferc1_simple['plant_id_pudl'],
                                 eia_simple['plant_id_pudl'])
    return pudl_simple

def ferc_eia_shared_plant_ids(pudl_engine):
    """Generates a list of PUDL plant IDs that appear in both FERC and EIA.

    Args:
        pudl_engine (str): A PUDL database connection.

    Returns:
        shared_plant_ids: A DataFrame containing all of the PUDL plant IDs that
        appear in both FERC and EIA.
    """
    ferc_plant_ids = pd.read_sql('''SELECT plant_id_pudl FROM plants_ferc''',
                                 pudl_engine)
    eia_plant_ids = pd.read_sql('''SELECT plant_id_pudl FROM plants_eia''',
                                pudl_engine)
    shared_plant_ids = np.intersect1d(ferc_plant_ids['plant_id_pudl'],
                                      eia_plant_ids['plant_id_pudl'])
    return shared_plant_ids

def ferc_pudl_plant_ids(pudl_engine):
    """Generates a list of PUDL plant IDs that correspond to FERC plants.

    Args:
        pudl_engine (str): A PUDL database connection.

    Returns:
        ferc_plant_ids: A DataFrame containing all of the PUDL plant IDs that
        correspond to FERC plants.
    """
    ferc_plant_ids = pd.read_sql('''SELECT plant_id_pudl FROM plants_ferc''',
                                 pudl_engine)
    return ferc_plant_ids

def eia_pudl_plant_ids(pudl_engine):
    """Generates a list of PUDL plant IDs that correspond to EIA plants.

    Args:
        pudl_engine (str): A PUDL database connection.

    Returns:
        eia_plant_ids: A DataFrame containing all the PUDL plant IDs that
        correspond to EIA plants.
    """
    eia_plant_ids = pd.read_sql('''SELECT plant_id_pudl FROM plants_eia''',
                                pudl_engine)
    return eia_plant_ids

def yearly_sum_eia(df, sum_by, columns=('plant_id_eia', 'generator_id')):
    """Groups an input dataframe by several columns, and calculates annual sums.

    TODO: Zane revisit

    The dataframe to group and sum is 'table'. The column to sum on an annual
    basis is 'sum_by' and 'columns' is the set of fields to group the dataframe
    by before summing.

    The dataframe can start with either a report_year or report_date field. If
    it's got a report_date, that's converted into an integer year field named
    report_year.

    Comments from Zane:
     - If we implement consistent report_year and report_date naming
       convention in our database tables, then I think we could eliminate
       the need to pass in a date/year column? If there's a report_date
       then we'd turn it into a report_year, and if there's a report_year, then
       it's ready to go.
     - Might want to do some assert() checking to make sure we have a valid
       date or year field in the dataframe that's passed in.
     - Does this need to be an EIA specific function? If we're using the same
       report_year and report_date convention in other data sources could we
       make it work for them as well?
     - Why did we end up converting things to integer years rather than using
       the native time-based grouping functions?
    """
    df['report_year'] = pd.to_datetime(df['report_date']).dt.year
    gb = df.groupby(by=columns)
    return gb.agg({sum_by: np.sum})

def consolidate_ferc1_expns(steam_df, min_capfac=0.6, min_corr=0.5):
    """
    Calculates non-fuel production and nonproduction costs from a steam
    DataFrame.

    Takes a DataFrame containing information from the plants_steam_ferc1 table
    and add columns representing the non-production costs, and non-fuel
    production costs, which are sums of other expense columns. Which columns
    are treated as production vs. non-production costs is determined based on
    the overall correlation between those column values and net_generation_mwh
    for the entire steam_df DataFrame.

    Args:
        steam_df (DataFrame): Data selected from the PUDL plants_steam_ferc1
            table, containing expense columns, prefixed with expns_
        min_capfac (float): Minimum capacity factor required for a plant's
            data to be used in determining which expense columns are
            production vs. non-production costs.
        min_corr (float): Minimum correlation with net_generation_mwh required
            to indicate that a given expense field should be considered a
            "production" cost.

    Returns:
        steam_df: A DataFrame containing all the same information as the
        original steam_df, but with two additional columns consolidating the
        non-fuel production and non-production costs for ease of calculation.

    """
    steam_df = steam_df.copy()
    # Calculate correlation of expenses to net power generation. Require a
    # minimum plant capacity factor of 0.6 so we the signal will be high,
    # but we'll still have lots of plants to look at:
    expns_corr = ferc1_expns_corr(steam_df, min_capfac=min_capfac)

    # We've already got fuel separately, and we know it's a production expense
    expns_corr.pop('expns_fuel')
    # Sort these expense fields into nonfuel production (nonfuel_px) or
    # non-production (npx) expenses.
    nonfuel_px = [k for k in expns_corr if expns_corr[k] >= min_corr]
    npx = [k for k in expns_corr if expns_corr[k] < min_corr]

    # The three main categories of expenses we're reporting:
    # - fuel production expenses (already in the table)
    # - non-fuel production expenses
    steam_df['expns_total_nonfuel_production'] = \
        steam_df[nonfuel_px].copy().sum(axis=1)
    # - non-production expenses
    steam_df['expns_total_nonproduction'] = steam_df[npx].copy().sum(axis=1)

    return steam_df

def ferc1_expns_corr(steam_df, min_capfac=0.6):
    """Calculates generation vs. expense correlation for FERC Form 1 plants.

    This function helped us identify which of the expns_* fields in the FERC
    Form 1 dataset represent production costs, and which are non-production
    costs, for the purposes of modeling marginal cost of electricity from
    various plants.  We expect the difference in expenses vs. generation to
    be more indicative of production vs. non-production costs for plants with
    higher capacity factors, and since what we're trying to do here is
    identify which *fields* in the FERC Form 1 data are production costs, we
    allow a capacity_factor threshold to be set -- analysis is only done for
    those plants with capacity factors larger than the threshold.

    Additionally, some types of plants simply do not have some types of
    expenses, so to keep those plants from dragging down otherwise meaningful
    correlations, any zero expense values are dropped before calculating the
    correlations.

    Args:
        steam_df (DataFrame): Data selected from the PUDL plants_steam_ferc1
            table, containing expense columns, prefixed with expns_
        min_capfac (float): Minimum capacity factor required for a plant's
            data to be used in determining generation vs expense correlations.

    Returns:
        expns_corr: A dictionary with expns_ field names as the keys, and
        correlations as the values.
    """
    steam_df = steam_df.copy()
    steam_df['capacity_factor'] = \
        (steam_df['net_generation_mwh'] / 8760 * steam_df['total_capacity_mw'])

    # Limit plants by capacity factor
    steam_df = steam_df[steam_df['capacity_factor'] > min_capfac]

    # This is all the expns_* fields, except for the per_mwh and total.
    cols_to_correlate = ['expns_operations',
                         'expns_fuel',
                         'expns_coolants',
                         'expns_steam',
                         'expns_steam_other',
                         'expns_transfer',
                         'expns_electric',
                         'expns_misc_power',
                         'expns_rents',
                         'expns_allowances',
                         'expns_engineering',
                         'expns_structures',
                         'expns_boiler',
                         'expns_plants',
                         'expns_misc_steam']

    expns_corr = {}
    for expns in cols_to_correlate:
        mwh_plants = steam_df.net_generation_mwh[steam_df[expns] != 0]
        expns_plants = steam_df[expns][steam_df[expns] != 0]
        expns_corr[expns] = np.corrcoef(mwh_plants, expns_plants)[0, 1]

    return expns_corr

def ferc_expenses(pudl_engine, pudl_plant_ids=(), require_eia=True,
                  min_capfac=0.6, min_corr=0.5):
    """
    Gathers operating expense data for a selection of FERC plants by PUDL ID.

    Args:
        pudl_engine: a connection to the PUDL database.
        pudl_plant_ids: list of PUDL plant IDs for which to pull expenses out
            of the FERC dataset. If it's an empty list, get all the plants.
        require_eia: Boolean (True/False). If True, then only return FERC
            plants which also appear in the EIA dataset.  Useful for when you
            want to merge the FERC expenses with other EIA data.
        min_capfac: the minimum plant capacity factor to use in
            determining whether an expense category is a production or
            non-production cost.
        min_corr: The threhold correlation to use in determining whether an
            expense is a production or non-production expense. If an expense
            has a correlation to net generation that is greater than or equal
            to this threshold, it is categorized as a production expense.

    Returns:
        ferc1_expns_corr: A dictionary of expense categories
            and their correlations to the plant's net electricity
            generation.
        steam_df: a DataFrame with all the operating expenses
            broken out for each simple FERC PUDL plant.
    """
    # All of the large steam plants from FERC:
    steam_df = pudl.output.ferc1.plants_steam_ferc1(pudl_engine)

    # Calculate the dataset-wide expense correlations, for the record.
    expns_corrs = ferc1_expns_corr(steam_df, min_capfac=min_capfac)
    # Lump the operating expenses based on those correlations. Note that we
    # could also do this lumping after limiting the set of plants that we're
    # reporting on.  However, doing it based on the entire dataset seems more
    # appropriate, given that these correlations are properties of the fields,
    # not the plants... or so we hope.
    steam_df = consolidate_ferc1_expns(steam_df,
                                       min_capfac=min_capfac,
                                       min_corr=min_corr)

    # If we are only looking at a specified subset of the FERC plants, then
    # here is where we limit the information that's returned:
    if len(pudl_plant_ids) > 0:
        steam_df = steam_df[steam_df.plant_id_pudl.isin(pudl_plant_ids)]

    if require_eia:
        # All of the EIA PUDL plant IDs
        eia_pudl = eia_pudl_plant_ids(pudl_engine)
        steam_df = steam_df[
            steam_df.plant_id_pudl.isin(eia_pudl.plant_id_pudl)]

    # Pass back both the expense correlations, and the plant data.
    return expns_corrs, steam_df

def fuel_ferc1_by_pudl(pudl_plant_ids, pudl_engine,
                       fuels=('gas', 'oil', 'coal'),
                       cols=('fuel_consumed_total_mmbtu',
                             'fuel_consumed_total_cost_mmbtu',
                             'fuel_consumed_total_cost_unit')):
    """
    Aggregate FERC Form 1 fuel data by PUDL plant id and, optionally, fuel.

    Args:
        pudl_plant_ids: which PUDL plants should we retain for aggregation?
        fuels: Should the columns listed in cols be broken out by each
            individual fuel? If so, which fuels do we want totals for? If
            you want all fuels lumped together, pass in 'all'.
        cols: which columns from the fuel_ferc1 table should be summed.

    Returns:
        fuel_df: a dataframe with pudl_plant_id, year, and the summed values
            specified in cols. If fuels is not 'all' then it also has a column
            specifying fuel type.

    """
    fuel_df = pudl.output.ferc1.fuel_ferc1_df(pudl_engine)

    # Calculate the total fuel heat content for the plant by fuel
    fuel_df = fuel_df[fuel_df.plant_id_pudl.isin(pudl_plant_ids)]

    if fuels == 'all':
        cols_to_gb = ['plant_id_pudl', 'report_year']
    else:
        # Limit to records that pertain to our fuels of interest.
        fuel_df = fuel_df[fuel_df['fuel'].isin(fuels)]
        # Group by fuel as well, so we get individual fuel totals.
        cols_to_gb = ['plant_id_pudl', 'report_year', 'fuel']

    fuel_df = fuel_df.groupby(cols_to_gb)[cols].sum()
    fuel_df = fuel_df.reset_index()

    return fuel_df

def steam_ferc1_by_pudl(pudl_plant_ids, pudl_engine,
                        cols=('net_generation_mwh', )):
    """
    Aggregate and return data from the steam_ferc1 table by pudl_plant_id.

    Args:
        pudl_plant_ids: A list of ids to include in the output.
        cols: The data columns that you want to aggregate and return.

    Returns:
        steam_df: A dataframe with columns for report_year, pudl_plant_id and
            cols, with the values in cols aggregated by plant and year.

    """
    steam_df = pudl.output.ferc1.plants_steam_ferc1_df(pudl_engine)
    steam_df = steam_df[steam_df.plant_id_pudl.isin(pudl_plant_ids)]
    steam_df = steam_df.groupby(['plant_id_pudl', 'report_year'])[cols].sum()
    steam_df = steam_df.reset_index()

    return steam_df

def frc_by_pudl(pudl_plant_ids, pudl_engine,
                fuels=('gas', 'oil', 'coal'),
                cols=('total_fuel_cost', )):
    """
    Aggregate fuel_receipts_costs_eia923 table for comparison with FERC Form 1.

    In order to correlate information between EIA 923 and FERC Form 1, we need
    to aggregate the EIA data annually, and potentially by fuel. This function
    groups fuel_receipts_costs_eia923 by pudl_plant_id, fuel, and year, and
    sums the columns of interest specified in cols, and returns a dataframe
    with the totals by pudl_plant_id, fuel, and year.

    Args:
        pudl_plant_ids: list of plant IDs to keep.
        fuels: list of fuel strings that we want to group by. Alternatively,
            this can be set to 'all' in which case fuel is not grouped by.
        cols: List of data columns which we are summing.

    Returns:
        A dataframe with the sums of cols, as grouped by pudl ID, year, and
            (optionally) fuel.

    """
    # Get all the EIA info from generation_fuel_eia923
    frc_df = pudl.output.eia923.fuel_receipts_costs_eia923(pudl_engine)
    # Limit just to the plants we're looking at
    frc_df = frc_df[frc_df.plant_id_pudl.isin(pudl_plant_ids)]
    # Just keep the columns we need for output:
    cols_to_keep = ['plant_id_pudl', 'report_date']
    cols_to_keep = cols_to_keep + cols
    cols_to_gb = [pd.Grouper(freq='A'), 'plant_id_pudl']

    if fuels != 'all':
        frc_df = frc_df[frc_df.fuel.isin(fuels)]
        cols_to_keep = cols_to_keep + ['fuel', ]
        cols_to_gb = cols_to_gb + ['fuel', ]

    # Pare down the dataframe to make it easier to play with:
    frc_df = frc_df[cols_to_keep]

    # Prepare to group annually
    frc_df['report_date'] = pd.to_datetime(frc_df['report_date'])
    frc_df.index = frc_df.report_date
    frc_df.drop('report_date', axis=1, inplace=True)

    # Group and sum of the columns of interest:
    frc_gb = frc_df.groupby(by=cols_to_gb)
    frc_totals_df = frc_gb[cols].sum()

    # Simplify and clean the DF for return:
    frc_totals_df = frc_totals_df.reset_index()
    frc_totals_df['report_year'] = frc_totals_df.report_date.dt.year
    frc_totals_df = frc_totals_df.drop('report_date', axis=1)
    frc_totals_df = frc_totals_df.dropna()

    return frc_totals_df

def gen_fuel_by_pudl(pudl_plant_ids, pudl_engine,
                     fuels=('gas', 'oil', 'coal'),
                     cols=('fuel_consumed_mmbtu',
                           'net_generation_mwh')):
    """
    Aggregate generation_fuel_eia923 table for comparison with FERC Form 1.

    In order to correlate informataion between EIA 923 and FERC Form 1, we need
    to aggregate the EIA data annually, and potentially by fuel. This function
    groups generation_fuel_eia923 by pudl_plant_id, fuel, and year, and sums
    the columns of interest specified in cols, and returns a dataframe with
    the totals by pudl_plant_id, fuel, and year.

    Args:
        pudl_plant_ids: list of plant IDs to keep.
        fuels: list of fuel strings that we want to group by. Alternatively,
            this can be set to 'all' in which case fuel is not grouped by.
        cols: List of data columns which we are summing.
    Returns:
        A dataframe with the sums of cols, as grouped by pudl ID, year, and
            (optionally) fuel.
    """
    # Get all the EIA info from generation_fuel_eia923
    gf_df = pudl.output.eia923.generation_fuel_eia923(pudl_engine)

    # Standardize the fuel codes (need to fix this in the DB!!!!)
    gf_df = gf_df.rename(columns={'fuel_type_code_pudl': 'fuel'})
    # gf_df['fuel'] = gf_df.fuel.replace(to_replace='petroleum', value='oil')

    # Select only the records that pertain to our target IDs
    gf_df = gf_df[gf_df.plant_id_pudl.isin(pudl_plant_ids)]

    cols_to_keep = ['plant_id_pudl', 'report_date']
    cols_to_keep = cols_to_keep + cols
    cols_to_gb = [pd.Grouper(freq='A'), 'plant_id_pudl']

    if fuels != 'all':
        gf_df = gf_df[gf_df.fuel.isin(fuels)]
        cols_to_keep = cols_to_keep + ['fuel', ]
        cols_to_gb = cols_to_gb + ['fuel', ]

    # Pare down the dataframe to make it easier to play with:
    gf_df = gf_df[cols_to_keep]

    # Prepare to group annually
    gf_df['report_date'] = pd.to_datetime(gf_df['report_date'])
    gf_df.index = gf_df.report_date
    gf_df.drop('report_date', axis=1, inplace=True)

    gf_gb = gf_df.groupby(by=cols_to_gb)
    gf_totals_df = gf_gb[cols].sum()
    gf_totals_df = gf_totals_df.reset_index()

    # Simplify date info for easy comparison with FERC.
    gf_totals_df['report_year'] = gf_totals_df.report_date.dt.year
    gf_totals_df = gf_totals_df.drop('report_date', axis=1)
    gf_totals_df = gf_totals_df.dropna()

    return gf_totals_df

def generator_proportion_eia923(g, id_col='plant_id_eia'):
    """
    Generate a dataframe with the proportion of generation for each generator.

    Args:
        g: a dataframe from either all of generation_eia923 or some subset of
        records from generation_eia923. The dataframe needs the following
        columns to be present:
            plant_id_eia, generator_id, report_date, net_generation_mwh

    Returns: a dataframe with:
            report_year, plant_id_eia, generator_id, proportion_of_generation
    """
    # Set the datetimeindex
    g = g.set_index(pd.DatetimeIndex(g['report_year']))
    # groupby plant_id_eia and by year
    g_yr = g.groupby([pd.Grouper(freq='A'), id_col, 'generator_id'])
    # sum net_gen by year by plant
    g_net_generation_per_generator = pd.DataFrame(
        g_yr.net_generation_mwh.sum())
    g_net_generation_per_generator = \
        g_net_generation_per_generator.reset_index(level=['generator_id'])

    # groupby plant_id_eia and by year
    g_net_generation_per_plant = g.groupby(
        [pd.Grouper(freq='A'), id_col])
    # sum net_gen by year by plant and convert to datafram
    g_net_generation_per_plant = pd.DataFrame(
        g_net_generation_per_plant.net_generation_mwh.sum())

    # Merge the summed net generation by generator with the summed net
    # generation by plant
    g_gens_proportion = g_net_generation_per_generator.merge(
        g_net_generation_per_plant, how="left", left_index=True,
        right_index=True)
    g_gens_proportion['proportion_of_generation'] = (
        g_gens_proportion.net_generation_mwh_x /
        g_gens_proportion.net_generation_mwh_y)
    # Remove the net generation columns
    g_gens_proportion = g_gens_proportion.drop(
        ['net_generation_mwh_x', 'net_generation_mwh_y'], axis=1)
    g_gens_proportion.reset_index(inplace=True)

    return g_gens_proportion

def capacity_proportion_eia923(g, id_col='plant_id_eia',
                               capacity='capacity_mw'):
    """
    Generate dataframe with proportion of plant capacity for each generator.

    Args:
        g: a dataframe from either all of generation_eia923 or some subset of
        records from generation_eia923. The dataframe needs the following
        columns to be present:
            generator_id, report_date, capacity_mw

        id_col: either plant_id_eia (default) or plant_id_pudl
        capacity: capacity_mw (default), summer_capacity_mw,
            or winter_capacity_mw

    Returns: a dataframe with:
            report_year, plant_id_eia, generator_id, proportion_of_capacity
    """
    # groupby plant_id_eia and by year
    g_net_capacity_per_plant = g.groupby(['report_year', id_col])
    # sum net_gen by year by plant and convert to datafram
    g_net_capacity_per_plant = pd.DataFrame(
        g_net_capacity_per_plant.capacity_mw.sum())
    g_net_capacity_per_plant.reset_index(inplace=True)

    # Merge the summed net generation by generator with the summed net
    # generation by plant
    g_capacity_proportion = g.merge(
        g_net_capacity_per_plant, on=[id_col, 'report_year'], how="left")
    g_capacity_proportion['proportion_of_plant_capacity'] = (
        g_capacity_proportion.capacity_mw_x /
        g_capacity_proportion.capacity_mw_y)
    # Remove the net generation columns
    g_capacity_proportion = g_capacity_proportion.rename(
        columns={'capacity_mw_x': 'capacity_gen_mw',
                 'capacity_mw_y': 'capacity_plant_mw'})

    return g_capacity_proportion

def values_by_generator_eia923(table_eia923, column_name, g):
    """
    Generate a dataframe with a plant value proportioned out by generator.

    Args:
        table_eia923: an EIA923 table (this has been tested with
        fuel_receipts_costs_eia923 and generation_fuel_eia923).
        column_name: a column name from the table_eia923.
        g: a dataframe from either all of generation_eia923 or some subset of
        records from generation_eia923. The dataframe needs the following
        columns to be present:
            plant_id_eia, generator_id, report_date, and net_generation_mwh.

    Returns: a dataframe with report_date, plant_id_eia, generator_id, and the
        proportioned value from the column_name.
    """
    # Set the datetimeindex
    table_eia923 = table_eia923.set_index(
        pd.DatetimeIndex(table_eia923['report_date']))
    # groupby plant_id_eia and by year
    table_eia923_gb = table_eia923.groupby(
        [pd.Grouper(freq='A'), 'plant_id_eia'])
    # sum fuel cost by year by plant
    table_eia923_sr = table_eia923_gb[column_name].sum()
    # Convert back into a dataframe
    table_eia923_df = pd.DataFrame(table_eia923_sr)
    column_name_by_plant = "{}_plant".format(column_name)
    table_eia923_df = table_eia923_df.rename(
        columns={column_name: column_name_by_plant})
    # get the generator proportions
    g_gens_proportion = generator_proportion_eia923(g)
    # merge the per generator proportions with the summed fuel cost
    g_generator = g_gens_proportion.merge(
        table_eia923_df, how="left", right_index=True, left_index=True)
    # calculate the proportional fuel costs
    g_generator["{}_generator".format(column_name)] = (
        g_generator[column_name_by_plant] *
        g_generator.proportion_of_generation)
    # drop the unneccessary columns
    g_generator = g_generator.drop(
        ['proportion_of_generation', column_name_by_plant], axis=1)
    return g_generator

def primary_fuel_ferc1(fuel_df, fuel_thresh=0.5):
    """
    Determine the primary fuel for plants listed in the PUDL fuel_ferc1 table.

    Given a selection of records from the PUDL fuel_ferc1 table, determine
    the primary fuel type for each plant (as identified by a unique
    combination of report_year, respondent_id, and plant_name).

    Args:
        fuel_df (DataFrame): a DataFrame selected from the PUDL fuel_ferc1
            table, with columns including report_year, respondent_id,
            plant_name, fuel, fuel_qty_burned, and fuel_avg_mmbtu_per_unit.
        fuel_thresh (float): What is the minimum proportion of a plant's
            annual fuel consumption in terms of heat content, that a fuel
            must account for, in order for that fuel to be considered the
            primary fuel.

    Returns:
        plants_by_primary_fuel (DataFrame): a DataFrame containing report_year,
            respondent_id, plant_name, and primary_fuel.
    """
    plants_by_heat = plant_fuel_proportions_ferc1(fuel_df)

    # On a per plant, per year basis, identify the fuel that made the largest
    # contribution to the plant's overall heat content consumed. If that
    # proportion is greater than fuel_thresh, set the primary_fuel to be
    # that fuel.  Otherwise, leave it None.
    plants_by_heat = plants_by_heat.set_index(['report_year',
                                               'respondent_id',
                                               'plant_name'])
    plants_by_heat = plants_by_heat.drop('total_mmbtu', axis=1)
    mask = plants_by_heat >= fuel_thresh
    plants_by_heat = plants_by_heat.where(mask)
    plants_by_heat['primary_fuel'] = plants_by_heat.idxmax(axis=1)
    return plants_by_heat[['primary_fuel', ]].reset_index()

def plant_fuel_proportions_ferc1(fuel_df):
    """Calculate annual fuel proportions by plant based on FERC data."""
    fuel_df = fuel_df.copy()

    fuel_df['total_mmbtu'] = \
        fuel_df['fuel_qty_burned'] * fuel_df['fuel_avg_mmbtu_per_unit']

    heat_df = fuel_df[['report_year',
                       'respondent_id',
                       'plant_name',
                       'fuel',
                       'total_mmbtu']]

    heat_pivot = heat_df.pivot_table(
        index=['report_year', 'respondent_id', 'plant_name'],
        columns='fuel',
        values='total_mmbtu')

    heat_pivot['total'] = heat_pivot.sum(axis=1, numeric_only=True)
    mmbtu_total = heat_pivot.copy()
    mmbtu_total = pd.DataFrame(mmbtu_total['total'])

    heat_pivot = heat_pivot.fillna(value=0)
    heat_pivot = heat_pivot.divide(heat_pivot.total, axis='index')
    heat_pivot = heat_pivot.drop('total', axis=1)
    heat_pivot = heat_pivot.reset_index()

    heat_pivot = heat_pivot.merge(mmbtu_total.reset_index())
    heat_pivot.rename(columns={'total': 'total_mmbtu'},
                      inplace=True)
    del heat_pivot.columns.name

    return heat_pivot

def plant_fuel_proportions_frc_eia923(frc_df, id_col='plant_id_eia'):
    """Calculate annual fuel proportions by plant from EIA923 fuel receipts."""
    frc_df = frc_df.copy()

    # Add a column with total fuel heat content per delivery
    frc_df['total_mmbtu'] = frc_df.fuel_qty_units * frc_df.average_heat_content

    # Drop everything but report_date, plant_id_eia, fuel_group_code,
    # total_mmbtu
    frc_df = frc_df[['report_date', 'plant_id_eia',
                     'plant_id_pudl', 'fuel_group_code', 'total_mmbtu']]

    # Group by report_date(annual), plant_id_eia, fuel_group_code
    frc_gb = frc_df.groupby(
        [id_col, pd.Grouper(freq='A'), 'fuel_group_code'])

    # Add up all the MMBTU for each plant & year. At this point each record
    # in the dataframe contains only information about a single fuel.
    heat_df = frc_gb.agg(np.sum)

    # Simplfy the DF a little before we turn it into a pivot table.
    heat_df = heat_df.reset_index()
    heat_df['year'] = pd.DatetimeIndex(heat_df['report_date']).year
    heat_df = heat_df.drop('report_date', axis=1)

    # Take the individual rows organized by fuel_group_code, and turn them into
    # columns, each with the total MMBTU for that fuel, year, and plant.
    heat_pivot = heat_df.pivot_table(
        index=['year', id_col],
        columns='fuel_group_code',
        values='total_mmbtu')

    # Add a column that has the *total* heat content of all fuels:
    heat_pivot['total'] = heat_pivot.sum(axis=1, numeric_only=True)

    # Replace any NaN values we got from pivoting with zeros.
    heat_pivot = heat_pivot.fillna(value=0)

    # Divide all columns by the total heat content, giving us the proportions
    # for each fuel instead of the heat content.
    heat_pivot = heat_pivot.divide(heat_pivot.total, axis='index')

    # Drop the total column (it's nothing but 1.0 values) and clean up the
    # index and columns a bit before returning the DF.
    heat_pivot = heat_pivot.drop('total', axis=1)
    heat_pivot = heat_pivot.reset_index()
    del heat_pivot.columns.name

    return heat_pivot

def primary_fuel_frc_eia923(frc_df, id_col='plant_id_eia', fuel_thresh=0.5):
    """Determine a plant's primary fuel from EIA923 fuel receipts table."""
    frc_df = frc_df.copy()

    # Figure out the heat content proportions of each fuel received:
    frc_by_heat = plant_fuel_proportions_frc_eia923(frc_df)

    # On a per plant, per year basis, identify the fuel that made the largest
    # contribution to the plant's overall heat content consumed. If that
    # proportion is greater than fuel_thresh, set the primary_fuel to be
    # that fuel.  Otherwise, leave it None.
    frc_by_heat = frc_by_heat.set_index([id_col, 'year'])
    mask = frc_by_heat >= fuel_thresh
    frc_by_heat = frc_by_heat.where(mask)
    frc_by_heat['primary_fuel'] = frc_by_heat.idxmax(axis=1)
    return frc_by_heat[['primary_fuel', ]].reset_index()

def plant_fuel_proportions_gf_eia923(gf_df):
    """Calculate annual fuel proportions by plant from EIA923 gen fuel."""
    gf_df = gf_df.copy()

    # Drop everything but report_date, plant_id_eia, fuel_type_code_pudl,
    # total_mmbtu
    gf_df = gf_df[['report_date',
                   'plant_id_eia',
                   'fuel_type_code_pudl',
                   'fuel_consumed_mmbtu']]

    # Set report_date as a DatetimeIndex
    gf_df = gf_df.set_index(pd.DatetimeIndex(gf_df['report_date']))

    # Group by report_date(annual), plant_id_eia, fuel_type_code_pudl
    gf_gb = gf_df.groupby(
        ['plant_id_eia', pd.Grouper(freq='A'), 'fuel_type_code_pudl'])

    # Add up all the MMBTU for each plant & year. At this point each record
    # in the dataframe contains only information about a single fuel.
    heat_df = gf_gb.agg(np.sum)

    # Simplfy the DF a little before we turn it into a pivot table.
    heat_df = heat_df.reset_index()
    heat_df['year'] = pd.DatetimeIndex(heat_df['report_date']).year
    heat_df = heat_df.drop('report_date', axis=1)

    # Take the individual rows organized by fuel_type_code_pudl, and turn them
    # into columns, each with the total MMBTU for that fuel, year, and plant.
    heat_pivot = heat_df.pivot_table(
        index=['year', 'plant_id_eia'],
        columns='fuel_type_code_pudl',
        values='fuel_consumed_mmbtu')

    # Add a column that has the *total* heat content of all fuels:
    heat_pivot['total'] = heat_pivot.sum(axis=1, numeric_only=True)

    # Replace any NaN values we got from pivoting with zeros.
    heat_pivot = heat_pivot.fillna(value=0)

    # Divide all columns by the total heat content, giving us the proportions
    # for each fuel instead of the heat content.
    heat_pivot = heat_pivot.divide(heat_pivot.total, axis='index')

    # Drop the total column (it's nothing but 1.0 values) and clean up the
    # index and columns a bit before returning the DF.
    heat_pivot = heat_pivot.drop('total', axis=1)
    heat_pivot = heat_pivot.reset_index()
    del heat_pivot.columns.name

    return heat_pivot

def primary_fuel_gf_eia923(gf_df, id_col='plant_id_eia', fuel_thresh=0.5):
    """Determines a plant's primary fuel from EIA923 generation fuel table.

    Args:
        gf_df (DataFrame):
        id_col ()
    Returns:
    """
    gf_df = gf_df.copy()

    # Figure out the heat content proportions of each fuel received:
    gf_by_heat = plant_fuel_proportions_gf_eia923(gf_df)

    # On a per plant, per year basis, identify the fuel that made the largest
    # contribution to the plant's overall heat content consumed. If that
    # proportion is greater than fuel_thresh, set the primary_fuel to be
    # that fuel.  Otherwise, leave it None.
    gf_by_heat = gf_by_heat.set_index([id_col, 'report_year'])
    mask = gf_by_heat >= fuel_thresh
    gf_by_heat = gf_by_heat.where(mask)
    gf_by_heat['primary_fuel'] = gf_by_heat.idxmax(axis=1)
    return gf_by_heat[['primary_fuel', ]].reset_index()

def fercplants(pudl_engine,
               plant_tables=('f1_steam',
                             'f1_gnrt_plant',
                             'f1_hydro',
                             'f1_pumped_storage'),
               years=pc.working_years['ferc1'],
               new=True,
               min_capacity=5.0):
    """Generate a list of FERC plants for matching with EIA plants.

    There are several kinds of FERC plants, with different information stored
    in different FERC database tables. FERC doesn't provide any kind of
    plant_id like EIA, so the unique identifier that we're using is a
    combination of the respondent_id (the utility) and plant_name.

    For each table in the FERC DB that contains per-plant information, we'll
    grab the respondent_id and plant_name, and join that with respondent_name
    so that the utility is more readily identifiable.  We'll also add a column
    indicating what table the plant came from, and return a DataFrame with
    those four columns in it, for use in the matching. That matching currently
    happens in an Excel spreadsheet, so you will likely want to output the
    resulting DataFrame as a CSV or XLSX file.

    The function can generate an exhaustive list of plants, or it can only grab
    plants from a particular range of years. It can also optionally grab only
    new plants i.e. those which do not appear in the existing PUDL database.
    This is useful for finding new plants when a new year of FERC data comes
    out.

    Args:
        f1_tables (list): A list of tables in the FERC Form 1 DB whose plants
            you want to get information about. Can include any of: f1_steam,
            f1_gnrt_plant, f1_hydro, and f1_pumped_storage.
        years (list): The set of years for which you wish to obtain plant by
            plant information.
        new (boolean): If True (the default) then return only those plants
            which appear in the years of FERC data being specified by years,
            and NOT also in the currently initialized PUDL DB.
        min_capacity (float): The smallest size plant, in MW, that should be
            included in the output. This avoids most of the plants being tiny.

    Returns:
        :mod:`pandas.DataFrame`: A DataFrame with four columns: respondent_id,
            respondent_name, plant_name, and plant_table.

    """
    # Need to be able to use years outside the "valid" range if we're trying
    # to get new plant ID info...
    if not new:
        for yr in years:
            assert yr in pc.working_years['ferc1']

    okay_tbls = ['f1_steam',
                 'f1_gnrt_plant',
                 'f1_hydro',
                 'f1_pumped_storage']

    # Function only knows how to work with these tables.
    for tbl in plant_tables:
        assert tbl in okay_tbls

    f1_engine = pudl.extract.ferc1.connect_db()

    # Need to make sure we have a populated metadata object, which isn't
    # always the case, since folks often are not initializing the FERC DB.
    pudl.extract.ferc1.define_db(max(pc.working_years['ferc1']),
                                 pc.ferc1_working_tables,
                                 pudl.extract.ferc1.ferc1_meta)
    f1_tbls = pudl.extract.ferc1.ferc1_meta.tables

    # FERC doesn't use the sme column names for the same values across all of
    # Their tables... but all of these are cpacity in MW.
    capacity_cols = {'f1_steam': 'tot_capacity',
                     'f1_gnrt_plant': 'capacity_rating',
                     'f1_hydro': 'tot_capacity',
                     'f1_pumped_storage': 'tot_capacity'}

    rspndnt_tbl = f1_tbls['f1_respondent_id']
    ferc1_plants_all = pd.DataFrame()
    for tbl in plant_tables:
        plant_select = sa.sql.select([
            f1_tbls[tbl].c.respondent_id,
            f1_tbls[tbl].c.plant_name,
            rspndnt_tbl.c.respondent_name
        ]).distinct().where(
            sa.and_(
                f1_tbls[tbl].c.respondent_id == rspndnt_tbl.c.respondent_id,
                f1_tbls[tbl].c.plant_name != '',
                f1_tbls[tbl].columns[capacity_cols[tbl]] >= min_capacity,
                f1_tbls[tbl].c.report_year.in_(years)
            )
        )
        # Add all the plants from the current table to our bigger list:
        new_plants = pd.read_sql(plant_select, f1_engine)
        new_plants = new_plants.rename(
            columns={'respondent_id': 'utility_id_ferc1',
                     'respondent_name': 'utility_name_ferc1'})
        new_plants = pudl.helpers.strip_lower(
            new_plants, columns=['plant_name', 'utility_name_ferc1'])
        new_plants['plant_table'] = tbl

        ferc1_plants_all = ferc1_plants_all.append(
            new_plants[['utility_id_ferc1',
                        'utility_name_ferc1',
                        'plant_name',
                        'plant_table']])

    # If we're only trying to get the NEW plants, then we need to see which
    # ones we've already got in the PUDL DB, and look at what's different.
    if new:
        ferc1_plants_all = ferc1_plants_all.set_index(
            ['utility_id_ferc1', 'plant_name'])

        pt = pudl.output.pudltabl.get_table_meta(pudl_engine)

        ferc1_plants_tbl = pt['plants_ferc']
        ferc1_plants_select = sa.sql.select([
            ferc1_plants_tbl.c.utility_id_ferc1,
            ferc1_plants_tbl.c.plant_name
        ]).distinct()
        ferc1_plants_old = pd.read_sql(ferc1_plants_select, pudl_engine)
        ferc1_plants_old = ferc1_plants_old.set_index(
            ['utility_id_ferc1', 'plant_name'])

        # Take the difference between the two table indexes -- I.e. get a
        # list of just the index values that appear in the FERC index, but
        # not in the PUDL index.
        new_index = ferc1_plants_all.index.difference(ferc1_plants_old.index)
        if new_index.size == 0:
            logger.info("No new plants found.")
            return
        ferc1_plants = ferc1_plants_all.loc[new_index].reset_index()
    else:
        ferc1_plants = ferc1_plants_all

    return ferc1_plants

def check_ferc1_tables(refyear=2017):
    """
    Test each FERC1 data year for compatibility with references year schema.

    Args:
        refyear (int): The reference year for testing compatibility of the
            database schema with a FERC Form 1 table and year.

    Returns:
        dict: A dictionary having database table names as keys, and lists of
            which years that table was compatible with the refernce year as
            values.

    """
    good_table_years = {}
    tables = list(pc.ferc1_dbf2tbl.values())
    # This is a special table, to which every other table refers, it will be
    # loaded alongside every table we test.
    tables.remove('f1_respondent_id')
    for table in tables:
        good_years = []
        print(f"'{table}': [", end="", flush=True)
        for yr in pc.data_years['ferc1']:
            try:
                pudl.extract.ferc1.init_db(
                    ferc1_tables=['f1_respondent_id', table],
                    refyear=refyear,
                    years=[yr, ],
                    def_db=True,
                    testing=True,
                    force_tables=True)
                good_years = good_years + [yr, ]
                print(f"{yr},", end=" ", flush=True)
            # generally bare except: statements are bad, but here we're really
            # just trying to test whether the ferc1 extraction fails for *any*
            # reason, and if not, mark that year as good, thus the # nosec
            except:  # noqa: E722  # nosec
                continue
            ferc1_engine = pudl.extract.ferc1.connect_db(testing=True)
            pudl.extract.ferc1.drop_tables(ferc1_engine)
        good_table_years[table] = good_years
        print("],", flush=True)

    return good_table_years


import random
import numpy as np
from utils import WordDictionary, check_word

class TerminalDisplay:
    guess_words:list[str]
    target_words:list[str]

    def __init__(self) -> None:
        self.guess_words = WordDictionary().load_words('dictionary_words_5.txt')
        self.target_words = WordDictionary().load_words('dictionary_words_answers.txt')

    def start_game(self, IA = None, word_to_guess=None):
        # Input function est une entrée de l'utilisateur par défaut, peut être changé pour connecter à une IA.
        # Une IA doit être une instance d'une class avec au moins les fonctions guess() et save_results()

        if not word_to_guess :
            word_to_guess = random.choice(self.target_words)

        word_found = False
        turn = 0
        while not word_found:
            turn += 1
            guess = ""
            if IA :
                # IA
                guess = IA.guess()
                print("Mot essayé par l'IA :", guess)
            else :
                # Utilisateur
                print("Entrez un mot à essayer (5 lettres): ")
                guess = input()

            rslt = np.array(check_word(word_to_guess, guess))
            print(rslt)
            if np.all([rslt==2]):
                print("Mot trouvé en ",turn,"coups")
                word_found=True
                return turn
            if IA :
                IA.save_results(guess,rslt)
                # print("possible words now : ", len(IA.possible_words))


#import

from pyimagesearch.transform import four_point_transform
from skimage.filters import threshold_local
import numpy as numpy
import argparse
import cv2
import imutils

# construct the arg parser
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required = True,
	help = "Path to the image to be scanned")
args = vars(ap.parse_args())
# load the image and compute the ratio of the old height
# to the new height, clone it, and resize it
image = cv2.imread(args["image"])
ratio = image.shape[0] / 500.0
orig = image.copy()
image = imutils.resize(image, height = 500)

refPt = []
cropping = False
def click_and_crop(event, x, y, flags, param):
	# grab references to the global variables
	global refPt, cropping
	# if the left mouse button was clicked, record the starting
	# (x, y) coordinates and indicate that cropping is being
	# performed
	if event == cv2.EVENT_LBUTTONDOWN:
		refPt = [(x, y)]
		cropping = True
	# check to see if the left mouse button was released
	elif event == cv2.EVENT_LBUTTONUP:
		# record the ending (x, y) coordinates and indicate that
		# the cropping operation is finished
		refPt.append((x, y))
		cropping = False
		# draw a rectangle around the region of interest
		cv2.rectangle(image, refPt[0], refPt[1], (0, 255, 0), 2)
		cv2.imshow("image", image)

# convert the image to grayscale, blur it, and find edges
# in the image
#print("image shape", image.shape[1])

y_end = image.shape[0]
x_end = image.shape[1]

gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (5, 5), 0)
edged = cv2.Canny(gray, 75, 200)

# show the original image and the edge detected image
print("STEP 1: Edge Detection")
#cv2.imshow("Image", image)
#cv2.imshow("Edged", edged)
#cv2.waitKey(0)
#cv2.destroyAllWindows()

# find the contours in the edged image, keeping only the
# largest ones, and initialize the screen contour
cnts = cv2.findContours(edged.copy(), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
cnts = sorted(cnts, key = cv2.contourArea, reverse = True)[:5]

flag = False
# loop over the contours
for c in cnts:
	# approximate the contour
	peri = cv2.arcLength(c, True)
	approx = cv2.approxPolyDP(c, 0.02 * peri, True)
	# if our approximated contour has four points, then we
	# can assume that we have found our screen
	if len(approx) == 4:
		screenCnt = approx
		print("flag yes")
		flag = True
		break

if flag == False :
	screenCnt = numpy.array([[[0,0]] ,[[x_end,0]] ,[[x_end, y_end]] ,[[0 ,y_end]]])
# show the contour (outline) of the piece of paper
print("STEP 2: Find contours of paper")
cv2.drawContours(image, [screenCnt], -1, (0, 255, 0), 2)
#cv2.imshow("Outline", image)
#cv2.waitKey(0)
#cv2.destroyAllWindows()

check_if_cropped = False

clone = image.copy()
cv2.namedWindow("image")
cv2.setMouseCallback("image", click_and_crop)
# keep looping until the 'q' key is pressed
while True:
	# display the image and wait for a keypress
	cv2.imshow("image", image)
	key = cv2.waitKey(1) & 0xFF
	# if the 'r' key is pressed, reset the cropping region
	if key == ord("r"):
		image = clone.copy()
	# if the 'c' key is pressed, break from the loop
	elif key == ord("c"):
		check_if_cropped = True
		if len(refPt) == 2:
			roi = clone[refPt[0][1]:refPt[1][1], refPt[0][0]:refPt[1][0]]
			cv2.imshow("ROI", roi)
			y_end_cr = roi.shape[0]
			x_end_cr = roi.shape[1]
			cv2.waitKey(0)
		break
	elif key == ord("s"):
		roi = clone.copy()
		break
# if there are two reference points, then crop the region of interest
# from teh image and display it

# close all open windows
cv2.destroyAllWindows()

if check_if_cropped == True :
	screenCnt = numpy.array([[[0,0]] ,[[x_end_cr,0]] ,[[x_end_cr, y_end_cr]] ,[[0 ,y_end_cr]]])

# apply the four point transform to obtain a top-downs
# view of the original image

#warped = four_point_transform(imutils.resize(roi, height = 500, width = 500), screenCnt.reshape(4, 2))
warped = four_point_transform(roi, screenCnt.reshape(4, 2))

# convert the warped image to grayscale, then threshold it
# to give it that 'black and white' paper effect

    # Threshold the HSV image to get only blue colors
warped = cv2.cvtColor(warped,cv2.COLOR_RGBA2GRAY)

T = threshold_local(warped, 11, offset = 5, method = "gaussian")
warped = (warped > T).astype("uint8") * 255

# show the original and scanned images
print("STEP 3: Apply perspective transform")
cv2.imshow("Original", imutils.resize(orig, height = 500))
#cv2.imshow("Scanned", imutils.resize(warped, height = 500, width = 500))
cv2.imshow("Scanned", warped)

cv2.waitKey(0)


#%%
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import numpy as np
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import confusion_matrix, plot_confusion_matrix
from sklearn.metrics import accuracy_score, mean_squared_error, mean_absolute_error, auc, roc_curve, plot_roc_curve
from sklearn.metrics import  precision_score, recall_score, f1_score, classification_report, roc_auc_score
from sklearn.model_selection import StratifiedKFold

from pre_pro import  X_train, X_test, y_train, y_test#, X_val, y_val #sem val por causa do kfold
#from pre_pro import PCA_X_test, PCA_X_train,

import tensorflow as tf
import os #disable logs
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

from tensorflow.keras.optimizers import SGD, Adam
from keras.callbacks import EarlyStopping
from keras.wrappers.scikit_learn import KerasClassifier
from keras.regularizers import l2
from keras.backend import clear_session
from ann_visualizer.visualize import ann_viz;
#%% K FOLD CROSS VALIDATION
kfold = StratifiedKFold(n_splits=2, shuffle=True, random_state=1)
cvscores = []

for train, test in kfold.split(X_train, y_train):
    # create model
    clear_session()
    ann = tf.keras.models.Sequential(name='MLPFullyConnected')
    ann.add(tf.keras.layers.InputLayer(input_shape=(44,)))
    ann.add(tf.keras.layers.Dense(units= 6, activation='relu'))
    ann.add(tf.keras.layers.Dense(units= 1, activation='sigmoid'))

    # Compile model
    optimizer = Adam()

    #optimizer = SGD(learning_rate=0.01, decay=1e-5)# , momentum=.9,

    ann.compile(optimizer=optimizer,
                loss= 'mean_squared_error',
                metrics= ['accuracy']) #adam

    early_stop = EarlyStopping(monitor='val_loss',
                            min_delta=0,
                            patience=5,
                            verbose=0,
                            mode='auto')  #testar patiance

    ann_history = ann.fit(X_train.iloc[train],y_train[train],
                        shuffle=True,
                        use_multiprocessing=True, #validation_split=0.1,
                        batch_size= 128,
                        epochs= 200,
                        validation_data = (X_train.iloc[test],y_train[test]),
                        callbacks=[early_stop])

    scores = ann.evaluate(X_test, y_test, verbose=0)
    print("%s: %.2f%%" % (ann.metrics_names[1], scores[1]*100))
    cvscores.append(scores[1] * 100)
#%%
print(cvscores)
print("%.2f%% (+/- %.2f%%)" % (np.mean(cvscores), np.std(cvscores)))

#%% KERAS ANN MLP
#maybe add dropout?
clear_session()

ann = tf.keras.models.Sequential(name='MLPFullyConnected')
ann.add(tf.keras.layers.InputLayer(input_shape=(44,)))
ann.add(tf.keras.layers.Dense(units= 6, activation='sigmoid')) #testar tgh
#ann.add(tf.keras.layers.Dense(units= 10, activation='relu')) #testar tgh
ann.add(tf.keras.layers.Dense(units= 1, activation='sigmoid')) #sigmoid

optimizer = Adam()

#testar sgd ainda.
#optimizer = SGD(learning_rate=0.01, momentum=.9, decay=1e-5,)#

ann.compile(optimizer=optimizer,
            loss= 'mean_squared_error',
            metrics= ['accuracy']) #adam

early_stop = EarlyStopping(monitor='val_loss',
                           min_delta=0,
                           patience=5,
                           verbose=0,
                           mode='auto')  #testar patiance

ann_history = ann.fit(X_train,y_train,
                      shuffle=True,
                      use_multiprocessing=True, #validation_split=0.1,
                      batch_size= 128,
                      epochs= 200,
                      validation_data = (X_val, y_val),
                      callbacks=[early_stop])

scores = ann.evaluate(X_test, y_test, batch_size=128)
#%%
#Loss
loss_train = ann_history.history['loss']
loss_val = ann_history.history['val_loss']
epochs = range(1,len(loss_train)+1)
plt.plot(epochs, loss_train, 'g', label='Training loss')
plt.plot(epochs, loss_val, 'b', label='validation loss')
plt.title('Training and Validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()
#%%
#Acuracia
print("%s: %.2f%%" % (ann.metrics_names[1], scores[1]*100))

ann.summary()
ann.get_config()
loss_train = ann_history.history['accuracy']
loss_val = ann_history.history['val_accuracy']
epochs = range(1,len(loss_train)+1)
plt.plot(epochs, loss_train, 'g', label='Training accuracy')
plt.plot(epochs, loss_val, 'b', label='validation accuracy')
plt.title('Training and Validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
plt.show()

#%% model visualization
from keras.utils.vis_utils import plot_model
plot_model(ann, to_file='model.png')
#%%confusion matrix
y_pred = ann.predict_classes(X_test)
#print(y_pred)
cma = confusion_matrix(y_test, y_pred) #labels='1,2'
#print(cma)

group_names = ['True Neg','False Pos','False Neg','True Pos']
group_counts = ['{0:0.0f}'.format(value) for value in
                cma.flatten()]
group_percentages = ['{0:.2%}'.format(value) for value in
                     cma.flatten()/np.sum(cma)]
labels = [f'{v1}\n{v2}\n{v3}' for v1, v2, v3 in
          zip(group_names,group_counts,group_percentages)]
labels = np.asarray(labels).reshape(2,2)
sns.heatmap(cma, annot=labels, fmt='', cmap='Blues')

#%%
print(classification_report(y_test, y_pred))
print(f'ROC AUC score: {roc_auc_score(y_test, y_pred)}')
print('Accuracy Score: ',accuracy_score(y_test, y_pred))
#%%
precision = precision_score(y_test, y_pred)  #Precision Score
recall = recall_score(y_test, y_pred)  #Recall Score
f1 = f1_score(y_test, y_pred)  #F1 Score

print(precision, recall, f1)
#%%
y_test.shape
#%%roc
false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)
roc_auc = auc(false_positive_rate, true_positive_rate)
sns.set_theme(style = 'white')
plt.figure(figsize = (8, 8))
plt.plot(false_positive_rate,true_positive_rate, color = '#b01717', label = 'AUC = %0.3f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1], linestyle = '--', color = '#174ab0')
plt.axis('tight')
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.legend()
plt.show()

#%%error histogram for train, test, validation

sns.histplot(data=ann_history.history['loss'], kde=True, log_scale=True, element='step', fill=False)

# %%


# Definition for a binary tree node.
class TreeNode:
  def __init__(self, x):
      self.val = x
      self.left = None
      self.right = None

class Solution:
  def maxDepth(self, roote) :
    if not root:
      return 0
    leftDepth = rightDepth = 0
    if root.left: leftDepth = self.maxDepth(root.left)
    if root.right: rightDepth = self.maxDepth(root.right)
    return max(leftDepth, rightDepth) + 1

a = [3,9,20,None,None,15,7]
s = Solution()

root = TreeNode(a)
s.maxDepth(root)


# -*- coding: utf-8 -*-
"""
Created on Thu Jul  6 09:09:54 2017

@author: user
"""
import operator
from math import log
import matplotlib.pyplot as plt

def calcShannonEnt(dataSet):
    numEntries = len(dataSet)
    labelCounts = {}
    for featVec in dataSet:
        currentLabel = featVec[-1]
        if currentLabel not in labelCounts.keys():
            labelCounts[currentLabel] = 0
        labelCounts[currentLabel] += 1
        shannonEnt = 0.0
    for key in labelCounts:
        prob = float(labelCounts[key])/numEntries
        shannonEnt -= prob * log(prob,2)
    return shannonEnt

def createDataSet():
    dataSet = [[1, 1, 'yes'],
    [1, 1, 'yes'],
    [1, 0, 'no'],
    [0, 1, 'no'],
    [0, 1, 'no']]
    labels = ['no surfacing','flippers']
    return dataSet, labels
"""
myDat,labels=createDataSet()
print(myDat)

print(calcShannonEnt(myDat))
myDat[0][-1]='maybe'
print(myDat)
print(calcShannonEnt(myDat))
"""
def splitDataSet(dataSet, axis, value):
    retDataSet = []
    for featVec in dataSet:
        if featVec[axis] == value:
            reducedFeatVec = featVec[:axis]
            reducedFeatVec.extend(featVec[axis+1:])
            retDataSet.append(reducedFeatVec)
    return retDataSet
"""
myDat,labels = createDataSet()

print(myDat)
print(splitDataSet(myDat,0,1))
print(splitDataSet(myDat,0,0))
"""

def chooseBestFeatureToSplit(dataSet):
    numFeatures = len(dataSet[0]) - 1
    baseEntropy = calcShannonEnt(dataSet)
    bestInfoGain = 0.0; bestFeature = -1
    for i in range(numFeatures):
        featList = [example[i] for example in dataSet]
        uniqueVals = set(featList)
        newEntropy = 0.0
        for value in uniqueVals:
            subDataSet = splitDataSet(dataSet, i, value)
            prob = len(subDataSet)/float(len(dataSet))
            newEntropy += prob * calcShannonEnt(subDataSet)
        infoGain = baseEntropy - newEntropy
        if (infoGain > bestInfoGain):
            bestInfoGain = infoGain
            bestFeature = i
    return bestFeature

myDat,labels = createDataSet()
chooseBestFeatureToSplit(myDat)
# print(myDat)

def majorityCnt(classList):
    classCount={}
    for vote in classList:
        if vote not in classCount.keys(): classCount[vote] = 0
        classCount[vote] += 1
    sortedClassCount = sorted(classCount.iteritems(),
    key=operator.itemgetter(1), reverse=True)
    return sortedClassCount[0][0]

def createTree(dataSet,labels):
    classList = [example[-1] for example in dataSet]
    if classList.count(classList[0]) == len(classList):
        return classList[0]#stop splitting when all of the classes are equal
    if len(dataSet[0]) == 1: #stop splitting when there are no more features in dataSet
        return majorityCnt(classList)
    bestFeat = chooseBestFeatureToSplit(dataSet)
    bestFeatLabel = labels[bestFeat]
    myTree = {bestFeatLabel:{}}
    del(labels[bestFeat])
    featValues = [example[bestFeat] for example in dataSet]
    uniqueVals = set(featValues)
    for value in uniqueVals:
        subLabels = labels[:]       #copy all of labels, so trees don't mess up existing labels
        myTree[bestFeatLabel][value] = createTree(splitDataSet(dataSet, bestFeat, value),subLabels)
    return myTree

myDat,labels = createDataSet()
myTree = createTree(myDat,labels)

# print(myTree)

decisionNode = dict(boxstyle="sawtooth", fc="0.8")
leafNode = dict(boxstyle="round4", fc="0.8")
arrow_args = dict(arrowstyle="<-")

def plotNode(nodeTxt, centerPt, parentPt, nodeType):
    createPlot.ax1.annotate(nodeTxt, xy=parentPt,
    xycoords='axes fraction',
    xytext=centerPt, textcoords='axes fraction',
    va="center", ha="center", bbox=nodeType, arrowprops=arrow_args)

def createPlot():
    fig = plt.figure(1, facecolor='white')
    fig.clf()
    createPlot.ax1 = plt.subplot(111, frameon=False)
    plotNode('a decision node', (0.5, 0.1), (0.1, 0.5), decisionNode)
    plotNode('a leaf node', (0.8, 0.1), (0.3, 0.8), leafNode)
    plt.show()

# createPlot()

def getNumLeafs(myTree):
    numLeafs = 0
    firstStr = myTree.keys()[0]
    secondDict = myTree[firstStr]
    for key in secondDict.keys():
        if type(secondDict[key]).__name__=='dict':
            numLeafs += getNumLeafs(secondDict[key])
        else:
            numLeafs +=1
    return numLeafs

def getTreeDepth(myTree):
    maxDepth = 0
    firstStr = myTree.keys()[0]
    secondDict = myTree[firstStr]
    for key in secondDict.keys():
        if type(secondDict[key]).__name__=='dict':
            thisDepth = 1 + getTreeDepth(secondDict[key])
        else:thisDepth = 1
        if thisDepth > maxDepth: maxDepth = thisDepth
    return maxDepth

def retrieveTree(i):
    listOfTrees =[{'no surfacing': {0: 'no', 1: {'flippers': \
    {0: 'no', 1: 'yes'}}}},
    {'no surfacing': {0: 'no', 1: {'flippers': \
    {0: {'head': {0: 'no', 1: 'yes'}}, 1: 'no'}}}}
    ]
    return listOfTrees[i]

print(retrieveTree (1))

def plotMidText(cntrPt, parentPt, txtString):
    xMid = (parentPt[0]-cntrPt[0])/2.0 + cntrPt[0]
    yMid = (parentPt[1]-cntrPt[1])/2.0 + cntrPt[1]
    createPlot.ax1.text(xMid, yMid, txtString)

def plotTree(myTree, parentPt, nodeTxt):
    numLeafs = getNumLeafs(myTree)
    getTreeDepth(myTree)
    firstStr = myTree.keys()[0]
    cntrPt = (plotTree.xOff + (1.0 + float(numLeafs))/2.0/plotTree.totalW,\
    plotTree.yOff)
    plotMidText(cntrPt, parentPt, nodeTxt)
    plotNode(firstStr, cntrPt, parentPt, decisionNode)
    secondDict = myTree[firstStr]
    plotTree.yOff = plotTree.yOff - 1.0/plotTree.totalD
    for key in secondDict.keys():
        if type(secondDict[key]).__name__=='dict':
            plotTree(secondDict[key],cntrPt,str(key))
        else:
            plotTree.xOff = plotTree.xOff + 1.0/plotTree.totalW
            plotNode(secondDict[key], (plotTree.xOff, plotTree.yOff),
            cntrPt, leafNode)
            plotMidText((plotTree.xOff, plotTree.yOff), cntrPt, str(key))
    plotTree.yOff = plotTree.yOff + 1.0/plotTree.totalD

def createPlot(inTree):
    fig = plt.figure(1, facecolor='white')
    fig.clf()
    axprops = dict(xticks=[], yticks=[])
    createPlot.ax1 = plt.subplot(111, frameon=False, **axprops)
    plotTree.totalW = float(getNumLeafs(inTree))
    plotTree.totalD = float(getTreeDepth(inTree))
    plotTree.xOff = -0.5/plotTree.totalW; plotTree.yOff = 1.0;
    plotTree(inTree, (0.5,1.0), '')
    plt.show()


'''
All the parameters used in the Simulation has been documented here.

Easily modifiable paramters with the quick access in this settings.py file \
    to achieve quick modifications especially during the training sessions.

Names of the parameters are self-explanatory therefore elimating the use of further comments.
'''

HOST = "localhost"
PORT = 2000
TIMEOUT = 20.0

CAR_NAME = 'model3'
EPISODE_LENGTH = 120
NUMBER_OF_VEHICLES = 30
NUMBER_OF_PEDESTRIAN = 10
CONTINUOUS_ACTION = True
VISUAL_DISPLAY = True

RGB_CAMERA = 'sensor.camera.rgb'
SSC_CAMERA = 'sensor.camera.semantic_segmentation'


"""Collective motion"""
import numba
import numpy as np
from cell_lists import add_to_cells, neighboring_cells, iter_nearest_neighbors
from numba import f8, i8
from numba.typing.typeof import typeof

from crowddynamics.core.sensory_region import is_obstacle_between_points
from crowddynamics.core.structures import obstacle_type_linear
from crowddynamics.core.vector2D import length, normalize, weighted_average, dot
from crowddynamics.simulation.agents import NO_LEADER, NO_TARGET

MISSING_NEIGHBOR = -1

@numba.jit([f8[:, :](f8[:, :])], nopython=True, nogil=True, cache=True)
def normalize_nx2(v):
    out = np.zeros_like(v)
    for i in range(len(v)):
        out[i, :] = normalize(v[i, :])
    return out

@numba.jit([(f8[:], f8[:], f8[:], f8[:], f8)],
           nopython=True, nogil=True, cache=True)
def herding_relationship(x1, x2, v1, v2, phi=np.pi / 2):
    """Determine leader-follower relationship

    Args:
        x1:
        x2:
        v1:
        v2:
        phi: Angle between [0, pi]

    Returns:
        (bool ,bool):
    """
    if length(v1) == 0 or length(v2) == 0:
        return False, False

    e_rel = normalize(x2 - x1)

    c_i = dot(e_rel, normalize(v1))
    c_j = -dot(e_rel, normalize(v2))

    # 1.0 = cos(0)
    cos_phi = np.cos(phi)
    if cos_phi < c_i < 1.0:
        if cos_phi < c_j < 1.0:
            return False, False
        else:
            return True, False
    else:
        if cos_phi < c_j < 1.0:
            return False, True
        else:
            return True, True

@numba.jit(nopython=True, nogil=True)
def set_neighbor(i, j, l, neighbors, distances, distances_max):
    argmax = np.argmax(distances[i, :])
    neighbors[i, argmax] = j
    distances[i, argmax] = l
    distances_max[i] = np.max(distances[i, :])

@numba.jit([(f8[:, :], f8, i8, i8[:], i8[:], i8[:],
             i8[:], i8[:], typeof(obstacle_type_linear)[:])],
           nopython=True, nogil=True, cache=True)
def find_nearest_neighbors(
        position, sight, size_nearest_other,
        cell_indices, neigh_cells, points_indices, cells_count,
        cells_offset, obstacles):
    size = len(position)

    neighbors = np.full(
        (size, size_nearest_other),
        fill_value=MISSING_NEIGHBOR, dtype=np.int64)
    '''Current nearest neighbours.'''

    distances = np.full(
        (size, size_nearest_other),
        fill_value=sight, dtype=np.float64)
    '''Distance to current nearest neighbours.'''

    distances_max = np.full(
        size, fill_value=sight, dtype=np.float64)
    '''Distance to furthest neighbor.'''

    for i, j in iter_nearest_neighbors(
            cell_indices, neigh_cells, points_indices, cells_count,
            cells_offset):
        # Test if line of sight is obstructed by an obstacle
        if is_obstacle_between_points(position[i], position[j], obstacles):
            continue

        l = length(position[i] - position[j])

        if l < distances_max[i]:
            set_neighbor(i, j, l, neighbors, distances, distances_max)

        if l < distances_max[j]:
            set_neighbor(j, i, l, neighbors, distances, distances_max)

    return neighbors

@numba.jit(nopython=True, nogil=True, cache=True)
def herding_interaction(is_herding, position, velocity, neighbors,
                        weight_position, phi):
    r"""Herding effect.

        .. math::
           \mathbf{\hat{e}_{herding}} = \mathcal{N}
           \left(\sum_{j \in Neigh} \mathbf{\hat{e}}_j\right)

    """
    # [(boolean[:], f8[:, :], f8[:, :], i8[:, :], f8, f8, i8[:])]
    new_direction = np.zeros_like(position)
    has_new_direction = np.zeros(position.shape[0], dtype=np.bool_)

    mean_position = np.zeros(position.shape[1])
    mean_velocity = np.zeros(velocity.shape[1])

    for i in range(position.shape[0]):
        if not is_herding[i]:
            continue

        num_neigh = 0

        for j in neighbors[i, :]:
            if j == MISSING_NEIGHBOR:
                continue

            is_heading_away, _ = herding_relationship(
                position[i], position[j], velocity[i], velocity[j], phi)

            if is_heading_away:
                mean_position[:] += position[j, :]
                mean_velocity[:] += velocity[j, :]
                num_neigh += 1

        if num_neigh > 0:
            new_direction[i, :] = normalize(weighted_average(
                normalize(mean_position[:] / num_neigh - position[i, :]),
                normalize(mean_velocity[:]),
                weight_position))
            has_new_direction[i] = True

        mean_position[:] = 0
        mean_velocity[:] = 0

    return new_direction, has_new_direction

@numba.jit(nopython=True, nogil=True, cache=True)
def leader_follower_interaction_brute(
        is_follower, is_leader, position, velocity, weight_position, phi,
        target, index_leader, obstacles, sight):
    has_strategy = np.zeros(len(position), dtype=np.bool_)
    new_direction = np.zeros_like(position)
    # new_target = np.zeros_like(target)
    # new_index_leader = np.zeros_like(index_leader)

    indices = np.arange(len(position))
    leaders = indices[is_leader]
    distances = np.zeros(leaders.shape, dtype=np.float64)

    # Iterate over followers
    for i in indices[is_follower]:
        behind_obstacle = 0
        heading_away = 0

        # Find distances to the leaders
        for k, j in enumerate(leaders):
            distances[k] = length(position[i] - position[j])

        # Iterate over leaders in orders from closest to furthest leader
        # and choose a strategy.
        for k in np.argsort(distances):
            if distances[k] > sight:
                continue

            j = leaders[k]

            if is_obstacle_between_points(position[i], position[j], obstacles):
                # We are not seeing this leader.
                leader = index_leader[i]
                # Check if we were following this leader before.
                if leader != NO_LEADER and leader == j:
                    behind_obstacle += 1
                    # Use navigation of this leader.
                    target[i] = target[leader]
                    has_strategy[i] = True
                    break
                else:
                    continue
            else:
                is_heading_away, _ = herding_relationship(
                    position[i], position[j], velocity[i], velocity[j], phi)

                if is_heading_away:
                    heading_away += 1
                    # We see this leader. Remember it and follow.
                    index_leader[i] = j
                    target[i] = NO_TARGET
                    new_direction[i, :] = normalize(weighted_average(
                        normalize(position[j, :] - position[i, :]),
                        normalize(velocity[j, :]), weight_position))
                    has_strategy[i] = True
                    break

        if behind_obstacle == 0 and heading_away == 0:
            leader = index_leader[i]
            if leader != NO_LEADER:
                target[i] = target[leader]
                has_strategy[i] = True

    return new_direction, has_strategy

def leader_follower_interaction(
        agents, obstacles, sight, phi=0.45 * np.pi,
        weight_position_leader=0.40):
    # Follow the leader
    direction_leader, has_strategy = leader_follower_interaction_brute(
        agents['is_follower'], agents['is_leader'], agents['position'],
        agents['velocity'], weight_position_leader, phi,
        agents['target'], agents['index_leader'], obstacles, sight)

    # Use familiar exits
    is_lost = ~has_strategy & agents['is_follower']
    agents['target'][is_lost] = agents['familiar_exit'][is_lost]

    return direction_leader

def leader_follower_with_herding_interaction(
        agents, obstacles, sight,
        size_nearest_other,
        phi=0.45 * np.pi,
        weight_position_herding=0.15,
        weight_position_leader=0.40,
        weight_direction_leader=0.65):
    position = agents['position']
    velocity = agents['velocity']
    is_leader = agents['is_leader']
    is_follower = agents['is_follower']
    sight_leader = 20.0

    cell_size = sight
    points_indices, cells_count, cells_offset, grid_shape = add_to_cells(
        agents['position'], cell_size)
    cell_indices = np.arange(len(cells_count))
    neigh_cells = neighboring_cells(grid_shape)

    neighbors = find_nearest_neighbors(
        position, sight,
        size_nearest_other, cell_indices, neigh_cells, points_indices,
        cells_count, cells_offset, obstacles)

    direction, has_direction = herding_interaction(
        is_follower, position, velocity, neighbors, weight_position_herding,
        phi)

    agents['target'][has_direction] = NO_TARGET

    direction_leader, has_strategy = leader_follower_interaction_brute(
        is_follower, is_leader, position, velocity, weight_position_leader,
        phi, agents['target'], agents['index_leader'], obstacles, sight_leader)

    # Use familiar exits
    is_lost = ~(has_direction | has_strategy) & is_follower
    agents['target'][is_lost] = agents['familiar_exit'][is_lost]

    return normalize_nx2(weighted_average(
        direction_leader, direction, weight_direction_leader))


import csv
from taumahi import kōmiri_kupu

pūtuhi = csv.DictWriter(open('aotearoa-nz.csv', 'w'),
    fieldnames = ('id', 'name', 'status', 'feat_type', 'latitude', 'longitude', 'reo', 'maybe', 'other'))
pūtuhi.writeheader()

# Use the kōmiri_kupu function to count words of each language type in the
# placenames

for rārangi in csv.DictReader(open('gaz_names.csv')):
    reo = [sum(x.values()) for x in kōmiri_kupu(rārangi['name'])]
    pūtuhi.writerow(dict(id =rārangi['\ufeffname_id'],
        name = rārangi['name'],
        status = rārangi['status'],
        feat_type = rārangi['feat_type'],
        latitude = rārangi['crd_latitude'],
        longitude = rārangi['crd_longitude'],
        reo = reo[0],
        maybe = reo[1],
        other = reo[2]
        ))


# Module imports
from flask import request, abort, current_app
from sqlalchemy import exc
from marshmallow import ValidationError

class masqlapi():
    """ RestfulAPI SQLA/MarshMallow/jSend class """

    def __init__(self, session, resource, roschema, rwschema):
        """ Constructor """
        self.session = session
        self.resource = resource
        self.roschema = roschema
        self.rwschema = rwschema

    def get(self, **kwargs):
        """ Get record for a given resource """
        try:
            current_app.logger.debug(f'kwargs:{kwargs}')
            obj = self.resource.query.filter_by(**kwargs).first()
        except ValueError as err:
            current_app.logger.error(err)
            abort(400, err.args)
        return obj

    def get_many(self, **kwargs):
        """ Gets zero or more records for a given resource """
        try:
            obj = self.resource.query.filter_by(**kwargs)
        except ValueError as err:
            current_app.logger.error(err)
            abort(400, err.args)
        return obj

    def getMethod(self, obj, many=False):
        """ GET one """
        # Dump object
        try:
            data = self.roschema().dump(obj, many=many)
        except ValidationError as error:
            errMsg = {
                "errorType":"Schema Dump ValidationError",
                "errMsg":error.messages
            }
            abort(400, errMsg)
        except :
            abort(500, "schema dump error with unknown reason")

        # Return entity
        return {'message': None, 'status_code': 200, 'status': 'success', 'data': data}
    def add(self,obj):
        """ add the object from manual create"""
        # Merge and Commit
        try:
            self.session.add(obj)
            self.session.commit()
            data = self.roschema().dump(obj)
            return {'message': 'OK - entity add successfully', 'status_code': 200, 'status': 'success', 'data': data}
        except exc.IntegrityError as err:
            self.session.rollback()
            return {'message': 'Conflict', 'status_code': 409, 'status': 'failure', 'data': err.orig.__str__().strip()}
        except Exception as err:
            self.session.rollback()
            raise err

    def update(self,obj):
        """ just update the object from manual patch"""
        # Merge and Commit
        try:
            current_app.logger.debug(f"obj before patch:{obj.__dict__}")
            self.session.add(obj)
            self.session.commit()
            data = self.roschema().dump(obj)
            return {'message': 'OK - entity updated successfully', 'status_code': 200, 'status': 'success', 'data': data}
        except exc.IntegrityError as err:
            self.session.rollback()
            return {'message': 'Conflict', 'status_code': 409, 'status': 'failure', 'data': err.orig.__str__().strip()}
        except Exception as err:
            self.session.rollback()
            raise err

    def patch(self, obj):
        """ PATCH """
        # Grab request data
        json = request.get_json()
        if not json:
            return {'message': 'Bad request', 'status_code': 400, 'status': 'failure', 'data': 'JSON input object is missing or cannot be parsed'}

        try:
            current_app.logger.debug(f"isObj shown:{obj}, json:{json}")
            data = self.rwschema().load(json, instance=obj, partial=True)
            # data = self.rwschema().dump(obj)
            # data_id = data.pop('id',None)
            # data = {k: v for k, v in data.items() if v is not None}
            # data = self.rwschema().load(data)

            # current_app.logger.debug(f"data before patching:{data}")
            # patch_data = self.rwschema().load(json, partial=True)
            # data.update(patch_data)
            # if data_id:
            #     data['id'] = data_id
            # current_app.logger.debug(f"data for patching:{data}")
        except ValidationError as error:
            errMsg = {
                "errorType":"Schema Load ValidationError",
                "errMsg":error.messages
            }
            abort(400, errMsg)
        except :
            abort(500, "schema load error with unknown reason")

        # Merge and Commit
        try:
            for key, value in data.items():
                setattr(obj,key, value)
            current_app.logger.debug(f"obj before patch:{obj.__dict__}")
            self.session.add(obj)
            self.session.commit()
            data = self.roschema().dump(obj)
            return {'message': 'OK - entity updated successfully', 'status_code': 200, 'status': 'success', 'data': data}
        except exc.IntegrityError as err:
            self.session.rollback()
            return {'message': 'Conflict', 'status_code': 409, 'status': 'failure', 'data': err.orig.__str__().strip()}
        except Exception as err:
            self.session.rollback()
            raise err

    def delete(self, obj):
        """ DELETE """
        # Delete object
        try:
            self.session.delete(obj)
            self.session.commit()
            return {'message': 'No data', 'status_code': 200, 'status': 'success', 'data': 'Entity deleted successfully'}
        except exc.IntegrityError as err:
            self.session.rollback()
            return {'message': 'Conflict', 'status_code': 409, 'status': 'failure', 'data': err.orig.__str__().strip()}, 409
        except Exception as err:
            self.session.rollback()
            raise err

    def post(self,uniqueField):
        """ POST """

        # if uniqueField is not a list we make it into one
        if not isinstance(uniqueField, list):
            uniqueField_list = [uniqueField]
        else:
            uniqueField_list = uniqueField
        # Grab request data
        json = request.get_json()
        if not json:
            return {'message': 'Bad request', 'status_code': 400, 'status': 'failure', 'data': 'JSON input object is missing or cannot be parsed'}, 400
        # print("json:{0}, type:{1}".format(json,type(json)))
        # Validate and deserialize input
        try:
            data = self.rwschema().load(json)
        except ValidationError as error:
            errMsg = {
                "errorType":"Schema Load ValidationError",
                "errMsg":error.messages
            }
            abort(400, errMsg)
        except :
            abort(500, "schema load error with unknown reason")

        query = {}
        for keys in uniqueField_list:
            if keys in data:
                query[keys] = data[keys]
        # Check for existing row based on passed JSON
        current_app.logger.info(f"query data in post function:{query}")
        existing = self.resource.query.filter_by(**query).first()

        if existing:
            obj=self.roschema().dump(existing)
            return {'message': 'OK - entity exists', 'status_code': 200, 'status': 'success', 'data': obj}

        # Add and Commit
        try:
            obj = self.resource(**data)
            self.session.add(obj)
            self.session.commit()
            data = self.roschema().dump(obj)
            return {'message': 'Created - entity created successfully', 'status_code': 201, 'status': 'success', 'data': data}, 201
        except exc.IntegrityError as err:
            self.session.rollback()
            return {'message': 'Conflict', 'status_code': 409, 'status': 'failure', 'data': err.orig.__str__().strip()},409
        except Exception as err:
            self.session.rollback()
            raise err


from helpers import *
from levelUnits import *
from gridClasses import *
from random import randint
from gridGeo import dist
from classtypes import inf

def genBoard(x,y,defCell=".",wallCell="x",empty=.1):
    x_size=x*7
    y_size=y*7
    grid=[[defCell for i in range(x_size)] for i in range(y_size)]
    def push_gunit(gunit,xo,yo):
        for y in range(len(gunit)):
            for x in range(len(gunit[0])):
                cell = gunit[y][x]
                if cell=="x":
                    if cell!=wallCell:
                        cell=wallCell
                    grid[y+yo][x+xo]=cell

    for rx in range(0,x):
        for ry in range(0,y):
            gunit = GUNITS[randint(0,len(GUNITS)-1)]#get random grid peice
            if randint(1,100)/100>empty:#chance of empty to not spawn
                push_gunit(gunit,rx*7,ry*7)

    for row in range(len(grid)):
        if row==0 or row==len(grid)-1:
            for i in range(len(grid[row])):
                grid[row][i]=wallCell
        else:
            grid[row][0]=wallCell
            grid[row][-1]=wallCell
    return grid

def spawnEnemies(crang,amount,canSpawnFunc,grid,minPlayerDist=-1,playerPos=[0,0],bad=[]):
    spawned=[]
    for i in range(amount):
        pos=[randint(crang[0][i],crang[1][i]) for i in range(2)]
        while (not canSpawnFunc(pos[0],pos[1])) or dist(pos,playerPos)<minPlayerDist or pos in bad:
            pos=[randint(crang[0][i],crang[1][i]) for i in range(2)]
        spawned.append(Unit(pos,randFrameName(),[randWeaponName(),randWeaponName()],randArmorName(),displayChar="*",kind="enemy"))
    return spawned

#spawnAllies
def spawnAllies(crang,amount,canSpawnFunc,grid,maxPlayerDist=inf(),playerPos=[0,0],bad=[]):
    spawned=[]
    for i in range(amount):
        pos=[randint(crang[0][i],crang[1][i]) for i in range(2)]
        while (not canSpawnFunc(pos[0],pos[1])) or (dist(pos,playerPos)>maxPlayerDist) or (pos in bad):
            pos=[randint(crang[0][i],crang[1][i]) for i in range(2)]
        spawned.append(Unit(pos,randFrameName(),[randWeaponName(),randWeaponName()],randArmorName(),displayChar="+",kind="ally"))
    return spawned

def get_free_loc(crang,grid,canSpawnFunc,bad=[],playerPos=[0,0],dist_range=[-1,inf()]):
    loc=[randint(crang[0][i],crang[1][i]) for i in range(2)]
    while (not canSpawnFunc(loc[0],loc[1])) or loc in bad or (not dist_range[0]<dist(loc,playerPos)<dist_range[1]):
        loc=[randint(crang[0][i],crang[1][i]) for i in range(2)]
    return loc


import requests
import re
from bs4 import BeautifulSoup

import time
import datetime
import random

starting_page = None
ending_page = None

base_url = "https://boardgamegeek.com/browse/boardgame/page/"

MIN_TIME = 1
MAX_TIME = 5

def sleep(MIN_TIME, MAX_TIME):
    time.sleep(random.randint(MIN_TIME*10, MAX_TIME*10)/10)

def boardgameinfo(info):
    links = info.find_all("a")

    rank = links[0].get("name")
    gameid = links[1].get("href").split("/")[2]
    gamename = links[2].text

    span = info.select("span")
    if len(span) > 0:
        releaseyear = span[0].text
    else:
        releaseyear = ""

    if len(releaseyear) > 0:
        if (releaseyear[0] == "(") & (releaseyear[-1] == ")"):
            releaseyear = releaseyear[1:-1]

    ratingdata = info.find_all("td", {"class": "collection_bggrating"})

    geekrating = ratingdata[0].text.strip()
    userrating = ratingdata[1].text.strip()
    numvotes = ratingdata[2].text.strip()

    return (rank, gameid, releaseyear, geekrating, userrating, numvotes, gamename)

def geturl(base_url, pagenumber):
    return f"{base_url}{pagenumber}"

if __name__ == "__main__":
    timenow = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    games_collected = 0

    for pagenumber in range(starting_page, ending_page+1):
        url = geturl(base_url, pagenumber)
        response = requests.get(url)
        while response.status_code != 200:
            print(response.status_code)
            sleep(MIN_TIME, MAX_TIME)
            response = requests.get(url)

        soup = BeautifulSoup(response.text, 'html.parser')

        rows = soup.find_all("tr", {"id": "row_"})

        if len(rows) > 0:
            for row in rows:
                rank, gameid, releaseyear, geekrating, userrating, numvotes, gamename = boardgameinfo(row)
                with open("../../data/boardgames.csv", "a+") as f:
                    record = f"{rank},{gameid},{releaseyear},{geekrating},{userrating},{numvotes},'{gamename}'\n"
                    f.write(record)

        games_collected += len(rows)

        print(f"{timenow}, Page: {pagenumber}, Games: {games_collected}")

        sleep(MIN_TIME, MAX_TIME)


'''
    Reference : https://github.com/rlcode/per
    Date: 2022.11.30
    Authored: Dala zhu
'''

import random
import math
import sys
import gymnasium as gym
import numpy as np
import numpy
import pylab
import torch
import torch.nn as nn

EPSILON_START = 1.0
EPSILON_FINAL = 0.01
EPSILON_DECAY = 1000
epsilon_by_frame = lambda i : EPSILON_FINAL + (EPSILON_START - EPSILON_FINAL) * math.exp(-1 * i / EPSILON_DECAY)

# SumTree
# a binary tree data structure where the parent’s value is the sum of its children
class SumTree:
    write = 0
    def __init__(self, capacity):
        self.capacity = capacity
        self.tree = numpy.zeros(2 * capacity - 1)  #start from 0
        self.data = numpy.zeros(capacity, dtype=object)
        self.n_entries = 0  #tree的leaf中被占用的个数，也即data中不为空的个数

    # update to the root node
    def _propagate(self, idx, change):
        parent = (idx - 1) // 2
        self.tree[parent] += change
        if parent != 0:
            self._propagate(parent, change)

    # find sample on leaf node
    def _retrieve(self, idx, s):
        left = 2 * idx + 1
        right = left + 1

        if left >= len(self.tree):
            return idx

        if s <= self.tree[left]:
            return self._retrieve(left, s)
        else:
            return self._retrieve(right, s - self.tree[left])

    def total(self):
        return self.tree[0]

    # store priority and sample
    def add(self, p, data):
        idx = self.write + self.capacity - 1

        self.data[self.write] = data
        self.update(idx, p)

        self.write += 1
        if self.write >= self.capacity:
            self.write = 0

        if self.n_entries < self.capacity:
            self.n_entries += 1

    # update priority
    def update(self, idx, p):
        change = p - self.tree[idx]
        self.tree[idx] = p
        self._propagate(idx, change)

    # get priority and sample
    def get(self, s):
        idx = self._retrieve(0, s)
        dataIdx = idx - self.capacity + 1

        return (idx, self.tree[idx], self.data[dataIdx])

class Memory:  # stored as ( s, a, r, s_, done) in SumTree
    e = 0.01  #ebusilon
    a = 0.6 #alpha
    beta = 0.4
    beta_increment_per_sampling = 0.001

    def __init__(self, capacity):
        self.tree = SumTree(capacity)
        self.capacity = capacity

    def _get_priority(self, error):
        return (np.abs(error) + self.e) ** self.a

    def add(self, error, sample):   #sample is (s, a, r, s_, done)
        p = self._get_priority(error)
        self.tree.add(p, sample)

    def sample(self, n):
        batch = []
        idxs = []

        segment = self.tree.total() / n  #抽取时为何会进行这样化等分抽取？
        priorities = []

        self.beta = np.min([1., self.beta + self.beta_increment_per_sampling])  #每次采样，beta都会增加，为何？

        for i in range(n):
            a = segment * i
            b = segment * (i + 1)

            s = random.uniform(a, b)   # sample from [a,b]
            (idx, p, data) = self.tree.get(s)
            priorities.append(p)
            batch.append(data)
            idxs.append(idx)

        sampling_probabilities = np.array(priorities) / self.tree.total()
        is_weight = np.power(self.tree.n_entries * sampling_probabilities, - self.beta)
        is_weight /= is_weight.max()

        return batch, idxs, is_weight #idxs is the index in tree.tree, not in tree.data

    def update(self, idx, error):
        p = self._get_priority(error)
        self.tree.update(idx, p)

class DuelingDQN(nn.Module):
    def __init__(self, n_state, n_action):

        super(DuelingDQN, self).__init__()
        self.n_state = n_state
        self.n_action = n_action

        self.feature = nn.Sequential(
            nn.Linear(self.n_state, 128),
            nn.ReLU()
        )

        self.advantage = nn.Sequential(
            nn.Linear(128, 128),
            nn.ReLU(),
            nn.Linear(128, self.n_action)
        )

        self.value = nn.Sequential(
            nn.Linear(128, 128),
            nn.ReLU(),
            nn.Linear(128, 1)
        )

    def forward(self, x):
        x = self.feature(x)
        advantage = self.advantage(x)
        value = self.value(x)
        return value + advantage - advantage.mean()  # V(s) + A(s,a) - mean(A(s,a)),这里是将max(A(s,a))换成了mean(A(s,a))

class Agent:
    def __init__(self, num_state, num_action):

        self.num_state = num_state
        self.num_action = num_action
        self.learning_rate = 1e-3
        self.gamma = 0.99
        self.batch_size = 64  # 如果太大训练不好训练
        self.load_model = False #超参，是否记载模型

        self.online_net = DuelingDQN(num_state, num_action)

        if self.load_model:
            self.online_net = torch.load('./save_model/cartpole_dqn')

        self.target_net = DuelingDQN(num_state, num_action)
        self.update_target()

        self.memo = Memory(capacity = 1000)

        self.optimizer = torch.optim.Adam(self.online_net.parameters(), lr= self.learning_rate)

    def act(self, state, epsilon):
        if random.random() > epsilon:
            state = torch.as_tensor(state).unsqueeze(0)
            q_value = self.online_net(state)
            # action = q_value.max(1)[1].item()
            max_q_idx = torch.argmax(input=q_value)
            action = max_q_idx.detach().item()
        else:
            action = random.randrange(self.num_action)
        return action

    def push(self, state, action, reward, next_state, done):
        # error最大，这个需要看论文确定
        error = max(self.memo.tree.tree[self.memo.capacity - 1: self.memo.tree.n_entries + self.memo.tree.capacity])
        self.memo.add(error = error, sample = (state, action, reward, next_state, done))

    def compute_td_loss(self):

        batch, idxs, is_weight = self.memo.sample(self.batch_size)
        batch_state, batch_action, batch_reward, batch_next_state, batch_done = zip(*batch)
        batch_state_tensor = torch.as_tensor(np.asarray(batch_state), dtype=torch.float32)
        batch_action_tensor = torch.as_tensor(np.asarray(batch_action), dtype=torch.int64).unsqueeze(-1)
        batch_reward_tensor = torch.as_tensor(np.asarray(batch_reward), dtype=torch.float32).unsqueeze(-1)  # unsqueeze 进行升维
        batch_done_tensor = torch.as_tensor(np.asarray(batch_done), dtype=torch.float32).unsqueeze(-1)
        batch_next_state_tensor = torch.as_tensor(np.asarray(batch_next_state), dtype=torch.float32)

        # compute double DQN , y_i
        next_q_values_online = self.online_net(batch_next_state_tensor)
        next_q_values_target = self.target_net(batch_next_state_tensor)

        next_q_value = next_q_values_target.gather(dim=1, index=torch.max(next_q_values_online, dim=1, keepdim=True)[1])

        pred_q_value = batch_reward_tensor + self.gamma * next_q_value * (1 - batch_done_tensor)

        # compute Q(s,a)
        q_values = self.online_net(batch_state_tensor)
        true_q_value = q_values.gather(1, batch_action_tensor)

        # update priority
        errors = torch.abs(pred_q_value - true_q_value).squeeze(dim = 1).detach().numpy()
        for i in range(self.batch_size):
            idx = idxs[i]
            self.memo.update(idx, errors[i])

        # compute loss
        loss = (true_q_value - pred_q_value.data).pow(2).mean()
        # loss = nn.functional.smooth_l1_loss(expected_q_value, q_value)

        # gradient descent
        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()
        return loss

    # update target net parameters
    def update_target(self):
        self.target_net.load_state_dict(self.online_net.state_dict())

class Environment:
    def __init__(self, env_id, render_mode):
        self.env = gym.make(env_id, render_mode = render_mode)

        self.num_action = self.env.action_space.n
        self.num_state = self.env.observation_space.shape[0]
        self.agent = Agent(self.num_state, self.num_action)
        self.updata_target_frame = 500
        self.n_episode = 500

    def train(self):
        eposide_rewards = []
        eposide_ids = []
        frame_losses = []
        frames = 0 # 记录所有的帧数
        for i_eposide in range(self.n_episode):
            done = False
            eposide_reward = 0
            # 初始化状态
            state, _ = self.env.reset()
            while not done:
                frames += 1
                epsilon = epsilon_by_frame(frames)
                action = self.agent.act(state, epsilon)
                next_state, reward, done, _, _ = self.env.step(action)

                # if an action make the episode end, then gives penalty of -100
                reward = reward if not done or eposide_reward == 499 else -10

                # add transitions to memory
                self.agent.push(state, action, reward, next_state, done)

                #compute td loss
                loss = self.agent.compute_td_loss()
                frame_losses.append(loss.item())

                # pylab.figure(1)
                # pylab.plot(frame_losses, 'r')
                # pylab.xlabel('step')
                # pylab.ylabel('loss')
                # pylab.savefig("./save_graph/cartpole_dqn_loss.png")

                eposide_reward += reward
                state = next_state

                if done:
                    eposide_reward = eposide_reward if eposide_reward == 500 else eposide_reward + 10
                    eposide_rewards.append(eposide_reward)
                    eposide_ids.append(i_eposide)

                    # plot the play time
                    # pylab.figure(2)
                    # pylab.plot(eposide_ids, eposide_rewards, 'b')
                    # pylab.xlabel('eposide')
                    # pylab.ylabel('eposide_reward')
                    # pylab.savefig("./save_graph/cartpole_dqn_reward.png")

                    print("episode:", i_eposide, "  eposids_reward:", eposide_reward, "  memory length:",
                          self.agent.memo.tree.n_entries, "  epsilon:", epsilon)

                    # if the mean of scores of last 10 episode is bigger than 490
                    # stop training
                    if np.mean(eposide_rewards[-min(10, len(eposide_rewards)):]) > 490:
                        torch.save(self.agent.online_net, "./save_model/cartpole_dqn")
                        sys.exit()

                if frames % self.updata_target_frame == 0:
                    self.agent.update_target()

if __name__ == '__main__':
    env_id = 'CartPole-v1'
    env = Environment(env_id, 'human')
    env.train()


# adapted from https://www.reddit.com/r/learnpython/comments/22tke1/use_python_to_send_keystrokes_to_games_in_windows/
# NOTE: only supports Win32 API!
# Consider using PyAutoGUI where cross-platform support is needed!

import ctypes, time
# Bunch of stuff so that the script can send keystrokes to game #

SendInput = ctypes.windll.user32.SendInput

# C struct redefinitions
PUL = ctypes.POINTER(ctypes.c_ulong)
class KeyBdInput(ctypes.Structure):
    _fields_ = [("wVk", ctypes.c_ushort),
                ("wScan", ctypes.c_ushort),
                ("dwFlags", ctypes.c_ulong),
                ("time", ctypes.c_ulong),
                ("dwExtraInfo", PUL)]

class HardwareInput(ctypes.Structure):
    _fields_ = [("uMsg", ctypes.c_ulong),
                ("wParamL", ctypes.c_short),
                ("wParamH", ctypes.c_ushort)]

class MouseInput(ctypes.Structure):
    _fields_ = [("dx", ctypes.c_long),
                ("dy", ctypes.c_long),
                ("mouseData", ctypes.c_ulong),
                ("dwFlags", ctypes.c_ulong),
                ("time",ctypes.c_ulong),
                ("dwExtraInfo", PUL)]

class Input_I(ctypes.Union):
    _fields_ = [("ki", KeyBdInput),
                 ("mi", MouseInput),
                 ("hi", HardwareInput)]

class Input(ctypes.Structure):
    _fields_ = [("type", ctypes.c_ulong),
                ("ii", Input_I)]

# Actuals Functions

def PressKey(hexKeyCode):
    extra = ctypes.c_ulong(0)
    ii_ = Input_I()
    ii_.ki = KeyBdInput( 0, hexKeyCode, 0x0008, 0, ctypes.pointer(extra) )
    x = Input( ctypes.c_ulong(1), ii_ )
    ctypes.windll.user32.SendInput(1, ctypes.pointer(x), ctypes.sizeof(x))

def ReleaseKey(hexKeyCode):
    extra = ctypes.c_ulong(0)
    ii_ = Input_I()
    ii_.ki = KeyBdInput( 0, hexKeyCode, 0x0008 | 0x0002, 0, ctypes.pointer(extra) )
    x = Input( ctypes.c_ulong(1), ii_ )
    ctypes.windll.user32.SendInput(1, ctypes.pointer(x), ctypes.sizeof(x))

# These are SCAN codes, not virtual codes!
keyMap = {
    '1': 0x02,
    '2': 0x03,
    '3': 0x04,
    '4': 0x05,
    '5': 0x06,
    '6': 0x07,
    '7': 0x08,
    '8': 0x09,
    '9': 0x0A,
    '0': 0x0B,
    'Q': 0x10,
    'W': 0x11,
    'E': 0x12,
    'R': 0x13,
    'T': 0x14,
    'Y': 0x15,
    'U': 0x16,
    'I': 0x17,
    'O': 0x18,
    'P': 0x19,
    'A': 0x1E,
    'S': 0x1F,
    'D': 0x20,
    'F': 0x21,
    'G': 0x22,
    'H': 0x23,
    'J': 0x24,
    'K': 0x25,
    'L': 0x26,
    'Z': 0x2C,
    'X': 0x2D,
    'C': 0x2E,
    'V': 0x2F,
    'B': 0x30,
    'N': 0x31,
    'M': 0x32,
    'LCTRL': 0x1D,
    'LSHIFT': 0x2A,
    'LALT': 0x38,
    'ENTER': 0x1C,
    'LARROW': 0x4B,
    'RARROW': 0x4D
}

delay = 1.5

class KeyNotSupportedError(Exception):
    pass

def CheckAndReturnKeyInput(key):
    if key not in keyMap:
        raise KeyNotSupportedError
    return keyMap[key]

def KeyPress(key):
    time.sleep(delay)
    keyCode = CheckAndReturnKeyInput(key)
    PressKey(keyCode) # press M
    time.sleep(.05)
    ReleaseKey(keyCode) #release M

def TwoKeyCombo(key1, key2):
    time.sleep(delay)
    keyCode1 = CheckAndReturnKeyInput(key1)
    keyCode2 = CheckAndReturnKeyInput(key2)
    PressKey(keyCode1)
    time.sleep(0.05)
    PressKey(keyCode2)
    time.sleep(0.05)
    ReleaseKey(keyCode2)
    time.sleep(0.05)
    ReleaseKey(keyCode1)