from collections import OrderedDict

import numpy as np
import pandas as pd

from ... import opcodes as OperandDef
from ...core import ENTITY_TYPE, recursive_tile
from ...serialization.serializables import (
    KeyField,
    AnyField,
    StringField,
    DataTypeField,
    BoolField,
    Int32Field,
)
from ...tensor.core import TENSOR_TYPE, TENSOR_CHUNK_TYPE
from ...tensor.datasource import (
    empty,
    tensor as astensor,
    from_series as tensor_from_series,
    from_dataframe as tensor_from_dataframe,
)
from ...tensor.statistics.quantile import quantile as tensor_quantile
from ..operands import DataFrameOperand, DataFrameOperandMixin
from ..core import DATAFRAME_TYPE
from ..datasource.from_tensor import series_from_tensor, dataframe_from_tensor
from ..initializer import DataFrame as create_df
from ..utils import parse_index, build_empty_df, find_common_type, validate_axis

class DataFrameQuantile(DataFrameOperand, DataFrameOperandMixin):
    _op_type_ = OperandDef.QUANTILE

    _input = KeyField("input")
    _q = AnyField("q")
    _axis = Int32Field("axis")
    _numeric_only = BoolField("numeric_only")
    _interpolation = StringField("interpolation")

    _dtype = DataTypeField("dtype")

    def __init__(
        self,
        q=None,
        interpolation=None,
        axis=None,
        numeric_only=None,
        dtype=None,
        gpu=None,
        output_types=None,
        **kw
    ):
        super().__init__(
            _q=q,
            _interpolation=interpolation,
            _axis=axis,
            _numeric_only=numeric_only,
            _dtype=dtype,
            _output_types=output_types,
            gpu=gpu,
            **kw
        )

    @property
    def input(self):
        return self._input

    @property
    def q(self):
        return self._q

    @property
    def interpolation(self):
        return self._interpolation

    @property
    def axis(self):
        return self._axis

    @property
    def numeric_only(self):
        return self._numeric_only

    def _set_inputs(self, inputs):
        super()._set_inputs(inputs)
        self._input = self._inputs[0]
        if isinstance(self._q, (TENSOR_TYPE, TENSOR_CHUNK_TYPE)):
            self._q = self._inputs[-1]

    def _calc_dtype_on_axis_1(self, a, dtypes):
        quantile_dtypes = []
        for name in dtypes.index:
            dt = tensor_quantile(
                tensor_from_series(a[name]),
                self._q,
                interpolation=self._interpolation,
                handle_non_numeric=not self._numeric_only,
            ).dtype
            quantile_dtypes.append(dt)
        return find_common_type(quantile_dtypes)

    def _call_dataframe(self, a, inputs):
        if self._numeric_only:
            empty_df = build_empty_df(a.dtypes)
            dtypes = empty_df._get_numeric_data().dtypes
        else:
            dtypes = a.dtypes
        if isinstance(self._q, TENSOR_TYPE):
            q_val = self._q
            pd_index = pd.Index([], dtype=q_val.dtype)
            name = None
            store_index_value = False
        else:
            q_val = np.asanyarray(self._q)
            pd_index = pd.Index(q_val)
            name = self._q if q_val.size == 1 else None
            store_index_value = True
        tokenize_objects = (a, q_val, self._interpolation, type(self).__name__)

        if q_val.ndim == 0 and self._axis == 0:
            index_value = parse_index(dtypes.index, store_data=store_index_value)
            shape = (len(dtypes),)
            # calc dtype
            dtype = self._calc_dtype_on_axis_1(a, dtypes)
            return self.new_series(
                inputs,
                shape=shape,
                dtype=dtype,
                index_value=index_value,
                name=name or dtypes.index.name,
            )
        elif q_val.ndim == 0 and self._axis == 1:
            index_value = a.index_value
            shape = (len(a),)
            # calc dtype
            dt = tensor_quantile(
                empty(a.shape[1], dtype=find_common_type(list(dtypes))),
                self._q,
                interpolation=self._interpolation,
                handle_non_numeric=not self._numeric_only,
            ).dtype
            return self.new_series(
                inputs,
                shape=shape,
                dtype=dt,
                index_value=index_value,
                name=name or index_value.name,
            )
        elif q_val.ndim == 1 and self._axis == 0:
            shape = (len(q_val), len(dtypes))
            index_value = parse_index(
                pd_index, *tokenize_objects, store_data=store_index_value
            )
            dtype_list = []
            for name in dtypes.index:
                dtype_list.append(
                    tensor_quantile(
                        tensor_from_series(a[name]),
                        self._q,
                        interpolation=self._interpolation,
                        handle_non_numeric=not self._numeric_only,
                    ).dtype
                )
            dtypes = pd.Series(dtype_list, index=dtypes.index)
            return self.new_dataframe(
                inputs,
                shape=shape,
                dtypes=dtypes,
                index_value=index_value,
                columns_value=parse_index(dtypes.index, store_data=True),
            )
        else:
            assert q_val.ndim == 1 and self._axis == 1
            shape = (len(q_val), a.shape[0])
            index_value = parse_index(
                pd_index, *tokenize_objects, store_data=store_index_value
            )
            pd_columns = a.index_value.to_pandas()
            dtype_list = np.full(len(pd_columns), self._calc_dtype_on_axis_1(a, dtypes))
            dtypes = pd.Series(dtype_list, index=pd_columns)
            return self.new_dataframe(
                inputs,
                shape=shape,
                dtypes=dtypes,
                index_value=index_value,
                columns_value=parse_index(
                    dtypes.index, store_data=True, key=a.index_value.key
                ),
            )

    def _call_series(self, a, inputs):
        if isinstance(self._q, TENSOR_TYPE):
            q_val = self._q
            index_val = pd.Index([], dtype=q_val.dtype)
            store_index_value = False
        else:
            q_val = np.asanyarray(self._q)
            index_val = pd.Index(q_val)
            store_index_value = True

        # get dtype by tensor
        a_t = astensor(a)
        self._dtype = dtype = tensor_quantile(
            a_t,
            self._q,
            interpolation=self._interpolation,
            handle_non_numeric=not self._numeric_only,
        ).dtype

        if q_val.ndim == 0:
            return self.new_scalar(inputs, dtype=dtype)
        else:
            return self.new_series(
                inputs,
                shape=q_val.shape,
                dtype=dtype,
                index_value=parse_index(
                    index_val,
                    a,
                    q_val,
                    self._interpolation,
                    type(self).__name__,
                    store_data=store_index_value,
                ),
                name=a.name,
            )

    def __call__(self, a, q_input=None):
        inputs = [a]
        if q_input is not None:
            inputs.append(q_input)
        if isinstance(a, DATAFRAME_TYPE):
            return self._call_dataframe(a, inputs)
        else:
            return self._call_series(a, inputs)

    @classmethod
    def _tile_dataframe(cls, op):
        from ...tensor.merge.stack import TensorStack

        df = op.outputs[0]
        if df.ndim == 1:
            if op.axis == 0:
                ts = []
                for name in df.index_value.to_pandas():
                    a = tensor_from_series(op.input[name])
                    t = tensor_quantile(
                        a,
                        op.q,
                        interpolation=op.interpolation,
                        handle_non_numeric=not op.numeric_only,
                    )
                    ts.append(t)
                try:
                    dtype = np.result_type(*[it.dtype for it in ts])
                except TypeError:
                    dtype = np.dtype(object)
                stack_op = TensorStack(axis=0, dtype=dtype)
                tr = stack_op(ts)
                r = series_from_tensor(
                    tr, index=df.index_value.to_pandas(), name=ts[0].op.q.item()
                )
            else:
                assert op.axis == 1
                empty_df = build_empty_df(op.input.dtypes)
                fields = empty_df._get_numeric_data().columns.tolist()
                t = tensor_from_dataframe(op.input[fields])
                tr = tensor_quantile(
                    t,
                    op.q,
                    axis=1,
                    interpolation=op.interpolation,
                    handle_non_numeric=not op.numeric_only,
                )
                r = series_from_tensor(tr, index=op.input.index, name=tr.op.q.item())
        else:
            assert df.ndim == 2
            if op.axis == 0:
                d = OrderedDict()
                for name in df.dtypes.index:
                    a = tensor_from_series(op.input[name])
                    t = tensor_quantile(
                        a,
                        op.q,
                        interpolation=op.interpolation,
                        handle_non_numeric=not op.numeric_only,
                    )
                    d[name] = t
                r = create_df(d, index=op.q)
            else:
                assert op.axis == 1
                empty_df = build_empty_df(op.input.dtypes)
                fields = empty_df._get_numeric_data().columns.tolist()
                t = tensor_from_dataframe(op.input[fields])
                tr = tensor_quantile(
                    t,
                    op.q,
                    axis=1,
                    interpolation=op.interpolation,
                    handle_non_numeric=not op.numeric_only,
                )
                if not op.input.index_value.has_value():
                    raise NotImplementedError
                # TODO(xuye.qin): use index=op.input.index when we support DataFrame.index
                r = dataframe_from_tensor(
                    tr, index=op.q, columns=op.input.index_value.to_pandas()
                )

        return (yield from recursive_tile(r))

    @classmethod
    def _tile_series(cls, op):
        a = tensor_from_series(op.input)
        t = tensor_quantile(
            a,
            op.q,
            interpolation=op.interpolation,
            handle_non_numeric=not op.numeric_only,
        )
        if isinstance(op.outputs[0], TENSOR_TYPE):
            r = t
        else:
            r = series_from_tensor(t, index=op.q, name=op.outputs[0].name)
        r = yield from recursive_tile(r)
        return [r]

    @classmethod
    def tile(cls, op):
        if isinstance(op.input, DATAFRAME_TYPE):
            tiled = yield from cls._tile_dataframe(op)
        else:
            tiled = yield from cls._tile_series(op)
        return tiled

def quantile_series(series, q=0.5, interpolation="linear"):
    """
    Return value at the given quantile.

    Parameters
    ----------
    q : float or array-like, default 0.5 (50% quantile)
        0 <= q <= 1, the quantile(s) to compute.
    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}

        This optional parameter specifies the interpolation method to use,
        when the desired quantile lies between two data points `i` and `j`:

            * linear: `i + (j - i) * fraction`, where `fraction` is the
              fractional part of the index surrounded by `i` and `j`.
            * lower: `i`.
            * higher: `j`.
            * nearest: `i` or `j` whichever is nearest.
            * midpoint: (`i` + `j`) / 2.

    Returns
    -------
    float or Series
        If ``q`` is an array or a tensor, a Series will be returned where the
        index is ``q`` and the values are the quantiles, otherwise
        a float will be returned.

    See Also
    --------
    core.window.Rolling.quantile
    numpy.percentile

    Examples
    --------
    >>> import mars.dataframe as md
    >>> s = md.Series([1, 2, 3, 4])
    >>> s.quantile(.5).execute()
    2.5
    >>> s.quantile([.25, .5, .75]).execute()
    0.25    1.75
    0.50    2.50
    0.75    3.25
    dtype: float64
    """

    if isinstance(q, ENTITY_TYPE):
        q = astensor(q)
        q_input = q
    else:
        q_input = None

    op = DataFrameQuantile(q=q, interpolation=interpolation, gpu=series.op.gpu)
    return op(series, q_input=q_input)

def quantile_dataframe(df, q=0.5, axis=0, numeric_only=True, interpolation="linear"):
    """
    Return values at the given quantile over requested axis.

    Parameters
    ----------
    q : float or array-like, default 0.5 (50% quantile)
        Value between 0 <= q <= 1, the quantile(s) to compute.
    axis : {0, 1, 'index', 'columns'} (default 0)
        Equals 0 or 'index' for row-wise, 1 or 'columns' for column-wise.
    numeric_only : bool, default True
        If False, the quantile of datetime and timedelta data will be
        computed as well.
    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}
        This optional parameter specifies the interpolation method to use,
        when the desired quantile lies between two data points `i` and `j`:
        * linear: `i + (j - i) * fraction`, where `fraction` is the
          fractional part of the index surrounded by `i` and `j`.
        * lower: `i`.
        * higher: `j`.
        * nearest: `i` or `j` whichever is nearest.
        * midpoint: (`i` + `j`) / 2.

    Returns
    -------
    Series or DataFrame
        If ``q`` is an array or a tensor, a DataFrame will be returned where the
          index is ``q``, the columns are the columns of self, and the
          values are the quantiles.
        If ``q`` is a float, a Series will be returned where the
          index is the columns of self and the values are the quantiles.

    See Also
    --------
    core.window.Rolling.quantile: Rolling quantile.
    numpy.percentile: Numpy function to compute the percentile.

    Examples
    --------
    >>> import mars.dataframe as md
    >>> df = md.DataFrame(np.array([[1, 1], [2, 10], [3, 100], [4, 100]]),
    ...                   columns=['a', 'b'])
    >>> df.quantile(.1).execute()
    a    1.3
    b    3.7
    Name: 0.1, dtype: float64

    >>> df.quantile([.1, .5]).execute()
           a     b
    0.1  1.3   3.7
    0.5  2.5  55.0

    Specifying `numeric_only=False` will also compute the quantile of
    datetime and timedelta data.

    >>> df = md.DataFrame({'A': [1, 2],
    ...                    'B': [md.Timestamp('2010'),
    ...                          md.Timestamp('2011')],
    ...                    'C': [md.Timedelta('1 days'),
    ...                          md.Timedelta('2 days')]})
    >>> df.quantile(0.5, numeric_only=False).execute()
    A                    1.5
    B    2010-07-02 12:00:00
    C        1 days 12:00:00
    Name: 0.5, dtype: object
    """
    if isinstance(q, ENTITY_TYPE):
        q = astensor(q)
        q_input = q
    else:
        q_input = None
    axis = validate_axis(axis, df)

    op = DataFrameQuantile(
        q=q,
        interpolation=interpolation,
        axis=axis,
        numeric_only=numeric_only,
        gpu=df.op.gpu,
    )
    return op(df, q_input=q_input)


import sys,os,math,string,time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from IPython.display import display, clear_output
from pyzillow.pyzillow import ZillowWrapper, GetDeepSearchResults
from sklearn.linear_model import ElasticNetCV
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn import ensemble, tree, linear_model
import seaborn as sns
import scipy.stats as st
import missingno as msno
import pickle
pd.options.display.max_rows = 200

df3 = pd.read_csv("downsized_7.csv")
df3['PTYPE'] = df3['PTYPE'].astype('O')

myfeat = ['key', 'PTYPE', 'LU', 'OWN_OCC', 'LAND_SF', 'YR_BUILT', 'YR_REMOD', 'GROSS_AREA', 'NUM_FLOORS','R_BLDG_STYL', 'R_ROOF_TYP', 'R_EXT_FIN', 'R_TOTAL_RMS', 'R_BDRMS','R_FULL_BTH', 'R_HALF_BTH', 'R_BTH_STYLE', 'R_KITCH', 'R_KITCH_STYLE','R_HEAT_TYP', 'R_AC', 'R_FPLACE', 'R_EXT_CND','R_INT_CND', 'R_INT_FIN', 'R_VIEW', 'R_TOTAL_BTH','MARKET_VALUE', "ZIPYR_TAX",'DIS0']
#df3.columns.to_series().groupby(df3.dtypes).groups

dumFeat = ['LU', 'R_BLDG_STYL', 'R_ROOF_TYP','R_EXT_FIN', 'R_BTH_STYLE', 'R_KITCH_STYLE', 'R_HEAT_TYP', 'R_AC','R_EXT_CND', 'R_OVRALL_CND', 'R_INT_CND', 'R_INT_FIN', 'R_VIEW', "PTYPE"]
df3 = pd.get_dummies(df3,columns=dumFeat)

all_features = df3.columns
index = np.argwhere(all_features=="MARKET_VALUE")
all_features = np.delete(all_features, index)
target_feature = "MARKET_VALUE"

#df3.columns.to_series().groupby(df3.dtypes).groups

my_feat = ['LU_R1', 'LU_R2', 'LU_R3', 'R_BLDG_STYL_BW', 'R_BLDG_STYL_CL',
        'R_BLDG_STYL_CN', 'R_BLDG_STYL_CP', 'R_BLDG_STYL_CV', 'R_BLDG_STYL_DK',
        'R_BLDG_STYL_DX', 'R_BLDG_STYL_OT', 'R_BLDG_STYL_RE', 'R_BLDG_STYL_RM',
        'R_BLDG_STYL_RN', 'R_BLDG_STYL_RR', 'R_BLDG_STYL_SD', 'R_BLDG_STYL_SL',
        'R_BLDG_STYL_TD', 'R_BLDG_STYL_TF', 'R_BLDG_STYL_TL', 'R_BLDG_STYL_VT',
        'R_ROOF_TYP_F', 'R_ROOF_TYP_G', 'R_ROOF_TYP_H', 'R_ROOF_TYP_L',
        'R_ROOF_TYP_M', 'R_ROOF_TYP_S', 'R_EXT_FIN_A', 'R_EXT_FIN_B',
        'R_EXT_FIN_C', 'R_EXT_FIN_F', 'R_EXT_FIN_G', 'R_EXT_FIN_M',
        'R_EXT_FIN_O', 'R_EXT_FIN_P', 'R_EXT_FIN_S', 'R_EXT_FIN_U',
        'R_EXT_FIN_V', 'R_EXT_FIN_W', 'R_BTH_STYLE_L', 'R_BTH_STYLE_M',
        'R_BTH_STYLE_N', 'R_BTH_STYLE_S', 'R_KITCH_STYLE_L', 'R_KITCH_STYLE_M',
        'R_KITCH_STYLE_N', 'R_KITCH_STYLE_S', 'R_HEAT_TYP_E', 'R_HEAT_TYP_F',
        'R_HEAT_TYP_N', 'R_HEAT_TYP_O', 'R_HEAT_TYP_P', 'R_HEAT_TYP_S',
        'R_HEAT_TYP_W', 'R_AC_C', 'R_AC_D', 'R_AC_N', 'R_AC_Y', 'R_EXT_CND_A',
        'R_EXT_CND_E', 'R_EXT_CND_F', 'R_EXT_CND_G', 'R_EXT_CND_P',
        'R_OVRALL_CND_A', 'R_OVRALL_CND_E', 'R_OVRALL_CND_F', 'R_OVRALL_CND_G',
        'R_OVRALL_CND_P', 'R_INT_CND_A', 'R_INT_CND_E', 'R_INT_CND_F',
        'R_INT_CND_G', 'R_INT_CND_P', 'R_INT_FIN_E', 'R_INT_FIN_N',
        'R_INT_FIN_S', 'R_VIEW_A', 'R_VIEW_E', 'R_VIEW_F', 'R_VIEW_G',
        'R_VIEW_P', 'PTYPE_101.0', 'PTYPE_104.0', 'PTYPE_105.0',
           'OWN_OCC','LAND_SF', 'YR_BUILT', 'YR_REMOD', 'GROSS_AREA',
        'NUM_FLOORS', 'R_TOTAL_RMS', 'R_BDRMS', 'R_FULL_BTH', 'R_HALF_BTH',
        'R_KITCH', 'R_FPLACE', 'R_TOTAL_BTH', 'DIS0', 'ZIPYR_TAX','key']
target_feat = "MARKET_VALUE"

df3.columns.to_series().groupby(df3.dtypes).groups

test19 = df3[df3["key"]==2019]
train19 = df3[df3["key"]!=2019]
X_train, X_test, Y_train, Y_test = train_test_split(train19[my_feat], train19[target_feat], test_size=0.2, random_state=1)

X_test = X_test.append(test19[my_feat]);
Y_test = Y_test.append(test19[target_feat]);

from sklearn.linear_model import ElasticNetCV
ElasticNetCVModel2 = ElasticNetCV(l1_ratio=[1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1],
                                  eps=0.0001, n_alphas=50, normalize=True, random_state=1,
                                  verbose=False, max_iter=1000,cv=8)

ElasticNetCVModel2.fit(X_train[my_feat],Y_train)

filename = 'finalized_model.sav'
pickle.dump(ElasticNetCVModel2, open(filename, 'wb'))

coef_val = pd.Series(ElasticNetCVModel2.coef_,my_feat)
print(coef_val)
#print(coef_val.sort_values(ascending=False))
#for index, feat in enumerate(X_train.columns):
#    print(index, feat, ElasticNetCVModel2.coef_[index])

print("train R^2 = ", ElasticNetCVModel2.score(X_train[my_feat],Y_train))
print("test R^2 = ", ElasticNetCVModel2.score(X_test[my_feat],Y_test))
print("L1_ratio = ", ElasticNetCVModel2.l1_ratio_)
#print("mse = ", ElasticNetCVModel2.mse_path_)

print(coef_val.sort_values(ascending=False))

#plt.scatter(X_test["GROSS_AREA"],Y_test)
#plt.scatter(X_test["GROSS_AREA"],ElasticNetCVModel2.predict(X_test[my_feat]))

#plt.scatter(X_test["GROSS_AREA"],Y_test)
#plt.scatter(X_test["GROSS_AREA"],ElasticNetCVModel2.predict(X_test[my_feat]))

df3["Residual"] =  -np.exp(ElasticNetCVModel2.predict(df3[my_feat])) + np.exp(df3["MARKET_VALUE"])
#RSS = (df3["Residual"]**2).sum()/len(df3)
#print(np.sqrt(RSS))

MAE = np.abs(df3["Residual"].values).sum()/len(df3)
print("\nmean absolute error (MAE) = ", MAE)


'''
Given a 2d grid map of '1's (land) and '0's (water), count the number of islands. An island is surrounded by water and is formed by connecting adjacent lands horizontally or vertically. You may assume all four edges of the grid are all surrounded by water.

Example 1:

Input:
11110
11010
11000
00000

Output: 1

Example 2:

Input:
11000
11000
00100
00011

Output: 3

'''

class Solution:
    def dfs(self, x: int, y: int, grid: List[List[str]]):
        if y < len(grid) and y >= 0 and x >= 0 and x < len(grid[y]):
            if grid[y][x] == '1':
                grid[y][x] = '2'
                self.dfs(x, y+1, grid)
                self.dfs(x+1, y, grid)
                self.dfs(x-1, y, grid)
                self.dfs(x, y-1, grid)

    def numIslands(self, grid: List[List[str]]) -> int:
        count = 0

        for rowIndex, row in enumerate(grid):
            for columnIndex, column in enumerate(row):
                if grid[rowIndex][columnIndex] == '1':
                    count = count + 1
                    self.dfs(columnIndex, rowIndex, grid)

        return count

'''
You want to go through every point in the 2D grid iteratively.
At every point run a dfs where you basically perform a recursive algorithm
that changes a '1' to a value differnt from '1' or '0' (in this case '2').
Then you will recurse in all 4 adjacent directions. This will make at most 1 island of 2's at every iterative step.
'''


from classes import Graph, Vertex
import heapq
from typing import *

def dijkstra(G: Graph, source: Vertex):
    source.set_distance(0)
    unvisitedQueue: list = [v for v in G]
    heapq.heapify(unvisitedQueue)
    while len(unvisitedQueue):
        current: Vertex = heapq.heappop(unvisitedQueue)
        current.set_visited()
        for nex in current.adjacent:
            if nex.visited:
                continue
            newDist = current.get_distance() + current.get_weight(nex)
            if newDist < nex.get_distance():
                nex.set_distance(newDist)
                nex.set_previous(current)
                print(
                    f"Updated: current={current.get_vertex_ID()},"
                    f"next={nex.get_vertex_ID()},"
                    f"newDist={newDist}"
                    )
        while len(unvisitedQueue):
            heapq.heappop(unvisitedQueue)
        unvisitedQueue: list = [
            v for v in G if not v.visited
            ]
        heapq.heapify(unvisitedQueue)

if __name__ == '__main__':
    g = Graph()
    for i in 'abcde':
        g.add_vertex(i)
    for v, w, c in zip('aacbcd', 'bcbede', [4, 1, 2, 4, 4, 4]):
        g.add_edge(v, w, c)
    print(g.get_edges())
    dijkstra(g, g.get_vertex('a'))


import logging
import time
from abc import abstractmethod
from copy import deepcopy
from enum import Enum, unique
from pprint import pformat

from .config import PktgenConf
from .logger import getLogger

# packet generator name
from .settings import PKTGEN, PKTGEN_DPDK, PKTGEN_IXIA, PKTGEN_IXIA_NETWORK, PKTGEN_TREX

# macro definition
TRANSMIT_CONT = "continuous"
TRANSMIT_M_BURST = "multi_burst"
TRANSMIT_S_BURST = "single_burst"

@unique
class STAT_TYPE(Enum):
    RX = "rx"
    TXRX = "txrx"

class PacketGenerator(object):
    """
    Basic class for packet generator, define basic function for each kinds of
    generators
    """

    def __init__(self, tester):
        self.logger = getLogger(PKTGEN)
        self.tester = tester
        self.__streams = []
        self._ports_map = []
        self.pktgen_type = None

    def _prepare_generator(self):
        raise NotImplementedError

    def prepare_generator(self):
        self._prepare_generator()

    def _get_port_pci(self, port_id):
        raise NotImplementedError

    def _convert_pktgen_port(self, port_id):
        """
        :param port_id:
            index of a port in packet generator tool
        """
        try:
            gen_pci = self._get_port_pci(port_id)
            if not gen_pci:
                msg = "can't get port {0} pci address".format(port_id)
                raise Exception(msg)
            for port_idx, info in enumerate(self.tester.ports_info):
                if "pci" not in info or info["pci"] == "N/A":
                    return -1
                tester_pci = info["pci"]
                if tester_pci == gen_pci:
                    msg = "gen port {0} map test port {1}".format(port_id, port_idx)
                    self.logger.debug(msg)
                    return port_idx
            else:
                port = -1
        except Exception as e:
            port = -1

        return port

    def _get_gen_port(self, tester_pci):
        raise NotImplementedError

    def _convert_tester_port(self, port_id):
        """
        :param port_id:
            index of a port in dts tester ports info
        """
        try:
            info = self.tester.ports_info[port_id]
            # limit to nic port, not including ixia port
            if "pci" not in info or info["pci"] == "N/A":
                return -1
            tester_pci = info["pci"]
            port = self._get_gen_port(tester_pci)
            msg = "test port {0} map gen port {1}".format(port_id, port)
            self.logger.debug(msg)
        except Exception as e:
            port = -1

        return port

    def add_stream(self, tx_port, rx_port, pcap_file):
        pktgen_tx_port = self._convert_tester_port(tx_port)
        pktgen_rx_port = self._convert_tester_port(rx_port)

        stream_id = len(self.__streams)
        stream = {
            "tx_port": pktgen_tx_port,
            "rx_port": pktgen_rx_port,
            "pcap_file": pcap_file,
        }
        self.__streams.append(stream)

        return stream_id

    def add_streams(self, streams):
        """' a group of streams"""
        raise NotImplementedError

    def config_stream(self, stream_id=0, opts={}):
        if self._check_options(opts) is not True:
            self.logger.error("Failed to configure stream[%d]" % stream_id)
            return
        stream = self.__streams[stream_id]
        stream["options"] = opts

    def config_streams(self, stream_ids, nic, frame_size, port_num):
        """all streams using the default option"""
        raise NotImplementedError

    def get_streams(self):
        return self.__streams

    def _clear_streams(self):
        raise NotImplementedError

    def clear_streams(self):
        """clear streams"""
        self._clear_streams()
        self.__streams = []

    def _set_stream_rate_percent(self, rate_percent):
        """set all streams' rate percent"""
        if not self.__streams:
            return
        for stream in self.__streams:
            stream["options"]["stream_config"]["rate"] = rate_percent

    def _set_stream_pps(self, pps):
        """set all streams' pps"""
        if not self.__streams:
            return
        for stream in self.__streams:
            stream["options"]["stream_config"]["pps"] = pps

    def reset_streams(self):
        self.__streams = []

    def __warm_up_pktgen(self, stream_ids, options, delay):
        """run warm up traffic before start main traffic"""
        if not delay:
            return
        msg = "{1} packet generator: run traffic {0}s to warm up ... ".format(
            delay, self.pktgen_type
        )
        self.logger.info(msg)
        self._start_transmission(stream_ids, options)
        time.sleep(delay)
        self._stop_transmission(stream_ids)

    def __get_single_throughput_statistic(self, stream_ids, stat_type=None):
        bps_rx = []
        pps_rx = []
        bps_tx = []
        pps_tx = []
        used_rx_port = []
        msg = "begin get port statistic ..."
        self.logger.info(msg)
        for stream_id in stream_ids:
            if self.__streams[stream_id]["rx_port"] not in used_rx_port:
                bps_rate, pps_rate = self._retrieve_port_statistic(
                    stream_id, "throughput"
                )
                used_rx_port.append(self.__streams[stream_id]["rx_port"])
                if stat_type and stat_type is STAT_TYPE.TXRX:
                    bps_tx.append(bps_rate[0])
                    pps_tx.append(pps_rate[0])

                if isinstance(bps_rate, tuple) and isinstance(pps_rate, tuple):
                    bps_rx.append(bps_rate[1])
                    pps_rx.append(pps_rate[1])
                else:
                    bps_rx.append(bps_rate)
                    pps_rx.append(pps_rate)
        if stat_type and stat_type is STAT_TYPE.TXRX:
            bps_tx_total = self._summary_statistic(bps_tx)
            pps_tx_total = self._summary_statistic(pps_tx)
            bps_rx_total = self._summary_statistic(bps_rx)
            pps_rx_total = self._summary_statistic(pps_rx)
            self.logger.info(
                "throughput: pps_tx %f, bps_tx %f" % (pps_tx_total, bps_tx_total)
            )
            self.logger.info(
                "throughput: pps_rx %f, bps_rx %f" % (pps_rx_total, bps_rx_total)
            )

            return (bps_tx_total, bps_rx_total), (pps_tx_total, pps_rx_total)
        else:
            bps_rx_total = self._summary_statistic(bps_rx)
            pps_rx_total = self._summary_statistic(pps_rx)
            self.logger.info(
                "throughput: pps_rx %f, bps_rx %f" % (pps_rx_total, bps_rx_total)
            )

            return bps_rx_total, pps_rx_total

    def __get_multi_throughput_statistic(
        self, stream_ids, duration, interval, callback=None, stat_type=None
    ):
        """
        duration: traffic duration (second)
        interval: interval of get throughput statistics (second)
        callback: a callback method of suite, which is used to do some actions
            during traffic lasting.

        Return: a list of throughput instead of a single tuple of pps/bps rate
        """
        time_elapsed = 0
        stats = []
        while time_elapsed < duration:
            time.sleep(interval)
            stats.append(self.__get_single_throughput_statistic(stream_ids, stat_type))
            if callback and callable(callback):
                callback()
            time_elapsed += interval
        return stats

    def measure_throughput(self, stream_ids=[], options={}):
        """
        Measure throughput on each tx ports

        options usage:
            rate:
                port rate percent, float(0--100). Default value is 100.

            delay:
                warm up time before start main traffic. If it is set, it will start
                a delay time traffic to make sure packet generator under good status.
                Warm up flow is ignored by default.

            interval:
                a interval time of get throughput statistic (second)
                If set this key value, pktgen will return several throughput statistic
                data within a duration traffic. If not set this key value, only
                return one statistic data. It is ignored by default.

            callback:
                this key works with ``interval`` key. If it is set, the callback
                of suite level will be executed after getting throughput statistic.
                callback method should define as below, don't add sleep in this method.

                def callback(self):
                    xxxx()

            duration:
                traffic lasting time(second). Default value is 10 second.

            stat_type(for trex only):
                STAT_TYPE.RX  return (rx bps, rx_pps)
                STAT_TYPE.TXRX return ((tx bps, rx_bps), (tx pps, rx_pps))
        """
        interval = options.get("interval")
        callback = options.get("callback")
        duration = options.get("duration") or 10
        delay = options.get("delay")
        if self.pktgen_type == PKTGEN_TREX:
            stat_type = options.get("stat_type") or STAT_TYPE.RX
        else:
            if options.get("stat_type") is not None:
                msg = (
                    "'stat_type' option is only for trex, "
                    "should not set when use other pktgen tools"
                )
                raise Exception(msg)
            stat_type = STAT_TYPE.RX
        self._prepare_transmission(stream_ids=stream_ids)
        # start warm up traffic
        self.__warm_up_pktgen(stream_ids, options, delay)
        # main traffic
        self._start_transmission(stream_ids, options)
        # keep traffic within a duration time and get throughput statistic
        if interval and duration:
            stats = self.__get_multi_throughput_statistic(
                stream_ids, duration, interval, callback, stat_type
            )
        else:
            time.sleep(duration)
            stats = self.__get_single_throughput_statistic(stream_ids, stat_type)
        self._stop_transmission(stream_ids)
        return stats

    def _measure_loss(self, stream_ids=[], options={}):
        """
        Measure lost rate on each tx/rx ports
        """
        delay = options.get("delay")
        duration = options.get("duration") or 10
        throughput_stat_flag = options.get("throughput_stat_flag") or False
        self._prepare_transmission(stream_ids=stream_ids)
        # start warm up traffic
        self.__warm_up_pktgen(stream_ids, options, delay)
        # main traffic
        self._start_transmission(stream_ids, options)
        # keep traffic within a duration time
        time.sleep(duration)
        if throughput_stat_flag:
            _throughput_stats = self.__get_single_throughput_statistic(stream_ids)
        self._stop_transmission(None)
        result = {}
        used_rx_port = []
        for stream_id in stream_ids:
            port_id = self.__streams[stream_id]["rx_port"]
            if port_id in used_rx_port:
                continue
            stats = self._retrieve_port_statistic(stream_id, "loss")
            tx_pkts, rx_pkts = stats
            lost_p = tx_pkts - rx_pkts
            if tx_pkts <= 0:
                loss_rate = 0
            else:
                loss_rate = float(lost_p) / float(tx_pkts)
                if loss_rate < 0:
                    loss_rate = 0
            result[port_id] = (loss_rate, tx_pkts, rx_pkts)
        if throughput_stat_flag:
            return result, _throughput_stats
        else:
            return result

    def measure_loss(self, stream_ids=[], options={}):
        """
        options usage:
            rate:
                port rate percent, float(0--100). Default value is 100.

            delay:
                warm up time before start main traffic. If it is set, it will
                start a delay time traffic to make sure packet generator
                under good status. Warm up flow is ignored by default.

            duration:
                traffic lasting time(second). Default value is 10 second.
        """
        result = self._measure_loss(stream_ids, options)
        # here only to make sure that return value is the same as dts/etgen format
        # In real testing scenario, this method can offer more data than it
        return list(result.values())[0]

    def _measure_rfc2544_ixnet(self, stream_ids=[], options={}):
        """
        used for ixNetwork
        """
        # main traffic
        self._prepare_transmission(stream_ids=stream_ids)
        self._start_transmission(stream_ids, options)
        self._stop_transmission(None)
        # parsing test result
        stats = self._retrieve_port_statistic(stream_ids[0], "rfc2544")
        tx_pkts, rx_pkts, pps = stats
        lost_p = tx_pkts - rx_pkts
        if tx_pkts <= 0:
            loss_rate = 0
        else:
            loss_rate = float(lost_p) / float(tx_pkts)
            if loss_rate < 0:
                loss_rate = 0
        result = (loss_rate, tx_pkts, rx_pkts, pps)
        return result

    def measure_latency(self, stream_ids=[], options={}):
        """
        Measure latency on each tx/rx ports

        options usage:
            rate:
                port rate percent, float(0--100). Default value is 100.

            delay:
                warm up time before start main traffic. If it is set, it will
                start a delay time transmission to make sure packet generator
                under correct status. Warm up flow is ignored by default.

            duration:
                traffic lasting time(second). Default value is 10 second.
        """
        delay = options.get("delay")
        duration = options.get("duration") or 10
        self._prepare_transmission(stream_ids=stream_ids, latency=True)
        # start warm up traffic
        self.__warm_up_pktgen(stream_ids, options, delay)
        # main traffic
        self._start_transmission(stream_ids, options)
        # keep traffic within a duration time
        time.sleep(duration)
        self._stop_transmission(None)

        result = {}
        used_rx_port = []
        for stream_id in stream_ids:
            port_id = self.__streams[stream_id]["rx_port"]
            if port_id in used_rx_port:
                continue
            stats = self._retrieve_port_statistic(stream_id, "latency")
            result[port_id] = stats
        self.logger.info(result)

        return result

    def _check_loss_rate(self, result, permit_loss_rate):
        """
        support multiple link peer, if any link peer loss rate happen set
        return value to False
        """
        for port_id, _result in result.items():
            loss_rate, _, _ = _result
            if loss_rate > permit_loss_rate:
                return False
        else:
            return True

    def measure_rfc2544(self, stream_ids=[], options={}):
        """check loss rate with rate percent dropping

        options usage:
            rate:
                port rate percent at first round testing(0 ~ 100), default is 100.

            pdr:
                permit packet drop rate, , default is 0.

            drop_step:
                port rate percent drop step(0 ~ 100), default is 1.

            delay:
                warm up time before start main traffic. If it is set, it will
                start a delay time traffic to make sure packet generator
                under good status. Warm up flow is ignored by default.

            duration:
                traffic lasting time(second). Default value is 10 second.
        """
        loss_rate_table = []
        rate_percent = options.get("rate") or float(100)
        permit_loss_rate = options.get("pdr") or 0
        self.logger.info("allow loss rate: %f " % permit_loss_rate)
        rate_step = options.get("drop_step") or 1
        result = self._measure_loss(stream_ids, options)
        status = self._check_loss_rate(result, permit_loss_rate)
        loss_rate_table.append([rate_percent, result])
        # if first time loss rate is ok, ignore left flow
        if status:
            # return data is the same with dts/etgen format
            # In fact, multiple link peer have multiple loss rate value,
            # here only pick one
            tx_num, rx_num = list(result.values())[0][1:]
            return rate_percent, tx_num, rx_num
        _options = deepcopy(options)
        # if warm up option  'delay' is set, ignore it in next work flow
        if "delay" in _options:
            _options.pop("delay")
        if "rate" in _options:
            _options.pop("rate")
        while not status and rate_percent > 0:
            rate_percent = rate_percent - rate_step
            if rate_percent <= 0:
                msg = "rfc2544 run under zero rate"
                self.logger.warning(msg)
                break
            self._clear_streams()
            # set stream rate percent to custom value
            self._set_stream_rate_percent(rate_percent)
            # run loss rate testing
            result = self._measure_loss(stream_ids, _options)
            loss_rate_table.append([rate_percent, result])
            status = self._check_loss_rate(result, permit_loss_rate)
        self.logger.info(pformat(loss_rate_table))
        self.logger.info("zero loss rate percent is %f" % rate_percent)
        # use last result as return data to keep the same with dts/etgen format
        # In fact, multiple link peer have multiple loss rate value,
        # here only pick one
        last_result = loss_rate_table[-1]
        rate_percent = last_result[0]
        tx_num, rx_num = list(last_result[1].values())[0][1:]
        return rate_percent, tx_num, rx_num

    def measure_rfc2544_with_pps(self, stream_ids=[], options={}):
        """
        check loss rate with pps bisecting.(not implemented)

        Currently, ixia/trex use rate percent to control port flow rate,
        pps not supported.
        """
        max_pps = options.get("max_pps")
        min_pps = options.get("min_pps")
        step = options.get("step") or 10000
        permit_loss_rate = options.get("permit_loss_rate") or 0.0001
        # traffic parameters
        loss_pps_table = []
        pps = traffic_pps_max = max_pps
        traffic_pps_min = min_pps

        while True:
            # set stream rate percent to custom value
            self._set_stream_pps(pps)
            # run loss rate testing
            _options = deepcopy(options)
            result = self._measure_loss(stream_ids, _options)
            loss_pps_table.append([pps, result])
            status = self._check_loss_rate(result, permit_loss_rate)
            if status:
                traffic_pps_max = pps
            else:
                traffic_pps_min = pps
            if traffic_pps_max - traffic_pps_min < step:
                break
            pps = (traffic_pps_max - traffic_pps_min) / 2 + traffic_pps_min

        self.logger.info("zero loss pps is %f" % pps)
        # use last result as return data to keep the same with dts/etgen format
        # In fact, multiple link peer have multiple loss rate value,
        # here only pick one
        return list(loss_pps_table[-1][1].values())[0]

    def measure_rfc2544_dichotomy(self, stream_ids=[], options={}):
        """check loss rate using dichotomy algorithm

        options usage:
            delay:
                warm up time before start main traffic. If it is set, it will
                start a delay time traffic to make sure packet generator
                under good status. Warm up flow is ignored by default.

            duration:
                traffic lasting time(second). Default value is 10 second.

            min_rate:
                lower bound rate percent , default is 0.

            max_rate:
                upper bound rate percent , default is 100.

            pdr:
                permit packet drop rate(<1.0), default is 0.

            accuracy :
                dichotomy algorithm accuracy, default 0.001.
        """
        if self.pktgen_type == PKTGEN_IXIA_NETWORK:
            return self._measure_rfc2544_ixnet(stream_ids, options)

        max_rate = options.get("max_rate") or 100.0
        min_rate = options.get("min_rate") or 0.0
        accuracy = options.get("accuracy") or 0.001
        permit_loss_rate = options.get("pdr") or 0.0
        duration = options.get("duration") or 10.0
        throughput_stat_flag = options.get("throughput_stat_flag") or False
        # start warm up traffic
        delay = options.get("delay")
        _options = {"duration": duration}
        if delay:
            self._prepare_transmission(stream_ids=stream_ids)
            self.__warm_up_pktgen(stream_ids, _options, delay)
            self._clear_streams()
        # traffic parameters for dichotomy algorithm
        loss_rate_table = []
        hit_result = None
        hit_rate = 0
        rate = traffic_rate_max = max_rate
        traffic_rate_min = min_rate
        while True:
            # run loss rate testing
            _options = {
                "throughput_stat_flag": throughput_stat_flag,
                "duration": duration,
            }
            result = self._measure_loss(stream_ids, _options)
            loss_rate_table.append([rate, result])
            status = self._check_loss_rate(
                result[0] if throughput_stat_flag else result, permit_loss_rate
            )
            # if upper bound rate percent hit, quit the left flow
            if rate == max_rate and status:
                hit_result = result
                hit_rate = rate
                break
            # if lower bound rate percent not hit, quit the left flow
            if rate == min_rate and not status:
                break
            if status:
                traffic_rate_min = rate
                hit_result = result
                hit_rate = rate
            else:
                traffic_rate_max = rate
            if traffic_rate_max - traffic_rate_min < accuracy:
                break
            rate = (traffic_rate_max - traffic_rate_min) / 2 + traffic_rate_min
            self._clear_streams()
            # set stream rate percent to custom value
            self._set_stream_rate_percent(rate)

        if throughput_stat_flag:
            if not hit_result or not hit_result[0]:
                msg = (
                    "expected permit loss rate <{0}> "
                    "not between rate {1} and rate {2}"
                ).format(permit_loss_rate, max_rate, min_rate)
                self.logger.error(msg)
                self.logger.info(pformat(loss_rate_table))
                ret_value = 0, result[0][0][1], result[0][0][2], 0
            else:
                self.logger.debug(pformat(loss_rate_table))
                ret_value = (
                    hit_rate,
                    hit_result[0][0][1],
                    hit_result[0][0][2],
                    hit_result[1][1],
                )
        else:
            if not hit_result:
                msg = (
                    "expected permit loss rate <{0}> "
                    "not between rate {1} and rate {2}"
                ).format(permit_loss_rate, max_rate, min_rate)
                self.logger.error(msg)
                self.logger.info(pformat(loss_rate_table))
                ret_value = 0, result[0][1], result[0][2]
            else:
                self.logger.debug(pformat(loss_rate_table))
                ret_value = hit_rate, hit_result[0][1], hit_result[0][2]
        self.logger.info("zero loss rate is %f" % hit_rate)

        return ret_value

    def measure(self, stream_ids, traffic_opt):
        """
        use as an unify interface method for packet generator
        """
        method = traffic_opt.get("method")
        if method == "throughput":
            result = self.measure_throughput(stream_ids, traffic_opt)
        elif method == "latency":
            result = self.measure_latency(stream_ids, traffic_opt)
        elif method == "loss":
            result = self.measure_loss(stream_ids, traffic_opt)
        elif method == "rfc2544":
            result = self.measure_rfc2544(stream_ids, traffic_opt)
        elif method == "rfc2544_with_pps":
            result = self.measure_rfc2544_with_pps(stream_ids, traffic_opt)
        elif method == "rfc2544_dichotomy":
            result = self.measure_rfc2544_dichotomy(stream_ids, traffic_opt)
        else:
            result = None

        return result

    def _summary_statistic(self, array=[]):
        """
        Summary all values in statistic array
        """
        summary = 0.000
        for value in array:
            summary += value

        return summary

    def _get_stream(self, stream_id):
        return self.__streams[stream_id]

    def _get_generator_conf_instance(self):
        conf_inst = PktgenConf(self.pktgen_type)
        pktgen_inst_type = conf_inst.pktgen_conf.get_sections()
        if len(pktgen_inst_type) < 1:
            msg = (
                "packet generator <{0}> has no configuration " "in pktgen.cfg"
            ).format(self.pktgen_type)
            raise Exception(msg)
        return conf_inst

    @abstractmethod
    def _prepare_transmission(self, stream_ids=[], latency=False):
        pass

    @abstractmethod
    def _start_transmission(self, stream_ids, options={}):
        pass

    @abstractmethod
    def _stop_transmission(self, stream_id):
        pass

    @abstractmethod
    def _retrieve_port_statistic(self, stream_id, mode):
        pass

    @abstractmethod
    def _check_options(self, opts={}):
        pass

    @abstractmethod
    def quit_generator(self):
        pass

class DpdkPacketGenerator(PacketGenerator):
    pass  # not implemented


#!/usr/bin/python3

import datetime
import os

import bme280
import boto3
import smbus2
from dotenv import main as dotenv

dotenv.load_dotenv()

s3_access_key = os.environ.get('S3ACCESSKEY')
s3_secret_key = os.environ.get('S3SECRETKEY')
location = os.environ.get('LOCATION')
address = int(os.environ.get('ADDRESS'), 0)

port = 1
bus = smbus2.SMBus(port)

filename = f'{location}-{datetime.datetime.now().strftime("%Y%m%d")}.csv'
targetdir = f'{os.path.dirname(os.path.abspath(__file__))}/data'
target = f'{targetdir}/{filename}'

bme280.load_calibration_params(bus, address)

if not os.path.exists(target):
    if not os.path.exists(targetdir):
        os.mkdir(targetdir)
    with open(target, "a") as f:
        f.write("timestamp,temp,humidity,airpressure\n")

with open(target, "a+") as f:
    bme280_data = bme280.sample(bus, address)
    timenow = bme280_data.timestamp.astimezone().strftime('%Y/%m/%d-%H:%M:%S')
    ambient_temperature = bme280_data.temperature
    humidity = bme280_data.humidity
    pressure = bme280_data.pressure
    f.write(f"{timenow},{ambient_temperature},{humidity},{pressure}\n")

with open(file=target, mode='rb') as f:
    bucket_name = 'tsutsui-test'
    s3 = boto3.client(
        "s3", aws_access_key_id=s3_access_key, aws_secret_access_key=s3_secret_key
    )

    s3.put_object(Bucket=bucket_name, Body=f, Key=filename)


"""empty message

Revision ID: 4f13c0b30718
Revises: 3ab1ffe0b504
Create Date: 2019-08-03 16:08:33.121530

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '4f13c0b30718'
down_revision = '3ab1ffe0b504'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_unique_constraint(None, 'question', ['PID'])
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(None, 'question', type_='unique')
    # ### end Alembic commands ###


import struct
import numpy as np
# 读取图片
import torch

def load_image_fromfile(filename):
    with open(filename, 'br') as fd:
        # 读取图像的信息
        header_buf = fd.read(16)   # 16字节，4个int整数
        # 按照字节解析头信息（具体参考python SL的struct帮助）  解包
        magic_, nums_, width_, height_ = struct.unpack('>iiii', header_buf)  # 解析成四个整数：>表示大端字节序，i表示4字节整数
        # 保存成ndarray对象
        imgs_ = np.fromfile(fd, dtype=np.uint8)
        imgs_ = imgs_.reshape(nums_, height_, width_)
    return imgs_

# 读取标签
def load_label_fromfile(filename):
    with open(filename, 'br') as fd:
        header_buf = fd.read(8)
        magic, nums = struct.unpack('>ii' ,header_buf)
        labels_ = np.fromfile(fd, np.uint8)
    return labels_

def load_data():
    # 读取训练数据集
    # 图片
    train_x = load_image_fromfile('data/train-images.idx3-ubyte')
    train_y = load_label_fromfile('data/train-labels.idx1-ubyte')
    # print(train_x.shape)# N数量 H W

    # 读取测试数据集
    # 图片
    test_x = load_image_fromfile('data/t10k-images.idx3-ubyte')
    test_y = load_label_fromfile('data/t10k-labels.idx1-ubyte')

    x = torch.Tensor(train_x).view(train_x.shape[0], 1, train_x.shape[1], train_x.shape[2])
    y = torch.LongTensor(train_y)

    t_x = torch.Tensor(test_x).view(test_x.shape[0], 1, test_x.shape[1], test_x.shape[2])
    t_y = torch.LongTensor(test_y)

    # 使用torch封装数据
    train_dataset = torch.utils.data.TensorDataset(x, y)
    test_dataset = torch.utils.data.TensorDataset(t_x, t_y)

    # 数据随机加载 按批切分   （数据加载器）
    train_loader = torch.utils.data.DataLoader(dataset=train_dataset, shuffle=True, batch_size=2000)
    test_loader = torch.utils.data.DataLoader(dataset=test_dataset, shuffle=True, batch_size=10000)

    return train_loader, test_loader


class CursoRepository:

    def __init__(self, connection):
        self.connection = connection

    def insert(self, curso):
        mycursor = self.connection.cursor()
        sql = "INSERT INTO curso (turno, codigo_universidade, codigo, codigo_modalidade) VALUES (%s, %s, %s, %s) RETURNING id"
        vals = (curso.getValues())
        mycursor.execute(sql, vals)
        self.connection.commit()
        curso.id = mycursor.fetchone()[0]
        return curso

    def findOrInsertIfNotExist(self, curso):
        findCurso = self.findByCodigos(curso.codigo, curso.codigoUniversidade)
        if findCurso is None:
            return self.insert(curso)
        else:
            curso.id = findCurso[0]
            return curso

    def findByCodigos(self, codigoCurso, codigoUniversidade):
        mycursor = self.connection.cursor()
        sql = "SELECT * FROM curso WHERE codigo = {} AND codigo_universidade = {}".format(codigoCurso, codigoUniversidade)
        mycursor.execute(sql)
        rows = mycursor.fetchall()
        if rows.__len__() > 0:
            return rows[0]
        else:
            return None


import PyQt5
from PyQt5 import QtCore, QtGui, QtWidgets

## class Ui_RecordingTutorial_Window
#  This class contains the recording tutorial window design
class Ui_RecordingTutorial_Window(object):
    def setupUi(self, RecordingTutorial_Window):
        RecordingTutorial_Window.setObjectName("RecordingTutorial_Window")
        RecordingTutorial_Window.resize(900, 600)

        ## Label containing the window's title
        self.RecordingTutorial_Title_Label = QtWidgets.QLabel(RecordingTutorial_Window)
        self.RecordingTutorial_Title_Label.setGeometry(QtCore.QRect(300, 20, 300, 50))

        font = QtGui.QFont()
        font.setPointSize(24)
        font.setBold(True)
        font.setWeight(75)
        self.RecordingTutorial_Title_Label.setFont(font)

        self.RecordingTutorial_Title_Label.setFrameShape(QtWidgets.QFrame.Panel)
        self.RecordingTutorial_Title_Label.setFrameShadow(QtWidgets.QFrame.Raised)
        self.RecordingTutorial_Title_Label.setLineWidth(4)

        self.RecordingTutorial_Title_Label.setAlignment(QtCore.Qt.AlignCenter)

        self.RecordingTutorial_Title_Label.setObjectName("RecordingTutorial_Title_Label")

        ## Label that contains the informative text to display to the user
        self.RecordingTutorial_Text_Label = QtWidgets.QLabel(RecordingTutorial_Window)
        self.RecordingTutorial_Text_Label.setGeometry(QtCore.QRect(100, 100, 700, 400))

        font = QtGui.QFont()
        font.setPointSize(18)
        self.RecordingTutorial_Text_Label.setFont(font)

        self.RecordingTutorial_Text_Label.setAlignment(QtCore.Qt.AlignHCenter|QtCore.Qt.AlignTop)
        self.RecordingTutorial_Text_Label.setWordWrap(True)

        self.RecordingTutorial_Text_Label.setObjectName("RecordingTutorial_Text_Label")

        ## Button to finish the tutorial
        self.Continue_Button = QtWidgets.QPushButton(RecordingTutorial_Window)
        self.Continue_Button.setGeometry(QtCore.QRect(375, 520, 150, 40))

        font = QtGui.QFont()
        font.setPointSize(16)
        font.setBold(True)
        font.setWeight(75)
        self.Continue_Button.setFont(font)

        self.Continue_Button.setObjectName("Continue_Button")

        self.retranslateUi(RecordingTutorial_Window)
        QtCore.QMetaObject.connectSlotsByName(RecordingTutorial_Window)

    ## function retranslateUi
    #  Adds text to the window's elements
    #  @param self The object pointer
    #  @param RecordingTutorial_Window The window whose elements's text must be modified
    def retranslateUi(self, RecordingTutorial_Window):
        _translate = QtCore.QCoreApplication.translate
        RecordingTutorial_Window.setWindowTitle(_translate("RecordingTutorial_Window", "Form"))
        self.RecordingTutorial_Title_Label.setText(_translate("RecordingTutorial_Window", "Recording tutorial"))
        self.RecordingTutorial_Text_Label.setText(_translate("RecordingTutorial_Window", "
In the Recording Window, you initially have two options: either you go back to the Configuration Window or you click the "Play" button, which will trigger a 10-second countdown.

At the end of it, the recording will start and the image corresponding to the gesture to perform will be displayed: images will be updated every 30 seconds.

When the recording has started you can manually stop it with the "Stop" button.
"))
        self.Continue_Button.setText(_translate("RecordingTutorial_Window", "Ok, I got it!"))


from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import str
from builtins import object
from past.builtins import basestring

import re
import types

import datalab.utils

from . import _utils

class SqlStatement(object):
  """A helper class for wrapping and manipulating SQL statements.
  """

  def __init__(self, sql, module=None):
    """ Initializes the SqlStatement.

    Args:
      sql: a string containing a SQL query with optional variable references.
      module: if defined in a %%sql cell, the parent SqlModule object for the SqlStatement.
    """
    self._sql = sql
    self._module = module

  def __str__(self):
    """Creates a string representation of this object.

    Returns:
      The string representation of this object.
    """
    return self._sql

  def __repr__(self):
    """Creates a friendly representation of this object.

    Returns:
      The friendly representation of this object.
    """
    return self._sql

  @property
  def sql(self):
    """ The (unexpanded) SQL for the SqlStatement. """
    return self._sql

  @property
  def module(self):
    """ The parent SqlModule for the SqlStatement, if any. """
    return self._module

  @staticmethod
  def _find_recursive_dependencies(sql, values, code, resolved_vars, resolving_vars=None):
    """ Recursive helper method for expanding variables including transitive dependencies.

    Placeholders in SQL are represented as $<name>. If '$' must appear within
    the SQL statement literally, then it can be escaped as '$$'.

    Args:
      sql: the raw SQL statement with named placeholders.
      values: the user-supplied dictionary of name/value pairs to use for placeholder values.
      code: an array of referenced UDFs found during expansion.
      resolved_vars: a ref parameter for the variable references completely resolved so far.
      resolving_vars: a ref parameter for the variable(s) we are currently resolving; if we see
          a dependency again that is in this set we know we have a circular reference.
    Returns:
      The formatted SQL statement with placeholders replaced with their values.
    Raises:
      Exception if a placeholder was found in the SQL statement, but did not
      have a corresponding argument value.
    """

    # Get the set of $var references in this SQL.
    dependencies = SqlStatement._get_dependencies(sql)
    for dependency in dependencies:
      # Now we check each dependency. If it is in complete - i.e., we have an expansion
      # for it already - we just continue.
      if dependency in resolved_vars:
        continue
      # Look it up in our resolution namespace dictionary.
      dep = datalab.utils.get_item(values, dependency)
      # If it is a SQL module, get the main/last query from the module, so users can refer
      # to $module. Useful especially if final query in module has no DEFINE QUERY <name> part.
      if isinstance(dep, types.ModuleType):
        dep = _utils.get_default_query_from_module(dep)
      # If we can't resolve the $name, give up.
      if dep is None:
        raise Exception("Unsatisfied dependency $%s" % dependency)
      # If it is a SqlStatement, it may have its own $ references in turn; check to make
      # sure we don't have circular references, and if not, recursively expand it and add
      # it to the set of complete dependencies.
      if isinstance(dep, SqlStatement):
        if resolving_vars is None:
          resolving_vars = []
        elif dependency in resolving_vars:
          # Circular dependency
          raise Exception("Circular dependency in $%s" % dependency)
        resolving_vars.append(dependency)
        SqlStatement._find_recursive_dependencies(dep._sql, values, code, resolved_vars,
                                                  resolving_vars)
        resolving_vars.pop()
        resolved_vars[dependency] = SqlStatement(dep._sql)
      else:
        resolved_vars[dependency] = dep

  @staticmethod
  def _escape_string(s):
    return '"' + s.replace('"', '\\"') + '"'

  @staticmethod
  def format(sql, args=None):
    """ Resolve variable references in a query within an environment.

    This computes and resolves the transitive dependencies in the query and raises an
    exception if that fails due to either undefined or circular references.

    Args:
      sql: query to format.
      args: a dictionary of values to use in variable expansion.

    Returns:
      The resolved SQL text with variables expanded.

    Raises:
      Exception on failure.
    """
    resolved_vars = {}
    code = []
    SqlStatement._find_recursive_dependencies(sql, args, code=code,
                                              resolved_vars=resolved_vars)

    # Rebuild the SQL string, substituting just '$' for escaped $ occurrences,
    # variable references substituted with their values, or literal text copied
    # over as-is.
    parts = []
    for (escape, placeholder, _, literal) in SqlStatement._get_tokens(sql):
      if escape:
        parts.append('$')
      elif placeholder:
        variable = placeholder[1:]
        try:
          value = resolved_vars[variable]
        except KeyError as e:
          raise Exception('Invalid sql. Unable to substitute $%s.' % e.args[0])

        if isinstance(value, types.ModuleType):
          value = _utils.get_default_query_from_module(value)

        if isinstance(value, SqlStatement):
          sql = value.format(value._sql, resolved_vars)
          value = '(%s)' % sql
        elif '_repr_sql_' in dir(value):
          # pylint: disable=protected-access
          value = value._repr_sql_()
        elif isinstance(value, basestring):
          value = SqlStatement._escape_string(value)
        elif isinstance(value, list) or isinstance(value, tuple):
          if isinstance(value, tuple):
            value = list(value)
          expansion = '('
          for v in value:
            if len(expansion) > 1:
              expansion += ', '
            if isinstance(v, basestring):
              expansion += SqlStatement._escape_string(v)
            else:
              expansion += str(v)
          expansion += ')'
          value = expansion
        else:
          value = str(value)
        parts.append(value)
      elif literal:
        parts.append(literal)

    expanded = ''.join(parts)
    return expanded

  @staticmethod
  def _get_tokens(sql):
    # Find escaped '$' characters ($$), "$<name>" variable references, lone '$' characters, or
    # literal sequences of character without any '$' in them (in that order).
    return re.findall(r'(\$\$)|(\$[a-zA-Z_][a-zA-Z0-9_\.]*)|(\$)|([^\$]*)', sql)

  @staticmethod
  def _get_dependencies(sql):
    """ Return the list of variables referenced in this SQL. """
    dependencies = []
    for (_, placeholder, dollar, _) in SqlStatement._get_tokens(sql):
      if placeholder:
        variable = placeholder[1:]
        if variable not in dependencies:
          dependencies.append(variable)
      elif dollar:
        raise Exception('Invalid sql; $ with no following $ or identifier: %s.' % sql)
    return dependencies


from flask import Flask, render_template
import urllib.request, json

app = Flask(__name__)

@app.route("/")
def load_main():
    key = open("key_nasa.txt", "r").read()
    url = "https://api.nasa.gov/planetary/apod?api_key=" + key

    response = urllib.request.urlopen(url)
    data = response.read()
    dict = json.loads(data)
    return render_template("main.html", image1=dict["hdurl"], desc=dict["explanation"])

if __name__ == "__main__":
    app.debug = True
    app.run()


from os import getenv
import re

def dist(p1, p2):
    return abs(p1[0]-p2[0]) + abs(p1[1]-p2[1])

class Solution:

    def __init__(self):
        self.input_txt = [list(map(int, re.findall(r'-*\d+', i))) for i in open("input.txt", 'r').read().split('\n')]
        self.row_to_test = 2000000
        self.max_dimension = 4000000
        self.sensors = {(i[0], i[1]): (i[2], i[3]) for i in self.input_txt}
        self.distances = {i: dist(i, self.sensors[i]) for i in self.sensors}
        self.seen_beacons = set(self.sensors.values())
        self.no_beacons = set()
        self.not_empty = set()

    def get_border(self, point, distance):
        point1 = (point[0] + distance + 1, point[1])
        point2 = (point[0] - distance - 1, point[1])
        point3 = point1
        point4 = point2
        border = {point1, point2, (point[0], point[1] + distance + 1), (point[0], point[1] - distance - 1)}
        for _ in range(distance):
            point1 = (point1[0] - 1, point1[1] + 1)
            point2 = (point2[0] + 1, point2[1] + 1)
            point3 = (point3[0] - 1, point3[1] - 1)
            point4 = (point4[0] + 1, point4[1] - 1)
            border.update([point1, point2, point3, point4])
        return {p for p in border if 0 <= p[0] <= self.max_dimension and 0 <= p[1] <= self.max_dimension}

    def calc_no_beacons(self):
        for s in self.sensors:
            d = dist(s, self.sensors[s])
            y_range = range(s[1]-d, s[1]+d)
            if self.row_to_test in y_range:
                if s[1] < self.row_to_test:
                    dist_to_check = y_range.stop - self.row_to_test
                else:
                    dist_to_check = self.row_to_test - y_range.start
                x_range = range(s[0] - dist_to_check, s[0] + dist_to_check + 1)
                self.no_beacons.update({i for i in x_range})
        self.no_beacons.difference_update({b[0] for b in self.seen_beacons if b[1] == self.row_to_test})

    def find_missing_spot(self):
        for s in self.sensors:
            d = dist(s, self.sensors[s])
            border = [b for b in self.get_border(s, d) if b not in self.not_empty]
            for p in border:
                distance_to_sensors = {s: dist(p, s) for s in self.sensors}
                if all(distance_to_sensors[x] > self.distances[x] for x in self.sensors):
                    return p
                else:
                    self.not_empty.add(p)

if __name__ == '__main__':
    sol = Solution()
    if getenv('part') == 'part1':
        sol.calc_no_beacons()
        print(len(sol.no_beacons))
    else:
        point = sol.find_missing_spot()
        print(point[0] * 4000000 + point[1])


import requests
#导入进程池模块
from multiprocessing import Pool
from requests.exceptions import RequestException
import re
import json

headers = {
'Host': 'maoyan.com',
'Connection': 'keep-alive',
'Cache-Control': 'max-age=0',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Referer': 'https://maoyan.com/films',
'Accept-Language': 'zh-CN,zh;q=0.9',
}

def get_one_page(url,headers,verify):
    try:
        sess = requests.session()

        # sess = requests.session()
        response = sess.get(url=url,headers=headers,verify=verify)

        #判断是否请求成功

        if response.status_code == 200:
            #若请求成功则返回，页面信息
            return response.text
        return None
    except:
        return None

#解析获取的网页
def parse_one_page(html):
    pattern = re.compile(r'
.*?movie-poster">.*?data-src="(.*?)".*?}">(.*?).*?integer">(.*?).*?fraction">(\d+)',re.S)
    items = re.findall(pattern,html)
    # print(items)
    for item in items:
        yield {
        'title':item[1],
        'image':item[0],
        'score':item[2]+item[3]
        }
#将数据写入文件
def write_to_file(content):
    #设置json数据显示中文
    with open('result.txt','a',encoding='utf-8') as f:
        #ensure_ascii=False 表示上传的json数据，不按照ascii码
        f.write(json.dumps(content,ensure_ascii=False)+'\n')
        f.close()

#获取响应信息
def main(offset):
    url = 'https://maoyan.com/films?showType=3&offset='+str(offset)
    # headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"}

    #调用请求
    html = get_one_page(url=url,headers=headers,verify=False)
    # print(html)
    for itme in parse_one_page(html):
        print(itme)
        write_to_file(itme)
if __name__ == '__main__':
    # for i in range(10):
    #     main(i*30)
    #创建进程池对象
    pool = Pool()
    pool.map(main,[i*10 for i in range(100)])


import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.nn import Parameter
from GlobalNetworkParam import GlobalNetworkParam
from NetworkConfig import NetworkConfig
from LocalNetworkParam import LocalNetworkParam
import torch.optim
from Utils import *
import time

class NetworkModel(nn.Module):

    Iter = 0

    def __init__(self, fm, compiler):
        super().__init__()
        self._fm = fm
        self._num_threads = NetworkConfig.NUM_THREADS
        self._compiler = compiler
        self._all_instances = None
        self._param = None
        self._cache_networks = True
        self._networks = None

    def get_instances(self):
        return self._all_instances

    def get_feature_manager(self):
        return self._fm

    def get_network_compiler(self):
        return self._compiler

    def split_instances_for_train(self, insts_before_split):
        eprint("#instances=", len(insts_before_split))
        insts = [None for i in range(len(insts_before_split) * 2)]

        k=0
        for i in range(0, len(insts), 2):
            insts[i + 1] = insts_before_split[k]
            insts[i] = insts_before_split[k].duplicate()
            insts[i].set_instance_id(-insts[i].get_instance_id())
            insts[i].set_weight(-insts[i].get_weight())
            insts[i].set_unlabeled()
            k = k + 1
        return insts

    def lock_it(self):
        gnp = self._fm.get_param_g()

        if gnp.is_locked():
            return

        weights_new = torch.nn.Parameter(torch.randn(gnp.size()))  # self.size
        #weights_new = torch.nn.Parameter(torch.Tensor(gnp.size()).fill_(0.0))  # self.size
        print('weights_new type:', type(weights_new))
        # weights_new.fill_(0.0)  ## TODO: need to randomly initialize

        self.weights = weights_new
        gnp.version = 0
        gnp._string_index.lock()
        gnp.locked = True
        eprint(gnp._size, " features")

    def assign_fws(self, insts):
        if self._networks == None:
            self._networks = [None for i in range(len(insts))]

        for network_id in range(len(insts)):
            if network_id % 100 == 0:
                eprint('.', end='')
            network = self.get_network(network_id)

            network.assign_fw()

        eprint()

    def train(self, train_insts, max_iterations):

        if NetworkConfig.GPU_ID >= 0:
            torch.cuda.set_device(NetworkConfig.GPU_ID)

        insts_before_split = train_insts #self.prepare_instance_for_compilation(train_insts)

        insts = self.split_instances_for_train(insts_before_split)

        self._param = LocalNetworkParam(self, self._fm, len(insts))
        self._fm.set_local_param(self._param)
        self._fm.enable_cache(len(insts))

        self._all_instances = insts
        if NetworkConfig.PRE_COMPILE_NETWORKS:
            self.pre_compile_networks(insts)
        keep_existing_threads = True if NetworkConfig.PRE_COMPILE_NETWORKS else False
        self.touch(insts, keep_existing_threads)

        self._param.finalize_it()

        #self._fm.get_param_g().lock_it()
        self.lock_it()
        self.assign_fws(insts)

        # optimizer = torch.optim.SGD(self.parameters(), lr = 0.01)  # lr=0.8
        optimizer = torch.optim.LBFGS(self.parameters())  # lr=0.8
        NetworkModel.Iter = 0
        for it in range(max_iterations):

            def closure():

                optimizer.zero_grad()

                all_loss = 0  ### scalar

                start_time = time.time()
                for i in range(len(self._all_instances)):
                    loss = self.forward(self.get_network(i))
                    all_loss -= loss
                    #loss.backward()

                end_time = time.time()
                diff_time = end_time - start_time
                print('Forward:', '\tTime=',diff_time)

                start_time = time.time()
                all_loss.backward()
                end_time = time.time()
                diff_time = end_time - start_time
                print('Backward:', '\tTime=', diff_time)

                print("Iteration ", NetworkModel.Iter,": Obj=",  all_loss.data[0], '\tTime=',diff_time)
                NetworkModel.Iter += 1

                return all_loss
            #print('bWeight:', self.weights)
            # print("bGrad:", self.weights.grad)
            optimizer.step(closure)

            if NetworkModel.Iter > max_iterations:
                break
            #print('aWeight:', self.weights)
            # print("aGrad:", self.weights.grad)

    def forward(self, network):
        return network.inside()

    def get_network(self, network_id):
        if self._cache_networks and self._networks[network_id] != None:
            return self._networks[network_id]

        inst = self._all_instances[network_id]

        network = self._compiler.compile(network_id, inst, self._param)
        #print("after compile: ", network)

        if self._cache_networks:
            self._networks[network_id] = network

        return network

    def touch(self, insts, keep_existing_threads = False):
        if self._networks == None:
            self._networks = [None for i in range(len(insts))]

        for network_id in range(len(insts)):
            if network_id % 100 == 0:
                eprint('.', end='')
            network =  self.get_network(network_id)

            network.touch()

        eprint()

    def test(self, instances):
        return self.decode(instances=instances)

    def decode(self, instances, cache_features=False):

        self._num_threads = NetworkConfig.NUM_THREADS
        eprint('#Threads: ', self._num_threads)

        self._all_instances = instances

        self._param = LocalNetworkParam(self, self._fm, len(instances))
        self._fm.set_local_param(self._param)

        instances_output = []

        for k in range(len(instances)):
            instance = instances[k]
            # print("decode ", instance.is_labeled)

            network = self._compiler.compile(k, instance, self._param)
            network.max()
            instance_output = self._compiler.decompile(network)
            instances_output.append(instance_output)

        return instances_output


import cv2
import numpy as np

img = cv2.imread("lionel-messi-gq-14feb18_getty_b.webp")
img2 = cv2.imread("dickens-lin-FLSbAX8dheQ-unsplash.jpg")

print(img.shape) #Returns the number of rows, columns and channels in the image
print(img.size) #Returns the number of pixels in the image
print(img.dtype) #The image's datatype

# Split the image into the 3 channels
b, g, r = cv2.split(img)

# Merge the image
img = cv2.merge((b,g,r))

# Resize the image
img = cv2.resize(img, (512,512))
img2 = cv2.resize(img2, (512,512))

# Add the images
dst = cv2.add(img, img2)

# Show the image
cv2.imshow("Image", dst)

# def mouse_click(event, x, y, flags, params):
#     img = cv2.imread("lionel-messi-gq-14feb18_getty_b.webp")
#     if event == cv2.EVENT_LBUTTONDOWN:
#         font = cv2.FONT_HERSHEY_SIMPLEX
#         print((x,y))
#         # img = cv2.putText(img, text, (x,y), font, 2, (255,4,3), 1)

#     # cv2.imshow("Image", img)

# cv2.setMouseCallback("Image", mouse_click)

cv2.waitKey(0)
cv2.destroyAllWindows()


import re

#https://sakkyoku.info/beginner/determine-the-key/ を参考
chordPattern_key = [
    ["C", "Dm", "Em", "F", "G", "Am", "Bm-5"],       #キー：C or Am
    ["G", "Am", "Bm", "C", "D", "Em", "F#m-5"],      #キー：G or Em
    ["D", "Em", "F#m", "G", "A", "Bm", "C#m-5"],     #キー：D or Bm
    ["A", "Bm", "C#m", "D", "E", "F#m", "G#m-5"],    #キー：A or F#m
    ["E", "F#m", "G#m", "A", "B", "C#m", "D#m-5"],   #キー：E or C#m
    ["B", "C#m", "D#m", "E", "F#", "G#m", "A#m-5"],  #キー：B or G#m
    ["F#", "G#m", "A#m", "B", "C#", "D#m", "Fm-5"],  #キー：F# or D#m
    ["F", "Gm", "Am", "Bb", "C", "Dm", "Em-5"],      #キー：F or Dm
    ["Bb", "Cm", "Dm", "Eb", "F", "Gm", "Am-5"],     #キー：Bb or Gm
    ["Eb", "Fm", "Gm", "Ab", "Bb", "Cm", "Dm-5"],    #キー：Eb or Cm
    ["Ab", "Bbm", "Cm", "Db", "Eb", "Fm", "Gm-5"],   #キー：Ab or Fm
    ["Db", "Ebm", "Fm", "Gb", "Ab", "Bbm", "Cm-5"]   #キー：Db or A#m
]

def analysisKey(chordsText):
    chords = chordsText.split(" ")
    key_probability = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]   #インデックスに対応するキーである確率が格納される
    key_probability_ans = [.0, .0, .0, .0, .0, .0, .0, .0, .0, .0, .0, .0]
    isMajor = [False, False, False, False, False, False, False, False, False, False, False, False]
    MajorPoints = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
    prevChordNums = [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]
    #コードごとにキーに対応するコードと一致するかを確かめる
    for chord in chords:
        mChord = re.sub("M|7|/[a-zA-Z#]+|sus[0-9]|add[0-9]|\(.+?\)|9", "", chord)

        for i in range(12):
            for j in range(7):
                if chordPattern_key[i][j] == mChord:       #あるキーのコードと一致したら
                    key_probability[i] += 1                #そのキーに対応するインデックス値の場所の値をインクリメント
                    #メジャーコード的な進行を検知
                    if prevChordNums[i] == 0 and j == 3:
                        MajorPoints[i] += 1
                    if prevChordNums[i] == 3 and j == 4:
                        MajorPoints[i] += 1
                    if prevChordNums[i] == 4 and j == 0:
                        MajorPoints[i] += 1
                    #マイナーコード的な進行を検知
                    if prevChordNums[i] == 5 and j == 1:
                        MajorPoints[i] -= 1
                    if prevChordNums[i] == 1 and j == 2:
                        MajorPoints[i] -= 1
                    if prevChordNums[i] == 2 and j == 5:
                        MajorPoints[i] -= 1
                    prevChordNums[i] = j
                    break
                #最後の要素だったら合致するコードが見つからなかったってことになる
                if j == 6:
                    prevChordNums[i] = -1   #不正な値をいれておく

    #正規化と解析率を求めるためのカウンター
    sum = 0
    for i in key_probability:
        sum += i

    if sum < (len(chords) * 0.7):            #もし7割以下が不明なコードなら空の解答を返す
        return (key_probability_ans, isMajor)

    for i in range(12):
        if sum == 0:
            print("error : divide 0")
        key_probability_ans[i] = key_probability[i] / sum       #正規化を行う

    isMajor = [MajorPoint >= 0 for MajorPoint in MajorPoints]

    return (key_probability_ans, isMajor)

def writeChordMarkDown(songName, songUrl, chordsText, key_probability):
    with open("mikuChord.md", mode="a") as markdownFile:
        markdownFile.write("# **[" + songName + "](" + songUrl + ")**\n\n")
        markdownFile.write("## KeyPercentage\n")

        #受け取ったkeyの可能性を表示する処理
        text = ""
        if key_probability[1][0] == True:
            text += "Cメジャー："
        else :
            text += "Aマイナー："
        text += str(round(key_probability[0][0] * 100, 2)) + "％  "

        if key_probability[1][1] == True:
            text += "Gメジャー："
        else :
            text += "Eマイナー："
        text += str(round(key_probability[0][1] * 100, 2)) + "％  "

        if key_probability[1][2] == True:
            text += "Dメジャー："
        else :
            text += "Bマイナー："
        text += str(round(key_probability[0][2] * 100, 2)) + "％  "

        if key_probability[1][3] == True:
            text += "Aメジャー："
        else :
            text += "F#マイナー："
        text += str(round(key_probability[0][3] * 100, 2)) + "％  "

        if key_probability[1][4] == True:
            text += "Eメジャー："
        else :
            text += "C#マイナー："
        text += str(round(key_probability[0][4] * 100, 2)) + "％  "

        if key_probability[1][5] == True:
            text += "Bメジャー："
        else :
            text += "G#マイナー："
        text += str(round(key_probability[0][5] * 100, 2)) + "％  "

        if key_probability[1][6] == True:
            text += "F#メジャー："
        else :
            text += "D#マイナー："
        text += str(round(key_probability[0][6] * 100, 2)) + "％  "

        if key_probability[1][7] == True:
            text += "Fメジャー："
        else :
            text += "Dマイナー："
        text += str(round(key_probability[0][7] * 100, 2)) + "％  "

        if key_probability[1][8] == True:
            text += "Bbメジャー："
        else :
            text += "Gマイナー："
        text += str(round(key_probability[0][8] * 100, 2)) + "％  "

        if key_probability[1][9] == True:
            text += "Ebメジャー："
        else :
            text += "Cマイナー："
        text += str(round(key_probability[0][9] * 100, 2)) + "％  "

        if key_probability[1][10] == True:
            text += "Abメジャー："
        else :
            text += "Fマイナー："
        text += str(round(key_probability[0][10] * 100, 2)) + "％  "

        if key_probability[1][11] == True:
            text += "Dbメジャー："
        else :
            text += "A#マイナー："
        text += str(round(key_probability[0][11] * 100, 2)) + "％  "

        markdownFile.write(text + "\n\n")

        #一番可能性が高いキーを特別に表示
        maxNum = max(key_probability[0])
        maxIndex = key_probability[0].index(maxNum)
        text = "## **"
        #表示するインデックスが指定されてること以外は上の処理と同様
        if maxIndex == 0:
            if key_probability[1][0] == True:
                text += "Cメジャー："
            else :
                text += "Aマイナー："
            text += str(round(key_probability[0][0] * 100, 2)) + "％"

        if maxIndex == 1:
            if key_probability[1][1] == True:
                text += "Gメジャー："
            else :
                text += "Eマイナー："
            text += str(round(key_probability[0][1] * 100, 2)) + "％"

        if maxIndex == 2:
            if key_probability[1][2] == True:
                text += "Dメジャー："
            else :
                text += "Bマイナー："
            text += str(round(key_probability[0][2] * 100, 2)) + "％"

        if maxIndex == 3:
            if key_probability[1][3] == True:
                text += "Aメジャー："
            else :
                text += "F#マイナー："
            text += str(round(key_probability[0][3] * 100, 2)) + "％"

        if maxIndex == 4:
            if key_probability[1][4] == True:
                text += "Eメジャー："
            else :
                text += "C#マイナー："
            text += str(round(key_probability[0][4] * 100, 2)) + "％"

        if maxIndex == 5:
            if key_probability[1][5] == True:
                text += "Bメジャー："
            else :
                text += "G#マイナー："
            text += str(round(key_probability[0][5] * 100, 2)) + "％"

        if maxIndex == 6:
            if key_probability[1][6] == True:
                text += "F#メジャー："
            else :
                text += "D#マイナー："
            text += str(round(key_probability[0][6] * 100, 2)) + "％"

        if maxIndex == 7:
            if key_probability[1][7] == True:
                text += "Fメジャー："
            else :
                text += "Dマイナー："
            text += str(round(key_probability[0][7] * 100, 2)) + "％"

        if maxIndex == 8:
            if key_probability[1][8] == True:
                text += "Bbメジャー："
            else :
                text += "Gマイナー："
            text += str(round(key_probability[0][8] * 100, 2)) + "％"

        if maxIndex == 9:
            if key_probability[1][9] == True:
                text += "Ebメジャー："
            else :
                text += "Cマイナー："
            text += str(round(key_probability[0][9] * 100, 2)) + "％"

        if maxIndex == 10:
            if key_probability[1][10] == True:
                text += "Abメジャー："
            else :
                text += "Fマイナー："
            text += str(round(key_probability[0][10] * 100, 2)) + "％"

        if maxIndex == 11:
            if key_probability[1][11] == True:
                text += "Dbメジャー："
            else :
                text += "A#マイナー："
            text += str(round(key_probability[0][11] * 100, 2)) + "％"
        text += "**"
        markdownFile.write(text + "\n\n")

        chords = chordsText.split(" ")
        markdownFile.write("### **")
        for i, chord in enumerate(chords):
            if i == len(chords) - 1:
                markdownFile.write(chord)
            else:
                markdownFile.write(chord + " | ")

        markdownFile.write("**\n
\n\n")

        #予測したキーに応じてローマ数字表記に書き換える
        singles = ['Ⅰ', 'Ⅱ', 'Ⅲ', 'Ⅳ', 'Ⅴ', 'Ⅵ', 'Ⅶ']
        minorSingles = ['Ⅲ', 'Ⅳ', 'Ⅴ', 'Ⅵ', 'Ⅶ', 'Ⅰ', 'Ⅱ']
        markdownFile.write("## 予測結果\n\n**")
        for i, chord in enumerate(chords):
            isKeyChord = False              #キーに含まれないコードは赤で表示する
            isOne = False                   #コードがⅠだったときにTrueになる
            isFour = False                  #コードがⅣだったとき
            isFive = False                  #コードがⅤだったとき
            isDominant = False              #前回のコードから5度下(4度上)のコードに進行するのを検知したらTrueにする
            numberChordName = chord         #もし書き換え不可だった時、元のコードを出力する
            mChord = re.sub("M|7|/[a-zA-Z#]+|sus[0-9]|add[0-9]|\(.+?\)|9", "", chord)
            for j in range(7):
                if chordPattern_key[maxIndex][j] == mChord:       #あるキーのコードと一致したら
                    isKeyChord = True                             #キーに属するコードだと分かったのでTrueにする
                    #次のコードが強進行的なニュアンスを含んでいたら
                    if i < len(chords) - 1:
                        if re.sub("M|7|/[a-zA-Z#]+|sus[0-9]|add[0-9]|\(.+?\)|9", "", chords[i + 1]) == chordPattern_key[maxIndex][(j + 3) % 7]:
                            isDominant = True
                    if key_probability[1][maxIndex] == True:       #メジャーコードだったら
                        if j == 0:
                            isOne = True                              #Ⅰだと分かったのでTrue
                        if j == 3:
                            isFour = True
                        if j == 4:
                            isFive = True
                        numberChordName = chord.replace(mChord ,singles[j])
                    else :
                        if j == 5:
                            isOne = True                              #Ⅰだと分かったのでTrue
                        if j == 1:
                            isFour = True
                        if j == 2:
                            isFive = True
                        numberChordName = chord.replace(mChord ,minorSingles[j])

            #コードの情報を書く前に、特定のコードを着色する処理を入れる
            if isKeyChord == False:
                    markdownFile.write("")
            elif isOne == True:
                    markdownFile.write("")
            elif isFour == True:
                    markdownFile.write("")
            elif isFive == True:
                    markdownFile.write("")

            #コードを書く処理
            if i == len(chords) - 1:
                markdownFile.write(numberChordName)
                #着色する処理の後処理(最後の出力以外の「|」を記述する処理のために後処理は2重化を許容してる)
                if isKeyChord == False or isOne == True or isFour == True or isFive == True:
                    markdownFile.write("")
            else:
                markdownFile.write(numberChordName)
                #着色する処理の後処理
                if isKeyChord == False or isOne == True or isFour == True or isFive == True:
                    markdownFile.write("")
                if isDominant == True:
                    markdownFile.write("")
                    markdownFile.write(" -> ")
                    markdownFile.write("")
                else :
                    markdownFile.write(" | ")

        markdownFile.write("**\n
\n\n")

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

def chordTransitionStatistics(chordsText, key_probability, chordTransisionPercentage):
    maxNum = max(key_probability[0])
    maxIndex = key_probability[0].index(maxNum)
    chords = chordsText.split(" ")
    for i, chord in enumerate(chords):
        for j in range(7):
            mChord = re.sub("M|7|/[a-zA-Z#]+|sus[0-9]|add[0-9]|\(.+?\)|9", "", chord)
            if chordPattern_key[maxIndex][j] == mChord:
                #次のコードを調べる
                if i < len(chords) - 1:
                    for k in range(7):
                        if re.sub("M|7|/[a-zA-Z#]+|sus[0-9]|add[0-9]|\(.+?\)|9", "", chords[i + 1]) == chordPattern_key[maxIndex][k]:
                            chordTransisionPercentage[j][k] += 1
    return chordTransisionPercentage

def retFirstChord(chordsText, key_probability):
    singles = ['Ⅰ', 'Ⅱ', 'Ⅲ', 'Ⅳ', 'Ⅴ', 'Ⅵ', 'Ⅶ']
    minorSingles = ['Ⅲ', 'Ⅳ', 'Ⅴ', 'Ⅵ', 'Ⅶ', 'Ⅰ', 'Ⅱ']
    #一番可能性が高いキーを取得
    maxNum = max(key_probability[0])
    maxIndex = key_probability[0].index(maxNum)
    mChord = re.sub("M|7|/[a-zA-Z#]+|sus[0-9]|add[0-9]|\(.+?\)|9", "", chordsText.split(" ")[0])
    for j in range(7):
        if chordPattern_key[maxIndex][j] == mChord:       #あるキーのコードと一致したら
            if key_probability[1][maxIndex] == True:       #メジャーコードだったら
                numberChordName = singles[j]
            else :                                         #マイナーコードだったら
                numberChordName = minorSingles[j]
            return numberChordName

    return ""

def min_max(x, axis=None):
    min = x.min(axis=axis, keepdims=True)
    max = x.max(axis=axis, keepdims=True)
    result = (x-min)/(max-min)
    return result

def writeChordStatistics(chordTransisionPercentage, firstChordNums):
    chordTransitionPercentage_normal = min_max(chordTransisionPercentage)
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.set_xlabel("chord number before transition")
    ax.set_ylabel("chord number after transition")
    ax.set_zlabel("Normalized probability")
    X1, X2 = np.mgrid[0:7, 0:7]
    """before = [3,0,4,1,5,2,6]
    after = [2,5,1,4,0,3,6]
    chord_sp = np.zeros((7, 7), dtype=float)
    for i in range(7):
        for j in range(7):
            chord_sp[i][j] = chordTransitionPercentage_normal[before[i]][after[j]]
    print(chord_sp)
    print()
    print(chordTransitionPercentage_normal)"""
    surf = ax.plot_surface(X1, X2, chordTransitionPercentage_normal, cmap="plasma", facecolor="w")
    plt.xticks([0,1,2,3,4,5,6], ["Ⅰ", "Ⅱ", "Ⅲ", "Ⅳ", "Ⅴ", "Ⅵ", "Ⅶ"])
    plt.yticks([0,1,2,3,4,5,6], ["Ⅰ", "Ⅱ", "Ⅲ", "Ⅳ", "Ⅴ", "Ⅵ", "Ⅶ"])
    #plt.xticks([0,1,2,3,4,5,6], ["Ⅳ", "Ⅰ", "Ⅴ", "Ⅱ", "Ⅵ", "Ⅲ", "Ⅶ"])
    #plt.yticks([0,1,2,3,4,5,6], ["Ⅲ", "Ⅵ", "Ⅱ", "Ⅴ", "Ⅰ", "Ⅳ", "Ⅶ"])
    fig.colorbar(surf)
    ax.set_title("chord transition statistics plot")
    plt.show()
    fig.savefig("chordTransitionImage.png")

    with open("mikuChordStatistics.md", mode="w") as sFile:
        singles = ['Ⅰ', 'Ⅱ', 'Ⅲ', 'Ⅳ', 'Ⅴ', 'Ⅵ', 'Ⅶ']
        chordTransitionPercentage_line = np.zeros((7, 7), dtype=float)
        sFile.write("# **初音ミク関係の曲のコード遷移確率**\n")
        sFile.write("## コード遷移の比重を表した画像\n")
        sFile.write("![コード遷移の統計画像](chordTransitionImage.png)\n")
        sFile.write("\n
\n\n")
        for i in range(7):
            sum = 0.0
            for j in range(7):
                sum += chordTransitionPercentage_normal[i][j]
            for j in range(7):
                chordTransitionPercentage_line[i][j] = chordTransitionPercentage_normal[i][j] / sum
        for i in range(7):
            sFile.write("## **" + singles[i] + "からの遷移確率**\n")
            for j in range(7):
                sFile.write("### **" + singles[j] + " : " + str(chordTransitionPercentage_line[i][j]*100) + "％**\n")
            sFile.write("\n
\n\n")

        sFile.write("## **開始のコード確率**\n")
        sum = 0
        for key in firstChordNums:
            if key != "":
                sum += firstChordNums[key]
        for key in firstChordNums:
            if key != "":
                sFile.write("### **" + key + " : " + str(float(firstChordNums[key]) / sum * 100) + "％**\n")
            else:
                sFile.write("### 有効なコード数 : " +  str(sum) + "\n")
                sFile.write("### 無効なコード数 : " +  str(firstChordNums[key]))
        sFile.write("\n
\n\n")

#メインの処理
with open("mikuChord.txt") as f:
    lines = [s.strip() for s in f.readlines()]
    with open("mikuChord.md", mode="w") as markdownFile:
        markdownFile.write("")
        print("マークダウンファイルをクリアしました")
    chordTransisionPercentage = np.zeros((7, 7), dtype=int)
    firstChordNums = {'Ⅰ':0, 'Ⅱ':0, 'Ⅲ':0, 'Ⅳ':0, 'Ⅴ':0, 'Ⅵ':0, 'Ⅶ':0, '':0}
    for oneLineText in lines:
        lineTexts = oneLineText.split(",", 2)       #区切るのは名前とURLとコード列の三つだけ、後の要素はコード進行なので分割しないでおく
        songName = lineTexts[0]
        songUrl = lineTexts[1]
        chordsText = lineTexts[2]
        key_probability = analysisKey(chordsText)
        writeChordMarkDown(songName, songUrl, chordsText, key_probability)
        firstChordNums[retFirstChord(chordsText, key_probability)] += 1
        #統計を出すための処理
        chordTransisionPercentage = chordTransitionStatistics(chordsText, key_probability, chordTransisionPercentage)

    writeChordStatistics(chordTransisionPercentage, firstChordNums)