import glob
# import seaborn as sns
import logging
import logging.handlers
import os
import platform
import sys
import argparse
import traceback
import warnings
from builtins import enumerate
from sklearn.metrics import mean_squared_error
from flaml import AutoML
from lightgbm import plot_importance

# import datetime as dt
# from datetime import datetime
# import pvlib
import matplotlib.dates as mdates
import matplotlib.cm as cm
# from sklearn.metrics import mean_absolute_error

import matplotlib as mpl
import matplotlib.pyplot as  plt
import numpy as np
from scipy.stats import linregress
import pandas as pd
import math
from scipy import spatial
from pandas.tseries.offsets import Day, Hour, Minute, Second
from scipy.interpolate import Rbf
from numpy import zeros, newaxis

# os.environ["PROJ_LIB"] = "C:\ProgramData\Anaconda3\Library\share"
# from pygem import IDW
# import eccodes
import pygrib
# import pykrige.kriging_tools as kt
import pytz
import requests
import datetime
import h2o
# from pycaret.regression import *
from sqlalchemy import create_engine
import re
import configparser
import sqlalchemy
from sqlalchemy.ext.declarative import declarative_base
import sqlalchemy
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, DateTime, Float
from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
import pymysql
import re
import configparser
import sqlalchemy
from sqlalchemy.ext.declarative import declarative_base
from scipy.stats import linregress
import argparse
from sklearn.model_selection import train_test_split
import numpy as np
import pandas as pd
# from sklearn.model_selection import train_test_split

import xarray as xr
from pvlib import location
from pvlib import irradiance

import h2o
from h2o.automl import H2OAutoML

from matplotlib import font_manager, rc

try:
    from pycaret.regression import *
except Exception as e:
    print("Exception : {}".format(e))

try:
    from pycaret.regression import *
except Exception as e:
    print("Exception : {}".format(e))

import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from keras import regularizers
from sklearn.metrics import mean_absolute_percentage_error, mean_squared_error
from sklearn.ensemble import RandomForestRegressor
import xgboost
# from sklearn.utils import safe_indexing

import os
import pickle
import ray

# =================================================
# 사용자 매뉴얼
# =================================================
# [소스 코드의 실행 순서]
# 1. 초기 설정 : 폰트 설정
# 2. 유틸리티 함수 : 초기화 함수 (로그 설정, 초기 변수, 초기 전달인자 설정) 또는 자주 사용하는 함수
# 3. 주 프로그램 :부 프로그램을 호출
# 4. 부 프로그램 : 자료 처리를 위한 클래스로서 내부 함수 (초기 변수, 비즈니스 로직, 수행 프로그램 설정)
# 4.1. 환경 변수 설정 (로그 설정) : 로그 기록을 위한 설정 정보 읽기
# 4.2. 환경 변수 설정 (초기 변수) : 입력 경로 (inpPath) 및 출력 경로 (outPath) 등을 설정
# 4.3. 초기 변수 (Argument, Option) 설정 : 파이썬 실행 시 전달인자 설정 (pyhton3 *.py argv1 argv2 argv3 ...)
# 4.4. 비즈니스 로직 수행 : 단위 시스템 (unit 파일명)으로 관리 또는 비즈니스 로직 구현

# =================================================
# 1. 초기 설정
# =================================================
warnings.filterwarnings("ignore")
# font_manager._rebuild()

plt.rc('font', family='Malgun Gothic')
plt.rc('axes', unicode_minus=False)
# sns.set(font="Malgun Gothic", rc={"axes.unicode_minus": False}, style='darkgrid')

# 그래프에서 마이너스 글꼴 깨지는 문제에 대한 대처
mpl.rcParams['axes.unicode_minus'] = False

# 타임존 설정
tzKst = pytz.timezone('Asia/Seoul')
tzUtc = pytz.timezone('UTC')
dtKst = datetime.timedelta(hours=9)

# =================================================
# 2. 유틸리티 함수
# =================================================
# 로그 설정
def initLog(env=None, contextPath=None, prjName=None):
    if env is None: env = 'local'
    if contextPath is None: contextPath = os.getcwd()
    if prjName is None: prjName = 'test'

    saveLogFile = "{}/{}_{}_{}_{}.log".format(
        contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'log', prjName)
        , platform.system()
        , platform.node()
        , prjName
        , datetime.datetime.now().strftime("%Y%m%d")
    )

    if not os.path.exists(os.path.dirname(saveLogFile)):
        os.makedirs(os.path.dirname(saveLogFile))

    # logger instance 생성
    log = logging.getLogger(prjName)

    if len(log.handlers) > 0:
        return log

    # format 생성
    format = logging.Formatter('%(asctime)s [%(name)s | %(lineno)d | %(filename)s] [%(levelname)-5.5s] %(message)s')

    # handler 생성
    streamHandler = logging.StreamHandler()
    fileHandler = logging.FileHandler(saveLogFile)

    # logger instance에 format 설정
    streamHandler.setFormatter(format)
    fileHandler.setFormatter(format)

    # logger instance에 handler 설정
    log.addHandler(streamHandler)
    log.addHandler(fileHandler)

    # logger instance로 log 기록
    log.setLevel(level=logging.INFO)

    return log

#  초기 변수 설정
def initGlobalVar(env=None, contextPath=None, prjName=None):
    if env is None: env = 'local'
    if contextPath is None: contextPath = os.getcwd()
    if prjName is None: prjName = 'test'

    # 환경 변수 (local, 그 외)에 따라 전역 변수 (입력 자료, 출력 자료 등)를 동적으로 설정
    # 즉 local의 경우 현재 작업 경로 (contextPath)를 기준으로 설정
    # 그 외의 경우 contextPath/resources/input/prjName와 같은 동적으로 구성
    globalVar = {
        'prjName': prjName
        , 'sysOs': platform.system()
        , 'contextPath': contextPath
        , 'resPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources')
        , 'cfgPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'config')
        , 'inpPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'input', prjName)
        , 'figPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'fig', prjName)
        , 'outPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'input', prjName)
        , 'movPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'movie', prjName)
        , 'logPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'log', prjName)
        , 'mapPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'mapInfo')
        , 'sysPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'config', 'system.cfg')
        , 'seleniumPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'config', 'selenium')
        , 'fontPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'config', 'fontInfo')
    }

    return globalVar

#  초기 전달인자 설정
def initArgument(globalVar, inParams):
    # 원도우 또는 맥 환경
    if globalVar['sysOs'] in 'Windows' or globalVar['sysOs'] in 'Darwin':
        inParInfo = inParams

        # 글꼴 설정
        plt.rc('font', family='Malgun Gothic')

    # 리눅스 환경
    if globalVar['sysOs'] in 'Linux':
        parser = argparse.ArgumentParser()

        for i, argv in enumerate(sys.argv[1:]):
            if not argv.__contains__('--'): continue
            parser.add_argument(argv)

        inParInfo = vars(parser.parse_args())

        # 글꼴 설정
        fontList = glob.glob('{}/{}'.format(globalVar['fontPath'], '*.ttf'))
        fontName = font_manager.FontProperties(fname=fontList[0]).get_name()
        plt.rcParams['font.family'] = fontName

    log.info("[CHECK] inParInfo : {}".format(inParInfo))

    for key, val in inParInfo.items():
        if val is None: continue
        # 전역 변수에 할당
        globalVar[key] = val

    # 전역 변수
    for key, val in globalVar.items():
        if env not in 'local' and key.__contains__('Path') and env and not os.path.exists(val):
            os.makedirs(val)

        globalVar[key] = val.replace('\\', '/')

        log.info("[CHECK] {}".format(val))

        # self 변수에 할당
        # setattr(self, key, val)

    return globalVar

# 딥러닝 예측
def makeDlModel(subOpt=None, xCol=None, yCol=None, inpData=None):

    log.info('[START] {}'.format('makeDlModel'))
    log.info('[CHECK] subOpt : {}'.format(subOpt))

    result = None

    try:

        saveModel = '{}/{}/{}-{}-{}-{}-{}-{}.model'.format(globalVar['modelPath'], subOpt['modelKey'], serviceName, subOpt['srvId'], 'final', 'h2o', 'for', '*')
        saveModelList = sorted(glob.glob(saveModel), reverse=True)
        xyCol = xCol.copy()
        xyCol.append(yCol)
        data = inpData[xyCol]
        dataL1 = data.dropna()

        # h2o.shutdown(prompt=False)

        if (not subOpt['isInit']):
            h2o.init()
            h2o.no_progress()
            subOpt['isInit'] = True

        # 학습 모델이 없을 경우
        if (subOpt['isOverWrite']) or (len(saveModelList) < 1):

            # 7:3에 대한 학습/테스트 분류
            # trainData, validData = train_test_split(dataL1, test_size=0.3)

            # dlModel = H2OAutoML(max_models=30, max_runtime_secs=99999, balance_classes=True, seed=123)
            dlModel = H2OAutoML(max_models=20, max_runtime_secs=99999, balance_classes=True, seed=123)

            #dlModel.train(x=xCol, y=yCol, training_frame=h2o.H2OFrame(trainData), validation_frame=h2o.H2OFrame(validData))
            #dlModel.train(x=xCol, y=yCol, training_frame=h2o.H2OFrame(dataL1), validation_frame=h2o.H2OFrame(dataL1))
            dlModel.train(x=xCol, y=yCol, training_frame=h2o.H2OFrame(dataL1), validation_frame=h2o.H2OFrame(dataL1))

            fnlModel = dlModel.get_best_model()
            #fnlModel = dlModel.leader

            # 학습 모델 저장
            saveModel = '{}/{}/{}-{}-{}-{}-{}-{}.model'.format(globalVar['modelPath'], subOpt['modelKey'], serviceName, subOpt['srvId'], 'final', 'h2o', 'for', datetime.datetime.now().strftime('%Y%m%d'))
            log.info('[CHECK] saveModel : {}'.format(saveModel))
            os.makedirs(os.path.dirname(saveModel), exist_ok=True)

            # h2o.save_model(model=fnlModel, path=os.path.dirname(saveModel), filename=os.path.basename(saveModel), force=True)
            fnlModel.save_mojo(path=os.path.dirname(saveModel), filename=os.path.basename(saveModel), force=True)
        else:
            saveModel = saveModelList[0]
            log.info('[CHECK] saveModel : {}'.format(saveModel))
            fnlModel = h2o.import_mojo(saveModel)

        result = {
            'msg': 'succ'
            , 'dlModel': fnlModel
            , 'saveModel': saveModel
            , 'isExist': os.path.exists(saveModel)
        }

        return result

    except Exception as e:
        log.error('Exception : {}'.format(e))
        return result

    finally:
        # try, catch 구문이 종료되기 전에 무조건 실행
        log.info('[END] {}'.format('makeDlModel'))

# 머신러닝 예측
def makeMlModel(subOpt=None, xCol=None, yCol=None, inpData=None):

    log.info('[START] {}'.format('makeMlModel'))
    log.info('[CHECK] subOpt : {}'.format(subOpt))

    result = None

    try:
        saveModel = '{}/{}/{}-{}-{}-{}-{}-{}.model.pkl'.format(globalVar['outPath'], subOpt['modelKey'], serviceName, subOpt['srvId'], 'final', 'pycaret', 'for', '*')
        saveModelList = sorted(glob.glob(saveModel), reverse=True)
        xyCol = xCol.copy()
        xyCol.append(yCol)
        data = inpData[xyCol]
        dataL1 = data.dropna()

        # 학습 모델이 없을 경우
        if (subOpt['isOverWrite']) or (len(saveModelList) < 1):

            # 7:3에 대한 학습/테스트 분류
            # trainData, validData = train_test_split(data, test_size=0.3)
            # trainData = inpData

            # from pycaret.regression import *
            # from pycaret.regression import setup
            # from pycaret.regression import compare_models

            # from sklearn.impute import SimpleImputer

            dataL1['is_weekday'] = dataL1['is_weekday'].astype('category')
            dataL1['is_spring'] = dataL1['is_spring'].astype('category')
            dataL1['is_summer'] = dataL1['is_summer'].astype('category')
            dataL1['is_fall'] = dataL1['is_fall'].astype('category')
            dataL1['is_winter'] = dataL1['is_winter'].astype('category')

            # dataL2 = dataL1[['CA_TOT', 'SG_PWRER_USE_AM']]

            pyModel = setup(
                data=dataL1
                , session_id=123
                , silent=True
                , target=yCol
                # 2022.11.02
                # , normalize=True
            )

            # pyModel = setup(
            #     data=dataL1
            #     , session_id=123
            #     , silent=True
            #     , target=yCol
            #     , remove_outliers= True
            #     , remove_multicollinearity = True
            #     , ignore_low_variance = True
            #     , normalize=True
            #     , transformation= True
            #     , transform_target = True
            #     , combine_rare_levels = True
            # )

            # 각 모형에 따른 자동 머신러닝
            # modelList = compare_models(sort='RMSE', n_select=3)
            # modelList = compare_models(sort='RMSE', n_select=2)
            modelList = compare_models(sort='RMSE')

            # 앙상블 모형
            # blendModel = blend_models(estimator_list=modelList, fold=10)

            # 앙상블 파라미터 튜닝
            # tuneModel = tune_model(blendModel, fold=10, choose_better=True)

            # 학습 모형
            fnlModel = finalize_model(modelList)

            # 학습 모형 저장
            saveModel = '{}/{}/{}-{}-{}-{}-{}-{}.model'.format(globalVar['outPath'], subOpt['modelKey'], serviceName, subOpt['srvId'], 'final', 'pycaret', 'for', datetime.datetime.now().strftime('%Y%m%d'))
            log.info('[CHECK] saveModel : {}'.format(saveModel))
            os.makedirs(os.path.dirname(saveModel), exist_ok=True)
            save_model(fnlModel, saveModel)

        else:
            saveModel = saveModelList[0]
            log.info('[CHECK] saveModel : {}'.format(saveModel))
            fnlModel = load_model(os.path.splitext(saveModel)[0])

        result = {
            'msg': 'succ'
            , 'mlModel': fnlModel
            , 'saveModel': saveModel
            , 'isExist': os.path.exists(saveModel)
        }

        return result

    except Exception as e:
        log.error('Exception : {}'.format(e))
        return result

    finally:
        # try, catch 구문이 종료되기 전에 무조건 실행
        log.info('[END] {}'.format('makeMlModel'))

### 주말이면 1, 평일이면 0 반환
def check_weekend(date):
    days = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
    day = date.weekday()
    if days[day] in ['Sat', 'Sun']:
        return 1
    else:
        return 0

### 봄 여부
def check_spring(date):
    month = date.month
    if month in [3, 4, 5]:
        return 1
    else:
        return 0

### 여름 여부
def check_summer(date):
    month = date.month
    if month in [6, 7, 8]:
        return 1
    else:
        return 0

### 가을 여부
def check_fall(date):
    month = date.month
    if month in [9, 10, 11]:
        return 1
    else:
        return 0

### 겨울 여부
def check_winter(date):
    month = date.month
    if month in [12, 1, 2]:
        return 1
    else:
        return 0

# 머신러닝 예측
def makeFlamlModel(subOpt=None, xCol=None, yCol=None, trainData=None, testData=None):
    log.info('[START] {}'.format('makeFlamlModel'))
    log.info('[CHECK] subOpt : {}'.format(subOpt))

    result = None

    try:
        saveModel = '{}/{}/{}-{}-{}-{}-{}-{}.model'.format(globalVar['outPath'], subOpt['modelKey'], serviceName, subOpt['srvId'], 'final', 'flaml', 'for', '*')
        saveModelList = sorted(glob.glob(saveModel), reverse=True)
        xyCol = xCol.copy()
        xyCol.append(yCol)

        trainDataL1 = trainData[xyCol]
        trainDataL2 = trainDataL1.dropna()

        testDataL1 = testData[xyCol]
        testDataL2 = testDataL1.dropna()

        # 학습 모델이 없을 경우
        if (subOpt['isOverWrite']) or (len(saveModelList) < 1):

            # 7:3에 대한 학습/테스트 분류
            # trainData, validData = train_test_split(dataL1, test_size=0.3)

            # 전체 학습 데이터
            # trainData = dataL1

            # flModel = MultiOutputRegressor(
            #     AutoML(
            #         task="regression"
            #         , metric='rmse'
            #         , time_budget=60
            #     )
            # )

            ray.init(num_cpus=12, ignore_reinit_error=True)

            flModel = AutoML(
                task="regression"
                , metric='rmse'
                # , time_budget=600
                # , early_stop = True
                , ensemble = False
                # , ensemble = True
                , seed = 123
                , time_budget=60
            )

            # 각 모형에 따른 자동 머신러닝
            flModel.fit(X_train=trainData[xCol], y_train=trainData[yCol], n_jobs=12)
            # flModel.fit(X_train=trainData[xCol], y_train=trainData[yCol], n_jobs=4, n_concurrent_trials=4)
            # flModel.fit(X_train=trainData[xCol], y_train=trainData[yCol], n_jobs=4)
            # flModel.fit(X_train=trainDataL2[xCol], y_train=trainDataL2[yCol], X_test=testDataL2[xCol], y_test=testDataL2[yCol])
            # flModel.fit(X_train=trainDataL2[xCol], y_train=trainDataL2[yCol])

            # 학습 모형 저장
            saveModel = '{}/{}/{}-{}-{}-{}-{}-{}.model'.format(globalVar['outPath'], subOpt['modelKey'], serviceName, subOpt['srvId'], 'final', 'flaml', 'for', datetime.datetime.now().strftime('%Y%m%d'))
            log.info('[CHECK] saveModel : {}'.format(saveModel))
            os.makedirs(os.path.dirname(saveModel), exist_ok=True)

            with open(saveModel, 'wb') as f:
                pickle.dump(flModel, f, pickle.HIGHEST_PROTOCOL)

        else:
            saveModel = saveModelList[0]
            log.info('[CHECK] saveModel : {}'.format(saveModel))

            with open(saveModel, 'rb') as f:
                flModel = pickle.load(f)

        result = {
            'msg': 'succ'
            , 'mlModel': flModel
            , 'saveModel': saveModel
            , 'isExist': os.path.exists(saveModel)
        }

        return result

    except Exception as e:
        log.error('Exception : {}'.format(e))
        return result

    finally:
        # try, catch 구문이 종료되기 전에 무조건 실행
        log.info('[END] {}'.format('makeFlamlModel'))

def short_term_predict(raw_data, CNSMR_NO):
    print("Predict {} Start!!".format(CNSMR_NO))
    data = raw_data.loc[raw_data['CNSMR_NO'] == CNSMR_NO, ['SRV', 'MESURE_DATE_TM', 'CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR', 'SG_PWRER_USE_AM']].reset_index(drop=True)
    data['MESURE_DATE_TM'] = pd.to_datetime(data['MESURE_DATE_TM'])
    data['is_weekday'] = data['MESURE_DATE_TM'].map(check_weekend)
    data['is_spring'] = data['MESURE_DATE_TM'].map(check_spring)
    data['is_summer'] = data['MESURE_DATE_TM'].map(check_summer)
    data['is_fall'] = data['MESURE_DATE_TM'].map(check_fall)
    data['is_winter'] = data['MESURE_DATE_TM'].map(check_winter)
    data = data.loc[:, ['CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR', 'SG_PWRER_USE_AM', 'is_weekday', 'is_spring', 'is_summer', 'is_fall', 'is_winter']]
    # 독립변수 목록 정의
    independent_variable = ['CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR', 'is_weekday', 'is_spring', 'is_summer', 'is_fall', 'is_winter']
    minmax_scaler = MinMaxScaler()
    minmax_scaler = minmax_scaler.fit_transform(data.loc[:, independent_variable])
    data.loc[:, independent_variable] = minmax_scaler

    ### 단기 예측
    t = 48
    X = np.array(data.loc[:, independent_variable])
    y = data['SG_PWRER_USE_AM']

    train_X = X[:-t, :]
    test_X = X[-t:, :]
    train_y = np.array(y[:-t])
    test_y = np.array(y[-t:])

    filename = './model/short_term/' + CNSMR_NO + '.model'
    model = pickle.load(open(filename, 'rb'))

    test_predict = model.predict(test_X)
    # print("RMSE: {}".format(np.sqrt(mean_squared_error(test_y, test_predict))))
    # print("MAPE: {}".format(mean_absolute_percentage_error(test_y, test_predict)))
    return train_X, test_X, train_y, test_y, model

# ================================================
# 4. 부 프로그램
# ================================================
class DtaProcess(object):
    # ================================================
    # 요구사항
    # ================================================
    # Python을 이용한 한국 월별 수출액과 코스피 종간 간의 회귀분석

    # ================================================================================================
    # 환경변수 설정
    # ================================================================================================
    global env, contextPath, prjName, serviceName, log, globalVar

    # env = 'local'  # 로컬 : 원도우 환경, 작업환경 (현재 소스 코드 환경 시 .) 설정
    env = 'dev'  # 개발 : 원도우 환경, 작업환경 (사용자 환경 시 contextPath) 설정
    # env = 'oper'  # 운영 : 리눅스 환경, 작업환경 (사용자 환경 시 contextPath) 설정

    if (platform.system() == 'Windows'):
        contextPath = os.getcwd() if env in 'local' else 'E:/04. TalentPlatform/Github/TalentPlatform-Python'
    else:
        contextPath = os.getcwd() if env in 'local' else '/SYSTEMS/PROG/PYTHON/PyCharm'

    prjName = 'test'
    serviceName = 'PRJ2022'

    # 4.1. 환경 변수 설정 (로그 설정)
    log = initLog(env, contextPath, prjName)

    # 4.2. 환경 변수 설정 (초기 변수)
    globalVar = initGlobalVar(env, contextPath, prjName)

    # ================================================================================================
    # 4.3. 초기 변수 (Argument, Option) 설정
    # ================================================================================================
    def __init__(self, inParams):

        log.info("[START] __init__ : {}".format("init"))

        try:
            # 초기 전달인자 설정 (파이썬 실행 시)
            # pyhton3 *.py argv1 argv2 argv3 ...
            initArgument(globalVar, inParams)

        except Exception as e:
            log.error("Exception : {}".format(e))
            raise e
        finally:
            log.info("[END] __init__ : {}".format("init"))

    # ================================================================================================
    # 4.4. 비즈니스 로직 수행
    # ================================================================================================
    def exec(self):

        log.info('[START] {}'.format("exec"))

        try:

            if (platform.system() == 'Windows'):

                # 옵션 설정
                sysOpt = {
                    # 시작/종료 시간
                    # 'srtDate': '2021-10-01'
                    # , 'endDate': '2021-10-10'
                    'srtDate': '2019-01-01'
                    , 'endDate': '2022-05-22'
                    # , 'endDate': '2021-11-01'

                    , 'stnId': '1'

                    #  딥러닝
                    , 'dlModel': {
                        # 초기화
                        'isInit': False

                        # 모형 업데이트 여부
                        , 'isOverWrite': True
                        #, 'isOverWrite': False

                        # 모형 버전 (날짜)
                        , 'modelVer': '*'
                    }

                    #  머신러닝
                    , 'mlModel': {
                        # 모델 업데이트 여부
                        # 'isOverWrite': True
                        'isOverWrite': False

                        # 모형 버전 (날짜)
                        , 'modelVer': '*'
                    }
                }

            else:

                # 옵션 설정
                sysOpt = {
                    # 시작/종료 시간
                    # 'srtDate': globalVar['srtDate']
                    # , 'endDate': globalVar['endDate']
                    # , 'stnId': globalVar['stnId']
                    'srtDate': '2019-01-01'
                    , 'endDate': '2022-05-22'
                    , 'stnId': '1'

                    #  딥러닝
                    , 'dlModel': {
                        # 초기화
                        'isInit': False

                        # 모형 업데이트 여부
                        , 'isOverWrite': True
                        #, 'isOverWrite': False

                        # 모형 버전 (날짜)
                        , 'modelVer': '*'
                    }

                    #  머신러닝
                    , 'mlModel': {
                        # 모델 업데이트 여부
                        'isOverWrite': True
                        #'isOverWrite': False

                        # 모형 버전 (날짜)
                        , 'modelVer': '*'
                    }
                }

            globalVar['inpPath'] = '/DATA/INPUT'
            globalVar['outPath'] = '/DATA/OUTPUTT'
            globalVar['figPath'] = '/DATA/FIG'

            # ******************************************************************************
            # 크몽 전문가 검증 테스트
            # ******************************************************************************
            inpFile = '{}/{}/{}.csv'.format(globalVar['inpPath'], serviceName, 'FINAL_PW_DATA')
            fileList = sorted(glob.glob(inpFile))
            # pw_data = pd.read_csv('FINAL_PW_DATA.csv')
            pw_data = pd.read_csv(fileList[0])
            pw_data['MESURE_DATE_TM'] = pd.to_datetime(pw_data['MESURE_DATE_TM'])
            # pw_data.columns

            # inpFile = '{}/{}/{}.csv'.format(globalVar['inpPath'], serviceName, 'FINAL_FORE_DATA')
            inpFile = '{}/{}/{}.csv'.format(globalVar['inpPath'], serviceName, 'FINAL_FORE_DATA_20221110')
            fileList = sorted(glob.glob(inpFile))
            # fore_data = pd.read_csv('FINAL_FORE_DATA.csv')
            fore_data = pd.read_csv(fileList[0])

            fore_data['ANA_DATE'] = pd.to_datetime(fore_data['ANA_DATE'])
            fore_data['DATE_TIME_KST'] = pd.to_datetime(fore_data['DATE_TIME_KST'])
            fore_data['DATE_TIME'] = fore_data['DATE_TIME_KST'] - dtKst

            fore_data['hour'] = fore_data['ANA_DATE'].dt.strftime("%H")
            fore_data = fore_data.loc[
                (fore_data['hour'] == '00')
                ].reset_index(drop=True)

            # fore_data['hourDiff'] = (fore_data['DATE_TIME'] - fore_data['ANA_DATE']) / pd.Timedelta(hours=1)
            # fore_data = fore_data.loc[
            #     (fore_data['hourDiff'] <= 5)
            #     ].reset_index(drop=True)

            # 분석 시간을 기준으로 0~5시간 내의 예보 시간 사용
            fore_data['hourDiff'] = (fore_data['DATE_TIME'] - fore_data['ANA_DATE']) / pd.Timedelta(hours=1)
            # fore_data = fore_data.loc[
            #     (fore_data['hourDiff'] <= 5)
            #     ].reset_index(drop=True)

            # fore_data.columns

            # 원본 데이터 : 15,319,449
            # orgData = fore_data.merge(pw_data, on=['SRV', 'MESURE_DATE_TM'])
            # orgData = fore_data.merge(pw_data, on=['SRV', 'MESURE_DATE_TM'])
            # raw_data = fore_data.merge(pw_data, on=['SRV', 'MESURE_DATE_TM'])
            # raw_data = fore_data.merge(pw_data, on=['SRV', 'DATE_TIME_KST'])
            raw_data = fore_data.merge(pw_data, how='left', left_on=['SRV', 'DATE_TIME_KST'], right_on=['SRV', 'MESURE_DATE_TM'])

            # raw_data.columns
            # orgData.columns

            # ==============================================================================
            # 단기 예측
            # ==============================================================================
            CNSMR_NO_list = list(set(raw_data['CNSMR_NO']))
            RMSE_list = []
            # MAPE_list = []

            for i, CNSMR_NO in enumerate(CNSMR_NO_list):

                # CNSMR_NO = 'SE0300104001404'
                # CNSMR_NO = 'SE0300111002104'

                saveFile = '{}/{}/model/short_term/{}.model'.format(globalVar['inpPath'], serviceName, CNSMR_NO)
                if (not os.path.exists(saveFile)): continue

                log.info("[CHECK] CNSMR_NO : {} / {}".format(CNSMR_NO, round(i / len(CNSMR_NO_list) * 100.0, 2)))
                # short_term_predict(raw_data, CNSMR_NO)

                # data = raw_data.loc[raw_data['CNSMR_NO'] == CNSMR_NO, ['SRV', 'MESURE_DATE_TM', 'CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR', 'SG_PWRER_USE_AM']].reset_index(drop=True)
                data = raw_data.loc[
                    raw_data['CNSMR_NO'] == CNSMR_NO
                    ].reset_index(drop=True)

                data['MESURE_DATE_TM'] = pd.to_datetime(data['MESURE_DATE_TM'])
                data['is_weekday'] = data['MESURE_DATE_TM'].map(check_weekend)
                data['is_spring'] = data['MESURE_DATE_TM'].map(check_spring)
                data['is_summer'] = data['MESURE_DATE_TM'].map(check_summer)
                data['is_fall'] = data['MESURE_DATE_TM'].map(check_fall)
                data['is_winter'] = data['MESURE_DATE_TM'].map(check_winter)

                data['is_weekday'] = data['is_weekday'].astype('category')
                data['is_spring'] = data['is_spring'].astype('category')
                data['is_summer'] = data['is_summer'].astype('category')
                data['is_fall'] = data['is_fall'].astype('category')
                data['is_winter'] = data['is_winter'].astype('category')

                ### 단기 예측
                t = 48

                # 중복 제거
                idxData = data.groupby(['DATE_TIME_KST'])['hourDiff'].idxmin()
                dataL1 = data.iloc[idxData.values].reset_index(drop=True)

                # 자동 머신러닝
                # xCol = ['CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR', 'is_weekday', 'is_spring', 'is_summer', 'is_fall', 'is_winter']
                xCol = ['CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR', 'is_weekday', 'is_spring', 'is_summer', 'is_fall', 'is_winter', 'CO', 'PM10', 'PM25']
                # xCol = ['CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'SWR', 'is_weekday', 'is_spring', 'is_summer', 'is_fall', 'is_winter']
                # xCol = ['CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR']
                yCol = 'SG_PWRER_USE_AM'

                trainData = dataL1[:-t].reset_index(drop=True)
                testData = dataL1[-t:].reset_index(drop=True)

                prdData = testData

                # plt.plot(testData['DATE_TIME_KST'], testData['TA'])
                # plt.plot(testData['DATE_TIME_KST'], testData['SG_PWRER_USE_AM'])
                # plt.plot(testData['DATE_TIME_KST'], testData['CO'])
                # plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%H'))
                # plt.show()

                sysOpt['mlModel'].update(
                    {
                        'srvId': CNSMR_NO
                        , 'modelKey': 'AI-FOR-20221101'
                        # , 'isOverWrite': False
                        , 'isOverWrite': True
                    }
                )

                # resPycaret = makeMlModel(sysOpt['mlModel'], xCol, yCol, trainData)
                # log.info('[CHECK] resPycaret : {}'.format(resPycaret))
                #
                # pycaretModel = resPycaret['mlModel']
                # prdData['pycaret'] = pycaretModel.predict(prdData)

                # 머신러닝 불러오기
                # resFlaml = makeFlamlModel(sysOpt['mlModel'], xCol, yCol, trainData)
                resFlaml = makeFlamlModel(sysOpt['mlModel'], xCol, yCol, trainData, testData)
                log.info('[CHECK] resFlaml : {}'.format(resFlaml))

                # 머신러닝 예측
                flamlModel = resFlaml['mlModel']
                prdData['flaml'] = flamlModel.predict(prdData)

                import optuna.integration.lightgbm as lgb

                lgbTrainData = lgb.Dataset(trainData[xCol], trainData[yCol])
                lgbTestData = lgb.Dataset(testData[xCol], testData[yCol], reference=lgbTrainData)

                # best_params, history = {}, []

                params = {
                    'objective' : 'regression'
                    , 'metric' : 'rmse'
                    , 'verbosity' : -1
                    , 'n_jobs' : -1
                }

                # model = lgb.train(params=params, train_set=dtrain, num_boost_round=5000, early_stopping_rounds=5, valid_sets=[dtrain, deval], verbose_eval=False)
                # model = lgb.train(params=params, train_set=dtrain, num_boost_round=5000, early_stopping_rounds=50, valid_sets=[dtrain, deval], verbose_eval=False)
                # model = lgb.train(params=params, train_set=dtrain, num_boost_round=10000, early_stopping_rounds=50, valid_sets=[dtrain, deval], verbose_eval=False)
                model = lgb.train(params=params, train_set=lgbTrainData, num_boost_round=10000, early_stopping_rounds=100, valid_sets=[lgbTrainData, lgbTestData], verbose_eval=False)

                saveModel = '{}/{}/{}-{}-{}-{}-{}.model'.format(globalVar['outPath'], 'AI-FOR-20221101', serviceName, CNSMR_NO, 'final', 'lightgbm', 'for')
                os.makedirs(os.path.dirname(saveModel), exist_ok=True)
                log.info('[CHECK] saveFile : {}'.format(saveModel))

                # if (not os.path.exists(saveModel)):
                #     pickle.dump(model, open(saveModel, 'wb'))
                pickle.dump(model, open(saveModel, 'wb'))

                model = pickle.load(open(saveModel, 'rb'))
                prdData['ML3'] = model.predict(data=testData[xCol])

                # 모델 영향도
                # mainTitle = '{}-{}_{}'.format(prdData['SRV'].iloc[0], CNSMR_NO, 'flaml')
                # saveImg = '{}/{}/{}.png'.format(globalVar['figPath'], sysOpt['mlModel']['modelKey'], mainTitle)
                # featData = pd.DataFrame([flamlModel.feature_names_in_, flamlModel.feature_importances_], index=['key', 'val']).transpose().sort_values(by=['val'], ascending=True)
                # plt.barh(featData['key'], featData['val'])
                # os.makedirs(os.path.dirname(saveImg), exist_ok=True)
                # plt.title(mainTitle)
                # plt.legend()
                # plt.tight_layout()
                # plt.savefig(saveImg, dpi=600, bbox_inches='tight')
                # plt.show()
                # plt.close()

                # mainTitle = '{}-{}_{}'.format(prdData['SRV'].iloc[0], CNSMR_NO, 'lightgbm')
                # saveImg = '{}/{}/{}.png'.format(globalVar['figPath'], sysOpt['mlModel']['modelKey'], mainTitle)
                # plot_importance(model)
                # plt.title(mainTitle)
                # plt.tight_layout()
                # plt.savefig(saveImg, dpi=600, bbox_inches='tight')
                # plt.show()
                # plt.close()

                mainTitle = '[48시간 단기] [{0:s}-{1:s}]'.format(prdData['SRV'].iloc[0], CNSMR_NO)
                saveImg = '{}/{}/{}.png'.format(globalVar['figPath'], serviceName, mainTitle)
                os.makedirs(os.path.dirname(saveImg), exist_ok=True)
                log.info('[CHECK] saveImg : {}'.format(saveImg))

                plt.plot(prdData['DATE_TIME_KST'], prdData['SG_PWRER_USE_AM'], marker='o', label='실측')
                # plt.plot(prdData['MESURE_DATE_TM'], prdData['prd'], label='예측 (RMSE : {:.2f}, {:.2f}%)'.format(
                #     mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['prd'], squared=False)
                #     , (mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['prd'], squared=False) / np.nanmean(prdData['SG_PWRER_USE_AM'])) * 100.0)
                #     )
                plt.plot(prdData['DATE_TIME_KST'], prdData['flaml'], label='예측 (RMSE : {:.2f}, {:.2f}%)'.format(
                    mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['flaml'], squared=False)
                    , (mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['flaml'], squared=False) / np.nanmean(prdData['SG_PWRER_USE_AM'])) * 100.0)
                    )
                plt.plot(prdData['DATE_TIME_KST'], prdData['ML3'], label='예측2 (RMSE : {:.2f}, {:.2f}%)'.format(
                    mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['ML3'], squared=False)
                    , (mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['ML3'], squared=False) / np.nanmean(prdData['SG_PWRER_USE_AM'])) * 100.0)
                    )
                # plt.plot(prdData['MESURE_DATE_TM'], prdData['ML4'], label='예측4 (RMSE : {:.2f}, {:.2f}%)'.format(
                #     mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['ML4'], squared=False)
                #     , (mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['ML4'], squared=False) / np.nanmean(prdData['SG_PWRER_USE_AM'])) * 100.0)
                #     )
                # plt.plot(prdData['MESURE_DATE_TM'], prdData['lazy'], label='예측 (lazy) : {:.3f}'.format(round(np.sqrt(mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['flaml'])), 2)))
                # plt.plot(prdData['MESURE_DATE_TM'], prdData['pycaret'], label='예측 (pycaret) : {:.3f}'.format(round(np.sqrt(mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['pycaret'])), 2)))

                plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y.%m.%d %H'))
                plt.gca().xaxis.set_major_locator(mdates.HourLocator(interval=3))
                plt.gcf().autofmt_xdate()
                # plt.xticks(rotation=45, ha='right')
                plt.xticks(rotation=45)
                plt.title(mainTitle)
                plt.legend()
                plt.tight_layout()
                plt.savefig(saveImg, dpi=600, bbox_inches='tight', transparent=True)
                # plt.show()
                plt.close()

        except Exception as e:
            log.error("Exception : {}".format(e))
            raise e

        finally:
            log.info('[END] {}'.format("exec"))

# ================================================
# 3. 주 프로그램
# ================================================
if __name__ == '__main__':

    print('[START] {}'.format("main"))

    try:

        # 파이썬 실행 시 전달인자를 초기 환경변수 설정
        inParams = {}

        print("[CHECK] inParams : {}".format(inParams))

        # 부 프로그램 호출
        subDtaProcess = DtaProcess(inParams)

        subDtaProcess.exec()

    except Exception as e:
        print(traceback.format_exc())
        sys.exit(1)

    finally:
        print('[END] {}'.format("main"))


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# -------------------------------
# SDA: Pin3
# SCL: Pin5

import smbus
import time

LCD_ADDRESS             = 0x3e
RGB_ADDRESS             = 0x60

'''
 *  @brief color define
'''
WHITE                   = 0
RED                     = 1
GREEN                   = 2
BLUE                    = 3

REG_RED                 = 0x04        # pwm2
REG_GREEN               = 0x03        # pwm1
REG_BLUE                = 0x02        # pwm0

REG_MODE1               = 0x00
REG_MODE2               = 0x01
REG_OUTPUT              = 0x08

'''
 *  @brief commands
'''
LCD_CLEARDISPLAY        = 0x01
LCD_RETURNHOME          = 0x02
LCD_ENTRYMODESET        = 0x04
LCD_DISPLAYCONTROL      = 0x08
LCD_CURSORSHIFT         = 0x10
LCD_FUNCTIONSET         = 0x20
LCD_SETCGRAMADDR        = 0x40
LCD_SETDDRAMADDR        = 0x80

'''
 *  @brief flags for display entry mode
'''
LCD_ENTRYRIGHT          = 0x00
LCD_ENTRYLEFT           = 0x02
LCD_ENTRYSHIFTINCREMENT = 0x01
LCD_ENTRYSHIFTDECREMENT = 0x00

'''
 *  @brief flags for display on/off control
'''
LCD_DISPLAYON           = 0x04
LCD_DISPLAYOFF          = 0x00
LCD_CURSORON            = 0x02
LCD_CURSOROFF           = 0x00
LCD_BLINKON             = 0x01
LCD_BLINKOFF            = 0x00

'''
 *  @brief flags for display/cursor shift
'''
LCD_DISPLAYMOVE         = 0x08
LCD_CURSORMOVE          = 0x00
LCD_MOVERIGHT           = 0x04
LCD_MOVELEFT            = 0x00

'''
 *  @brief flags for function set
'''
LCD_8BITMODE            = 0x10
LCD_4BITMODE            = 0x00
LCD_2LINE               = 0x08
LCD_1LINE               = 0x00
LCD_5x10DOTS            = 0x04
LCD_5x8DOTS             = 0x00
'''
BUS = smbus.SMBus(1)

class LCD:
    def __init__(self, row, col):
        self._row = row
        self._col = col
        print("LCD _row=%d _col=%d"%(self._row,self._col))

    def print(self,arg):
        if(isinstance(arg,int)):
            arg=str(arg)

        for x in bytearray(arg):
           self.write(x)
'''
class LCD1602():
    def __init__(self, col=16, row=2, i2c=None):
        self._showcontrol = LCD_DISPLAYON | LCD_CURSOROFF | LCD_BLINKOFF
        self._showmode = LCD_ENTRYLEFT | LCD_ENTRYSHIFTDECREMENT
        if i2c == None:
            self._i2c = smbus.SMBus(1)
        else:
            self._i2c = i2c
        self._row = row
        self._col = col
        print("LCD: _row=%d, _col=%d"%(self._row,self._col))

        #super(LCD1602, self).__init__(row, col)
        for cmd in(
        LCD_FUNCTIONSET | LCD_2LINE,
        LCD_DISPLAYCONTROL | LCD_DISPLAYON | LCD_CURSOROFF | LCD_BLINKOFF,
        LCD_CLEARDISPLAY,
        LCD_ENTRYMODESET | LCD_ENTRYLEFT | LCD_ENTRYSHIFTDECREMENT
        ):
            self.command(cmd)
            time.sleep(0.1)
        self.setReg(REG_MODE1, 0)
        self.setReg(REG_OUTPUT, 0xff)
        self.setReg(REG_MODE2, 0x20)
        self.setRGB(255, 255, 255)

    def printLCD(self,arg):
        if(isinstance(arg,int)):
            arg=str(arg)

        for x in bytearray(arg):
           self.write(x)

    def command(self,cmd):
        b = bytearray(2)
        b[0] = 0x80
        b[1] = cmd
        self._i2c.write_byte_data(LCD_ADDRESS, b[0], b[1])

    def write(self,data):
        b = bytearray(2)
        b[0] = 0x40
        b[1] = data
        self._i2c.write_byte_data(LCD_ADDRESS, b[0], b[1])

    def setReg(self,reg,data):
        b = bytearray(1)
        b[0] = data
        self._i2c.write_byte_data(RGB_ADDRESS,reg,b[0])

    def setRGB(self,r,g,b):
        self.setReg(REG_RED,  r)
        self.setReg(REG_GREEN,  g)
        self.setReg(REG_BLUE,  b)

    def setCursor(self,col,row):
        if(row == 0):
            col |= 0x80
        else:
            col |= 0xc0
        self.command(col)

    def clear(self):
        self.command(LCD_CLEARDISPLAY)
        time.sleep(0.002)

    def setPWM(self, colorAddr, pwm):
        self.setReg(colorAddr, pwm)

    def autoscroll(self):
        self._showmode |= LCD_ENTRYSHIFTINCREMENT
        self.command(LCD_ENTRYMODESET | self._showmode)

    def noAutoscroll(self):
        self._showmode &= ~LCD_ENTRYSHIFTINCREMENT
        self.command(LCD_ENTRYMODESET | self._showmode)

    def display(self):
        self._showcontrol |= LCD_DISPLAYON
        self.command(LCD_DISPLAYCONTROL | self._showcontrol)

    def noDisplay(self):
        self._showcontrol &= ~LCD_DISPLAYON
        self.command(LCD_DISPLAYCONTROL | self._showcontrol)

    def blink(self):
        self._showcontrol |= LCD_BLINKON
        self.command(LCD_DISPLAYCONTROL | self._showcontrol)

    def stopBlink(self):
        self._showcontrol &= ~LCD_BLINKON
        self.command(LCD_DISPLAYCONTROL | self._showcontrol)

    def cursor(self):
        self._showcontrol |= LCD_CURSORON
        self.command(LCD_DISPLAYCONTROL | self._showcontrol)

    def noCursor(self):
        self._showcontrol &= ~LCD_CURSORON
        self.command(LCD_DISPLAYCONTROL | self._showcontrol)

    def leftToRight(self):
        self._showmode |= LCD_ENTRYLEFT
        self.command(LCD_ENTRYMODESET | self._showmode)

    def rightToLeft(self):
        self._showmode &= ~LCD_ENTRYLEFT
        self.command(LCD_ENTRYMODESET | self._showmode)

    def scrollDisplayLeft(self):
        self.command(LCD_CURSORSHIFT | LCD_DISPLAYMOVE | LCD_MOVELEFT)

    def scrollDisplayRight(self):
        self.command(LCD_CURSORSHIFT | LCD_DISPLAYMOVE | LCD_MOVERIGHT)

    def home(self):
        self.command(LCD_RETURNHOME)    # set cursor position to zero
        time.sleep(2)                   # this command takes a long time!

    def customSymbol(self, location, charmap):
        location &= 0x7 # we only have 8 locations 0-7
        self.command(LCD_SETCGRAMADDR | (location << 3))

        for x in charmap:
            self.write(x)

if __name__ == '__main__':
    print("========Raspberry Pi 3 I2C LCD1602 TEST===========")
    num = 0
    lcd=LCD1602()
    lcd.setRGB(0,0,255)
    #lcd.setCursor(0,0)
    lcd.printLCD("Hello world!")

    #lcd.setCursor(5,1)
    #lcd.printLCD("RGBBacklight TEST")
    #lcd.home()
    #lcd.printLCD(3322)
    while True:
        lcd.setCursor(0, 1)
        lcd.printLCD(num)
        num += 1
        time.sleep(1)


import os

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential

def get_data():
    # load data from mnist dataset
    (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()

    # standardize data: reshape images, so that all images are of height: 28, width: 28, & depth: 1
    x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
    x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)

    # convert data to floats
    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')

    # normalize RGB values
    x_train /= 255
    x_test /= 255

    y_train = tf.keras.utils.to_categorical(y_train, 10)
    y_test = tf.keras.utils.to_categorical(y_test, 10)

    return x_train, y_train, x_test, y_test

def create_model():
    x_train, y_train, x_test, y_test = get_data()

    # initialize number of classes to 10 (1 for each digit)
    num_classes = 10

    # create model
    # include dropout: exclude 20% of nodes in next layer
    model = Sequential([
        layers.Conv2D(32, kernel_size=(3, 3), padding='same', activation='relu', input_shape=(28, 28, 1)),
        layers.MaxPooling2D(),
        layers.Conv2D(64, kernel_size=(3, 3), padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Dropout(0.2),
        layers.Flatten(),
        layers.Dense(128, activation='relu'),
        layers.Dense(num_classes, activation='softmax')
    ])

    # compile model
    model.compile(keras.optimizers.Adam(),
                  loss=tf.keras.losses.categorical_crossentropy,
                  metrics=['accuracy'])

    # train model
    epochs = 10
    model.fit(
        x_train, y_train,
        batch_size=64,
        validation_data=(x_test, y_test),
        epochs=epochs
    )

    # save model
    model.save('model.h5')

if __name__=='__main__':
    if not os.path.exists('model.h5'):
        create_model()
    model = keras.models.load_model('model.h5')

    _, _, x_test, y_test = get_data()
    print(model.evaluate(x_test, y_test, False))


import pandas as pd
from pycaret.classification import predict_model, load_model
class PredictChurn:

    def load_data(filepath):
        """
        Loads churn data into a DataFrame from a string filepath.
        """
        df = pd.read_csv(filepath, index_col='customerID')
        return df

    def make_predictions(df):
        """
        Uses the pycaret best model to make predictions on data in the df dataframe.
        """
        model = load_model('gbc')
        predictions = predict_model(model, data=df)
        predictions.rename({'Label': 'of Churn', 'Score': 'Probability'}, axis=1, inplace=True)
        predictions['of Churn'].replace({1: 'Churn', 0: 'No Churn'},
                                                inplace=True)
        return predictions[{'of Churn', 'Probability'}]

    if __name__ == "__main__":
        df = load_data('new_churn_data.csv')
        predictions = make_predictions(df)
        print(predictions)


#Q.15)Python Program to Find the Factorial of a Number
num = int(input("number::"))
fact = 1
if num < 0:
    print("not possible")
elif num == 0:
    print("factorial of zero is always 1")
else:
    for i in range(1, num+1):
        fact = fact * i
    print('The factorial of',num, "is:",fact)

Question16-Python Program to Find the Factorial of a Number.py


import cv2, math, numpy as np
# --------------------- START PARAMETERS -----------------------
# h,s,v range of the object to be tracked - get these values with hsv_color_picker.py
color_values = 128,119,55,157,255,126
#color_values = 115,110,45,175,255,186
color_values = 126,107,7,206,255,152

# Capture your source video
cap = cv2.VideoCapture('/home/stephen/Desktop/642.MP4')
# Pick a path to save the data
output_path = '/home/stephen/Desktop/data/1.csv'
# How big is a ball? This is the dark blue circle (9 to 60)
ball_size = 15
# How big of a colored swatch is necessary to deam it as a ball (25 to 81 is good)
contour_ball_size = 15
global max_snap_distance, max_direction_deviation
# The tracker 'snaps' onto the ball's position. 45 is good - light blue circle
max_snap_distance = 20
# The ball shouldn't change direction drastically 9 is good for 120fps
max_direction_deviation = 35
deviation_warning = 5
# If the tracker gets lost, the last few tracking values are erroneous
# Preserving the history allows the user can skip back a bit when a tracker error occurs
p_imgs, history_length, history_idx = [], 30, 0
# -------------------- END PARAMETERS ----------------------
# Parameters for the text in the user instructions
font, scale, color, thick = cv2.FONT_HERSHEY_SIMPLEX, .5, (255,0,0), 1
# Takes an image, and a lower and upper bound
# Returns only the parts of the image in bounds
def only_color(frame, color_valeus, p_position):
    # Create a mask
    mask = np.zeros((frame.shape[0], frame.shape[1]), np.uint8)
    # Draw a circle around the last known position of the ball
    # Everything outside the circle is value=0
    # Only objects inside the circle are possible juggling balls
    mask = cv2.circle(mask, p_position, max_snap_distance, 255, -1)
    # Mask out the image
    frame = cv2.bitwise_and(frame, frame, mask=mask)
    # Convert BGR to HSV
    b,r,g,b1,r1,g1 = color_values
    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
    # Define range of blue color in HSV
    lower, upper = np.array([b,r,g]), np.array([b1,r1,g1])
    # Threshold the HSV image to get only blue colors
    mask = cv2.inRange(hsv, lower, upper)
    # Bitwise-AND mask and original image
    res = cv2.bitwise_and(frame,frame, mask= mask)
    #cv2.imshow('mask', mask)
    return res, mask
# Finds the largest contour in a list of contours
# Returns a single contour
def largest_contour(contours):
    c = max(contours, key=cv2.contourArea)
    return c[0]

# Takes an image and the threshold value returns the contours
def get_contours(im):
    imgray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)
    _ ,thresh = cv2.threshold(imgray,0,255,0)
    contours, _ = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
    return contours

# Finds the center of a contour
# Takes a single contour
# Returns (x,y) position of the contour
def contour_center(c):
    M = cv2.moments(c)
    try: center = int(M['m10']/M['m00']), int(M['m01']/M['m00'])
    except: center = 0,0
    return center

# Finds the most likely position of the ball in an image using color
# Returns xy coordinates
def find_ball_by_color(img, color_values, ball_position):
    # Filter the image for the specific ball color
    img, mask = only_color(img, color_values, ball_position)
    # Find the contours in the image
    contours = get_contours(img)
    # Start with values that are definitely out of the snap range
    nearest_ball = 100000000
    nearest_snap_possibility = (10000,10000)
    # Iterate through the contours
    for contour in contours:
        # Check if a contour is big enough to be considered a ball
        if cv2.contourArea(contour)>contour_ball_size:
            # Get the center of the contour
            center = contour_center(contour)
            # This contour is the nearest one to the ball, it is the most likely snap position
            if distance(center, ball_position) < nearest_ball:
                # Save this as the nearest likely ball
                nearest_snap_possibility = center
    # Return the mostly likely location of the ball in the image
    return nearest_snap_possibility

def distance(a,b): return math.sqrt((a[0]-b[0])**2+(a[1]-b[1])**2)

# Parameters of lk optical flow
lk_params = dict(winSize = (ball_size, ball_size),
                 maxLevel = 5,
                 criteria = (cv2.TERM_CRITERIA_EPS |
                             cv2.TERM_CRITERIA_COUNT, 5, 0.03))

# Mouse callback function
global click_list
positions, click_list = [], []
def callback(event, x, y, flags, param):
    if event == 1: click_list.append((x,y))
cv2.namedWindow('img')
cv2.setMouseCallback('img', callback)

# Initialize frame_number
frame_number = 0
wait_time = 0

# Initialize tracker value, this will get reset by the user
p0 = np.array([[[0,0]]], np.float32)
fails = 0
# Main loop
while True:

    # Program crashes if user tries to use the history before it is built completely
    if len(p_imgs)< history_length - history_idx: print("Rebuilding history...")

    # Read frame of video (either from the source or the history)
    if history_idx == 0:
        ret, img = cap.read()
        try:
            #img = cv2.resize(img, (1908, 1080)) #resize the image so it fills screen
            #img = cv2.blur(img, (15,15))
            pass
        except: break
        # Save the frame to the history
        if ret: p_imgs.append(img.copy())
    else:
        # Read a frame from the history
        img = p_imgs.pop(0)
        history_idx -= 1

    # Break if the video is over
    try:h,w,e = img.shape
    except: break

    # Optical flow works in single channel, so save the previous gray image
    try: old_gray = img_gray.copy()
    except: old_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    try: img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    except: break

    # Track using optical flow
    # p1 is the value produced by the optical flow tracker
    p1, _, _ = cv2.calcOpticalFlowPyrLK(old_gray, img_gray,p0, None, **lk_params)

    # Convert the coordinates to integers for drawing
    xy  = int(p1[0][0][0]), int(p1[0][0][1])

    # Get the nearest likely snap position
    snap_position = find_ball_by_color(img, color_values, xy)
    # Draw a circle around the snap position showing the max snap distance and max directional deviation
    cv2.circle(img, snap_position, max_snap_distance, (255,255,0), 1)
    cv2.circle(img, snap_position, max_direction_deviation, (0,255,255), 1)
    # Is the likely position found using color close to the optical flow tracker
    if distance(snap_position, xy) < max_snap_distance:
        # The color value is a good one, so snap to that
        xy = snap_position
        p1 = np.array([[[snap_position[0], snap_position[1]]]], np.float32)

    # Update the predicted tracking point
    p0 = p1

    # If there are at least two positions, use those positions to calculate a likely position
    # where the ball will be in the future, if the ball isn't there, tracker failure!
    if len(positions)>2:
        # Get the last two tracking locations
        past, present = positions[-2:]
        # Calculate the predicted future position of the ball
        dx, dy = present[0]-past[0], present[1]-past[1]
        future = present[0]+dx, present[1]+dy
        #print(dx,dy, distance(future, xy))
        cv2.circle(img, future, 2, (255, 255, 255), -1)
        # If the tracking location and the predicted location don't match, stop the tracker and tell the user
        if distance(future, xy) > deviation_warning:
            print("Direction Warning: " , distance(future, xy))
        else:
            if fails > 0: fails -=1
        if distance(future, xy) > max_direction_deviation:
            print("Direction Fail: ", distance(future, xy))
            wait_time = 0

    # Make circle around ball to show the tracking point
    cv2.circle(img, xy, ball_size, (255,2,1), 1)

    # Write instructions to user
    if wait_time == 0:
        cv2.putText(img, "Click on ball and press F5 to reset tracker", (50,50), font, scale, color, thick, cv2.LINE_AA)
        if len(positions)>0:
            cv2.putText(img, "...or press F6 to continue to the next frame", (50,80), font, scale, color, thick, cv2.LINE_AA)
            cv2.putText(img, "...or press F7 to autotrack slowly", (50,110), font, scale, color, thick, cv2.LINE_AA)
            cv2.putText(img, "...or press F8 to autotrack at max speed", (50,140), font, scale, color, thick, cv2.LINE_AA)
            cv2.putText(img, "...or press F4 to use the predicted value (white dot)", (50,170), font, scale, color, thick, cv2.LINE_AA)
    if wait_time == 1 or wait_time == 25:
        cv2.putText(img, "Auto-Tracking", (50,50), font, scale, color, thick, cv2.LINE_AA)
        cv2.putText(img, "Press any key on mistake", (50,80), font, scale, color, thick, cv2.LINE_AA)
        cv2.putText(img, "Press F6 to pause", (50,110), font, scale, color, thick, cv2.LINE_AA)

    # Draw the previous ball positions on the screen
    idx = 0
    while idx < 20 and idx < len(positions)-3:
        a,b = positions[-(idx+2)], positions[-(idx+1)]
        cv2.line(img, a,b, (0,255,86), 4)
        idx += 1

    # Show frame and wait
    cv2.imshow('img', img)
    k = cv2.waitKey(wait_time)
    if k ==27: break

    # User presses F4, use the the predicted value
    if k == 193:
        xy = future
        p0 = np.array([[xy]], np.float32)
    # User Presses F6, advance to the next frame and wait
    if k == 195: wait_time = 0
    # User presses F7, play slowly
    if k == 196: wait_time = 65
    # User presses F8, max computer resources
    if k == 197: wait_time = 1
    # User wants to reset the tracker to the last click position
    if k == 194: #F5
        # Get the user's input
        xy = click_list[-1]
        # Set the tracker to this user input
        p0 = np.array([[xy]], np.float32)
        # Advance to the next frame, but pause
        wait_time = 0

    # Add tracked location to data
    positions.append(xy)

    # If there are enough items in the history, remove the oldest one
    if len(p_imgs)>history_length: p_imgs.pop(0)

    # User has paused any other key because the tracker has gotten lost
    if k < 190 and k >10:
        # Pause
        wait_time = 0
        # Get the history length
        history_idx = history_length
        # Remove the last few positions, that are errors from a broken tracker
        positions = positions[:-history_length]

# Close the window and release the source video
cv2.destroyAllWindows()
cap.release()

# Write data to a csv (spreadsheet) file
import csv
with open(output_path, 'w') as csvfile:
    fieldnames = ['x_position', 'y_position']
    writer = csv.DictWriter(csvfile, fieldnames = fieldnames)
    writer.writeheader()
    for position in positions:
        x, y = position[0], position[1]
        writer.writerow({'x_position': x, 'y_position': y})
print( 'finished writing data')


from github import Github, RateLimitExceededException
import os

from django.shortcuts import render
from django.http import Http404
from django.http import HttpResponse
from .ghproj import GithubProject

def index(request):
    reponame = os.environ.get("REPO_NAME")
    domain = request.GET.get('domain')
    if domain:
        return status(request, domain)
    try:
        context = { 'domains': GithubProject.list_domains(reponame) }
        return render(request, 'status/index.html', context)
    except RateLimitExceededException:
        raise Http404(f"Rate limit exceeded, sorry!")

def status(request, domain):
    reponame = os.environ.get("REPO_NAME")
    try:
        issues, ok = GithubProject.get_domain(reponame, domain)
        if issues or ok:
            context = {'reponame': reponame, 'domain': domain, 'issues': issues, 'ok': ok }
            return render(request, 'status/status.html', context)
        raise Http404(f"No such domain {domain}")
    except RateLimitExceededException:
        raise Http404(f"Rate limit exceeded, sorry!")


from requests_html import HTMLSession
import os
from dotenv import load_dotenv
load_dotenv()

playlist_url = os.getenv('APPLE_MUSIC_PLAYLIST_URL')
storage_directory = os.getenv('STORAGE_DIRECTORY')
session = HTMLSession()
r = session.get(playlist_url)
cwd = os.path.abspath(__file__)

artist_item_data_attribute_name = 'data-test-song-artist-url'

tracklists_items = r.html.find('.tracklist-item--song .tracklist-item__text__headline')
artists_items = r.html.find(f'.tracklist-item--song [{artist_item_data_attribute_name}]')
tracklists_headlines = list(map(lambda tracklist_item: tracklist_item.text, tracklists_items))
artists_headlines = list(map(lambda artist_item: artist_item.attrs[artist_item_data_attribute_name], artists_items))

songs = [f'{artist} {tracklists_headlines[i]}' for i, artist in enumerate(artists_headlines)]

for song in songs:
  download_result = os.popen(f'instantmusic -p -s "{song}"').read()

  if storage_directory:
    file_name = download_result.split('[ffmpeg] Destination: ', 1)[1].split('\n', 1)[0]
    os.system(f'mv "./{file_name}" {storage_directory}/')


# 创建一个背景为蓝色的pygame窗口
import sys #使用其中工具退出游戏

import pygame

class Game:
    def __init__(self):
        pygame.init() # 初始化背景设置
        self.screen = pygame.display.set_mode((1920,1080)) #设置窗口
        self.bg_color = (0,104,120) #设置背景色
        pygame.display.set_caption("一个练习窗口") #设置标题

    def run_game(self):
        #游戏主循环
        while True:

          for event in pygame.event.get() : #监视事件
              if event.type == pygame.QUIT:  #按x就结束程序
                  sys.exit()

          self.screen.fill(self.bg_color) #绘制屏幕
          pygame.display.flip() #最近绘制的屏幕可见

# 创建实例
my_exercise=Game()
my_exercise.run_game()


#!/usr/bin/env python3

# Usage: ./dummy_data_generator.py > file.sql
#  Note: this might only work with POSIX shells

# If you are using windows you can either copy paste the output into a file
# or reconsider using a better operating system.

from random import randint, uniform, choice

models_dict = {
    "Smartphone" : {
        "Apple" : ["iPhone SE", "iPhone 14"],
        "Samsung" : ["Galaxy S21", "Galaxy S7", "Galaxy A72"]
    },
    "TV" : {
        "LG" : ["B2 55\"", "C1 77\""]
    },
    "Laptop" : {
        "Apple" : ["MacBook Air 2022", "MacBook Pro 2020"],
        "Razer" : ["Razer Blade Stealth 2018", "Razer Blade 2020"]
    }
}

descriptions = [
    "Wooow look at this description!!",
    "Have a nice day!!",
    "This device does not work and will never work. Please do not waste your time on trying to fix this thing",
    "Soll ich die Beschreibungen auf Deutsch schreiben?",
    "Je m''appelle Paul",
    "J''adore le kebab",
    "Roma in Italiā est.",
    "These description where written while tracking my time.",
    "Das Gerät hat einen dubiosen Geruch; ich befürchte es funktioniert nicht mehr."
]

states = ["available", "reserved", "sold"]

# This will transform the weird hashmap into a list of tuples
# e.g. {"Smartphone" : {"Apple" : ["iPhone SE", "iPhone 14"} will become
#   [("Smartphone", "Apple", "iPhone SE"), ("Smartphone", "Apple", "iPhone 14")]
models = []
for typ, x in models_dict.items():
    for brand, y in x.items():
        for model in y:
            models.append((typ, brand, model))

# coordinates of vienna
# the backend (and this script) use WGS 84 (EPSG 4326) coordinates
longitude_r = (16.18, 16.58)
latitude_r = (48.11, 48.32)

user_id_r = (1, 20)
location_id_r = (1, 40)
collect_appointment_id_r = (1, 100)

num_items = 1000
num_models = len(models)

print("-- this was generated by the dummy_data_generator.py")
print("-- item models")

for i, (typ, brand, model) in enumerate(models):
    print(f"INSERT INTO item_model VALUES ({i+1}, '{typ}', '{brand}', '{model}');")

print("\n-- items")

for _ in range(num_items):
    model_id = randint(1, num_models)
    user_id = randint(*user_id_r)
    location_id = randint(*location_id_r)

    long = uniform(*longitude_r)
    lat = uniform(*latitude_r)
    description = choice(descriptions)

    status = choice(states)
    reservee = "NULL"
    if status != "available":
        # maybe check if it is not the same as user id
        reservee = randint(*user_id_r)

    collect_appointment = "NULL"
    if status == "sold":
        collect_appointment = randint(*collect_appointment_id_r)

    print("INSERT INTO item (description, model_id, owner, location, status, reservee, collect_appointment, position) VALUES",
          f"('{description}', {model_id}, {user_id}, {location_id}, '{status}'::ITEM_STATUS, {reservee}, {collect_appointment},",
          f"ST_Point({long}, {lat}, 4326));")


import gradio as gr
from utils import summarize_text

demo = gr.Interface(
    summarize_text,
    inputs=[
        gr.Textbox(label="Youtube link", placeholder="Enter Youtube link here..."),
        gr.Dropdown(["auto", "en"], label="Language", value="auto"),
    ],
    outputs=[
        gr.Textbox(label="Audio2Text", placeholder="Content of the Youtube video", lines=5),
        gr.Textbox(label="Summarization", placeholder="Main content of the Youtube video", lines=5)
    ],
    examples=[["https://www.youtube.com/watch?v=Uew5BbvmLks", "en"], ["https://www.youtube.com/watch?v=p79GmLNLMrY", "en"], ["https://www.youtube.com/watch?v=cZv-m3Bvnn4", "auto"]],
    allow_flagging="never"
    )

if __name__ == '__main__':
    demo.launch()


import argparse
import datetime
import logging
import os
import random
import numpy as np
from sklearn.metrics import f1_score, accuracy_score
import torch.nn as nn
import torch.nn.functional as F
import torchquantum as tq
import torchquantum.functional as tqf
import torch
from torch.utils.data import DataLoader, dataset
from torch.autograd import Variable

from ExpDataset import MustardDataset
from hyper_mt import HyperModel
from m2model_train import M2Model

class Q_Model(nn.Module):
  def __init__(self,bszi):
    super().__init__()
    self.encoder= tq.GeneralEncoder([
        {'input_idx': [0], 'func': 'rx', 'wires': [0]},
        {'input_idx': [1], 'func': 'rx', 'wires': [1]},
        {'input_idx': [2], 'func': 'rx', 'wires': [2]},
        {'input_idx': [3], 'func': 'rx', 'wires': [3]},
        {'input_idx': [4], 'func': 'rx', 'wires': [4]},
        {'input_idx': [5], 'func': 'rx', 'wires': [5]},
      ])

    self.sar_L=nn.Linear(3,3)
    self.sen_L=nn.Linear(3,3)

    self.measure = tq.MeasureAll(tq.PauliZ)

    self.re_sar_L=nn.Linear(3,2)
    self.re_sen_L=nn.Linear(3,3)

  def forward(self,sa,sen):

    device=tq.QuantumDevice(n_wires=6,bsz=sa.shape[0],device='cuda')
    # sa,sen=self.classM(txt,img,audio)
    sa=self.sar_L(sa)
    sen=self.sen_L(sen)

    qin=torch.concat([sen,sa],1)
    self.encoder(device,qin)

    tq.cx(device,[0,1])
    tq.cx(device,[1,2])
    tq.cx(device,[2,0])

    tq.cx(device,[3,4])
    tq.cx(device,[4,5])
    tq.cx(device,[5,3])

    x=self.measure(device)
    sar=x[:,3:6]

    sar=self.re_sar_L(sar)
    sen=x[:,0:3]

    sen=self.re_sen_L(sen)
    return sar,sen

def trainEval():

    batchsize = 500
    epochs = 40

    all_epochs=280

    now = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')

    stateSavePATH = f'result/state/{all_epochs}_Epochs_{now}/state.pt'
    modelSavePATH = f'result/state/{all_epochs}_Epochs_{now}/model.pt'

    data = MustardDataset(datatye='train')
    valData = MustardDataset(datatye='dev')

    data_loader = DataLoader(
        data,
        batch_size=batchsize,
        shuffle=True,
        pin_memory=True
        )
    val_loader = DataLoader(
        valData,
        batch_size=50,
        shuffle=True,
        pin_memory=True,
        )
    class_model=torch.load('result/state/80_Epochs_2023-06-24-12-51-18/model.pt')
    class_model.eval()
    model = Q_Model(bszi=batchsize).cuda()

    lossFun = nn.CrossEntropyLoss().cuda()
    lossFun_val = nn.CrossEntropyLoss().cuda()

    optimizer = torch.optim.Adam(
        model.parameters(), lr=0.05, weight_decay=0.015)

    total = sum([param.nelement() for param in model.parameters()])
    print(f"Number of parameter: {total}")

    train_step = 0
    os.mkdir(f'result/state/{all_epochs}_Epochs_{now}')
    for _ in range(epochs):
        logger.info('epoch:'+str(_))

        for batch in data_loader:
            train_step += 1

            model.train()
            textInput = batch[0][0].cuda().to(torch.float32)
            imageInput = batch[0][1].to(torch.float32).cuda()
            audioInput = batch[0][2].to(torch.float32).cuda()

            sarLabel = batch[1][0].to(torch.float32).cuda()
            sentLabel = batch[1][1].to(torch.float32).cuda()

            with torch.no_grad():
               sar, sent= class_model.Qin(textInput, imageInput,audioInput)

            sar,sent=model(sar,sent)

            sarArgmax = torch.argmax(sarLabel, dim=-1)
            sentArgmax = torch.argmax(sentLabel, dim=-1)
            # emoArgmax = torch.argmax(emoLabel, dim=-1)
            # logger.info(f'sar:{sarArgmax} sent:{sentArgmax} emo:{emoArgmax}\n')
            loss1 = lossFun(sar, sarArgmax)
            loss2 = lossFun(sent, sentArgmax)
            # loss3 = lossFun(emo, emoArgmax)
            loss = (loss1 + loss2 )/2

            loss.requires_grad_(True)

            logger.info('loss1:%f loss2:%f loss:%f\n'
                        % (loss1.item(),
                            loss2.item(),
                            loss.item()))

            if train_step%(epochs-1) == 0:
                label_sar = np.argmax(
                    sarLabel.cpu().detach().numpy(), axis=-1)
                label_sent = np.argmax(
                    sentLabel.cpu().detach().numpy(), axis=-1)
                # label_emo = np.argmax(
                #     emoLabel.cpu().detach().numpy(), axis=-1)
                pred_sar = np.argmax(
                    sar.cpu().detach().numpy(), axis=1)
                pred_sent = np.argmax(
                    sent.cpu().detach().numpy(), axis=1)

                sar_f1 = f1_score(label_sar, pred_sar, average='micro')
                sent_f1 = f1_score(label_sent, pred_sent, average='micro')
                # emo_f1 = f1_score(label_emo, pred_emo, average='micro')
                sar_acc = accuracy_score(
                    label_sar, pred_sar)
                sent_acc = accuracy_score(
                    label_sent, pred_sent)

                logger.info(('train result sar-f1:%f sent-f1:%f ' +
                            'sar-acc:%f sent-acc:%f \n')
                            % (sar_f1, sent_f1,
                            sar_acc, sent_acc))

                model.eval()
                with torch.no_grad():
                    outputsar, outputsent, outputemo = [], [], []
                    tarsar, tarsent, taremo = [], [], []
                    for batch in val_loader:
                        textInput = batch[0][0].cuda().to(torch.float32)
                        imageInput = batch[0][1].cuda().to(torch.float32)
                        audioInput = batch[0][2].cuda().to(torch.float32)

                        sarLabel = batch[1][0].to(torch.float32).cuda()
                        sentLabel = batch[1][1].to(torch.float32).cuda()

                        with torch.no_grad():
                           sar, sent=class_model.Qin(textInput, imageInput,audioInput)

                        sar,sent=model(sar,sent)

                        sarArgmax = torch.argmax(sarLabel, dim=-1)
                        sentArgmax = torch.argmax(sentLabel, dim=-1)

                        loss1_val = lossFun_val(sar, sarArgmax)
                        loss2_val = lossFun_val(sent, sentArgmax)

                        loss_val = (loss1_val + loss2_val)/2

                        logger.info('val loss1:%f loss2:%f loss:%f\n'
                        % (loss1_val.item(),
                            loss2_val.item(),
                            loss_val.item()))

                        label_sar = np.argmax(
                            sarLabel.cpu().detach().numpy(), axis=-1)
                        label_sent = np.argmax(
                            sentLabel.cpu().detach().numpy(), axis=-1)

                        pred_sar = np.argmax(
                            sar.cpu().detach().numpy(), axis=1)
                        pred_sent = np.argmax(
                            sent.cpu().detach().numpy(), axis=1)

                        outputsar.append(pred_sar)
                        outputsent.append(pred_sent)

                        tarsar.append(label_sar)
                        tarsent.append(label_sent)

                    outputsar = np.concatenate(
                        np.array(outputsar),dtype=np.int64)
                    outputsent = np.concatenate(
                        np.array(outputsent),dtype=np.int64)

                    tarsar = np.concatenate(
                        np.array(tarsar),dtype=np.int64)
                    tarsent = np.concatenate(
                        np.array(tarsent),dtype=np.int64)

                    sar_f1 = f1_score(
                        tarsar, outputsar, average='micro')
                    sent_f1 = f1_score(
                        tarsent, outputsent, average='micro')

                    sar_acc = accuracy_score(
                        tarsar, outputsar)
                    sent_acc = accuracy_score(
                        tarsent, outputsent)

                    logger.info(('val-result sar-f1:%f sent-f1:%f' +
                                ' sar-acc:%f sent-acc:%f\n')
                                % (sar_f1, sent_f1,
                                sar_acc, sent_acc))

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

        torch.save(model.state_dict(), stateSavePATH)
        torch.save(model, modelSavePATH)

def testModel(modelPATH):
    testData = MustardDataset(datatye='test')
    batchsize = 32
    data_loader = DataLoader(
        testData,
        batch_size=batchsize,
        shuffle=True,
        pin_memory=True
        )
    classmodel=torch.load(ClassModelPath)

    model=torch.load(modelPATH)
    model.eval()
    classmodel.eval()
    with torch.no_grad():
        outputsar, outputsent, outputemo = [], [], []
        tarsar, tarsent, taremo = [], [], []
        for batch in data_loader:
            textInput = batch[0][0].cuda().to(torch.float32)
            imageInput = batch[0][1].cuda().to(torch.float32)
            wavInput = batch[0][2].cuda().to(torch.float32)

            sarLabel = batch[1][0].to(torch.float32).cuda()
            sentLabel = batch[1][1].to(torch.float32).cuda()
            emoLabel = batch[1][2].to(torch.float32).cuda()
            with torch.no_grad():
                sar, sent=classmodel(textInput, wavInput)
            inq=torch.cat((sar,sent),dim=1).cuda()
            sar,sent=model(inq)

            label_sar = np.argmax(
                sarLabel.cpu().detach().numpy(), axis=-1)
            label_sent = np.argmax(
                sentLabel.cpu().detach().numpy(), axis=-1)

            pred_sar = np.argmax(
                sar.cpu().detach().numpy(), axis=1)
            pred_sent = np.argmax(
                sent.cpu().detach().numpy(), axis=1)

            outputsar.append(pred_sar)
            outputsent.append(pred_sent)

            tarsar.append(label_sar)
            tarsent.append(label_sent)

        outputsar = np.concatenate(
            np.array(outputsar, dtype=object))
        outputsent = np.concatenate(
            np.array(outputsent, dtype=object))

        tarsar = np.concatenate(
            np.array(tarsar, dtype=object))
        tarsent = np.concatenate(
            np.array(tarsent, dtype=object))

        sar_f1 = f1_score(
            tarsar, outputsar, average='micro')
        sent_f1 = f1_score(
            tarsent, outputsent, average='micro')

        sar_acc = accuracy_score(
            tarsar, outputsar)
        sent_acc = accuracy_score(
            tarsent, outputsent)

        print('test tarsar:', tarsar)
        print('test outputsar:', outputsar)
        print('test tarsent:', tarsent)
        print('test outputsent:', outputsent)
        logger.info(('test-result sar-f1:%f sent-f1:%f' +
                    'sar-acc:%f sent-acc:%f \n')
                    % (sar_f1, sent_f1,
                    sar_acc, sent_acc))

if __name__ == "__main__":
    logging.basicConfig(filemode='w')
    logger = logging.getLogger(__name__)
    logger.setLevel(level=logging.INFO)
    handler = logging.FileHandler(
        "result/log/new_log.txt")
    handler.setLevel(logging.INFO)
    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    logger.info("**********Start print log**********")

    trainEval()
    # testModel('result/state/10_Epochs_2023-06-18-05-41-09/model.pt')


import requests
from lxml import html as parser
import random

req = requests.Session()
user_agent_list = [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
]
user_agent = random.choice(user_agent_list)
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
    "Dnt": "1",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": user_agent,
}
req.headers = headers

def getSearchURL(search, page):
    baseURL = 'https://www.amazon.com/s?k={}&s=review-rank&page={}&language=es'
    search = search.replace(' ','+')
    return baseURL.format(search, page)

def getProductData(product_list):

    products = []
    for product in product_list:
        data = {}
        asin = product.get('data-asin')
        data['ASIN'] = asin

        img_list = product.xpath('//div[@data-asin="'+asin+'"]/div/span/div/div/div[2]/div[1]/div/div/span/a/div/img')
        if len(img_list)==0:
            continue
        data['IMG'] = img_list[0].get('src')

        price_list = product.xpath('//div[@data-asin="'+asin+'"]/div/span/div/div/div[2]/div[2]/div/div/div[3]/div[1]/div/div[1]/div/a/span/span[2]')
        if len(price_list)==0:
            continue
        data['PRICE'] = price_list[0].text_content()

        title_list = product.xpath('//div[@data-asin="'+asin+'"]/div/span/div/div/div[2]/div[2]/div/div/div[1]/h2/a/span/text()')
        if len(title_list)==0:
            continue
        data['TITLE'] = title_list[0]

        products += [data]
    return products

def getAllProducts(min_products, search):
    page = 0
    product_list = []
    while len(product_list)<min_products:
        page += 1
        url = getSearchURL(search, str(page))
        res = req.get(url)
        data = res.content
        html = parser.fromstring(data)
        products = html.xpath('//div[@data-asin]')
        product_list += getProductData(products)
        print(len(product_list))

    return product_list

if __name__ == '__main__':

    for product in products:
        print('---------------------------------------------')
        print('ASIM:    '+product['ASIN'])
        print('IMAGEN:  '+product['IMG'])
        print('TITLE:   '+product['TITLE'])
        print('PRICE:   '+product['PRICE'])

import json

def lambda_handler(event, context):
    search = event.get('search')
    count = event.get('count')

    products = getAllProducts(count, search)

    return {
        'statusCode': 200,
        'products': products
    }

    #first_image = asin_data[3].xpath('//span[@data-component-type="s-product-image"]').xpath('//a')


import pygame
import random
import math
from pygame import mixer

LENGTH = 600
BREADTH = 700
GameOver = False

# Screen Setup
pygame.init()
screen = pygame.display.set_mode((BREADTH, LENGTH))
pygame.display.set_caption("Covid Blaster")

# BackGround
icon = pygame.image.load("./spaceship.png")
pygame.display.set_icon(icon)
bg = pygame.image.load("./SafePlanet.jpg")

# Music
mixer.music.load("./SpaceMusic.mp3")
mixer.music.play(-1)

# Enemy
EnemyImg = []
EnemyX = []
EnemyY = []
EnemyChangeX = []
EnemyChangeY = []
num_of_enemies = 8

for i in range(num_of_enemies):
    EnemyImg.append(pygame.image.load("./alien.png"))
    EnemyX.append(random.randint(0, 636))
    EnemyY.append(random.randint(50, 150))
    EnemyChangeX.append(-3)
    EnemyChangeY.append(20)

# Player
PlayerImg = pygame.image.load("./spaceship.png")
PlayerX = 480
PlayerY = 510
PlayerChangeX = 0

# Bullet
BulletImg = pygame.image.load("./syringe1.png")
BulletX = 0
BulletY = 510
BulletChangeY = 0
BulletChangeY = 10
BulletFire = False

#   SCORE
score_value = 0
font = pygame.font.Font('freesansbold.ttf', 32)
Ammo_value = 0
font = pygame.font.Font('freesansbold.ttf', 32)

textX = 10
textY = 10

# Game Over
over_font = pygame.font.Font('freesansbold.ttf', 32)
over_font1 = pygame.font.Font('freesansbold.ttf', 64)

def Show_score(x,y):
    score = font.render("Virus Remaining : " + str(num_of_enemies),True,(0,0,255))
    screen.blit(score,(x,y))
    # Ammo = font.render("Bullets Fired : " + str(Ammo_value),True,(50,205,50))
    # screen.blit(Ammo,(700,y))

def Game_Over(x,y):
    screen.blit(bg, (0, 0))
    over = over_font1.render("Congratratulations,",True,(0,0,0))
    screen.blit(over,(x,y))
    over2 = over_font.render("You Have Defeated COVID-19 ",True,(0,0,0))
    screen.blit(over2,(x,y+70))

def player(PlayerX, PlayerY):
    screen.blit(PlayerImg, (PlayerX, PlayerY))

def enemy(X, Y,i):
    screen.blit(EnemyImg[i], (X, Y))

def bullet(BulletX, BulletY):
    global BulletFire
    BulletFire = True
    screen.blit(BulletImg, (BulletX+16, BulletY+10))

def collision(X,Y,BulletX,BulletY):
    dis = math.sqrt(math.pow(X-BulletX,2)+math.pow(Y-BulletY,2))

    if dis <30:
        return True
    else:
        return False

while not GameOver:
    screen.blit(bg, (0, 0))
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            GameOver = True
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_LEFT:
                PlayerChangeX = -5
            if event.key == pygame.K_RIGHT:
                PlayerChangeX = 5
            if event.key == pygame.K_SPACE:
                if not BulletFire:
                    # Ammo_value += 1
                    Bullet_sound = mixer.Sound("./laser.wav")
                    Bullet_sound.play()
                    BulletChangeY = -5
                    BulletX = PlayerX
                    bullet(BulletX,BulletY)

        if event.type == pygame.KEYUP:
            if event.key == pygame.K_LEFT or event.key == pygame.K_RIGHT:
                PlayerChangeX = 0

    # Player Motion
    PlayerX = PlayerX + PlayerChangeX
    if PlayerX <= 0:
        PlayerX = 0
    elif PlayerX >= 636:
        PlayerX = 636

    # Enemy Motion
    for i in range(num_of_enemies):

        if num_of_enemies == 0:
            Game_Over(0,0)
            break

        EnemyX[i] = EnemyX[i] + EnemyChangeX[i]
        if EnemyX[i] <= 0:
            EnemyChangeX[i] = 2
            EnemyY[i] += 20
        elif EnemyX[i] >= 636:
            EnemyChangeX[i] = -2
            EnemyY[i] += 20

    # screen.fill((0,0,0))
        col = collision(EnemyX[i],EnemyY[i],BulletX,BulletY)
        if col:
            Contact_sound = mixer.Sound("./explosion.wav")
            Contact_sound.play()
            BulletY = 510
            BulletFire = False
            score_value +=1
            # print(score_value)
            num_of_enemies -=1
            # EnemyX[i] = random.randint(0, 936)
            # EnemyY[i] = random.randint(50, 150)

        enemy(EnemyX[i], EnemyY[i],i)

    if BulletY <= 0:
        BulletY = 510
        BulletFire =False
    if BulletFire:
        bullet(BulletX,BulletY)
        BulletY += BulletChangeY

    player(PlayerX, PlayerY)
    Show_score(textX,textY)
    if num_of_enemies == 0:
        Game_Over(0,0)
    pygame.display.update()


from app.extensions import db

class Person(db.Model):
    def __init__(self, name):
        self.name = name

    def validate(self):
        # Check that the name is not empty
        if not self.name:
            return False, 'nameIsEmpty'

        # Check that the name only contains letters A-Ö, a-ö, space, hyphen and apostrophe
        for letter in self.name:
            if not (letter.isalpha() or letter == ' ' or letter == '-' or letter == "'"):
                return False, 'containsIllegalChars'

        # Check that the name is not longer than 50 characters
        if len(self.name) > 50:
            return False, 'nameTooLong'

        return True, None

    def sterlize(self):
        #remove trailing and leading whitespace
        self.name = self.name.strip()

    __tablename__ = 'persons'
    person_id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    name = db.Column(db.Text)