# -*- coding: utf-8 -*-

import os, sys
from .config import *
from .importExport import *
import zipfile as zp
import requests # OA 1/10
import shutil

class Menus:
    def __init__(self, gui, core):
        self.gui,self.core = gui,core
        self.cfg = Config(core)
        self.dialogs = self.cfg.dialogs
        self.gtyp = self.cfg.gtyp

    def OnNew(self,evt=None):
        """creates a new file"""
        #self.askSave(evt)
        dlg = self.dialogs.myFileDialog('New')
        self.core.fileDir,self.core.fileName = dlg.getsetFile(self.gui,'New Model',"*.iqpht;*.orti")
        if self.core.fileDir == '': return
        self.core.addin.initAddin()
        self.core.initAll()
        if self.gtyp =='qt':
            self.gui.visu.setVisu(self.core)
            self.gui.updateTitle()

    def OnOpen(self,evt=None):
        """opens a file"""
        if self.core.fileDir!=None:
            self.askSave(evt)
        dlg = self.dialogs.myFileDialog('Open')
        fDir,fName =dlg.getsetFile(self.gui,'Open','*.iqpht;*.orti');
        if fName == '': return
        self.onOpen1(fDir,fName) # OA 16/12/18 to separate if file dir is already known
        if self.core.mfUnstruct: self.gui.onGridMesh('Mesh') # OA 18/2/20 removed from core, set here
        self.gui.onSetMediaNb(getNmedia(self.core),getNlayers(self.core))  # OA 14/3/21

    def onOpen1(self,fDir,fName): # OA 16/12/18 to be able to use only that part
        self.core.openModel(fDir,fName)
        a = self.core.makeTtable()
        listSpec = self.core.addin.chem.getListSpecies() # just the names
        self.core.addin.set3D()
        mtype = self.core.dicaddin['Model']['group']
        self.gui.guiShow.openModel() # OA 14/10
        if self.gtyp =='qt':
            if 'obs.1' in list(self.core.diczone['Observation'].dic.keys()):
                onames = self.core.diczone['Observation'].dic['obs.1']['name']
                #self.gui.guiShow.setNames('Observation_Zone_L',onames) #EV 19/12/18
            self.gui.visu.setVisu(self.core)
            self.gui.updateTitle()
        if self.gtyp=='qgis':
            self.gui.visu.removeOrtiLayers()
            self.gui.visu.zonesCore2qgs()
        self.gui.varBox.chooseCategory(mtype)
        self.gui.visu.initDomain()
        tl2 = self.core.getTlist2()
        self.gui.guiShow.setNames('Model_Tstep_L',tl2)
        self.gui.guiShow.setChemSpecies(listSpec)
        self.dialogs.onMessage(self.gui,'File opened')

    def OnSave(self,evt=None):
        if self.core.fileDir!=None:
            if self.gtyp =='qt':
                self.gui.updateTitle()
            if self.gtyp=='qgis':
                self.gui.visu.zonesQgs2core()
            self.core.saveModel(str(self.core.fileDir),str(self.core.fileName))
            self.dialogs.onMessage(self.gui,'File saved')
        else :
            self.OnSaveAs(evt)

    def OnSaveAs(self,evt=None):
        dlg = self.dialogs.myFileDialog('Save')
        fDir,fName = dlg.getsetFile(self.gui,'Save as','*.iqpht;*.orti')
        if fName == '': return
        self.core.saveModel(str(fDir),str(fName))
        if self.gtyp =='qt': #EV 27/11/18
            self.gui.updateTitle()
        if self.gtyp=='qgis':
            self.gui.visu.zonesQgs2core()

    def OnImportVersion1(self,evt=None):
        dlg = self.dialogs.myFileDialog()
        fDir,fName =dlg.getsetFile(self.gui,'Open',"*.ipht");#print fDir,fName
        importer = impFile(self.gui,self.core)
        importer.impVersion1(fDir,fName)

    def OnImportModflowAscii(self,evt=None):
        dlg = self.dialogs.myFileDialog()
        fDir,fName =dlg.getsetFile(self.gui,'Open',"*.nam");#print fDir,fName
        importer = impAsciiModflow(self.core,fDir,fName)
        importer.readAll()

    def askSave(self,evt=None):
        if self.gtyp=='qgis': #EV 27/11/18
            if self.core.fileDir==None:return
        message = self.dialogs.onQuestion(self.gui,"Do you want to save the Orti file?") # OA modif 22/10/18
        if message == 'Yes':
            self.OnSave(evt)
        else : return
        #message.Destroy()

    def OnImportData(self,evt=None):
        """import external data to be used for representation"""
        dlg = self.dialogs.myFileDialog()
        fDir,fName =dlg.getsetFile(self.gui,'Open data file',"*.txt")
        if fDir == None: return
        else :
            self.data=self.core.importData(fDir,fName)
            self.dialogs.onMessage(self.gui,'Data imported')

    def OnImportHead(self,evt=None):
        m=impObsData(self.gui,self.core,'Head')
        m.show()

    def OnImportTracer(self,evt=None):
        m=impObsData(self.gui,self.core,'Tracer')
        m.show()

    def OnImportChemistry(self,evt=None):
        m=impObsData(self.gui,self.core,'Chemistry')
        m.show()

    #def OnImportSolutions(self,evt=None):                               #EV 14/11/19
    #    """import a text file to store solutions"""
    #    dlg = self.dialogs.myFileDialog()
    #    fDir,fName =dlg.getsetFile(self.gui,'Open solutions',"*.txt")
    #    if fDir == None: return
    #    else :
    #        self.core.importSolutions(fDir,fName)
    #        self.dialogs.onMessage(self.gui,'Solutions imported')
    #        #self.gui.OnMessage("Fichier donnees importe")

    def OnImportUserSpecies(self,evt=None):
        dlg = self.dialogs.myFileDialog()
        fDir,fName =dlg.getsetFile(self.gui,'Open solutions',"*.txt;*.out")
        if fDir == None: return
        elif fName == 'selected' :
            d = self.core.addin.pht3d.readSelectOut(fDir)
            #self.gui.guiShow.userSpecies = d
            self.gui.guiShow.setNames('Chemistry_User_L',list(d.keys()))
        else :
            f1= open(fDir+os.sep+fName+'.txt')
            dicSp = {}
            for l in f1:
                if '=' in l:
                    a,b=l.split('=');dicSp[a]=b
            self.gui.guiShow.setUserSpecies(dicSp)
            nameBox = 'Chemistry_User_L'
            self.gui.guiShow.setNames(nameBox,list(dicSp.keys()))
        self.dialogs.onMessage(self.gui,'User species imported')

    def OnImportPostfixSpecies(self,evt=None): # oa added 30/6/19 (some modifs, added core.)
        if 'postfix.phrq' in os.listdir(self.core.fileDir):
            #print('ok')
            if 'selected.out' in os.listdir(self.core.fileDir):
                d = self.core.addin.pht3d.readSelectOut(self.core.fileDir)
                if self.gui != None:
                    self.gui.guiShow.setUserSpecies(d);#print 'core203',d
                    self.gui.guiShow.setNames('Chemistry_User_L',list(d.keys()))
                self.dialogs.onMessage(self.gui,'Postfix species imported')

    def OnExportParm(self,evt=None): # added 28/3/17 oa
        model,line,media = self.gui.currentModel,self.gui.currentLine,self.gui.currentMedia
        if not line : #EV 11/12/19
            self.dialogs.onMessage(self.gui,'Select a parameter to export')
        else :
            name = line.replace('.','')
            fname = self.core.fileDir+os.sep+ name
            data = self.core.getValueLong(model,line,0)[media2layers(self.core,media)[0]] # returns just the 1st layer of the considered medium
            savetxt(fname+'.txt',data)
            self.dialogs.onMessage(self.gui,'File '+name+' saved')

    def OnExportResu(self,evt=None): # modif 28/3/17 oa for correct name
        if not self.gui.guiShow.curName : #EV 11/12/19
             self.dialogs.onMessage(self.gui,'Select a result to export')
        else :
            if self.gui.guiShow.curSpecies != True :
                name = self.gui.guiShow.curName+self.gui.guiShow.curSpecies;# OA modified 10/5/17
            else :
                name = self.gui.guiShow.curName
            fname = self.core.fileDir+os.sep+ name
            data = self.gui.guiShow.data[-1]
            savetxt(fname+'.txt',data)
            self.dialogs.onMessage(self.gui,'File '+name+' saved')

    def OnExportParmVtk(self,evt=None): # added 28/3/17 oa
        model,line = self.gui.currentModel,self.gui.currentLine
        if not line : #EV 11/12/19
            self.dialogs.onMessage(self.gui,'Select a parameter to export')
            return
        dlg = self.dialogs.myFileDialog('Save')
        fDir,fName = dlg.getsetFile(self.gui,'Save vtk',"*.vtk")
        if fName :
            name = line.replace('.','')
            data = self.core.getValueLong(model,line,0)
            s = writeVTKstruct(self.core,model,data)
            f1=open(fDir+os.sep+fName+'.vtk','w');f1.write(s);f1.close()
            self.dialogs.onMessage(self.gui,'File '+fName+' saved')

    def OnExportResuVtk(self,evt=None): # modif 28/3/17 oa for correct name
        model = self.gui.currentModel
        data = self.gui.guiShow.arr3#;print('menu 151',shape(data))
        if not shape(data) : #EV 11/12/19
            self.dialogs.onMessage(self.gui,'Select a result to export')
            return
        dlg = self.dialogs.myFileDialog('Save')
        fDir,fName = dlg.getsetFile(self.gui,'Save vtk',"*.vtk")
        if fName :
            s = writeVTKstruct(self.core,model,data)
            f1=open(fDir+os.sep+fName+'.vtk','w');f1.write(s);f1.close()
            self.dialogs.onMessage(self.gui,'file '+fName+' saved')

    def OnExportResuVtkAll(self,evt=None): # modif 28/3/17 oa for correct name
        model,g = self.gui.currentModel,self.gui.guiShow
        name = g.curName
        dlg = self.dialogs.myFileDialog('Save')
        fDir,fName = dlg.getsetFile(self.gui,'Save vtk',"*.vtk")
        tlist = self.core.getTlist2()
        if fName :
            for it in range(len(tlist)):
                data = self.gui.guiShow.getArray3D(g.curGroup,name,it,g.curSpecies)
                s = writeVTKstruct(self.core,model,data)
                f1=open(fDir+os.sep+fName+str(it)+'.vtk','w')
                f1.write(s);f1.close()
            s = '{"file-series-version" : "1.0",\n "files" : \n [ \n'
            for it in range(len(tlist)):
                t = tlist[it]
                s += '{ "name" : "'+name+str(it)+'.vtk", "time" : '+str(t)+'},\n'
            s += '] \n }'
            f1=open(fDir+os.sep+fName+'.series','w');f1.write(s);f1.close()
        self.dialogs.onMessage(self.gui,'file '+fName+' saved')

    def OnHelpI(self,evt=None): #,lang):
        """calling help file"""
        os.startfile(self.gui.mainDir+os.sep+'doc'+os.sep+"interfaceHelp.chm")
    def OnHelpM(self,evt=None): #,lang):
        """calling help file"""
        os.startfile(self.gui.mainDir+os.sep+'doc'+os.sep+"modelsHelp.chm")

    def OnDownloadLast(self,evt=None):
        self.onDownload('master')

    def OnDownloadDev(self,evt=None):
        self.onDownload('develop')

    def OnDownloadLocal(self,evt=None): # Remove from orti3dGui.py EV 10/10/18
        dlg = self.dialogs.myFileDialog()
        fDir,fName =dlg.getsetFile(self.gui,'Open','*.zip');#print fDir,fName
        self.onDownload(fDir+os.sep+fName,'local')

    def onDownload(self,fname,opt='web'):
        maindir=self.gui.mainDir
        dirdoc=maindir+os.sep+'doc'
        dirutil=maindir+os.sep+'utils'
        dirbin=maindir+os.sep+'bin'
        dirdoc=os.path.normpath(dirdoc)
        dirutil=os.path.normpath(dirutil)
        dirbin=os.path.normpath(dirbin)
        if self.cfg.typInstall=='python':
            dirlib=maindir+os.sep+'ilibq'
        else :
            dirlib=maindir+os.sep+'lib'+os.sep+'ilibq' # EV 23/10/18 new exe version
        dirlib=os.path.normpath(dirlib)
        lfu=os.listdir(dirutil)
        if 'newlib.zip' in lfu:
            os.system('copy '+dirutil+os.sep+'newlib.zip '+dirutil+os.sep+'oldlib.zip')
        f2=dirutil+os.sep+'newlib.zip'
        if opt=='web':
            htname = 'https://www.github.com/ORTI3D/ORTI3D/archive/'+fname+'.zip'
            r = requests.get(htname)
            with open(f2,"wb") as code: # OA 1/10
                code.write(r.content) # OA 1/10
        else :
            f2 = fname+'.zip'
        znew=zp.ZipFile(f2,'r')
        if self.cfg.typInstall=='python': #the python version
            znew.extractall(dirutil)
            #os.system('xcopy /Y '+dirutil+os.sep+'ORTI3D-'+fname+os.sep+'ilibq '+dirlib)
            os.system('xcopy /Y /E '+dirutil+os.sep+'ORTI3D-'+fname+' '+dirlib) #EV 26/11/18
            print ('ok')
            for n in os.listdir(dirlib):
                if ('.chm' in n) or ('.pdf' in n):
                    os.system('move '+dirlib+os.sep+n+' '+dirdoc)
                if ('.png' in n) or ('.dbs' in n): #EV 30/09/19 add .png
                    os.system('move '+dirlib+os.sep+n+' '+dirutil)
                if ('.exe' in n) : #EV 30/09/19
                    os.system('move '+dirlib+os.sep+n+' '+dirbin)
        else : # the windows version
            znew.extractall(dirutil) # EV 23/10/18 new exe version
            os.system('xcopy /Y /E '+dirutil+os.sep+'ORTI3D-'+fname+' '+dirlib) #EV 26/11/18
            for n in os.listdir(dirlib): #EV 30/09/19
                if ('.chm' in n) or ('.pdf' in n) : #EV 30/09/19 add .png
                    os.system('move '+dirlib+os.sep+n+' '+dirdoc)
                if ('.png' in n) or ('.dbs' in n): #EV 30/09/19 add .png
                    os.system('move '+dirlib+os.sep+n+' '+dirutil)
                if ('.exe' in n) : #EV 30/09/19
                    os.system('move '+dirlib+os.sep+n+' '+dirbin)
            #zlib=zp.ZipFile(dirlib,'r'); #print 'menu dwnload 157', dirlib
            #zlib.extractall(maindir+os.sep+'temp')
            #zlib.close()
            #shutil.rmtree(maindir+os.sep+'temp'+os.sep+'ilibq')
            #znew.extractall(maindir+os.sep+'ilib1')
            #os.system('xcopy /Y '+maindir+os.sep+'ilib1'+os.sep+'ORTI3D-'+fname+os.sep+'iliblast '+maindir+os.sep+'temp'+os.sep+'ilibq')
            #self.zip_folder(maindir+os.sep+'temp',maindir+os.sep+'zout.zip')
            #os.chdir(maindir)
            #os.system('del '+dirlib)
            #os.system('rename zout.zip library.zip')
            #shutil.rmtree('temp')
            #shutil.rmtree('ilib1')
        znew.close()
        if self.gui.gtyp!='qgis':
            self.dialogs.onMessage(self.gui,'lib changed, ORTi3D will stop, then restart it')
            sys.exit()
        else :
            self.dialogs.onMessage(self.gui,'lib changed, please reloading qORTi3D plugin')

    def zip_folder(self,folder_path, output_path):
        parent_folder = os.path.dirname(folder_path)
        # Retrieve the paths of the folder contents.
        contents = os.walk(folder_path)
        zip_file = zp.ZipFile(output_path, 'w', zp.ZIP_DEFLATED)
        for root, folders, files in contents:
            for folder_name in folders:
                absolute_path = os.path.join(root, folder_name)
                relative_path = absolute_path.replace(parent_folder + '\\', '')
                relative_path = relative_path.replace('temp\\','')
                zip_file.write(absolute_path, relative_path)
            for file_name in files:
                absolute_path = os.path.join(root, file_name)
                relative_path = absolute_path.replace(parent_folder + '\\', '')
                relative_path= relative_path.replace('temp\\','')
                zip_file.write(absolute_path, relative_path)

    def OnBackVersion(self,evt=None):
        dirutil=self.gui.mainDir+os.sep+'utils'
        lf=os.listdir(dirutil)
        if 'oldlib.zip' not in lf: self.dialogs.onMessage('sorry no old lib')
        zin=zp.ZipFile(dirutil+os.sep+'oldlib.zip','r')
        zin.extractall(self.gui.mainDir+os.sep+'ilibq')
        self.dialogs.onMessage(self.gui,'lib changed, iPht3D will stop, then restart')
        self.gui.Destroy()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import math

import REDACTED
from absl import app
from absl import flags
from absl import logging
import tensorflow.compat.v2 as tf
from REDACTED.tf2_bert import optimization
from REDACTED.tf2_bert.bert import bert_models
from REDACTED.tf2_bert.bert import common_flags
from REDACTED.tf2_bert.bert import configs
from REDACTED.tf2_bert.bert import input_pipeline
from REDACTED.tf2_common.modeling import model_training_utils
from REDACTED.tf2_common.modeling import performance
from REDACTED.tf2_common.utils.misc import distribution_utils
from REDACTED.tf2_common.utils.mlp_log import mlp_log

flags.DEFINE_string('input_files', None,
                    'File path to retrieve training data for pre-training.')
# Model training specific flags.
flags.DEFINE_integer(
    'max_seq_length', 128,
    'The maximum total input sequence length after WordPiece tokenization. '
    'Sequences longer than this will be truncated, and sequences shorter '
    'than this will be padded.')
flags.DEFINE_integer('max_predictions_per_seq', 20,
                     'Maximum predictions per sequence_output.')
flags.DEFINE_integer('train_batch_size', 32, 'Total batch size for training.')
flags.DEFINE_integer('num_steps_per_epoch', 1000,
                     'Total number of training steps to run per epoch.')
flags.DEFINE_float('warmup_steps', 10000,
                   'Warmup steps for Adam weight decay optimizer.')
flags.DEFINE_integer('stop_steps', None,
                     'The number of steps to stop training.')
flags.DEFINE_bool('do_eval', False, 'Whether to run eval.')
flags.DEFINE_bool('device_warmup', False,
                  'Whether or not to enable device warmup.')
flags.DEFINE_integer('steps_between_eval', 10000,
                     'Steps between an eval. Is multiple of steps per loop.')
flags.DEFINE_integer('steps_before_eval_start', 0,
                     'Steps before starting eval.')
flags.DEFINE_integer('num_eval_samples', 10000, 'Number of eval samples.')
flags.DEFINE_integer('eval_batch_size', 32, 'Total batch size for training.')
flags.DEFINE_float('weight_decay_rate', 0.01,
                   'The weight_decay_rate value for the optimizer.')
flags.DEFINE_float('beta_1', 0.9, 'The beta_1 value for the optimizer.')
flags.DEFINE_float('beta_2', 0.999, 'The beta_2 value for the optimizer.')
flags.DEFINE_float('epsilon', 1e-6, 'The epsilon value for the optimizer.')
flags.DEFINE_integer('num_accumulation_steps', 1,
                     'number of steps to accumulate with large batch size.')
flags.DEFINE_float(
    'stop_threshold', default=0.712, help=('Stop threshold for MLPerf.'))
common_flags.define_common_bert_flags()

FLAGS = flags.FLAGS

def get_pretrain_dataset_fn(input_file_pattern, seq_length,
                            max_predictions_per_seq, global_batch_size,
                            is_training, use_synthetic, num_eval_samples):
  """Returns input dataset from input file string."""
  def _dataset_fn(ctx=None):
    """Returns tf.data.Dataset for distributed BERT pretraining."""
    if use_synthetic:
      input_patterns = ''
    else:
      input_patterns = input_file_pattern.split(',')
    batch_size = ctx.get_per_replica_batch_size(global_batch_size)

    train_dataset = input_pipeline.create_pretrain_dataset(
        input_patterns=input_patterns,
        seq_length=seq_length,
        max_predictions_per_seq=max_predictions_per_seq,
        batch_size=batch_size,
        is_training=is_training,
        use_synthetic=use_synthetic,
        input_pipeline_context=ctx,
        num_eval_samples=num_eval_samples)
    return train_dataset

  return _dataset_fn

def get_loss_fn(loss_factor=1.0):
  """Returns loss function for BERT pretraining."""

  def _bert_pretrain_loss_fn(unused_labels, losses, **unused_args):
    return tf.reduce_mean(losses) * loss_factor

  return _bert_pretrain_loss_fn

def clip_by_global_norm(grads_and_vars):
  grads, tvars = list(zip(*grads_and_vars))
  (grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0)
  return zip(grads, tvars)

def run_customized_training(
    strategy, optimizer_type, weight_decay_rate, beta_1, beta_2, epsilon,
    bert_config, max_seq_length, max_predictions_per_seq, model_dir,
    steps_per_epoch, steps_per_loop, epochs, initial_lr, warmup_steps,
    input_files, train_batch_size, do_eval, eval_batch_size, num_eval_samples,
    custom_callbacks, init_checkpoint, steps_between_eval,
    steps_before_eval_start, stop_threshold, explicit_allreduce,
    allreduce_bytes_per_pack, enable_checkpoint_and_summary, device_warmup):
  """Run BERT pretrain model training using low-level API."""
  mlp_log.mlperf_print('cache_clear', True)
  mlp_log.mlperf_print('init_start', None)
  mlp_log.mlperf_print('global_batch_size', train_batch_size)
  mlp_log.mlperf_print('max_sequence_length', max_seq_length)
  mlp_log.mlperf_print('max_predictions_per_seq', max_predictions_per_seq)
  mlp_log.mlperf_print('opt_base_learning_rate', initial_lr)
  mlp_log.mlperf_print('opt_lamb_weight_decay_rate', weight_decay_rate)
  mlp_log.mlperf_print('opt_lamb_beta_1', beta_1)
  mlp_log.mlperf_print('opt_lamb_beta_2', beta_2)
  mlp_log.mlperf_print('opt_gradient_accumulation_steps',
                       FLAGS.num_accumulation_steps)
  mlp_log.mlperf_print('opt_learning_rate_warmup_steps', warmup_steps)
  mlp_log.mlperf_print('num_warmup_steps', warmup_steps)
  mlp_log.mlperf_print('start_warmup_step', 0)
  mlp_log.mlperf_print('opt_epsilon', epsilon)
  mlp_log.mlperf_print('eval_samples', num_eval_samples)
  mlp_log.mlperf_print('opt_lamb_learning_rate_decay_poly_power', 1.0)
  mlp_log.mlperf_print('opt_learning_rate_training_steps',
                       steps_per_epoch * epochs)
  mlp_log.mlperf_print('train_samples',
                       train_batch_size * steps_per_epoch * epochs)
  train_input_fn = get_pretrain_dataset_fn(
      input_file_pattern=input_files,
      seq_length=max_seq_length,
      max_predictions_per_seq=max_predictions_per_seq,
      global_batch_size=train_batch_size,
      is_training=True,
      num_eval_samples=num_eval_samples,
      use_synthetic=False)
  eval_input_fn = None
  if do_eval:
    eval_input_fn = get_pretrain_dataset_fn(
        input_file_pattern=input_files,
        seq_length=max_seq_length,
        max_predictions_per_seq=max_predictions_per_seq,
        global_batch_size=eval_batch_size,
        is_training=False,
        num_eval_samples=num_eval_samples,
        use_synthetic=False)
  synthetic_train_input_fn = None
  if device_warmup:
    synthetic_train_input_fn = get_pretrain_dataset_fn(
        input_file_pattern=None,
        seq_length=max_seq_length,
        max_predictions_per_seq=max_predictions_per_seq,
        global_batch_size=train_batch_size,
        is_training=True,
        num_eval_samples=1,
        use_synthetic=True)

  def _get_pretrain_model():
    """Gets a pretraining model."""
    pretrain_model, core_model, core_pretrain_model = bert_models.pretrain_model(
        bert_config, max_seq_length, max_predictions_per_seq)
    optimizer = optimization.create_optimizer(initial_lr,
                                              steps_per_epoch * epochs,
                                              warmup_steps, optimizer_type,
                                              weight_decay_rate, beta_1, beta_2,
                                              epsilon)
    pretrain_model.optimizer = performance.configure_optimizer(
        optimizer,
        use_float16=common_flags.use_float16(),
        use_graph_rewrite=common_flags.use_graph_rewrite())
    return pretrain_model, core_model, core_pretrain_model

  trained_model, masked_lm_accuracy, run_steps = model_training_utils.run_customized_training_loop(
      strategy=strategy,
      model_fn=_get_pretrain_model,
      loss_fn=get_loss_fn(
          loss_factor=1.0 /
          strategy.num_replicas_in_sync if FLAGS.scale_loss else 1.0),
      model_dir=model_dir,
      train_input_fn=train_input_fn,
      steps_per_epoch=steps_per_epoch,
      steps_per_loop=steps_per_loop,
      epochs=epochs,
      eval_input_fn=eval_input_fn,
      eval_steps=math.ceil(num_eval_samples / eval_batch_size),
      steps_between_eval=steps_between_eval,
      steps_before_eval_start=steps_before_eval_start,
      sub_model_export_name='pretrained/bert_model',
      init_checkpoint=init_checkpoint,
      custom_callbacks=custom_callbacks,
      device_warmup=device_warmup,
      synthetic_train_input_fn=synthetic_train_input_fn,
      explicit_allreduce=explicit_allreduce,
      post_allreduce_callbacks=[clip_by_global_norm],
      allreduce_bytes_per_pack=allreduce_bytes_per_pack,
      enable_checkpoint_and_summary=enable_checkpoint_and_summary,
      num_accumulation_steps=FLAGS.num_accumulation_steps,
      stop_steps=FLAGS.stop_steps,
      stop_threshold=FLAGS.stop_threshold)

  return trained_model, masked_lm_accuracy, run_steps

def run_bert_pretrain(strategy, custom_callbacks=None):
  """Runs BERT pre-training."""
  bert_config = configs.BertConfig.from_json_file(FLAGS.bert_config_file)
  if not strategy:
    raise ValueError('Distribution strategy is not specified.')

  # Runs customized training loop.
  logging.info('Training using customized training loop TF 2.0 with distrubuted'
               'strategy.')

  performance.set_mixed_precision_policy(common_flags.dtype())

  _, masked_lm_accuracy, run_steps = run_customized_training(
      strategy=strategy,
      optimizer_type=FLAGS.optimizer_type,
      weight_decay_rate=FLAGS.weight_decay_rate,
      beta_1=FLAGS.beta_1,
      beta_2=FLAGS.beta_2,
      epsilon=FLAGS.epsilon,
      bert_config=bert_config,
      max_seq_length=FLAGS.max_seq_length,
      max_predictions_per_seq=FLAGS.max_predictions_per_seq,
      model_dir=FLAGS.model_dir,
      steps_per_epoch=FLAGS.num_steps_per_epoch,
      steps_per_loop=FLAGS.steps_per_loop,
      epochs=FLAGS.num_train_epochs,
      initial_lr=FLAGS.learning_rate,
      warmup_steps=FLAGS.warmup_steps,
      input_files=FLAGS.input_files,
      train_batch_size=FLAGS.train_batch_size,
      eval_batch_size=FLAGS.eval_batch_size,
      do_eval=FLAGS.do_eval,
      num_eval_samples=FLAGS.num_eval_samples,
      steps_between_eval=FLAGS.steps_between_eval,
      steps_before_eval_start=FLAGS.steps_before_eval_start,
      stop_threshold=FLAGS.stop_threshold,
      explicit_allreduce=FLAGS.explicit_allreduce,
      allreduce_bytes_per_pack=FLAGS.allreduce_bytes_per_pack,
      enable_checkpoint_and_summary=FLAGS.enable_checkpoint_and_summary,
      custom_callbacks=custom_callbacks,
      init_checkpoint=FLAGS.init_checkpoint,
      device_warmup=FLAGS.device_warmup)
  return masked_lm_accuracy, run_steps

# copybara:strip_begin
# Hparams wrapper should be REDACTED only, not open source.
def hparams_run_wrapper(hparams):
  """Convenience wrapper for running with hyperparameters."""
  print('HParams: ', hparams)

  if 'learning_rate' in hparams:
    print('Overwriting learning_rate with ', hparams.learning_rate)
    FLAGS.learning_rate = hparams.learning_rate
  if 'beta_1' in hparams:
    print('Overwriting beta_1 with ', hparams.beta_1)
    FLAGS.beta_1 = hparams.beta_1
  if 'beta_2' in hparams:
    print('Overwriting beta_2 with ', hparams.beta_2)
    FLAGS.beta_1 = hparams.beta_2
  if 'warmup_steps' in hparams:
    print('Overwriting warmup_steps with ', hparams.warmup_steps)
    FLAGS.warmup_steps = hparams.warmup_steps
  if 'num_steps_per_epoch' in hparams:
    print('Overwriting num_steps_per_epoch with ', hparams.num_steps_per_epoch)
    FLAGS.num_steps_per_epoch = hparams.num_steps_per_epoch
  if 'weight_decay_rate' in hparams:
    print('Overwriting weight_decay_rate with ', hparams.weight_decay_rate)
    FLAGS.weight_decay_rate = hparams.weight_decay_rate

  if not FLAGS.model_dir:
    FLAGS.model_dir = '/tmp/bert20/'
  strategy = distribution_utils.get_distribution_strategy(
      distribution_strategy=FLAGS.distribution_strategy,
      num_gpus=FLAGS.num_gpus,
      all_reduce_alg=FLAGS.all_reduce_alg,
      tpu_address=FLAGS.tpu)
  if strategy:
    print('***** Number of cores used : ', strategy.num_replicas_in_sync)

  return run_bert_pretrain(strategy)

# copybara:strip_end

def main(_):
  # Users should always run this script under TF 2.x
  tf.enable_v2_behavior()

  if not FLAGS.model_dir:
    FLAGS.model_dir = '/tmp/bert20/'
  strategy = distribution_utils.get_distribution_strategy(
      distribution_strategy=FLAGS.distribution_strategy,
      num_gpus=FLAGS.num_gpus,
      all_reduce_alg=FLAGS.all_reduce_alg,
      tpu_address=FLAGS.tpu)
  if strategy:
    print('***** Number of cores used : ', strategy.num_replicas_in_sync)

  run_bert_pretrain(strategy)

if __name__ == '__main__':
  app.run(main)


import math
import sys
CACHESIZE = int(sys.argv[1])
ASSOC = int(sys.argv[2])
REPLACEMENT = ""
if int(sys.argv[3]) == 0:
    REPLACEMENT = "LRU"
elif int(sys.argv[3]) == 1:
    REPLACEMENT = "FIFO"
WRITETYPE = ""
if int(sys.argv[4]) == 0:
    WRITETYPE = "WT"
elif int(sys.argv[4]) == 1:
    WRITETYPE = "WB"
TRACEFILE = sys.argv[5]
BLOCKSIZE = 64
VERBOSE = False
num_sets = int((CACHESIZE / BLOCKSIZE) / ASSOC)

def main():
    if VERBOSE:
        printParameters()
    f = open(TRACEFILE, "r")
    lines = f.readlines()
    cache = cacheStruct()
    hits = 0
    misses = 0
    writes = 0
    for i in range(len(lines)):
        num_set_bits = math.log(num_sets,2)
        op = lines[i][0]
        address = int(lines[i][2:], 16)
        set_num = int(address/BLOCKSIZE) % num_sets
        tag_address = int(address/BLOCKSIZE)
        hit,write = cache.index[set_num].lookup(tag_address, op)
        if hit == 1:
            hits += 1
        elif hit == 0:
            misses += 1
        writes += write

    print(misses/ (misses + hits))
    print(writes)
    print(misses)
    return (misses/ (misses + hits)), writes, misses

class cacheMember():
    def __init__(self):
        self.tags = []
        self.dirtys = []

    def lookup(self, add, op):
        hit = 0
        write = 0
        read = 0
        for tag in self.tags:
            if tag == add:
                hit = 1

        if REPLACEMENT == "FIFO":
            if hit == 0:
                self.tags.append(add)
                if len(self.tags) > ASSOC:
                    removed = self.tags[0]
                    self.tags.pop(0)
                    if removed in self.dirtys:
                        write = 1
                        self.dirtys.remove(removed)

        elif REPLACEMENT == "LRU":
            removed = -1
            if hit == 1:
                self.tags.remove(add)
            self.tags.append(add)
            if len(self.tags) > ASSOC:
                removed = self.tags[0]
                self.tags.pop(0)
                if removed in self.dirtys:
                    write = 1
                    self.dirtys.remove(removed)

        if op == 'W':
            if WRITETYPE == "WT":
                write = 1
            elif WRITETYPE == "WB":
                if not add in self.dirtys:
                    self.dirtys.append(add)

        return hit, write

class cacheStruct():
    def __init__(self):
        self.index = []
        for i in range(num_sets):
            self.index.append(cacheMember())
    def report(self):
        for tag in self.index:
            print(tag.tags)
            print("dirtys", tag.dirtys)
            print()
        return

def printParameters():
    print(CACHESIZE)
    print("\t", type(CACHESIZE))
    print(ASSOC)
    print("\t", type(ASSOC))
    print(REPLACEMENT)
    print("\t", type(REPLACEMENT))
    print(WRITETYPE)
    print("\t", type(WRITETYPE))
    print(TRACEFILE)
    print("\t", type(TRACEFILE))

if __name__ == '__main__':
    main()


import os
from functools import cached_property

from pandas import read_csv

from app import DATA_DIRPATH

GTZAN_DIRPATH = os.path.join(DATA_DIRPATH, "gtzan")
GENRES_DIRPATH = os.path.join(GTZAN_DIRPATH, "genres_original")

class AudioFile:

    def __init__(self, audio_filepath):
        self.audio_filepath = audio_filepath

        self.genre = audio_filepath.split("/")[-2]
        self.audio_filename = audio_filepath.split("/")[-1]

        self.genre_dirpath = os.path.join(GENRES_DIRPATH, self.genre)

class GenreDataset:

    #def __init__(self, gtzan_dirpath=GTZAN_DIRPATH):
    #    self.gtzan_dirpath = gtzan_dirpath
    #    self.genres_dirpath = os.path.join(self.gtzan_dirpath, "genres_original")

    @cached_property
    def genres(self):
        return sorted([genre for genre in os.listdir(GENRES_DIRPATH) if genre not in [".DS_Store"]])

    #@cached_property
    #def audio_filepaths(self):
    #    filepaths = []
    #    for genre in self.genres:
    #        genre_dirpath = os.path.join(GENRES_DIRPATH, genre)
    #        audio_filenames = sorted([fname for fname in os.listdir(genre_dirpath) if fname.endswith(".wav")])
    #        #print(genre, len(audio_filenames))
    #        for audio_filename in audio_filenames:
    #            audio_filepath = os.path.join(genre_dirpath, audio_filename)
    #            filepaths.append(audio_filepath)
    #    return filepaths

    @cached_property
    def audio_files(self):
        files = []
        for genre in self.genres:
            genre_dirpath = os.path.join(GENRES_DIRPATH, genre)
            audio_filenames = sorted([fname for fname in os.listdir(genre_dirpath) if fname.endswith(".wav")])
            #print(genre, len(audio_filenames))

            for audio_filename in audio_filenames:
                audio_filepath = os.path.join(genre_dirpath, audio_filename)
                files.append(AudioFile(audio_filepath))

        return files

    def features_csv_filepath(self, track_length, n_mfcc):
        features_dirpath = os.path.join(GTZAN_DIRPATH, f"features_{track_length}s")
        os.makedirs(features_dirpath, exist_ok=True)
        return os.path.join(features_dirpath, f"mfcc_{n_mfcc}_features.csv")

    def read_features_csv(self, track_length, n_mfcc):
        csv_filepath = self.features_csv_filepath(track_length, n_mfcc)
        return read_csv(csv_filepath)


# -*- coding: utf-8 -*-

import codecs
import xml.etree.ElementTree as ET

# 读取待修改文件
updateTree = ET.parse("volun_qa.aiml")
root = updateTree.getroot()
f = codecs.open("志愿者常见问题问答.txt", 'r', 'utf8')
for line in f:
    # 创建新节点并添加为root的子节点
    category = ET.Element("category")
    root.append(category)

    fields = line.strip().split('|')
    if len(fields) < 2:
        break

    pattern = ET.Element("pattern")
    pattern.text = fields[0]
    category.append(pattern)

    template = ET.Element("template")
    template.text = fields[1]
    category.append(template)

f.close()

updateTree.write("volun_qa.aiml", encoding='utf-8')


from __future__ import annotations

from sqlalchemy import Column, Integer, String
from sqlalchemy.orm import relationship

from .. import database

class Organization(database.Base):
    __tablename__ = "organization"

    id = Column(Integer, primary_key=True)
    name = Column(String, nullable=False)
    slug = Column(String, nullable=False)
    external_slug = Column(String)

    items = relationship("Item")
    users = relationship("User")
    sentry_installations = relationship("SentryInstallation")

    def __init__(self, name: str, slug: str, external_slug: str) -> None:
        self.name = name
        self.slug = slug
        self.external_slug = external_slug

    def __repr__(self) -> str:
        return f"<Organization #{self.id}: {self.slug}>"


import scrapy

class Test(scrapy.Spider):
    name = 'xxx'
    start_urls = ['https://github.com/shiyanlou?tab=repositories']

    def parse(self, r):
        for i in r.xpath('//li[contains(@class, "col-12")]'):
            yield {
                'name': i.css('a::text').extract_first().strip(),
                'update_time': i.xpath('.//relative-time/@datetime').extract_first()
            }
        url = r.xpath('//div[@class="pagination"]/a')[-1].css('::attr(href)').extract_first()
        if r.xpath('//div[@class="pagination"]/a')[-1].css('::text').extract_first() == 'Next':
            yield r.follow(url, callback=self.parse)


import sys
from instagur import app
from instagur.database import db_session, init_db, init_engine

app.config.from_pyfile('../config/config.py')
if len(sys.argv) == 2:
    conf = sys.argv[1]
    print(f'Loading additional config {conf}')
    app.config.from_pyfile(f'config/{conf}')

init_engine(app.config['DATABASE_URI'])
init_db()
app.run(port=8080)


import requests
import json
import pandas as pd

def OrientQuery(query,database):

    url = "http://localhost:2480/command/CustomerE2E/sql"

    payload = json.dumps({
        "command": "select * from COLUMN"
    })
    headers = {
        'Authorization': 'Basic cm9vdDpyb290',
        'Content-Type': 'application/json',
        'Cookie': 'OSESSIONID=OS1647213389295-1231421075967031879'
    }

    response = requests.request("POST", url, headers=headers, data=payload)
    #data = json.loads(response.text)
    #df = pd.json_normalize(data['result'])

    return (response.text)


from typing import List

class Solution:
    def exist(self, board: List[List[str]], word: str) -> bool:

        def find(ix, i, j, readed):
            readed.append((i, j))
            if ix == len(word):
                return True
            if board[i][j] != word[ix]:
                return False
            rslt = []
            candi = [(i-1, j), (i+1, j), (i, j-1), (i, j+1)]
            for nxt_i, nxt_j in candi:
                if nxt_i < 0 or nxt_i >= len(board) or nxt_j < 0 or nxt_j >= len(board[0]) or (nxt_i, nxt_j) in readed:
                    continue
                r = find(ix+1, nxt_i, nxt_j)
                print(i, j, r, nxt_i, nxt_j)
                rslt.append(r)
            print(i, j, rslt)
            if sum(rslt) == 0:
                return False
            return True

        for i in range(len(board)):
            for j in range(len(board[0])):
                readed = []
                print('*************************')
                if find(0, i, j):
                    return True
        return False

s = Solution()
board = [["A","B","C","E"],
         ["S","F","C","S"],
         ["A","D","E","E"]]
word = "ABFS"
print(s.exist(board, word))


from run_game import RunGame
from gestures import Choices
from human import Human
from comphuman import Computer

if __name__ == '__main__':

    run_game = RunGame().run_gesture()
    player1 = Computer()
    player2 = Human()
    correct_value = Choices()
    print(Choices().Gesture)
    print()
    RunGame().run_game(player1, player2)
    # game_gesture = Choices().show_choices()
    Choices().is_valid(correct_value)
    RunGame().winner(player1, player2)
print("\n---------------\n")


import torch
import torch.nn as nn
import torch.nn.functional as F
from .TopicDistQuant import TopicDistQuant
from .TSC import TSC

class TSCTM(nn.Module):
    '''
        Mitigating Data Sparsity for Short Text Topic Modeling by Topic-Semantic Contrastive Learning. EMNLP 2022

        Xiaobao Wu, Anh Tuan Luu, Xinshuai Dong.

        Note: This implementation does not include TSCTM with augmentations. For augmentations, see https://github.com/BobXWu/TSCTM.
    '''

    def __init__(self, vocab_size, num_topics=50, en_units=200, temperature=0.5, weight_contrast=1.0):
        super().__init__()

        self.fc11 = nn.Linear(vocab_size, en_units)
        self.fc12 = nn.Linear(en_units, en_units)
        self.fc21 = nn.Linear(en_units, num_topics)

        self.mean_bn = nn.BatchNorm1d(num_topics)
        self.mean_bn.weight.requires_grad = False
        self.decoder_bn = nn.BatchNorm1d(vocab_size)
        self.decoder_bn.weight.requires_grad = False

        self.fcd1 = nn.Linear(num_topics, vocab_size, bias=False)

        for m in self.modules():
            if isinstance(m, (nn.Conv2d, nn.Linear)):
                nn.init.xavier_uniform_(m.weight)
                if m.bias is not None:
                    nn.init.zeros_(m.bias)

        self.topic_dist_quant = TopicDistQuant(num_topics, num_topics)
        self.contrast_loss = TSC(temperature, weight_contrast)

    def get_beta(self):
        return self.fcd1.weight.T

    def encode(self, inputs):
        e1 = F.softplus(self.fc11(inputs))
        e1 = F.softplus(self.fc12(e1))
        return self.mean_bn(self.fc21(e1))

    def decode(self, theta):
        d1 = F.softmax(self.decoder_bn(self.fcd1(theta)), dim=1)
        return d1

    def get_theta(self, inputs):
        theta = self.encode(inputs)
        softmax_theta = F.softmax(theta, dim=1)
        return softmax_theta

    def forward(self, inputs):
        theta = self.encode(inputs)
        softmax_theta = F.softmax(theta, dim=1)

        quant_rst = self.topic_dist_quant(softmax_theta)

        recon = self.decode(quant_rst['quantized'])
        loss = self.loss_function(recon, inputs) + quant_rst['loss']

        features = torch.cat([F.normalize(theta, dim=1).unsqueeze(1)], dim=1)
        contrastive_loss = self.contrast_loss(features, quant_idx=quant_rst['encoding_indices'])
        loss += contrastive_loss

        return {'loss': loss, 'contrastive_loss': contrastive_loss}

    def loss_function(self, recon_x, x):
        loss = -(x * (recon_x).log()).sum(axis=1)
        loss = loss.mean()
        return loss


from typing import Any, List
from uuid import UUID

from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session

from app import crud, models, schemas
from app.api import deps

router = APIRouter()

@router.get("/", response_model=List[schemas.Definition])
def read_definitions(
    db: Session = Depends(deps.get_db),
    skip: int = 0,
    limit: int = 100,
    current_user: models.User = Depends(deps.get_current_active_user),
) -> Any:
    """
    Retrieve definitions.
    """
    if crud.user.is_superuser(current_user):
        definitions = crud.definition.get_multi(db, skip=skip, limit=limit)
    else:
        definitions = crud.definition.get_multi_by_owner(
            db=db, owner_uuid=current_user.uuid, skip=skip, limit=limit
        )
    return definitions

@router.get("/{uuid}", response_model=schemas.Definition)
def read_definition(
    *,
    db: Session = Depends(deps.get_db),
    uuid: UUID,
) -> Any:
    """
    Get definition by ID.
    """
    definition = crud.definition.get_by_uuid(db=db, uuid=uuid)
    if not definition:
        raise HTTPException(status_code=404, detail="Definition not found")
    return definition

# @router.post("/{uuid}", response_model=schemas.Definition)
# def create_definition(
#     *,
#     db: Session = Depends(deps.get_db),
#     uuid: UUID,
#     definition_in: schemas.DefinitionCreate,
#     current_user: models.User = Depends(deps.get_current_active_user),
# ) -> Any:
#     """
#     Create new definition.
#     """
#     definition = crud.definition.create_with_owner(db=db, obj_in=definition_in, owner_uuid=current_user.uuid)
#     return definition

# @router.put("/{id}", response_model=schemas.Definition)
# def update_definition(
#     *,
#     db: Session = Depends(deps.get_db),
#     id: int,
#     definition_in: schemas.DefinitionUpdate,
#     current_user: models.User = Depends(deps.get_current_active_user),
# ) -> Any:
#     """
#     Update an definition.
#     """
#     definition = crud.definition.get(db=db, id=id)
#     if not definition:
#         raise HTTPException(status_code=404, detail="Definition not found")
#     if not crud.user.is_superuser(current_user) and (definition.owner_uuid != current_user.uuid):
#         raise HTTPException(status_code=400, detail="Not enough permissions")
#     definition = crud.definition.update(db=db, db_obj=definition, obj_in=definition_in)
#     return definition

# @router.delete("/{id}", response_model=schemas.Definition)
# def delete_definition(
#     *,
#     db: Session = Depends(deps.get_db),
#     id: int,
#     current_user: models.User = Depends(deps.get_current_active_user),
# ) -> Any:
#     """
#     Delete an definition.
#     """
#     definition = crud.definition.get(db=db, id=id)
#     if not definition:
#         raise HTTPException(status_code=404, detail="Definition not found")
#     if not crud.user.is_superuser(current_user) and (definition.owner_uuid != current_user.uuid):
#         raise HTTPException(status_code=400, detail="Not enough permissions")
#     definition = crud.definition.remove(db=db, id=id)
#     return definition


# coding: utf-8

# In[ ]:

from scipy import sparse
import numpy as np

train_text_tfidf=sparse.load_npz("./npz/train_apitext_valuetext.npz")
train_api_tfidf=sparse.load_npz("./npz/train_api_set_x.npz")
train_value_tfidf=sparse.load_npz("./npz/train_value_set_x.npz")
train_count_fea=np.load("./npy/train_count_fea_x.npy")

train_x=sparse.hstack((train_text_tfidf,train_api_tfidf))
train_x=sparse.hstack((train_x,train_value_tfidf))
train_x=sparse.hstack((train_x,train_count_fea))
print(train_x.shape)

# In[ ]:

test_text_tfidf=sparse.load_npz("./npz/test_apitext_valuetext.npz")
test_api_tfidf=sparse.load_npz("./npz/test_api_set_x.npz")
test_value_tfidf=sparse.load_npz("./npz/test_value_set_x.npz")
test_count_fea=np.load("./npy/test_count_fea_x.npy")

test_x=sparse.hstack((test_text_tfidf,test_api_tfidf))
test_x=sparse.hstack((test_x,test_value_tfidf))
test_x=sparse.hstack((test_x,test_count_fea))
print(test_x.shape)

# In[ ]:

import numpy as np
train_y=np.load("./npy/data1_train_y.npy")

# In[ ]:

from sklearn.model_selection import  StratifiedKFold
import lightgbm as lgb
from sklearn.metrics import  log_loss
nfold=10

kf=StratifiedKFold(nfold)
train_x=train_x.tocsr()
test_x=test_x.tocsr()

params = {
    'max_depth':-1,
    'boosting_type':'gbdt',
    'num_leaves': 31,
    'objective': 'multiclass',
    'min_data_in_leaf': 200,
    'learning_rate': 0.02,
    'max_bin':255,
    'feature_fraction': 0.64,
    'bagging_fraction': 0.8,
    'bagging_freq': 1,
    'metric': 'multi_logloss',
    'num_class':6,
    'num_threads': -1,
    'is_unbalance':True,
}
import numpy as np
oof_train=np.zeros((train_x.shape[0],6))
oof_test=np.zeros((test_x.shape[0],6))
oof_test_skf=np.zeros((nfold,test_x.shape[0],6))
logloss_csv=[]

for i,(train,test) in  enumerate(kf.split(train_x,train_y)):
    print("*******第 %d 折********"%i)

    x_tr=train_x[train]
    x_te=train_x[test]
    y_tr=train_y[train]
    y_te=train_y[test]

    lgb_train = lgb.Dataset(x_tr, y_tr)
    lgb_eval = lgb.Dataset(x_te,y_te)
    lgb_train = lgb.Dataset(x_tr, y_tr)
    lgb_eval = lgb.Dataset(x_te,y_te,reference=lgb_train)
    gbm = lgb.train(params,
                lgb_train,
                num_boost_round=1000,
                valid_sets=lgb_eval,
                early_stopping_rounds=10)
    eva_pre=gbm.predict(x_te,num_iteration=gbm.best_iteration)
    oof_train[test]=eva_pre
    logloss=log_loss(y_te,eva_pre)
    print("valid logloss %s"%logloss)
    logloss_csv.append(logloss)
    pre=gbm.predict(test_x,num_iteration=gbm.best_iteration)
    oof_test_skf[i,:]=pre

print("mean loss  %f"%np.mean(logloss_csv))

# In[ ]:

oof_test=oof_test_skf.mean(axis=0)
score=np.mean(logloss_csv)
print(score)

# In[ ]:

# import numpy as np
# def check_ans(result):
#     new_result=[]
#     for i in result:
#         m=i.sum()
#         for j in range(0,6):
#             i[j]=i[j]/m
#         if abs(i.sum()-1)>=1e-6:
#             print(i.sum()-1)
#         new_result.append(i)
#     return new_result

# In[ ]:

# import numpy as np

# res=check_ans(oof_test)

# In[ ]:

# #
# test_prob=pd.DataFrame(oof_test)
# test_prob.columns=['prob0','prob1','prob2','prob3','prob4','prob5']
# test_prob['file_id']=test_data['file_id']

# In[ ]:

# test_prob=test_prob[['file_id','prob0','prob1','prob2','prob3','prob4','prob5']]
# test_prob.head(20)

# In[ ]:

# test_prob.to_csv("lgb_submit_%f.csv"%np.mean(logloss_csv),index=False)

# In[ ]:

# test_prob.info()

# In[ ]:

#保存train_prob
import pandas as pd
train_num=1
train_prob=pd.DataFrame(oof_train)
train_prob.columns=['class_%s'%i for i  in   range(0,6)]
test_prob=pd.DataFrame(oof_test)
test_prob.columns=['class_%s'%i for i in range(0,6)]
train_prob.to_csv("./stacking_csv/cv10/train%d_lgb_train_prob_cv10_%f.csv"%(train_num,score),index=False)
test_prob.to_csv("./stacking_csv/cv10/train%d_lgb_test_prob_cv10_%f.csv"%(train_num,score),index=False)

# In[ ]:

np.mean(logloss_csv)

# In[ ]:

from sklearn.metrics import  classification_report
y_pr=[np.argmax(i) for i in eva_pre]
print(classification_report(y_te,y_pr))


import cv2
import numpy as np
import mediapipe as mp
from src import string_constants, var, landmark_condition
from pynput.keyboard import Controller, Key

keyboard = Controller()
mp_hands = mp.solutions.hands
hands = mp_hands.Hands()
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles

def put_text_in_middle(image, text):
    text_size = cv2.getTextSize(text, var.font, var.scale_default, var.thickness_double)[0]

    text_x = (image.shape[1] - text_size[0]) // 2
    text_y = (image.shape[0] + text_size[1]) // 2

    cv2.putText(image, text, (text_x, text_y), var.font, var.scale_default, var.white_color, var.thickness_double)
    return image

def format_number_lead_zero(num):
    return str(num) if num >= 10 else '0' + str(num)

def add_normal_text_to_right_bottom(image, text):
    x = 10
    y = image.shape[0] - 10
    cv2.putText(image, text, (x, y), var.font, var.scale_half, var.white_color, var.thickness_double)

def add_warning_text_to_bottom_left(image, text):
    x = 10
    y = image.shape[0] - 30
    cv2.putText(image, text, (x, y), var.font, var.scale_half, var.white_color, var.thickness_double)

def draw_hand_bounding_box(frame, hand_landmarks, player_name):
    image_height, image_width, _ = frame.shape
    landmark_list = []
    for landmark in hand_landmarks.landmark:
        x = int(landmark.x * image_width)
        y = int(landmark.y * image_height)
        landmark_list.append((x, y))

    x, y, w, h = cv2.boundingRect(np.array(landmark_list))
    cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)

    text_size = cv2.getTextSize(player_name,
                                var.font, var.scale_half,
                                var.thickness_double
                                )[0]

    cv2.rectangle(frame, (x - 1, y - text_size[1] - var.space_between),
                  (x + text_size[0], y + text_size[1] - 2 * var.space_between - 1),
                  var.green_color, var.border_fill)
    cv2.putText(frame, player_name,
                (x, y - var.space_between),
                var.font, var.scale_half,
                var.white_color, var.thickness_double)

def get_hand_move(hand_landmarks):
    landmarks = hand_landmarks.landmark
    if all([landmarks[i].y < landmarks[i + 3].y for i in range(9, 20, 4)]):
        return string_constants.ROCK
    elif landmarks[13].y < landmarks[16].y and landmarks[17].y < landmarks[20].y:
        return string_constants.SCISSORS
    else:
        return string_constants.PAPER

def draw_multiline_text(image, text):
    lines = text.split('\n')
    total_text_height = len(lines) * cv2.getTextSize(lines[0], var.font, var.scale_half, var.thickness_double)[0][1]
    start_y = (image.shape[0] - total_text_height) // 2
    try:
        margin_x = cv2.getTextSize(lines[1], var.font, var.scale_half, var.thickness_double)[0][0]
    except Exception as e:
        margin_x = cv2.getTextSize(lines[0], var.font, var.scale_half, var.thickness_double)[0][0]
    start_x = (image.shape[1] - margin_x) // 2
    for line in lines:
        text_size = cv2.getTextSize(line, var.font, var.scale_half, var.thickness_double)[0]
        # start_x = (image.shape[1] - text_size[0]) // 2
        cv2.putText(image, line, (start_x, start_y), var.font, var.scale_half, var.white_color, var.thickness_double)
        start_y = int(start_y + text_size[1] * var.line_height)
    return image

def check_winner(player1, player2):
    if player1 == player2:
        return string_constants.TIED
    elif player1 == string_constants.PAPER and player2 == string_constants.ROCK:
        return string_constants.PLAYER1_WIN
    elif player1 == string_constants.ROCK and player2 == string_constants.SCISSORS:
        return string_constants.PLAYER1_WIN
    elif player1 == string_constants.SCISSORS and player2 == string_constants.PAPER:
        return string_constants.PLAYER1_WIN
    else:
        return string_constants.PLAYER2_WIN

def draw_landmarks(frame, hand_landmarks):
    mp_drawing.draw_landmarks(
        frame, hand_landmarks, mp_hands.HAND_CONNECTIONS,
        mp_drawing_styles.get_default_hand_landmarks_style(),
        mp_drawing_styles.get_default_hand_connections_style()
    )

def game1_running(clock, success, game_text, player1, player2, results):
    if 0 <= clock <= 20:
        success = True
        game_text = string_constants.start
    elif clock < 50:
        game_text = f"{5 - clock // 10}..."
    # elif clock < 40:
    #     game_text = "2..."
    # elif clock < 50:
    #     game_text = "1..."
    elif clock < 60:
        game_text = string_constants.process
    elif clock == 60:
        hand_landmarks = results.multi_hand_landmarks
        if hand_landmarks and len(hand_landmarks) == 2:
            player1 = get_hand_move(hand_landmarks[0])
            player2 = get_hand_move(hand_landmarks[1])
        else:
            success = False
            game_text = string_constants.FAIL
    elif clock < var.MAX_TIME:
        if success:
            game_text = f"{string_constants.PLAYER1} {player1}.\n{string_constants.PLAYER2} {player2}.\n"
            game_text += check_winner(player1, player2)
            print(game_text)
    return clock, success, game_text, player1, player2

def get_current_option_and_warming(frame, text_frame):
    results = hands.process(frame)
    if results.multi_hand_landmarks is None:
        return None
    if len(results.multi_hand_landmarks) == 1:
        hand_landmarks = results.multi_hand_landmarks[0]
        draw_hand_bounding_box(text_frame, hand_landmarks, string_constants.PLAYER1)
        if landmark_condition.is_five(hand_landmarks):
            return 5
        if landmark_condition.is_one(hand_landmarks):
            return 1
        if landmark_condition.is_three(hand_landmarks):
            return 3
        if landmark_condition.is_two(hand_landmarks):
            return 2
        else:
            return None
    elif len(results.multi_hand_landmarks) == 2:
        add_warning_text_to_bottom_left(text_frame, string_constants.warn_one_hand)
    return None

def detect_rock_gestures(hand_landmarks):
    if (hand_landmarks.landmark[8].y > hand_landmarks.landmark[5].y) \
            and (hand_landmarks.landmark[12].y > hand_landmarks.landmark[9].y) \
            and (hand_landmarks.landmark[16].y > hand_landmarks.landmark[13].y):
        return string_constants.ROCK

def put_text_horizontal(image, is_horizontal=False):
    cv2.putText(image, string_constants.waiting_to_exit if is_horizontal else string_constants.tutor_to_exit,
                var.position_horizontal_text,
                var.font,
                var.scale_half, var.white_color, var.thickness_double)

def is_quite_game(is_quit, flag_game, option, game_clock):
    if is_quit:
        return False, False, None, var.MIN_TIME, var.MIN_TIME
    return is_quit, flag_game, option, var.MIN_TIME, game_clock

def change_video_capture_size(video_capture):
    video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, var.new_width)
    video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, var.new_height)
    pass


# n이랑 m은 input만 받고 활용 안 한다.
n = int(input())
# input 받은 숫자들을 map을 통해서 리스트에 담는다.
temp1 = list(map(int, input().split()))
# 숫자들을 곱을 num1에 할당해준다.
num1 = 1
for i in temp1:
    num1 *=i

# 위와 동일
m = int(input())
temp2 = list(map(int, input().split()))
num2 = 1
for j in temp2:
    num2 *=j

# 유클리드 호제법을 통해 최대공약수를 구하는 함수를 정의한다.
def euclid_gcd(a, b): # a > b
    res = 0
    while b:
        res = a % b
        a = b
        b = res
    return a

result = euclid_gcd(num1, num2)
# 9자리 이하일 때는 그냥 출력한다.
if len(str(result)) <= 9:
    print(result)
# 10자리 이상일 때는 슬라이싱을 통해 뒤에 9자리만 출력한다.
else:
    print(str(result)[-9:len(str(result))])


import matplotlib.pyplot as plt
import matplotlib as mpl
import numpy as np
import time as ti
import pandas as pd
from scipy import stats
import random as rd
from matplotlib.font_manager import FontProperties
from scipy import interpolate
font = FontProperties(fname=r'C:/Users/user/Anaconda3/Lib/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSans.ttc')

def linear_regression(x,y):
  x=np.concatenate((np.ones((x.shape[0],1)),x[:,np.newaxis]),axis=1)
  y=y[:,np.newaxis]
  beta=np.matmul(np.matmul(np.linalg.inv(np.matmul(x.T,x)),x.T),y)
  return beta
pass

path = 'well.csv'
file = pd.read_csv(path)
print(file['level'])
print(file['ELE'])
print(len(file['level']))

by_hand = linear_regression(file['ELE'], file['level'])
print(by_hand[0])
print(by_hand[1])
dis = by_hand[0]
m = by_hand[1]
str_dis=str(dis)
str_m=str(m)
xs = np.linspace(0,150, 5)
ys = dis+m*xs
plt.xlabel("地表高程(m)", fontproperties=font, fontsize=15)
plt.ylabel("水頭(m)", fontproperties=font, fontsize=15)
plt.plot(xs ,ys, color='green')
plt.scatter(file['ELE'], file['level'], color='red' )
plt.text(10, 100, "水頭=" + str_m + "高程" + str_dis + "(單位:公尺)" , fontproperties=font, fontsize=10)
plt.show()


from django.shortcuts import render, get_object_or_404, redirect
import urllib
# Create your views here.
def oauth(request):
    code = request.GET['code']
    print('code= ' + str(code))

    client_id = '13839c3b390207f7c0c7eaa789139e34'
    redirect_uri = 'http://127.0.0.1:8000/account/login/kakao/callback'
    access_token_request_uri = "https://kauth.kakao.com/oauth/token?grant_type=authorization_code&"

    access_token_request_uri += "client_id=" + client_id
    access_token_request_uri += "&redirect_uri=" + redirect_uri
    access_token_request_uri += "&code=" + code
    print(access_token_request_uri)

    access_token_request_uri_data = requests.get(access_token_request_uri)
    json_data = access_token_request_uri_data.json()
    access_token = json_data['access_token']
    print(access_token)

    user_profile_info_uri = "https://kapi.kakao.com/v2/user/me?access_token="
    user_profile_info_uri += str(access_token)
    print(user_profile_info_uri)

    user_profile_info_uri_data = requests.get(user_profile_info_uri)
    user_json_data = user_profile_info_uri_data.json()
    user_nickname = user_json_data['properties']['nickname']
    print(user_nickname)

    return redirect('index')

def kakao_login(request):
    login_request_uri = 'https://kauth.kakao.com/oauth/authorize?'
    client_id = '13839c3b390207f7c0c7eaa789139e34'
    redirect_uri = 'http://127.0.0.1:8000/account/login/kakao/callback'

    login_request_uri += 'client_id=' + client_id
    login_request_uri += '&redirect_uri=' + redirect_uri
    login_request_uri += '&response_type=code'

    return redirect(login_request_uri)


#//-------------------------------------------------------------------
#/*/{Protheus.doc} MATC070 -
# 	CASO DE USO test_FIN001_CT001 - Inclusão de desmontagem:
#	CASO DE USO test_FIN001_CT002 - Visualização de desmontagem.
#	CASO DE USO test_FIN001_CT003 - Estorno de Desmontagem.
#
#  - Seguintes recursos de tela: 1o Nivel, ShowF4.
#  - Controle de Endereço, Controle de Rastro com Lote e Sub-Lote
#  - Informando uma porcentagem de rateio abaixo de 100%
#
#INCLUSÃO DE MOVIMENTACAÇÃO
#TABELA DE MOVIMENTAÇÃO SD3
#
#@author PEDRO ANTONIO MISSAGLIA
#@since 26/06/2019
#@version P12
#/*/
#//-------------------------------------------------------------------
from tir import Webapp
import unittest
import time

class MATC070(unittest.TestCase):

	@classmethod
	def setUpClass(inst):
		inst.oHelper = Webapp()
		inst.oHelper.Setup('SIGAEST','21/06/2019','T1','D MG 01')
		inst.oHelper.Program('MATC070')
		#inst.oHelper.SetLateralMenu('Atualizações > Movimentações > Internas > Desmontagem')

	def test_MATC070_CT001(self):

		self.oHelper.SearchBrowse('D MG 01 ESTSE0000000000000000000000037')
		self.oHelper.SetButton('Consulta')
		self.oHelper.SetButton('Imprimir')
		self.oHelper.SetButton('Cancelar')
		self.oHelper.SetButton('Ordem')
		self.oHelper.SetButton('Ok')
		self.oHelper.SetButton('Ok')
		self.oHelper.AssertTrue()

	@classmethod
	def tearDownClass(inst):
		inst.oHelper.TearDown()

if __name__ == '__main__':
	unittest.main()


from functools import partial
import logging
import math
from typing import Tuple, List, Callable, Any, Dict, Sequence, Optional

import torch
import torch.nn as nn

@torch.jit.script
def get_padding_size(length, dap_size):
    return (int(length / dap_size) + int((length % dap_size)!=0)) * dap_size - length

@torch.jit.script
def padding_feat(feat:torch.Tensor, ps1:torch.Tensor, ps2:torch.Tensor, is_mask:bool=False):
    if ps1 == 0 and ps2 == 0:
        return feat
    else:
        if is_mask:
            feat = torch.nn.functional.pad(feat, (0, int(ps1), 0, int(ps2)))
        else:
            feat = torch.nn.functional.pad(feat, (0, 0, 0, int(ps1), 0, int(ps2)))
        return feat

@torch.jit.script
def split_feat(feat, ps1, ps2, has_batch=False):
    if ps1 == 0 and ps2 == 0:
        return feat
    else:
        if has_batch:
            if ps1 != 0:
                feat = feat[:, :-ps1]
            if ps2 != 0:
                feat = feat[:, :, :-ps2]
        else:
            if ps1 != 0:
                feat = feat[:-ps1]
            if ps2 != 0:
                feat = feat[:, :-ps2]
        return feat

def permute_final_dims(tensor, inds):
    zero_index = -1 * len(inds)
    first_inds = [i for i in range(len(tensor.shape[:zero_index]))]
    return tensor.permute(first_inds + [zero_index + i for i in inds])

def flatten_final_dims(t, no_dims):
    return t.reshape(t.shape[:-no_dims] + (-1,))

def one_hot(x, num_classes, dtype=torch.float32):
    x_one_hot = torch.zeros(*x.shape, num_classes, dtype=dtype, device=x.device)
    x_one_hot.scatter_(-1, x.long().unsqueeze(-1), 1)
    return x_one_hot

def _fetch_dims(tree):
    shapes = []
    tree_type = type(tree)
    if tree_type is dict:
        for v in tree.values():
            shapes.extend(_fetch_dims(v))
    elif tree_type is list or tree_type is tuple:
        for t in tree:
            shapes.extend(_fetch_dims(t))
    elif tree_type is torch.Tensor:
        shapes.append(tree.shape)
    elif tree_type is torch.nn.parameter.Parameter:
        shapes.append(tree.shape)
    else:
        raise ValueError(f"{tree_type} Not supported")

    return shapes

def dict_map(fn, dic, leaf_type):
    new_dict = {}
    for k, v in dic.items():
        if type(v) is dict:
            new_dict[k] = dict_map(fn, v, leaf_type)
        else:
            new_dict[k] = tree_map(fn, v, leaf_type)

    return new_dict

def tree_map(fn, tree, leaf_type):
    if isinstance(tree, dict):
        return dict_map(fn, tree, leaf_type)
    elif isinstance(tree, list):
        return [tree_map(fn, x, leaf_type) for x in tree]
    elif isinstance(tree, tuple):
        return tuple([tree_map(fn, x, leaf_type) for x in tree])
    elif isinstance(tree, leaf_type):
        return fn(tree)
    else:
        print(type(tree))
        raise ValueError("Not supported")

tensor_tree_map = partial(tree_map, leaf_type=torch.Tensor)

@torch.jit.ignore
def _flat_idx_to_idx(
    flat_idx: int,
    dims: Tuple[int],
) -> Tuple[int]:
    idx = []
    for d in reversed(dims):
        idx.append(flat_idx % d)
        flat_idx = flat_idx // d

    return tuple(reversed(idx))

@torch.jit.ignore
def _get_minimal_slice_set(
    start: Sequence[int],
    end: Sequence[int],
    dims: int,
    start_edges: Optional[Sequence[bool]] = None,
    end_edges: Optional[Sequence[bool]] = None,
) -> Sequence[Tuple[int]]:
    """
        Produces an ordered sequence of tensor slices that, when used in
        sequence on a tensor with shape dims, yields tensors that contain every
        leaf in the contiguous range [start, end]. Care is taken to yield a
        short sequence of slices, and perhaps even the shortest possible (I'm
        pretty sure it's the latter).

        end is INCLUSIVE.
    """
    # start_edges and end_edges both indicate whether, starting from any given
    # dimension, the start/end index is at the top/bottom edge of the
    # corresponding tensor, modeled as a tree
    def reduce_edge_list(l):
        tally = 1
        for i in range(len(l)):
            reversed_idx = -1 * (i + 1)
            l[reversed_idx] *= tally
            tally = l[reversed_idx]

    if(start_edges is None):
        start_edges = [s == 0 for s in start]
        reduce_edge_list(start_edges)
    if(end_edges is None):
        end_edges = [e == (d - 1) for e,d in zip(end, dims)]
        reduce_edge_list(end_edges)

    # Base cases. Either start/end are empty and we're done, or the final,
    # one-dimensional tensor can be simply sliced
    if(len(start) == 0):
        return [tuple()]
    elif(len(start) == 1):
        return [(slice(start[0], end[0] + 1),)]

    slices = []
    path = []

    # Dimensions common to start and end can be selected directly
    for s,e in zip(start, end):
        if(s == e):
            path.append(slice(s, s + 1))
        else:
            break

    path = tuple(path)
    divergence_idx = len(path)

    # start == end, and we're done
    if(divergence_idx == len(dims)):
        return [tuple(path)]

    def upper():
        sdi = start[divergence_idx]
        return [
            path + (slice(sdi, sdi + 1),) + s for s in
            _get_minimal_slice_set(
                start[divergence_idx + 1:],
                [d - 1 for d in dims[divergence_idx + 1:]],
                dims[divergence_idx + 1:],
                start_edges=start_edges[divergence_idx + 1:],
                end_edges=[1 for _ in end_edges[divergence_idx + 1:]]
            )
        ]

    def lower():
        edi = end[divergence_idx]
        return [
            path + (slice(edi, edi + 1),) + s for s in
            _get_minimal_slice_set(
                [0 for _ in start[divergence_idx + 1:]],
                end[divergence_idx + 1:],
                dims[divergence_idx + 1:],
                start_edges=[1 for _ in start_edges[divergence_idx + 1:]],
                end_edges=end_edges[divergence_idx + 1:],
            )
        ]

    # If both start and end are at the edges of the subtree rooted at
    # divergence_idx, we can just select the whole subtree at once
    if(start_edges[divergence_idx] and end_edges[divergence_idx]):
        slices.append(
            path + (slice(start[divergence_idx], end[divergence_idx] + 1),)
        )
    # If just start is at the edge, we can grab almost all of the subtree,
    # treating only the ragged bottom edge as an edge case
    elif(start_edges[divergence_idx]):
        slices.append(
            path + (slice(start[divergence_idx], end[divergence_idx]),)
        )
        slices.extend(lower())
    # Analogous to the previous case, but the top is ragged this time
    elif(end_edges[divergence_idx]):
        slices.extend(upper())
        slices.append(
            path + (slice(start[divergence_idx] + 1, end[divergence_idx] + 1),)
        )
    # If both sides of the range are ragged, we need to handle both sides
    # separately. If there's contiguous meat in between them, we can index it
    # in one big chunk
    else:
        slices.extend(upper())
        middle_ground = end[divergence_idx] - start[divergence_idx]
        if(middle_ground > 1):
            slices.append(
                path + (slice(start[divergence_idx] + 1, end[divergence_idx]),)
            )
        slices.extend(lower())

    return [tuple(s) for s in slices]

@torch.jit.ignore
def _chunk_slice(
    t: torch.Tensor,
    flat_start: int,
    flat_end: int,
    no_batch_dims: int,
) -> torch.Tensor:
    """
        Equivalent to

            t.reshape((-1,) + t.shape[no_batch_dims:])[flat_start:flat_end]

        but without the need for the initial reshape call, which can be
        memory-intensive in certain situations. The only reshape operations
        in this function are performed on sub-tensors that scale with
        (flat_end - flat_start), the chunk size.
    """

    batch_dims = t.shape[:no_batch_dims]
    start_idx = list(_flat_idx_to_idx(flat_start, batch_dims))
    # _get_minimal_slice_set is inclusive
    end_idx = list(_flat_idx_to_idx(flat_end - 1, batch_dims))

    # Get an ordered list of slices to perform
    slices = _get_minimal_slice_set(
        start_idx,
        end_idx,
        batch_dims,
    )

    sliced_tensors = [t[s] for s in slices]

    return torch.cat(
        [s.view((-1,) + t.shape[no_batch_dims:]) for s in sliced_tensors]
    )

def chunk_layer(
    layer: Callable,
    inputs: Dict[str, Any],
    chunk_size: int,
    no_batch_dims: int,
    low_mem: bool = False,
    _out: Any = None,
    _add_into_out: bool = False,
) -> Any:
    """
    Implements the "chunking" procedure described in section 1.11.8.

    Layer outputs and inputs are assumed to be simple "pytrees,"
    consisting only of (arbitrarily nested) lists, tuples, and dicts with
    torch.Tensor leaves.

    Args:
        layer:
            The layer to be applied chunk-wise
        inputs:
            A (non-nested) dictionary of keyworded inputs. All leaves must
            be tensors and must share the same batch dimensions.
        chunk_size:
            The number of sub-batches per chunk. If multiple batch
            dimensions are specified, a "sub-batch" is defined as a single
            indexing of all batch dimensions simultaneously (s.t. the
            number of sub-batches is the product of the batch dimensions).
        no_batch_dims:
            How many of the initial dimensions of each input tensor can
            be considered batch dimensions.
        low_mem:
            Avoids flattening potentially large input tensors. Unnecessary
            in most cases, and is ever so slightly slower than the default
            setting.
    Returns:
        The reassembled output of the layer on the inputs.
    """
    if not (len(inputs) > 0):
        raise ValueError("Must provide at least one input")

    initial_dims = [shape[:no_batch_dims] for shape in _fetch_dims(inputs)]
    orig_batch_dims = tuple([max(s) for s in zip(*initial_dims)])

    def _prep_inputs(t):
        if(not low_mem):
            if not sum(t.shape[:no_batch_dims]) == no_batch_dims:
                t = t.expand(orig_batch_dims + t.shape[no_batch_dims:])
            t = t.reshape(-1, *t.shape[no_batch_dims:])
        else:
            t = t.expand(orig_batch_dims + t.shape[no_batch_dims:])
        return t

    prepped_inputs = tensor_tree_map(_prep_inputs, inputs)
    prepped_outputs = None
    if(_out is not None):
        reshape_fn = lambda t: t.view([-1] + list(t.shape[no_batch_dims:]))
        prepped_outputs = tensor_tree_map(reshape_fn, _out)

    flat_batch_dim = 1
    for d in orig_batch_dims:
        flat_batch_dim *= d

    no_chunks = flat_batch_dim // chunk_size + (
        flat_batch_dim % chunk_size != 0
    )

    i = 0
    out = prepped_outputs
    for _ in range(no_chunks):
        # Chunk the input
        if(not low_mem):
            select_chunk = (
                lambda t: t[i : i + chunk_size] if t.shape[0] != 1 else t
            )
        else:
            select_chunk = (
                partial(
                    _chunk_slice,
                    flat_start=i,
                    flat_end=min(flat_batch_dim, i + chunk_size),
                    no_batch_dims=len(orig_batch_dims)
                )
            )

        chunks = tensor_tree_map(select_chunk, prepped_inputs)

        # Run the layer on the chunk
        output_chunk = layer(**chunks)

        # Allocate space for the output
        if out is None:
            allocate = lambda t: t.new_zeros((flat_batch_dim,) + t.shape[1:])
            out = tensor_tree_map(allocate, output_chunk)

        # Put the chunk in its pre-allocated space
        out_type = type(output_chunk)
        if out_type is dict:
            def assign(d1, d2):
                for k, v in d1.items():
                    if type(v) is dict:
                        assign(v, d2[k])
                    else:
                        if(_add_into_out):
                            v[i: i + chunk_size] += d2[k]
                        else:
                            v[i: i + chunk_size] = d2[k]

            assign(out, output_chunk)
        elif out_type is tuple:
            for x1, x2 in zip(out, output_chunk):
                if(_add_into_out):
                    x1[i: i + chunk_size] += x2
                else:
                    x1[i : i + chunk_size] = x2
        elif out_type is torch.Tensor:
            if(_add_into_out):
                out[i: i + chunk_size] += output_chunk
            else:
                out[i: i + chunk_size] = output_chunk
        else:
            raise ValueError("Not supported")

        i += chunk_size

    reshape = lambda t: t.view(orig_batch_dims + t.shape[1:])
    out = tensor_tree_map(reshape, out)

    return out


# -*- coding: utf-8 -*-
from flask import json

from tests.base import BaseTestCase

from got.models.comments import Comments

class SeriesTestCase(BaseTestCase):
    def test_get_all_episodes(self):
        self.insert_sample_got_data(title="t1", season=1, episode=1)
        self.insert_sample_got_data(title="t2", season=1, episode=2)
        self.insert_sample_got_data(title="t3", season=2, episode=1)
        response = self.app.get("/v1/")
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content_type, "application/json")

        expcted_responsre = {"game_of_thrones": []}
        season = 1
        episode = 0
        for id in range(1, 4):
            if id == 3:
                episode = 1
                season = 2
            else:
                episode += 1

            expcted_responsre["game_of_thrones"].append(
                {
                    "title": f"t{id}",
                    "released": "2021-01-01",
                    "id": id,
                    "imdb_id": "112211221",
                    "imdb_rating": 1.0,
                    "season": season,
                    "episode": episode,
                }
            )
        self.assertDictEqual(json.loads(response.data), expcted_responsre)

    def test_get_episode_by_id(self):
        self.insert_sample_got_data(title="t1", season=1, episode=1)
        self.insert_sample_got_data(title="t2", season=1, episode=2)
        response = self.app.get("/v1/1")
        expected_response = {
            "game_of_thrones": [
                {
                    "title": "t1",
                    "released": "2021-01-01",
                    "id": 1,
                    "imdb_id": "112211221",
                    "imdb_rating": 1.0,
                    "season": 1,
                    "episode": 1,
                }
            ]
        }

        self.assertDictEqual(json.loads(response.data), expected_response)

    def test_get_imdb_high_rate(self):
        self.insert_sample_got_data(title="t1", season=1, episode=1, imdb_rate=9.0)
        self.insert_sample_got_data(title="t2", season=1, episode=2, imdb_rate=8.8)
        self.insert_sample_got_data(title="t3", season=2, episode=1, imdb_rate=7.8)
        self.insert_sample_got_data(title="t4", season=2, episode=2, imdb_rate=8.9)

        response = self.app.get("/v1/high_rate/")

        expected_response = {
            "game_of_thrones": [
                {
                    "title": "t1",
                    "released": "2021-01-01",
                    "id": 1,
                    "imdb_id": "112211221",
                    "imdb_rating": 9.0,
                    "season": 1,
                    "episode": 1,
                },
                {
                    "title": "t4",
                    "released": "2021-01-01",
                    "id": 4,
                    "imdb_id": "112211221",
                    "imdb_rating": 8.9,
                    "season": 2,
                    "episode": 2,
                },
            ]
        }

        self.assertDictEqual(json.loads(response.data), expected_response)

    def test_get_imdb_high_rate_season(self):
        self.insert_sample_got_data(title="t1", season=1, episode=1, imdb_rate=9.0)
        self.insert_sample_got_data(title="t2", season=1, episode=2, imdb_rate=8.8)
        self.insert_sample_got_data(title="t3", season=2, episode=1, imdb_rate=7.8)
        self.insert_sample_got_data(title="t4", season=2, episode=2, imdb_rate=8.9)

        response = self.app.get("/v1/high_rate/2")

        expected_response = {
            "game_of_thrones": [
                {
                    "title": "t4",
                    "released": "2021-01-01",
                    "id": 4,
                    "imdb_id": "112211221",
                    "imdb_rating": 8.9,
                    "season": 2,
                    "episode": 2,
                },
            ]
        }

        self.assertDictEqual(json.loads(response.data), expected_response)

class CommentsTestCase(BaseTestCase):
    def test_read_comments_of_an_episode(self):
        self.insert_sample_comment(body="body1", author="auther1")
        self.insert_sample_comment(body="body2", author="auther2")
        self.insert_sample_comment(body="body3", episode_id=2)
        response = self.app.get("/v1/comment/1")

        actual_response = json.loads(response.data)

        actual_comments = actual_response["comments"]
        for id in range(1, 2):
            actual_author = actual_comments[id - 1]["author"]
            self.assertEqual(actual_author, f"auther{id}")

    def test_post_comment(self):
        self.insert_sample_got_data()
        sample_comment = {
            "body": "foo bar foo",
            "author": "Amir",
        }
        response = self.app.post("/v1/comment/1", json=sample_comment)

        comments_from_db = Comments.query.all()

        self.assertEqual(len(comments_from_db), 1)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(comments_from_db[0].body, sample_comment["body"])
        self.assertEqual(comments_from_db[0].author, sample_comment["author"])
        self.assertEqual(comments_from_db[0].episode_id, 1)

    def test_post_comment_with_invalid_payload_returns_400(self):
        self.insert_sample_got_data()
        invalid_payload = {"foo": "bar"}
        response = self.app.post("/v1/comment/1", json=invalid_payload)
        comments_from_db = Comments.query.all()

        self.assertEqual(response.status_code, 400)
        self.assertEqual(len(comments_from_db), 0)

    def test_post_comment_with_invalid_episode_returns_400(self):
        self.insert_sample_got_data()
        sample_comment = {
            "body": "foo bar foo",
            "author": "Amir",
        }
        response = self.app.post("/v1/comment/2", json=sample_comment)
        comments_from_db = Comments.query.all()

        self.assertEqual(response.status_code, 400)
        self.assertEqual(len(comments_from_db), 0)

    def test_delete_comment(self):
        self.insert_sample_comment()
        comments_before_delete = Comments.query.all()
        response = self.app.delete("/v1/comment/1")
        comments_after_delete = Comments.query.all()

        self.assertEqual(response.status_code, 200)
        self.assertEqual(len(comments_before_delete), 1)
        self.assertEqual(len(comments_after_delete), 0)

    def test_delete_comment_with_invalid_id_returns_400(self):
        # database is empty so comment 1 does not exist!
        response = self.app.delete("/v1/comment/1")
        self.assertEqual(response.status_code, 400)

    def test_update_comment(self):
        self.insert_sample_comment(body="first body", author="person 1")

        payloads = [
            {"body": "second body"},
            {"author": "person 2"},
            {"episode_id": 2},
            {"body": "third body", "author": "person 3", "episode_id": 3},
        ]
        for update_payload in payloads:
            response = self.app.put("/v1/comment/1", json=update_payload)
            comment = Comments.query.filter(Comments.id == 1).first()
            comment_dict = comment.to_dict()

            self.assertEqual(response.status_code, 200)
            for key, value in update_payload.items():
                self.assertEqual(comment_dict[key], value)

    def test_update_comment_with_invalid_payload_returns_400(self):
        self.insert_sample_comment()
        invalid_payloads = [None, {}, {"foo": "bar"}]

        for payload in invalid_payloads:
            response = self.app.put("/v1/comment/1", json=payload)
            self.assertEqual(response.status_code, 400)

    def test_update_comment_with_invalid_id_returns_400(self):
        self.insert_sample_comment()
        payload = {"body": "test"}
        not_exist_id = 2
        response = self.app.put(f"/v1/comment/{not_exist_id}", json=payload)
        self.assertEqual(response.status_code, 400)


# Written by Seonwoo Min, Seoul National University (mswzeus@gmail.com)
# Some parts of the code were referenced from or inspired by below
# - Tristan Bepler's code (https://github.com/tbepler/protein-sequence-embedding-iclr2019)
# PLUS

""" P-ELMo model classes and functions """

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.utils.rnn import PackedSequence, pack_padded_sequence, pad_packed_sequence

from plus.model.plus_rnn import PLUS_RNN

class P_ELMo(PLUS_RNN):
    """ P-ELMo model """
    def __init__(self, cfg):
        super(PLUS_RNN, self).__init__()

        # LM hidden states -> EM input model (ELMo-style; use hidden states as input to EM model)
        self.fc_lm = nn.Linear(cfg.lm_dim, cfg.lm_proj_dim)
        self.x_embed = nn.Embedding(cfg.input_dim-1, cfg.lm_proj_dim, padding_idx=cfg.input_dim - 2)
        self.rnn = nn.LSTM(cfg.lm_proj_dim, cfg.hidden_dim, cfg.num_layers, bidirectional=True, batch_first=True)
        self.fc = nn.Linear(2 * cfg.hidden_dim, cfg.embedding_dim)

        # language modeling decoder
        self.decoder = nn.Linear(2 * cfg.hidden_dim, cfg.input_dim-1)

        # ordinal classification
        # initialize ordinal regression bias with 10 following ICLR19_Bepler
        bias_init = 10
        if cfg.num_classes is not None:
            self.ordinal_weight = nn.Parameter(torch.ones(1, cfg.num_classes))
            self.ordinal_bias = nn.Parameter(torch.zeros(cfg.num_classes) + bias_init)

    def forward(self, x_aa, x_lm, lengths):
        order = torch.argsort(lengths, descending=True)
        order_rev = torch.argsort(order)
        x_aa, x_lm, lengths = x_aa[order], x_lm[order], lengths[order]

        # language model + x embedding
        total_length = x_aa.shape[1] - 2
        for i in range(x_aa.size(0)): x_aa[i, 1:lengths[i] - 1] -= 1
        x_aa = pack_padded_sequence(x_aa[:, 1:-1], lengths - 2, batch_first=True)
        x_aa = PackedSequence(self.x_embed(x_aa.data), x_aa.batch_sizes)
        x_lm = pack_padded_sequence(x_lm, lengths - 2, batch_first=True)
        x_lm = PackedSequence(self.fc_lm(x_lm.data), x_lm.batch_sizes)
        h = PackedSequence(nn.ReLU()(x_aa.data + x_lm.data), x_aa.batch_sizes)

        # feed-forward bidirectional RNN
        self.rnn.flatten_parameters()
        h, _ = self.rnn(h)
        h = pad_packed_sequence(h, batch_first=True, total_length=total_length)[0][order_rev]

        # final projection
        z = self.fc(h)
        return z, h

    def em(self, h, lengths, cpu=False):
        # get representations with different lengths from the collated single matrix
        e = [None] * len(lengths)
        for i in range(len(lengths)):
            if cpu: e[i] = h[i, :lengths[i] - 2].cpu()
            else:   e[i] = h[i, :lengths[i] - 2]
        return e

class P_ELMo_lm(nn.Module):
    """ language model for P-ELMo """
    def __init__(self, cfg):
        super(P_ELMo_lm, self).__init__()
        self.embed = nn.Embedding(cfg.input_dim, cfg.input_dim-1, padding_idx=cfg.input_dim-1)

        input_dim = cfg.input_dim - 1
        layers = []
        for _ in range(cfg.num_layers):
            layers.append(nn.LSTM(input_dim, cfg.hidden_dim, 1, batch_first=True))
            input_dim = cfg.hidden_dim
        self.rnn = nn.ModuleList(layers)

        self.decoder = nn.Linear(cfg.hidden_dim, cfg.input_dim-1)

    def forward(self, x, lengths):
        x = self.embed(x)
        h_fwd, h_rvs = x[:, :-1], x[:, 1:]
        h_fwd, h_rvs = self.transform(h_fwd, self.reverse(h_rvs))

        b, l, d = h_fwd.size()
        h_fwd = self.decoder(h_fwd.contiguous().view(-1, d))
        h_rvs = self.decoder(h_rvs.contiguous().view(-1, d))

        zero = h_fwd.new_zeros((b, 1, h_fwd.size(1)))
        h_fwd = torch.cat((zero, h_fwd.view(b, l, -1)), 1)
        h_rvs = torch.cat((h_rvs.view(b, l, -1), zero), 1)
        logits = F.log_softmax(h_fwd + h_rvs, dim=2)

        return logits

    def load_weights(self, pretrained_model):
        # load pretrained_model weights
        state_dict = self.state_dict()
        for key, value in torch.load(pretrained_model, map_location=torch.device('cpu')).items():
            if key.startswith("module"): key = key[7:]
            if key in state_dict and value.shape == state_dict[key].shape: state_dict[key] = value
        self.load_state_dict(state_dict)

    def encode(self, x, lengths):
        order = torch.argsort(lengths, descending=True)
        order_rev = torch.argsort(order)
        x, lengths = x[order], lengths[order]

        with torch.no_grad():
            x = self.embed(x)
            h_fwd, h_rvs = x[:, :-1], x[:, 1:]
            total_length = x.shape[1] - 1
            h_fwd = pack_padded_sequence(h_fwd, lengths - 1, batch_first=True)
            h_rvs = pack_padded_sequence(h_rvs, lengths - 1, batch_first=True)
            h = self.transform(h_fwd, self.reverse(h_rvs, total_length), total_length, last_only=False)[order_rev]

        return h

    def reverse(self, h, total_length=None):
        # reverse given input tensor for reverse direction RNN
        if isinstance(h, PackedSequence):
            h, lengths = pad_packed_sequence(h, batch_first=True, total_length=total_length)
            h_rvs = h.clone().zero_()
            for i in range(h.size(0)):
                n = lengths[i]
                idx = [j for j in range(n - 1, -1, -1)]
                idx = torch.LongTensor(idx).to(h.device)
                h_rvs[i, :n] = h[i].index_select(0, idx)
            # repack h_rvs
            h_rvs = pack_padded_sequence(h_rvs, lengths, batch_first=True)
        else:
            idx = [i for i in range(h.size(1) - 1, -1, -1)]
            idx = torch.LongTensor(idx).to(h.device)
            h_rvs = h.index_select(1, idx)
        return h_rvs

    def transform(self, h_fwd, h_rvs, total_length=None, last_only=True):
        # transform into forward and reverse rnn hidden states
        h = []
        for l in range(len(self.rnn)):
            self.rnn[l].flatten_parameters()
            h_fwd, _ = self.rnn[l](h_fwd)
            h_rvs, _ = self.rnn[l](h_rvs)
            if not last_only:
                h.append(pad_packed_sequence(h_fwd, batch_first=True, total_length=total_length)[0][:, :-1])
                h.append(pad_packed_sequence(self.reverse(h_rvs), batch_first=True, total_length=total_length)[0][:, 1:])

        if not last_only:
            return torch.cat(h, 2)
        else:
            return h_fwd, self.reverse(h_rvs)