# implementation adapted from:
# https://github.com/yabufarha/ms-tcn/blob/master/model.py
import torch
import torch.nn as nn
import torch.nn.functional as F
import copy

class MultiStageModel(nn.Module):
    def __init__(self, hparams):
        self.num_stages = hparams.mstcn_stages  # 4 #2
        self.num_layers = hparams.mstcn_layers  # 10  #5
        self.num_f_maps = hparams.mstcn_f_maps  # 64 #64
        self.dim = hparams.mstcn_f_dim  #2048 # 2048
        self.num_classes = hparams.out_features  # 7
        self.causal_conv = hparams.mstcn_causal_conv
        print(
            f"num_stages_classification: {self.num_stages}, num_layers: {self.num_layers}, num_f_maps:"
            f" {self.num_f_maps}, dim: {self.dim}")
        super(MultiStageModel, self).__init__()
        self.stage1 = SingleStageModel(self.num_layers,
                                       self.num_f_maps,
                                       self.dim,
                                       self.num_classes,
                                       causal_conv=self.causal_conv)
        self.stages = nn.ModuleList([
            copy.deepcopy(
                SingleStageModel(self.num_layers,
                                 self.num_f_maps,
                                 self.num_classes,
                                 self.num_classes,
                                 causal_conv=self.causal_conv))
            for s in range(self.num_stages - 1)
        ])
        self.smoothing = False

    def forward(self, x):
        out_classes = self.stage1(x)
        outputs_classes = out_classes.unsqueeze(0)
        for s in self.stages:
            out_classes = s(F.softmax(out_classes, dim=1))
            outputs_classes = torch.cat(
                (outputs_classes, out_classes.unsqueeze(0)), dim=0)
        return outputs_classes

    @staticmethod
    def add_model_specific_args(parser):  # pragma: no cover
        mstcn_reg_model_specific_args = parser.add_argument_group(
            title='mstcn reg specific args options')
        mstcn_reg_model_specific_args.add_argument("--mstcn_stages",
                                                   default=4,
                                                   type=int)
        mstcn_reg_model_specific_args.add_argument("--mstcn_layers",
                                                   default=10,
                                                   type=int)
        mstcn_reg_model_specific_args.add_argument("--mstcn_f_maps",
                                                   default=64,
                                                   type=int)
        mstcn_reg_model_specific_args.add_argument("--mstcn_f_dim",
                                                   default=2048,
                                                   type=int)
        mstcn_reg_model_specific_args.add_argument("--mstcn_causal_conv",
                                                   action='store_true')
        return parser

class SingleStageModel(nn.Module):
    def __init__(self,
                 num_layers,
                 num_f_maps,
                 dim,
                 num_classes,
                 causal_conv=False):
        super(SingleStageModel, self).__init__()
        self.conv_1x1 = nn.Conv1d(dim, num_f_maps, 1)

        self.layers = nn.ModuleList([
            copy.deepcopy(
                DilatedResidualLayer(2**i,
                                     num_f_maps,
                                     num_f_maps,
                                     causal_conv=causal_conv))
            for i in range(num_layers)
        ])
        self.conv_out_classes = nn.Conv1d(num_f_maps, num_classes, 1)

    def forward(self, x):
        out = self.conv_1x1(x)
        for layer in self.layers:
            out = layer(out)
        out_classes = self.conv_out_classes(out)
        return out_classes

class DilatedResidualLayer(nn.Module):
    def __init__(self,
                 dilation,
                 in_channels,
                 out_channels,
                 causal_conv=False,
                 kernel_size=3):
        super(DilatedResidualLayer, self).__init__()
        self.causal_conv = causal_conv
        self.dilation = dilation
        self.kernel_size = kernel_size
        if self.causal_conv:
            self.conv_dilated = nn.Conv1d(in_channels,
                                          out_channels,
                                          kernel_size,
                                          padding=(dilation *
                                                   (kernel_size - 1)),
                                          dilation=dilation)
        else:
            self.conv_dilated = nn.Conv1d(in_channels,
                                          out_channels,
                                          kernel_size,
                                          padding=dilation,
                                          dilation=dilation)
        self.conv_1x1 = nn.Conv1d(out_channels, out_channels, 1)
        self.dropout = nn.Dropout()

    def forward(self, x):
        out = F.relu(self.conv_dilated(x))
        if self.causal_conv:
            out = out[:, :, :-(self.dilation * 2)]
        out = self.conv_1x1(out)
        out = self.dropout(out)
        return (x + out)

class DilatedSmoothLayer(nn.Module):
    def __init__(self, causal_conv=True):
        super(DilatedSmoothLayer, self).__init__()
        self.causal_conv = causal_conv
        self.dilation1 = 1
        self.dilation2 = 5
        self.kernel_size = 5
        if self.causal_conv:
            self.conv_dilated1 = nn.Conv1d(7,
                                           7,
                                           self.kernel_size,
                                           padding=self.dilation1 * 2 * 2,
                                           dilation=self.dilation1)
            self.conv_dilated2 = nn.Conv1d(7,
                                           7,
                                           self.kernel_size,
                                           padding=self.dilation2 * 2 * 2,
                                           dilation=self.dilation2)

        else:
            self.conv_dilated1 = nn.Conv1d(7,
                                           7,
                                           self.kernel_size,
                                           padding=self.dilation1 * 2,
                                           dilation=self.dilation1)
            self.conv_dilated2 = nn.Conv1d(7,
                                           7,
                                           self.kernel_size,
                                           padding=self.dilation2 * 2,
                                           dilation=self.dilation2)
        self.conv_1x1 = nn.Conv1d(7, 7, 1)
        self.dropout = nn.Dropout()

    def forward(self, x):
        x1 = self.conv_dilated1(x)
        x1 = self.conv_dilated2(x1[:, :, :-4])
        out = F.relu(x1)
        if self.causal_conv:
            out = out[:, :, :-((self.dilation2 * 2) * 2)]
        out = self.conv_1x1(out)
        out = self.dropout(out)
        return (x + out)


import os
import re
from pprint import pprint

import requests
from bs4 import BeautifulSoup

# Regex
num_pattern = re.compile(r"#(?P<num>\d{3})")
rel_url_pattern = re.compile(r"^/pokedex/\w+")

class PokemonInfo:
    def __init__(self, img_url, number, name, type_list, full_url):
        self.img_url = img_url
        self.number = number
        self.name = name
        self.type_list = type_list
        self.full_url = full_url

    def __str__(self):
        return f"<Pokemon: {self.name} (#{self.number:03d})>"

def href_relpath(href):
    return href and rel_url_pattern.match(href)

base_url = "https://pokemondb.net"
url = f"{base_url}/pokedex/national"
page = requests.get(url)

soup = BeautifulSoup(page.content, "html.parser")

info_cards = soup.find_all("div", class_="infocard")

pokemon_list = []
output_fixture = []

for ic in info_cards:
    print(ic)
    img_url = ic.find("span", class_="img-sprite")["data-src"]
    number = int(ic.find("small", string=lambda x: num_pattern.match(x)).text.replace("#", ""))
    name = ic.find("a", class_="ent-name").text
    types = ic.find_all("a", class_="itype")
    type_list = []
    for t in types:
        type_list.append(t.text)

    pokemon_url = ic.find_all("a")
    full_url = None
    for pu in pokemon_url:
        if rel_url_pattern.match(pu["href"]):
            full_url = f"{base_url}{pu['href']}"
            break

    pokemon = PokemonInfo(img_url, number, name, type_list, full_url)
    pokemon_list.append(pokemon)

if not os.path.exists("images"):
    os.makedirs("images")

for pokemon in pokemon_list:
    img_data = requests.get(pokemon.img_url)
    with open(f"images/{pokemon.name}.png", "wb") as fp:
        fp.write(img_data.content)

    output_fixture.append({
        "model": "api.Pokemon",
        "name": pokemon.name,
        "pk": pokemon.number,
        "types": []
    })


'''
@File  : coin-change-2.py
@Author: Swift
@Date  : 2021/4/29 17:52
@Link  : https://leetcode-cn.com/problems/coin-change-2/
@Desc  : 518. 零钱兑换 II
@Method:
'''

class Solution:

    def change(self, amount: int, coins):
        dp = [0]*(amount+1)
        dp[0] = 1
        for val in coins:
            for i in range(val, amount+1):
                dp[i] += dp[i-val]
        return dp[amount]

if __name__ == '__main__':
    solution = Solution()
    coins = [1,5,2]
    amount = 5
    count = solution.change(amount, coins)
    print(count)


import torch
import torch.nn as nn
import torch.nn.functional as F

class Head(nn.Module):
	'''
	Author: Sungjun Han
	Description: FFN layers for encoder or decoder
	input_size : int
	output_size : int
	activation : str
	dropout : float < 1
	'''

	def __init__(self,
				 input_size,
				 output_size,
				 activation,
				 dropout):

		super().__init__()
		self.head = nn.Sequential(
							nn.LayerNorm(input_size),
							nn.Linear(input_size,output_size),
							activation,
							nn.Dropout(dropout),
							)

	def forward(self, x):
		return self.head(x)

class StaticEmbeddingMixture(nn.Module):
	'''
	Author: Sungjun Han
	Description: static embedding model using max/min/average operatino for pooling

	embedding : torch tensor
	hidden_encoder_size : int
	hidden_decoder_size : int
	num_encoder_layers : int
	num_decoder_layers : int
	dropout : float < 1
	pooling : str in ['max', 'sum', 'prod']
	'''

	def __init__(self,
				 embedding,
				 hidden_encoder_size,
				 hidden_decoder_size,
				 num_encoder_layers,
				 num_decoder_layers,
				 dropout,
				 pooling = 'max'):

		super().__init__()

		self.embedding = embedding
		self.pooling = pooling
		hidden_size = embedding.weight.size(1)

		self.encoder_premise = nn.ModuleList([Head(hidden_size, hidden_encoder_size, nn.ReLU(), dropout)])
		for _ in range(num_encoder_layers-1):
			self.encoder_premise.append(Head(hidden_encoder_size, hidden_encoder_size, nn.ReLU(), dropout))

		self.encoder_hyp = nn.ModuleList([Head(hidden_size, hidden_encoder_size, nn.ReLU(), dropout)])
		for _ in range(num_encoder_layers-1):
			self.encoder_hyp.append(Head(hidden_encoder_size, hidden_encoder_size, nn.ReLU(), dropout))

		self.decoder = nn.ModuleList([Head(hidden_encoder_size*5, hidden_decoder_size, nn.ReLU(), dropout)])
		for _ in range(num_decoder_layers-2):
			self.decoder.append(Head(hidden_decoder_size, hidden_decoder_size, nn.ReLU(), dropout))
		self.decoder.append(nn.Linear(hidden_decoder_size, 1))

		self.loss_fn = nn.BCEWithLogitsLoss()

		if pooling == 'sum':
			self.pooling = torch.sum

		elif pooling =='product':
			self.pooling = torch.prod
		else:
			raise ValueError

	def forward(self, p, h1, h2, y=None ):

		p = self.embedding(p) # B X L X H
		h1 = self.embedding(h1)
		h2 = self.embedding(h2)

		#pool
		p  = self.pooling(p, dim=1) #B X H
		h1 = self.pooling(h1, dim=1)
		h2 = self.pooling(h2, dim=1)

		#encode
		for encoder in self.encoder_premise:
			p = encoder(p)

		for encoder in self.encoder_hyp:
			h1 = encoder(h1)
			h2 = encoder(h2)

		#concatenate [p - h1, p - h2, p , h1, h2]
		#or [p*h1, p*h2, torch.abs(p-h1), torch.abs(p-h2)]
		logit = torch.cat([p, h1, h2, p-h1, p-h2],dim=-1)

		for decoder in self.decoder:
			logit = decoder(logit)

		if y is None:
			return logit

		loss = self.loss_fn(logit.view(-1), y.view(-1))
		return logit, loss

class StaticEmbeddingRNN(nn.Module):
	'''
	Author: Sungjun Han
	Description: static embedding model using RNN for pooling

	embedding : torch tensor
	hidden_encoder_size : int
	hidden_decoder_size : int
	num_encoder_layers : int
	num_decoder_layers : int
	dropout : float < 1
	bidirectional : bool
	'''
	def __init__(self,
				 embedding,
				 hidden_encoder_size,
				 hidden_decoder_size,
				 num_encoder_layers,
				 num_decoder_layers,
				 dropout,
				 bidirectional=True):

		super().__init__()

		self.embedding = embedding
		hidden_size = embedding.weight.size(1)
		self.hidden_encoder_size = hidden_encoder_size
		self.hidden_decoder_size = hidden_decoder_size
		self.num_encoder_layers = num_encoder_layers
		self.num_decoder_layers = num_decoder_layers
		self.bidirectional = bidirectional

		self.encoder_premise = nn.GRU(hidden_size, hidden_encoder_size, num_encoder_layers, batch_first = True, dropout=dropout, bidirectional=bidirectional)
		self.encoder_hyp = nn.GRU(hidden_size, hidden_encoder_size, num_encoder_layers, batch_first = True, dropout=dropout, bidirectional=bidirectional)

		if bidirectional:
			hidden_encoder_size *= 2

		self.decoder = nn.ModuleList([Head(hidden_encoder_size*5, hidden_decoder_size, nn.ReLU(), dropout)])
		for _ in range(num_decoder_layers-2):
			self.decoder.append(Head(hidden_decoder_size, hidden_decoder_size, nn.ReLU(),dropout))
		self.decoder.append(Linear(hidden_decoder_size, 1))

		self.loss_fn = nn.BCEWithLogitsLoss()

	def forward(self, p, h1, h2, y=None):

		p = self.embedding(p)
		h1 = self.embedding(h1)
		h2 = self.embedding(h2)

		#encode
		p, _ = self.encoder_premise(p) # only use the last
		h1, _ = self.encoder_hyp(h1)
		h2, _ = self.encoder_hyp(h2)

		if self.bidirectional:
			B, L, _ = p.size()
			p = p.view(B, L, self.hidden_encoder_size, -1)

			_, L, __ = h1.size()
			h1 = h1.view(B, L, self.hidden_encoder_size, -1)

			_, L, __ = h2.size()
			h2 = h2.view(B, L, self.hidden_encoder_size, -1)

			p = torch.cat([p[:,-1,:,0],p[:,0,:,1]], dim=-1)
			h1 = torch.cat([h1[:,-1,:,0],h1[:,0,:,1]],dim=-1)
			h2 = torch.cat([h2[:,-1,:,0],h2[:,0,:,1]],dim=-1)
		else:
			p = p[:,-1,:]
			h1 = h1[:,-1,:]
			h2 = h2[:,-1,:]

		#concatenate (p - h1, p - h2, p , h1, h2)
		logit = torch.cat([p, h1, h2, p-h1, p-h2],dim=-1)

		#logit = torch.cat([p*h1, p-h1, p*h2, p-h2],dim=-1)

		for decoder in self.decoder:
			logit = decoder(logit)

		if y is None:
			return logit

		loss = self.loss_fn(logit.view(-1), y.view(-1))
		return logit, loss

class StaticEmbeddingCNN(nn.Module):
	'''
		Author: Anastasiia
		Description: static embedding model using CNN for pooling

		the CNN was assembled by following the tutotial by Chris Tran
		#https://chriskhanhtran.github.io/posts/cnn-sentence-classification/
	'''

	def __init__(self,
				 embedding,
				 hidden_decoder_size,
				 dropout,
				 kernel_sizes = [3,4,5],
				 num_kernels = [20,20,20]):

		super().__init__()

		self.embedding = embedding
		embedding_size = embedding.weight.size(1)

		self.conv1d_list_prem = nn.ModuleList([nn.Conv1d(in_channels=embedding_size,
														out_channels=num_kernels[i],
														kernel_size=kernel_sizes[i])
													for i in range(len(kernel_sizes))
												])

		self.conv1d_list_hyp = nn.ModuleList([nn.Conv1d(in_channels=embedding_size,
														out_channels=num_kernels[i],
														kernel_size=kernel_sizes[i])
													for i in range(len(kernel_sizes))
												])

		self.decoder = nn.Sequential(
							nn.Linear(sum(num_kernels)*4,hidden_decoder_size),
							nn.ReLU(),
							nn.Dropout(dropout),
							nn.LayerNorm(hidden_decoder_size),
							nn.Linear(hidden_decoder_size,1))
		self.loss_fn = nn.BCEWithLogitsLoss()

	def forward(self, p, h1, h2, y=None):

		p = self.embedding(p) # B X L X H
		h1 = self.embedding(h1)
		h2 = self.embedding(h2)

		#reshape for Conv1d
		p = p.permute(0, 2, 1) # B X H X L
		h1 = h1.permute(0, 2, 1)
		h2 = h2.permute(0, 2, 1)

		#CNN + ReLU
		p_conv_list = [F.relu(conv1d(p)) for conv1d in self.conv1d_list_prem]
		h1_conv_list = [F.relu(conv1d(h1)) for conv1d in self.conv1d_list_hyp]
		h2_conv_list = [F.relu(conv1d(h2)) for conv1d in self.conv1d_list_hyp]

		#max_pooling
		p_pool_list = [F.max_pool1d(p_conv, kernel_size=p_conv.shape[2])
			for p_conv in p_conv_list]
		h1_pool_list = [F.max_pool1d(h1_conv, kernel_size=h1_conv.shape[2])
			for h1_conv in h1_conv_list]
		h2_pool_list = [F.max_pool1d(h2_conv, kernel_size=h2_conv.shape[2])
			for h2_conv in h2_conv_list]

		#concatenate to get a single feature vector
		p = torch.cat([p_pool.squeeze(dim=2) for p_pool in p_pool_list],
						dim=1)
		h1 = torch.cat([h1_pool.squeeze(dim=2) for h1_pool in h1_pool_list],
						dim=1)
		h2 = torch.cat([h2_pool.squeeze(dim=2) for h2_pool in h2_pool_list],
						dim=1)

		#concatenate (p - h1, p - h2, p , h1, h2)
		#concat = torch.cat([p, h1, h2, p-h1, p-h2],dim=-1)
		concat = torch.cat([p*h1, p-h1, p*h2, p-h2],dim=-1)
		logit = self.decoder(concat)
		logit = logit.view(-1)

		if y is None:
			return logit

		loss = self.loss_fn(logit, y.view(-1))
		return logit, loss


import os, sys, json, time, base64
from flask import Flask, jsonify, request
import requests
import utils, load_balance, global_vars
import logging
import redis
import cfg_tasks
from utils import send_file_to_jobe

#======================================================
# Flask app initializtion
#======================================================
app = Flask(__name__)

#======================================================
# Let gunicotn takeover logger
#======================================================
if __name__ != '__main__':
    gunicorn_logger = logging.getLogger('gunicorn.error')
    app.logger.handlers = gunicorn_logger.handlers
    app.logger.setLevel(gunicorn_logger.level)

#======================================================
# API call: get_languages
#
# Returns a json containing supporting languages
# Something like this:
# [["cpp", "7.4.0"], ["python3", "3.6.5"]]
#
# Returns:
#  200 on success
#  400 on on illegal request parameters
#   (but when will that ever happened?)
#======================================================
@app.route(global_vars.URI_prefix + '/languages', methods = ['GET'])
def get_languages():
    # Get data from Celery
    result = cfg_tasks.get_data.delay('sorted_lang').get()
    if result[0]:
        return jsonify(json.loads(result[1])), 200
    else:
        return jsonify([]), 200

#======================================================
# API call: put_file
#
# Like the name sugguests, it put files in the server
# via PUT request in parameter file_contents.
#
# It save the file to this proxy instead of the actual
# jobe server until submit_run is called.
#
# It won't decode the base64 content sent to this proxy
# since the file content is non of its business and we
# can save some cpu cycles by not doing it.
#
# Returns:
#  204 on success
#  400 on contents are not a valid base-64 encoding
#  403 if the server does not trust the client to
#   provide a unique file ID
#   - NOT impelemented atm.
#======================================================
@app.route(global_vars.URI_prefix + '/files/<file_uid>', methods = ['PUT'])
def put_file(file_uid):
    # Code for decode base64
    #f.write(base64.b64decode(data['file_contents']))

    # Check if the file exist or not
    if os.path.exists(global_vars.PATH_PREFIX_file_cache + file_uid):
        # To save cpu cycle, we just ignore it if the requested file exists.
        return '', 204
    else:
        try:
            data = request.get_data()
            with open(global_vars.PATH_PREFIX_file_cache + file_uid, 'wb') as f:
                f.write(data)
            return '', 204
        except:
            app.logger.error('[put_file] Something went wrong writing file "%s"', file_uid)
            return '', 500

#======================================================
# API call: check_file
#
# Like the name sugguests, it checks cached files in
# the server via HEAD request.
#
# It checks if the file is present in this proxy
# instead of the actual jobe server.
#
# Returns:
#  204 on success
#  400 on illegal parameter
#  404 on file not found
#======================================================
@app.route(global_vars.URI_prefix + '/files/<file_uid>', methods = ['HEAD'])
def check_file(file_uid):
    # Check if the file exist or not
    if os.path.exists(global_vars.PATH_PREFIX_file_cache + file_uid):
        return '', 204
    else:
        return '', 404

#======================================================
# API call: submit_run
#
# The elephant in the room. It check if the file needed
# is in this proxy, then decide which jobe server to
# sned to and retrive and return result back to client.
#
# Returns:
#  200 when the runs is sucessful and with result
#  202 when the run is pending on jobe
#   - which is impossible because jobe server only runs
#   in immediate mode right now
#  400 on illegal parameter
#  404 when file needed is not found on the proxy
#======================================================
@app.route(global_vars.URI_prefix + '/runs', methods = ['POST'])
def submit_runs():
    request_data = request.get_json()
    # TODO: Check parameters
    #if 'run_spec' in request_data

    # Check if the file(s) exist on the proxy
    file_list = request_data['run_spec']['file_list']
    for file_pair in file_list:
        if not os.path.exists(global_vars.PATH_PREFIX_file_cache + file_pair[0]):
            return '', 404

    # Read in working_server from Celery
    working_server = cfg_tasks.get_data.delay('working_server').get()
    if working_server[0]:
        working_server = json.loads(working_server[1])
    else:
        app.logger.error('[submit_run] Error loading working_server')
        return jsonify([]), 500 # TODO: Compose the right respose so coderunner can display error msg. (It can be done, right?)

    # Delete every server that doesn't support the language(lang_id)
    # Pop the list of servers to be excluded
    # Why do I write it like this? It's unreadable.
    for server in [server for server in working_server if request_data['run_spec']['language_id'] not in working_server[server]]:
        working_server.pop(server, None)

    while True: # Auto resend loop
        # Choose a jobe URL from working_sever
        jobe_url = load_balance.lb_random(working_server, request_data['run_spec']['language_id']) # Use random for now
        if jobe_url == 'Nope':
            app.logger.error("[submit_run] Don't use __weight as language id!!")
            return '', 500 # TODO: Compose the right respose so coderunner can display error msg. (It can be done, right?)
        app.logger.info('[submit_run] Selected Jobe server: %s', jobe_url)

        # Send submit run to the server.
        # TODO: Log
        return_data = None
        return_code = None
        dead = False
        # First run:
        run_result = utils.send_run_to_jobe(jobe_url, request_data)
        if run_result[0]: # First run success
            return_data = run_result[1]
            return_code = 200
        elif run_result[2] == 404: # First run file not found
            send_file_result = utils.send_file_to_jobe(jobe_url, file_list)
            if send_file_result[0]: # Send file(s) success
                app.logger.info('[submit_run] File(s) sent to %s successfully', jobe_url)
                # Second run:
                run_result = utils.send_run_to_jobe(jobe_url, request_data)
                if run_result[0]: # Second run success
                    return_data = run_result[1]
                    return_code = 200
                else: # Second run fail
                    app.logger.warning('[submit_runs] Jobe: %s fail running submit, reason: %i, %s', jobe_url, run_result[2], run_result[3])
                    dead = True
            else: # Send file(s) fail
                app.logger.warning('[submit_runs] Fail sending file to Jobe: %s, reason: %i, %s', jobe_url, send_file_result[1], send_file_result[2])
                dead = True
        else: # First run fail
            app.logger.warning('[submit_runs] Jobe: %s fail running submit, reason: %i, %s', jobe_url, run_result[2], run_result[3])
            dead = True

        if dead:
            cfg_tasks.sus_jobe.delay(jobe_url)
            working_server.pop(jobe_url) # Exclude the jobe from local var
            if working_server == {}: # No more working server to choose
                app.logger.error("[submit_run] Running out of working server!!")
                return_data = ''
                return_code = 500
                break
        else:
            break

    return jsonify(return_data), return_code # TODO: Compose the right respose so coderunner can display error msg.

#======================================================
# API call: post_file
#
# Like the name sugguests, it put files in the server
# via POST request in parameter file_contents.
#
# It save the file to this proxy instead of the actual
# jobe server until submit_run is called.
#
# Since isn't implemented by jobe yet, it only return
# 403 in the current state.
#
# Returns:
# =========ONLY RESPONSES 403 IN CURRENT STATE=========
#  200 on success
#  400 on contents are not a valid base-64 encoding
#======================================================
@app.route(global_vars.URI_prefix + '/files', methods = ['POST'])
def post_file():
    return '', 403

#======================================================
# API call: get_run_status
#
# Like the name sugguests, it gets status of a
# pending run.
#
# It's not being implemented by jobe right now.
# Only Returns 404 in current state.
#
# Returns:
# =========ONLY RESPONSES 404 IN CURRENT STATE=========
#======================================================
@app.route(global_vars.URI_prefix + '/runresults/id', methods = ['GET'])
def get_run_status():
    return '', 404

#======================================================
# Not in the API call: force_update
#
# It force update working_server.json and
# sorted_lang.json.
#
# It does the same thing when genrating these two but
# without the file pre-checks.
#
# Returns:
#
#======================================================
@app.route(global_vars.URI_prefix + '/force_update', methods = ['GET'])
def force_update():
    if cfg_tasks.update_jobe_list.delay().get():
        update_result = cfg_tasks.update_working_server.delay().get()
        if update_result:
            data_result = cfg_tasks.get_data.delay('sorted_lang').get()
            if data_result[0]:
                return jsonify(json.loads(cfg_tasks.get_data.delay('sorted_lang').get()[1])), 200
    return jsonify([]), 200

if __name__ == '__main__':
    app.run()


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

#%load_ext google.colab.data_table

import random
from datetime import datetime, timedelta

first_names = [
    "Maria Carmen",
    "Maria",
    "Carmen",
    "Josefa",
    "Isabel",
    "Ana Maria",
    "Maria Dolores",
    "Maria Pilar",
    "Maria Teresa",
    "Ana",
    "Laura",
    "Francisca",
    "Antonia",
    "Dolores",
    "Maria Angeles",
    "Cristina",
    "Marta",
    "Maria Jose",
    "Maria Isabel",
    "Pilar",
    "Maria Luisa",
    "Lucia",
    "Concepcion",
    "Elena",
    "Mercedes",
    "Manuela",
    "Rosa Maria",
    "Raquel",
    "Sara",
    "Maria Jesus",
    "Paula",
    "Juana",
    "Teresa",
    "Rosario",
    "Encarnacion",
    "Beatriz",
    "Rosa",
    "Nuria",
    "Silvia",
    "Montserrat",
    "Julia",
    "Patricia",
    "Irene",
    "Monica",
    "Andrea",
    "Rocio",
    "Angela",
    "Maria Mar",
    "Margarita",
    "Sonia",
    "Sandra",
    "Susana",
    "Alicia",
    "Yolanda",
    "Alba",
    "Maria Josefa",
    "Marina",
    "Natalia",
    "Maria Rosario",
    "Inmaculada",
    "Angeles",
    "Esther",
    "Maria Mercedes",
    "Ana Isabel",
    "Eva",
    "Veronica",
    "Amparo",
    "Noelia",
    "Maria Rosa",
    "Maria Victoria",
    "Maria Concepcion",
    "Carolina",
    "Claudia",
    "Eva Maria",
    "Catalina",
    "Consuelo",
    "Victoria",
    "Lorena",
    "Ana Belen",
    "Maria Antonia",
    "Maria Elena",
    "Miriam",
    "Emilia",
    "Nerea",
    "Luisa",
    "Ines",
    "Maria Nieves",
    "Gloria",
    "Lidia",
    "Carla",
    "Aurora",
    "Esperanza",
    "Josefina",
    "Sofia",
    "Milagros",
    "Olga",
    "Celia",
    "Maria Soledad",
    "Purificacion",
]

store_names = [
    "Situwala",
    "Yakkha",
    "Gnawale",
    "Buhyo",
    "Kutal",
    "Bindukar",
    "Upeti",
    "Mana",
    "Badhyo",
    "Barme",
    "Dhnaju",
    "Kami",
    "Baidhaya",
    "Ma",
    "Ghimere",
    "Sangami",
    "Ghotane",
    "Kewat",
    "Singtan",
    "Chitrakr",
    "Khwaonjoo",
    "Manjhi",
]

city_names = [
    "Távora",
    "Cataguases",
    "Médici",
    "Bressane",
    "Onça",
    "Brígida",
    "Parati",
    "Mônica",
    "Pantano",
    "Anastácio",
    "Ostras",
    "Jequitibá",
]

def get_datetime_string(min_year=1980, max_year=datetime.now().year):
    # generate a datetime in format yyyy-mm-dd hh:mm:ss.000000
    start = datetime(min_year, 1, 1, 0, 0, 0)
    years = max_year - min_year + 1
    end = start + timedelta(days=365 * years)
    return start + (end - start) * random.random()

get_total_amount = lambda: round(random.random() * 500, 2)

# Distributed evenly, not realistic.
get_number_of_products = lambda: random.randint(0, 200)

get_cashier = lambda: random.choice(first_names)

get_store_name = lambda: random.choice(store_names)

get_city = lambda: random.choice(city_names)

get_discount_card = lambda: random.random() > 0.5

def get_random_data(fn, n):
    return [fn() for _ in range(n)]

num_rows = 200

data = {
    "datetime": get_random_data(get_datetime_string, num_rows),
    "total_amount": get_random_data(get_total_amount, num_rows),
    "number_of_products": get_random_data(get_number_of_products, num_rows),
    "cashier": get_random_data(get_cashier, num_rows),
    "store": get_random_data(get_store_name, num_rows),
    "city": get_random_data(get_city, num_rows),
    "discount_card_used": get_random_data(get_discount_card, num_rows),
}

df = pd.DataFrame(data=data)

## Dropping columns
print(df)

# Read current columns
print(df.columns)

# Drop single column
print(df.drop(columns='city'))

# Drop multiple columns
print(df.drop(columns=['city','number_of_products','discount_card_used']))

# Immutable by default, use inplace=True to change the original
#df.drop(columns='city', inplace=True)
#print(df)

## Renaming columns

# Simplest method: assigning a list
# Literal values
#print(df.columns)
#new_column_names = ["dt", "total", "num_products", "cashier", "store", "place", "card"]
#df.columns = new_column_names
#print(df)

# We can also generate a list based on df.columns
#print(df.columns)
#new_column_names = [col.split("_")[0] for col in df.columns]
#df.columns = new_column_names
#print(df.columns)

# Or use .str. methods
#print(df.columns)
#new_column_names = df.columns.str.capitalize()
#df.columns = new_column_names
#print(df.columns)
#print(df)

# Or use df.rename to selectively rename columns
#df.rename(columns={"number_of_products": "num_products", "discount_card_used": "card_used"})

## Changing the order of the columns

# We can use loc to (a) select all rows and (b) select all columns but in a different order
df2 = df.loc[:,['discount_card_used','store', 'cashier', 'datetime',  'total_amount', 'number_of_products', 'city']]
print(df2)

# But because we can get the list of columns with df.columns we can also slice and combine that
cols = df.columns.to_list()
print(cols)
new_cols = cols[-3:] + cols[:-3]
print(df.loc[:,new_cols])


__all__ = (
    "tabengine",
)

def _str_percent_decode(str):
    return str.replace("%20", " ").replace("%25", "%")

import os
import sys
#from ibus import Property
import keysyms
import modifier
import re

from socket import *

class KeyEvent:
    all_mods = [
        '', #no modifier
        'A', # alt
        'AC',# alt ctrl
        'ACS', #alt ctrl shift
        'AS',
        'C',
        'CS',
        'S'
        ]
    def __init__(self, keyval, chars, state):
        self.code = keyval
        self.mask = state
        self.name = chars

        if self.name and not (len(self.name) == 1 and ord(self.name) < ord(' ')):
            if self.name == ' ':
                self.name = 'space'
            else:
                self.mask &= ~modifier.SHIFT_MASK
        else:
            self.name = keysyms.keycode_to_name(self.code).lower()

        if self.name in ("control_l",
                         "control_r",
                         "alt_l",
                         "alt_r",
                         "shift_l",
                         "shift_r",
                         ):
            self.name = ""
            return

        mods = ''
        if self.mask & modifier.ALT_MASK:
            mods += 'A'
        if self.mask & modifier.CONTROL_MASK:
            mods += 'C'
        if self.mask & modifier.SHIFT_MASK:
            mods += 'S'

        if mods != '':
            self.name = mods + ' ' + self.name
    def __str__(self):
        return self.name

class tabengine ():
    '''The IM Engine for Tables'''

    _page_size = 10

    def __init__ (self):
        self.sock = socket(AF_INET, SOCK_STREAM)
        self.sock.connect(("localhost", 31415))
        self.sock = self.sock.makefile("rwb", 0)

        self.clear_data()
        # self._lookup_table = ibus.LookupTable (tabengine._page_size)

        self._name = 'sdim'
        self._config_section = "engine/%s" % self._name

        # config module
        self._on = True

    def clear_data(self):
        self._preedit_str = ''
        self._cands = []
        self._aux_str = ''
        self._commit_str = ''
        self._cands_str = ''
        self._cand_idx = '0'
        self._active = ''

    def reset (self):
        self._update_ui ()
        pass

    def _update_preedit (self):
        '''Update Preedit String in UI'''
        _str = self._preedit_str
        sdim_ui.update_preedit_text(_str)

    def _update_aux (self):
        '''Update Aux String in UI'''
        _aux = self._aux_str
        if _aux:
            sdim_ui.update_auxiliary_text(_aux)
        else:
            sdim_ui.hide_auxiliary_text()

    def _update_lookup_table (self):
        '''Update Lookup Sdim in UI'''
        if self._cands_str == '':
            sdim_ui.hide_lookup_table()
            return

        _cands = self._cands_str.split()
        _cands = [_str_percent_decode(str) for str in _cands]

        sdim_ui.clear_lookup_table()

        for cand in _cands:
            sdim_ui.append_candidate(cand)
            pass

        index = int(self._cand_idx) % 10
        sdim_ui.set_cursor_pos_in_current_page(index)
        sdim_ui.update_lookup_table()

    def _update_ui (self):
        '''Update User Interface'''
        self._update_lookup_table ()
        self._update_preedit ()
        self._update_aux ()
        self.commit_string()

    def commit_string (self):
        if self._commit_str == '':
            return
        commit = self._commit_str
        self._commit_str = ''
        sdim_ui.commit_text(commit)

    def process_key_event(self, keyval, chars, state):
        '''Process Key Events
        Key Events include Key Press and Key Release,
        modifier means Key Pressed
        '''
        key = KeyEvent(keyval, chars, state)
        # ignore NumLock mask

        result = self._process_key_event (key)
        return result

    def _process_key_event (self, key):
        '''Internal method to process key event'''
        key = str(key)
        if key == '':
            return False
        if self._preedit_str == '' and len(key) != 1:
            return False
        self._really_process_key(key)
        self._update_ui()
        return True

    def _really_process_key (self, key):
        self.sock.write("keyed " + key + "\n")
        self.clear_data()
        while True:
            line = self.sock.readline()
            if not line:
                break
            line = line[:-1]

            if line.find('commit: ') == 0:
                self._commit_str = line[len('commit: '):]

            elif line.find('hint: ') == 0:
                self._aux_str = line[len('hint: '):]

            elif line.find('comp: ') == 0:
                self._preedit_str = line[len('comp: '):]

            elif line.find('cands: ') == 0:
                self._cands_str = line[len('cands: '):]

            elif line.find('cand_index: ') == 0:
                self._cand_idx = line[len('cand_index: '):]

            elif line.find('active: ') == 0:
                self._active = line[len('active: '):]

            elif line == "end:":
                break

            else:
                self._aux_str = line

    def focus_in (self):
        if self._on:
            self._update_ui ()

    def focus_out (self):
        pass

    def enable (self):
        self._on = True
        self.focus_in()

    def disable (self):
        self.reset()
        self._on = False


import os
import numpy as np
import matplotlib.pyplot as plt
from scipy import io
from ukf_functions import *
from panorama import *
from transforms3d import taitbryan

def main():
    ''' modify this part accordingly '''
    # flags
    UKF = True # use UKF or just gyro data for estimate
    Panorama = True # generate panorama or not
    Estimate = True # the panorama is based on estimate or ground truth, must be set as True if no vicon data provided
    # dataset idx
    idx = 11
    # mat file location+prefix
    imu_prefix = "imu/imuRaw"
    vicon_prefix = "vicon/viconRot"
    cam_prefix = "cam/cam"

    # load data
    imu_ts, imu_vals, vicon_ts, vicon_euler = load_data(idx, imu_prefix, vicon_prefix)

    # Unscented Kalman Filter
    # init
    qk = np.array([1,0,0,0]) # last mean in quaternion
    Pk = np.identity(3) * 0.1 # last cov in vector
    Q = np.identity(3) * 2 # process noise cov
    R = np.identity(3) * 2 # measurement noise cov

    time = imu_ts.shape[0]
    ukf_euler = np.zeros((time, 3))  # represent orientation in euler angles

    for t in range(time):
        # extract sensor data
        acc = imu_vals[t,:3]
        gyro = imu_vals[t,3:]

        # Prediction
        X = compute_sigma_pts(qk, Pk, Q)

        if t == time-1: # last iter
            dt = np.mean(imu_ts[-10:] - imu_ts[-11:-1])
        else:
            dt = imu_ts[t+1] - imu_ts[t]
        Y = process_model(X, gyro, dt)

        q_pred, P_pred, W = prediction(Y, qk)

        if UKF:
            # Measurement
            vk, Pvv, Pxz = measurement_model(Y, acc, W, R)

            # Update
            K = np.dot(Pxz,np.linalg.inv(Pvv)) # Kalman gain
            qk, Pk = update(q_pred, P_pred, vk, Pvv, K)

        else:
            # estimate just based on control input gyro
            qk, Pk = q_pred, P_pred

        # save for visualization
        ukf_euler[t, :] = taitbryan.quat2euler(qk)

    np.save('result/'+'ukf'+str(idx),ukf_euler)

    # orientation plot UKF + only gyro
    orientation_plot(idx, imu_ts, ukf_euler, vicon_ts, vicon_euler)

    # panoramic by image stitching
    if Panorama:
        if Estimate:
            panorama(idx, cam_prefix, imu_ts, ukf_euler)
        else:
            panorama(idx, cam_prefix, vicon_ts, vicon_euler)
    return 0

def load_data(idx, imu_prefix, vicon_prefix):
    # load data from imu/vicon, calibrate and save

    # load imu
    imu = io.loadmat(imu_prefix+str(idx)+".mat")
    imu_vals = np.array(imu['vals'])
    imu_ts = np.array(imu['ts']).T

    # scale and bias based on IMU reference
    acc_x = -imu_vals[0,:]
    acc_y = -imu_vals[1,:]
    acc_z = imu_vals[2,:]
    acc = np.array([acc_x, acc_y, acc_z]).T

    Vref = 3300
    acc_sensitivity = 330
    acc_scale_factor = Vref/1023/acc_sensitivity
    acc_bias = np.mean(acc[:10], axis = 0) - (np.array([0,0,1])/acc_scale_factor)
    acc = (acc-acc_bias)*acc_scale_factor

    gyro_x = imu_vals[4,:]
    gyro_y = imu_vals[5,:]
    gyro_z = imu_vals[3,:]
    gyro = np.array([gyro_x, gyro_y, gyro_z]).T

    gyro_sensitivity = 3.33
    gyro_scale_factor = Vref/1023/gyro_sensitivity
    gyro_bias = np.mean(gyro[:10], axis = 0)
    gyro = (gyro-gyro_bias)*gyro_scale_factor*(np.pi/180)

    imu_vals = np.hstack((acc,gyro))

    # vicon not provided
    if not os.path.exists(vicon_prefix+str(idx)+".mat"):
        return imu_ts, imu_vals, None, None
    # load vicon
    vicon = io.loadmat(vicon_prefix+str(idx)+".mat")
    vicon_vals = np.array(vicon['rots'])
    vicon_ts = np.array(vicon['ts']).T

    n = np.shape(vicon_vals)[2]
    vicon_euler = np.zeros((n,3))
    for i in range(n):
        R = vicon_vals[:,:,i]
        vicon_euler[i] = taitbryan.mat2euler(R)

    return imu_ts, imu_vals, vicon_ts, vicon_euler

def orientation_plot(idx, imu_ts, ukf_euler, vicon_ts, vicon_euler):
    # no vicon data
    if vicon_ts is None and vicon_euler is None:
        plt.figure(1)
        plt.subplot(3, 1, 1)
        ukf, = plt.plot(imu_ts, ukf_euler[:, 0], 'r', label='UKF Estimate')
        plt.title('Z-Yaw')
        plt.ylabel('Angle [rad]')
        plt.legend(handles=[ukf])

        plt.subplot(3, 1, 2)
        ukf, = plt.plot(imu_ts, ukf_euler[:, 1], 'r', label='UKF Estimate')
        plt.title('Y-Pitch')
        plt.ylabel('Angle [rad]')
        plt.legend(handles=[ukf])

        plt.subplot(3, 1, 3)
        ukf, = plt.plot(imu_ts, ukf_euler[:, 2], 'r', label='UKF Estimate')
        plt.title('X-Roll')
        plt.ylabel('Angle [rad]')
        plt.legend(handles=[ukf])

        plt.savefig('result/orientation' + str(idx) + '.png')

    else:
        plt.figure(1)
        plt.subplot(3, 1, 1)
        true, = plt.plot(vicon_ts,  vicon_euler[:, 0], 'g', label='Ground Truth')
        ukf, = plt.plot(imu_ts, ukf_euler[:, 0], 'r', label='UKF Estimate')
        plt.title('Z-Yaw')
        plt.ylabel('Angle [rad]')
        plt.legend(handles=[true, ukf])

        plt.subplot(3, 1, 2)
        true, = plt.plot(vicon_ts, vicon_euler[:, 1], 'g', label='Ground Truth')
        ukf, = plt.plot(imu_ts, ukf_euler[:, 1], 'r', label='UKF Estimate')
        plt.title('Y-Pitch')
        plt.ylabel('Angle [rad]')
        plt.legend(handles=[true, ukf])

        plt.subplot(3, 1, 3)
        true, = plt.plot(vicon_ts, vicon_euler[:, 2], 'g', label='Ground Truth')
        ukf, = plt.plot(imu_ts, ukf_euler[:, 2], 'r', label='UKF Estimate')
        plt.title('X-Roll')
        plt.ylabel('Angle [rad]')
        plt.legend(handles=[true, ukf])

        plt.savefig('result/orientation'+str(idx)+'.png')

if __name__ == '__main__':
    main()


import sys, os, datetime

import numpy as np
import torch
from torch.utils.tensorboard import SummaryWriter

# add the path to the files from this project
sys.path.append(os.path.dirname(__file__))

import world
import utils
import losses
import mesh_utils
from utils import *
import data_utils
import train_utils
from point_tri_net import PointTriNet_Mesher

### Experiment options

args = world.ArgsObject()
world.args = args

# System parameters
args.experiment_name = "fit_model"
args.run_name = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S")
experiments_dir = os.path.join(os.path.dirname(__file__), "..")
args.run_dir = os.path.join(os.path.dirname(__file__), "../training_runs", args.run_name)
args.log_dir = os.path.join(args.run_dir, "logs")
args.dataset_dir = os.path.join(experiments_dir, "..", "data")
args.debug_checks = False
args.disable_cuda = False

# set some defaults
set_args_defaults(args)
world.device = args.device
world.dtype = args.dtype

# Experiment parameters
args.load_weights = None

args.w_watertight = 1.0
args.w_dist_surf_tri = 1.0
args.w_dist_tri_surf = 1.0
args.w_overlap_kernel = 0.01

# Algorithm parameters
args.n_mesh_rounds = 5

# Training parameters
args.epochs = 3                         # Number of epochs to train for
args.lr = 1e-4                          # "Learning rate"
args.lr_decay = .5                      # How much to decrease the learning rate by, applied every decay_step samples (lr = lr * lr_decay)
args.decay_step = 10e99                 # Decay lr after processing this many samples (_not_ batches, samples)
args.batch_size = 1                     # Batch size (note accum parameter below)
args.batch_accum = 8                    # Accumulate over this many batches before stepping gradients
args.eval_every = 2048                  # Evaluate on the validation set after this many samples
args.eval_size = 512                    # Use this much of the validation set to evaluate (if less than full validation set size)

print("Beginning run " + str(args.run_name))

# Ensure the run directory exists
ensure_dir_exists(args.run_dir)

## Initialize the tensorboard writer
world.tb_writer = SummaryWriter(log_dir=args.log_dir)

# Load the dataset
T = []
train_dataset = data_utils.PointSurfaceDataset(dir_with_meshes="data/train/", transforms=T)
val_dataset = data_utils.PointSurfaceDataset(dir_with_meshes="data/val/", transforms=T)

train_loader = torch.utils.data.DataLoader(
    train_dataset,
    batch_size=world.args.batch_size,
    shuffle=True,
    num_workers=4,
    pin_memory=True,
    drop_last=True,
    # collate_fn=train_dataset.collate_fn
)
val_loader = torch.utils.data.DataLoader(
    val_dataset,
    batch_size=world.args.batch_size,
    shuffle=True,
    num_workers=4,
    pin_memory=True,
    drop_last=True,
    # collate_fn=val_dataset.collate_fn
)

# Construct a model
model = PointTriNet_Mesher()

if args.load_weights:
    weights_path = os.path.join(world.args.load_weights,"model_state_dict.pth")
    model.load_state_dict(torch.load(weights_path))

# Calls the model on a batch
def call_model_fn(model, batch, trainer=None):
    points = batch['vert_pos'].to(world.device)
    candidate_triangles, candidate_probs, proposal_triangles, proposal_probs = model.predict_mesh(points, n_rounds=args.n_mesh_rounds, sample_last=True)
    return {
            "candidates" : candidate_triangles, "probs": candidate_probs,
            "proposals" : proposal_triangles, "proposal_probs" : proposal_probs
           }

# Construct a loss
def loss_fn(batch, model_outputs, viz_extra=False, trainer=None):
    B = batch['vert_pos'].shape[0]

    # Data from the sample
    vert_pos_batch = batch['vert_pos'].to(world.device)
    surf_pos_batch = batch['surf_pos'].to(world.device)
    surf_normal_batch = batch['surf_normal'].to(world.device)

    # Outputs from model
    all_candidates = model_outputs['candidates']
    all_candidate_probs = model_outputs['probs']
    all_proposals = model_outputs['proposals']
    all_proposal_probs = model_outputs['proposal_probs']

    # Accumulate loss
    need_grad = all_candidate_probs.requires_grad
    total_loss = torch.tensor(0.0, dtype=vert_pos_batch.dtype, device=vert_pos_batch.device, requires_grad=need_grad)

    # Evaluate loss one batch entry at a time
    for b in range(B):

        vert_pos = vert_pos_batch[b, :]
        candidates = all_candidates[b,:,:]
        candidate_probs = all_candidate_probs[b, :]
        proposals = all_proposals[b,:,:]
        proposal_probs = all_proposal_probs[b, :]

        surf_pos = surf_pos_batch[b, :]
        surf_normal = surf_normal_batch[b, :]

        # Add all the terms
        loss_terms = losses.build_loss(args, vert_pos, candidates, candidate_probs, surf_pos=surf_pos, surf_normal=surf_normal, n_sample=1000)

        loss_terms["proposal_match"] = losses.match_predictions(
            candidates, candidate_probs.detach(),
            proposals, proposal_probs)

        this_loss = torch.tensor(0.0, dtype=vert_pos_batch.dtype, device=vert_pos_batch.device, requires_grad=need_grad)
        for t in loss_terms:
            this_loss = this_loss + loss_terms[t]

        # Log some stats
        if trainer is not None:
            if trainer.training:
                prefix = "train_"
                it = trainer.curr_iter + b
            else:
                prefix = "val_"
                it = trainer.eval_iter + b

            # log less
            if it % 10 == 0:

                for t in loss_terms:
                    world.tb_writer.add_scalar(prefix + t, loss_terms[t].item(), it)

                world.tb_writer.add_scalar(prefix + "sample loss", this_loss.item(), it)

                if it % 1000 == 0:
                    world.tb_writer.add_histogram(prefix + 'triangle_probs', candidate_probs.detach(), it)
                    world.tb_writer.add_histogram(prefix + 'triangle_proposal_probs', proposal_probs.detach(), it)

                world.tb_writer.add_scalar(prefix + "prob mean", torch.mean(candidate_probs).item(), it)
                world.tb_writer.add_scalar(prefix + "prob stddev", torch.std(candidate_probs).item(), it)

                if not trainer.training:
                    trainer.add_eval_stat_entry("prob mean", torch.mean(candidate_probs).item())
                    trainer.add_eval_stat_entry("prob std", torch.std(candidate_probs).item())
                    for t in loss_terms:
                        trainer.add_eval_stat_entry(t, loss_terms[t].item())

        total_loss = total_loss + this_loss

    return total_loss / B

# Train
with torch.autograd.set_detect_anomaly(world.debug_checks):

    trainer = train_utils.MyTrainer(
        args=world.args,
        model=model,
        call_model_fn=call_model_fn,
        loss_fn=loss_fn,
        train_loader=train_loader,
        val_loader=val_loader,
    )

    trainer.train()


from django.shortcuts import render
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound
from django.core.exceptions import ObjectDoesNotExist
import os
import json
from .models import Course
from django.db.models import Q

# Create your views here.
TRUE_SET = {"true", "yes", "y", "t", "1"}

def lookup(request, subject_id=None):
    """
    Provides a full JSON description of the course specified by the given subject
    ID.
    """
    if subject_id is None:
        return HttpResponseBadRequest("Provide a subject ID to look up a course.")
    try:
        c = Course.public_courses().get(subject_id=subject_id)
        return HttpResponse(json.dumps(c.to_json_object()), content_type="application/json")
    except ObjectDoesNotExist:
        return HttpResponseNotFound("No subject found with the given ID")

def department(request, dept=None):
    """
    Provides a list of JSON descriptions of the courses whose subject IDs begin
    with the given department code. If a boolean GET parameter for "full" is
    specified, it will indicate whether the full JSON description is included.
    """
    if dept is None:
        return HttpResponseBadRequest("Provide a department to look up its courses.")
    if "full" in request.GET:
        full = request.GET["full"].lower() in TRUE_SET
    else:
        full = False
    courses = Course.public_courses().filter(subject_id__startswith=dept + ".").order_by("subject_id")
    return HttpResponse(json.dumps([c.to_json_object(full=full) for c in courses]), content_type="application/json")

def list_all(request):
    """
    Provides a list of JSON descriptions of all courses in the database. If a
    boolean GET parameter for "full" is specified, it will indicate whether the
    full JSON description is included.
    """
    if "full" in request.GET:
        full = request.GET["full"].lower() in TRUE_SET
    else:
        full = False
    return HttpResponse(json.dumps([c.to_json_object(full=full) for c in Course.public_courses().order_by("subject_id")]), content_type="application/json")

def offered_filter(offered_value):
    """Constructs a Q filter based on the given offered value, or throws a
    ValueError if the value is inappropriate."""
    offered_value = offered_value.lower()
    if offered_value == "off":
        return None
    elif offered_value == "fall":
        return Q(offered_fall=True)
    elif offered_value == "spring":
        return Q(offered_spring=True)
    elif offered_value == "iap":
        return Q(offered_IAP=True)
    elif offered_value == "summer":
        return Q(offered_summer=True)
    else:
        raise ValueError

def level_filter(level_value):
    """Constructs a Q filter based on the given level value, or throws a
    ValueError if the value is inappropriate."""
    level_value = level_value.lower()
    if level_value == "off":
        return None
    elif level_value == "undergrad":
        return Q(level="U")
    elif level_value == "grad":
        return Q(level="G")
    else:
        raise ValueError

def ci_filter(ci_value):
    """Constructs a Q filter based on the given CI value, or throws a
    ValueError if the value is inappropriate."""
    ci_value = ci_value.lower()
    if ci_value == "off":
        return None
    elif ci_value == "cih":
        return Q(communication_requirement="CI-H")
    elif ci_value == "cihw":
        return Q(communication_requirement="CI-HW")
    elif ci_value == "not-ci":
        return Q(communication_requirement__isnull=True) | Q(communication_requirement="")
    else:
        raise ValueError

def hass_filter(hass_value):
    """Constructs a Q filter based on the given HASS value, or throws a
    ValueError if the value is inappropriate."""
    hass_value = hass_value.lower()
    if hass_value == "off":
        return None
    elif hass_value == "any":
        return ~Q(hass_attribute="")
    elif hass_value in {"a", "s", "h"}:
        return Q(hass_attribute__icontains="HASS-" + hass_value.upper())
    else:
        raise ValueError

def gir_filter(gir_value):
    """Constructs a Q filter based on the given GIR value, or throws a
    ValueError if the value is inappropriate."""
    gir_value = gir_value.lower()
    if gir_value == "off":
        return None
    elif gir_value == "any":
        return ~Q(gir_attribute="")
    elif gir_value in {"lab", "rest"}:
        return Q(gir_attribute=gir_value.upper())
    else:
        raise ValueError

def construct_search_query(search_term, type):
    """Constructs a Q filter based on the given search term, using the given
    search type (contains, matches, etc.)."""
    if type == "contains":
        return Q(subject_id__icontains=search_term) | Q(title__icontains=search_term)
    elif type == "matches":
        return Q(subject_id__iexact=search_term) | Q(title__iexact=search_term)
    elif type == "starts":
        return Q(subject_id__istartswith=search_term) | Q(title__istartswith=search_term)
    elif type == "ends":
        return Q(subject_id__iendswith=search_term) | Q(title__iendswith=search_term)
    else:
        raise ValueError

def search(request, search_term=None):
    """
    Searches the catalog database for courses matching the given search term.
    The following case-insensitive GET parameter options are available:

    type: The match type to use with the search term. Possible values: "contains"
        (default), "matches", "starts", "ends"
    gir: Whether to filter by GIR. Possible values: "off" (default), "any", "lab",
        "rest"
    hass: Whether to filter by HASS fulfillment. Possible values: "off" (default),
        "any", "a", "s", "h"
    ci: Whether to filter by CI fulfillment. Possible values: "off" (default),
        "cih", "cihw", "not-ci"
    offered: Which semester the course is offered. Possible values: "off"
        (default), "fall", "spring", "IAP", "summer"
    level: The level of the course. Possible values: "off" (default), "undergrad",
        "grad"
    full: Boolean indicating whether to return the full course description.
        Possible values: "n" (default), "y"

    TODO: search fields, schedule conflicts
    """
    if search_term is None:
        return HttpResponseBadRequest("Must provide a search term.")

    try:
        # Construct query
        query = construct_search_query(search_term, request.GET.get("type", "contains"))
        if "offered" in request.GET:
            filter = offered_filter(request.GET["offered"])
            if filter is not None: query &= filter
        if "level" in request.GET:
            filter = level_filter(request.GET["level"])
            if filter is not None: query &= filter
        if "gir" in request.GET:
            filter = gir_filter(request.GET["gir"])
            if filter is not None: query &= filter
        if "hass" in request.GET:
            filter = hass_filter(request.GET["hass"])
            if filter is not None: query &= filter
        if "ci" in request.GET:
            filter = ci_filter(request.GET["ci"])
            if filter is not None: query &= filter
    except ValueError:
        return HttpResponseBadRequest("Invalid filter value")

    # Search by query
    results = Course.public_courses().filter(query)
    if "full" in request.GET:
        full = request.GET["full"].lower() in TRUE_SET
    else:
        full = False
    return HttpResponse(json.dumps([c.to_json_object(full=full) for c in results]), content_type="application/json")


import requests
from fastapi import APIRouter
news = APIRouter()
import requests
import os
from bson import json_util
from datetime import datetime
api_key = os.environ.get("API_key")

import uuid
from pydantic import BaseModel, Field
from pymongo import MongoClient
CONNECTION_STRING = 'mongodb://localhost:27017'
client = MongoClient(CONNECTION_STRING)
db = client['news_artikel']
collection = db['news']

class News(BaseModel):
  id: str = Field(default_factory=uuid.uuid4, alias="_id")
  description: str
  title:str
  url:str
  urlToImage:str
  author:str
  created_at:datetime = Field(default_factory=datetime.utcnow)

def create_news():
    url = ('https://newsapi.org/v2/everything?'
           'q=Apple&'
           'from=2023-06-04&'
           'sortBy=popularity&'
           f'apiKey={api_key}')

    response = requests.get(url)
    data = response.json()

    articl = []
    count=0
    for article in data["articles"]:
        description = article["description"]
        title = article["title"]
        urlToImage = article["urlToImage"]
        url = article["url"]
        author = article["author"]
        created_at = datetime.utcnow()
        articl.append({
            "description": description,
            "title": title,
            "urlToImage": urlToImage,
            "url": url,
            "author": author,
            "created_at": created_at
        })
        values =[[
            description,
            title,
            urlToImage,
            url,
            author,
            created_at,
        ]]
        result =[]
        collection.insert_many([{
            "description": value[0],
            "titel": value[1],
            "urlToImage": value[2],
            "url":value[3],
            "autor":value[4],
            "created_at":value[5]
        }for value in values])
        result.append({description,title,urlToImage,url,author, created_at})
        count+=1
        if count>=2:
           break

    return(result)

@news.get("/news")
def get_news():
    articles = create_news()
    result = collection.find({}, {"_id": 0}).sort("created_at", -1).limit(2)
    result_dicts = [doc for doc in result]
    return result_dicts


from django.db import models
from django.core.validators import RegexValidator
from django.contrib.auth.models import AbstractBaseUser, BaseUserManager

# Create your models here.

class MasterKelas(models.Model):
    id_kelas                = models.IntegerField(primary_key=True)
    category_kelas          = (
                                ('10', '10'),
                                ('11', '11'),
                                ('12', '12'),
                                ('13', '13'),
                            )
    kelas                   = models.CharField(max_length=100, choices=category_kelas)

    def __str__(self):
        return f'{self.id_kelas}. {self.kelas}'

class MasterJurusan(models.Model):
    id_jurusan              = models.IntegerField(primary_key=True)
    category_jurusan        = (
                                ('KGSP', 'KGSP'),
                                ('SIJA', 'SIJA'),
                                ('TEDK', 'TEDK'),
                                ('TFLM', 'TFLM'),
                                ('TMPO', 'TMPO'),
                                ('TTL', 'TTL'),
                            )
    jurusan                 = models.CharField(max_length=100, choices=category_jurusan)

    def __str__(self):
        return f'{self.id_jurusan}. {self.jurusan}'

class MasterSiswa(models.Model):
    nisn                    = models.CharField(primary_key=True, unique=True,  max_length=10, validators=[RegexValidator(r'^\d{1,10}$')])
    name                    = models.CharField(max_length=100)
    username                = models.CharField(max_length=16, unique=True)
    password                = models.CharField(max_length=32)
    id_kelas                = models.ForeignKey(MasterKelas, on_delete=models.CASCADE)
    id_jurusan              = models.ForeignKey(MasterJurusan, on_delete=models.CASCADE)

    def __str__(self):
        return f'{self.nisn}, {self.name}'

class Absensi(models.Model):
    id_absensi              = models.ForeignKey(MasterSiswa, on_delete=models.CASCADE)
    daily                   = models.DateField()
    status                  = models.BooleanField()
    checkin                 = models.DateTimeField(auto_now_add=True)
    checkout                = models.DateTimeField(null=True, blank=True, default=None)

    def __str__(self):
        return f'{self.id_absensi}'


from pyqode.qt import QtCore
from pyqode.qt.QtTest import QTest
from pyqode.core.api import TextHelper
from pyqode.core import modes
from test.helpers import editor_open

def get_mode(editor):
    return editor.modes.get(modes.SmartBackSpaceMode)

def test_enabled(editor):
    mode = get_mode(editor)
    assert mode.enabled
    mode.enabled = False
    mode.enabled = True

@editor_open(__file__)
def test_key_pressed(editor):
    QTest.qWait(1000)
    TextHelper(editor).goto_line(21, 4)
    # QTest.qWait(2000)
    assert editor.textCursor().positionInBlock() == 4
    QTest.keyPress(editor, QtCore.Qt.Key_Backspace)


from xarm.servo_controller import *
from xarm.cartesian import compute_ik, compute_fk
from xarm.inverse_kinematics import FREE_ANGLE

speed = 1

def set_speed(value: float) -> None:

    global speed

    if 0 < value <= 10:
        speed = value
    else:
        raise ValueError("speed must be greater than 0 and less or equal to 10")

def grip_open() -> None:
    move_servo(1, 200, int(1000 / speed))

def grip_close(value=None) -> None:

    if value is not None:

        if not 0 <= value <= 1000:
            raise ValueError("value must be between 0 and 1000")

        value = int((value - 0) * (650 - 200) / (1000 - 0) + 200)

        move_servo(1, value, int(1000 / speed))
    else:
        move_servo(1, 650, int(1000 / speed))

def precompute_moves() -> None:

    """Parse all application and if saved compute data is not present or different, pre-compute each moves (inverse kinematics)"""

    pass

def movej(joint: tuple, time: int) -> None:

    """Move each servomotors to joint position within time.
        joint: tuple(j1, j2, j3, j4, j5) | jx: 0-1000
        time: 0-65535 milliseconds
    """

    move_servos((2, 3, 4, 5, 6), joint[::-1], int(time / speed))

def movel(point: tuple, time: int, orientation=500, approach_angle=FREE_ANGLE, waypoints=1) -> None:

    """Move arm to point position within time.
            point: tuple(x, y, z)
            time: 0-65535 milliseconds
            orientation: the orientation of the hand
            approach_angle: calculates the angles considering a specific approach angle
            waypoints: number of points through which the arm will pass (Allows a more linear movement)
    """

    if waypoints > 1:
        current = get_position(cartesian=True)
        step_values = [abs(p - c) / waypoints for p, c in zip(point, current)]

        iks = list()
        for i in range(1, waypoints + 1):
            way = (current[0] + (i * step_values[0]), current[1] + (i * step_values[1]), current[2] + (i * step_values[2]))
            iks.append(compute_ik(way, orientation, approach_angle))

        for ik in iks:
            move_servos((6, 5, 4, 3, 2), ik, int(time / waypoints / speed))
    else:
        move_servos((6, 5, 4, 3, 2), compute_ik(point, orientation, approach_angle), int(time / speed))

def appro(point: tuple, offset: tuple) -> tuple:

    """Add offset to point.
            point: tuple(x, y, z)
            offset: tuple(x, y, z)
            return: tuple(x, y, z)
    """

    new_point = list()
    for p, o in zip(point, offset):
        new_point.append(p + o)

    return tuple(new_point)

def get_position(cartesian=False) -> tuple:

    """Return each servos position or return position in cartesian referential.
            return: tuple(j1, j2, j3, j4, j5) | jx: 0-1000
            or
            return: tuple(x, y, z)
    """

    position = get_servos_position((6, 5, 4, 3, 2))

    if not cartesian:
        return position
    else:
        return compute_fk(position)

def power_on() -> None:

    """Power on each servos motor"""

    sleep(0.1)

    position = get_position()
    movej(position, 100)

def power_off() -> None:

    """Power off each servos motor"""

    sleep(0.1)
    unload_servos((1, 2, 3, 4, 5, 6))


from typing import Any
from ..._group import project_add_group
from ....task.task import Task
from ....task.decorator import python_task
from ....task.cmd_task import CmdTask
from ....task.resource_maker import ResourceMaker
from ....runner import runner
from ....helper import util
from ....helper.codemod.add_import_module import add_import_module
from ....helper.codemod.append_code_to_function import append_code_to_function
from ....helper.file.text import read_text_file_async, write_text_file_async
from .._common.input import (
    project_dir_input, app_name_input, module_name_input, entity_name_input,
    plural_entity_name_input, main_column_name_input
)
from .._common.helper import validate_project_dir
from .add_navigation import create_add_navigation_task

import asyncio
import os

current_dir = os.path.dirname(__file__)
codemod_dir = os.path.join(current_dir, 'nodejs', 'codemod')

@python_task(
    name='validate',
    inputs=[
        project_dir_input, app_name_input, module_name_input, entity_name_input
    ],
)
async def validate(*args: Any, **kwargs: Any):
    project_dir = kwargs.get('project_dir')
    validate_project_dir(project_dir)
    app_name = kwargs.get('app_name')
    module_name = kwargs.get('module_name')
    entity_name = kwargs.get('entity_name')
    snake_module_name = util.to_snake_case(module_name)
    snake_entity_name = util.to_snake_case(entity_name)
    automation_dir = os.path.join(
        project_dir, '_automate', util.to_snake_case(app_name)
    )
    if not os.path.exists(automation_dir):
        raise Exception(
            f'Automation directory does not exist: {automation_dir}'
        )
    app_dir = os.path.join(
        project_dir, 'src', f'{util.to_kebab_case(app_name)}'
    )
    if not os.path.exists(app_dir):
        raise Exception(f'App directory does not exist: {app_dir}')
    module_path = os.path.join(app_dir, 'src', 'module', snake_module_name)
    if not os.path.exists(module_path):
        raise Exception(f'Module directory does not exist: {module_path}')
    entity_path = os.path.join(module_path, 'entity', snake_entity_name)
    if os.path.exists(entity_path):
        raise Exception(f'Entity directory already exists: {entity_path}')

copy_resource = ResourceMaker(
    name='copy-resource',
    inputs=[
        project_dir_input,
        app_name_input,
        module_name_input,
        entity_name_input,
        plural_entity_name_input,
        main_column_name_input,
    ],
    upstreams=[validate],
    replacements={
        'appName': '{{input.app_name}}',
        'moduleName': '{{input.module_name}}',
        'entityName': '{{input.entity_name}}',
        'pluralEntityName': '{{input.plural_entity_name}}',
        'columnName': '{{input.column_name}}',
    },
    template_path=os.path.join(current_dir, 'template'),
    destination_path='{{ input.project_dir }}',
    excludes=[
        '*/__pycache__',
    ]
)

@python_task(
    name='register-crud',
    inputs=[
        project_dir_input,
        app_name_input,
        module_name_input,
        entity_name_input,
        plural_entity_name_input,
        main_column_name_input,
    ],
    upstreams=[
        copy_resource,
    ]
)
async def register_crud(*args: Any, **kwargs: Any):
    task: Task = kwargs.get('_task')
    project_dir = kwargs.get('project_dir', '.')
    app_name = kwargs.get('app_name')
    module_name = kwargs.get('module_name')
    entity_name = kwargs.get('entity_name')
    kebab_app_name = util.to_kebab_case(app_name)
    snake_module_name = util.to_snake_case(module_name)
    snake_entity_name = util.to_snake_case(entity_name)
    await asyncio.gather(
        asyncio.create_task(register_api(
            task, project_dir, kebab_app_name, snake_module_name,
            snake_entity_name
        )),
        asyncio.create_task(register_rpc(
            task, project_dir, kebab_app_name, snake_module_name,
            snake_entity_name
        )),
        asyncio.create_task(register_permission(
            task, project_dir, kebab_app_name, snake_module_name,
            snake_entity_name
        )),
    )

prepare_codemod = CmdTask(
    name='prepare-codemod',
    cwd=codemod_dir,
    cmd=[
        'npm install --save-dev',
        'node_modules/.bin/tsc'
    ]
)

add_navigation = create_add_navigation_task(
    upstreams=[
        copy_resource,
        prepare_codemod,
    ]
)

@python_task(
    name='fastapp-crud',
    group=project_add_group,
    upstreams=[
        register_crud,
        add_navigation
    ],
    runner=runner
)
async def add_fastapp_crud(*args: Any, **kwargs: Any):
    task: Task = kwargs.get('_task')
    task.print_out('Success')

async def register_api(
    task: Task,
    project_dir: str,
    kebab_app_name: str,
    snake_module_name: str,
    snake_entity_name: str
):
    module_api_file_path = os.path.join(
        project_dir, 'src', kebab_app_name, 'src', 'module', snake_module_name,
        'api.py'
    )
    register_function_path = '.'.join([
        'module', snake_module_name, 'entity', snake_entity_name, 'api'
    ])
    register_function = f'register_{snake_entity_name}_api'
    task.print_out(f'Read code from: {module_api_file_path}')
    code = await read_text_file_async(module_api_file_path)
    task.print_out(
        f'Add import "register_api" as "{register_function}" ' +
        f'from "{register_function_path}" to the code'
    )
    code = add_import_module(
        code=code,
        module_path=register_function_path,
        resource='register_api',
        alias=register_function
    )
    task.print_out(f'Add "{register_function}" call to the code')
    code = append_code_to_function(
        code=code,
        function_name='register_api',
        new_code=f'{register_function}(logger, app, authorizer, rpc_caller, publisher)' # noqa
    )
    task.print_out(f'Write modified code to: {module_api_file_path}')
    await write_text_file_async(module_api_file_path, code)

async def register_rpc(
    task: Task,
    project_dir: str,
    kebab_app_name: str,
    snake_module_name: str,
    snake_entity_name: str
):
    module_rpc_file_path = os.path.join(
        project_dir, 'src', kebab_app_name, 'src', 'module', snake_module_name,
        'rpc.py'
    )
    register_function_path = '.'.join([
        'module', snake_module_name, 'entity', snake_entity_name, 'rpc'
    ])
    register_function = f'register_{snake_entity_name}_rpc'
    task.print_out(f'Read code from: {module_rpc_file_path}')
    code = await read_text_file_async(module_rpc_file_path)
    task.print_out(
        f'Add import "register_rpc" as "{register_function}" ' +
        f'from "{register_function_path}" to the code'
    )
    code = add_import_module(
        code=code,
        module_path=register_function_path,
        resource='register_rpc',
        alias=register_function
    )
    task.print_out(f'Add "{register_function}" call to the code')
    code = append_code_to_function(
        code=code,
        function_name='register_rpc',
        new_code=f'{register_function}(logger, rpc_server, rpc_caller, publisher)' # noqa
    )
    task.print_out(f'Write modified code to: {module_rpc_file_path}')
    await write_text_file_async(module_rpc_file_path, code)

async def register_permission(
    task: Task,
    project_dir: str,
    kebab_app_name: str,
    snake_module_name: str,
    snake_entity_name: str
):
    module_register_permission_file_path = os.path.join(
        project_dir, 'src', kebab_app_name, 'src', 'module', 'auth',
        'register_permission.py'
    )
    task.print_out(f'Read code from: {module_register_permission_file_path}')
    code = await read_text_file_async(module_register_permission_file_path)
    code = append_code_to_function(
        code=code,
        function_name='register_permission',
        new_code='\n'.join([
            'await ensure_entity_permission(',
            f"    module_name='{snake_module_name}', entity_name='{snake_entity_name}'", # noqa
            ')'
        ])
    )
    task.print_out(
        f'Add "ensure_entity_permission" call for {snake_entity_name} ' +
        'to the code'
    )
    task.print_out(
        f'Write modified code to: {module_register_permission_file_path}'
    )
    await write_text_file_async(module_register_permission_file_path, code)


import re
import shutil
import os
import shlex
import subprocess
import sys

debug = False
default_encoding = sys.getdefaultencoding()

def check_param(obj, obj_type):
    """Assert obj as type obj_type."""

    try:
        obj_type(obj)
    except Exception:
        raise AssertionError(
            "error: parameter ("
            + str(type(obj))
            + ": "
            + str(obj)
            + ") must be of "
            + str(obj_type)
            + "!"
        )

def replace(infile, outfile, tokens, values, formats=None, encoding=default_encoding):
    """
    Replace values in a file using delimiters.

    Parameters
    ----------
    infile : str
        Name of template file that will be parsed.
    outfile : str
        Name of file that will received the template parsed.
        If equal to *None* or to *infile*, the file will be replaced in-place.
    tokens : list of str
        Regexes that will be replaced.
        When dealing with overlapping tokens, longest tokens should go first.
    values : list
        Values (can be string, float, ...) that will replace
        the tokens. The list must have the same size as tokens.
    formats : list of str, optional
        A list of formats for printing the values in the parsed files at
        tokens locations
        see https://docs.python.org/2/library/string.html#formatspec
    encoding : str, optional
        File encoding
        see http://docs.python.org/2/library/codecs.html#codec-base-classes

    Raises
    ------
    AssertionError
        parameters badly set
    EOFError
        a token has not been found

    Examples
    --------
    >>> import openturns.coupling_tools as ct
    >>> # write a template file
    >>> with open('prgm.dat.in', 'w') as f:
    ...     count = f.write('E=@E_VAR F=-PF-')
    >>> # replace tokens from template
    >>> ct.replace('prgm.dat.in', 'prgm.dat',
    ...     tokens=["@E_VAR", '-PF-'], values=[1.4, '5'])
    >>> # display file
    >>> with open('prgm.dat', 'r') as f:
    ...     print(f.read())
    E=1.4 F=5
    """
    if len(tokens) != len(values):
        raise AssertionError("Error: tokens size is != values size!")
    check_param(tokens, list)
    check_param(values, list)
    if formats is not None:
        check_param(formats, list)
        if len(values) != len(formats):
            raise AssertionError("Error: values size is != formats size!")
    else:
        formats = ["{0}"] * len(values)

    inplace = False
    if (outfile is None) or (infile == outfile):
        inplace = True
        outfile = infile + ".temporary_outfile"

    infile_handle = open(infile, "rb")
    outfile_handle = open(outfile, "wb")

    regex_tokens = []
    found_tokens = []
    for token in tokens:
        regex_tokens.append(re.compile(token))
        found_tokens.append(False)

    for line in infile_handle:
        line = line.decode(encoding)
        i = 0
        for regex_token in regex_tokens:
            found = regex_token.search(line)
            while found is not None:
                found_tokens[i] = True
                line = line.replace(found.group(0), formats[i].format(values[i]))
                found = regex_token.search(line)
            i += 1

        outfile_handle.write(line.encode(encoding))

    infile_handle.close()
    outfile_handle.close()

    for token, found_token in zip(tokens, found_tokens):
        if not found_token:
            raise EOFError("Error: token '" + token + "' not found!")

    if inplace:
        os.remove(infile)
        shutil.move(outfile, infile)

class OTCalledProcessError(subprocess.CalledProcessError):
    def __str__(self):
        err_msg = (
            (":\n" + self.stderr[:200].decode()) if self.stderr is not None else ""
        )
        return super(OTCalledProcessError).__str__(self) + err_msg

def execute(
    cmd,
    cwd=None,
    shell=False,
    executable=None,
    hide_win=True,
    check=True,
    capture_output=False,
    timeout=None,
    env=None,
):
    """
    Launch an external process.

    Parameters
    ----------
    cmd : str
        Command line to execute, e.g.: "echo 42"
    cwd : str
        Current directory of the executed command.
    shell : bool, default=False
        If set to True, the command is started in a shell (bash).
    executable : str, default=False
        path to the shell. e.g. /bin/zsh.
    hide_win : str, default=True
        Hide cmd.exe popup on windows platform.
    check : bool, default=True
        If set to True: raise RuntimeError if return code of process != 0
    capture_output : bool, default=False
        Whether the output/error streams will be captured
    timeout : int
        Process timeout (Python >=3.3 only)
        On timeout and if psutil is available the children of the process
        are killed before the process itself
    env : dict, default=None
        Environment variables mapping for the new process

    Returns
    -------
    cp : subprocess.CompletedProcess
        Process state info

    Raises
    ------
    RuntimeError
        could not run

    Examples
    --------
    >>> import openturns.coupling_tools as ct
    >>> cp = ct.execute('echo 42', capture_output=True, shell=True)
    >>> cp.returncode
    0
    >>> int(cp.stdout)
    42
    """

    # split cmd if not in a shell before passing it to os.execvp()
    posix = os.name == "posix"
    process_args = cmd if shell else shlex.split(cmd, posix=posix)

    # override startupinfo to hide windows console
    startupinfo = None
    if hide_win and sys.platform.startswith("win"):
        startupinfo = subprocess.STARTUPINFO()
        startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW

    stdout = subprocess.PIPE if capture_output else None
    stderr = subprocess.PIPE if capture_output else None
    process = subprocess.Popen(
        process_args,
        shell=shell,
        cwd=cwd,
        executable=executable,
        stdout=stdout,
        stderr=stderr,
        startupinfo=startupinfo,
        env=env,
    )
    stdout_data = None
    stderr_data = None
    try:
        stdout_data, stderr_data = process.communicate(timeout=timeout)
    except subprocess.TimeoutExpired:
        # kill children is psutil is available
        try:
            import psutil
        except ImportError:
            pass
        else:
            parent = psutil.Process(process.pid)
            for child in parent.children(recursive=True):
                child.kill()
        process.kill()
        raise

    returncode = process.poll()

    # check return code
    if check and returncode:
        raise OTCalledProcessError(returncode, cmd, stdout_data, stderr_data)

    return subprocess.CompletedProcess(
        cmd, returncode, stdout=stdout_data, stderr=stderr_data
    )

def get_regex(filename, patterns, encoding=default_encoding):
    """
    Get values from a file using regex.

    Parameters
    ----------
    filename : str
        The name of the file to parse
    patterns : list of str
        Regex patterns that will permit one to get the values
        see https://docs.python.org/2/library/re.html for available patterns
        The value to be searched must be surrounded by parenthesis.
    encoding : str
        File encoding
        see http://docs.python.org/2/library/codecs.html#codec-base-classes

    Returns
    -------
    results : list of float
        Each value corresponds to each pattern.
        If nothing has been found, the corresponding value is set to None.

    Raises
    ------
    AssertionError
        parameters badly set
    EOFError
        no value found

    Examples
    --------
    >>> import openturns.coupling_tools as ct
    >>> # write an output file
    >>> with open('results.out', 'w') as f:
    ...     count = f.write('@E=-9.5E3')
    >>> # parse file with regex
    >>> ct.get_regex('results.out', patterns=[r"(\\R)"])
    [-9500.0]
    """

    if not isinstance(patterns, list) or len(patterns) == 0:
        raise AssertionError("error: patterns parameter badly set!")

    results = [None] * len(patterns)

    re_patterns = []
    for pattern in patterns:
        # OT-like shortcuts
        pattern = pattern.replace(
            r"\R", r"[+-]? *(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?"
        )
        pattern = pattern.replace(r"\I", r"[+-]? *\d+")

        re_patterns.append(re.compile(pattern))

    file_handle = open(filename, "rb")

    for line in file_handle:
        line = line.decode(encoding)
        i = 0
        for re_pattern in re_patterns:
            match = re_pattern.search(line)
            if match:
                results[i] = float(match.group(1))
            i += 1

    file_handle.close()

    for result, pattern in zip(results, patterns):
        if result is None:
            raise EOFError("error: no pattern (" + pattern + ") found!")

    return results

def get_real_from_line(line):
    """
    Try to get a real value from the beginning of a text line.

    Parameters
    ----------
    line : str
        Line to parse

    Returns
    -------
    value : float or int
        Found value. Raise an exception if nothing found.
    """

    # \S*: spaces are allowed at the beginning of the real
    real_regex = r"\s*[+-]? *(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?"
    re_real = re.compile(real_regex)

    # get the value
    match = re_real.match(line)
    if match:
        result = float(line[match.start(): match.end()])
    else:
        raise EOFError(
            "error: real not found at the beginning of this line: " "(" + line + ")!"
        )

    return result

def read_line(handle, seek=0, encoding=default_encoding):
    """
    Get next line.

    Parameters
    ----------
    seek : int
        if seek < 0: stop reading seek before the end of the file
    encoding : str
        file encoding, see http://docs.python.org/2/library/codecs.html#codec-base-classes
    """
    line = handle.readline().decode(encoding)

    if seek < 0 and line != "":
        if handle.tell() > -seek:
            if debug:
                sys.stderr.write("line before cut ->" + line + "<-\n")

            # cut the end of the line
            line = line[: len(line) - (handle.tell() + seek)]
            # after cut, if the line is empty, add a \n to behave like
            # readline() python function
            if line == "":
                line = "\n"
            # any further read will return ''
            handle.seek(0, os.SEEK_END)

            if debug:
                sys.stderr.write("line after cut ->" + line + "<-\n")

    if debug:
        sys.stderr.write("read_line: ->" + line + "<-\n")

    return line

def get_line_col(
    filename, skip_line=0, skip_col=0, col_sep=None, seek=0, encoding=default_encoding
):
    """
    Get a value at specific line/columns coordinates.

    Parameters
    ----------
    filename : str
        The name of the file that will be parsed
    skip_line : int, default=0
        Number of lines skipped
        If skip_line < 0: count lines backward from the end of the file.
        Be careful: a last empty line is taken into account too!
        Default: 0: no line skipped
    skip_col : int, default=0
        Number of columns skipped from the beginning or end of the line.
        If skip_col < 0: count col backward from the end of the line.
        Default: 0: no column skipped
    col_sep : str
        Column separator
        Default: None: whitespace separator, see str.split
    seek : int, default=0
        if > 0, consider the file starts at pos seek.
        if < 0, consider the file ends at pos -seek (and NOT (end-(-seek))!).
        Default: 0: consider the whole file.
    encoding : str
        File encoding
        see http://docs.python.org/2/library/codecs.html#codec-base-classes

    Returns
    -------
    result : float
        Value to retrieve

    Examples
    --------
    >>> import openturns.coupling_tools as ct
    >>> with open('results.out', 'w') as f:
    ...     count = f.write('1.1 1.2 1.3 1.4')
    >>> ct.get_line_col(filename='results.out', skip_col=2)
    1.3
    """
    check_param(filename, str)
    check_param(skip_line, int)
    check_param(skip_col, int)
    check_param(seek, int)

    handle = open(filename, "rb")
    if seek > 0:
        handle.seek(seek)

    if debug:
        sys.stderr.write(
            "get_line_col(skip_line="
            + str(skip_line)
            + ", skip_col="
            + str(skip_col)
            + ", seek="
            + str(seek)
            + ")\n"
        )

    # skip line backward
    if skip_line < 0:
        # cache position of each beginning of line
        # last elt of the list is the last inserted
        lines_cache = []
        # determine number of elt to cache
        lines_cache_size = -skip_line

        # build lines cache
        previous_pos = handle.tell()
        line = read_line(handle, seek, encoding)
        while line != "":
            # append to cache
            lines_cache.append(previous_pos)
            if len(lines_cache) > lines_cache_size:
                # todo: list not optimized to del before the end
                del lines_cache[0]
            previous_pos = handle.tell()

            line = read_line(handle, seek, encoding)
            if debug:
                sys.stderr.write("lines_cache: " + str(lines_cache) + "\n")

        if len(lines_cache) < lines_cache_size:
            err_msg = (
                "error: the file has less than " + str(lines_cache_size) + " lines!"
            )
            handle.close()
            raise EOFError(err_msg)
        else:
            handle.seek(lines_cache[0])
            line_found = read_line(handle, seek, encoding)
            if debug:
                sys.stderr.write("line found: ->" + line_found + "<-\n")
    # skip line forward
    else:
        while skip_line >= 0:
            line = read_line(handle, seek, encoding)
            if skip_line > 0 and line == "":
                handle.close()
                raise EOFError('error: the file has less lines than "skip_line"!')
            skip_line -= 1
        line_found = line

    handle.close()

    # get the good col
    if skip_col != 0:
        try:
            line_found = line_found.split(col_sep)[skip_col]
        except Exception:
            raise EOFError("error: value not found on this line: (" + line_found + ")!")

    # get the value
    result = get_real_from_line(line_found)

    return result

def get_value(
    filename,
    token=None,
    skip_token=0,
    skip_line=0,
    skip_col=0,
    col_sep=None,
    encoding=default_encoding,
):
    """
    Get a value from a file using a delimiter and/or offsets.

    This function is optimized to be rather fast and takes low memory on human
    readable file.

    Parameters
    ----------
    filename : str
        The name of the file that will be parsed
    token : str
        A regex that will be searched.
        The value right after the token is returned.
        Default: None (no token searched)
    skip_token : int
        The number of tokens that will be skipped before getting
        the value. If set to != 0, the corresponding token parameter must
        not be equal to None.
        If skip_tokens < 0: count tokens backward from the end of the file.
        Default: 0: no token skipped
    skip_line : int
        Number of lines skipped from the token found.
        If corresponding token equal None, skip from the beginning of the file.
        If corresponding token != None, skip from the token.
        If skip_line < 0: count lines backward from the token or from the end
        of the file. Be careful: a last empty line is taken into account too.
        Default: 0: no line skipped
    skip_col : int
        Number of columns skipped from the token found.
        If corresponding token = None, skip words from the beginning
        of the line.
        If corresponding token != None, skip words from the token.
        If skip_col < 0: count col backward from the end of the line or from
        the token.
        Default: 0: no column skipped
    col_sep : str
        Column separator
        Default: None: whitespace separator, see str.split
    encoding : str
        File encoding
        see http://docs.python.org/2/library/codecs.html#codec-base-classes

    Returns
    -------
    value : float
        Value found

    Raises
    ------
    AssertionError
        parameters badly set
    EOFError
        no value found

    Examples
    --------
    using a single token

    >>> import openturns.coupling_tools as ct
    >>> with open('results.out', 'w') as f:
    ...     count = f.write('Y1=2.0, Y2=-6.6E56')
    >>> ct.get_value('results.out', token='Y1=')
    2.0

    using token and skip_tokens

    >>> with open('results.out', 'w') as f:
    ...     count = f.write('Y1=2.6 Y1=6.0 # temperature 2')
    >>> ct.get_value('results.out', token='Y1=', skip_token=1)
    6.0

    using column & line

    >>> with open('results.out', 'w') as f:
    ...     count = f.write('1.1 1.2 1.3 1.4')
    >>> ct.get_value(filename='results.out', skip_col=2)
    1.3
    """
    if debug:
        sys.stderr.write(
            "get_value(token="
            + str(token)
            + ", skip_token="
            + str(skip_token)
            + ", skip_line="
            + str(skip_line)
            + ", skip_col="
            + str(skip_col)
            + ")\n"
        )

    # check parameters
    check_param(filename, str)
    if token is not None:
        check_param(token, str)
    elif skip_token != 0:
        raise AssertionError(
            "error: skip_token parameter needs token " "parameter to be set!"
        )
    check_param(skip_token, int)
    check_param(skip_line, int)
    check_param(skip_col, int)

    result = None
    if not token:
        result = get_line_col(
            filename, skip_line, skip_col, col_sep=col_sep, encoding=encoding
        )
    else:
        handle = open(filename, "rb")

        re_token = re.compile(token)

        # store previous begin of line pos
        line_pos = handle.tell()

        # store token pos [elt 0: start pos of the token, elt 1: end of the
        # token]
        if skip_token >= 0:
            token_pos = None
        else:
            token_pos_cache = []

        line = handle.readline().decode(encoding)
        while line != "":
            for token_match in re_token.finditer(line):
                if skip_token > 0:
                    # token found but it's not the good one
                    skip_token -= 1
                elif skip_token == 0:
                    # token found
                    token_pos = [
                        line_pos + token_match.start(),
                        line_pos + token_match.end(),
                    ]
                    if debug:
                        sys.stderr.write("skip_token == 0, line: " + line + "\n")
                    break
                else:
                    # token wanted in revert order: we first cache them all
                    token_pos_cache.append(
                        [line_pos + token_match.start(), line_pos + token_match.end()]
                    )

            if skip_token >= 0 and token_pos is not None:
                # token found
                break

            line_pos = handle.tell()
            line = handle.readline().decode(encoding)

        # get the token from the cache
        if skip_token < 0 and len(token_pos_cache) >= -skip_token:
            token_pos = token_pos_cache[skip_token]

        if token_pos is None:
            handle.close()
            raise EOFError("error: no token (" + token + ") was found!")

        if skip_line == 0 and skip_col == 0:
            # get the real right after the token
            handle.seek(token_pos[1])
            line = handle.readline().decode(encoding)
            if debug:
                sys.stderr.write("first token, line_found: " + line + "\n")
            result = get_real_from_line(line)
            handle.close()  # fixme: no multiple close?
        else:
            # get the real by skipping from the token
            if skip_line < 0 or (skip_line == 0 and skip_col < 0):
                # search before the token
                seek_pos = -token_pos[0]
                skip_line -= 1  # skip line from the token
            else:
                # search after the token
                seek_pos = token_pos[1]
            handle.close()
            result = get_line_col(
                filename,
                skip_line,
                skip_col,
                col_sep=col_sep,
                seek=seek_pos,
                encoding=encoding,
            )

    return result

def get(
    filename,
    tokens=None,
    skip_tokens=None,
    skip_lines=None,
    skip_cols=None,
    encoding=default_encoding,
):
    """
    Get several values from a file using delimiters and/or offsets.

    This is equivalent to get_value
    Each following parameters must be a list. The first element of each list
    will be used to get the first value, the 2nd ...
    If the parameter is set to None, the list is set to the default value of
    get_value(), i.e skip_lines=None => [0, 0, ...].

    Parameters
    ----------
    filename : str
        The name of the file that will be parsed
    tokens : list of str
        see get_value
    skip_tokens : list of int
        see get_value
    skip_lines : list of int
        see get_value()
    skip_cols : list of int
        see get_value()
    encoding : str
        File encoding, see
        http://docs.python.org/2/library/codecs.html#codec-base-classes

    Returns
    -------
    result : list of float
        Values parsed

    Raises
    ------
    AssertionError
        parameters badly set
    EOFError
        no value found

    Examples
    --------
    using tokens

    >>> import openturns.coupling_tools as ct
    >>> with open('results.out', 'w') as f:
    ...     count = f.write('Y1=2.0, Y2=-6.6E2')
    >>> ct.get(filename='results.out', tokens=['Y1=', 'Y2='])
    [2.0, -660.0]
    """
    # test parameters and determine the number of value to return
    nb_values = None
    if tokens is not None:
        nb_values = len(tokens)

    err_msg = "error: skip_tokens parameter needs tokens parameter to be set!"
    if skip_tokens is not None:
        if nb_values is None:
            raise AssertionError(err_msg)
        elif nb_values != len(skip_tokens):
            raise AssertionError(err_msg)

    if skip_lines is not None:
        if nb_values is None:
            nb_values = len(skip_lines)
        elif nb_values != len(skip_lines):
            raise AssertionError(err_msg)

    if skip_cols is not None:
        if nb_values is None:
            nb_values = len(skip_cols)
        elif nb_values != len(skip_cols):
            raise AssertionError(err_msg)

    if nb_values is None:
        raise AssertionError("error: no parameters have been set!")

    # init properly every list
    if tokens is None:
        tokens = [None] * nb_values
    if skip_tokens is None:
        skip_tokens = [0] * nb_values
    if skip_lines is None:
        skip_lines = [0] * nb_values
    if skip_cols is None:
        skip_cols = [0] * nb_values

    results = [None] * nb_values

    # possible optimization: get values in one pass
    for n in range(nb_values):
        results[n] = get_value(
            filename,
            tokens[n],
            skip_tokens[n],
            skip_lines[n],
            skip_cols[n],
            encoding=encoding,
        )

    return results