import pandas as pd
from pandas import DataFrame as df
import datetime
from datetime import datetime as dt
from single_trial import *

def run_experiment(pid):
  try:
    print('Loading config_{}.csv...'.format(pid), end='\r')
    config = pd.read_csv(
        os.path.join(PATH_INPUT, 'config_{}.csv'.format(pid)),
        keep_default_na=False).to_dict(orient='records')
    print('Loading config_{}.csv...done'.format(pid))

    print('Loading block_parameters_{}.csv...'.format(pid), end='\r')
    block_params = pd.read_csv(
        os.path.join(PATH_INPUT, 'block_parameters_{}.csv'.format(pid)),
        keep_default_na=False).to_dict(orient='records')
    print('Loading block_parameters_{}.csv...done'.format(pid))

    print('Loading experimental_parameters_{}.csv...'.format(pid), end='\r')
    experimental_params = pd.read_csv(
        os.path.join(PATH_INPUT, 'experimental_parameters_{}.csv'.format(pid)),
        keep_default_na=False).to_dict(orient='records')
    print('Loading experimental_parameters_{}.csv...done'.format(pid))
  except BaseException as error:
    exit('\n{}\n{}\n{}\n{}'.format(error,
                                  '#######################',
                                  '### EXITING PROGRAM ###',
                                  '#######################'))
  stims = STIMULI
  try:
    print('Loading calibration_parameters_{}.csv...'.format(pid), end='\r')
    calibration_params = pd.read_csv(
        os.path.join(PATH_INPUT, 'calibration_parameters_{}.csv'.format(pid)),
        keep_default_na=False).to_dict(orient='records')[0]
    print('Loading calibration_parameters_{}.csv...done\n'.format(pid))

    origin_left =  (monitorunittools.pix2deg(calibration_params['x_offset_LE_final'], MONITOR),
                    monitorunittools.pix2deg(calibration_params['y_offset_LE_final'], MONITOR))
    origin_right = (monitorunittools.pix2deg(calibration_params['x_offset_RE_final'], MONITOR),
                    monitorunittools.pix2deg(calibration_params['y_offset_RE_final'], MONITOR))
    stims = init_stimuli(origin_left, origin_right)
  except BaseException as error:
    print('calibration_parameters_{}.csv not found! Using default values.\n'.format(pid))

  os.makedirs(PATH_OUTPUT, exist_ok=True)
  config_out = os.path.join(PATH_OUTPUT, 'out_config_{}.csv'.format(pid))
  block_out = os.path.join(PATH_OUTPUT, 'out_block_parameters_{}.csv'.format(pid))
  exp_out = os.path.join(PATH_OUTPUT, 'out_experimental_parameters_{}.csv'.format(pid))

  config_df = df.from_dict(config)
  config_df.to_csv(config_out, mode='w+', index=False,
                   header=not os.path.exists(config_out))

  iteration = block_number = block_start_time = inter_block_pause_duration = 0
  last_block = block_params[-1]['block']

  show_instructions()
  show_pre_trial_countdown(stims)

  print(origin_left)

  for exp_param in experimental_params:
    iteration += 1
    trial = Trial(stims, exp_param, iteration)

    if trial.block > block_number:
      if not block_number == 0:
        block_end_time = dt.now().strftime('%Y-%m-%d %H:%M:%S.%f')
        block_params[block_number-1]['end_time'] = block_end_time
        block_df = df.from_dict(block_params)
        block_df.to_csv(block_out, mode='w+', index=False,
                        header=not os.path.exists(block_out))
        print('### BLOCK {} END: {}'.format(block_number, block_end_time))
        show_inter_block_countdown(inter_block_pause_duration)
        show_pre_trial_countdown(stims)

      inter_block_pause_duration = block_params[block_number]['inter_block_pause_duration']
      block_start_time = dt.now().strftime('%Y-%m-%d %H:%M:%S.%f')
      block_params[block_number]['start_time'] = block_start_time
      block_df = df.from_dict(block_params)
      block_df.to_csv(block_out, mode='w+', index=False,
                      header=not os.path.exists(block_out))
      block_number = trial.block
      print('### BLOCK {} BEGIN: {}'.format(block_number, block_start_time))

    response = run_trial(trial)

    if response:
      exp_param['response'] = response[0].name
      exp_param['response_time_ms'] = [response[0].rt * 1000]
    else:
      exp_param['response'] = 'NA'
      exp_param['response_time_ms'] = ['NA']

    exp_df = df.from_dict(exp_param)
    exp_df.to_csv(exp_out, mode='a+', index=False,
                  header=not os.path.exists(exp_out))

  block_end_time = dt.now().strftime('%Y-%m-%d %H:%M:%S.%f')
  block_params[block_number-1]['end_time'] = block_end_time
  block_df = df.from_dict(block_params)
  block_df.to_csv(block_out, mode='w+', index=False,
                  header=not os.path.exists(block_out))
  print('### BLOCK {} END: {}'.format(block_number, block_end_time))

  print('Experiment completed:', block_end_time)
  print('Results written to:', PATH_OUTPUT)

  show_completion_message()

def show_instructions():
  print('### EXPERIMENT BEGIN: {}'.format(dt.now().strftime('%Y-%m-%d %H:%M:%S.%f')))
  print('AWAITING RESPONSE', end='\r')
  instructions = visual.TextBox2(WINDOW, text=INSTRUCTIONS,
                                 alignment='center', color='black', bold=True,
                                 letterHeight=28, size=[99999, None])
  instructions.draw()
  WINDOW.flip()

  KEYBOARD.clearEvents()
  response = KEYBOARD.waitKeys(keyList=['space'], waitRelease=False)
  print('Response: {} {}'.format(response[0].name, dt.now().strftime('%Y-%m-%d %H:%M:%S.%f')))
  WINDOW.flip()
  core.wait(0.5)

def show_pre_trial_countdown(stims):
  countdown_stim_L = visual.TextStim(WINDOW)
  countdown_stim_R = visual.TextStim(WINDOW)
  countdown_stim_L.bold = countdown_stim_R.bold = True
  countdown_stim_L.colorSpace = countdown_stim_R.colorSpace = 'rgb'
  countdown_stim_L.color = countdown_stim_R.color = 'black'
  countdown_stim_L.units = countdown_stim_R.units = 'deg'
  countdown_stim_L.size = countdown_stim_R.size = R2_SIZE
  countdown_stim_L.pos = [stims['left'][0]['x'], stims['left'][0]['y']]
  countdown_stim_R.pos = [stims['right'][0]['x'], stims['right'][0]['y']]

  for i in range(3, 0, -1):
    print('COUNTDOWN:', datetime.timedelta(0, i), end='\r')
    countdown_stim_L.text = countdown_stim_R.text = '{}...'.format(i)
    countdown_stim_L.draw()
    countdown_stim_R.draw()
    WINDOW.flip()
    core.wait(1)
  print('      BEGIN       ', end='\r')

  countdown_stim_L.text = countdown_stim_R.text = 'BEGIN'
  countdown_stim_L.draw()
  countdown_stim_R.draw()
  WINDOW.flip()
  core.wait(1)

  WINDOW.flip()
  core.wait(1)

def show_inter_block_countdown(secs):
  print('### INTER-BLOCK PAUSE: {} ms'.format(secs))
  pause_message = visual.TextBox2(WINDOW, text='',
                                  alignment='center', color='black', bold=True,
                                  letterHeight=32, size=[99999, None])
  KEYBOARD.clearEvents()

  for i in range(round(secs/1000), -1, -1):
    response = KEYBOARD.getKeys(keyList=['space'], waitRelease=False)
    if response: break

    time_left = datetime.timedelta(0, i)
    print('COUNTDOWN: {}'.format(time_left), end='\r')
    pause_message.text = 'You may now take a short break.\n\nREMAINING TIME: {}\n\nPress [SPACE BAR] to continue now.'.format(time_left)
    pause_message.draw()
    WINDOW.flip()
    core.wait(secs=1, hogCPUperiod=1)
  print()

  WINDOW.flip()
  core.wait(1)

def show_completion_message():
  message = 'Thank you for your time.\n\nThe experiment is now over.\n\nPress [SPACE BAR] to end this session.'
  completion_message = visual.TextBox2(WINDOW, text=message,
                                 alignment='center', color='black', bold=True,
                                 letterHeight=32, size=[99999, None])
  completion_message.draw()
  WINDOW.flip()

  KEYBOARD.clearEvents()
  KEYBOARD.waitKeys(keyList=['space'], waitRelease=False)
  WINDOW.flip()
  core.wait(0.5)


import torch
import torch.nn.functional as F
import torch.nn as nn
import numpy as np

class BinaryDiceLoss(nn.Module):
    def __init__(self, smooth=1, p=2, reduction='mean'):
        super(BinaryDiceLoss, self).__init__()
        self.smooth = smooth
        self.p = p
        self.reduction = reduction

    def forward(self, predict, target):
        assert predict.shape[0] == target.shape[0], "predict & target batch size don't match"
        predict = predict.contiguous().view(predict.shape[0], -1)
        target = target.contiguous().view(target.shape[0], -1)

        num = torch.sum(torch.mul(predict, target), dim=1)
        den = torch.sum(predict, dim=1) + torch.sum(target, dim=1) + self.smooth

        dice_score = 2*num / den
        loss_avg = 1 - dice_score.mean()

        return loss_avg

class DiceLoss4BraTS(nn.Module):
    def __init__(self, weight=None, ignore_index=None, **kwargs):
        super(DiceLoss4BraTS, self).__init__()
        self.kwargs = kwargs
        self.weight = weight
        self.ignore_index = ignore_index

    def forward(self, predict, target):
        assert predict.shape == target.shape, 'predict %s & target %s shape do not match' % (predict.shape, target.shape)
        dice = BinaryDiceLoss(**self.kwargs)
        total_loss = 0
        predict = F.sigmoid(predict)

        for i in range(target.shape[1]):
            if i != self.ignore_index:
                dice_loss = dice(predict[:, i], target[:, i])
                if self.weight is not None:
                    assert self.weight.shape[0] == target.shape[1], \
                        'Expect weight shape [{}], get[{}]'.format(target.shape[1], self.weight.shape[0])
                    dice_loss *= self.weights[i]
                total_loss += dice_loss

        return total_loss/(target.shape[1]-1 if self.ignore_index!=None else target.shape[1])

class BCELoss4BraTS(nn.Module):
    def __init__(self, ignore_index=None, **kwargs):
        super(BCELoss4BraTS, self).__init__()
        self.kwargs = kwargs
        self.ignore_index = ignore_index
        self.criterion = nn.BCEWithLogitsLoss()

    def weighted_BCE_cross_entropy(self, output, target, weights = None):
        if weights is not None:
            assert len(weights) == 2
            output = torch.clamp(output, min=1e-7, max=1-1e-7)
            bce = weights[1] * (target * torch.log(output)) + \
                  weights[0] * ((1-target) * torch.log((1-output)))
        else:
            output = torch.clamp(output, min=1e-3, max=1 - 1e-3)
            bce = target * torch.log(output) + (1-target) * torch.log((1-output))
        return torch.neg(torch.mean(bce))

    def forward(self, predict, target):
        assert predict.shape == target.shape, 'predict & target shape do not match'
        total_loss = 0
        for i in range(target.shape[1]):
            if i != self.ignore_index:
                bce_loss = self.criterion(predict[:, i], target[:, i])
                total_loss += bce_loss

        return total_loss.mean()

class BCELossBoud(nn.Module):
    def __init__(self, weight=None, ignore_index=None, **kwargs):
        super(BCELossBoud, self).__init__()
        self.kwargs = kwargs
        self.weight = weight
        self.ignore_index = ignore_index
        self.criterion = nn.BCEWithLogitsLoss()

    def weighted_BCE_cross_entropy(self, output, target, weights = None):
        if weights is not None:
            assert len(weights) == 2
            output = torch.clamp(output, min=1e-3, max=1-1e-3)
            bce = weights[1] * (target * torch.log(output)) + \
                  weights[0] * ((1-target) * torch.log((1-output)))
        else:
            output = torch.clamp(output, min=1e-3, max=1 - 1e-3)
            bce = target * torch.log(output) + (1-target) * torch.log((1-output))
        return torch.neg(torch.mean(bce))

    def forward(self, predict, target):

        bs, category, depth, width, heigt = target.shape
        bce_loss = []
        for i in range(predict.shape[1]):
            pred_i = predict[:,i]
            targ_i = target[:,i]
            tt = np.log(depth * width * heigt / (target[:, i].cpu().data.numpy().sum()+1))
            bce_i = self.weighted_BCE_cross_entropy(pred_i, targ_i, weights=[1, tt])
            bce_loss.append(bce_i)

        bce_loss = torch.stack(bce_loss)
        total_loss = bce_loss.mean()
        return total_loss


import numpy as np
import torch
import torch.nn as nn

from libcity.model import loss
from libcity.model.abstract_traffic_state_model import AbstractTrafficStateModel

class SLSTM(nn.Module):
    def __init__(self, feature_dim, hidden_dim, device, p_interval):
        super(SLSTM, self).__init__()
        self.hidden_dim = hidden_dim
        self.cell_dim = hidden_dim
        self.p_interval = p_interval
        self.f_gate = nn.Sequential(
            nn.Linear(feature_dim + hidden_dim, self.cell_dim),
            nn.Softmax(dim=1)
        )
        self.i_gate = nn.Sequential(
            nn.Linear(feature_dim + hidden_dim, self.cell_dim),
            nn.Softmax(dim=1)
        )
        self.o_gate = nn.Sequential(
            nn.Linear(feature_dim + hidden_dim, self.hidden_dim),
            nn.Softmax(dim=1)
        )
        self.g_gate = nn.Sequential(
            nn.Linear(feature_dim + hidden_dim, self.cell_dim),
            nn.Tanh()
        )

        self.tanh = nn.Tanh()

        self.device = device

    def forward(self, x):
        # (T, B * N, 2E)
        h = torch.zeros((x.shape[1], self.hidden_dim)).unsqueeze(dim=0).repeat(self.p_interval, 1, 1).to(self.device)
        # (P, B * N, 2E)
        c = torch.zeros((x.shape[1], self.hidden_dim)).unsqueeze(dim=0).repeat(self.p_interval, 1, 1).to(self.device)
        # (P, B * N, 2E)

        T = x.shape[0]

        for t in range(T):
            x_ = x[t, :, :]  # (B * N, 2E)
            x_ = torch.cat((x_, h[t % self.p_interval]), 1)  # (B * N, 2E + 2E)

            f = self.f_gate(x_)  # (B * N, 2E)

            i = self.i_gate(x_)  # (B * N, 2E)

            o = self.o_gate(x_)  # (B * N, 2E)

            g = self.g_gate(x_)  # (B * N, 2E)

            c = f * c[t % self.p_interval] + i * g  # (B * N, 2E)

            c = self.tanh(c)  # (B * N, 2E)

            h[t % self.p_interval] = o * c  # (B * N, 2E)

        return h[(T - 1) % self.p_interval]  # (B * N, 2E)

class MutiLearning(nn.Module):
    def __init__(self, fea_dim, device):
        super(MutiLearning, self).__init__()
        self.fea_dim = fea_dim
        self.transition = nn.Parameter(data=torch.randn(self.fea_dim, self.fea_dim).to(device), requires_grad=True)
        self.project_in = nn.Parameter(data=torch.randn(self.fea_dim, 1).to(device), requires_grad=True)
        self.project_out = nn.Parameter(data=torch.randn(self.fea_dim, 1).to(device), requires_grad=True)

    def forward(self, x: torch.Tensor):
        # (B, N, 2E)
        x_t = x.permute(0, 2, 1)  # (B, 2E, N)

        x_in = torch.matmul(x, self.project_in)  # (B, N, 1)

        x_out = torch.matmul(x, self.project_out)  # (B, N, 1)

        x = torch.matmul(x, self.transition)
        # (B, N, 2E)
        x = torch.matmul(x, x_t)
        # (B, N, N)

        x = x.unsqueeze(dim=-1).unsqueeze(dim=1)
        x_in = x_in.unsqueeze(dim=-1).unsqueeze(dim=1)
        x_out = x_out.unsqueeze(dim=-1).unsqueeze(dim=1)

        return x, x_in, x_out

class GraphConvolution(nn.Module):
    def __init__(self, feature_dim, embed_dim, device, use_bias=False):
        super(GraphConvolution, self).__init__()
        self.feature_dim = feature_dim
        self.embed_dim = embed_dim
        self.activation = nn.ReLU()

        weight = torch.randn((self.feature_dim, self.embed_dim))
        self.weight = nn.Parameter(data=weight.to(device), requires_grad=True)
        if use_bias:
            self.bias = nn.Parameter(data=torch.zeros(self.embed_dim).to(device), requires_grad=True)
        else:
            self.bias = None

    def forward(self, x, a):
        # (B, N, N)
        embed = torch.matmul(a, x)
        # (B, N, N)
        embed = torch.matmul(embed, self.weight)
        # (B, N, E)
        if self.bias is not None:
            embed += self.bias

        return embed

class GCN(nn.Module):
    def __init__(self, feature_dim, embed_dim, device):
        super(GCN, self).__init__()
        self.feature_dim = feature_dim
        self.embed_dim = embed_dim
        self.gcn = nn.ModuleList([
            GraphConvolution(feature_dim, embed_dim, device),
            GraphConvolution(embed_dim, embed_dim, device)
        ])

    def forward(self, input_seq, adj_seq):
        embed = []
        for i in range(input_seq.shape[1]):
            frame = input_seq[:, i, :, :]  # (B, N, N)
            adj = adj_seq[:, i, :, :]  # (B, N, N)

            for m in self.gcn:
                frame = m(frame, adj)
            # (B, N, E)

            embed.append(frame)

        return torch.stack(embed, dim=1)

def generate_geo_adj(distance_matrix: np.matrix):
    distance_matrix = torch.Tensor(distance_matrix)  # (N, N)
    distance_matrix = distance_matrix * distance_matrix
    sum_cost_vector = torch.sum(distance_matrix, dim=1, keepdim=True)  # (N, 1)
    weight_matrix = distance_matrix / sum_cost_vector
    weight_matrix[range(weight_matrix.shape[0]), range(weight_matrix.shape[1])] = 1
    return weight_matrix  # (N, N)

def generate_semantic_adj(demand_matrix, device):
    # (B, T, N, N)
    adj_matrix = demand_matrix.clone()
    in_matrix = adj_matrix.permute(0, 1, 3, 2)

    adj_matrix[adj_matrix > 0] = 1

    adj_matrix[in_matrix > 0] = 1

    degree_vector = torch.sum(adj_matrix, dim=3, keepdim=True)
    # (B, T, N, 1)

    sum_degree_vector = torch.matmul(adj_matrix, degree_vector)
    # (B, T, N, 1)

    weight_matrix = torch.matmul(1 / (sum_degree_vector + 1e-3), degree_vector.permute((0, 1, 3, 2)))  # (B, T, N, N)

    weight_matrix[:, :, range(weight_matrix.shape[2]), range(weight_matrix.shape[3])] = 1

    return weight_matrix

class GEML(AbstractTrafficStateModel):
    def __init__(self, config, data_feature):
        super().__init__(config, data_feature)
        self.num_nodes = self.data_feature.get('num_nodes')
        self._scaler = self.data_feature.get('scaler')
        self.output_dim = config.get('output_dim')
        self.device = config.get('device', torch.device('cpu'))
        self.input_window = config.get('input_window', 1)
        self.output_window = config.get('output_window', 1)

        self.p_interval = config.get('p_interval', 1)
        self.embed_dim = config.get('embed_dim')
        self.batch_size = config.get('batch_size')
        self.loss_p0 = config.get('loss_p0', 0.5)
        self.loss_p1 = config.get('loss_p1', 0.25)
        self.loss_p2 = config.get('loss_p2', 0.25)

        dis_mx = self.data_feature.get('adj_mx')
        self.geo_adj = generate_geo_adj(dis_mx) \
            .repeat(self.batch_size * self.input_window, 1) \
            .reshape((self.batch_size, self.input_window, self.num_nodes, self.num_nodes)) \
            .to(self.device)

        self.GCN_ge = GCN(self.num_nodes, self.embed_dim, self.device)
        self.GCN_se = GCN(self.num_nodes, self.embed_dim, self.device)

        # self.LSTM = nn.LSTM(2 * self.embed_dim, 2 * self.embed_dim)
        self.LSTM = SLSTM(2 * self.embed_dim, 2 * self.embed_dim, self.device, self.p_interval)

        self.mutiLearning = MutiLearning(2 * self.embed_dim, self.device)

    def forward(self, batch):
        x = batch['X'].squeeze(dim=-1)
        # (B, T, N, N)
        x_ge_embed = self.GCN_ge(x, self.geo_adj[:x.shape[0], ...])
        # (B, T, N, E)

        x_se_embed = self.GCN_se(x, self.semantic_adj)

        # (B, T, N, E)
        x_embed = torch.cat([x_ge_embed, x_se_embed], dim=3)
        # (B, T, N, 2E)
        x_embed = x_embed.permute(1, 0, 2, 3)
        # (T, B, N, 2E)
        x_embed = x_embed.reshape((self.input_window, -1, 2 * self.embed_dim))
        # (T, B * N, 2E)

        # _, (h, _) = self.LSTM(x_embed)
        # x_embed_pred = h[0].reshape((self.batch_size, -1, 2 * self.embed_dim))
        x_embed_pred = self.LSTM(x_embed).reshape((x.shape[0], -1, 2 * self.embed_dim))
        # (B, N, 2E)

        out = self.mutiLearning(x_embed_pred)

        return out

    def calculate_loss(self, batch):
        y_true = batch['y']  # (B, TO, N, N, 1)
        y_in_true = torch.sum(y_true, dim=-2, keepdim=True)  # (B, TO, N, 1)
        y_out_true = torch.sum(y_true.permute(0, 1, 3, 2, 4), dim=-2, keepdim=True)  # (B, TO, N, 1)
        y_pred, y_in, y_out = self.predict(batch)

        y_true = self._scaler.inverse_transform(y_true[..., :self.output_dim])
        y_in_true = self._scaler.inverse_transform(y_in_true[..., :self.output_dim])
        y_out_true = self._scaler.inverse_transform(y_out_true[..., :self.output_dim])

        y_pred = self._scaler.inverse_transform(y_pred[..., :self.output_dim])
        y_in = self._scaler.inverse_transform(y_in[..., :self.output_dim])
        y_out = self._scaler.inverse_transform(y_out[..., :self.output_dim])

        loss_pred = loss.masked_mse_torch(y_pred, y_true)
        loss_in = loss.masked_mse_torch(y_in, y_in_true)
        loss_out = loss.masked_mse_torch(y_out, y_out_true)
        return self.loss_p0 * loss_pred + self.loss_p1 * loss_in + self.loss_p2 * loss_out

    def predict(self, batch):
        x = batch['X']  # (B, T, N, N, 1)
        self.semantic_adj = generate_semantic_adj(x.squeeze(dim=-1), self.device)
        assert x.shape[-1] == 1 or print("The feature_dim must be 1")
        y_pred = []
        y_in_pred = []
        y_out_pred = []
        x_ = x.clone()
        for i in range(self.output_window):
            batch_tmp = {'X': x_}
            y_, y_in_, y_out_ = self.forward(batch_tmp)  # (B, 1, N, N, 1)
            y_pred.append(y_.clone())
            y_in_pred.append(y_in_.clone())
            y_out_pred.append(y_out_.clone())

            x_ = torch.cat([x_[:, 1:, :, :, :], y_], dim=1)

        y_pred = torch.cat(y_pred, dim=1)  # (B, TO, N, N, 1)
        y_in_pred = torch.cat(y_in_pred, dim=1)  # (B, TO, N, 1)
        y_out_pred = torch.cat(y_out_pred, dim=1)  # (B, TO, N, 1)
        return y_pred, y_in_pred, y_out_pred


from odoo import api, fields, models, _
from odoo.exceptions import ValidationError
import logging
import time

_logger = logging.getLogger(__name__)
DF = "%Y-%m-%d"
EPS = 0.00001

class Posting(models.Model):
    _inherit = "wc.posting"

    #over write close_date operation for fixing f540 issues
    #this was kind of bug , so overwrite the method by the method below.
    #the source code was copied , and modified part of it.
    @api.multi
    def close_date(self):
        start = time.time()
        _logger.debug("***#close_date: start")

        first = True
        for rec in self:
            rec.add_details()
            rec.state = 'closed'

            if first:
                first = False

                #delete all draft transactions
                #account transactions
                trans = self.env['wc.account.transaction'].sudo().search([
                    ('state','=','draft'),
                    ('company_id','=',self.env.user.company_id.id)
                ])
                _logger.debug("*DRAFT trans: %s", trans)
                if trans:
                    #f540 fix
                    #trans.unlink()
                    raise ValidationError(_("Draft record is remaining in %s! Please confirm or delete it before closing.") % ('account transactions '))

                coll_lines = self.env['wc.collection.line'].sudo().search([
                    ('state','=','draft'),
                    ('company_id','=',self.env.user.company_id.id)
                ])
                _logger.debug("*DRAFT coll lines: %s", coll_lines)
                if coll_lines:
                    #f540 fix
                    #coll_lines.unlink()
                    raise ValidationError(_("Draft record is remaining in %s! Please confirm or delete it before closing.") % ('collection line '))

                collection = self.env['wc.collection'].sudo().search([
                    ('state','=','draft'),
                    ('company_id','=',self.env.user.company_id.id)
                ])
                _logger.debug("*DRAFT collection: %s", collection)
                if collection:
                    #f540 fix
                    #for coll in collection:
                    #    coll.loan_id = False
                    #    coll.loan_payment_id = False
                    #collection.unlink()
                    raise ValidationError(_("Draft record is remaining in %s! Please confirm or delete it before closing.") % ('collection '))

                pay = self.env['wc.loan.payment'].sudo().search([
                    ('state','=','draft'),
                    ('company_id','=',self.env.user.company_id.id)
                ])
                _logger.debug("*DRAFT loan payment: %s", pay)
                if pay:
                    #f540 fix
                    #for p in pay:
                    #    p.collection_id = False
                    #pay.unlink()
                    raise ValidationError(_("Draft record is remaining in %s! Please confirm or delete it before closing.") % ('payment transactions '))

        _logger.debug("***#close_date: stop elapsed=%s", time.time() - start)


from manim import *

class StoC(Scene):
    def construct(self):
        self.add(TextMobject("StoC"))
        self.wait(2)
        self.play(FadeOut(self.mobjects[0]))
        circle = Circle(radius=1.5, color=BLUE)
        self.play(ShowCreation(circle))
        square = Square(side_length=1.5, color=RED)
        self.play(Transform(circle, square))
        self.play(FadeOut(square))


import time
import os
from datetime import datetime
opentime = datetime.now().time()
print('Initalizing setup...')
time.sleep(2)
s1 = open('config.config', "w")
s1.write('localhost = 1\nscripts = scripts\nstart.config\nonlocal=0\nlocalToken="33434An3ikNm3am"\na4a4a4a4a4a4a4a4a4a4a4a4a\ns--------------------------\nbreak()')
s1.close()
print('Config file successfully written.')
time.sleep(1)
print('creating EZlog file...')
s2 = open('EZlog', "w")
s2.write('# log setup\nlogSetup = true\nnil\n')
s2.close()
time.sleep(2)
s3 = open('EZsetvar', 'w')
s3.write('var=1\nvar2\nvar3\nYou shouldnt be reading this file xd')
s3.close()
print('Setting variables..')
time.sleep(2)
print('Creating Directories..')
os.mkdir('EZpack1')
os.mkdir('EZpack2')
os.mkdir('EZpack3')
os.mkdir('EZpack4')
os.mkdir('EZpack5') #Creates Different Packages.
os.mkdir('logKeeper') # The directory that keeps log files
time.sleep(5)
print('Writing config...')
con = open('config.config', "w")
con.write('config.rewrite = true\nEZpack1\nEZpack2\nEZpack3\nEZpack4\nEZpack5\nEZlog\n')
con.close()
time.sleep(0.90)
print('config file rewritten.')
time.sleep(0.50)
print('Initalizing Final Setup..')
coc = open('config.config', "a")
coc.write('config.files.start = 1')
coc.close()

time.sleep(3)
print('Setup Complete! Say ezsetup log for what files were made/deleted')
setchance = input('>_')
if setchance.strip() == 'ezsetup log':
    print(opentime)
    print('created config.config')
    print(opentime)
    print('created EZLog File.')
    print(opentime)
    print('Created EZsetvar')
    print(opentime)
    print('Set 6 Packages, EZpack1 EZpack2 EZpack3 EZpack4 EZpack5 LogKeeper')

elif setchance.strip() == '':
    print('exiting...')
time.sleep(0.60)

print('Exit slip')
exit = input('>')


import torch
from torch import nn
from torch.utils.data import DataLoader, Dataset
from torchvision import datasets, transforms
from torchinfo import summary

import os
import pathlib
import random
from PIL import Image
from pathlib import Path
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from typing import Tuple, Dict, List
from tqdm.auto import tqdm
from timeit import default_timer as timer

def main():
    # Setup device-agnostic code
    device = "cuda" if torch.cuda.is_available() else "cpu"

    # Setup path to data folder
    data_path = Path("data/")
    image_path = data_path / "pizza_steak_sushi"

    # Setup train and testing paths
    train_dir = image_path / "train"
    test_dir = image_path / "test"

    # Set seed
    random.seed(42)

    # 1. Get all image paths (* means "any combination")
    image_path_list = list(image_path.glob("*/*/*.jpg"))

    # 2. Get random image path
    random_image_path = random.choice(image_path_list)

    # 3. Get image class from path name (the image class is the name of the directory where the image is stored)
    image_class = random_image_path.parent.stem

    # 4. Open image
    img = Image.open(random_image_path)

    # 5. Print metadata
    print(f"Random image path: {random_image_path}")
    print(f"Image class: {image_class}")
    print(f"Image height: {img.height}")
    print(f"Image width: {img.width}")

    # Open with local image opener
    # img.show()

    # We can also visualize using matplotlib

    # Turn the image into an array
    img_as_array = np.asarray(img)

    # Plot the image with matplotlib
    plt.figure(figsize=(10, 7))
    plt.imshow(img_as_array)
    plt.title(f"Image class: {image_class} | Image shape: {img_as_array.shape} -> [height, width, color_channels]")
    plt.axis(False)
    plt.show()

    # Write transform for image
    data_transform = transforms.Compose([
        # Resize the images to 64x64
        transforms.Resize(size=(64, 64)),
        # Flip the images randomly on the horizontal
        transforms.RandomHorizontalFlip(p=0.5), # p = probability of flip, 0.5 = 50% chance
        # Turn the image into a torch.Tensor
        transforms.ToTensor() # this also converts all pixel values from 0 to 255 to be between 0.0 and 1.0
    ])

    def plot_transformed_images(image_paths, transform, n=3, seed=42):
        """Plots a series of random images from image_paths.

        Will open n image paths from image_paths, transform them
        with transform and plot them side by side.

        Args:
            image_paths (list): List of target image paths.
            transform (PyTorch Transforms): Transforms to apply to images.
            n (int, optional): Number of images to plot. Defaults to 3.
            seed (int, optional): Random seed for the random generator. Defaults to 42.
        """
        random.seed(seed)
        random_image_paths = random.sample(image_paths, k=n)
        for image_path in random_image_paths:
            with Image.open(image_path) as f:
                fig, ax = plt.subplots(1, 2)
                ax[0].imshow(f)
                ax[0].set_title(f"Original \nSize: {f.size}")
                ax[0].axis("off")

                # Transform and plot image
                # Note: permute() will change shape of image to suit matplotlib
                # (PyTorch default is [C, H, W] but Matplotlib is [H, W, C])
                transformed_image = transform(f).permute(1, 2, 0)
                ax[1].imshow(transformed_image)
                ax[1].set_title(f"Transformed \nSize: {transformed_image.shape}")
                ax[1].axis("off")

                fig.suptitle(f"Class: {image_path.parent.stem}", fontsize=16)
                plt.show()

    plot_transformed_images(image_path_list,
                            transform=data_transform,
                            n=3)

    # Option 1: Loading Image Data Using ImageFolder
    train_data = datasets.ImageFolder(root=train_dir, # target folder of images
                                    transform=data_transform, # transforms to perform on data (images)
                                    target_transform=None) # transforms to perform on labels (if necessary)

    test_data = datasets.ImageFolder(root=test_dir,
                                    transform=data_transform)

    print(f"Train data:\n{train_data}\nTest data:\n{test_data}")

    # Get class names as a list
    class_names = train_data.classes

    img, label = train_data[0][0], train_data[0][1]
    print(f"Image tensor:\n{img}")
    print(f"Image shape: {img.shape}")
    print(f"Image datatype: {img.dtype}")
    print(f"Image label: {label}")
    print(f"Label datatype: {type(label)}")

    # Rearrange the order of dimensions
    img_permute = img.permute(1, 2, 0)

    # Print out different shapes (before and after permute)
    print(f"Original shape: {img.shape} -> [color_channels, height, width]")
    print(f"Image permute shape: {img_permute.shape} -> [height, width, color_channels]")

    # Plot the image
    plt.figure(figsize=(10, 7))
    plt.imshow(img.permute(1, 2, 0))
    plt.axis("off")
    plt.title(class_names[label], fontsize=14)
    # plt.show()

    train_dataloader = DataLoader(dataset=train_data,
                                batch_size=1, # how many samples per batch?
                                num_workers=1, # how many subprocesses to use for data loading? (higher = more)
                                shuffle=True) # shuffle the data?

    test_dataloader = DataLoader(dataset=test_data,
                                batch_size=1,
                                num_workers=1,
                                shuffle=False) # don't usually need to shuffle testing data

    print(train_dataloader, test_dataloader)

    # Option 2: Loading Image Data with a Custom Dataset

    # Make function to find classes in target directory
    def find_classes(directory: str) -> Tuple[List[str], Dict[str, int]]:
        """Finds the class folder names in a target directory.

        Assumes target directory is in standard image classification format.

        Args:
            directory (str): target directory to load classnames from.

        Returns:
            Tuple[List[str], Dict[str, int]]: (list_of_class_names, dict(class_name: idx...))

        Example:
            find_classes("food_images/train")
            >>> (["class_1", "class_2"], {"class_1": 0, ...})
        """
        # 1. Get the class names by scanning the target directory
        classes = sorted(entry.name for entry in os.scandir(directory) if entry.is_dir())

        # 2. Raise an error if class names not found
        if not classes:
            raise FileNotFoundError(f"Couldn't find any classes in {directory}.")

        # 3. Create a dictionary of index labels (computers prefer numerical rather than string labels)
        class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)}
        return classes, class_to_idx

    print(find_classes(train_dir))

    # 1. Subclass torch.utils.data.Dataset
    class ImageFolderCustom(Dataset):

        # 2. Initialize with a targ_dir and transform (optional) parameter
        def __init__(self, targ_dir: str, transform=None) -> None:

            # 3. Create class attributes
            # Get all image paths
            self.paths = list(pathlib.Path(targ_dir).glob("*/*.jpg")) # note: you'd have to update this if you've got .png's or .jpeg's
            # Setup transforms
            self.transform = transform
            # Create classes and class_to_idx attributes
            self.classes, self.class_to_idx = find_classes(targ_dir)

        # 4. Make function to load images
        def load_image(self, index: int) -> Image.Image:
            "Opens an image via a path and returns it."
            image_path = self.paths[index]
            return Image.open(image_path)

        # 5. Overwrite the __len__() method (optional but recommended for subclasses of torch.utils.data.Dataset)
        def __len__(self) -> int:
            "Returns the total number of samples."
            return len(self.paths)

        # 6. Overwrite the __getitem__() method (required for subclasses of torch.utils.data.Dataset)
        def __getitem__(self, index: int) -> Tuple[torch.Tensor, int]:
            "Returns one sample of data, data and label (X, y)."
            img = self.load_image(index)
            class_name  = self.paths[index].parent.name # expects path in data_folder/class_name/image.jpeg
            class_idx = self.class_to_idx[class_name]

            # Transform if necessary
            if self.transform:
                return self.transform(img), class_idx # return data, label (X, y)
            else:
                return img, class_idx # return data, label (X, y)

    # Augment train data
    train_transforms = transforms.Compose([
        transforms.Resize((64, 64)),
        transforms.RandomHorizontalFlip(p=0.5),
        transforms.ToTensor()
    ])

    # Don't augment test data, only reshape
    test_transforms = transforms.Compose([
        transforms.Resize((64, 64)),
        transforms.ToTensor()
    ])

    train_data_custom = ImageFolderCustom(targ_dir=train_dir,
                                        transform=train_transforms)
    test_data_custom = ImageFolderCustom(targ_dir=test_dir,
                                        transform=test_transforms)

    # Check for equality amongst our custom Dataset and ImageFolder Dataset
    print((len(train_data_custom) == len(train_data)) & (len(test_data_custom) == len(test_data)))
    print(train_data_custom.classes == train_data.classes)
    print(train_data_custom.class_to_idx == train_data.class_to_idx)

    # 1. Take in a Dataset as well as a list of class names
    def display_random_images(dataset: torch.utils.data.dataset.Dataset,
                            classes: List[str] = None,
                            n: int = 10,
                            display_shape: bool = True,
                            seed: int = None):

        # 2. Adjust display if n too high
        if n > 10:
            n = 10
            display_shape = False
            print(f"For display purposes, n shouldn't be larger than 10, setting to 10 and removing shape display.")

        # 3. Set random seed
        if seed:
            random.seed(seed)

        # 4. Get random sample indexes
        random_samples_idx = random.sample(range(len(dataset)), k=n)

        # 5. Setup plot
        plt.figure(figsize=(16, 8))

        # 6. Loop through samples and display random samples
        for i, targ_sample in enumerate(random_samples_idx):
            targ_image, targ_label = dataset[targ_sample][0], dataset[targ_sample][1]

            # 7. Adjust image tensor shape for plotting: [color_channels, height, width] -> [color_channels, height, width]
            targ_image_adjust = targ_image.permute(1, 2, 0)

            # Plot adjusted samples
            plt.subplot(1, n, i+1)
            plt.imshow(targ_image_adjust)
            plt.axis("off")
            if classes:
                title = f"class: {classes[targ_label]}"
                if display_shape:
                    title = title + f"\nshape: {targ_image_adjust.shape}"
            plt.title(title)

        plt.show()

    # Display random images from ImageFolder created Dataset
    display_random_images(train_data,
                        n=5,
                        classes=class_names,
                        seed=None)

    # Display random images from ImageFolderCustom Dataset
    display_random_images(train_data_custom,
                        n=12,
                        classes=class_names,
                        seed=None) # Try setting the seed for reproducible images

    # Turn custom loaded images into DataLoader's
    train_dataloader_custom = DataLoader(dataset=train_data_custom, # use custom created train Dataset
                                        batch_size=1, # how many samples per batch?
                                        num_workers=0, # how many subprocesses to use for data loading? (higher = more)
                                        shuffle=True) # shuffle the data?

    test_dataloader_custom = DataLoader(dataset=test_data_custom, # use custom created test Dataset
                                        batch_size=1,
                                        num_workers=0,
                                        shuffle=False) # don't usually need to shuffle testing data

    train_transforms = transforms.Compose([
        transforms.Resize((224, 224)),
        transforms.TrivialAugmentWide(num_magnitude_bins=31), # how intense
        transforms.ToTensor() # use ToTensor() last to get everything between 0 & 1
    ])

    # Don't need to perform augmentation on the test data
    test_transforms = transforms.Compose([
        transforms.Resize((224, 224)),
        transforms.ToTensor()
    ])

    # Get all image paths
    image_path_list = list(image_path.glob("*/*/*.jpg"))

    # Plot random images
    plot_transformed_images(
        image_paths=image_path_list,
        transform=train_transforms,
        n=3,
        seed=None
    )

    # Create simple transform
    simple_transform = transforms.Compose([
        transforms.Resize((64, 64)),
        transforms.ToTensor(),
    ])

    # 1. Load and transform data
    train_data_simple = datasets.ImageFolder(root=train_dir, transform=simple_transform)
    test_data_simple = datasets.ImageFolder(root=test_dir, transform=simple_transform)

    # 2. Turn data into DataLoaders

    # Setup batch size and number of workers
    BATCH_SIZE = 32
    # NUM_WORKERS = os.cpu_count() // 2
    NUM_WORKERS = 0
    print(f"Creating DataLoader's with batch size {BATCH_SIZE} and {NUM_WORKERS} workers.")

    # Create DataLoader's
    train_dataloader_simple = DataLoader(train_data_simple,
                                        batch_size=BATCH_SIZE,
                                        shuffle=True,
                                        num_workers=NUM_WORKERS)

    test_dataloader_simple = DataLoader(test_data_simple,
                                        batch_size=BATCH_SIZE,
                                        shuffle=False,
                                        num_workers=NUM_WORKERS)

    class TinyVGG(nn.Module):
        """
        Model architecture copying TinyVGG from:
        https://poloclub.github.io/cnn-explainer/
        """
        def __init__(self, input_shape: int, hidden_units: int, output_shape: int) -> None:
            super().__init__()
            self.conv_block_1 = nn.Sequential(
                nn.Conv2d(in_channels=input_shape,
                        out_channels=hidden_units,
                        kernel_size=3, # how big is the square that's going over the image?
                        stride=1, # default
                        padding=1), # options = "valid" (no padding) or "same" (output has same shape as input) or int for specific number
                nn.ReLU(),
                nn.Conv2d(in_channels=hidden_units,
                        out_channels=hidden_units,
                        kernel_size=3,
                        stride=1,
                        padding=1),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2,
                            stride=2) # default stride value is same as kernel_size
            )
            self.conv_block_2 = nn.Sequential(
                nn.Conv2d(hidden_units, hidden_units, kernel_size=3, padding=1),
                nn.ReLU(),
                nn.Conv2d(hidden_units, hidden_units, kernel_size=3, padding=1),
                nn.ReLU(),
                nn.MaxPool2d(2)
            )
            self.classifier = nn.Sequential(
                nn.Flatten(),
                # Where did this in_features shape come from?
                # It's because each layer of our network compresses and changes the shape of our inputs data.
                nn.Linear(in_features=hidden_units*16*16,
                        out_features=output_shape)
            )

        def forward(self, x: torch.Tensor):
            # x = self.conv_block_1(x)
            # print(x.shape)
            # x = self.conv_block_2(x)
            # print(x.shape)
            # x = self.classifier(x)
            # print(x.shape)
            # return x
            return self.classifier(self.conv_block_2(self.conv_block_1(x))) # <- leverage the benefits of operator fusion

    torch.manual_seed(42)
    model_0 = TinyVGG(input_shape=3, # number of color channels (3 for RGB)
                    hidden_units=10,
                    output_shape=len(train_data.classes)).to(device)

    print(model_0)

    # 1. Get a batch of images and labels from the DataLoader
    img_batch, label_batch = next(iter(train_dataloader_simple))

    # 2. Get a single image from the batch and unsqueeze the image so its shape fits the model
    img_single, label_single = img_batch[0].unsqueeze(dim=0), label_batch[0]
    print(f"Single image shape: {img_single.shape}\n")

    # 3. Perform a forward pass on a single image
    model_0.eval()
    with torch.inference_mode():
        pred = model_0(img_single.to(device))

    # 4. Print out what's happening and convert model logits -> pred probs -> pred label
    print(f"Output logits:\n{pred}\n")
    print(f"Output prediction probabilities:\n{torch.softmax(pred, dim=1)}\n")
    print(f"Output prediction label:\n{torch.argmax(torch.softmax(pred, dim=1), dim=1)}\n")
    print(f"Actual label:\n{label_single}")

    summary(model_0, input_size=[1, 3, 64, 64]) # do a test pass through of an example input size

    def train_step(model: torch.nn.Module, dataloader: torch.utils.data.DataLoader, loss_fn: torch.nn.Module, optimizer: torch.optim.Optimizer):
        # Put model in train mode
        model.train()

        # Setup train loss and train accuracy values
        train_loss, train_acc = 0, 0

        # Loop through data loader data batches
        for batch, (X, y) in enumerate(dataloader):
            # Send data to target device
            X, y = X.to(device), y.to(device)

            # 1. Forward pass
            y_pred = model(X)

            # 2. Calculate  and accumulate loss
            loss = loss_fn(y_pred, y)
            train_loss += loss.item()

            # 3. Optimizer zero grad
            optimizer.zero_grad()

            # 4. Loss backward
            loss.backward()

            # 5. Optimizer step
            optimizer.step()

            # Calculate and accumulate accuracy metric across all batches
            y_pred_class = torch.argmax(torch.softmax(y_pred, dim=1), dim=1)
            train_acc += (y_pred_class == y).sum().item()/len(y_pred)

        # Adjust metrics to get average loss and accuracy per batch
        train_loss = train_loss / len(dataloader)
        train_acc = train_acc / len(dataloader)
        return train_loss, train_acc

    def test_step(model: torch.nn.Module, dataloader: torch.utils.data.DataLoader, loss_fn: torch.nn.Module):
        # Put model in eval mode
        model.eval()

        # Setup test loss and test accuracy values
        test_loss, test_acc = 0, 0

        # Turn on inference context manager
        with torch.inference_mode():
            # Loop through DataLoader batches
            for batch, (X, y) in enumerate(dataloader):
                # Send data to target device
                X, y = X.to(device), y.to(device)

                # 1. Forward pass
                test_pred_logits = model(X)

                # 2. Calculate and accumulate loss
                loss = loss_fn(test_pred_logits, y)
                test_loss += loss.item()

                # Calculate and accumulate accuracy
                test_pred_labels = test_pred_logits.argmax(dim=1)
                test_acc += ((test_pred_labels == y).sum().item()/len(test_pred_labels))

        # Adjust metrics to get average loss and accuracy per batch
        test_loss = test_loss / len(dataloader)
        test_acc = test_acc / len(dataloader)
        return test_loss, test_acc

    # 1. Take in various parameters required for training and test steps
    def train(model: torch.nn.Module, train_dataloader: torch.utils.data.DataLoader, test_dataloader: torch.utils.data.DataLoader, optimizer: torch.optim.Optimizer, loss_fn: torch.nn.Module = nn.CrossEntropyLoss(), epochs: int = 5):

        # 2. Create empty results dictionary
        results = {"train_loss": [],
            "train_acc": [],
            "test_loss": [],
            "test_acc": []
        }

        # 3. Loop through training and testing steps for a number of epochs
        for epoch in tqdm(range(epochs)):
            train_loss, train_acc = train_step(model=model,
                                            dataloader=train_dataloader,
                                            loss_fn=loss_fn,
                                            optimizer=optimizer)
            test_loss, test_acc = test_step(model=model,
                dataloader=test_dataloader,
                loss_fn=loss_fn)

            # 4. Print out what's happening
            print(
                f"Epoch: {epoch+1} | "
                f"train_loss: {train_loss:.4f} | "
                f"train_acc: {train_acc:.4f} | "
                f"test_loss: {test_loss:.4f} | "
                f"test_acc: {test_acc:.4f}"
            )

            # 5. Update results dictionary
            results["train_loss"].append(train_loss)
            results["train_acc"].append(train_acc)
            results["test_loss"].append(test_loss)
            results["test_acc"].append(test_acc)

        # 6. Return the filled results at the end of the epochs
        return results

    # Set random seeds
    torch.manual_seed(42)

    # Set number of epochs
    NUM_EPOCHS = 5

    # Recreate an instance of TinyVGG
    model_0 = TinyVGG(input_shape=3, # number of color channels (3 for RGB)
                    hidden_units=10,
                    output_shape=len(train_data.classes)).to(device)

    # Setup loss function and optimizer
    loss_fn = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(params=model_0.parameters(), lr=0.001)

    # Start the timer
    from timeit import default_timer as timer
    start_time = timer()

    # Train model_0
    model_0_results = train(model=model_0,
                            train_dataloader=train_dataloader_simple,
                            test_dataloader=test_dataloader_simple,
                            optimizer=optimizer,
                            loss_fn=loss_fn,
                            epochs=NUM_EPOCHS)

    # End the timer and print out how long it took
    end_time = timer()
    print(f"Total training time: {end_time-start_time:.3f} seconds")

    def plot_loss_curves(results: Dict[str, List[float]]):
        """Plots training curves of a results dictionary.

        Args:
            results (dict): dictionary containing list of values, e.g.
                {"train_loss": [...],
                "train_acc": [...],
                "test_loss": [...],
                "test_acc": [...]}
        """

        # Get the loss values of the results dictionary (training and test)
        loss = results['train_loss']
        test_loss = results['test_loss']

        # Get the accuracy values of the results dictionary (training and test)
        accuracy = results['train_acc']
        test_accuracy = results['test_acc']

        # Figure out how many epochs there were
        epochs = range(len(results['train_loss']))

        # Setup a plot
        plt.figure(figsize=(15, 7))

        # Plot loss
        plt.subplot(1, 2, 1)
        plt.plot(epochs, loss, label='train_loss')
        plt.plot(epochs, test_loss, label='test_loss')
        plt.title('Loss')
        plt.xlabel('Epochs')
        plt.legend()

        # Plot accuracy
        plt.subplot(1, 2, 2)
        plt.plot(epochs, accuracy, label='train_accuracy')
        plt.plot(epochs, test_accuracy, label='test_accuracy')
        plt.title('Accuracy')
        plt.xlabel('Epochs')
        plt.legend()
        plt.show()

    plot_loss_curves(model_0_results)

    # Create training transform with TrivialAugment
    train_transform_trivial_augment = transforms.Compose([
        transforms.Resize((64, 64)),
        transforms.TrivialAugmentWide(num_magnitude_bins=31),
        transforms.ToTensor()
    ])

    # Create testing transform (no data augmentation)
    test_transform = transforms.Compose([
        transforms.Resize((64, 64)),
        transforms.ToTensor()
    ])

    # Turn image folders into Datasets
    train_data_augmented = datasets.ImageFolder(train_dir, transform=train_transform_trivial_augment)
    test_data_simple = datasets.ImageFolder(test_dir, transform=test_transform)

    BATCH_SIZE = 32
    NUM_WORKERS = 0

    torch.manual_seed(42)
    train_dataloader_augmented = DataLoader(train_data_augmented,
                                            batch_size=BATCH_SIZE,
                                            shuffle=True,
                                            num_workers=NUM_WORKERS)

    test_dataloader_simple = DataLoader(test_data_simple,
                                        batch_size=BATCH_SIZE,
                                        shuffle=False,
                                        num_workers=NUM_WORKERS)

    # Create model_1 and send it to the target device
    torch.manual_seed(42)
    model_1 = TinyVGG(
        input_shape=3,
        hidden_units=10,
        output_shape=len(train_data_augmented.classes)).to(device)

    # Set random seeds
    torch.manual_seed(42)
    torch.cuda.manual_seed(42)

    # Set number of epochs
    NUM_EPOCHS = 5

    # Setup loss function and optimizer
    loss_fn = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(params=model_1.parameters(), lr=0.001)

    # Start the timer
    from timeit import default_timer as timer
    start_time = timer()

    # Train model_1
    model_1_results = train(model=model_1,
                            train_dataloader=train_dataloader_augmented,
                            test_dataloader=test_dataloader_simple,
                            optimizer=optimizer,
                            loss_fn=loss_fn,
                            epochs=NUM_EPOCHS)

    # End the timer and print out how long it took
    end_time = timer()
    print(f"Total training time: {end_time-start_time:.3f} seconds")

    plot_loss_curves(model_1_results)

    model_0_df = pd.DataFrame(model_0_results)
    model_1_df = pd.DataFrame(model_1_results)
    print(model_0_df)

    # Setup a plot
    plt.figure(figsize=(15, 10))

    # Get number of epochs
    epochs = range(len(model_0_df))

    # Plot train loss
    plt.subplot(2, 2, 1)
    plt.plot(epochs, model_0_df["train_loss"], label="Model 0")
    plt.plot(epochs, model_1_df["train_loss"], label="Model 1")
    plt.title("Train Loss")
    plt.xlabel("Epochs")
    plt.legend()

    # Plot test loss
    plt.subplot(2, 2, 2)
    plt.plot(epochs, model_0_df["test_loss"], label="Model 0")
    plt.plot(epochs, model_1_df["test_loss"], label="Model 1")
    plt.title("Test Loss")
    plt.xlabel("Epochs")
    plt.legend()

    # Plot train accuracy
    plt.subplot(2, 2, 3)
    plt.plot(epochs, model_0_df["train_acc"], label="Model 0")
    plt.plot(epochs, model_1_df["train_acc"], label="Model 1")
    plt.title("Train Accuracy")
    plt.xlabel("Epochs")
    plt.legend()

    # Plot test accuracy
    plt.subplot(2, 2, 4)
    plt.plot(epochs, model_0_df["test_acc"], label="Model 0")
    plt.plot(epochs, model_1_df["test_acc"], label="Model 1")
    plt.title("Test Accuracy")
    plt.xlabel("Epochs")
    plt.legend()
    plt.show()

    # Download custom image
    import requests

    # Setup custom image path
    custom_image_path = data_path / "04-pizza-dad.jpeg"

    # Download the image if it doesn't already exist
    if not custom_image_path.is_file():
        with open(custom_image_path, "wb") as f:
            # When downloading from GitHub, need to use the "raw" file link
            request = requests.get("https://raw.githubusercontent.com/mrdbourke/pytorch-deep-learning/main/images/04-pizza-dad.jpeg")
            print(f"Downloading {custom_image_path}...")
            f.write(request.content)
    else:
        print(f"{custom_image_path} already exists, skipping download.")

    # Read in custom image
    import torchvision
    custom_image_uint8 = torchvision.io.read_image(str(custom_image_path))

    # Print out image data
    print(f"Custom image tensor:\n{custom_image_uint8}\n")
    print(f"Custom image shape: {custom_image_uint8.shape}\n")
    print(f"Custom image dtype: {custom_image_uint8.dtype}")

    # Load in custom image and convert the tensor values to float32
    custom_image = torchvision.io.read_image(str(custom_image_path)).type(torch.float32)

    # Divide the image pixel values by 255 to get them between [0, 1]
    custom_image = custom_image / 255.

    # Print out image data
    print(f"Custom image tensor:\n{custom_image}\n")
    print(f"Custom image shape: {custom_image.shape}\n")
    print(f"Custom image dtype: {custom_image.dtype}")

    # Plot custom image
    plt.imshow(custom_image.permute(1, 2, 0)) # need to permute image dimensions from CHW -> HWC otherwise matplotlib will error
    plt.title(f"Image shape: {custom_image.shape}")
    plt.axis(False)
    plt.show()

    # Create transform pipleine to resize image
    custom_image_transform = transforms.Compose([
        transforms.Resize((64, 64)),
    ])

    # Transform target image
    custom_image_transformed = custom_image_transform(custom_image)

    # Print out original shape and new shape
    print(f"Original shape: {custom_image.shape}")
    print(f"New shape: {custom_image_transformed.shape}")

    model_1.eval()
    with torch.inference_mode():
        # Add an extra dimension to image
        custom_image_transformed_with_batch_size = custom_image_transformed.unsqueeze(dim=0)

        # Print out different shapes
        print(f"Custom image transformed shape: {custom_image_transformed.shape}")
        print(f"Unsqueezed custom image shape: {custom_image_transformed_with_batch_size.shape}")

        # Make a prediction on image with an extra dimension
        custom_image_pred = model_1(custom_image_transformed.unsqueeze(dim=0).to(device))

    # Print out prediction logits
    print(f"Prediction logits: {custom_image_pred}")

    # Convert logits -> prediction probabilities (using torch.softmax() for multi-class classification)
    custom_image_pred_probs = torch.softmax(custom_image_pred, dim=1)
    print(f"Prediction probabilities: {custom_image_pred_probs}")

    # Convert prediction probabilities -> prediction labels
    custom_image_pred_label = torch.argmax(custom_image_pred_probs, dim=1)
    print(f"Prediction label: {custom_image_pred_label}")

    # Find the predicted label
    custom_image_pred_class = class_names[custom_image_pred_label.cpu()] # put pred label to CPU, otherwise will error

    def pred_and_plot_image(model: torch.nn.Module,
                        image_path: str,
                        class_names: List[str] = None,
                        transform=None,
                        device: torch.device = device):
        """Makes a prediction on a target image and plots the image with its prediction."""

        # 1. Load in image and convert the tensor values to float32
        target_image = torchvision.io.read_image(str(image_path)).type(torch.float32)

        # 2. Divide the image pixel values by 255 to get them between [0, 1]
        target_image = target_image / 255.

        # 3. Transform if necessary
        if transform:
            target_image = transform(target_image)

        # 4. Make sure the model is on the target device
        model.to(device)

        # 5. Turn on model evaluation mode and inference mode
        model.eval()
        with torch.inference_mode():
            # Add an extra dimension to the image
            target_image = target_image.unsqueeze(dim=0)

            # Make a prediction on image with an extra dimension and send it to the target device
            target_image_pred = model(target_image.to(device))

        # 6. Convert logits -> prediction probabilities (using torch.softmax() for multi-class classification)
        target_image_pred_probs = torch.softmax(target_image_pred, dim=1)

        # 7. Convert prediction probabilities -> prediction labels
        target_image_pred_label = torch.argmax(target_image_pred_probs, dim=1)

        # 8. Plot the image alongside the prediction and prediction probability
        plt.imshow(target_image.squeeze().permute(1, 2, 0)) # make sure it's the right size for matplotlib
        if class_names:
            title = f"Pred: {class_names[target_image_pred_label.cpu()]} | Prob: {target_image_pred_probs.max().cpu():.3f}"
        else:
            title = f"Pred: {target_image_pred_label} | Prob: {target_image_pred_probs.max().cpu():.3f}"
        plt.title(title)
        plt.axis(False)
        plt.show()

    # Pred on our custom image
    pred_and_plot_image(model=model_1,
                        image_path=custom_image_path,
                        class_names=class_names,
                        transform=custom_image_transform,
                        device=device)

if __name__ == '__main__':
    main()


from unittest.mock import Mock

import pytest

from libpythonproo import github_api

@pytest.fixture
def avatar_url(mocker):
    resp_mock = Mock()
    url = 'https://avatars.githubusercontent.com/u/94086373?v=4'
    resp_mock.json.return_value = {
        'login': 'gustavossandrin', 'id': 94086373, 'node_id': 'U_kgDOBZuk5Q',
        'avatar_url': url
    }
    get_mock = mocker.patch('libpythonproo.github_api.requests.get')
    get_mock.return_value = resp_mock
    return url

def teste_buscar_avatar(avatar_url):
    url = github_api.buscar_avatar('gustavossandrin')
    assert avatar_url == url

def teste_buscar_avatar_intregracao():
    url = github_api.buscar_avatar('gustavossandrin')
    assert 'https://avatars.githubusercontent.com/u/94086373?v=4' == url


from sklearn import datasets
import Point
import Prototype
import random as rd
import copy
import numpy as np

class PointGenerator:

    def __init__(self):
        return

    def digits(self):
        "Generate the digit distribution"
        digit = datasets.load_digits()
        return self.matrixAndLabelsToPoints(digit.data, digit.target)

    def uniformPoints(self,n, dim, a, b):
        #anciennement pointsUniforme
        "Retourne un liste de points générés"
        X=[]
        for i in range(0,n):
            xi = Point.Point(-1,[i for i in range(dim)],30)
            for j in range(dim):
                xi.setCoordinate(rd.uniform(a,b), j)
            xi.setId(i)

            if (xi.getCoordinate(0) < (b+a)/2):
                xi.setLabel(0)
            else :
                xi.setLabel(1)

            X += [xi]
        return X

    def splitDistrib(self, X, f):
        "Splits the distribution in half"
        x1 = X[0:(int)(f*len(X))]
        x2 = X[(int)(f*len(X)):len(X)]
        return x1, x2

    def uniformPointsWithoutLabel(self,n, dim, a, b):
        #anciennement pointsUniformeWithoutLabel
        "Generate a list of points without label"
        X=[]
        for i in range(0,n):
            xi = Point.Point(-1,[i for i in range(dim)],30)
            for j in range(dim):
                xi.setCoordinate(rd.uniform(a,b), j)
            xi.setId(i)

            X += [xi]
        return X

    def putLabelsOnHalfSplittingUniform(self, pointsList, a,b):
        #anciennement putLabelsMoitieUniform
        "Put labels on a separated in half uniform distribution"
        workingList = copy.deepcopy(pointsList)
        j=0
        for i in workingList:
            if (i.getCoordinate(0)<((b+a)/2)):
                i.setLabel(0)
                j+=1
            else :
                i.setLabel(1)
        return workingList

    def matrixAndLabelsToPoints(self, X, labelsList):
        "Generate Point objects from their coordinates and their label"
        points = []
        for i in range(X.shape[0]):
            coordinates = []
            for j in range(X.shape[1]):
                coordinates.append(X[i,j])
            points.append(Point.Point(i, coordinates, labelsList[i]))
        return points

    def pointsToMatrixAndLabels(self, points):
        "Generate the coordinates matrix and label list from Point objects"
        X = np.zeros((len(points), len(points[0].getCoordinates())))
        Y = np.zeros((len(points)))
        for i in range(len(points)):
            for j in range(len(points[i].getCoordinates())):
                X[i,j] = points[i].getCoordinate(j)
            Y[i] = points[i].getLabel()
        return X,Y

    def blobToPoints(self, blobs, labelsList):
        "Generate points forming a blob"
        points = []
        for k in range(len(blobs)):
            X = blobs[k]
            for i in range(X.shape[0]):
                coordinates = []
                for j in range(X.shape[1]):
                    coordinates.append(X[i,j])
                points.append(Point.Point(i, coordinates, labelsList[i]))
        return points

    def generateNoisyCircles(self, n_samples):
        (X, labs) = datasets.make_circles(n_samples=n_samples, factor=.5,noise=.05)
        return self.matrixAndLabelsToPoints(X, labs)

    def generateNoisyMoons(self, n_samples):
        (X, labs) = datasets.make_moons(n_samples=n_samples, noise=.05)
        return self.matrixAndLabelsToPoints(X, labs)

    def generateBlobs(self, n_samples):
        blobs = datasets.make_blobs(n_samples=n_samples, random_state=8)
        print(blobs)
        (X, labs) = blobs
        return self.matrixAndLabelsToPoints(X, labs)

    def generateDeformation(self, nPointsSide, delta, length):
        angle = delta
        nPoints = 2 * nPointsSide + 1
        X = np.zeros((nPoints,2))
        for i in range(nPointsSide):
            x = X[2*i,:]
            X[2*i+2,:] = [x[0] + length * np.sin(angle), x[1] + length * np.cos(angle)]
            X[2*i+1,:] = [X[2*i+2,0], -X[2*i+2,1]]
            angle = angle + delta
        return X

    def generateBlob(self, nPoints):
        rhos = np.random.rand(nPoints)
        thetas = 2 * np.pi * np.random.rand(nPoints)
        x = rhos * np.cos(thetas)
        y = rhos * np.sin(thetas)

        return np.transpose(np.vstack((x,y)))

    def eraseLabels(self, points):
        pointsCopy = copy.deepcopy(points)
        for i in pointsCopy:
            i.setLabel(30)
        return pointsCopy

    def generateWholeDriftProblemByStep(self, nbPoints, thetaMax, nbEtapes):
        listeEtapes = []
        for i in range(nbEtapes):
            theta = (i+1)*thetaMax/nbEtapes
            listeEtapes.append(self.generateWholeDriftProblemTheta(nbPoints, theta))
        return listeEtapes

    def generateWholeDriftProblemTheta(self, nbrePoints, theta = 0):
        nPointsSide = nbrePoints - 1
        radius = 1
        nPointsBlob = nbrePoints
        nPoints = 2 * nPointsSide + 1
        maxAngle = 2 * np.pi / nPoints
        L0 = 1.0 / nPointsSide
        L1 = 2 * radius * np.sin(np.pi / nPoints)
        delta = maxAngle * theta
        length = L0 + (L1 - L0) * theta
        X = self.generateDeformation(nPointsSide, delta, length)
        blob = [1.0, 0.0] + 0.25 * self.generateBlob(nPointsBlob)
        #return blobs, transf, Y
#        print(len(transf[0]))
#        print(len(Y))
#        return (self.matrixAndLabelsToPoints(blobs, Y), self.matrixAndLabelsToPoints(transf, Y))
        return self.matrixAndLabelsToPoints(X, np.zeros((len(X)))) + self.matrixAndLabelsToPoints(blob, np.ones((len(X))))

    def slicePoints(self, X, labels):
        #anciennement decouper_X
        listXByLabels = []
        for label in labels:
            labelI = []
            for i in X:
                if(i.getLabel() == label):
                    labelI.append(i)
            listXByLabels.append(labelI)
        return listXByLabels

    def generateUnevenPrototypes(self, m, X, labels):
        "In that case, m is a list containing the number of prototypes for each label"
        listP = []
        listXByLabel = self.slicePoints(X, labels)
        indexesList = []
        for i in range(len(labels)):
            indexesList.append([])
        maxM=max(m)
        for i in range(len(m)):
            for j in range(len(labels)):
                if i > m[j] - 1:
                    continue
                proto=Prototype.Prototype(-1,[i for i in range(len(X[0].getCoordinates()))],30,[])
                index = rd.randint(0, len(listXByLabel[j])-1)
                while index in indexesList[j]:
                    index = rd.randint(0, len(listXByLabel[j])-1)
                indexesList[j].append(index)
                for k in range(len(listXByLabel[j][index].getCoordinates())):
                    proto.setCoordinate(listXByLabel[j][index].getCoordinate(k),k)
                proto.label = labels[j]
                proto.reseau = []
                proto.id = i + labels[j]*maxM
                listP.append(proto)
        return listP

    def generatePrototypes(self, m, X, labels):
        "There is the same number of prototype for each label"
        listP = []
        listXByLabel = self.slicePoints(X, labels)
        indexesList = []
        for l in range(len(labels)):
            indexesList.append([])
        for i in range(m):
            for j in range(len(labels)):
                proto=Prototype.Prototype(-1,[i for i in range(len(X[0].getCoordinates()))],30,[])
                index = rd.randint(0, len(listXByLabel[j])-1)
                while index in indexesList[j]:
                    index = rd.randint(0, len(listXByLabel[j])-1)
                indexesList[j].append(index)
                for k in range(len(listXByLabel[j][index].getCoordinates())):
                    proto.setCoordinate(listXByLabel[j][index].getCoordinate(k),k)
                proto.label = labels[j]
                proto.network = []
                proto.id = i + labels[j]*m
                listP.append(proto)
        return listP


from netCDF4 import Dataset as NetCDFFile

from mpas_tools.planar_hex import make_planar_hex_mesh
from mpas_tools.io import write_netcdf
from mpas_tools.mesh.conversion import convert, cull
from mpas_tools.logging import check_call

from compass.model import make_graph_file
from compass.step import Step

class SetupMesh(Step):
    """
    A step for creating a mesh and initial condition for enthalpy benchmark
    test cases
    """
    def __init__(self, test_case):
        """
        Create the step

        Parameters
        ----------
        test_case : compass.TestCase
            The test case this step belongs to
        """
        super().__init__(test_case=test_case, name='setup_mesh')
        self.add_output_file(filename='graph.info')
        self.add_output_file(filename='landice_grid.nc')

    # no setup() method is needed

    def run(self):
        """
        Run this step of the test case
       """
        logger = self.logger
        section = self.config['enthalpy_benchmark']
        nx = section.getint('nx')
        ny = section.getint('ny')
        dc = section.getfloat('dc')
        levels = section.get('levels')

        dsMesh = make_planar_hex_mesh(nx=nx, ny=ny, dc=dc, nonperiodic_x=True,
                                      nonperiodic_y=True)

        write_netcdf(dsMesh, 'grid.nc')

        dsMesh = cull(dsMesh, logger=logger)
        dsMesh = convert(dsMesh, logger=logger)
        write_netcdf(dsMesh, 'mpas_grid.nc')

        args = ['create_landice_grid_from_generic_MPAS_grid.py',
                '-i', 'mpas_grid.nc',
                '-o', 'landice_grid.nc',
                '-l', levels,
                '--thermal']

        check_call(args, logger)

        make_graph_file(mesh_filename='landice_grid.nc',
                        graph_filename='graph.info')

        _setup_initial_conditions(section, 'landice_grid.nc')

def _setup_initial_conditions(section, filename):
    """ Add the initial conditions for enthalpy benchmark A """
    thickness = section.getfloat('thickness')
    basal_heat_flux = section.getfloat('basal_heat_flux')
    surface_air_temperature = section.getfloat('surface_air_temperature')
    temperature = section.getfloat('temperature')

    with NetCDFFile(filename, 'r+') as gridfile:
        thicknessVar = gridfile.variables['thickness']
        bedTopography = gridfile.variables['bedTopography']
        basalHeatFlux = gridfile.variables['basalHeatFlux']
        surfaceAirTemperature = gridfile.variables['surfaceAirTemperature']
        temperatureVar = gridfile.variables['temperature']

        thicknessVar[:] = thickness
        bedTopography[:] = 0
        basalHeatFlux[:] = basal_heat_flux
        surfaceAirTemperature[:] = surface_air_temperature
        temperatureVar[:] = temperature


import datetime
import sys

now = datetime.datetime.now()
today = now.strftime('%Y-%m-%d')
timestamp = now.strftime('%Y%m%d %H:%M:%S')
short_title = sys.argv[1] if len(sys.argv) > 1 else 'post'
path = '_posts/%s-%s.md' % (today, short_title)

print(f"path: {path}")

header = """
---
layout: post
title:  "%s"
date:  %s +0800
categories: default
tags:
 - blogging
---
""" % (short_title, timestamp)

with open(path, "w+") as f:
    f.write(header)

if len(sys.argv) == 1:
    print('***WARN: default title is used***')


import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
#from scipy import fft, arange
import numpy as np
from pylab import figure
import const as c

def to_plot(out_buffers, inp_signal_buff):

    figure(1)
    ax1 = plt.subplot(611)
    ax1.plot(c.t, inp_signal_buff)
    ax1.set_title('Вход приемника (сигнал + помехи)  ' + str("fs = ") +
                  str(c.fs) + " Hz")
    ax1.grid(True)

    ax2 = plt.subplot(612, sharex=ax1)
    ax2.plot(c.t, out_buffers[0])
    ax2.set_title('Сигнал на выходе канального фильтра (400 - 800 Гц)  ' +
                  str("fs = ") + str(c.fs) + " Hz")
    ax2.grid(True)

    ax3 = plt.subplot(613)
    ax3.plot(c.t2, out_buffers[1])
    ax3.plot(c.t2, out_buffers[2])
    ax3.set_title('Сигнал на выходе когерентного детектора  ' + str("fs = ") +
                  str(c.fs2) + " Hz")
    ax3.grid(True)

    ax5 = plt.subplot(614)
    ax5.plot(c.t2, out_buffers[3], label='8Hz')
    ax5.plot(c.t2, out_buffers[4], label='12Hz')
    ax5.set_title('Сигналы на выходах фильтров 8 и 12 Гц ' + str("fs = ") +
                  str(c.fs2) + " Hz")
    ax5.grid(True)
    ax5.legend()

    ax7 = plt.subplot(615)
    ax7.plot(c.t2, out_buffers[5], label='8Hz')
    ax7.plot(c.t2, out_buffers[6], label='12Hz')
    ax7.plot(c.t2, out_buffers[7], label='8Hz')
    ax7.plot(c.t2, out_buffers[8], label='12Hz')
    ax7.set_title('Сигналы на входах\выходах компараторов ' + str("fs = ") +
                  str(c.fs2) + " Hz")
    ax7.grid(True)
    ax7.legend()

    ax11 = plt.subplot(616)
    ax11.plot(c.t2, out_buffers[9], label='уровень сигнал\шум 8 Гц')
    ax11.plot(c.t2, out_buffers[10], label='уровень сигнал\шум 12 Гц')
    plt.ylim(0, 40)
    ax11.set_title('Индикатор сигнал/шум в канале (разницы\
     уровней в каналах 8 и 12 Гц '+ str("fs = ")\
     + str(c.fs2) + " Hz")
    ax11.grid(True)
    ax11.legend()

def plotSpectrum(y):
    """
	Function to plot the time domain and frequency domain signal
	"""
    figure(2)
    plt.ylim(0, 100)
    plt.xlim(0, 2000)
    plt.grid(True)
    #plt.legend()
    plt.magnitude_spectrum(y, Fs=c.fs, scale='dB')
    plt.ylabel('Уровень (dB)')
    plt.xlabel('Частота (Hz)')


#Size of the points reduced on purpose to run faster

#Important modules
import csv
from plotly.graph_objs import Scattergeo, Layout
from plotly import offline
from datetime import datetime

#Explore the structure of the data
filename = "data/world_fires_7_day.csv"

with open(filename) as f:
	reader = csv.reader(f)
	header_row = next(reader)
	print(header_row)

	#Extracting fire information
	brightness, lons, lats, daynight, date = [], [], [], [], []

	#Look for the relevant information
	lat_index = header_row.index("latitude")
	lon_index = header_row.index("longitude")
	bright_index = header_row.index("bright_ti4")
	dn_index = header_row.index("daynight")
	day_index = header_row.index("acq_date")

	for row in reader:

		try:
			bright = float(row[bright_index])
			lat = float(row[lat_index])
			lon = float(row[lon_index])
			dn = row[dn_index]
			day = datetime.strptime(row[day_index], "%Y-%m-%d")
		except:
			print(f"N/A")
		else:
			lats.append(lat)
			lons.append(lon)
			brightness.append(bright)
			daynight.append(dn)
			date.append(day.date())

data = [{
		'type': 'scattergeo',
		"lon":lons,
		"lat": lats,
		"text": daynight,
		"marker": {
			"size":[brightn/100 for brightn in brightness],
			"color": [brightn/100 for brightn in brightness],
			"colorscale": "Hot",
			"reversescale": True,
			"colorbar": {"title": "Brightness/100"},
			},
		}]

my_layout = Layout(title= "Global Fires 2018")

fig = {"data": data, "layout": my_layout}
offline.plot(fig, filename = "global_fires.html")

f.close()


#!/usr/bin/env python3
import requests
import json
from bs4 import BeautifulSoup
import os
import socket
from time import sleep
import sys

def wait_till_online():
    try:
        host = socket.gethostbyname("tagesschau.de")
    except:
        sleep(1)
        print("offline")
        wait_till_online()

def get_latest(links):
    return  sorted(links)[-1]

def scrape_links(url):
    page = requests.get(url)
    soup = BeautifulSoup(page.content, 'html.parser')
    # players = soup.find_all('div',  {"class": "ts-mediaplayer ts-mediaplayer--einszueins ts-mediaplayer--list"})

    players = soup.find_all('div',  {"class": "ts-mediaplayer"})
    links = []
    for player_item in players:
        attribs = player_item.attrs
        data = attribs["data-config"]
        try:
            title = json.loads(data)["mc"]["_title"]
            if "tsde" in title or title == "Ganze Sendung":
                if "Gebärdensprache" not in data:
                    links.append(json.loads(data)["mc"]['_mediaArray'][0]["_mediaStreamArray"][4]["_stream"])
        except KeyError:
            pass
    return links

player = "castnow"
if len(sys.argv) > 1:
    player = sys.argv[1]
url = "https://www.tagesschau.de/multimedia/"
wait_till_online()
links = scrape_links(url)
url = get_latest(links)
cmd = f"{player} {url}"
os.system(cmd)


from rest_framework import serializers
from datetime import date,timedelta
from authen.models import *
from mahalla.models import *

class CategoriyaPeopleSerializers(serializers.ModelSerializer):
    class Meta:
        model = Categoriya
        fields = ['id','name']

class UserSektorSerializers(serializers.ModelSerializer):
    class Meta:
        model = CustumUsers
        fields = ['id','first_name','last_name','position','phone']

class RegionAllSerializers(serializers.ModelSerializer):
    class Meta:
        model = Region
        fields = ['id','name']

class MahallaAllSerializers(serializers.ModelSerializer):
    class Meta:
        model = Mahalla
        fields = ['id','name','id_sektor']

class SektorSerializers(serializers.ModelSerializer):
    class Meta:
        model = Sektor
        fields = ['id','name']

class DistirckSerializers(serializers.ModelSerializer):
    id_sektor = SektorSerializers(read_only=True,many=True)
    class Meta:
        model = District
        fields = ['id','name','id_sektor']

class TaslCategoriyaSerializers(serializers.ModelSerializer):
    class Meta:
        model = TaskCategoriya
        fields = '__all__'

class TaskPeopleSerializers(serializers.ModelSerializer):
    id_user = UserSektorSerializers(read_only=True)
    id_task_category = TaslCategoriyaSerializers(read_only=True)
    class Meta:
        model = Tasks
        fields = '__all__'

class PeopleAllSerializers(serializers.ModelSerializer):
    id_categor = CategoriyaPeopleSerializers(read_only=True)
    responsible_employee = UserSektorSerializers(read_only=True)
    id_mahalla = MahallaAllSerializers(read_only=True)
    id_region = RegionAllSerializers(read_only=True)
    id_sektor = SektorSerializers(read_only=True)
    id_districk = DistirckSerializers(read_only=True)
    people = TaskPeopleSerializers(many=True,read_only=True)
    class Meta:
        model = People
        fields = ('id','full_name','birth_date','phone','village','additional_information','id_categor','id_mahalla','id_sektor','id_region','id_districk','responsible_employee','create_user','people')

class PeopleCreateSerializers(serializers.ModelSerializer):
    class Meta:
        model = People
        fields = ['full_name','birth_date','phone','village','additional_information','id_categor','id_mahalla','id_sektor','id_region','id_districk','responsible_employee','create_user',]
    def create(self, validated_data):
        user_create = People.objects.create(**validated_data)
        user_create.id_mahalla = self.context.get('id_mahalla')
        user_create.id_sektor = self.context.get('id_sektor')
        user_create.id_region = self.context.get('id_region')
        user_create.id_districk = self.context.get('id_districk')
        user_create.create_user = self.context.get('create_user')
        user_create.save()
        return user_create

class TaskSerializers(serializers.ModelSerializer):
    class Meta:
        model = TaskCategoriya
        fields = ['id','name']

class TaskUser(serializers.ModelSerializer):
    class Meta:
        model = CustumUsers
        fields = ['id','first_name','last_name','phone','position']

class UserSerializersAll(serializers.ModelSerializer):
    class Meta:
        model = CustumUsers
        fields = ['id','first_name','last_name']

class TaskAllSerializers(serializers.ModelSerializer):
    id_user = TaskUser(read_only=True)
    class Meta:
        model = Tasks
        fields = '__all__'

class TaskCrudSerializers(serializers.ModelSerializer):
    class Meta:
        model = Tasks
        fields = ['id','id_people','id_task_category','id_user','task','is_user','comment','date_line']
    def create(self, validated_data):
        task_create = Tasks(
            id_task_category = validated_data['id_task_category'],
            id_user = validated_data['id_user'],
            task = validated_data['task'],
            comment = validated_data['comment']
        )
        task_create.id_people = self.context.get('id_people')
        for item in TaskCategoriya.objects.all():
            if validated_data['id_task_category'] == item:
                x = date.today()+timedelta(days=item.date)
        task_create.date_line = x
        task_create.save()
        return task_create

class TaskCreateFiles(serializers.ModelSerializer):
    class Meta:
        model = Tasks
        fields = ['id','id_people','files','is_user','is_task']
    def update(self, instance, validated_data):
        instance.files = validated_data.get('files',instance.files)
        instance.is_user = validated_data.get('is_user',instance.is_user)
        instance.save()
        return instance


class DeploymentInfoRepository(object):

    def __init__(self, model_constructor, control_node_repo, license_repo, sensor_repo):
        self.__model_constructor = model_constructor
        self.__control_node_repository = control_node_repo
        self.__license_repository = license_repo
        self.__sensor_repository = sensor_repo

    def get_deployment_info(self):
        info = self.__model_constructor(
            self.__control_node_repository.get_control_node(),
            self.__license_repository.get_license(),
            self.__sensor_repository.get_sensors()
        )

        return info


#!/usr/bin/env python3

import socket
from xml.dom import minidom
import sys
HOST = '0.0.0.0'  # The server's hostname or IP address
PORT = 12345        # The port used by the server
PORT_IN = 23456
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

f=open(sys.argv[1], "r")
if f.mode == 'r':
    contents =f.read()

class Connection:
        def send_to_server(self):

            s.connect((HOST, PORT))
            #s.sendall(b'<create><symbol sym="AAA"><account id="666">100</account></symbol></create>')
            s.sendall(bytes(contents, 'utf-8'))
            #data = s.recv(1024)
            #s.sendall(b'<transactions id="10"><order sym="BBB" amount="1000" limit="100"/></transactions>')
        def receive(self):
            xml = ''

            data = s.recv(10240)

            xml += str(data,'utf-8')
                #conn.sendall(data)
            print(minidom.parseString(xml).toprettyxml(indent = "    "))

def main():
    con = Connection()
    con.send_to_server()
    con.receive()

if __name__ == "__main__":
    main()


from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
from enum import Enum

class SummaryModel:

    def __init__(self, model_type: str, **kwargs):
        # I used this construction method to make it easier to add new models and model types in the future.
        if model_type in Models.__members__.keys():
            self.model = Models[model_type].value(**kwargs)
        else:
            raise Exception(f"Model {model_type} not abailable.\nAvailable models are: {Models.__members__.keys()}")

    def get_summary(self, text):
        return self.model.get_summary(text)

class SummaryModelTest(SummaryModel):
    def __init__(self):
        pass

    def get_summary(self, text):
        summary = f"Test summary of string {text}"
        return summary

class SummaryModelTransformer(SummaryModel):
    def __init__(self, hf_model_name: str = "sshleifer/distilbart-cnn-12-6"):
        self.tokenizer = AutoTokenizer.from_pretrained(hf_model_name)
        self.model = AutoModelForSeq2SeqLM.from_pretrained(hf_model_name)

    def get_summary(self, text):
        assert type(text) == str, f"Input text must be a string but found {type(text)}"
        inputs = self.tokenizer.encode(text, return_tensors="pt", max_length=512)
        outputs = self.model.generate(inputs, max_length=150, min_length=40, length_penalty=2.0, num_beams=4,
                                      early_stopping=True)
        summary = self.tokenizer.batch_decode(outputs)[0]
        return summary[7:-4]  # remove useless characters given as output by the tokenizer

class Models(Enum):
    TRANSFORMERS = SummaryModelTransformer
    # NLTK = SummaryModelTest
    TEST = SummaryModelTest

if __name__ == '__main__':
    # simple AI model test
    text = "this is a test text from which we want to extract a good summary!"
    summary_model = SummaryModel('TRANSFORMERS', hf_model_name="sshleifer/distilbart-cnn-12-6")
    # summary_model = SummaryModel('nothing')
    summary = summary_model.get_summary(text)
    print(summary)


import torch
import torch.nn as nn
import torch.nn.functional as F

from .conv2d_cd import Conv2d_cd
from .dropout import Dropout

__all__ = ['mobilenetv2']

def _make_divisible(v, divisor, min_value=None):
    """
    This function is taken from the original tf repo.
    It ensures that all layers have a channel number that is divisible by 8
    It can be seen here:
    https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py
    :param v:
    :param divisor:
    :param min_value:
    :return:
    """
    if min_value is None:
        min_value = divisor
    new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)
    # Make sure that round down does not go down by more than 10%.
    if new_v < 0.9 * v:
        new_v += divisor
    return new_v

def conv_3x3_bn(inp, oup, stride, theta):
    return nn.Sequential(
        Conv2d_cd(inp, oup, 3, stride, 1, bias=False, theta=theta),
        nn.BatchNorm2d(oup),
        nn.ReLU6(inplace=True)
    )

def conv_3x3_in(inp, oup, stride, theta):
    return nn.Sequential(
        Conv2d_cd(inp, oup, 3, stride, 1, bias=False, theta=theta),
        nn.InstanceNorm2d(oup),
        nn.ReLU6(inplace=True)
    )

def conv_1x1_bn(inp, oup):
    return nn.Sequential(
        nn.Conv2d(inp, oup, 1, 1, 0, bias=False),
        nn.BatchNorm2d(oup),
        nn.ReLU6(inplace=True)
    )

def conv_1x1_in(inp, oup):
    return nn.Sequential(
        nn.Conv2d(inp, oup, 1, 1, 0, bias=False),
        nn.InstanceNorm2d(oup),
        nn.ReLU6(inplace=True)
    )

class InvertedResidual(nn.Module):
    def __init__(self, inp, oup, stride, expand_ratio,
                 prob_dropout, type_dropout, sigma, mu, theta):
        super().__init__()
        assert stride in [1, 2]
        hidden_dim = round(inp * expand_ratio)
        self.identity = stride == 1 and inp == oup
        self.dropout2d = Dropout(dist=type_dropout, sigma=sigma, mu=mu, p=prob_dropout)
        if expand_ratio == 1:
            self.conv = nn.Sequential(
                # dw
                Conv2d_cd(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False, theta=theta),
                nn.BatchNorm2d(hidden_dim),
                nn.ReLU6(inplace=True),
                # pw-linear
                nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),
                nn.BatchNorm2d(oup),
            )
        else:
            self.conv = nn.Sequential(
                # pw
                nn.Conv2d(inp, hidden_dim, 1, 1, 0, bias=False),
                nn.BatchNorm2d(hidden_dim),
                nn.ReLU6(inplace=True),
                # dw
                Conv2d_cd(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False, theta=theta),
                nn.BatchNorm2d(hidden_dim),
                nn.ReLU6(inplace=True),
                # pw-linear
                nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),
                nn.BatchNorm2d(oup),
            )

    def forward(self, x):
        if self.identity:
            return x + self.dropout2d(self.conv(x))
        else:
            return self.dropout2d(self.conv(x))

class MobileNetV2(nn.Module):
    def __init__(self, width_mult=1., prob_dropout=0.1, type_dropout='bernoulli',
                 prob_dropout_linear=0.5, embeding_dim=1280, mu=0.5, sigma=0.3,
                 theta=0, multi_heads=True, scaling=1):
        super().__init__()
        # setting of inverted residual blocks
        self.multi_heads = multi_heads
        self.scaling = scaling
        self.prob_dropout_linear = prob_dropout_linear
        self.cfgs = [
            # t, c, n, s
            [1,  16, 1, 1],
            [6,  24, 2, 2],
            [6,  32, 3, 2],
            [6,  64, 4, 2],
            [6,  96, 3, 1],
            [6, 160, 3, 2],
            [6, 320, 1, 1],
        ]

        # building first layer
        input_channel = _make_divisible(32 * width_mult, 4 if width_mult == 0.1 else 8)
        layers = [conv_3x3_bn(3, input_channel, 2, theta=theta)]
        # building inverted residual blocks
        block = InvertedResidual
        for t, c, n, s in self.cfgs:
            output_channel = _make_divisible(c * width_mult, 4 if width_mult == 0.1 else 8)
            for i in range(n):
                layers.append(block(input_channel, output_channel,
                                    s if i == 0 else 1, t,
                                    prob_dropout=prob_dropout,
                                    type_dropout=type_dropout,
                                    mu=mu, sigma=sigma, theta=theta))
                input_channel = output_channel
        self.features = nn.Sequential(*layers)
        # building last several layers
        self.conv_last = conv_1x1_bn(input_channel, embeding_dim)
        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        self.spoofer = nn.Linear(embeding_dim, 2)
        if self.multi_heads:
            self.lightning = nn.Linear(embeding_dim, 5)
            self.spoof_type = nn.Linear(embeding_dim, 11)
            self.real_atr = nn.Linear(embeding_dim, 40)

    def forward(self, x):
        x = self.features(x)
        x = self.conv_last(x)
        return x

    def forward_to_onnx(self,x):
        x = self.features(x)
        x = self.conv_last(x)
        x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        spoof_out = self.spoofer(x)
        if isinstance(spoof_out, tuple):
            spoof_out = spoof_out[0]
        probab = F.softmax(spoof_out*self.scaling, dim=-1)
        return probab

    def make_logits(self, features):
        output = self.avgpool(features)
        output = output.view(output.size(0), -1)
        spoof_out = self.spoofer(output)
        if self.multi_heads:
            type_spoof = self.spoof_type(output)
            lightning_type = self.lightning(output)
            real_atr = torch.sigmoid(self.real_atr(output))
            return spoof_out, type_spoof, lightning_type, real_atr
        return spoof_out

    def spoof_task(self, features):
        output = self.avgpool(features)
        output = output.view(output.size(0), -1)
        spoof_out = self.spoofer(output)
        return spoof_out

def mobilenetv2(**kwargs):
    """
    Constructs a MobileNet V2 model
    """
    return MobileNetV2(**kwargs)


from code.skipgram import SkipGram
from code.utils import get_test_set
from sklearn.decomposition import PCA
import numpy as np
import copy

def evaluate_w2v(w2v, sentences):
    """
    Parameters:
    - w2v (SkipGram): Pre-trained SkipGram
    - sentences (list): Clean evaluation data
    Output:
    - errors (np.array): MAE between predicted similarity and ground-truth
    """
    # Get predictions and labels
    predictions = np.array([simil(w2v, sent) for sent in sentences])
    labels = np.array([float(sent[2]) for sent in sentences])

    # Remove None
    idx = np.where(predictions != None)

    # Predictions and labels
    predictions = predictions[idx]
    labels = labels[idx]

    # Compute mean error
    errors = np.abs(predictions*10 - labels)

    return(errors)

def pca_w2v(w2v, n_components):
    """
    Apply PCA on the embeddings on a SkipGram object.
    Parameters:
    - w2v (SkipGram): Pre-trained SkipGram
    - n_components (int): Number of components to keep
    Output:
    - new_w2v (SkipGram)
    """
    new_w2v = copy.deepcopy(w2v)
    pca = PCA(n_components)
    new_w2v.w1 = pca.fit_transform(new_w2v.w1.T)
    new_w2v.w1 = new_w2v.w1.T
    return(new_w2v)

def most_similar(sg, word):
    """
    Make it a static method
    """
    results = {}
    for w in sg.word2idx.keys():
        results[w] = sg.similarity(word, w)

    results = dict(sorted(results.items(), key=lambda x: x[1], reverse=True))

    return(results)

def simil(w2v, sent):
    try:
        return(w2v.similarity(sent[0], sent[1]))
    except:
        return(None)