from decimal import Decimal
from datetime import datetime
from bokeep.book_transaction import \
    Transaction, FinancialTransaction, FinancialTransactionLine, \
    BoKeepTransactionNotMappableToFinancialTransaction

ZERO = Decimal(0)
NEG_1 = Decimal(-1)

class TrustTransaction(Transaction):
    def __init__(self, trust_module):
        self.trust_module = trust_module
        self.transfer_amount = Decimal(0)
        if len(trust_module.get_trustors().values()) == 0:
            self.set_trustor(None)
        else:
            self.set_trustor(trust_module.get_trustors().values()[0])
        self.memo = ''
        self.trans_date = datetime.today()
        self.set_id(-1)

    def get_financial_transactions(self):
        # you should throw BoKeepTransactionNotMappableToFinancialTransaction
        # under some conditions
        cash_line = FinancialTransactionLine(self.get_transfer_amount())
        if hasattr(self.trust_module, 'cash_account'):
            cash_line.account_spec = self.trust_module.cash_account

        #use the cash line's memo field for the memo
        cash_line.line_memo = self.get_memo()

        liability_line = \
            FinancialTransactionLine(self.get_transfer_amount() * NEG_1)

        if hasattr(self.trust_module, 'trust_liability_account'):
            if self.get_trustor() != None:
                liability_line.create_account_if_missing = True
                liability_line.account_spec = \
                    self.trust_module.trust_liability_account + \
                    (self.get_trustor().name,)
            # else we rely on the failure due to account_spec being missing
            # should really throw
            # BoKeepTransactionNotMappableToFinancialTransaction
            # instead
        # else ditto as above...
        fin_trans = FinancialTransaction( (cash_line, liability_line) )

        if self.get_trustor() != None:
            fin_trans.description = self.get_trustor().name
        fin_trans.trans_date = self.trans_date
        # If a previous version without an ID is being used, don't attempt to
        # set the checknum with an ID that doesn't even exist.
        # Remove the if line once we're at version 1.2.0.
        if hasattr(self, '_TrustTransaction__id'):
            fin_trans.chequenum = self.__id
        fin_trans.currency = self.trust_module.get_currency()
        # should add chequenum at some point, legal aid requested this
        # for ordering
        return ( fin_trans, )

    def get_transfer_amount(self):
        return self.transfer_amount

    def set_trustor(self, trustor):
        from bokeep.plugins.trust import Trustor
        assert(trustor == None or trustor.__class__ == Trustor)
        self.trustor = trustor

    def set_id(self, id):
        """Sets the ID of this trust transaction.  It's used as the ID in the
        backend, or as the GnuCash plugin would call it, the chequenum."""
        self.__id = id

    def get_trustor(self):
        """Returns the trustor (object) associated with this transaction."""
        return self.trustor

    def get_displayable_amount(self):
        return TrustTransaction.get_transfer_amount(self)

    def get_memo(self):
        # Remove "if line" and just return "self.memo" once backwards
        # compatibility with BoKeep 1.0.2's transaction database is no longer
        # desired.  Perhaps at BoKeep 1.2.0.
        if hasattr(self, 'memo'):
            return self.memo
        else:
            return ''

class TrustMoneyInTransaction(TrustTransaction):
    pass

class TrustMoneyOutTransaction(TrustTransaction):
    def get_transfer_amount(self):
        # when Trust money is paid out, the cash line needs to be negative
        # (credit) and the liability line needs to be positive (debit)
        # see TrustTransaction.get_financial_transactions
        return TrustTransaction.get_transfer_amount(self) * NEG_1


"""Tests for the audit endpoints"""

import bson
from fastapi.testclient import TestClient
from pymongo import MongoClient
from auth_checker import AuthChecker
from util import db_connect
from main import app
from util.authorization import get_authorization
from tests.override_get_authorization import override_get_authorization

client = TestClient(app)
app.dependency_overrides[get_authorization] = override_get_authorization

class TestAuditController:
    def mock_mongo_client(self, collection_name):
        """Mock a MongoDB database"""
        return MongoClient(
            "mongodb://localhost:27017").clearance_service[collection_name]

    def test_search_actions(self, monkeypatch):
        """Tests the search_actions endpoint"""
        monkeypatch.setattr(db_connect,
                            "get_clearance_collection",
                            self.mock_mongo_client)
        monkeypatch.setattr(AuthChecker,
                            "check_authorization",
                            lambda *_, **__: None)

        ca_collection = self.mock_mongo_client("audit")

        audit_records = [
            {
                "_id": bson.ObjectId(),
                "assigner_id": "test_assigner",
                "assignee_id": "test_assignee",
                "clearance_id": None,
                "timestamp": 0,
                "message": "test_message",
            }
        ]

        ca_collection.insert_many(audit_records)

        response = client.get("/audit/",
                              headers={"Authorization": "Bearer token"})

        assert response.status_code == 200

    def test_search_actions_by_assigner_pagination(self, monkeypatch):
        """Test the search_actions_by_assigner endpoint with pagination"""
        monkeypatch.setattr(db_connect,
                            "get_clearance_collection",
                            self.mock_mongo_client)
        monkeypatch.setattr(AuthChecker,
                            "check_authorization",
                            lambda *_, **__: None)

        ca_collection = self.mock_mongo_client("audit")

        audit_records = [
            {
                "_id": bson.ObjectId(),
                "assigner_id": "test_assigner",
                "assignee_id": "test_assignee",
                "clearance_id": None,
                "timestamp": 0,
                "message": "test_message",
            }
        ]

        ca_collection.insert_many(audit_records)

        response = client.get("/audit/?limit=1",
                              headers={"Authorization": "Bearer token"})

        assert response.status_code == 200


import torch
import torch.nn as nn
from sklearn import metrics
from scipy.stats import exponweib
from utils.question_pair import MAX_LEN, VOC_SIZE

def pairs_auc(test_pairs, all_scores):
  """
  calculate the auc scores
  """
  pair2scores = {}
  for i,pair in enumerate(test_pairs):
    pair2scores[pair] =  all_scores[i]
  sorted_pairs = sorted(test_pairs, key = lambda x:pair2scores[x])
  pred = sorted(all_scores)
  y = [2 - pair.is_duplicate  for pair in sorted_pairs]
  fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=2)
  return metrics.auc(fpr, tpr)

def model_scorer(cand, ref, model, tokenizer, MAX_LEN = MAX_LEN):
  """
  This function assign a average of the mean cross entropy of
  1) generating ref conditioned on cand
  2) generating cand conditioned on ref
  """
  model.eval()
  model = model.to('cuda')
  criterion = nn.CrossEntropyLoss(ignore_index=tokenizer.pad_token_id, reduction = 'mean')
  all_scores = []
  tokens1 = tokenizer(ref, max_length = MAX_LEN,
                              truncation = True, padding = "max_length",
                              add_special_tokens = True, return_token_type_ids = False,\
                              return_attention_mask = False, return_tensors = 'pt')['input_ids']
  tokens2 = tokenizer(cand, max_length = MAX_LEN,
                              truncation = True, padding = "max_length",
                              add_special_tokens = True, return_token_type_ids = False,\
                              return_attention_mask = False, return_tensors = 'pt')['input_ids']
  tokens1 = tokens1.to('cuda')
  tokens2 = tokens2.to('cuda')
  num_batch = tokens1.size(0)//32
  if tokens1.size(0) % 32 != 0:
        num_batch += 1
  for j in range(num_batch):
    x1 = tokens1[j*32:j*32+32,:]
    x2 = tokens2[j*32:j*32+32,:]
    with torch.no_grad():
        batch_size = x1.size(0)
#         losses = model(x1, x2).cpu().detach().numpy()
        conditional_logits_1 = model(x1, x2).reshape(batch_size,-1, VOC_SIZE)
        conditional_logits_2 = model(x2, x1).reshape(batch_size,-1, VOC_SIZE)
        losses = []
        for i in range(batch_size):
          losses_1 = criterion(conditional_logits_1[i], x2[i])
          losses_2 = criterion(conditional_logits_2[i], x1[i])
          losses.append((losses_1 + losses_2)/2)

    all_scores.append(torch.Tensor(losses).reshape(batch_size,-1))
  all_scores = torch.cat(all_scores, dim = 0)
  return all_scores

def bert_scorer(cand, ref, model, tokenizer, MAX_LEN = MAX_LEN):
  """
  use the naive model (basic bert or roberta model) to assign scores to each cand-ref pair
  """
  model.eval()
  model = model.to('cuda')
  all_scores = []
  sf = nn.Softmax(dim = -1)
  tokens = tokenizer(cand, ref, max_length = MAX_LEN*2,
                              truncation = True, padding = "max_length",
                              add_special_tokens = True, return_token_type_ids = False,\
                              return_attention_mask = False, return_tensors = 'pt')['input_ids']

  tokens = tokens.to('cuda')
  num_batch = tokens.size(0)//32
  if tokens.size(0) % 32 != 0:
        num_batch += 1
  for j in range(num_batch):
    x = tokens[j*32:j*32+32,:]
    with torch.no_grad():
        batch_size = x.size(0)
        outputs = sf(model(x).reshape(batch_size, 2))[:,1]
    all_scores.append(outputs.reshape(batch_size,-1))
  all_scores = torch.cat(all_scores, dim = 0)
  return all_scores.cpu()

def distribution_scorer(cand, ref, model, tokenizer):
  """
  Returns the cross_entropy measure by the model of
  generating (cand concat ref) without a condition
  """
  model.eval()
  model = model.to('cuda')
  criterion = nn.CrossEntropyLoss(ignore_index=tokenizer.pad_token_id, reduction = 'mean')
  all_scores = []
  empty_ref = ['' for c in cand]
  tokens1 = tokenizer(empty_ref, max_length = MAX_LEN*2,
                              truncation = True, padding = "max_length",
                              add_special_tokens = True, return_token_type_ids = False,\
                              return_attention_mask = False, return_tensors = 'pt')['input_ids']
  tokens2 = tokenizer(cand, ref, max_length = MAX_LEN*2,
                              truncation = True, padding = "max_length",
                              add_special_tokens = True, return_token_type_ids = False,\
                              return_attention_mask = False, return_tensors = 'pt')['input_ids']
  tokens1 = tokens1.to('cuda')
  tokens2 = tokens2.to('cuda')
  num_batch = tokens1.size(0)//32
  if tokens1.size(0) % 32 != 0:
        num_batch += 1
  for j in range(num_batch):
    x1 = tokens1[j*32:j*32+32,:]
    x2 = tokens2[j*32:j*32+32,:]
    with torch.no_grad():
        batch_size = x1.size(0)
        conditional_logits_1 = model(x1, x2).reshape(batch_size,-1, VOC_SIZE)
        losses = []
        for i in range(batch_size):
          losses_1 = criterion(conditional_logits_1[i], x2[i])
          losses.append(losses_1 )

    all_scores.append(torch.Tensor(losses).reshape(batch_size,-1))
  all_scores = torch.cat(all_scores, dim = 0)
  return all_scores

def robust_predictions(positive_model, negative_model, distribution_model,\
                       dev_pairs, test_pairs, tokenizer, discriminative_model= None, C = 3):
    """
    this method assembles all the models to make robust predictions for paraphrase identification
    the outputs are numerical scores, to make classification results, compare the scores with 0,
    >0 will be paraphrases
    """
    thresh = .1
    #first fit a weibull distribution to dev set
    candidates1 = [pair.question1 for pair in dev_pairs]
    candidates2 = [pair.question2 for pair in dev_pairs]
    benchmark_scores = distribution_scorer(candidates1, candidates2, distribution_model, tokenizer)
    a,c, loc, scale = exponweib.fit(benchmark_scores)

    candidates1 = [pair.question1 for pair in test_pairs]
    candidates2 = [pair.question2 for pair in test_pairs]
    #calculate the lambda for each testing sample (called distribution_weights)
    distribution_scores = distribution_scorer(candidates1, candidates2, distribution_model, tokenizer)
    distribution_weights = 1 - torch.Tensor(exponweib.cdf(distribution_scores,a,c, loc, scale))
    #thresholding
    distribution_weights = torch.where(distribution_weights>thresh, 1, distribution_weights)

    #calculate the scores from the positive and negative model
    positive_scores = model_scorer(candidates1, candidates2, positive_model, tokenizer)
    negative_scores = model_scorer(candidates1, candidates2, negative_model, tokenizer)

    #assemble all the scores
    scores = positive_scores - distribution_weights.reshape(-1,1)* negative_scores \
    - ( 1 - distribution_weights.reshape(-1,1))*C

    #calculate the scores from the discriminative model
    if discriminative_model is not None:
        discriminative_scores = bert_scorer(candidates1, candidates2, discriminative_model, tokenizer)
        discriminative_scores = (.5 - discriminative_scores )*1000
        scores += (distribution_weights > thresh)*discriminative_scores
    return scores


import os
import numpy as np
from glob import glob
import open3d as o3d
import argparse
import time
import pyquaternion

# https://github.com/intel-isl/Open3D/issues/2#issuecomment-610683341
def text_3d(text, pos, direction=None, degree=0.0, font="OpenSans-Regular.ttf", font_size=16):
    """
    Generate a 3D text point cloud used for visualization.
    :param text: content of the text
    :param pos: 3D xyz position of the text upper left corner
    :param direction: 3D normalized direction of where the text faces
    :param degree: in plane rotation of text
    :param font: Name of the font - change it according to your system
    :param font_size: size of the font
    :return: o3d.geoemtry.PointCloud object
    """
    if direction is None:
        direction = (0., 0., 1.)

    from PIL import Image, ImageFont, ImageDraw
    from pyquaternion import Quaternion

    font_obj = ImageFont.truetype(font, font_size)
    font_dim = font_obj.getsize(text)

    img = Image.new('RGB', font_dim, color=(255, 255, 255))
    draw = ImageDraw.Draw(img)
    draw.text((0, 0), text, font=font_obj, fill=(0, 0, 0))
    img = np.asarray(img)
    img_mask = img[:, :, 0] < 128
    indices = np.indices([*img.shape[0:2], 1])[:, img_mask, 0].reshape(3, -1).T

    pcd = o3d.geometry.PointCloud()
    pcd.colors = o3d.utility.Vector3dVector(img[img_mask, :].astype(float) / 255.0)
    pcd.points = o3d.utility.Vector3dVector(indices / 100.0)

    raxis = np.cross([0.0, 0.0, 1.0], direction)
    if np.linalg.norm(raxis) < 1e-6:
        raxis = (0.0, 0.0, 1.0)
    trans = (Quaternion(axis=raxis, radians=np.arccos(direction[2])) *
             Quaternion(axis=direction, degrees=degree)).transformation_matrix
    trans[0:3, 3] = np.asarray(pos)
    pcd.transform(trans)
    return pcd

def visualize_pcd_auto_play(cfg):
    WINDOW_WIDTH = cfg.window_width  # change this if needed
    WINDOW_HEIGHT = cfg.window_height  # change this if needed
    file_type = cfg.file_type
    ply_base_dir = cfg.ply_folder
    ply_files = [pf for pf in os.listdir(ply_base_dir) if pf.endswith(file_type)]
    if not len(ply_files) > 0:
        print("empty directory, exiting...")
        return
    # first pcd file
    source_pcd = o3d.io.read_point_cloud(os.path.join(ply_base_dir, ply_files[0]))
    min_pos = np.min(source_pcd.points, axis=0)
    max_pos = np.max(source_pcd.points, axis=0)
    shift_diff_pos = (max_pos - min_pos) / 10
    text_pos = min_pos - shift_diff_pos
    text_pcd = text_3d(text="idx: {} | file: {}".format(0, ply_files[0]), pos=text_pos)
    tmp_pcd_text_points = np.vstack([source_pcd.points, text_pcd.points])
    source_pcd.points = o3d.utility.Vector3dVector(tmp_pcd_text_points)
    vis = o3d.visualization.Visualizer()
    vis.create_window(width=WINDOW_WIDTH, height=WINDOW_HEIGHT)
    vis.add_geometry(source_pcd)
    ctr = vis.get_view_control()
    parameters = o3d.io.read_pinhole_camera_parameters(cfg.camera_view)
    ctr.convert_from_pinhole_camera_parameters(parameters)
    for idx in range(1, len(ply_files)):
        tmp_pcd = o3d.io.read_point_cloud(os.path.join(ply_base_dir, ply_files[idx]))
        min_pos = np.min(tmp_pcd.points, axis=0)
        max_pos = np.max(tmp_pcd.points, axis=0)
        shift_diff_pos = (max_pos - min_pos) / 10
        text_pos = min_pos - shift_diff_pos
        text_pcd = text_3d(text="idx: {} | file: {}".format(0, ply_files[0]), pos=text_pos)
        tmp_pcd_text_points = np.vstack([tmp_pcd.points, text_pcd.points])
        source_pcd.points = o3d.utility.Vector3dVector(tmp_pcd_text_points)
        vis.update_geometry(source_pcd)
        vis.poll_events()
        vis.update_renderer()
        time.sleep(cfg.pause_time)

def visualize_pcd_with_key_bind(cfg):
    WINDOW_WIDTH = cfg.window_width  # change this if needed
    WINDOW_HEIGHT = cfg.window_height  # change this if needed
    file_type = cfg.file_type
    ply_base_dir = cfg.ply_folder
    ply_files = [pf for pf in os.listdir(ply_base_dir) if pf.endswith(file_type)]
    if not len(ply_files) > 0:
        print("empty directory, exiting...")
        return
    source_pcd = o3d.io.read_point_cloud(os.path.join(ply_base_dir, ply_files[0]))
    min_pos = np.min(source_pcd.points, axis=0)
    max_pos = np.max(source_pcd.points, axis=0)
    shift_diff_pos = (max_pos - min_pos) / 10
    text_pos = min_pos - shift_diff_pos
    text_pcd = text_3d(text="idx: {} | file: {}".format(0, ply_files[0]), pos=text_pos)
    tmp_pcd_text_points = np.vstack([source_pcd.points, text_pcd.points])
    source_pcd.points = o3d.utility.Vector3dVector(tmp_pcd_text_points)
    num_ply_file = len(ply_files)
    rotating = False
    pointer = 0
    vis = o3d.visualization.VisualizerWithKeyCallback()

    def key_action_callback(vis, action, mods):
        nonlocal rotating
        # print(action)
        # print(o3d.visualization.gui.KeyEvent.type)
        if action == 1:  # key down
            rotating = True
        elif action == 0:  # key up
            rotating = False
        return True

    def animation_callback(vis):
        nonlocal rotating
        nonlocal pointer
        nonlocal num_ply_file
        nonlocal ply_base_dir
        nonlocal ply_files
        if rotating:
            pointer += 1
            cur_ply_f_idx = pointer % num_ply_file
            tmp_pcd = o3d.io.read_point_cloud(os.path.join(ply_base_dir, ply_files[cur_ply_f_idx]))
            min_pos = np.min(tmp_pcd.points, axis=0)
            max_pos = np.max(tmp_pcd.points, axis=0)
            shift_diff_pos = (max_pos - min_pos) / 10
            text_pos = min_pos - shift_diff_pos
            text_pcd = text_3d(text="idx: {} | file: {}".format(cur_ply_f_idx, ply_files[cur_ply_f_idx]), pos=text_pos)
            tmp_pcd_text_points = np.vstack([tmp_pcd.points, text_pcd.points])
            source_pcd.points = o3d.utility.Vector3dVector(tmp_pcd_text_points)
            vis.update_geometry(source_pcd)
            vis.poll_events()
            vis.update_renderer()
            rotating = False

    # key_action_callback will be triggered when there's a keyboard press, release or repeat event
    vis.register_key_action_callback(32, key_action_callback)  # space

    # animation_callback is always repeatedly called by the visualizer
    vis.register_animation_callback(animation_callback)

    vis.create_window(width=WINDOW_WIDTH, height=WINDOW_HEIGHT)
    vis.add_geometry(source_pcd)
    vis.run()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Ply Visualizer')
    parser.add_argument('--keyboard', type=bool, default=False,
                        help='Use Keyboard interaction or auto-play.')
    parser.add_argument('--ply_folder', type=str, default="./ply",
                        help='the folder of ply files')
    parser.add_argument('--file_type', type=str, default=".ply",
                        help='the folder of ply files')
    parser.add_argument('--window_width', type=int, default="640",
                        help='the width of visualization window')
    parser.add_argument('--pause_time', type=float, default="0.2",
                        help='time to sleep for each frame, only effective in auto-play mode')
    parser.add_argument('--window_height', type=int, default="480",
                        help="the height of visualization window")
    parser.add_argument('--camera_view', type=str, default="ScreenCamera_2021-06-29-14-56-27.json",
                        help='The screen camera pose json file, if you don\'t have one record it with key P when you are in an open3d window')
    args = parser.parse_args()
    if args.keyboard:
        visualize_pcd_with_key_bind(args)
    else:
        visualize_pcd_auto_play(args)


n1, n2, n3 = map(int, input('Digite três números: ').split())
def maior(x, y, z):
  if x > y and x > z:
    m = x
  elif y > x and y > z:
    m = y
  else:
    m = z
  return m
print(maior(n1, n2, n3))


import re
import logging
from ..index_tests import key_fields
from faker import Faker
from copy import deepcopy

LOGGER = logging.getLogger("pytest-splunk-addon")
host_ipv4, dvc_ipv4 = 51, 0
src_ipv4, dest_ipv4 = 0, 0
host_ipv6, dvc_ipv6 = 0, 0
src_ipv6, dest_ipv6 = 0, 0
host_count, fqdn_count = 0, 0
url_ip_count = 0
host_ipv4_octet_count, dvc_ipv4_octet_count = -1, 0

ip_rules = {
    "src": {
        "ipv4": "10.1.",
        "ipv6": "fdee:1fe4:2b8c:3261",
    },
    "dest": {
        "ipv4": "10.100.",
        "ipv6": "fdee:1fe4:2b8c:3262",
    },
    "dvc": {
        "ipv4": "172.16.",
        "ipv6": "fdee:1fe4:2b8c:3263",
    },
    "host": {
        "ipv4": "172.16.",
        "ipv6": "fdee:1fe4:2b8c:3264",
    },
    "url": {
        "ip_host": "192.168.",
    },
}

class SampleEvent(object):
    """
    This class represents an event which will be ingested in Splunk.

    Args:
        event_string (str): Event content
        metadata (dict): Contains metadata for the event
        sample_name (str): Name of the file containing this event
    """

    def __init__(self, event_string, metadata, sample_name, requirement_test_data=None):
        self.event = event_string
        self.key_fields = dict()
        self.time_values = list()
        self.metadata = metadata
        self.sample_name = sample_name
        self.host_count = 0
        self.requirement_test_data = requirement_test_data

    def update(self, new_event):
        """
        This method updates the event content

        Args:
            new_event (str): Event content
        """
        LOGGER.debug("Updated the event {} with {}".format(self.event, new_event))
        self.event = new_event

    def get_host(self):
        """
        Returns a unique host value
        """
        global host_count
        host_count += 1
        LOGGER.debug(
            "Creating host value: {}-{}-{}".format(
                "host", self.sample_name, str(host_count)
            )
        )
        return "{}-{}-{}".format("host", self.sample_name, str(host_count))

    def get_field_host(self, rule):
        """
        Returns unique host value for the key fields src, dest, host, dvc

        Args:
            rule (str): Type of rule either src, host, dest, dvc
        """
        global host_count
        host_count += 1
        LOGGER.debug(
            "Creating field with value: {}-{}{}".format(rule, "sample_host", host_count)
        )
        return "{}-{}{}".format(rule, "sample_host", host_count)

    def get_field_fqdn(self, rule):
        """
        Returns unique fqdn value for the key fields src, dest, host, dvc

        Args:
            rule (str): Type of rule either src, host, dest, dvc
        """
        global fqdn_count
        fqdn_count += 1
        LOGGER.debug(
            "Creating fgdn field with value: {}_{}.{}{}.com".format(
                rule, "sample_host", "sample_domain", fqdn_count
            )
        )
        return "{}_{}.{}{}.com".format(rule, "sample_host", "sample_domain", fqdn_count)

    def get_ipv4(self, rule):
        """
        Returns Ipv4 Address as per the rule.

        Args:
            rule (str): Type of rule either src, host, dest, dvc.
            If the value is not one of the key field it will return a randomly generated Ipv4 address.
        """
        if rule == "src":
            global src_ipv4
            src_ipv4 += 1
            addr = [int(src_ipv4 / 256) % 256, src_ipv4 % 256]
            LOGGER.debug(
                "Creating ipv4 field with value: {}".format(
                    "".join(
                        [ip_rules.get(rule)["ipv4"], str(addr[0]), ".", str(addr[1])]
                    )
                )
            )
            return "".join(
                [ip_rules.get(rule)["ipv4"], str(addr[0]), ".", str(addr[1])]
            )
        elif rule == "host":
            global host_ipv4, host_ipv4_octet_count
            host_ipv4_octet_count += 1
            if host_ipv4_octet_count > 255:
                host_ipv4 += 1
                host_ipv4_octet_count = host_ipv4_octet_count % 256
            if host_ipv4 == 101:
                host_ipv4 = 51
            LOGGER.debug(
                "Creating ipv4 field with value: {}".format(
                    "".join(
                        [
                            ip_rules.get(rule)["ipv4"],
                            str(host_ipv4 % 101),
                            ".",
                            str(host_ipv4_octet_count % 256),
                        ]
                    )
                )
            )
            return "".join(
                [
                    ip_rules.get(rule)["ipv4"],
                    str(host_ipv4 % 101),
                    ".",
                    str(host_ipv4_octet_count % 256),
                ]
            )
        elif rule == "dvc":
            global dvc_ipv4, dvc_ipv4_octet_count
            dvc_ipv4 += 1
            dvc_ipv4_octet_count += 1
            LOGGER.debug(
                "Creating ipv4 field with value: {}".format(
                    "".join(
                        [
                            ip_rules.get(rule)["ipv4"],
                            str(dvc_ipv4 % 51),
                            ".",
                            str(dvc_ipv4_octet_count % 256),
                        ]
                    )
                )
            )
            return "".join(
                [
                    ip_rules.get(rule)["ipv4"],
                    str(dvc_ipv4 % 51),
                    ".",
                    str(dvc_ipv4_octet_count % 256),
                ]
            )
        elif rule == "dest":
            global dest_ipv4
            dest_ipv4 += 1
            addr = [int(dest_ipv4 / 256) % 256, dest_ipv4 % 256]
            LOGGER.debug(
                "Creating ipv4 field with value: {}".format(
                    "".join(
                        [ip_rules.get(rule)["ipv4"], str(addr[0]), ".", str(addr[1])]
                    )
                )
            )
            return "".join(
                [ip_rules.get(rule)["ipv4"], str(addr[0]), ".", str(addr[1])]
            )
        elif rule == "url":
            global url_ip_count
            url_ip_count += 1
            addr = [int(url_ip_count / 256) % 256, url_ip_count % 256]
            LOGGER.debug(
                "Creating ipv4 field with value: {}".format(
                    "".join(
                        [ip_rules.get(rule)["ip_host"], str(addr[0]), ".", str(addr[1])]
                    )
                )
            )
            return "".join(
                [ip_rules.get(rule)["ip_host"], str(addr[0]), ".", str(addr[1])]
            )
        else:
            temp_ipv4 = Faker().ipv4()
            LOGGER.debug("Creating ipv4 field with value: {}".format(temp_ipv4))
            return temp_ipv4

    def get_ipv6(self, rule):
        """
        Returns Ipv6 Address as per the rule.

        Args:
            rule (str): Type of rule either src, host, dest, dvc.
            If the value is not one of the key field it will return a randomly generated Ipv6 address.
        """
        if rule == "src":
            global src_ipv6
            ipv6 = src_ipv6 % (int("ffffffffffffffff", 16))
            src_ipv6 += 1
        elif rule == "host":
            global host_ipv6
            ipv6 = host_ipv6 % (int("ffffffffffffffff", 16))
            host_ipv6 += 1
        elif rule == "dvc":
            global dvc_ipv6
            ipv6 = dvc_ipv6 % (int("ffffffffffffffff", 16))
            dvc_ipv6 += 1
        elif rule == "dest":
            global dest_ipv6
            ipv6 = dest_ipv6 % (int("ffffffffffffffff", 16))
            dest_ipv6 += 1
        else:
            temp_ipv4 = Faker().ipv6()
            LOGGER.debug("Creating ipv6 field with value: {}".format(temp_ipv4))
            return temp_ipv4

        hex_count = hex(ipv6)
        non_zero_cnt = len(hex_count[2:])
        addr = "{}{}".format("0" * (16 - non_zero_cnt), hex_count[2:])
        LOGGER.debug(
            "Creating ipv6 field with value: {}:{}".format(
                ip_rules.get(rule)["ipv6"],
                ":".join(addr[i : i + 4] for i in range(0, len(addr), 4)),
            )
        )
        return "{}:{}".format(
            ip_rules.get(rule)["ipv6"],
            ":".join(addr[i : i + 4] for i in range(0, len(addr), 4)),
        )

    def get_token_count(self, token):
        """
        Returns the token count in event

        Args:
            token (str): Token name
        """
        return len(re.findall(token, self.event, flags=re.MULTILINE))

    def get_token_extractions_count(self, token):
        """
        Returns minimum number of occurrence count if token not found in event but is in extracted fields

        Args:
            token (str): Token name
        """
        tokens_in_extractions = 0
        if (
            self.requirement_test_data is not None
            and "cim_fields" in self.requirement_test_data.keys()
        ):
            for extracted_field in self.requirement_test_data["cim_fields"].values():
                if isinstance(extracted_field, str):
                    tokens_in_extractions += len(re.findall(token, extracted_field))
                elif isinstance(extracted_field, list):
                    for each_filed in extracted_field:
                        tokens_in_extractions += len(re.findall(token, each_filed))
        return 1 if tokens_in_extractions > 0 else 0

    def replace_token(self, token, token_values):
        """
        Replaces the token value in event

        Args:
            token (str): Token name
            token_values (list/str): Value(s) to be replaced in the token
        """
        # TODO: How to handle dependent Values with list of token_values
        if isinstance(token_values, list):
            sample_tokens = re.finditer(token, self.event, flags=re.MULTILINE)

            for _, token_value in enumerate(token_values):
                token_value = token_value.value
                match_object = next(sample_tokens)
                match_str = (
                    match_object.group(0)
                    if len(match_object.groups()) == 0
                    else match_object.group(1)
                )
                match_str = re.escape(match_str)
                self.event = re.sub(
                    match_str,
                    lambda x: str(token_value),
                    self.event,
                    1,
                    flags=re.MULTILINE,
                )
        else:
            self.event = re.sub(
                token, lambda x: str(token_values), self.event, flags=re.MULTILINE
            )

    def register_field_value(self, field, token_values):
        """
        Registers the value for the key fields in its SampleEvent object

        Args:
            field (str): Token field name
            token_values (list/str): Token value(s) which are replaced in the key fields
        """
        if field == "_time":
            time_list = (
                token_values if isinstance(token_values, list) else [token_values]
            )
            self.time_values.extend([i.key for i in time_list])
        elif field in key_fields.KEY_FIELDS:
            if isinstance(token_values, list):
                for token_value in token_values:
                    self.key_fields.setdefault(field, []).append(str(token_value.key))
            else:
                self.key_fields.setdefault(field, []).append(str(token_values.key))

    def update_requirement_test_field(self, field, token, token_values):
        if field != "_time":
            if (
                self.requirement_test_data is not None
                and "cim_fields" in self.requirement_test_data.keys()
            ):
                for cim_field, value in self.requirement_test_data[
                    "cim_fields"
                ].items():
                    if token in value:
                        if isinstance(token_values, list):
                            if len(token_values) == 1:
                                self.requirement_test_data["cim_fields"][
                                    cim_field
                                ] = value.replace(token, str(token_values[0].key))
                            else:
                                self.requirement_test_data["cim_fields"][cim_field] = [
                                    value.replace(token, str(token_value.key))
                                    for token_value in token_values
                                ]
                        else:
                            self.requirement_test_data["cim_fields"][
                                cim_field
                            ] = value.replace(token, str(token_values.key))

    def get_key_fields(self):
        """
        Returns the key field value from event
        """
        return self.key_fields

    @classmethod
    def copy(cls, event):
        """
        Copies the SampleEvent object into a new one.
        Args:
            event (SampleEvent): Event object which has to be copied

        Returns:
            Copy of the SampleEvent object
        """
        new_event = cls("", {}, "")
        new_event.__dict__ = event.__dict__.copy()
        new_event.key_fields = event.key_fields.copy()
        new_event.time_values = event.time_values[:]
        new_event.metadata = deepcopy(event.metadata)
        new_event.requirement_test_data = deepcopy(event.requirement_test_data)
        return new_event

    def update_metadata(self, event, metadata, key_fields):
        """
        Processes the syslog formated samples
        Format::

            '***SPLUNK*** source=<source> sourcetype=<sourcetype> \
            field_1       field2        field3 \
            ##value1##    ##value2##   ##value3##'

        Args:
            event (str): event string containing raw syslog data
            metadata (dict): Contains metadata for the event

        Returns:
            Syslog event and the updated metadata
        """
        try:
            if isinstance(event, str) and event.startswith("***SPLUNK***"):
                header, event = event.split("\n", 1)

                for meta_field in re.findall(r"[\w]+=[^\s]+", header):
                    field, value = meta_field.split("=")
                    if field == "host":
                        metadata[field] = f"host_{metadata[field]}"
                        key_fields["host"] = list([metadata["host"]])
                    else:
                        metadata[field] = value

            return event, metadata, key_fields

        except KeyError as error:
            LOGGER.error(f"Unexpected data found. Error: {error}")
            raise error


from bs4 import BeautifulSoup
import requests
import pandas as pd
import pymongo
import time
import numpy as np
from splinter import Browser
from webdriver_manager.chrome import ChromeDriverManager

def scrape():

    # Run splinter executable
    executable_path = {'executable_path': ChromeDriverManager().install()}
    browser = Browser('chrome', **executable_path, headless=False)

    # Scrape the NASA Mars News Site and collect the latest News Title and Paragraph Text.
    # Assign the text to variables for later reference.

    # URL of page to be scraped
    url = 'https://mars.nasa.gov/news/'

    # Retrieve page in splinter browser
    browser.visit(url)

    time.sleep(5)

    # Scrape page into Soup
    html = browser.html
    soup = BeautifulSoup(html, "lxml")

    # Retrieve the parent divs for all headlines and preview text
    results = soup.find_all("div", class_="list_text")

    # Create empty lists to append with headlines and preview text data
    headers = list()
    preview_texts = list()

    # loop over results to get headlines and preview text data
    for result in results:
        # scrape the article header
        header = result.find("div", class_="content_title").text

        # scrape the article subheader
        preview_text = result.find("div", class_="article_teaser_body").text

        # print article data
        print('-----------------')
        print(header)
        print(preview_text)

        # Append lists with headlines and preview texts
        headers.append(header)
        preview_texts.append(preview_text)

    # URL of JPL page to be scraped
    jpl_url = "https://data-class-jpl-space.s3.amazonaws.com/JPL_Space/index.html"

    # Retrieve page in splinter browser
    browser.visit(jpl_url)

    time.sleep(5)

    # HTML object
    jpl_html = browser.html

    # Retrieve the parent divs for all articles
    jpl_soup = BeautifulSoup(jpl_html, 'lxml')

    # Retrieve featured image relative URL
    featured_image = jpl_soup.find("img", class_="headerimage")["src"]

    # Clean up the webpage URL to combine with
    # featured image relative URL to create full URL
    clean_url = jpl_url.split("index.html")

    # Concatenate URLs to create full featured image URL
    featured_image_url = clean_url[0] + featured_image

    print(featured_image_url)

    # URL of Space Facts page to be scraped
    facts_url = "https://space-facts.com/mars/"

    # Get all tables in URL via pandas
    tables = pd.read_html(facts_url)

    # Save relevant table as DataFrame and export to html
    df = tables[0]
    df = df.rename(columns={0:"Description",1:"Mars"})
    mars_table = df.to_html(index=False)

    # URL of Mars Hemispheres to be scraped
    hemi_url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"

    # Retrieve page in splinter browser
    browser.visit(hemi_url)

    # HTML object
    hemi_html = browser.html
    hemi_soup = BeautifulSoup(hemi_html, 'lxml')

    # Retrieve the parent divs for all articles
    hemi_results = hemi_soup.find_all("div", class_="item")

    # Retrieve all hemisphere links as a list
    hemispheres = list()

    for r in hemi_results:
        hemisphere = r.find("h3").text
        hemi_split = hemisphere.split(" Enhanced")
        hemi_clean = hemi_split[0]
        hemispheres.append(hemi_clean)

    hemisphere_image_urls = list()

    for hemi in hemispheres:
        try:
            # click on links with partial match "Enhanced"
            time.sleep(2)

            browser.click_link_by_partial_text(f"{hemi}")

            # get full image webpage URL and save as BeautifulSoup
            hemi_image_html = browser.html
            image_soup = BeautifulSoup(hemi_image_html, 'lxml')

            # find full image name and URL
            title = image_soup.find("h2", class_="title").text
            img_url = image_soup.find("div", class_="downloads").find("a")["href"]

            # remove word "Enhanced" from hemisphere name
            title_split = title.split(" Enhanced")
            title_clean = title_split[0]

            # print to check the code works
            print(title_clean)
            print(img_url)

            # append list with dictionary
            hemisphere_image_urls.append({"title":title_clean, "img_url":img_url})

            time.sleep(2)

            # click back
            browser.back()

        # stop code if all images scraped
        except ElementDoesNotExist:
            print("Scraping Complete")

    # Store data in a dictionary
    mars_data = {
        "headers": headers,
        "preview_texts": preview_texts,
        "featured_image_url": featured_image_url,
        "mars_table": mars_table,
        "hemispheres": hemispheres,
        "hemisphere_image_urls": hemisphere_image_urls,
    }

    # Close the browser after scraping
    browser.quit()

    # Return results
    return mars_data


# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

import os
import sys

from six.moves import xrange

from collections import Mapping

import tensorflow as tf

from polyaxon.datasets.converters.base import BaseConverter

class ImageReader(object):
    """Base ImageReader class that provides an operation to read/encode/decode an image."""
    def __init__(self, channels=3):
        self._placeholder = tf.placeholder(dtype=tf.string)
        self._image = self.decoder(channels)

    def decoder(self, channels):
        raise NotImplementedError

    def read(self, session, image_data, processing_fn=None):
        _image = self._image
        if processing_fn:
            _image = processing_fn(session, _image)
        return image_data, session.run(_image, feed_dict={self._placeholder: image_data})

class PNGImageReader(ImageReader):
    """A png image class reader"""
    def decoder(self, channels):
        return tf.image.decode_png(self._placeholder, channels=channels)

class PNGNumpyImageReader(ImageReader):
    """A numpy png image class reader"""

    def __init__(self, shape=None):
        self._placeholder = tf.placeholder(dtype=tf.uint8, shape=shape)
        self._image = tf.image.encode_png(self._placeholder)

    def decoder(self, channels):
        pass

    def read(self, session, image_data, processing_fn=None):
        _, image = super(PNGNumpyImageReader, self).read(session, image_data, processing_fn)
        return image, image

class JPGNumpyImageReader(ImageReader):
    """A jpeg numpy image class reader"""
    def __init__(self, shape=None):
        self._placeholder = tf.placeholder(dtype=tf.uint8, shape=shape)
        self._image = tf.image.encode_jpeg(self._placeholder)

    def decoder(self, channels):
        pass

    def read(self, session, image_data, processing_fn=None):
        _, image = super(JPGNumpyImageReader, self).read(session, image_data, processing_fn)
        return image, image

class JPEGImageReader(ImageReader):
    """A jpeg image class reader"""
    def decoder(self, channels):
        return tf.image.decode_jpeg(self._placeholder, channels=channels)

class ImagesToTFExampleConverter(BaseConverter):
    """Converts images to a TFRecords of TF-Example protos.

    Each record is TF-Example protocol buffer which contain a single image and label.

    Args:
        classes: `dict` or `list`. The data classes.
            e.g. ['zero', 'one', 'two', ...] or {0: 'cats', 1: 'dogs'}
        colorspace: `str`. The color space of the images, 'rgb', 'grayscale' ...
        image_format: `str`. The format of the images, 'png', 'jpeg'.
        channels: `int`. The number of channels of the images, e.g. 1, 3.
        image_reader: `ImageReader` instance. If `None` an image reader is created automatically
            based on the `image_format`.
        height: `int`. If provided the the height per image will not be stored in the TFRecord,
            only in the meta_data file.
        width: `int`. If provided the the width per image will not be stored in the TFRecord,
            only in the meta_data file.
        store_filenames: `bool`. If `True` the filename of the image will be stored as a feature.
    """
    def __init__(self, classes, colorspace, image_format, channels, image_reader=None,
                 height=None, width=None, store_filenames=False):
        if image_reader:
            self.image_reader = image_reader
        else:
            if image_format == 'png':
                self.image_reader = PNGImageReader(channels=channels)
            elif image_format == 'jpeg':
                self.image_reader = JPEGImageReader(channels=channels)

        if isinstance(classes, Mapping):
            self.classes = list(classes.values())
            self.labels_to_classes = classes
        elif isinstance(classes, list):
            assert len(set(classes)) == len(classes), '`classes` must contain unique values.'
            self.classes = classes
            self.labels_to_classes = dict(zip(range(len(classes)), classes))
        else:
            raise TypeError('`classes` type is unsupported, received `{}`'.format(type(classes)))

        self.classes_to_labels = {v: k for k, v in self.labels_to_classes.items()}
        self.colorspace = colorspace
        self.image_format = image_format
        self.channels = channels
        self.store_filenames = store_filenames
        self.height = height
        self.width = width

    def get_meta_data(self):
        return {
            'classes': self.classes,
            'num_classes': len(self.classes),
            'labels_to_classes': self.labels_to_classes,
            'colorspace': self.colorspace,
            'image_format': self.image_format,
            'channels': self.channels,
            'height': self.height,
            'width': self.width
        }

    def get_image_features(self, image):
        height = image.shape[0]
        width = image.shape[1]
        assert image.shape[2] == self.channels
        if self.height:
            assert height == self.height
        if self.width:
            assert height == self.width
        return height, width

    def create_example(self, image_data, encoded_image, label, filename=None):
        if hasattr(encoded_image, 'shape'):
            height, width = self.get_image_features(image=encoded_image)
        else:
            # the image could a bytes string
            height, width = self.height, self.width

        if height is None:
            raise ValueError('No `height` will be stored for the images. '
                             'If all images have the `height`, please provide '
                             'the `height`  at instantiation.')

        if width is None:
            raise ValueError('No `width` will be stored for the images. '
                             'If all images have the `width`, please provide '
                             'the `width`  at instantiation.')

        features = {
            'image/class/label': self.to_int64_feature(label),
            'image/encoded': self.to_bytes_feature(tf.compat.as_bytes(image_data)),
            'image/width': self.to_int64_feature(width),
            'image/height': self.to_int64_feature(height),
            'image/format': self.to_bytes_feature(self.image_format.encode()),
            'image/colorspace': self.to_bytes_feature(tf.compat.as_bytes(self.colorspace)),
            'image/channels': self.to_int64_feature(self.channels),
        }
        if self.store_filenames:
            filename_feature = self.to_bytes_feature(tf.compat.as_bytes(os.path.basename(filename)))
            features['image/filename'] = filename_feature

        return tf.train.Example(features=tf.train.Features(feature=features))

    def convert(self, session, writer, images, labels, total_num_items, start_index=0,
                filenames=None, processing_fn=None, post_processing_fn=None):

        if self.store_filenames and not filenames:
            raise ValueError('`filenames` is required to store the filename in TF-Example.'
                             'Either provide a list of `filenames` or '
                             'set `store_filenames` to `False`')

        for i in xrange(start_index, len(images)):
            sys.stdout.write('\r>> Converting image %d/%d' % (i + 1, total_num_items))
            sys.stdout.flush()

            image_data, encoded_image = self.image_reader.read(
                session=session, image_data=images[i], processing_fn=processing_fn)
            if post_processing_fn:
                _, image_data = post_processing_fn(session, encoded_image)
            example = self.create_example(image_data, encoded_image, labels[i],
                                          filenames[i] if self.store_filenames else None)
            writer.write(example.SerializeToString())


from web3 import Web3
from dotenv import load_dotenv
import json
import os

load_dotenv()

account = '0x52Ab9c52054B6df51E00ffdDFd8B1Ad31493E621'
private_key = os.getenv('PRIVATE_KEY')
node_endpoint = os.getenv('NODE_ENDPOINT')

with open('abi.json') as json_file:
    abi = json.load(json_file)

w3 = Web3(Web3.HTTPProvider(node_endpoint))
contract_address = '0x8D1372dA80F876f148066E29C667702EEbE8E0bC'
token = w3.eth.contract(address=contract_address, abi=abi)

# Check user's balance
balance = token.functions.balanceOf(account).call()
symbol = token.functions.symbol().call()
print('Balance of ' + account + ' is ' + str(w3.fromWei(balance, 'ether')) + ' ' + symbol)

# Transfer tokens
recipient = '0x52Ab9c52054B6df51E00ffdDFd8B1Ad31493E621'
amount = w3.toWei('1', 'ether')

nonce = w3.eth.getTransactionCount(account)
tx = {
    'nonce': nonce,
    'to': contract_address,
    'gas': 2000000,
    'gasPrice': w3.toWei('50', 'gwei'),
    'data': token.encodeABI(fn_name="transfer", args=[recipient, amount])
}

signed_tx = w3.eth.account.sign_transaction(tx, private_key)

tx_hash = w3.eth.send_raw_transaction(signed_tx.rawTransaction)

print('https://rinkeby.etherscan.io/tx/' + w3.toHex(tx_hash))


import json
import os
from logging import info, debug
from random import shuffle

from torch import tensor
from torch.utils import data

from .config import (
    DATASET,
    PROTEINNET_URL,
    ASTRAL_SECONDARY_STRUCTURES_URL,
    PDB_SECONDARY_STRUCTURES_URL,
    BATCH_SIZE
)
from .download import download_and_unpack, download_and_extract
from .sequences import dssp_to_3_state, create_windows

def load_secondary_structures() -> dict[str, tensor]:
    """Download secondary structure data (not part of main ProteinNet download), and convert
    DSSP sequences to 3-state encodes."""
    pdb = json.loads(download_and_extract(PDB_SECONDARY_STRUCTURES_URL))
    astral = json.loads(download_and_extract(ASTRAL_SECONDARY_STRUCTURES_URL))
    structures = pdb | astral

    return {id_: dssp_to_3_state(sequences['DSSP'])
            for id_, sequences in structures.items()}

def read_protein_data(path: str) -> dict[str, tuple[str, tensor, tensor]]:
    """Download ProteinNet (if necessary), then parse a ProteinNet "human readable" record at `path`.
    The output is a dict mapping protein ID to a tuple of (primary, pssm, secondary).
    Where:
        primary is the primary sequence as a string.
        pssm is the scoring matrix as a 21xlen(primary) tensor. The extra (last) dimension
            is all zeros. This is used later in the windowing step.
        secondary is the 3-state secondary sequence as a 1xlen(primary) tensor. This is
            mapped from the DSSP that ProteinNet provides natively.
    The format is described here:
        <https://github.com/aqlaboratory/proteinnet/blob/master/docs/proteinnet_records.md>."""
    download_and_unpack(PROTEINNET_URL)

    info(f'Parsing {path}...')

    with open(path) as f:
        file = f.read()

    result = {}

    # proteins are separated by blank line
    proteins = file.split('\n\n')
    for p in proteins:
        # empty section marks end of file
        if not p:
            break

        lines = p.split('\n')

        id_ = lines[1].split('#', maxsplit=1)[-1]

        primary = lines[3]

        try:
            secondary = secondary_structures[id_]
        except KeyError:
            # some sequences in ProteinNet are not in the DSSP dataset so ignore these
            debug(f'{id_!r} not in DSSP data')
            continue

        # the "evolutionary" section consists of 21 rows of tab-separated floats.
        # the first 20 dimensions are the PSSM, followed by information content.
        evolutionary = lines[5:26]

        pssm = [[float(f) for f in v.split('\t')] for v in evolutionary]

        result[id_] = (primary, tensor(pssm), secondary)

    info(f'Loaded {len(result)} sequences from {path}.')
    return result

secondary_structures = load_secondary_structures()

class ProteinDataset(data.Dataset):
    def __init__(self, file: str):
        path = os.path.join(DATASET, file)
        protein_data = read_protein_data(path)

        info(f'Creating windows for {path} ...')
        samples = []
        for (primary, pssm, secondary) in protein_data.values():
            samples.extend(create_windows(pssm, secondary))
        info(f'Windowing complete for {path} (created {len(samples)} windows).')

        self.samples = samples

    def __len__(self):
        return len(self.samples)

    def __getitem__(self, item):
        return self.samples[item]

    def shuffle(self):
        """Shuffle the dataset. Useful as an option in analysis."""
        shuffle(self.samples)

class ProteinDataLoader(data.DataLoader):
    def __init__(self, file: str, batch_size: int = BATCH_SIZE):
        super().__init__(ProteinDataset(file), batch_size=batch_size)


from . import sky_blueprint, weather
from app import Session
from flask import jsonify, request
from app.models.models import User, City, Satellites, Stars, StarsFluxV, StarsParallax
from app.models.schemas import SatellitesSchema, StarsSchema
from flask_jwt_extended import jwt_required, get_jwt_identity
from app.api.planet import Planet as PlanetClass
from datetime import datetime
from app.models.models import Planet, PlanetCoordinates
from app.models.schemas import PlanetSchema, PlanetCoordinatesSchema
from app.api.planet import load_planet_coordinates
from app.api.key import api_key_require
from app.api.satellite import Satellite

@sky_blueprint.route('/stars', methods=['GET'])
def get_stars_filter_by_limit():
    """
    optional parameters:
    limit=Value: int, Default=100
    sort=Value: str, Default=stars, Possible: flux_v, parallax, stars
    key=Value: str, Default=None, Keys are registered in database table key

    Returns list of stars limited by the given integer
    and sorted by chosen parameter

    Notice: if limit is greater than actual number of stars it will return all of them
    """

    session = Session()

    # checking api key
    key = api_key_require(request, session)
    if key:
        return key

    try:
        limit = request.args.get('limit', 100)
        mode = request.args.get('sort', 'stars')
        limit = int(limit)
    except ValueError:
        return {'message': 'Wrong input data provided'}, 400
    except Exception:
        return {'message': 'internal server error'}, 500

    if limit < 0:
        return {'message': 'Wrong input data provided'}, 400

    if mode == 'flux_v':
        model = StarsFluxV
    elif mode == 'parallax':
        model = StarsParallax
    elif mode != 'stars':
        return {'message': 'Wrong sort type provided'}, 400
    else:
        model = Stars

    result = []
    stars = session.query(model).limit(limit)

    for star in stars:
        result.append(StarsSchema().dump(star))

    session.close()
    return jsonify(result)

@sky_blueprint.route('/satellites', methods=['GET'])
def get_satellites_filter_by_limit():
    """
    limit=Value: int, Default=100
    key=Value: str, Default=None, Keys are registered in database table key

    Returns list of satellites limited by the given integer

    Notice: if limit is greater than actual number of satellite it will return all of them
    """

    session = Session()

    # checking api key
    key = api_key_require(request, session)
    if key:
        return key

    try:
        limit = request.args.get('limit', 100)
        limit = int(limit)
    except ValueError:
        return {'message': 'Wrong input data provided'}, 400
    except Exception:
        return {'message': 'internal server error'}, 500

    if limit < 0:
        return {'message': 'Wrong input data provided'}, 400

    result = []
    satellites = session.query(Satellites).limit(limit)

    for satellite in satellites:
        result.append(SatellitesSchema().dump(satellite))

    session.close()
    return jsonify(result)

@sky_blueprint.route('/weather', methods=['GET'])
@jwt_required()
def get_weather_for_user():
    """
    Before using this method make sure the api key in
    config.py is provided

    ?city=Value: str, Default=user`s location

    Returns current weather for the city.
    In case there is a trouble with city you will get 401

    Notice: all the api errors including api key errors return the same
    message and code as the city error
    """

    session = Session()
    current_identity_username = get_jwt_identity()

    user = session.query(User).filter_by(username=current_identity_username).first()
    if not user:
        return {'message': 'User not found'}, 404

    city_object = session.query(City).filter_by(id=user.city_id).first()
    if not city_object:
        return {'message': 'City not found in database'}, 404

    try:
        city = request.args.get('city', city_object.name)
    except Exception:
        return {'message': 'internal server error'}, 500

    data = weather.in_the_city(city)
    if 'cod' not in data:
        return jsonify(data), 200
    elif data['message'] == 'city not found':
        return {'message': 'Wrong city provided'}, 401
    else:
        return {'message': 'internal server error'}, 500

@sky_blueprint.route('/load_coordinates/<start_time>/<stop_time>', methods=['POST'])
@jwt_required()
def load_coordinates(start_time, stop_time):
    """
    start_time: YYYY-MM-DD
    stop_time: YYYY-MM-DD

    Loads coordinates for all the planet table objects in
    time interval into table planet_coordinates
    Returns success message

    Notice: works only for admin (with admin JWT token)
    """

    session = Session()
    current_identity_username = get_jwt_identity()

    if current_identity_username != 'admin':
        return {'message': 'Access is denied'}, 403

    planets = session.query(Planet).all()
    if not planets:
        return {'message': 'The planet table is empty'}, 200

    for planet in planets:
        try:
            coordinates = PlanetClass.get_dec_and_ra_in_time_interval(planet.name, start_time, stop_time)
        except ValueError:
            return {'message': 'Wrong input data provided'}, 400
        except Exception:
            return {'message': 'internal server error'}, 500

        loading_result = load_planet_coordinates(planet, coordinates, session)
        if loading_result:
            return {'message': 'internal server error'}, 500

    session.commit()
    session.close()
    return {'message': 'success'}

@sky_blueprint.route('/planets', methods=['GET'])
def get_planets():
    """
    ?key=Value: str, Default=None, Keys are registered in database table key

    Returns list of dicts
    {'name': planet_name, 'information': planet_dict, 'coordinates': coordinates_dict}
    where coordinates_dict is an planet_coordinates table object dict which for today`s date
    and planet_dict is planet table object dict for current planet.

    !!! In case there are no date for today in the date base the coordinates_dict will be empty.
    """

    session = Session()
    date = datetime.today().strftime('%Y-%m-%d')

    # checking api key
    key = api_key_require(request, session)
    if key:
        return key

    try:
        planets = session.query(Planet).all()
        result = []
        for planet in planets:
            if planet.name in ('Sun', 'Moon'):
                continue
            information = PlanetSchema().dump(planet)
            db_coordinates = session.query(PlanetCoordinates).filter_by(planet_id=planet.id, date=date).first()
            coordinates = PlanetCoordinatesSchema().dump(db_coordinates)
            result.append({'name': planet.name, 'information': information, 'coordinates': coordinates})
    except Exception:
        return {'message': 'internal server error'}, 500

    session.close()
    return jsonify(result)

@sky_blueprint.route('/Sun', methods=['GET'])
def get_sun():
    """
    ?key=Value: str, Default=None, Keys are registered in database table key

    Returns dict
    {'name': 'Sun', 'information': planet_dict, 'coordinates': coordinates_dict}
    where coordinates_dict is an planet_coordinates table object dict which for today`s date
    and planet_dict is planet table object dict for Sun.

    !!! In case there are no date for today in the date base the coordinates_dict will be empty.
    """

    session = Session()
    date = datetime.today().strftime('%Y-%m-%d')

    # checking api key
    key = api_key_require(request, session)
    if key:
        return key

    try:
        planet = session.query(Planet).filter_by(name='Sun').first()
        information = PlanetSchema().dump(planet)
        db_coordinates = session.query(PlanetCoordinates).filter_by(planet_id=planet.id, date=date).first()
        coordinates = PlanetCoordinatesSchema().dump(db_coordinates)
        result = {'name': planet.name, 'information': information, 'coordinates': coordinates}
    except Exception:
        return {'message': 'internal server error'}, 500

    session.close()
    return jsonify(result)

@sky_blueprint.route('/Moon', methods=['GET'])
def get_moon():
    """
    ?key=Value: str, Default=None, Keys are registered in database table key

    Returns dict
    {'name': 'Moon', 'information': planet_dict, 'coordinates': coordinates_dict}
    where coordinates_dict is an planet_coordinates table object dict which for today`s date
    and planet_dict is planet table object dict for Moon.

    !!! In case there are no date for today in the date base the coordinates_dict will be empty.
    """

    session = Session()
    date = datetime.today().strftime('%Y-%m-%d')

    # checking api key
    key = api_key_require(request, session)
    if key:
        return key

    try:
        planet = session.query(Planet).filter_by(name='Moon').first()
        information = PlanetSchema().dump(planet)
        db_coordinates = session.query(PlanetCoordinates).filter_by(planet_id=planet.id, date=date).first()
        coordinates = PlanetCoordinatesSchema().dump(db_coordinates)
        result = {'name': planet.name, 'information': information, 'coordinates': coordinates}
    except Exception:
        return {'message': 'internal server error'}, 500

    session.close()
    return jsonify(result)

@sky_blueprint.route('/satellite/<int:id>', methods=['GET'])
def get_current_satellite_coord(id):
    """
    Calculate satellite's position at current time

    Returns dict
    {'norad_id': satellite's norad, 'latitude': satellite's latitude, 'longitude': satellite's longitude,
    'elevation': satellite's elevation}
    """

    session = Session()

    try:
        satellite = session.query(Satellites).filter_by(norad_id=id).first()
        if not satellite:
            return {'message': 'Satellite not found in database'}, 404

        sat = Satellite(id)

        dateTimeObj = datetime.now()
        timestampStr = dateTimeObj.strftime("%d/%m/%y %H:%M:%S")

        tle = "2 "
        tle += str(id) + " "
        tle += satellite['inclination'] + " "
        tle += satellite['ascending_node_longitude'] + " "
        tle += satellite['eccentricity'] + " "
        tle += satellite['pericenter_argument'] + " "
        tle += satellite['average_anomaly'] + " "
        tle += satellite['call_frequency']

        lat, lon, elev = sat.get_satellite_lon_lat(tle, timestampStr)
        result = {'norad_id': id, 'latitude': lat, 'longitude': lon, 'elevation': elev}
    except Exception:
        return {'message': 'internal server error'}, 500

    session.close()
    return jsonify(result)

@sky_blueprint.route('/satellite/<int:id>/<string:date>/<string:time>', methods=['GET'])
def get_future_satellite_coord(id, date, time):
    """
    Get satellite's future position

    future date: DD-MM-YY
    future time: HH:MM:SS
    EXAMPLE: 21-01-22
             11:39:19

    Returns dict
    {'norad_id': satellite's norad, 'latitude': satellite's latitude, 'longitude': satellite's longitude,
    'elevation': satellite's elevation}
    """

    session = Session()

    try:
        satellite = session.query(Satellites).filter_by(norad_id=id).first()
        if not satellite:
            return {'message': 'Satellite not found in database'}, 404

        sat = Satellite(id)

        tle = "2 "
        tle += str(id) + " "
        tle += satellite['inclination'] + " "
        tle += satellite['ascending_node_longitude'] + " "
        tle += satellite['eccentricity'] + " "
        tle += satellite['pericenter_argument'] + " "
        tle += satellite['average_anomaly'] + " "
        tle += satellite['call_frequency']

        date = date.replace("-", "/")
        date_time = date + " " + time

        lat, lon, elev = sat.get_satellite_lon_lat(tle, date_time)
        result = {'norad_id': id, 'latitude': lat, 'longitude': lon, 'elevation': elev}
    except Exception:
        return {'message': 'internal server error'}, 500

    session.close()
    return jsonify(result)

@sky_blueprint.route('/satellite/passes/<int:id>/<string:city>', methods=['GET'])
def get_satellite_passes(id, city):
    """
    Get satellite passes through 2 days relative to an observer.
    """

    session = Session()

    try:
        satellite = session.query(Satellites).filter_by(norad_id=id).first()
        if not satellite:
            return {'message': 'Satellite not found in database'}, 404

        sat = Satellite(id)

        tle = "2 "
        tle += str(id) + " "
        tle += satellite['inclination'] + " "
        tle += satellite['ascending_node_longitude'] + " "
        tle += satellite['eccentricity'] + " "
        tle += satellite['pericenter_argument'] + " "
        tle += satellite['average_anomaly'] + " "
        tle += satellite['call_frequency']

        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
        # IT IS NOT FINISHED
        # WAITING FOR ADDING CITY COORDINATES TO DATABASE CITY TABLE
        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

        # city = session.query(City).filter_by(name=city).first()
        # if not city:
        #             return {'message': 'City not found in database'}, 404
        # observer_lat = city['latitude']
        # observer_long = city['longitude']
        # result = sat.get_satellite_observation(tle, observer_lat, observer_long)
        result = []
    except Exception:
        return {'message': 'internal server error'}, 500

    session.close()
    return jsonify(result)


from pprint import pprint
import csv
import re

if __name__ == '__main__':
    with open("phonebook_raw.csv") as f:
        rows = csv.reader(f, delimiter=",")
        contacts_list = list(rows)
    my_list = []
    for contacts in contacts_list:
        temp_list = []
        my_str = ','.join(contacts)
        pattern = r"(\+7|8)(\s*|)(\(|)(\d{3})(\)|)(\s*|-)(\d{3})(\s*|-)(\d{2})(\s*|-)(\d{2})"
        subst = r"+7(\4)\7-\9-\11"
        telephone = re.sub(pattern, subst, my_str)

        pattern = r"\(?(доб\.)\s*(\d+)\)?"
        subst = r"\1\2"
        telephone_2 = re.sub(pattern, subst, telephone)

        pattern = r"^([а-яА-Я]+).(\w+)(,| )+((\w+)?)(,| )+((\w+)?)(,| )+([^abd-zA-Z,+]+)?(,| )+" \
                  r"(\+7[^,]+)?(,| )+([a-zA-Z0-9@.]+)?"
        subst = r"\1,\2,\5,\8,\10,\12,\14"
        structure = re.sub(pattern, subst, telephone_2)

        temp_list += structure.split(',')
        my_list.append(temp_list)

    contacts_dict = {}
    for name in my_list:
        k = 0
        if name[0] not in contacts_dict:
            contacts_dict[name[0]] = []
            for record in name:
                contacts_dict[name[0]].append(record)
        else:
            temp_list = []
            for record in contacts_dict[name[0]]:
                if record == '':
                    temp_list.append(k)
                    contacts_dict[name[0]].pop(k)
                    contacts_dict[name[0]].insert(k, name[k])
                    k += 1
                else:
                    k += 1
    new_contacts_list = []
    for i in contacts_dict.values():
        new_contacts_list.append(i)
    pprint(new_contacts_list)

    with open("phonebook.csv", "w") as f:
        datawriter = csv.writer(f, delimiter=',')
        datawriter.writerows(new_contacts_list)


def slopeStyleScore(scores):
    min = 1000000
    max = -1000000
    sum = 0

    for element in scores:
        if element < min:
            min = element

        if element > max:
            max = element

    scores.remove(max)
    scores.remove(min)

    for element in scores:
        sum += element

    average = sum / len(scores)
    return round(average, 2)

print(slopeStyleScore([94, 95, 95, 95, 90]))


import scrapy
from urllib3 import request

from wymusic.items import WymusicItem
import re

class WyspiderSpider(scrapy.Spider):
    name = 'wyspider'
    allowed_domains = ['music.163.com']
    start_urls = ['https://music.163.com/discover/playlist']
    page=1

    def parse(self, response):

        # 清洗song_id
        def clean(lt):
            res = []
            for i in lt:
                res.append(int(re.findall(r"\d+\.?\d*", i)[0]))
            return res

        current_page=response.xpath("//ul[@class='m-cvrlst f-cb']/li")

        for aim in current_page:
            msc=WymusicItem()

            msc['playlist_name']=aim.xpath("div[@class='u-cover u-cover-1']/a/@title").extract()[0]
            msc['playlist_id']=clean(aim.xpath("div[@class='u-cover u-cover-1']/a/@href").extract())[0]

            link=aim.xpath("div[@class='u-cover u-cover-1']/a/@href").extract()[0]
            url="https://music.163.com"+link
            if url:
                yield scrapy.Request(url,meta={'item':msc},callback=self.parse_music)

            pass

        next_page=response.xpath("//div[@class='u-page']/a[@class='zbtn znxt']/@href").extract()
        if next_page:
            self.page+=1
            next_link=response.urljoin(next_page[0])
            yield scrapy.Request(next_link,callback=self.parse)

    def parse_music(self, response):

        # 清洗song_id
        def clean(lt):
            res = []
            for i in lt:
                res.append(int(re.findall(r"\d+\.?\d*", i)[0]))
            return res

        msc=response.meta['item']
        aims=response.xpath("//ul[@class='f-hide'][1]/li/a")

        for aim in aims:
            msc['music_name']=aim.xpath("text()").extract()
            msc['music_id']=clean(aim.xpath("@href").extract())[0]

            yield msc

        pass


"Relax the pressure in a simulation."

from ASE.Dynamics.Dynamics import Dynamics

class RelaxPressure(Dynamics):
    """This class implements Behrendsen's method to relax the pressure."""

    def __init__(self, atoms, p=0.0, coeff=0.1):
        Dynamics.__init__(self, atoms)
        self.p = p
        self.coeff = coeff

    def _Step(self):
        stress = self.atoms.GetStress()
        ucell = self.atoms.GetUnitCell()
        p = - sum(stress[0:3]) / 3.0
        delV = self.coeff * (p - self.p)
        ucell *= (1.0 + delV/3.0)
        self.atoms.SetUnitCell(ucell)


alphabet = "abcdefghijklmnopqrstuvwxyz"

test_dups = ["zzz","dog","bookkeeper","subdermatoglyphic","subdermatoglyphics"]

test_miss = ["zzz","subdermatoglyphic","the quick brown fox jumps over the lazy dog"]

# From Section 11.2 of:

# Downey, A. (2015). Think Python: How to think like a computer scientist. Needham, Massachusetts: Green Tree Press.
#this is the histogram that creates a dictionary with all the letters used and count on those letters
def histogram(s):
     d = dict()
     for c in s:
          if c not in d:
               d[c] = 1
          else:
               d[c] += 1
     return d
#has_duplicates sends the string to the histogram to get a dictionary abck and loop through it to determine if it has duplicates
def has_duplicates(str):
    histodic=histogram(str)
    keys=histodic.keys()
    for x in keys:

        if histodic[x]>1:
            return True
        else:
            return False
#test_dups_print is a function in charge of iterating through the test_dups and send them through has_duplicates which in turn returns false or true, booleans used to determine the print result
def test_dups_print():
    global test_dups
    for x in test_dups:
        if has_duplicates(x)== True:
            print(x+ " has duplicates")
        else:
            print(x+ " has no duplicates")
test_dups_print()
#missing letters takes in a str and turns the alpahabet into a list so it can then be turned into a dict, it iterates throguh the str and logs the letters into the dictionary and then it appends missing letters to a list and then returns a str with results
def missing_letters(str):
    global alphabet
    splited=list(alphabet)
    newdict=dict.fromkeys(splited,0)
    for x in str:
        if x==" ":
            continue
        newVal=newdict[x]+1
        newdict.update({x:newVal})
    missing=[]
    for y in newdict:
        if newdict[y]==0:
            missing.append(y)
    if len(missing)== 0:
        return str+" uses all the letters "
    else:
        return str+" is missing letters "+ "".join(missing)

#test_miss_print is in charge of iterating through test_miss and sending the test cases to missing_letters and printing the result
def test_miss_print():
    global test_miss
    for x in test_miss:
        print(missing_letters(x))
test_miss_print()


import pandas as pd
import numpy as np

# Building an N-gram based language models

# Function that computes n-grams count for each arbitrary n number
def count_n_grams(data,n,start_token="<s>",end_token="<e>"):
    # Initialize dictionary for n_grams and their cummulative count
    n_grams = {}
    # Loop through data
    for sentence in range(len(data)):
        sentences = [start_token]*n + list(data[sentence]) + [end_token]
        sentences = tuple(sentences)
        for i in range(0,len(sentences)-n+1):
            # Obtain n-gram from i to i+n
            n_gram = sentences[i:i+n]
            # Check if n-gram is in dictionary
            if n_gram in n_grams.keys():
                # Increment count
                n_grams[n_gram] += 1
            else:
                # if not, set count to one
                n_grams[n_gram] = 1
        # Return dictionary of n_grams and their cummulative count
        return n_grams

# Function that calculates the numerator and denominator to obtain estimated probability of interest
def estimate_probability(word,previous_n_gram,n_gram_counts,n_plus1_gram_counts,vocabulary_size,k=1.0):
    # Check if previous n-gram is of datatype "list"
    if type(previous_n_gram) == list:
        # Convert list to tuple
        previous_n_grams = tuple(previous_n_gram)
    else:
        # Otherwise, first convert to list then to tuple
        previous_n_grams = tuple([previous_n_gram])
    previous_n_gram_count = n_gram_counts.get(previous_n_grams,0)
    # Apply k-smoothing to handle zero counts in denominator
    denominator = previous_n_gram_count + vocabulary_size*1
    n_plus1_gram = previous_n_grams + tuple([word])
    n_plus1_gram_count = n_plus1_gram_counts.get(n_plus1_gram,0)
    # Apply smoothing to numerator
    numerator = n_plus1_gram_count + k
    # Calculating probability
    probability = numerator/denominator
    # Return calculated probability
    return probability

# Function to calculate probabilities for all possible words
def estimate_probabilities(previous_n_gram,n_gram_counts,n_plus1_gram_counts,vocabulary,k=1.0):
    previous_n_gram = previous_n_gram
    # Add <e> <unk> to the vocabulary <s> is not needed since it should not appear as the next word
    vocabulary = vocabulary + ["<e>","<unk>"]
    vocabulary_size = len(vocabulary)
    # Instantiate dictionary of probabilities
    probabilities = {}
    # Loop through vocabulary
    for word in vocabulary:
        # Obtain estimated probability for each word
        probability = estimate_probability(word,previous_n_gram,n_gram_counts,n_plus1_gram_counts,vocabulary_size,k=k)
        # Append word and it's probability to dictionary
        probabilities[word] = probability
    # Return estimated probabilities
    return probabilities

# Function to create frequency matrix
def make_count_matrix(n_plus1_gram_counts, vocabulary):
    vocabulary = vocabulary + ["<e>","<unk>"]
    n_grams = []
    for n_plus1_gram in n_plus1_gram_counts.keys():
        n_gram = n_plus1_gram[0:-1]
        n_grams.append(n_gram)
    n_grams = list(set(n_grams))
    row_index = {n_gram:i for i,n_gram in enumerate(n_grams)}
    col_index = {word:j for j,word in enumerate(vocabulary)}
    nrow = len(n_grams)
    ncol = len(vocabulary)
    count_matrix = np.zeros((nrow,ncol))
    for n_plus1_gram,count in n_plus1_gram_counts.items():
        n_gram = n_plus1_gram[0:-1]
        word = n_plus1_gram[-1]
        if word not in vocabulary:
            continue
        i = row_index[n_gram]
        j = col_index[word]
        count_matrix[i,j] = count
    count_matrix = pd.DataFrame(count_matrix,index=n_grams,columns=vocabulary)
    return count_matrix

# Function to calculate the probabilities of each word given the previous n-gram
def make_probability_matrix(n_plus1_gram_counts,vocabulary,k):
    count_matrix = make_count_matrix(n_plus1_gram_counts,vocabulary)
    count_matrix += k
    prob_matrix = count_matrix.div(count_matrix.sum(axis=1),axis=0)
    return prob_matrix

# Function to implement loss function as perplexity score
def calculate_perplexity(sentences,n_gram_counts,n_plus1_gram_counts,vocabulary_size,k=1.0):
    n = len(list(n_gram_counts.keys())[0])
    sentence = ["<s>"]*n + sentences + ["<e>"]
    sentence = tuple(sentence)

    N = len(sentence)
    product_pi = 1.0
    for i in range(n,N):
        n_gram = sentence[i-n]
        word = sentence[i]
        probability = estimate_probability(word,n_gram,n_gram_counts,n_plus1_gram_counts,vocabulary_size,k)
        product_pi *= 1/probability

    # Nth root of the product
    perplexity = product_pi**(1/float(N))
    perplexity = float(perplexity)

    return perplexity

# Function to define auto-complete system
def suggest_a_word(previous_n_gram,n_gram_counts,n_plus1_gram_counts,vocabulary,k=1.0,start_with=None):
    probabilities = estimate_probabilities(previous_n_gram,n_gram_counts,n_plus1_gram_counts,vocabulary,k=k)
    suggestion = None
    max_prob = 0
    for word,prob in probabilities.items():
        if start_with != None:
            if word.startswith(start_with)==False:
                continue
        if prob > max_prob:
            suggestion = word
            max_prob = prob
    return suggestion,max_prob


# Error & Exception handling
import os
import logging
from src.logexception.exceptionhandler import ExceptionHandler

def parse_csv_and_get_columns(filename):

    try:
        logger = logging.getLogger('my_module')
        if(os.path.exists(filename)):
            csvFile = open(filename, 'r')
            lines = csvFile.readlines()
            for line in lines[1:]:
                val = line.split(",")
                test_str_div = val[0] / val[11]

        else:
            logging.info('File not found')
            ExceptionHandler()

    except IOError:
        logger.error('IO Error | An error occurred trying to read the file.')

    except Exception as ex:
        logger.error('TypeErrorException |  Please check the operands type')
        raise ExceptionHandler(' |  Please check the operands type')

    except Exception as ex:
        logger.error('OtherException | Incorrect value or type error.')

if __name__ == "__main__":
    parse_csv_and_get_columns(filename="data/flights.csv")


from django.db import models
from utils import slugify, process_string

class Constant(models.Model):
    identifier = models.CharField(max_length=50)
    name = models.CharField(max_length=50)

    def __unicode__(self):
        return '{}'.format(self.name)

    @staticmethod
    def find_or_create(model, name):
        name = process_string(name).capitalize()
        identifier = slugify(name)

        try:
            instance = model.objects.filter(
                identifier=identifier)[0]
        except:
            instance = model.objects.create(
                identifier=identifier,
                name=name
            )

        return instance

    class Meta:
        abstract = True


from typing import List, Generator, Any

from nltk.tokenize.toktok import ToktokTokenizer
# from nltk.corpus import stopwords
# STOPWORDS = stopwords.words('russian')
import pymorphy2

from deeppavlov.core.models.component import Component
from deeppavlov.core.common.registry import register
from deeppavlov.models.tokenizers.utils import detokenize, ngramize
from deeppavlov.core.common.log import get_logger

logger = get_logger(__name__)

@register('ru_tokenizer')
class RussianTokenizer(Component):
    """
    Tokenize or lemmatize a list of documents.
    Return list of tokens or lemmas, without sentencizing.
    Works only for Russian language.
    """

    def __init__(self, stopwords: list = None, ngram_range: List[int] = None, lemmas=False,
                 lowercase: bool = None, alphas_only: bool = None, **kwargs):
        """
        :param stopwords: a set of words to skip
        :param ngram_range: range for producing ngrams, ex. for unigrams + bigrams should be set to
        [1, 2], for bigrams only should be set to [2, 2]
        :param lemmas: weather to perform lemmatizing or not while tokenizing, currently works only
        for the English language
        :param lowercase: perform lowercasing or not
        :param alphas_only: should filter numeric and alpha-numeric types or not
        """
        if ngram_range is None:
            ngram_range = [1, 1]
        self.stopwords = stopwords or []
        self.tokenizer = ToktokTokenizer()
        self.lemmatizer = pymorphy2.MorphAnalyzer()
        self.ngram_range = tuple(ngram_range)  # cast JSON array to tuple
        self.lemmas = lemmas
        self.lowercase = lowercase
        self.alphas_only = alphas_only
        self.tok2morph = {}

    def __call__(self, batch):
        if isinstance(batch[0], str):
            if self.lemmas:
                return list(self._lemmatize(batch))
            else:
                return list(self._tokenize(batch))
        if isinstance(batch[0], list):
            return [detokenize(doc) for doc in batch]
        raise TypeError(
            "StreamSpacyTokenizer.__call__() is not implemented for `{}`".format(type(batch[0])))

    def _tokenize(self, data: List[str], ngram_range=(1, 1), lowercase=True)\
            -> Generator[List[str], Any, None]:
        """
        Tokenize a list of documents.
        :param data: a list of documents to process
        :param ngram_range: range for producing ngrams, ex. for unigrams + bigrams should be set to
        (1, 2), for bigrams only should be set to (2, 2)
        :param batch_size: the number of documents to process at once
        :param lowercase: whether to perform lowercasing or not
        :return: a single processed doc generator
        """
        # DEBUG
        # size = len(data)
        _ngram_range = self.ngram_range or ngram_range

        if self.lowercase is None:
            _lowercase = lowercase
        else:
            _lowercase = self.lowercase

        for i, doc in enumerate(data):
            # DEBUG
            # logger.info("Tokenize doc {} from {}".format(i, size))
            tokens = self.tokenizer.tokenize(doc)
            if _lowercase:
                tokens = [t.lower() for t in tokens]
            filtered = self._filter(tokens)
            processed_doc = ngramize(filtered, ngram_range=_ngram_range)
            yield from processed_doc

    def _lemmatize(self, data: List[str], ngram_range=(1, 1)) -> \
            Generator[List[str], Any, None]:
        """
        Lemmatize a list of documents.
        :param data: a list of documents to process
        :param ngram_range: range for producing ngrams, ex. for unigrams + bigrams should be set to
        (1, 2), for bigrams only should be set to (2, 2)
        :return: a single processed doc generator
        """
        # DEBUG
        # size = len(data)
        _ngram_range = self.ngram_range or ngram_range

        tokenized_data = list(self._tokenize(data))

        for i, doc in enumerate(tokenized_data):
            # DEBUG
            # logger.info("Lemmatize doc {} from {}".format(i, size))
            lemmas = []
            for token in doc:
                try:
                    lemma = self.tok2morph[token]
                except KeyError:
                    lemma = self.lemmatizer.parse(token)[0].normal_form
                    self.tok2morph[token] = lemma
                lemmas.append(lemma)
            filtered = self._filter(lemmas)
            processed_doc = ngramize(filtered, ngram_range=_ngram_range)
            yield from processed_doc

    def _filter(self, items, alphas_only=True):
        """
        Make ngrams from a list of tokens/lemmas
        :param items: list of tokens, lemmas or other strings to form ngrams
        :param alphas_only: should filter numeric and alpha-numeric types or not
        :return: filtered list of tokens/lemmas
        """
        if self.alphas_only is None:
            _alphas_only = alphas_only
        else:
            _alphas_only = self.alphas_only

        if _alphas_only:
            filter_fn = lambda x: x.isalpha() and not x.isspace() and x not in self.stopwords
        else:
            filter_fn = lambda x: not x.isspace() and x not in self.stopwords

        return list(filter(filter_fn, items))

    def set_stopwords(self, stopwords):
        self.stopwords = stopwords


# Desired IVs
V6 = [31,31,31,31,31,31]
S0 = [31,31,31,31,31,00]
A0 = [31,00,31,31,31,31]

useFilters = True
MaxResults = 1000
doResearch = True

# Go to root of PyNXBot
import signal
import sys
import json
sys.path.append('../')

from lookups import Util
from nxbot import RaidBot
from rng import XOROSHIRO,Raid
from structure import Den

def signal_handler(signal, frame): #CTRL+C handler
    print("Stop request")
    b.close()

config = json.load(open("../config.json"))
b = RaidBot(config["IP"])

signal.signal(signal.SIGINT, signal_handler)

seed = None

for ii in range(RaidBot.DENCOUNT):
    if ii > 189:
        den = Den(b.readDen(ii + 32))
    elif ii > 99:
        den = Den(b.readDen(ii + 11))
    else:
        den = Den(b.readDen(ii))
    if den.isActive():
        spawn = den.getSpawn(denID = ii, isSword = b.isPlayingSword)
        currShinyLock = 0
        if ii > 189:
            info = f"[CT] denID: {ii-189}"
        elif ii > 99:
            info = f"[IoA] denID: {ii-99}"
        else:
            info = f"denID: {ii+1}"
        info += f"    {den.stars()}★    Species: {Util.STRINGS.species[spawn.Species()]}"
        if spawn.IsGigantamax():
            info += " G-Max"
        if den.isEvent():
            info += "    Event"
            currShinyLock = spawn.ShinyFlag()
        if den.isWishingPiece():
            if currShinyLock != 2:
                info += f"    Next Shiny Frame: {Raid.getNextShinyFrame(den.seed())}"
            else:
                info += f"    Next Shiny Frame: 0"
            seed = den.seed()
            info = "!!! " + info
            piecedSpawn = spawn
            piecedShinyLock = currShinyLock
        print(info)
        r = Raid(seed = den.seed(), TID = b.TID, SID = b.SID, flawlessiv = spawn.FlawlessIVs(), shinyLock = currShinyLock, ability = spawn.Ability(), gender = spawn.Gender(), species = spawn.Species(), altform = spawn.AltForm())
        r.print()
        print()

# Choose RNGable den to calculate frames
if seed is not None and doResearch:
    print('\nWishing Piece Den Prediction:\n')
    i = 0
    while i < MaxResults:
        r = Raid(seed, TID = b.TID, SID = b.SID, flawlessiv = piecedSpawn.FlawlessIVs(), shinyLock = piecedShinyLock, ability = piecedSpawn.Ability(), gender = piecedSpawn.Gender(), species = piecedSpawn.Species(), altform = piecedSpawn.AltForm())
        seed = XOROSHIRO(seed).next()
        if useFilters:
            if (r.ShinyType != 'None' or r.IVs == V6 or r.IVs == S0 or r.IVs == A0) and Util.STRINGS.natures[r.Nature] == 'Careful':
                print(f"Frame:{i}")
                r.print()
                print()
        else:
            print(f"Frame:{i}")
            r.print()
            print()
        i += 1

b.close()


from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^orders/settlement/$', views.OrderSettlementView.as_view()),  # 去结算

    url(r'^orders/commit/$', views.OrderCommitView.as_view()),  # 提交订单

    url(r'^orders/success/$', views.OrderSuccessView.as_view()),  # 提交订单成功

    url(r'^orders/comment/$', views.GoodsComment.as_view()),  # 商品评价

    url(r'^comments/(?P<sku_id>\d+)/$', views.CommentView.as_view()),  # 商品评论展示

]


import datetime
from cmath import log
from typing import Any, Dict, List

import boto3
import pytz
from botocore.exceptions import ClientError
from odd_collector_sdk.domain.adapter import AbstractAdapter
from odd_models.models import DataEntity, DataEntityList, DataEntityType

from odd_collector_aws.domain.plugin import SQSPlugin

from .sqs_generator import SqsGenerator

SCHEMA_FILE_URL = (
    "https://raw.githubusercontent.com/opendatadiscovery/opendatadiscovery-specification/"
    "main/specification/extensions/sqs.json"
)

class Adapter(AbstractAdapter):
    def __init__(self, config: SQSPlugin) -> None:
        self._account_id = boto3.client(
            "sts",
            aws_access_key_id=config.aws_access_key_id,
            aws_secret_access_key=config.aws_secret_access_key,
            region_name=config.aws_region,
        ).get_caller_identity()["Account"]
        self._sqs_client = boto3.client(
            "sqs",
            aws_access_key_id=config.aws_access_key_id,
            aws_secret_access_key=config.aws_secret_access_key,
            region_name=config.aws_region,
        )

        self.__oddrn_generator = SqsGenerator(
            cloud_settings={"region": config.aws_region, "account": self._account_id}
        )

    def get_data_entity_list(self) -> DataEntityList:
        return DataEntityList(
            data_source_oddrn=self.get_data_source_oddrn(),
            items=self.get_data_entities(),
        )

    def get_data_entities(self) -> List[DataEntity]:
        try:
            sqs_queues = []
            for queue in self._sqs_client.list_queues()["QueueUrls"]:
                queue_name = queue.split("/")[-1]
                queue_attributes = self._sqs_client.get_queue_attributes(
                    QueueUrl=queue, AttributeNames=["All"]
                )["Attributes"]
                created_at = datetime.datetime.fromtimestamp(
                    int(queue_attributes["CreatedTimestamp"]), tz=pytz.utc
                )
                updated_at = datetime.datetime.fromtimestamp(
                    int(queue_attributes["LastModifiedTimestamp"]), tz=pytz.utc
                )
                del queue_attributes["CreatedTimestamp"]
                del queue_attributes["LastModifiedTimestamp"]
                del queue_attributes["Policy"]
                queue_attributes["AccountID"] = self._account_id
                metadata = [
                    {
                        "schema_url": (
                            f"{SCHEMA_FILE_URL}#/definitions/SQSDataSetExtension"
                        ),
                        "metadata": queue_attributes,
                    }
                ]
                sqs_queues.append(
                    DataEntity(
                        name=queue_name,
                        oddrn=self.__oddrn_generator.get_oddrn_by_path(
                            "queue", queue_name
                        ),
                        type=DataEntityType.KAFKA_TOPIC,
                        created_at=created_at,
                        updated_at=updated_at,
                        metadata=metadata,
                        dataset={"field_list": []},
                    )
                )
        except ClientError:
            raise
        else:
            return sqs_queues

    def get_data_source_oddrn(self) -> str:
        return self.__oddrn_generator.get_data_source_oddrn()


import zmq

context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.connect("tcp://127.0.0.1:5555")

for request in range(10):
    print("sending request %s" % request)
    socket.send(b"hello")

    message = socket.recv()
    print("Recevied reply %s [ %s ]" % (request, message))


# -*- coding: utf-8 -*-
"""
Convenience function to clean Hirex-Sr signals after spectral line fitting.

@author: sciortino
"""

import numpy as np

def clean_moments(mf_time,mf_maxChan, t_min, t_max, gathered_moments, BR_THRESH=2.0, BR_STD_THRESH=0.1, BR_REL_THRESH=0.5, normalize=True):
    ''' Re-organize gathered moments into a numpy array.
    Exclude brightness data points whose value exceeds 'BR_THRESH'.

    Parameters:
    mf_time: Time array in the BSFC Moment-Fitter object, given in mf.time
    mf_maxChan: number of channels in the BSFC Moment-Fitter object, given in mf.maxChan
    t_min [s]: minimum time to be kept in results
    t_max [s]: maximum time to be kept in results
    gathered_moments: result of running bsfc_run_mpi.py, saved in ./bsfc_fits

    BR_THRESH: threshold to exclude outliers based on their brightness value
    BR_STDS_THRESH: threshold to exclude outliers based on their brightness standard deviation
    BR_REL_THRESH: threshold to exclude outliers based on their relative uncertainty

    normalize: normalize brightness to largest value across chords.
    '''
    # activate this for future BSFC runs:
    #ind_sel = np.where(np.logical_and(mf_time>t_min, mf_time<t_max))
    #time_sel= np.array(mf_time)[ind_sel] #[tidx_min: tidx_max]
    #tidx_min = ind_sel[0][0]
    #tidx_max = ind_sel[0][-1] +1  # python indexing

    tidx_min = np.argmin(np.abs(mf_time - t_min))
    tidx_max = np.argmin(np.abs(mf_time - t_max))
    time_sel = np.array(mf_time)[tidx_min:tidx_max]

    # get individual spectral moments
    moments_vals = np.empty((tidx_max-tidx_min,mf_maxChan,3))
    moments_stds = np.empty((tidx_max-tidx_min,mf_maxChan,3))
    moments_vals[:] = None
    moments_stds[:] = None

    #from IPython import embed
    #embed()
    for tbin in np.arange(tidx_max-tidx_min):
        for chbin in np.arange(mf_maxChan):
            moments_vals[tbin,chbin,0] = gathered_moments[tbin,chbin][0][0]
            moments_stds[tbin,chbin,0] = gathered_moments[tbin,chbin][1][0]
            moments_vals[tbin,chbin,1] = gathered_moments[tbin,chbin][0][1]
            moments_stds[tbin,chbin,1] = gathered_moments[tbin,chbin][1][1]
            moments_vals[tbin,chbin,2] = gathered_moments[tbin,chbin][0][2]
            moments_stds[tbin,chbin,2] = gathered_moments[tbin,chbin][1][2]

    # exclude values with brightness value or uncertainty greater than certain values
    moments_vals[:,:,0][np.logical_or(moments_vals[:,:,0] > BR_THRESH , moments_stds[:,:,0] > BR_STD_THRESH)] = np.nan
    moments_stds[:,:,0][np.logical_or(moments_vals[:,:,0] > BR_THRESH , moments_stds[:,:,0] > BR_STD_THRESH)] = np.nan
    moments_vals[:,:,1][np.logical_or(moments_vals[:,:,0] > BR_THRESH , moments_stds[:,:,0] > BR_STD_THRESH)] = np.nan
    moments_stds[:,:,1][np.logical_or(moments_vals[:,:,0] > BR_THRESH , moments_stds[:,:,0] > BR_STD_THRESH)] = np.nan
    moments_vals[:,:,2][np.logical_or(moments_vals[:,:,0] > BR_THRESH , moments_stds[:,:,0] > BR_STD_THRESH)] = np.nan
    moments_stds[:,:,2][np.logical_or(moments_vals[:,:,0] > BR_THRESH , moments_stds[:,:,0] > BR_STD_THRESH)] = np.nan

    # threshold on relative uncertainty
    '''
    moments_vals[:,:,0][moments_stds[:,:,0]/(moments_vals[:,:,0]+1e-10) > BR_REL_THRESH] = np.nan
    moments_stds[:,:,0][moments_stds[:,:,0]/(moments_vals[:,:,0]+1e-10) > BR_REL_THRESH] = np.nan
    '''
    if normalize:
        # normalize brightness to largest value
        idx1,idx2 = np.unravel_index(np.nanargmax(moments_vals[:,:,0]), moments_vals[:,:,0].shape)
        max_br = moments_vals[idx1,idx2,0]
        max_br_std = moments_stds[idx1,idx2,0]

        moments_vals[:, :,0] = moments_vals[:,:,0]/ max_br
        moments_stds[:,:,0] = np.sqrt((moments_stds[:,:,0]/max_br)**2.0 + (( moments_vals[:,:,0]/max_br)*(max_br_std/max_br))**2.0)

    return moments_vals, moments_stds, time_sel


import collections

class TreeNode:

    def __init__(self, val, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Sol:

    def helper(self,node, lst):

        if len(lst)==0:
            return None

        n=len(lst)-1
        m = n//2

        left_child_idx = (m-1)//2
        right_child_idx = m+(n-m-1)//2

        node.left = TreeNode(lst[left_child_idx])
        node.right = TreeNode(lst[right_child_idx])

        self.helper(node.left, lst[0:m-1])
        self.helper(node.right, lst[m+1:])

    def main(self, lst):

        n = len(lst)
        m = (n-1)//2
        root = TreeNode(lst[m])
        node = root
        self.helper(node, lst)
        return root

    def bfs(self,root):

        q = collections.deque([root])
        res=[]
        while(q):
            l = len(q)
            lst=[]
            for _ in range(l):
                ele = q.popleft()
                if ele:
                    lst.append(ele.val)
                    if ele.left:
                        q.append(ele.left)
                    if ele.right:
                        q.append(ele.right)

            res.append(lst)
        return res

obj = Sol()
lst = [1,2,3,4,5,6,7,8,9]
root = obj.main(lst)
print("--")
print(root.val)
print(root.left.val)
print(root.right.val)
print("--")

"""
[1,2,3,4,5,6,7,8,9]

                           5
                 2                   7
            1        3            6      8
                        4                   9

"""

res = obj.bfs(root)

print("res: {}".format(res))

"""
        1
    2       3
4       5  6  7
"""
node1 = TreeNode(1)
node2 = TreeNode(2)
node3 = TreeNode(3)
node4 = TreeNode(4)
node5 = TreeNode(5)
node6 = TreeNode(6)
node7 = TreeNode(7)

node1.left = node2
node1.right = node3

node2.left = node4
node2.right = node5

node3.left = node6
node3.right = node7

print("----")
print("----")

res = obj.bfs(node1)

print("res: {}".format(res))

"""

          5
    3            7
1       3      7   9

"""