from time import sleep
import serial
from test.lib import Ser

def comm(send, read):
    byte = '485 rx-tx test data'
    ser_send = serial.Serial(send, 115200)
    ser_read = serial.Serial(read, 115200)
    ser_read.flushInput()
    ser_send.write(byte.encode('utf-8'));
    sleep(0.1)
    count = ser_read.inWaiting()
    count = ser_read.read(count)
    if str(count).find('485 rx-tx test data') > -1:
        return True
    return False

def test_485(d):
    result = [{
        "flag": True,
        "message": "485 send_receive pass!"
    }]
    comm_result = []
    try:
        ser = Ser()
        ser.openCom(port_name = d['serial_485_sub'])
        read = ser.run(bytes.fromhex('68 05 00 42 00 11 02 12 00 00 1d 85 16'), 87)
        ser.closrCom()
        if read == False or str(read).find('RS485') == -1:
            result[0]['flag'] = False
            result[0]['message'] = "Serial Protocol error!"
            return result
        comm_result.append(comm(d['serial_send'], d['serial_read']))
        comm_result.append(comm(d['serial_read'], d['serial_send']))
        print(comm_result)

    except Exception as e:
        print(e)
        result[0]['flag'] = False
        result[0]['message'] = "ttyACM serial error!"
        return result
    return result


import re

def is_phishing_email(email_subject):
    # Perform pattern matching to detect potential phishing indicators
    phishing_patterns = ['urgent', 'verify', 'account', 'security', 'suspicious']
    for pattern in phishing_patterns:
        if re.search(pattern, email_subject, re.IGNORECASE):
            return True
    return False

def send_response(email_sender):
    # Send an automated response to the sender of the phishing email
    response = """Dear {},

Thank you for your email. We appreciate your vigilance in reporting potential phishing emails. Our security team will investigate the matter accordingly.

Best regards,
Your Organization""".format(email_sender)
    # Code to send the response email goes here (implementation depends on your email sending method)

# Example usage
email_subject = input("Enter the subject of the email: ")
email_sender = input("Enter the sender of the email: ")

if is_phishing_email(email_subject):
    send_response(email_sender)
    print("Automated response sent to the sender.")
else:
    print("This email is not identified as a phishing attempt.")

Security/Security Incident Response Automation/12-automated-phishing-response.py


"""Sometimes we want to extract a specific bit for
comparison or testing"""
"""Shifting a register."""
import rp2pio
import adafruit_pioasm
import array

# extracting bit 2
bit_to_extract = 30
program = f"""
.program extract_bit
    pull block

; print initial state
    mov isr, osr
    push noblock    ; and use PUSH to put it on the receive FIFO

; extract - by shifting
    in osr, {bit_to_extract}  ; get n bits
    in null, 31  ; keep only 1 bit
    push noblock
"""

assembled = adafruit_pioasm.assemble(program)

## set up a statemachine
sm = rp2pio.StateMachine(
    assembled,
    frequency=2000,
)

buffer = array.array("I", [0])

sm.write(array.array("I", [0b01101000_00000000_00000000_00000000]))
# read the data
sm.readinto(buffer)
print("Initial Y: {0} 0b{0:032b} 0x{0:x}".format(buffer[0]))

sm.readinto(buffer)
print("Y bit extracted:  {0} 0b{0:032b} 0x{0:x}".format(buffer[0]))


# Importing all the libraries
from flask import Flask, request, jsonify
from flask_sqlalchemy import SQLAlchemy
import pandas as pd
from sqlalchemy import create_engine
import sqlite3

# creating a server
app = Flask(__name__)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

# Function to create a database
def create_database():
    csv_file = 'data.csv'  # Data file to read
    engine = create_engine('sqlite:///sheertex.db', echo=False)  # Create a sqlite db
    with open(f'{csv_file}', 'r') as file:  # open the csv file
        data_df = pd.read_csv(f'{csv_file}',
                              dtype={'id': int, 'created_at': str, 'customer': str, 'state': str, 'price': int,
                                     'currency': str, 'items': str, 'sku': str, 'quantity': int, 'total': int}) #schema for the dataframe
        print(data_df, data_df.columns)
    data_df.to_sql('orders_data', con=engine, index=False, index_label=None,
                   if_exists='replace')  # Replace if exist and table is named orders_data
    conn = sqlite3.connect('sheertex.db', check_same_thread=False)  # connect to the DB
    return conn

# Calling create_database function
conn = create_database()
if not conn:  # Verifying whether db is created.
    raise RuntimeError("Cannot create DB")

@app.route('/api/orders/<string:start_date>/<string:end_date>',
           methods=['GET'])  # Using GET method to retreive the data
def get_orders(start_date, end_date):
    try:
        cur = conn.cursor()  # To allow python code to execute sql commands in the database session
        all_orders = cur.execute(
            "select * from orders_data where created_at BETWEEN  " + start_date + " AND " + end_date + "")  # Extracting the start_date and End_date from the api
        result = dict(orders=all_orders.fetchall())  # converting all_orders to dictionary type
        return result  # return the result
    except:  # To check if is not able to fetch the data from the API
        print("Cannot fetch the details from the API")

if __name__ == '__main__':
    app.run('127.0.0.1', 5000, debug=True)


# ========================================================================
# 구성 시나리오
# 1. "{Category}-{Folder NAME}-{Year}-{Status}" 형식의 폴더를 구성
#   ex. "(NLP)-(RNN Overview}-{2019)-(Processing)"
# 1-1. 폴더 안에 MarkDown 파일를 만들고 논문리뷰 작성, 파일명은 논문명
#   ex. "Recurrent Neural Networks (RNNs) : A gentle Introduction and Overview"
# 2. Local 환경에서 1~1.1 단계수행 후 github repository에 push 수행
# 3. 정상적으로 push수행 시 Git action에 의해 CI/CD가 수행
# 4. 정상적으로 Git Action에 의해 CI/CD 수행 시 'update_readme.py'가 수행
# 5. update_readme.py는 새로 푸쉬된 폴더 명과 그 안의 파일명을 추출해 README.md를 업데이트 한다.
#       category = "NLP" -> category
#       date = "23 Nov 2019" -> yeaer
#       title = "Paper Title" -> title
#       status = "processing" -> status
#   "| {category} | {year} | {title} | {status} |"의 형태로 README에 추가됨.

import os

readme_file_path = "/home/runner/work/ghost_nlp/ghost_nlp/README.md"
base_dir = r'/home/runner/work/ghost_nlp/ghost_nlp/'
# readme_file_path = "../README.md" # TEST Code
# base_dir = "../" # TEST Code

each_dir_to_time = []
for each_dir_name in os.listdir(base_dir):
    ignore_list = ['script', 'LICENSE', 'requirements.txt']
    if each_dir_name in ignore_list:
        continue
    if '.' not in each_dir_name:
        each_dir_to_time.append([each_dir_name, os.path.getctime(base_dir+each_dir_name)])

most_recent_dir = max(each_dir_to_time, key=lambda x: x[1])[0]
tmp = list(list(most_recent_dir.split('-')))
print(f"each_dir_to_time : {each_dir_to_time}")
category, year, folderName, status = tmp[0], tmp[1], tmp[2], tmp[3]
# print(tmp)
# print(category, folderName, year, status)
curr_folder_path = f"/home/runner/work/ghost_nlp/ghost_nlp/{most_recent_dir}/"
# curr_folder_path = f"../{most_recent_dir}" # Testing Code

target_paper_review_file = ""
checkList = []
for item in os.listdir(curr_folder_path):
    checkList.append(item)
    if ".md" in item or ".markdown" in item:
        target_paper_review_file += item
title = target_paper_review_file.split('.')[0]

res = f"| {category} | {year} | {title} | {status} |"

# Update README.md Contents
readme = open(readme_file_path, "a+")
readme.write(res+'\n')

readme.close()


from __future__ import annotations
from datetime import datetime

from django.db import models
from django.core.validators import MinLengthValidator
from django.urls import reverse

from django.utils.translation import gettext_lazy as _

from common.utils.stocks import get_date_status
from finance_majordomo.users.models import User, Portfolio

class Asset(models.Model):
    asset_types = [
        ('stocks', 'stocks'),
        ('bonds', 'bonds'),
        ('currencies', 'currencies'),
        ('funds', 'funds')
    ]

    creation_date = models.DateTimeField(
        auto_now_add=True, verbose_name="Дата создания")

    asset_type = models.CharField(max_length=20,
                                  choices=asset_types,
                                  verbose_name='Тип актива')

    secid = models.CharField(
        max_length=30,
        verbose_name="ID инструмента",
        unique=True
    )

    name = models.CharField(
        max_length=100,
        verbose_name="Имя акции"
    )

    isin = models.CharField(
        max_length=100,
        verbose_name="ISIN",
        blank=True,
        unique=True
    )

    currency = models.CharField(
        max_length=10,
        verbose_name="Валюта номинала",
        blank=True
    )

    issuedate = models.DateField(
        verbose_name="Дата начала торгов",
        blank=True,
        null=True
    )

    latname = models.CharField(
        max_length=100,
        verbose_name="Английское наименование",
        blank=True
    )

    isqualifiedinvestors = models.BooleanField(
        verbose_name="Бумаги для квалифицированных инвесторов",
        blank=True
    )

    morningsession = models.BooleanField(
        verbose_name="Допуск к утренней дополнительной торговой сессии",
        blank=True
    )

    eveningsession = models.BooleanField(
        verbose_name="Допуск к вечерней дополнительной торговой сессии",
        blank=True
    )

    typename = models.CharField(
        max_length=100,
        verbose_name="Вид/категория ценной бумаги",
        blank=True
    )

    group = models.CharField(
        max_length=100,
        verbose_name="Код типа инструмента",
        blank=True
    )

    type = models.CharField(
        max_length=100,
        verbose_name="Тип бумаги",
        blank=True
    )

    groupname = models.CharField(
        max_length=100,
        verbose_name="Тип инструмента",
        blank=True
    )

    primary_boardid = models.CharField(
        max_length=100,
        verbose_name="Основной режим торгов",
        blank=True
    )

    portfolios = models.ManyToManyField(
        Portfolio,
        through='AssetOfPortfolio',
        through_fields=('asset', 'portfolio'),
        related_name='asset',
    )

    users = models.ManyToManyField(
        User,
        through='AssetsOfUser',
        through_fields=('asset', 'user'),
        related_name='asset1'
    )

    latest_accrual_update = models.DateField(
        verbose_name='Дата последнего обновления информации о начислениях',
        blank=True,
        null=True
    )

    def __str__(self):
        return self.latname

    def get_delete_url(self):
        return reverse("stocks:delete_asset", kwargs={'pk': self.pk})

    def get_related_object(self):
        if self.group == "stock_shares":
            return getattr(self, 'stock')
        elif self.group == "stock_bonds":
            return getattr(self, 'bond')

class Stock(Asset):
    latest_dividend_update = models.DateField(
        verbose_name='Дата последнего обновления информации о дивидендах',
        blank=True,
        null=True
    )

    def __str__(self):
        return self.latname

    def get_delete_url(self):
        return reverse("delete_stock", kwargs={'pk': self.pk})

    class Meta:
        verbose_name = "Акция"
        verbose_name_plural = "Акции"
        ordering = ['creation_date', 'secid', 'name']

class StocksOfUser(models.Model):
    user = models.ForeignKey(
        User,
        on_delete=models.CASCADE
    )

    stock = models.ForeignKey(
        Stock,
        on_delete=models.CASCADE
    )

    class Meta:
        verbose_name = "Акции пользователя"
        verbose_name_plural = "Акции пользователей"
        ordering = ['stock']

class Bond(Asset):

    startdatemoex = models.DateField(
        verbose_name="Дата начала торгов на MOEX",
        blank=True
    )
    buybackdate = models.DateField(
        verbose_name="Дата байбека",
        blank=True,
        null=True
    )
    matdate = models.DateField(
        verbose_name="Дата погашения",
        blank=True
    )
    couponpercent = models.DecimalField(
        max_digits=10,
        decimal_places=4,
        verbose_name="Текущий процент",
        blank=True,
        null=True
    )
    couponfrequency = models.IntegerField(
        verbose_name="Частота выплаты купона в год",
        blank=True
    )
    couponvalue = models.DecimalField(
        max_digits=10,
        decimal_places=4,
        verbose_name="Текущий купон",
        blank=True,
        null=True
    )
    days_to_redemption = models.IntegerField(
        verbose_name="Дней до погашения",
        blank=True,
        null=True
    )
    latest_coupon_update = models.DateField(
        verbose_name='Дата последнего обновления информации о купонах',
        blank=True,
        null=True
    )

    face_value = models.DecimalField(
        max_digits=10,
        decimal_places=4,
        verbose_name="Номинал",
        blank=True,
        null=True
    )

    def __str__(self):
        return self.latname

    def get_delete_url(self):
        return reverse("delete_stock", kwargs={'pk': self.pk})

    class Meta:
        verbose_name = "Облигация"
        verbose_name_plural = "Облигации"
        ordering = ['creation_date', 'secid', 'name']

class BondOfUser(models.Model):
    user = models.ForeignKey(
        User,
        on_delete=models.CASCADE
    )

    bond = models.ForeignKey(
        Bond,
        on_delete=models.CASCADE
    )

    class Meta:
        verbose_name = "Облигации пользователя"
        verbose_name_plural = "Облигации пользователей"
        ordering = ['bond']

class ProdCalendar(models.Model):
    date_status_choice = [
        ('0', 'Working'),
        ('1', 'Nonworking'),
    ]

    date = models.DateField(
        max_length=10,
        verbose_name='Дата'
    )

    date_status = models.CharField(
        max_length=10,
        choices=date_status_choice,
        verbose_name='Статус дня'
    )

    @staticmethod
    def get_date(date: datetime.date) -> ProdCalendar:
        try:
            prod_date = ProdCalendar.objects.get(date=date)

        except ProdCalendar.DoesNotExist:
            try:
                prod_date = ProdCalendar.objects.create(
                    date=date,
                    date_status=get_date_status(date)
                )

            except Exception as e:
                print(e, 'ne smog poluchit date status from internet')
                raise Exception

        return prod_date

    class Meta:
        ordering = ['date', 'date_status']

class AssetsHistoricalData(models.Model):
    asset = models.ForeignKey(Asset, on_delete=models.CASCADE)

    tradedate = models.DateField()
    numtrades = models.IntegerField(null=True)
    value = models.DecimalField(max_digits=15, decimal_places=2, null=True)
    open = models.DecimalField(max_digits=13, decimal_places=5, null=True)
    low = models.DecimalField(max_digits=13, decimal_places=5, null=True)
    high = models.DecimalField(max_digits=13, decimal_places=5, null=True)
    legalcloseprice = models.DecimalField(max_digits=13, decimal_places=5)
    waprice = models.DecimalField(max_digits=13, decimal_places=5, null=True)
    close = models.DecimalField(max_digits=13, decimal_places=5, null=True)
    volume = models.IntegerField(null=True)

    waval = models.DecimalField(max_digits=13, decimal_places=5, null=True)
    trendclspr = models.DecimalField(max_digits=7, decimal_places=3, null=True)

    couponpercent = models.DecimalField(max_digits=13, decimal_places=5,
                                        null=True)
    couponvalue = models.DecimalField(max_digits=13, decimal_places=5,
                                      null=True)
    yieldclose = models.DecimalField(max_digits=13, decimal_places=5, null=True)

    is_closed = models.BooleanField(blank=False)
    update_time = models.DateTimeField(blank=True, null=True)

    class Meta:
        unique_together = ['asset', 'tradedate']

# class SharesHistoricalData(models.Model):
#     share = models.ForeignKey(Stock, on_delete=models.CASCADE)
#
#     tradedate = models.DateField()
#     numtrades = models.IntegerField(null=True)
#     value = models.DecimalField(max_digits=15, decimal_places=2, null=True)
#     open = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     low = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     high = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     legalcloseprice = models.DecimalField(max_digits=13, decimal_places=5)
#     waprice = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     close = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     volume = models.IntegerField(null=True)
#
#     waval = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     trendclspr = models.DecimalField(max_digits=7, decimal_places=3, null=True)
#
#     is_closed = models.BooleanField(blank=False)
#     update_time = models.DateTimeField(blank=True, null=True)
#
#     class Meta:
#         unique_together = ['share', 'tradedate']

# class BondsHistoricalData(models.Model):
#     bond = models.ForeignKey(Bond, on_delete=models.CASCADE)
#
#     tradedate = models.DateField()
#     numtrades = models.IntegerField(null=True)
#     value = models.DecimalField(max_digits=15, decimal_places=2, null=True)
#     open = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     low = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     high = models.DecimalField(max_digits=13, vm4decimal_places=5, null=True)
#     legalcloseprice = models.DecimalField(max_digits=13, decimal_places=5)
#     waprice = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     close = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     volume = models.IntegerField(null=True)
#
#     couponpercent = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     couponvalue = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     yieldclose = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#
#
#     #waval = models.DecimalField(max_digits=13, decimal_places=5, null=True)
#     #trendclspr = models.DecimalField(max_digits=7, decimal_places=3, null=True)
#
#     is_closed = models.BooleanField(blank=False)
#     update_time = models.DateTimeField(blank=True, null=True)
#
#     class Meta:
#         unique_together = ['bond', 'tradedate']

class AssetOfPortfolio(models.Model):
    asset = models.ForeignKey(
        Asset,
        on_delete=models.CASCADE
    )

    portfolio = models.ForeignKey(
        Portfolio,
        on_delete=models.CASCADE
    )

    # def get_purchase_price(self, currency=None, date=None):
    #     from ..transactions.services.transaction_calculation_services import \
    #         get_purchase_price
    #     return get_purchase_price(
    #         self.portfolio.id, self.asset.id, currency=currency, date=date
    #     )

    class Meta:
        verbose_name = "Объекты портфеля"
        verbose_name_plural = "Объекты портфеля"
        ordering = ['portfolio']

class AssetsOfUser(models.Model):
    user = models.ForeignKey(
        User,
        on_delete=models.CASCADE
    )

    asset = models.ForeignKey(
        Asset,
        on_delete=models.CASCADE
    )

    class Meta:
        verbose_name = "Активы пользователя"
        verbose_name_plural = "Активы пользователей"
        ordering = ['user']


import argparse
import asyncio
import os
import random
import string
import sys
import uuid
from datetime import datetime
import urllib.request

from hashids import Hashids
from quart import Quart, request, websocket, abort, redirect

async def get_data(obj):
    files = await obj.form

    if files:
        data = "\n".join((
            " ".join(i) if isinstance(i, list) else str(i) for i in files.values()
        ))

        if data.strip() == "":
            data = (await obj.get_data()).decode()
    else:
        data = (await obj.get_data()).decode()

    return data

class PubSub:
    def __init__(self):
        self.buffer = []
        self.subscribers = {}

    def push(self, data):
        self.buffer.append(data)

    async def get(self):
        identifier = uuid.uuid4()
        self.subscribers[identifier] = len(self.buffer)

        while True:
            while len(self.buffer) != self.subscribers[identifier]:
                yield self.buffer[self.subscribers[identifier]]
                self.subscribers[identifier] += 1
            await asyncio.sleep(0.1)

class ArgsHandler:
    def __init__(self):
        self.dir = "logs"
        self.port = 8000
        self.url = "ws://localhost:8000"

args = ArgsHandler()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="This program logs any request with any data it gets.")

    parser.add_argument("-d", "--dir", type=str, help="Directory to write requests to. Can alternatively be set with "
                                                      "NETECHO_DIR in envvars.", default="logs")
    parser.add_argument("-p", "--port", type=int, help="File to write data to. Can alternatively be set with "
                                                       "NETECHO_PORT in envvars.", default=8000)
    parser.add_argument("-u", "--url", type=str, help="url netecho runs on, for websockets. Can alternatively be set "
                                                      "with NETECHO_URL in envvars.", default="ws://localhost:8000")

    args = parser.parse_args(sys.argv[1:])

args.port = int(os.environ.get('NETECHO_PORT', args.port))
args.dir = os.environ.get('NETECHO_DIR', args.dir)
args.url = os.environ.get('NETECHO_URL', args.url)

app = Quart(__name__)

if not os.path.exists(args.dir):
    os.mkdir(args.dir)

symlinkpath = os.path.join(args.dir, "latest")

try:
    os.unlink(symlinkpath)
except:
    pass

filename = f"{datetime.now().strftime('%Y-%m-%d-%H:%M:%S')}.log"
filepath = os.path.join(args.dir, filename)
logfile = open(filepath, "a", buffering=1)
os.symlink(filename, symlinkpath)

hashids = Hashids(salt=''.join(random.choice(string.printable) for i in range(50)), min_length=8)

buffers = dict()

@app.route('/', methods=['POST'])
async def log():
    print(await get_data(request), file=logfile)
    return '', 200

@app.route('/stdout', methods=['POST'])
async def logstdout():
    print(await get_data(request), file=sys.stdout)
    return '', 200

@app.route('/stderr', methods=['POST'])
async def logstderr():
    print(await get_data(request), file=sys.stderr)
    return '', 200

@app.route('/', methods=['GET'])
async def public():
    next_url = hashids.encode(len(buffers))
    buffers[next_url] = PubSub()
    return redirect(f"/{next_url}")

@app.route("/<key>", methods=['GET'])
async def route_id(key: str):
    if key not in buffers or key == "favicon.ico":
        abort(404)
    return f"""

use echo "whatever" | curl -LF 'f=<-' -H 'Host:{str(args.url).split('://', 2)[1]}' {str(args.url).replace("ws", "http", 1)}/{key}
             to post to this url.
 If no DNS is available, it is possible to replace the last?? {str(args.url).replace("ws", "http", 1)}/
            with the netecho server's IP address. This works even if netecho runs behind a reverse proxy because of the -H argument.

            """

@app.route('/<key>', methods=['POST'])
async def logkey(key):
    if key not in buffers:
        abort(404)

    buffers[key].push((await get_data(request)).replace("<", "<"))
    return '', 200

@app.websocket("/<key>/ws")
async def route_ws(key):
    async for i in buffers[key].get():
        await websocket.send(i)

print(f"running on port:{args.port} logging to {filepath}")
print(f"running on port:{args.port} logging to {filepath}", file=logfile)

if __name__ == '__main__':
    app.run(
        host='0.0.0.0',
        port=args.port,
    )


# -*- coding: UTF-8 -*-

'''
Given a non-empty array of integers, return the k most frequent elements.
'''

# 1. Quick Select
# 2. Minimum Heap
# 3. collections.Counter()

class heapify(object):
    def topKFrequent(self, nums, k):
        """
        :type nums: List[int]
        :type k: int
        :rtype: List[int]
        """
        def swap(nums, index1, index2):
            nums[index1], nums[index2] = nums[index2], nums[index1]

        def flowup(nums, data):
            nums.append(data)
            index = len(nums) - 1
            parent_index = index >> 1
            while parent_index is not None and parent_index != 0:
                if nums[index][1] < nums[parent_index][1]:
                    swap(nums, index, parent_index)
                    index = parent_index
                    parent_index = index >> 1
                else:
                    break
            return

        def bubbledown(nums, data):
            nums.append(data)
            if nums[1][1] > nums[len(nums)-1][1]:
                del nums[len(nums) - 1]
                return
            else:
                swap(nums, 1, len(nums) - 1)
                del nums[len(nums) - 1]
            last_index = len(nums) - 1
            index = 1
            while True:
                smallest_index = index
                if 2 * index <= last_index and nums[2 * index][1] < nums[smallest_index][1]:
                    smallest_index = 2 * index
                if 2 * index + 1 <= last_index and nums[2 * index + 1][1] < nums[smallest_index][1]:
                    smallest_index = 2 * index + 1
                if smallest_index == index:
                    break
                swap(nums, index, smallest_index)
                index = smallest_index
            return

        map_dict = {}
        for i in nums:
            if i in map_dict.keys():
                map_dict[i] += 1
            else:
                map_dict[i] = 1

        map_arr = list(map_dict.items())
        heap = [(0, 0)]
        for i in map_arr:
            if len(heap) < k + 1:
                flowup(heap, i)
            else:
                bubbledown(heap, i)
        return [i[0] for i in heap[-1:0:-1]]

class Solution:
    def topKFrequent(self, nums, k):
        def heapify(arr, n, i):
            smallest = i  # 构造根节点与左右子节点
            l = 2 * i + 1
            r = 2 * i + 2
            if l < n and arr[l][1] < arr[i][1]:  # 如果左子节点在范围内且小于父节点
                smallest = l
            if r < n and arr[r][1] < arr[smallest][1]:
                smallest = r
            if smallest != i:  # 递归基:如果没有交换，退出递归
                arr[i], arr[smallest] = arr[smallest], arr[i]
                heapify(arr, n, smallest)  # 确保交换后，小于其左右子节点

        # 哈希字典统计出现频率
        map_dict = {}
        for item in nums:
            if item not in map_dict.keys():
                map_dict[item] = 1
            else:
                map_dict[item] += 1

        map_arr = list(map_dict.items())
        lenth = len(map_dict.keys())
        # 构造规模为k的minheap
        if k <= lenth:
            k_minheap = map_arr[:k]
            # 从后往前建堆，避免局部符合而影响递归跳转，例:2,1,3,4,5,0
            for i in range(k // 2 - 1, -1, -1):
                heapify(k_minheap, k, i)
            # 对于k:, 大于堆顶则入堆，维护规模为k的minheap
            for i in range(k, lenth): # 堆建好了，没有乱序，从前往后即可
                if map_arr[i][1] > k_minheap[0][1]:
                    k_minheap[0] = map_arr[i] # 入堆顶
                    heapify(k_minheap, k, 0)  # 维护 minheap
        # 如需按顺序输出，对规模为k的堆进行排序
        # 从尾部起，依次与顶点交换再构造minheap，最小值被置于尾部
        for i in range(k - 1, 0, -1):
            k_minheap[i], k_minheap[0] = k_minheap[0], k_minheap[i]
            k -= 1 # 交换后，维护的堆规模-1
            heapify(k_minheap, k, 0)
        return [item[0] for item in k_minheap]

class BucketSort:
    def topKFrequent(self, nums, k):
        map_dict = {}
        for i in nums:
            map_dict[i] = map_dict.get(i, 0) + 1
        max_time = max(map_dict.values())
        bucket_list = [[] for i in range(max_time+1)]
        for key, value in map_dict.items():
            bucket_list[value].append(key)
        res = []
        for i in range(max_time, -1, -1):
            if bucket_list[i]:
                res.extend(bucket_list[i])
            if len(res) >= k:
                return res[:k]

if __name__ == '__main__':
    print(BucketSort().topKFrequent(nums=[1,1,1,1,1,2,2,2,3,3,4,4,5,6], k=4))
    for i in range(6 // 2 - 1, -1, -1):
        print(i)


import sys
sys.path.append('../')
import cv2
import os.path
from help import show_picture as sp
from help import text_augment as ta
from tqdm import tqdm
import copy
import random
from multiprocessing import Pool
import math

ShowPicture = False
Channels = 3
TrainTextAug = False
TextAugRandom = False
TrainTextAug_manual = True
test = False
ResizeH = 608
ResizeW = 800
AugLoop = 100  # 229 per Loop
Aug_path = '../dataset/icdar2013_train_augment/'
Aug_GT_path = '../dataset/icdar2013_train_augment_gt/'
if not os.path.exists(Aug_path):
    os.mkdir(Aug_path)
if not os.path.exists(Aug_GT_path):
    os.mkdir(Aug_GT_path)

def classMapping(cls):
    class_list = ['foreground']

    if cls not in class_list:
        print('The current class is wrong.')
        os._exit(0)
    else:
        return str(class_list.index(cls))

# 获取样本信息
def get_data(input_path, PATH_NOTICE=True):
    all_imgs = {}
    classes_count = {}
    class_mapping = {}
    image_num = 0
    image_width_sum = 0
    image_width_max = 0
    image_width_min = 5000
    image_height_sum = 0
    image_height_max = 0
    image_height_min = 5000

    # 把样本以字典形式存储
    with open(input_path, 'r', encoding='utf-8') as f:
        print('Parsing annotation files')
        for line in tqdm(f):
            line_split = line.strip().split('\t')
            # if len(line_split) > 11:

            # print(line_split)
            (filename, x1, y1, x2, y2, class_name) = line_split
            # print(class_name)

            if x1 == x2 == y1 == y2 == '-1':
                continue
            else:
                if int(x1) < 0 or int(x2) < 0 or int(y1) < 0 or int(y2) < 0:
                    print('The coordinates are out of the image.')
                    os._exit(0)

                if PATH_NOTICE:
                    filename = filename[1:]

                class_name = 'foreground'

                if class_name not in classes_count:
                    classes_count[class_name] = 1
                else:
                    classes_count[class_name] += 1

                if classMapping(class_name) not in class_mapping:
                    class_mapping[classMapping(class_name)] = class_name  # 注意，mapping设置

                if filename not in all_imgs:

                    all_imgs[filename] = {}
                    # print (os.path.isfile(filename))
                    img = cv2.imread(filename)
                    # print(img.shape)
                    (rows, cols, channels) = img.shape

                    # print(img.shape)
                    # resize_img = cv2.resize(img, (ResizeShape_width, ResizeShape_height))
                    # height_ratio = ResizeShape_height / rows
                    # width_ratio = ResizeShape_width / cols

                    # (rows, cols, channels) = resize_img.shape
                    # print(resize_img.shape)
                    # input('Stop!')
                    if channels != 3:
                        print('There is a mistake.')
                        os._exit(0)

                    image_num += 1
                    image_width_sum += cols
                    image_height_sum += rows

                    if cols > image_width_max:
                        image_width_max = cols
                    if cols < image_width_min:
                        image_width_min = cols
                    if rows > image_height_max:
                        image_height_max = rows
                    if rows < image_height_min:
                        image_height_min = rows

                    all_imgs[filename]['filepath'] = filename
                    all_imgs[filename]['height'] = rows
                    all_imgs[filename]['width'] = cols
                    all_imgs[filename]['channel'] = channels
                    # all_imgs[filename]['pixel'] = img

                    # all_imgs[filename]['ignore'] = int(ign)

                    # all_imgs[filename]['width_ratio'] = width_ratio
                    # all_imgs[filename]['height_ratio'] = height_ratio

                    all_imgs[filename]['bboxes'] = []
                    # FLAG = np.random.randint(0, 39)
                    # if FLAG > 0:
                    all_imgs[filename]['imageset'] = 'train'
                    # else:
                    #     all_imgs[filename]['imageset'] = 'validation'

                # 每个图有多个框
                # x1 = max(int(math.ceil((float(x1) + 1) * width_ratio - 1)), 0)
                # x2 = min(int(math.floor(float(x2) * width_ratio)), cols - 1)
                # y1 = max(int(math.ceil((float(y1) + 1) * height_ratio - 1)), 0)
                # y2 = min(int(math.floor(float(y2) * height_ratio)), rows - 1)

                all_imgs[filename]['bboxes'].append({'class': classMapping(class_name),
                                                     'x1': int(x1), 'y1': int(y1), 'x2': int(x2), 'y2': int(y2)})

        all_data = []
        for key in all_imgs:
            all_data.append(all_imgs[key])

        print('The max width is {}.'.format(image_width_max))
        print('The min width is {}.'.format(image_width_min))
        print('The average width is {}.'.format(image_width_sum / image_num))

        print('The max height is {}.'.format(image_height_max))
        print('The min height is {}.'.format(image_height_min))
        print('The average height is {}.'.format(image_height_sum / image_num))

        return all_data, class_mapping, classes_count

def showPicture(one_image):
    coords = []
    for bn in range(len(one_image['bboxes'])):
        coords.append([one_image['bboxes'][bn]['x1'], one_image['bboxes'][bn]['y1'],
                       one_image['bboxes'][bn]['x2'], one_image['bboxes'][bn]['y2']])
    sp.show_pic(one_image['pixel'], coords)

def generateAugData(img, bboxes, path, iteration, mark):
    img_path = os.path.join(Aug_path, mark + str(iteration) + '_' + os.path.basename(path))
    cv2.imwrite(img_path, img)
    gt_path = os.path.join(Aug_GT_path, mark + str(iteration) + '_' + os.path.basename(path).split('.')[0] + '.txt')
    with open(gt_path, 'w') as f:
        for i in range(len(bboxes)):
            x1 = bboxes[i]['x1']
            y1 = bboxes[i]['y1']
            x2 = bboxes[i]['x2']
            y2 = bboxes[i]['y2']
            class_name = bboxes[i]['class']
            f.write('{} {} {} {} {}\n'.format(x1, y1, x2, y2, class_name))

    tmp_dict = {}
    tmp_dict['filepath'] = 'Augmented file, No path.'
    tmp_dict['width'] = img.shape[1]
    tmp_dict['height'] = img.shape[0]
    tmp_dict['channel'] = Channels
    tmp_dict['pixel'] = img
    tmp_dict['bboxes'] = bboxes
    # showPicture(tmp_dict)
    return tmp_dict

def readImage(path):
    return cv2.imread(path)

def textAugment(all_data, Aug_width=False, Aug_height=False, Aug_resize=False,
                Aug_rotate=False, Aug_rotate_height=False, Aug_rotate_width=False,
                Aug_add_noise=False, Aug_change_light=False, Aug_crop=False,
                Aug_shift=False, iteration=None):
    text_aug = ta.TextAugment()

    if Aug_width:
        for i in range(len(all_data)):
            img, bboxes = text_aug.resizeTextWidth(readImage(all_data[i]['filepath']), all_data[i]['bboxes'])
            tmp_dict = generateAugData(img, bboxes, all_data[i]['filepath'], iteration, mark='wid')
            if ShowPicture:
                showPicture(tmp_dict)
            # images.append(tmp_dict)
        # image_02 =copy.deepcopy(image_)
    # input('stop1')

    if Aug_height:
        for i in range(len(all_data)):
            img, bboxes = text_aug.resizeTextHeight(readImage(all_data[i]['filepath']), all_data[i]['bboxes'])
            tmp_dict = generateAugData(img, bboxes, all_data[i]['filepath'], iteration, mark='hei')
            if ShowPicture:
                showPicture(tmp_dict)
            # images.append(tmp_dict)
        # image_03 =copy.deepcopy(image_02)

    if Aug_resize:
        for i in range(len(all_data)):
            img, bboxes = text_aug.resizeText(readImage(all_data[i]['filepath']), all_data[i]['bboxes'])
            tmp_dict = generateAugData(img, bboxes, all_data[i]['filepath'], iteration, mark='res')
            if ShowPicture:
                showPicture(tmp_dict)
            # images.append(tmp_dict)

    if Aug_rotate:
        for i in range(len(all_data)):
            img, bboxes = text_aug.rotateText(readImage(all_data[i]['filepath']), all_data[i]['bboxes'])
            tmp_dict = generateAugData(img, bboxes, all_data[i]['filepath'], iteration, mark='rot')
            if ShowPicture:
                showPicture(tmp_dict)
            # images.append(tmp_dict)

    if Aug_rotate_height:
        for i in range(len(all_data)):
            img, bboxes = text_aug.resizeTextHeight(readImage(all_data[i]['filepath']), all_data[i]['bboxes'])
            img, bboxes = text_aug.rotateText(img, bboxes)
            tmp_dict = generateAugData(img, bboxes, all_data[i]['filepath'], iteration, mark='rhe')
            if ShowPicture:
                showPicture(tmp_dict)
            # images.append(tmp_dict)

    if Aug_rotate_width:
        for i in range(len(all_data)):
            img, bboxes = text_aug.resizeTextWidth(readImage(all_data[i]['filepath']), all_data[i]['bboxes'])
            img, bboxes = text_aug.rotateText(img, bboxes)
            tmp_dict = generateAugData(img, bboxes, all_data[i]['filepath'], iteration, mark='rwi')
            if ShowPicture:
                showPicture(tmp_dict)
            # images.append(tmp_dict)

    if Aug_add_noise:
        for i in range(len(all_data)):
            img = text_aug.addNoise(readImage(all_data[i]['filepath']))
            tmp_dict = generateAugData(img, all_data[i]['bboxes'], all_data[i]['filepath'], iteration, mark='noi')
            if ShowPicture:
                showPicture(tmp_dict)
            # images.append(tmp_dict)

    if Aug_change_light:
        for i in range(len(all_data)):
            img = text_aug.changeLight(readImage(all_data[i]['filepath']))
            tmp_dict = generateAugData(img, all_data[i]['bboxes'], all_data[i]['filepath'], iteration, mark='lig')
            if ShowPicture:
                showPicture(tmp_dict)
            # images.append(tmp_dict)

    if Aug_crop:
        for i in range(len(all_data)):
            img, bboxes = text_aug.cropText(readImage(all_data[i]['filepath']), all_data[i]['bboxes'])
            tmp_dict = generateAugData(img, bboxes, all_data[i]['filepath'], iteration, mark='crp')
            if ShowPicture:
                showPicture(tmp_dict)
            # images.append(tmp_dict)

    if Aug_shift:
        for i in range(len(all_data)):
            img, bboxes = text_aug.shiftText(readImage(all_data[i]['filepath']), all_data[i]['bboxes'])
            tmp_dict = generateAugData(img, bboxes, all_data[i]['filepath'], iteration, mark='shf')
            if ShowPicture:
                showPicture(tmp_dict)

def textAugment_random(all_data, Aug_width=False, Aug_height=False, Aug_resize=False,
                       Aug_rotate=False, Aug_rotate_height=False, Aug_rotate_width=False,
                       Aug_add_noise=False, Aug_change_light=False, Aug_crop=False,
                       Aug_shift=False, iteration=None):
    text_aug = ta.TextAugment()
    img = readImage(all_data['filepath'])
    bboxes = all_data['bboxes']

    if Aug_width:
        img, bboxes = text_aug.resizeTextWidth(img, bboxes)

    if Aug_height:
        img, bboxes = text_aug.resizeTextHeight(img, bboxes)

    if Aug_resize:
        img, bboxes = text_aug.resizeText(img, bboxes)

    if Aug_rotate:
        img, bboxes = text_aug.rotateText(img, bboxes)

    if Aug_rotate_height:
        img, bboxes = text_aug.resizeTextHeight(img, bboxes)

    if Aug_rotate_width:
        img, bboxes = text_aug.resizeTextWidth(img, bboxes)

    if Aug_add_noise:
        img = text_aug.addNoise(img)

    if Aug_change_light:
        img = text_aug.changeLight(img)

    if Aug_crop:
        img, bboxes = text_aug.cropText(img, bboxes)

    if Aug_shift:
        img, bboxes = text_aug.shiftText(img, bboxes)

    tmp_dict = generateAugData(img, bboxes, all_data['filepath'], iteration, mark='mix')
    if ShowPicture:
        showPicture(tmp_dict)

def textAugment_manual(all_data, Aug_width=False, Aug_height=False, Aug_resize=False,
                       Aug_rotate=False, Aug_rotate_height=False, Aug_rotate_width=False,
                       Aug_add_noise=False, Aug_change_light=False, Aug_crop=False,
                       Aug_shift=False, Aug_color=False, Aug_spot=False, iteration=None):

    text_aug = ta.TextAugment()
    img = readImage(all_data['filepath'])
    bboxes = all_data['bboxes']

    if Aug_width:
        img, bboxes = text_aug.resizeTextWidth(img, bboxes)

    if Aug_height:
        img, bboxes = text_aug.resizeTextHeight(img, bboxes)

    if Aug_resize:
        img, bboxes = text_aug.resizeText(img, bboxes)

    if Aug_rotate:
        img, bboxes = text_aug.rotateText(img, bboxes)

    if Aug_rotate_height:
        img, bboxes = text_aug.resizeTextHeight(img, bboxes)

    if Aug_rotate_width:
        img, bboxes = text_aug.resizeTextWidth(img, bboxes)

    if Aug_add_noise:
        img = text_aug.addNoise(img)

    if Aug_change_light:
        img = text_aug.changeLight(img)

    if Aug_crop:
        img, bboxes = text_aug.cropText(img, bboxes)

    if Aug_shift:
        img, bboxes = text_aug.shiftText(img, bboxes)

    if Aug_color:
        img, bboxes = text_aug.randomColor(img, bboxes)

    if Aug_spot:
        img, bboxes = text_aug.randomSpot(img, bboxes)

    tmp_dict = generateAugData(img, bboxes, all_data['filepath'], iteration, mark='mix')
    if ShowPicture:
        showPicture(tmp_dict)

def multi_pro(all_images, end_index):

    print("end image is:" + end_index.__str__())
    flag = []
    cnt = 1
    flag.append([False] * 5)

    for ii in range(5):
        tmp = [False] * 5
        tmp[ii] = True
        flag.append(tmp)
        cnt += 1

    # rs = [0, 1]
    # rt = [2, 3, 4]
    # nl = [5, 6]
    # cs = [7, 8]
    # cl = [9]
    # sp = [10]
    # sw = [rs, rt, nl, cs, cl, sp]

    # rs = [0, 1]
    rt = [0]
    nl = [1]
    cs = [2]
    cl = [3]
    sp = [4]
    sw = [rt, nl, cs, cl]

    for ii in range(3):
        for jj in range(ii + 1, 4):
            a = sw[ii]
            b = sw[jj]
            for aa in a:
                for bb in b:
                    tmp = [False] * 5
                    tmp[bb] = True
                    tmp[aa] = True
                    flag.append(tmp)
                    cnt += 1

    if test:
        flag = [[False, False, False, False, False]]
    print(flag)
    print(cnt)

    for ii in range(cnt):

        print('The loop is ', ii)
        Aug_width = False
        Aug_height = False
        Aug_resize = False

        Aug_rotate = flag[ii][0]
        Aug_rotate_height = False
        Aug_rotate_width = False

        Aug_add_noise = False
        Aug_change_light = flag[ii][1]

        Aug_crop = flag[ii][2]
        Aug_shift = False

        Aug_color = flag[ii][3]

        Aug_spot = flag[ii][4]

        for j in tqdm(range(len(all_images))):
            print(all_images[j]['filepath'])
            textAugment_manual(copy.deepcopy(all_images[j]), Aug_width=Aug_width, Aug_height=Aug_height,
                               Aug_resize=Aug_resize, Aug_rotate=Aug_rotate,
                               Aug_rotate_height=Aug_rotate_height,
                               Aug_rotate_width=Aug_rotate_width, Aug_add_noise=Aug_add_noise,
                               Aug_change_light=Aug_change_light, Aug_crop=Aug_crop,
                               Aug_shift=Aug_shift, Aug_color=Aug_color, Aug_spot=Aug_spot, iteration=ii)

def main():
    # 注释文件
    ANNOTATIONS_FILE = '../sample/icdar2013_annotations_train_set.txt'
    # 返回样本的所有信息，各类别数目，以及类别映射
    all_images, classes_count, class_mapping = get_data(ANNOTATIONS_FILE, PATH_NOTICE=False)
    print(len(all_images))
    print(classes_count)
    print(class_mapping)

    if TextAugRandom:
        MultiNum = 28  # 并发线程数
        print("MultiProcess:" + MultiNum.__str__())
        # 新建指定数量的进程池用于对多进程进行管理
        pool = Pool(processes=MultiNum)

        for ii in range(AugLoop):
            print('The loop is ', ii)
            pool.apply_async(multi_pro, args=(all_images, ii))

        pool.close()
        pool.join()

    # input('STOP!!')

    if TrainTextAug:
        for ii in range(AugLoop):
            aug_img = copy.deepcopy(all_images)

            textAugment(copy.deepcopy(aug_img), Aug_width=True, iteration=ii)
            textAugment(copy.deepcopy(aug_img), Aug_height=True, iteration=ii)
            # textAugment(copy.deepcopy(aug_img), Aug_resize=True, iteration=ii)
            textAugment(copy.deepcopy(aug_img), Aug_rotate=True, iteration=ii)
            textAugment(copy.deepcopy(aug_img), Aug_rotate_height=True, iteration=ii)
            textAugment(copy.deepcopy(aug_img), Aug_rotate_width=True, iteration=ii)
            textAugment(copy.deepcopy(aug_img), Aug_change_light=True, iteration=ii)
            textAugment(copy.deepcopy(aug_img), Aug_crop=True, iteration=ii)
            textAugment(copy.deepcopy(aug_img), Aug_shift=True, iteration=ii)
            textAugment(copy.deepcopy(aug_img), Aug_add_noise=True, iteration=ii)

    if TrainTextAug_manual:

        MultiNum = 28  # 并发线程数
        print("MultiProcess:" + MultiNum.__str__())
        # 新建指定数量的进程池用于对多进程进行管理
        pool = Pool(processes=MultiNum)

        # 划分任务, tmp 代表每一个进程完成的任务数
        tmp = int(math.ceil(len(all_images) / MultiNum))
        # print(tmp)
        for i in range(MultiNum):
            print("MultiProcess:" + i.__str__())
            pool.apply_async(multi_pro, args=(all_images[i * tmp:i * tmp + tmp], i * tmp + tmp))

        pool.close()
        pool.join()

if __name__ == '__main__':
    main()


def subtree(V):
    global cnt
    if V == 0:
        return
    cnt += 1
    subtree(left[V])
    subtree(right[V])

T = int(input())
for tc in range(1, T + 1):
    E, N = map(int, input().split())
    inp = list(map(int, input().split()))

    left = [0] * (E + 2)
    right = [0] * (E + 2)

    for i in range(len(inp) // 2):
        p, c = inp[i * 2], inp[i * 2 + 1]
        if not left[p]:
            left[p] = c
        else:
            right[p] = c

    cnt = 0
    subtree(N)
    print(f'#{tc} {cnt}')


import torch
import torch.nn as nn
import torch.nn.functional as F

class Model(nn.Module):
    def __init__(self,in_chans=1,kernel_size=5,padding=2,stride=2):
        super(Model,self).__init__()
        self.f1=nn.Conv2d(in_chans, 6, kernel_size=kernel_size,padding=padding)
        self.f2=nn.Conv2d(6, 16, kernel_size=kernel_size)
        self.l1=nn.Linear(1600, 120)
        self.l2= nn.Linear(120, 84)
        self.l3= nn.Linear(84, 10)
        self.GAP=nn.AvgPool2d(kernel_size=kernel_size, stride=stride)

    def forward(self, input):
        x=self.f1(input)
        x=nn.Sigmoid()(x)
        x=self.f2(x)
        x=nn.Sigmoid()(x)
        x=self.GAP(x)
        x=nn.Flatten()(x)
        x=self.l1(x)
        x=nn.Sigmoid()(x)
        x=self.l2(x)
        x=nn.Sigmoid()(x)
        x=self.l3(x)

        return x

if __name__=='__main__':
    model=Model()
    input = torch.randn(1,1,28,28)
    from thop import profile
    flops, params = profile(model, inputs=(input,))
    print(flops, params)


######################################################################
#       Filename: data-processing.py                                 #
#                                                                    #
#       Author: Mirza Avdic                                          #
#       Date:  Jul 07, 2018                                          #
#                                                                    #
######################################################################

import os
import re
import StringIO
import shutil
from utilities import *
import datetime
import subprocess

bashCommand = "rm -r *backup*"
try:
    output = subprocess.check_output(['bash','-c', bashCommand])
except subprocess.CalledProcessError as e:
    print("All backup folders already deleted")

# Name of the folder where the processed data is stored
processed_folder_absolute = "measurement-records-processed-absolute"
processed_folder_relative = "measurement-records-processed-relative"
# Creating folders
if os.path.exists(processed_folder_absolute):
    # make a backup of the old data
    shutil.copytree(processed_folder_absolute, processed_folder_absolute + "-backup-" +  datetime.datetime.now().strftime('%d-%m-%Y-%H-%M-%S'))
    # Clear previous data if exists
    shutil.rmtree(processed_folder_absolute, ignore_errors=True)
    # create a fresh one
    os.mkdir(processed_folder_absolute)
else:
    shutil.rmtree(processed_folder_absolute, ignore_errors=True)

if os.path.exists(processed_folder_relative):
    # make a backup of the old data
    shutil.copytree(processed_folder_relative, processed_folder_relative + "-backup-" +  datetime.datetime.now().strftime('%d-%m-%Y-%H-%M-%S'))
    # Clear previous data if exists
    shutil.rmtree(processed_folder_relative, ignore_errors=True)
    # create a fresh one
    os.mkdir(processed_folder_relative)
else:
    shutil.rmtree(processed_folder_relative, ignore_errors=True)

scenarios=[]

# find all scenarios for which the measurements have been done
for dirname, dirnames, filenames in os.walk('./measurement-records-raw'):
    # print path to all subdirectories first.
    expr=(subdirname for subdirname in dirnames if re.match('scenario[0-9]', subdirname))
    for subdirname in expr:
        print(os.path.join(dirname, subdirname))
        scenario = Scenario(os.path.join(dirname, subdirname))
        scenarios.append(scenario)

    # print path to all filenames.
    #expr=(filename for filename in filenames if re.match('sw_*', filename))
    #for filename in expr:
    #    scenario = Scenario(path_from_list(dirname.split("/")[1:-1]))
    #    print(scenario.path)
    #    print(os.path.join(dirname, filename))

print("Considered scenarios:")
print(scenarios)
print("")
for scenario in scenarios:
    print("Processing scenario: " + scenario.path.split("/")[-1])
    scenario.process_scenario()
    print(scenario.succ_iteration_indexes)
    print(scenario.unsucc_iteration_indexes)
    scenario.create_output_absolute(processed_folder_absolute + "/" + scenario.path.split("/",2)[2])
    scenario.create_output_relative(processed_folder_relative + "/" + scenario.path.split("/",2)[2])
    #for switch in scenario.switches.values():
    #    print(switch.name)
    #    print(switch.get_min_max_ip_assigned_timestamp())
    print(scenario.get_iteration_summary(0))


from asyncio.windows_events import NULL
from cProfile import label
import datetime as dt
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.colors as mtp
from mplcursors import cursor as cursor
#cursor(hover=True)
import seaborn as sns
import decimal
import os
uniqueUser = pd.read_csv(r'C:\Unimi\VisualizzazioneScientifica\dataset\open_wifi_project\OWM_SciVis\Datasets\uniqueuserzone.csv', sep=';')
uniqueUser = pd.DataFrame (uniqueUser, columns=[ 'Tipologia_API', 'Zona', 'Data', 'Valore'])
uniqueUser['Data'] = pd.to_datetime(uniqueUser['Data'])
sns.set()
colorArray = [[1, 0 , 0, 1]]
trendUser = uniqueUser.groupby("Data")["Valore"].sum().reset_index(name = "Somma")
#print(trendUser.iloc[1])
trendUser.plot(x = "Data", y = "Somma", title = "Trend utenti unici giornalieri in tutta Milano", xlabel="", linestyle="solid", legend=False, linewidth=1.5,  color="#ff0000")
plt.xticks(rotation=0)
plt.show()

Settimana = {
    0: 'Lunedì',
    1: 'Martedì',
    2: 'Mercoledì',
    3: 'Giovedì',
    4: 'Venerdì',
    5: 'Sabato',
    6: 'Domenica',
}
Municipio = {
    "OWM Zona1" : "Municipio 1",
    "OWM Zona2" : "Municipio 2",
    "OWM Zona3" : "Municipio 3",
    "OWM Zona4" : "Municipio 4",
    "OWM Zona5" : "Municipio 5",
    "OWM Zona6" : "Municipio 6",
    "OWM Zona7" : "Municipio 7",
    "OWM Zona8" : "Municipio 8",
    "OWM Zona9" : "Municipio 9",
}
asseX = {
    'x': list(Settimana.values())
}
def dayOfTheWeek(data):
    return dt.datetime.weekday(data)
colorArray = [[1, 0 , 0, 1]]
cmap = mtp.ListedColormap(colorArray)

uniqueUser['DayOfTheWeek'] = list(map(dayOfTheWeek, uniqueUser['Data']))
uniqueUser['DayOfTheWeekString'] = list(map(lambda x: Settimana[x], uniqueUser['DayOfTheWeek']))
uniqueUserByDayANDZone = uniqueUser.groupby(['Zona', 'DayOfTheWeek', 'DayOfTheWeekString'])['Valore'].sum().reset_index(name = 'Total')

for i in range(1,10,1):

    zonaInteressata = "OWM Zona"+str(i)
    MunicipioInteressato = "Municipio " +str(i)
    print(zonaInteressata)
    uniqueUserZoneVariable= uniqueUserByDayANDZone.loc[uniqueUserByDayANDZone['Zona'] == zonaInteressata, ['Zona', 'DayOfTheWeek', 'DayOfTheWeekString', 'Total']]
    uniqueUserZoneVariable.sort_values(by='DayOfTheWeek')
    print(uniqueUserZoneVariable)
    plotZone = uniqueUserZoneVariable.plot.bar(x = "DayOfTheWeekString", y= "Total", legend = False, xlabel = "",tick_label=1, title = MunicipioInteressato, colormap = cmap)

    for p in plotZone.patches:
        plotZone.annotate(str(p.get_height()), (p.get_x()+(p.get_width()*0.5), p.get_height() *1.005),  ha="center")
    plt.xticks(rotation=0)
    plt.show()
    #print("i incrementata", i)

zoneMoreUtilized = uniqueUser.groupby(['Zona'])['Valore'].mean().reset_index(name = "Mean")
zoneMoreUtilized['Zona'] = list(map(lambda x: Municipio[x], zoneMoreUtilized['Zona']))
print(zoneMoreUtilized)
plotMoreUtilized = zoneMoreUtilized.plot.bar( x = "Zona", y= "Mean", legend = False, xlabel = "", title = "Media utenti giornalieri per Municipio", colormap = cmap)
for p in plotMoreUtilized.patches:
    plotMoreUtilized.annotate(str(round(p.get_height(),2)), (p.get_x()+(p.get_width()*0.5), p.get_height() *1.005),  ha="center")
plt.xticks(rotation = 360)
plt.show()


from math import exp

import torch
import torch.nn.functional as F
from torch.autograd import Variable
from scipy.spatial.distance import minkowski

def gaussian(window_size, sigma):
    gauss = torch.Tensor(
        [
            exp(-((x - window_size // 2) ** 2) / float(2 * sigma**2))
            for x in range(window_size)
        ]
    )
    return gauss / gauss.sum()

def create_window_3D(window_size, channel):
    _1D_window = gaussian(window_size, 1.5).unsqueeze(1)
    _2D_window = _1D_window.mm(_1D_window.t())
    _3D_window = (
        _1D_window.mm(_2D_window.reshape(1, -1))
        .reshape(window_size, window_size, window_size)
        .float()
        .unsqueeze(0)
        .unsqueeze(0)
    )
    window = Variable(
        _3D_window.expand(
            channel, 1, window_size, window_size, window_size
        ).contiguous()
    )
    return window

def ssim_map_3D(y, y_pred, window_size=11, size_average=True):
    img1 = torch.from_numpy(y)[None, :, :, :, :]  # add a dimension
    img2 = torch.from_numpy(y_pred)[None, :, :, :, :]
    (_, channel, _, _, _) = img1.shape
    window = create_window_3D(window_size, channel)

    if img1.is_cuda:
        window = window.cuda(img1.get_device())
    window = window.type_as(img1)

    mu1 = F.conv3d(img1, window, padding=window_size // 2, groups=channel)
    mu2 = F.conv3d(img2, window, padding=window_size // 2, groups=channel)

    mu1_sq = mu1.pow(2)
    mu2_sq = mu2.pow(2)

    mu1_mu2 = mu1 * mu2

    sigma1_sq = (
        F.conv3d(img1 * img1, window, padding=window_size // 2, groups=channel) - mu1_sq
    )
    sigma2_sq = (
        F.conv3d(img2 * img2, window, padding=window_size // 2, groups=channel) - mu2_sq
    )
    sigma12 = (
        F.conv3d(img1 * img2, window, padding=window_size // 2, groups=channel)
        - mu1_mu2
    )

    C1 = 0.01**2
    C2 = 0.03**2

    ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / (
        (mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2)
    )
    return ssim_map[0][0]

def mse_in_mask(X, Y, mask):
    return ((X - Y)**2).mean(where=mask.astype(bool))

def ssim_in_mask(X, Y, mask):
    ssim_map = ssim_map_3D(X, Y).numpy()
    return ssim_map.mean(where=mask.astype(bool))

def minkowski_distance(X, Y):
    return minkowski(X, Y, p=10)


# pylint: disable=locally-disabled, invalid-name
"""
Manage mongodb connection and model layer
"""

from logging import getLogger

import motor.motor_tornado

logger = getLogger()
client = None
db = None

def init(db_name):
    """ initializes mongo db connection """
    logger.debug('init model, db_name: %s', db_name)

    global client, db
    client = motor.motor_tornado.MotorClient()
    db = client[db_name]


'''
Author: Marek Cmero
Functions for perfect phylogeny and incomplete phylogeny algorithms
'''

import numpy as np
import copy
import string
import subprocess
from mgraph import MGraph

def get_duplicates(items):
    '''
    returns the indices of all duplicates in a list
    @param items a 1D list
    '''
    locs = []

    for i in range(len(items)):
        loc_tmp = [i]
        item = items[i]
        start_at = i+1
        while True:
            try:
                loc = items.index(item,start_at+1)
            except ValueError:
                break
            else:
                loc_tmp.append(loc)
                start_at = loc

        if len(loc_tmp)>1:
            locs.append(loc_tmp)

    return(locs)

def remove_duplicates(m_prime, c_prime):
    '''
    remove any duplicate columns and merge associated features
    @param m_prime M' matrix
    @param c_prime features list corresponding to columns of M'
    '''
    m_prime_tmp = map(lambda x: '.'.join(map(str,x)),np.rot90(m_prime)[::-1])
    dups = get_duplicates(m_prime_tmp)

    n_dup = len(dups)
    to_del = []
    if n_dup > 0:
        for idx,dup in enumerate(dups):
            to_del.extend(dup[1:])
            c_prime[dup[0]] = '_'.join(c_prime[dup])

    m_prime = np.delete(m_prime,to_del,axis=1)
    c_prime = np.delete(c_prime,to_del)

    return(m_prime, c_prime)

def remove_zero_entry_cols(m3,c3):
    '''
    remove any columns that have no 0 entries
    @param m3 the M matrix
    @param c3 column features corresponding to the M matrix
    '''
    mi = np.empty(0,dtype='int')
    ncol = len(m3[0])
    idxs_to_delete = [i for i in range(ncol) if not np.any(m3[:,i]==0)]
    m3 = np.delete(m3,idxs_to_delete,axis=1)
    c3 = np.delete(c3,idxs_to_delete)
    return(m3,c3)

def get_m_prime(m,c):
    '''
    Construct M' matrix
    Obtain binary encoding for each feature, sort by binary value,
    then rotate and reverse the resulting matrix
    @param nshared nxm matrix of samples (cols) and
    @param svrot is true, the matrix is of form rows = svs, cols = samples
    '''
    m_prime, c_prime = remove_zero_entry_cols(m,c)
    m_prime, c_prime = remove_duplicates(m_prime, c_prime)
    m_prime = np.rot90(m_prime)

# count binary score of columns
    binary_strings = []
    for col in m_prime:
        col = np.array([ci if ci>0 else 0  for ci in col])
        col_string = '0b'+''.join(map(str,col))
        binary_strings.append(int(col_string,2))

# sort by binary score
    order = np.argsort(binary_strings)[::-1]
    m_prime = m_prime[order]
    m_prime = np.rot90(m_prime)[::-1] #rotate again
    c_order = (len(c_prime) - 1) - order #translate order of rotated matrix to order of columns
    c_prime = c_prime[c_order]

    return(m_prime,c_prime)

def get_k1_matrix(m_prime,features):
    '''
    Generate k1 from m' matrix
    Allows for checking of perfect phylogeny
    @param mp the m prime matrix
    @param nodes corresponding to the matrix
    '''
    ncol = len(m_prime[0])
    k = np.empty( [0,ncol], dtype='|S15' )

    for m in m_prime:
        row_feats = features[m!=0] #features in the row
        mrow = np.zeros(ncol,dtype='|S15')
        mrow.fill('0')

        for idx,feature in enumerate(row_feats):
            mrow[idx] = feature

        n_feat = len(row_feats)
        if n_feat < ncol:
            mrow[n_feat]='#'

        k = np.append(k,[mrow],axis=0)

    return(k)

def perfect_phylogeny_exists(k1,features):
    '''
    Determine whether perfect phylogeny exists from a k1 matrix
    @param k1 the k1 matrix (output from get_k1_matrix)
    @param features the column features
    '''
    locations = []
    for feature in features:
        present_at = set([])
        for k_i in k1:
            [ present_at.add(loc_list) for loc_list in list(np.where(k_i==feature)[0]) ]
        locations.append(present_at)

    loc_test = np.array([len(loc_list)>1 for loc_list in locations])
    if np.any(loc_test):
        print('No phylogeny found!')
        return(False)
    else:
        print('Success! Found phylogeny!\nK1 matrix:')
        print(k1)
        return(True)

def get_k(k,q,m_graph):
    '''
    return the first K vector with E[K] >= 1
    @param k holds the connection vector, initialise with set()
    @param q chain of connected vertices
    @param m_graph graph object containing matrix connections
    '''
    if not q:
        return(k)
    else:
        q1 = q.pop()
        k.add(q1)
        pairs = m_graph.get_pairs_containing(q1)
        pairs = set([node for pair in pairs for node in pair]) #flatten set of elements
        for node in pairs:
            if node not in k:
                q.add(node)
        return(get_k(k,q,m_graph))

def solve_incomplete_phylogeny(m,s,c):
    '''
    implement Pe'er incomplete phylogeny algorithm
    @param m M matrix
    @param s samples/species (rows)
    @param c features (columns)
    '''
    m3, c3 = get_m_prime(m,c)

    m_graph = MGraph()
    m_graph.build_graph(m3,s,c3)
    m_pairs = m_graph.get_edge_pairs()

    q = set(m_pairs[0]) #pick the first elements as k
    k = get_k(set(),q,m_graph)
    t = [set(s)] #initialise a tree

    while len(m_pairs) > 1:
        while len(k) < 3: #
            for n in k:
                m_graph.delNode(n)
            m_pairs = m_graph.get_edge_pairs()
            if len(m_pairs) > 1:
                q = set(m_pairs[0])
                k = get_k(set(),q,m_graph)
            else:
                break

        s_prime   = set(s).intersection(k)
        s_indexes = np.array([np.where(s_i==s)[0][0] for s_i in s_prime])
        m_tmp     = m3.copy()[s_indexes]

        print('k:')
        print(k)
        print("S':")
        print(s_prime)

        u = []
        c_in_k  = set(c).intersection(k)
        for c_i in c_in_k:
            c_index = np.where(c_i==c3)[0]
            m_col = m_tmp[:,c_index:c_index+1]
            if np.all(m_col!=0):
                cm = np.where(c_i==c3)[0]
                u.append(c_i)

        if not u:
            break
        else:
            print('u:')
            print(u)
            t.append(s_prime)
            for n in u:
                m_graph.delNode(n)
            m_pairs = m_graph.get_edge_pairs()
            k = get_k(set(),set(m_pairs[0]),m_graph)
            print('Tree:')
            print(t)

    m_new = m3.copy()

    c_indexes = [np.where(c3==x)[0][0] for x in u]
    for s_set in t[1:]:
        s_prime = set(s).intersection(s_set)
        s_indexes = np.array([np.where(s_i==s)[0][0] for s_i in s_prime])
        m_tmp = m3.copy()[s_indexes]

        for c_i in c3:
            c_idx = np.where(c_i==c3)[0]
            m_col = m_tmp[:,c_index:c_index+1]
            if np.all(m_col!=0) and np.any(m_col==-1):
                cm = np.where(c_i==c3)[0]
                m_col[m_col==-1] = 1
                m_new[s_indexes,c_idx:c_idx+1] = m_col

    for i in range(len(c3)):
        tcol = m_new[:,i:i+1]
        tcol[tcol==-1] = 0
        m_new[:,i:i+1] = tcol

    print("New M':")
    m_new, c_prime = get_m_prime(m_new,c3)
    k1 = get_k1_matrix(m_new,c_prime)
    print(m_new)

    return(m_new, c_prime, k1, t)

def write_dot(k1,s,outname):
    '''
    takes a k1 matrix and writes a dot source file of the node
    connections and edges, then converts the dot to a postscript file.
    @param k1 matrix output from perfect phylogeny checking
    @param s samples (rows)
    '''
    dot_lines = []

    for r_idx,k_row in enumerate(k1):
        for c_idx,k_i in enumerate(k_row):
            if k_i=='#': break

            k_next = k_row[c_idx+1]
            if c_idx==0:
                dot_lines.append('\troot [label=""];\n')
                dot_lines.append('\troot -> node_%s [label="%s"];\n' % (k_i,k_i))
            if k_next=='#':
                dot_lines.append('\tnode_%s [label=""];\n' % k_i)
                dot_lines.append('\tnode_%s -> %s;\n' % (k_i,s[r_idx]))
                break
            dot_lines.append('\tnode_%s [label=""];\n\tnode_%s [label=""];\n' % (k_i,k_next))
            dot_lines.append('\tnode_%s -> node_%s [label="%s"];\n' % (k_i,k_next,k_next))

    dot_lines = np.unique(np.array(dot_lines))
    with open('%s.dot'%outname,'w') as fout:
        fout.write('digraph {\n')
        fout.write('\tgraph[size="7.75,10.25"]\n')
        for line in dot_lines:
            fout.write(line)
        fout.write('}\n')

    subprocess.call(['dot','-Tps','%s.dot'%outname],stdout=open('%s.ps'%outname,'w'))


# from dolphindb import *
import dolphindb as ddb

HOST = "localhost"
PORT = 8801
s = ddb.session(HOST, PORT,"admin","123456")
WORK_DIR = "C:/Tutorials_EN/data"

trade=s.loadText(WORK_DIR+"/example.csv")

# convert the imported DolphinDB table object into a pandas DataFrame
df = trade.toDF()

if s.existsDatabase(WORK_DIR+"/valuedb"):
    s.dropDatabase(WORK_DIR+"/valuedb")
s.database('db', partitionType=ddb.VALUE, partitions=["AMZN","NFLX", "NVDA"], dbPath=WORK_DIR+"/valuedb")
trade = s.loadTextEx("db",  tableName='trade',partitionColumns=["TICKER"], filePath=WORK_DIR + "/example.csv")
print(trade.toDF())

trade = s.loadTable(tableName="trade", dbPath=WORK_DIR + "/valuedb")

s.database('db', partitionType=ddb.VALUE, partitions=["AMZN","NFLX", "NVDA"], dbPath="")

# "dbPath='db'" means that the system uses database handle 'db' to import data into in-memory partitioned table trade
trade=s.loadTextEx(dbPath="db", partitionColumns=["TICKER"], tableName='trade', filePath=WORK_DIR + "/example.csv")
print(trade.rows)
print(trade.cols)
print(trade.schema)

trade = s.loadTable(tableName="trade",dbPath=WORK_DIR+"/valuedb", partitions=["NFLX","NVDA"], memoryMode=True)

#show how many rows in the table
print(trade.rows)

import os
if s.existsDatabase(WORK_DIR+"/valuedb"  or os.path.exists(WORK_DIR+"/valuedb")):
    s.dropDatabase(WORK_DIR+"/valuedb")
s.database(dbName='db', partitionType=ddb.VALUE, partitions=["AMZN","NFLX", "NVDA"], dbPath=WORK_DIR+"/valuedb")
t = s.loadTextEx("db",  tableName='trade',partitionColumns=["TICKER"], filePath=WORK_DIR + "/example.csv")

trade = s.loadTableBySQL(tableName="trade", dbPath=WORK_DIR+"/valuedb", sql="select * from trade where date>2010.01.01")
print(trade.rows)

trade=s.ploadText(WORK_DIR+"/example.csv")
print(trade.rows)

import pandas as pd
import numpy as np

df = pd.DataFrame({'id': np.int32([1, 2, 3, 4, 3]), 'value':  np.double([7.8, 4.6, 5.1, 9.6, 0.1]), 'x': np.int32([5, 4, 3, 2, 1])})
s.upload({'t1': df})
print(s.run("t1.value.avg()"))

trade = s.loadTable(tableName="trade",dbPath=WORK_DIR+"/valuedb", partitions="AMZN")
print(trade.rows)

dt = s.table(data={'id': [1, 2, 2, 3],
                   'ticker': ['APPL', 'AMNZ', 'AMNZ', 'A'],
                   'price': [22, 3.5, 21, 26]}).executeAs("test")

# load table "test" on DolphinDB server(just uploaded from Python)
print(s.loadTable("test").toDF())

trade = s.loadTable(tableName="trade",dbPath=WORK_DIR+"/valuedb")
top10 = trade.top(10).executeAs("top10")
trade.append(top10)
trade = s.loadTable(tableName="trade",dbPath=WORK_DIR+"/valuedb")
print(trade.rows)

trade=s.loadText(WORK_DIR+"/example.csv")
top10 = trade.top(10).executeAs("top10")
t1=trade.append(top10)

print(t1.rows)

trade = s.loadTable(tableName="trade", dbPath=WORK_DIR+"/valuedb", memoryMode=True)
trade = trade.update(["VOL"],["999999"]).where("TICKER=`AMZN").where(["date=2015.12.16"]).execute()
t1=trade.where("ticker=`AMZN").where("VOL=999999")
print(t1.toDF())

trade = s.loadTable(tableName="trade", dbPath=WORK_DIR+"/valuedb", memoryMode=True)
trade.delete().where('date<2013.01.01').execute()
print(trade.rows)

trade = s.loadTable(tableName="trade", dbPath=WORK_DIR + "/valuedb", memoryMode=True)
t1=trade.drop(['ask', 'bid'])
print(t1.top(5).toDF())

# s.dropTable(WORK_DIR + "/valuedb", "trade")
# trade = s.loadTable(tableName="trade", dbPath=WORK_DIR + "/valuedb", memoryMode=True)

trade = s.loadTable(tableName="trade",dbPath=WORK_DIR+"/valuedb", memoryMode=True)
print(trade.select(['ticker','date','bid','ask','prc','vol']).toDF())

# trade = s.loadTable(tableName="trade",dbPath=WORK_DIR+"/valuedb", memoryMode=True)
# print(trade.where('TICKER=`AMZN').where('date>2016.12.15').sort('date').exec('prc'))
# print(trade.where('TICKER=`AMZN').where('date>2016.12.15').exec('count(prc)'))

trade = s.loadTable(tableName="trade",dbPath=WORK_DIR+"/valuedb", memoryMode=True)

# save filtered result to DolphinDB server variable "t1" through function "executeAs"
t1=trade.select(['date','bid','ask','prc','vol']).where('TICKER=`AMZN').where('bid!=NULL').where('ask!=NULL').where('vol>10000000').sort('vol desc').executeAs("t1")
print(t1.toDF())
print(t1.rows)

trade = s.loadTable(tableName="trade",dbPath=WORK_DIR+"/valuedb")
print(trade.select("ticker, date, vol").where("bid!=NULL, ask!=NULL, vol>50000000").toDF())

trade = s.loadTable(tableName="trade",dbPath=WORK_DIR+"/valuedb")
print(trade.select('ticker').groupby(['ticker']).count().sort(bys=['ticker desc']).toDF())

trade = s.table(dbPath=WORK_DIR+"/valuedb", data="trade")
t1 = s.table(data={'TICKER': ['AMZN', 'AMZN', 'AMZN'], 'date': ['2015.12.31', '2015.12.30', '2015.12.29'], 'open': [695, 685, 674]})
print(trade.merge(t1,on=["TICKER","date"]).toDF())

trade = s.table(dbPath=WORK_DIR+"/valuedb", data="trade")
t1 = s.table(data={'TICKER': ['AMZN', 'AMZN', 'AMZN'], 'date': ['2015.12.31', '2015.12.30', '2015.12.29'], 'open': [695, 685, 674]})
print("left left left")
print(trade.merge(t1,how="left", left_on=["TICKER","date"]).select("*").toDF())

t2 = s.table(data={'TICKER': ['NFLX', 'NFLX', 'NFLX'], 'date': ['2015.12.31', '2015.12.30', '2015.12.29'], 'open': [892, 185, 1000]})
print("outer outer outer")
print(t1.merge(t2,how="outer", on=["TICKER","date"]).select("*").toDF())

# HOST = "localhost"
# PORT = 8801
# s = ddb.session()
# s.connect(HOST, PORT,"admin","123456")
# if s.existsDatabase("dfs://valuedb"):
#     s.dropDatabase("dfs://valuedb")
# s.database('db', partitionType=ddb.VALUE, partitions=["AMZN","NFLX", "NVDA"], dbPath="dfs://valuedb")
# trade=s.loadTextEx(dbPath="dfs://valuedb", partitionColumns=["TICKER"], tableName='trade', filePath=WORK_DIR + "/example.csv")
# print(trade.rows)
# s.dropPartition("dfs://valuedb", partitionPaths=["/AMZN", "/NFLX"])
# trade = s.loadTable(tableName="trade", dbPath="dfs://valuedb")
# print(trade.rows)
# print(trade.select("distinct TICKER").toDF())

trade = s.table(dbPath=WORK_DIR+"/valuedb", data="trade")
t1 = s.table(data={'TICKER1': ['AMZN', 'AMZN', 'AMZN'], 'date1': ['2015.12.31', '2015.12.30', '2015.12.29'], 'open': [695, 685, 674]})
print(trade.merge(t1,left_on=["TICKER","date"], right_on=["TICKER1","date1"]).select("*").toDF())


"""Common fixtures"""
# pylint: disable=unused-argument, redefined-outer-name
from types import SimpleNamespace

import pytest
import responses

from django.test.client import Client
from rest_framework.test import APIClient
from wagtail.core.models import Site

from applications.constants import AppStates, VALID_SUBMISSION_TYPE_CHOICES
from applications.factories import (
    BootcampApplicationFactory,
    ApplicationStepFactory,
    BootcampRunApplicationStepFactory,
    ApplicationStepSubmissionFactory,
)
from backends.edxorg import EdxOrgOAuth2
from klasses.factories import InstallmentFactory
from profiles.factories import UserFactory

@pytest.fixture
def social_extra_data():
    """Some fake data for populating social auth"""
    yield {"access_token": "fooooootoken", "refresh_token": "baaaarrefresh"}

@pytest.fixture
def user(db, social_extra_data):
    """Creates a user"""
    # create a user
    user = UserFactory.create()
    user.social_auth.create(provider="not_edx")
    user.social_auth.create(
        provider=EdxOrgOAuth2.name,
        uid="{}_edx".format(user.username),
        extra_data=social_extra_data,
    )
    yield user

@pytest.fixture
def staff_user(db, user):
    """Staff user fixture"""
    user.is_staff = True
    user.save()

@pytest.fixture
def user_client(user):
    """Django test client that is authenticated with the user"""
    client = Client()
    client.force_login(user)
    return client

@pytest.fixture
def staff_client(staff_user):
    """Django test client that is authenticated with the staff user"""
    client = Client()
    client.force_login(staff_user)
    return client

@pytest.fixture
def user_drf_client(user):
    """DRF API test client that is authenticated with the user"""
    client = APIClient()
    client.force_authenticate(user=user)
    return client

@pytest.fixture
def admin_drf_client(admin_user):
    """DRF API test client with admin user"""
    client = APIClient()
    client.force_authenticate(user=admin_user)
    return client

@pytest.fixture
def mocked_responses():
    """Mocked responses for requests library"""
    with responses.RequestsMock() as rsps:
        yield rsps

@pytest.fixture
def mock_context(mocker, user):
    """Mocked context for serializers"""
    return {"request": mocker.Mock(user=user)}

@pytest.fixture()
def wagtail_site():
    """Fixture for Wagtail default site"""
    return Site.objects.get(is_default_site=True)

@pytest.fixture()
def home_page(wagtail_site):
    """Fixture for the home page"""
    return wagtail_site.root_page

@pytest.fixture()
def awaiting_submission_app():
    """Fixture for testing application submission types"""
    application = BootcampApplicationFactory.create(
        state=AppStates.AWAITING_USER_SUBMISSIONS.value
    )
    installment = InstallmentFactory.create(bootcamp_run=application.bootcamp_run)
    app_steps = [
        ApplicationStepFactory.create(
            submission_type=VALID_SUBMISSION_TYPE_CHOICES[0][0]
        ),
        ApplicationStepFactory.create(
            submission_type=VALID_SUBMISSION_TYPE_CHOICES[1][0]
        ),
    ]
    run_steps = [
        BootcampRunApplicationStepFactory.create(
            bootcamp_run=application.bootcamp_run, application_step=app_steps[0]
        ),
        BootcampRunApplicationStepFactory.create(
            bootcamp_run=application.bootcamp_run, application_step=app_steps[1]
        ),
    ]
    ApplicationStepSubmissionFactory.create(
        bootcamp_application=application, run_application_step=run_steps[0]
    )

    return SimpleNamespace(
        application=application, run_steps=run_steps, installment=installment
    )


#This is the handler for rolling dice
from random import randrange

#Dice Rolling Function, rolls XdY and returns the username and results from each roll
#Input example /roll 2d20
async def rolldice(message):
        #If the user has a nick set for a server use that for their name, otherwise use Dicord User ID
        if message.author.nick != None:
            roller = message.author.nick
        else:
            roller = message.author
        try:
            dice = message.content.split('d')[0]
            dice = dice.split(' ')[1]
            sides = message.content.split('d')[1]
        except Exception as e:
            print(e)
            await message.channel.send(f"{roller},you provided an invalid roll format. Example: 1d20")
            return
        #We need to limit the number of dice and sides to prevent message limits/abuse
        if int(dice) > 25:
            await message.channel.send(f"{roller}, I don't have {dice} dice. I'm not some sort of mechanical dice camel")
            return
        if int(sides) > 100:
            await message.channel.send(f"{roller}, NO, that is far too many sides. Please do try to be reasonable")
            return
        if int(sides) < 1:
            await message.channel.send(f"{roller}, you must specify at least one side")
            return
        if int (dice) < 1:
            await message.channel.send(f"{roller}, you must roll at least 1 die")
            return

        #We need to add 1 to the number of sides because randrange is indexed strangely
        sides_indexed = int(sides)
        sides_indexed += 1
        #Now we need to roll the number of dice and store the results in an array
        rollcount = 0
        rollout = []
        while rollcount < int(dice):
            roll = randrange(1, sides_indexed)
            rollout.append(roll)
            rollcount += 1

        await message.channel.send(f"{roller} rolled {dice}d{sides}: {rollout}")

if __name__== "__main__":
        main()


import sys
input = sys.stdin.readline

N = int(input())
stores = list(map(int, input().split()))
cnt = 0

for i in range(N):
    if stores[i] == cnt % 3:
        cnt += 1
print(cnt)


# -*- coding: utf-8 -*-
import base64
import io
import json
import os
from datetime import datetime
from os.path import join, basename
from tempfile import TemporaryDirectory
from typing import Optional
from asgiref.sync import sync_to_async
from data_url import DataURL
from django.shortcuts import redirect
from django_q.tasks import async_task, result
from django_q.models import Task, OrmQ
from iscc_generator.schema import NftPackage, QueuedTasks
from ninja import Router, File, Form, Schema, UploadedFile
from iscc_generator.base import get_or_404
from iscc_generator.codegen.spec import IsccCodePostRequest
from iscc_generator.models import IsccCode, Media, Nft
from iscc_generator.schema import AnyObject
from iscc_generator.storage import media_obj_from_path
from iscc_generator.tasks import iscc_generator_task, nft_generator_task
from iscc_generator.utils import normalize_web3_address
from iscc_generator.codegen.spec import (
    NftPostRequest,
    MediaEmbeddedMetadata,
    NftFrozen,
    MediaID,
)
from constance import config
import iscc_sdk as idk
import iscc_core as ic
import iscc_schema as iss

router = Router()

class Message(Schema):
    detail: str

class TaskResponse(Schema):
    id: str
    name: str
    started: datetime
    stopped: Optional[datetime]
    success: Optional[bool]
    attempt_count: Optional[int]
    result: Optional[dict]

@router.post(
    "/iscc_code",
    response={201: iss.IsccMeta, 202: TaskResponse, 400: Message, 500: Message},
    summary="create iscc",
    tags=["iscc"],
    operation_id="iscc-code-create",
    exclude_none=True,
    by_alias=True,
)
async def iscc_code_create(
    request,
    source_file: Optional[UploadedFile] = File(
        None, description="The file used for generating the ISCC"
    ),
    meta: IsccCodePostRequest = Form(...),
):
    """
    ## Generate an ISCC for a media asset.

    Provide at least a `source_file` or optionally a `source_url`. If processing finishes fast
    you will receive an `IsccResponse`. If processing exeeds the configured time limit you will
    receive a `TaskResponse` to poll for the result at /task/{task_id}.

    Metadata supplied with the file upload will be embedded into the media asset (if possible).
    You can download the modified media asset based on the `media_id` field of the returned
    ISCC Metadata with a GET request to /media/<media_id>.
    """
    # validate the request
    if not source_file and not meta.source_url:
        return 400, Message(detail="Either source_file or source_url is required")
    if meta.meta:
        if meta.meta.startswith("data:"):
            try:
                DataURL.from_url(meta.meta)
            except Exception:
                return 500, Message(detail="Invalid Data-URL in field meta")
        else:
            try:
                json.loads(meta.meta)
            except Exception:
                return 500, Message(detail="Invalid JSON-string in field meta")

    media_obj = None
    # Create Media object if source_file provided (source_url will be handled by worker task).
    if source_file:
        try:
            media_obj = await sync_to_async(Media.objects.create)(source_file=source_file)
        except Exception as e:
            return 500, Message(detail=str(e))

    # create IsccCode object
    iscc_obj = await sync_to_async(IsccCode.objects.create)(source_file=media_obj, **meta.dict())

    # start processing
    task_id = await sync_to_async(async_task)(iscc_generator_task, iscc_obj.pk)

    # wait for result with timeout
    task_result = await async_wait_for_task(task_id)
    if task_result:
        iscc_obj = await sync_to_async(IsccCode.objects.get)(pk=iscc_obj.pk)
        return 201, iscc_obj.result
    # return task-id instead
    else:
        task = await async_find_task(task_id)
        if not task:
            return 503, Message(detail="Task not found")
        return 202, task

@router.get(
    "/iscc_code/{iscc}",
    response={200: iss.IsccMeta, 404: Message},
    exclude_defaults=False,
    exclude_unset=True,
    summary="get iscc",
    tags=["iscc"],
    operation_id="iscc-code-get",
)
async def iscc_code_get(request, iscc: str):
    """
    Get metadata for previously generated ISCC-CODE
    """
    try:
        iscc = ic.iscc_normalize(iscc)
        objs = await sync_to_async(list)(IsccCode.objects.filter(iscc=iscc))
        return objs[0].result
    except Exception:
        return 404, Message(detail="ISCC-CODE not found")

@router.delete(
    "/iscc_code/{iscc}",
    response={200: Message, 404: Message},
    summary="delete iscc",
    tags=["iscc"],
    operation_id="iscc-code-delete",
)
async def iscc_code_delete(request, iscc: str):
    """
    Delete ISCC-CODE entry from database.
    """
    try:
        iscc = ic.iscc_normalize(iscc)
        iscc_code_obj: IsccCode = await sync_to_async(IsccCode.objects.get)(iscc=iscc)
        await sync_to_async(iscc_code_obj.delete)()
        return 200, Message(detail="ISCC-CODE deleted")
    except Exception:
        return 404, Message(detail="ISCC-CODE not found")

####################################################################################################
# /media endpoints                                                                                #
####################################################################################################

@router.post(
    "/media",
    tags=["media"],
    operation_id="post-media",
    response={201: MediaID, 400: Message, 500: Message},
    summary="upload media",
)
async def media_post(request, source_file: UploadedFile = File(None)):
    """Upload Media."""
    if not source_file:
        return 400, Message(detail="No file sent")

    try:
        media_obj = await sync_to_async(Media.objects.create)(source_file=source_file)
    except Exception as e:
        return 500, Message(detail=e.__class__.__name__)
    return 201, MediaID(media_id=media_obj.flake)

@router.get(
    "/media/{media_id}",
    tags=["media"],
    operation_id="get-media",
    summary="download media",
)
async def media_get(request, media_id):
    media_obj: Media = await get_or_404(Media, media_id)
    return redirect(media_obj.source_file.url)

@router.delete(
    "/media/{media_id}",
    tags=["media"],
    operation_id="delete-media",
    response={200: Message, 404: None},
    summary="delete media",
)
async def media_delete(request, media_id: str):
    """Delete media asset"""
    media_obj: Media = await get_or_404(Media, media_id)
    await sync_to_async(media_obj.delete)()

    return 200, Message(detail="Media asset deleted")

@router.post(
    "media/metadata/{media_id}",
    tags=["media"],
    operation_id="embed-metadata",
    response={201: MediaID, 400: Message},
    summary="embed metadata",
)
async def media_metadata_embed(request, media_id: str, meta: MediaEmbeddedMetadata):
    """Embeds metadata into media asset and returns `media_id` of new asset."""
    if not meta.dict(exclude_none=True):
        return 400, Message(detail="No embeddable metadata provided")

    media_obj: Media = await get_or_404(Media, media_id)
    new_media_obj = await async_media_metadata_embed(media_obj, meta)
    return 201, MediaID(media_id=new_media_obj.flake)

@sync_to_async
def async_media_metadata_embed(media_obj: Media, meta: MediaEmbeddedMetadata):
    """
    Embed metadata into a media file.

    # TODO - we need to download/reupload the file - move this to a worker task
    """
    meta = iss.IsccMeta.parse_obj(meta.dict())
    # Copy file to local storage
    filename = basename(media_obj.source_file.name)
    with TemporaryDirectory() as tempdir:
        tmpfile_path = join(tempdir, filename)
        with open(tmpfile_path, "wb") as tmpfile:
            with media_obj.source_file.open("rb") as infile:
                data = infile.read(1024 * 1024)
                while data:
                    tmpfile.write(data)
                    data = infile.read(1024 * 1024)
        # Embed metadata
        new_file = idk.embed_metadata(tmpfile_path, meta=meta)

    # Create new media object
    new_media_object = media_obj_from_path(new_file, original=media_obj)
    os.remove(new_file)

    return new_media_object

@router.get(
    "media/metadata/{media_id}",
    tags=["media"],
    operation_id="get-media-metadata",
    response={200: MediaEmbeddedMetadata, 400: None, 404: None},
    exclude_none=True,
    summary="extract metadata",
)
async def media_metadata_get(request, media_id: str):
    """Reads and returns embedded metadata from the media asset."""
    media_obj: Media = await get_or_404(Media, media_id)
    return media_obj.metadata

####################################################################################################
# /nft endpoints                                                                                #
####################################################################################################

@router.post(
    "/nft",
    tags=["nft"],
    operation_id="post-nft",
    summary="create nft",
    response={
        201: NftPackage,
        202: TaskResponse,
        400: Message,
        404: Message,
        422: None,
        500: Message,
    },
    exclude_unset=False,
    exclude_defaults=False,
    exclude_none=True,
    by_alias=True,
)
async def nft_post(request, item: NftPostRequest):
    """Creates an NFT package"""
    item = normalize_web3_address(item)
    media_obj_image = await get_or_404(Media, item.media_id_image)
    if item.media_id_animation:
        media_obj_animation = await get_or_404(Media, item.media_id_animation)
    else:
        media_obj_animation = None
    data = item.dict(exclude={"media_id_image", "media_id_animation"})
    nft_obj = await sync_to_async(Nft.objects.create)(
        media_id_image=media_obj_image, media_id_animation=media_obj_animation, **data
    )

    # start processing
    task_id = await sync_to_async(async_task)(nft_generator_task, nft_obj.pk)

    # wait for result with timeout
    task_result = await async_wait_for_task(task_id)
    if task_result:
        await sync_to_async(nft_obj.refresh_from_db)()
        result = await filter_nft_metadata(nft_obj.result)
        return 201, result
    # return task-id instead
    else:
        task = await async_find_task(task_id)
        if not task:
            return 503, Message(detail="Task not found")
        return 202, task

@sync_to_async
def filter_nft_metadata(meta: dict) -> dict:
    """Filter `nft_metadata` fields according to configuration"""
    exclude = config.NFT_EXCLUDE_FIELDS
    if not exclude:
        return meta
    exclude = set(field.strip() for field in exclude.split(","))
    obj = iss.IsccMeta.parse_obj(meta["nft_metadata"])
    meta["nft_metadata"] = obj.dict(exclude=exclude)
    return meta

@router.get(
    "/nft/{nft_id}",
    tags=["nft"],
    operation_id="get-nft",
    summary="get nft",
    response={200: NftPackage, 404: Message},
    exclude_none=True,
)
async def nft_get(request, nft_id: str):
    nft_obj: Nft = await get_or_404(Nft, nft_id)
    return nft_obj.result

@router.delete(
    "/nft/{nft_id}",
    tags=["nft"],
    operation_id="delete-nft",
    response={200: Message, 404: None},
    summary="delete nft",
)
async def nft_delete(request, nft_id: str):
    nft_obj: Nft = await get_or_404(Nft, nft_id)
    await sync_to_async(nft_obj.delete)()

    return 200, Message(detail="Media asset deleted")

@router.post(
    "/freeze",
    tags=["nft"],
    operation_id="freeze-nft",
    summary="freeze nft",
    response={200: NftFrozen, 400: Message},
)
async def nft_freeze(request, anyobject: AnyObject):
    """
    Creates a Token-ID and IPFS CIDv1 for NFT metadata.

    Post the contents of the `nft_metadata` field from the `/nft` entpoint to this endpoint if you
    want to mint your NFT with a Token-ID that is derived from the IPFS CIDv1 of the nft metadata.

    NFT-Freezing will use deterministic JCS serialization (RFC8785) to create the IPFS payload.
    The payload is than used to create the Token-ID and IPFS CIDv1 URI (base16). The deterministic
    payload is also returned (base64 encoded) for publishing to IPFS after the NFT has been minted.
    """
    data = anyobject.dict().get("__root__")
    if not data:
        return 400, Message(detail="No JSON data provided")

    canonical = ic.json_canonical(data)
    cid = ic.cidv1_hex(io.BytesIO(canonical))
    token_id_num = ic.cidv1_to_token_id(cid)
    return NftFrozen(
        token_id_hex=cid.lstrip("f01551220"),
        token_id_num=str(token_id_num),
        metadata_ipfs_uri=f"ipfs://{cid}",
        metadata_ipfs_payload=base64.b64encode(canonical).decode("ascii"),
    )

####################################################################################################
# /task endpoints                                                                                #
####################################################################################################

@router.get(
    "/task/{task_id}",
    response={200: TaskResponse, 404: Message},
    exclude_none=True,
    tags=["task"],
    operation_id="get-task",
    summary="get task",
)
async def get_task(request, task_id: str):
    task = await async_find_task(task_id)
    if not task:
        return 404, Message(detail="Task not found")
    return 200, task

@router.get(
    "/queued_tasks",
    tags=["task"],
    response=QueuedTasks,
    summary="number of queued tasks",
)
async def get_health(request):
    """Returns number of queued tasks."""
    queued_tasks = await sync_to_async(OrmQ.objects.count)()
    return QueuedTasks(queued_tasks=queued_tasks)

####################################################################################################
# Sync to Async functions                                                                          #
####################################################################################################

@sync_to_async
def async_create_generator_task(iscc_pk) -> str:
    return async_task(iscc_generator_task, iscc_pk)

@sync_to_async
def async_wait_for_task(task_id):
    return result(task_id, config.PROCESSING_TIMEOUT * 1000)

@sync_to_async
def async_find_task(task_id):
    task = None
    try:
        task = Task.objects.get(id=task_id)
    except Task.DoesNotExist:
        for queued in OrmQ.objects.all():
            if queued.task_id() == task_id:
                task = queued.task()
                break
    return task

@sync_to_async
def async_get_iscc_code(pk: Optional[int] = None, iscc: Optional[str] = None):
    try:
        if pk:
            return IsccCode.objects.get(pk=pk)
        elif iscc:
            return IsccCode.objects.get(iscc=iscc)
    except IsccCode.DoesNotExist:
        return None