import requests
from bs4 import BeautifulSoup
import tweepy # Library Used For twitter

auth = tweepy.OAuthHandler('NZIjCl060lClQIcGIvQZY7Er5','FFafAZfS4PuCIBajvzmTQLU2jW140DeQsDBxhnWneEUk82TvOQ') # this is one of the tokens and secret keys to initialize the API

auth.set_access_token('905298941830418432-kqrO4GcUWlSUt3M50mnVYadYrPuemAD','rztBW3cTEcudo3wK5lGDxzjjTarHRwJa5YCrcAMvA8nxn') # this is one of the tokens and secret keys to initialize the API

api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)

URL = 'https://www.google.com/search?q=weather+in+mumbai&rlz=1C1CHBF_enIN868IN868&oq=weather+&aqs=chrome.0.69i59j69i57j69i60l3.1072j0j7&sourceid=chrome&ie=UTF-8'
headers = {"User_Agent" : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36'}

page = requests.get(URL, headers=headers)

soup = BeautifulSoup(page.content, 'html.parser')

temp = soup.find(id= "wob_tm").get_text() # this accesses the website where the data is, and finds the correct ID for the current temperature

login= api.me()
api.update_status(temp) # this tweets out the temperature in Mumbai at any given time


# importing the libraries
from datetime import timedelta
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from airflow.utils.dates import days_ago

# DAG argument Block (Task 1.1)
default_args = {
    'owner': 'Smit Shah',
    'start_date': days_ago(0),
    'email': ['smitshah@gmail.com'],
    'email_on_failure': True,
    'email_on_retry': True,
    'retries': 1,
    'retry_delay': timedelta(minutes=5),
}

# DAG Definition (Task 1.2)
dag = DAG(
    dag_id='etl_toll_data',
    schedule_interval=timedelta(days=1),
    default_args=default_args,
    description='Apache Airflow Final Assignment',
)

# Create a task to unzip_data (Task 1.3)
unzip_data = BashOperator(
    task_id='unzip_data',
    dag=dag,
    bash_command='tar -xzf /home/project/airflow/dags/finalassignment/staging/tolldata.tgz',
)

# Create a task to extract data from csv file (Task 1.4)
extract_data_from_csv = BashOperator(
    task_id='extract_data_from_csv',
    dag=dag,
    bash_command='cut -d"," -f1-4 /home/project/airflow/dags/finalassignments/staging/vehicle-data.csv > /home/project/airflow/dags/finalassignments/staging/csv_data.csv',
)

# Create a task to extract data from tsv file (Task 1.5)
extract_data_from_tsv = BashOperator(
    task_id='extract_data_from_tsv',
    dag=dag,
    bash_command='cut -f5-7 /home/project/airflow/dags/finalassignments/staging/tollplaza-data.tsv > /home/project/airflow/dags/finalassignments/staging/tsv_data.csv',
)

# Create a task to extract data from fixed width file (Task 1.6)
extract_data_from_fixed_width = BashOperator(
    task_id='extract_data_from_fixed_width',
    dag=dag,
    bash_command="cat payment-data.txt | tr -s '[:space:]' | cut -d'  ' -f11,12 > /home/project/airflow/dags/finalassignments/staging/fixed_width_data.csv",
)

# Create a task to consolidate data extracted from previous tasks (Task 1.7)
consolidate_data = BashOperator(
    task_id='consolidate_data',
    dag=dag,
    bash_command='paste /home/project/airflow/dags/finalassignments/staging/csv_data.csv /home/project/airflow/dags/finalassignments/staging/tsv_data.csv /home/project/airflow/dags/finalassignments/staging/fixed_width_data.csv > /home/project/airflow/dags/finalassignments/staging/extracted_data.csv',
)

# Transform and load the data (Task1.8)
transform_data = BashOperator(
    task_id='transform_data',
    dag=dag,
    bash_command='tr "[a-z]" "[A-Z]" < /home/project/airflow/dags/finalassignments/staging/extracted_data.csv > /home/project/airflow/dags/finalassignments/staging/transformed_data.csv',
)

# Define the task pipeline (Task 1.9)
unzip_data >> extract_data_from_csv >> extract_data_from_tsv >> extract_data_from_fixed_width >> consolidate_data >> transform_data

Course-8-ETL and Data Pipelines with Shell, Airflow and Kafka/C8-Week5/Task 1 Airflow DAG/etl_toll_data.py


# 백준_1107 - [브루트 포스]리모컨 (08/12)
import sys
input = sys.stdin.readline

goal_ch = int(input())
broke_num = int(input())
if broke_num != 0:
    broke_btu = set(map(int, input().split()))
else:
    broke_btu = set()

ans = abs(100 - goal_ch)

for i in range(1000001):
    n = set(list(map(int, str(i))))

    if not n & broke_btu:
        ans = min(ans, len(str(i)) + abs(goal_ch - i))

print(ans)


from flask import Flask, render_template, request, redirect, session, flash

app = Flask(__name__)
app.secret_key = 'maicondouglasfilipiaki'

class Jogo:
    def __init__(self, nome, categoria, console):
        self.nome = nome
        self.categoria = categoria
        self.console = console

jogo1 = Jogo('Super Mario', 'Acao', 'SNES')
jogo2 = Jogo('Pokemon', 'RPG', 'GAMEBOY')
jogo3 = Jogo('Mortal Kombat', 'Luta', 'SNES')
lista = [jogo1, jogo2, jogo3]

@app.route('/')
def index():
    return render_template('lista.html', titulo='Jogos', jogos=lista)

@app.route('/novo')
def novo():
    if 'usuario_logado' not in session or session['usuario_logado'] == None:
        flash('Faca o login para entrar')
        return redirect('/login?proxima=novo')
    return render_template('novo.html', titulo='Novo Jogo')

@app.route('/criar', methods=['POST',])
def criar():
    nome = request.form['nome']
    categoria = request.form['categoria']
    console = request.form['console']
    jogo = Jogo(nome, categoria, console)
    lista.append(jogo)
    return redirect('/')

@app.route('/login', methods=['GET',])
def login():
    proxima = request.args.get('proxima')
    return render_template('login.html', titulo="Login", proxima=proxima)

@app.route('/autenticar', methods=['POST',])
def autenticar():
    if 'mestrar' == request.form['senha']:
        session['usuario_logado'] = request.form['usuario']
        flash(request.form['usuario'] + ' logou com sucesso!')
        return redirect('/{}'.format(request.form['proxima']))
    else:
        flash('Tente novamente')
        return redirect('/login')

@app.route('/logout')
def logout():
    session['usuario_logado'] = None
    flash('Nenhum usuario logado')
    return redirect('/')

app.run(debug=True)


import os
import sys
import asyncio
import signal
from binance import AsyncClient, BinanceSocketManager
from dotenv import load_dotenv
load_dotenv()

symbol = sys.argv[1]
entry_price = float(sys.argv[2])
entry_order_id = sys.argv[3]
stop_order_id = sys.argv[4]
qty = float(sys.argv[5])

stop_loss_margin = 0.995 # 0.5% stop loss margin
take_profit_margin = 1.0085 # 0.85% take profit margin

take_profit_bid = entry_price * take_profit_margin
stop_loss_bid = entry_price * stop_loss_margin

async def main():
    client = await AsyncClient.create(api_key=os.getenv("API_KEY_PROD"), api_secret=os.getenv("SECRET_KEY_PROD"))
    bm = BinanceSocketManager(client)
    # start any sockets here, i.e a trade socket
    ts = bm.trade_socket(symbol=symbol)
    # then start receiving messages
    async with ts as tscm:
        while True:
            res = await tscm.recv()
            print(res)
            if float(res["p"]) >= take_profit_bid:
                print("Take profit")
                await client.cancel_order(symbol=symbol, orderId=stop_order_id)
                await client.create_order(symbol=symbol, side="SELL", type="MARKET", quantity=qty)
                break
            if float(res["p"]) <= stop_loss_bid:
                print("Stop loss")
                break

    await client.close_connection()
    os.kill(os.getppid(), signal.SIGTERM)

if __name__ == "__main__":
    loop = asyncio.get_event_loop()
    loop.run_until_complete(main())


class Car:

    def __init__(self, manufacturer, model, make, transmission, color):
        self.manufacturer = manufacturer
        self.model = model
        self.make = make
        self.transmission = transmission
        self.color = color

    def accelerate(self):
        print(self.manufacturer + " " + self.model + " is moving")

    def stop(self):
        print(self.manufacturer + " " + self.model + " has stopped")

car1 = Car("Hyundai", "Creta", "2023", "Manual", "White")
car2 = Car("Maruti", "GrandVitara", "2023", "Manual", "Red")
car3 = Car("Skoda", "Kushaq", "2023", "Automatic", "Black")

car1.accelerate()
car1.stop()
car2.accelerate()
car2.stop()
car3.accelerate()
car3.stop()


# Problem link: https://leetcode.com/problems/remove-nth-node-from-end-of-list/
# Tags: Facebook, Amazon, Microsoft

"""
Given the head of a linked list, remove the nth node from the end of the list
and return its head.

Example 1:
Input: head = [1,2,3,4,5], n = 2
Output: [1,2,3,5]

Example 2:
Input: head = [1], n = 1
Output: []

Example 3:
Input: head = [1,2], n = 1
Output: [1]

Constraints:
The number of nodes in the list is sz.
1 <= sz <= 30
0 <= Node.val <= 100
1 <= n <= sz
"""

# Definition for singly-linked list.
class ListNode:
    def __init__(self, val=0, next=None):
        self.val  = val
        self.next = next

class Solution:
    def removeNthFromEnd(self, head: Optional[ListNode], n: int) -> Optional[ListNode]:
        dummy       = ListNode(0)
        dummy.next  = head
        curr        = dummy
        target_prev = dummy

        for idx in range(n+1):
            curr = curr.next

        while curr != None:
            curr        = curr.next
            target_prev = target_prev.next

        target_prev.next = target_prev.next.next
        return dummy.next


# -*- coding: utf-8 -*-

# -- Sheet --

from random import*

alphabet=['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']

def shuffle_alphabet(): # Return static rotor
    rotor1=alphabet.copy()
    shuffle(rotor1)
    return rotor1
"""
First step : Search the letter in alphabet. Save his index
Second step : Search in the first rotor the associate letter
Third step : Search this last letter in the alphabet and save his index
Fourth step : Search in the second rotor the index of third step
Fifth step : Return the letter
"""
def code(a,b,m): #Input : 2 rotors and a message
    code_mess=[]

    for i in range(len(m)):

        if m[i]==' ':
            code_mess.append(' ')
            continue

        for j in range(len(alphabet)):
            if alphabet[j]==m[i]:
                tmp=j #Index - First step done.

        indRotor1=a[tmp] #Second step - Done

        for k in range(len(alphabet)):
            if alphabet[k]==indRotor1:
                tmp1=k # Third step - Done

        indRotor2=b[tmp1]

        code_mess.append(indRotor2)

    for l in range(len(code_mess)):
        print(code_mess[l],end='')
    print("")

    #Return

"""
First step : Search the letter in the second rotor. Save his index
Second step : Search in the alphabet at the index
Third step : Search the letter in the first rotor and save his index
Fourth step : Search in the alphabet at this last index
"""
def decode(a,b,m): # Input : 2 rotors (same as in code algorithme) and the crypted message
    decode_mess=[]
    for i in range(len(m)):

        if m[i]==' ':
            decode_mess.append(' ')
            continue

        for j in range(len(b)):
            if b[j]==m[i]:
                tmp=j #First step - Done

        alphaPosi=alphabet[tmp] #Second step - Done

        for k in range(len(a)):
            if a[k]==alphaPosi:
                tmp1=k #Third step - Done

        finalLetter=alphabet[tmp1] #Fourth step - Done

        decode_mess.append(finalLetter)

    for l in range(len(decode_mess)):
        print(decode_mess[l],end='')

    print("")

def rotation(a):
    b=a.copy()
    for m in range(len(a)):
        if m==len(a)-1:
            b[0]=a[m]
        else:
            b[m+1]=a[m]
    return b

def rotation_finale(a,b): #Temporaire
    compt_rota=0
    rota_b=0
    if compt_rota==(len(a)-1):
        b=rotation(b)
        a=rotation(a)
        print(a)
        print(b)
        compt_rota+=1
        rota_b+=1
        print("compteur : ",compt_rota)
        print("Rotation de b : ",rota_b)
        print("")
        compt_rota=0
    else:
        a=rotation(a)
        print(a)
        print(b)
        compt_rota+=1
        print("compteur : ", compt_rota)
        print("Rotation de b : ",rota_b)
        print("")

def code_enigma(a,b,m):
    compt_rota=0
    rota_b=0
    code_mess=[]

    for i in range(len(m)):

        if m[i]==' ':
            code_mess.append(' ')
            continue

        for j in range(len(alphabet)):
            if alphabet[j]==m[i]:
                tmp=j #Index - First step done.

        indRotor1=a[tmp] #Second step - Done

        for k in range(len(alphabet)):
            if alphabet[k]==indRotor1:
                tmp1=k # Third step - Done

        indRotor2=b[tmp1]
        code_mess.append(indRotor2)

        if compt_rota==(len(a)-1):
            b=rotation(b)
            a=rotation(a)
            rota_b+=1
            compt_rota=0
        else:
            a=rotation(a)
            compt_rota+=1

    for l in range(len(code_mess)):
        print(code_mess[l],end='')
    print("")

    return code_mess

saveRotor1=['f', 'g', 'r', 'o', 'w', 'd', 'c', 'i', 'y', 't', 'p', 'z', 'u', 'a', 'h', 'm', 'l', 'q', 's', 'j', 'b', 'x', 'k', 'n', 'v', 'e']
saveRotor2=['u', 'i', 'f', 'e', 'w', 'x', 'z', 't', 'n', 'j', 'q', 'a', 'm', 's', 'h', 'c', 'l', 'o', 'v', 'd', 'b', 'y', 'g', 'k', 'p', 'r']
print(alphabet)
print("")
print(saveRotor1)
print(saveRotor2)
print("")

code(saveRotor1,saveRotor2,"je suis en classe de nsi")
decode(saveRotor1,saveRotor2,"dg vipv gu orxvvg hg uvp")


# Sofia Sackett
# XOR Cipher

def main():
    question = print("Enter 'e' to encipher or 'd' to decipher.")
    mode = input("Choose mode: ")

    if mode == 'e' or mode == 'E':
        text = input("Enter the text that you would like enciphered: ").lower()
        key = input("Enter the keyword: ").lower()
        print(encipher(text, key))

    elif mode == 'd' or mode == 'D':
        text = input("Enter the text that you would like deciphered: ")
        key = input("Enter the keyword: ").lower()
        print(decipher(text, key))

    else:
        print("An error occured. Please enter 'e' to encipher or 'd' to decipher.")

def encipher(text, key):
    ciphertext = ""
    count = 0
    for i in range(len(text)):
        container = ord(text[i]) ^ ord(key[count])
        ciphertext += hex(container)[2:].zfill(2)
        count += 1
        if count >= len(key):
            count = 0
    return ciphertext

def decipher(text, key):
    toUnicode = ""
    for i in range(0, len(text), 2):
        toUnicode += bytes.fromhex(text[i:i+2]).decode('utf-8')
    plaintext = ""
    count = 0
    for i in range(len(toUnicode)):
        container = ord(toUnicode[i]) ^ ord(key[count])
        plaintext += chr(container)
        count += 1
        if count >= len(key):
            count = 0
    return plaintext

main()


# ID успешной посылки: 86821452
class Stack:
    def __init__(self):
        self.items = []

    def push(self, item):
        self.items.append(item)

    def pop(self):
        return self.items.pop()

def main():
    arr = input().split()
    x = Stack()
    symbol_operation = {
        '+': lambda v1, v2: v2 + v1,
        '-': lambda v1, v2: v2 - v1,
        '/': lambda v1, v2: v2 // v1,
        '*': lambda v1, v2: v2 * v1
    }
    for elem in arr:
        if elem in symbol_operation:
            el1 = x.pop()
            el2 = x.pop()
            res = symbol_operation[elem](el1, el2)
            x.push(res)
        else:
            elem = int(elem)
            x.push(elem)
    print(x.pop())

if __name__ == '__main__':
    main()


# std
import logging
from typing import Optional
from datetime import datetime

# project
from . import FinishedSignageConditionChecker
from src.notifier import Event, EventService, EventType, EventPriority
from src.chia_log.parsers.finished_signage_point_parser import FinishedSignagePointMessage
from src.chia_log.handlers.util.calculate_skipped_signage_points import calculate_skipped_signage_points

class NonSkippedSignagePoints(FinishedSignageConditionChecker):
    """Check that the full node did not skip any signage points.
    If there are signage points missing, this could indicate connection
    issues which prevent the farmer from participating in all challenges.
    """

    def __init__(self):
        logging.debug("Enabled check for finished signage points.")
        self._last_signage_point_timestamp: datetime = datetime.fromtimestamp(0)
        self._last_signage_point: int = 0

    def check(self, obj: FinishedSignagePointMessage) -> Optional[Event]:
        if self._last_signage_point == 0:
            self._last_signage_point_timestamp = obj.timestamp
            self._last_signage_point = obj.signage_point
            return None

        event = None
        valid, skipped = calculate_skipped_signage_points(
            self._last_signage_point_timestamp, self._last_signage_point, obj.timestamp, obj.signage_point
        )

        if not valid:
            return None

        # To reduce notification spam, only send notifications for skips larger than 1
        if skipped == 1:
            logging.info(
                f"Detected {skipped} skipped signage point."
                "This is expected to happen occasionally and not a reason for concern."
            )
        elif skipped >= 2:
            message = f"Experiencing networking issues? Skipped {skipped} signage points!"
            logging.warning(message)
            event = Event(
                type=EventType.USER,
                priority=EventPriority.NORMAL,
                service=EventService.FULL_NODE,
                message=message,
            )

        self._last_signage_point_timestamp = obj.timestamp
        self._last_signage_point = obj.signage_point
        return event


#220721 / dp / 편집거리

a = input().rstrip()
b = input().rstrip()

la = len(a)
lb = len(b)
diff = abs(la-lb)

dp = [[0] * (lb+1) for _ in range(la+1)]
for i in range(1,la+1):
    for j in range(1,lb+1):
        dp[i][j] = max(dp[i-1][j],dp[i][j-1])
        if a[i-1] == b[j-1]:
            dp[i][j] += 1

print(diff + la - dp[-1][-1])


from .composer import Composer
from .mangalib_parser import MangaLibBook, load_chapter
from .status import StatusMessage

from uuid import uuid4 as random_token
import asyncio
from io import BytesIO

from dtelbot.inline_keyboard import markup, button
from telethon import TelegramClient
from telethon.sessions.string import StringSession

class MangaParser:
    def __init__(self, bot, telethon_params):
        self.composer = Composer()
        self.bot = bot
        self.telethon_session = telethon_params[0]
        self.telethon_api_id = telethon_params[1]
        self.telethon_api_hash = telethon_params[2]
        self.bot_name = None
        self.callback_timesleep = 2
        self.per_page = 10
        self.files = {}
        self.nav_codes = lambda c: 10**(len(c)-1) if c[0] == '>' else -10**(len(c)-1) if c[0] == '<' else  0

    async def set_bot_name(self):
        result = await (self.bot.getme().send())
        self.bot_name = '@' + result['result']['username']

    def navbar(self, count, nav_text, buttons=[]):
        if count > 10:
            buttons = [button('<', callback_data='{} <'.format(nav_text))] + buttons + [button('>', callback_data='{} >'.format(nav_text))]
        if count > 100:
            buttons = [button('<<', callback_data='{} <<'.format(nav_text))] + buttons + [button('>>', callback_data='{} >>'.format(nav_text))]
        if count > 1000:
            buttons = [button('<<<', callback_data='{} <<<'.format(nav_text))] + buttons + [button('>>>', callback_data='{} >>>'.format(nav_text))]
        return buttons

    def main_menu(self, source, id):
        return [
                [button('Volumes', callback_data='vls {} {} 0 -'.format(source, id))],
                [button('Download', callback_data='d {} {}'.format(source, id))]
            ]

    async def load_mangalib_book(self, session, id, new=False):
        from_session = session.get('mangalib')
        if from_session is None:
            from_session = {}
            session['mangalib'] = from_session
        book = from_session.get(id) if not new else None
        if not book:
            book = await MangaLibBook(id).load()
            if book:
                from_session[id] = book
        return book if book.loaded else None

    async def send_manga(self, chat_id, source, id, thumbnail, name, other_name, lang, description, chapters, volumes):
        await self.bot.photo(thumbnail, chat_id=chat_id, caption='{}\n{}[{}]\n{}\nVolumes: {}\nChapters: {}'.format(name, other_name, lang, description, len(volumes), len(chapters)),
             parse_mode='HTML', reply_markup=markup(self.main_menu(source, id))).send()

    async def send_mangalib(self, chat_id, id, session):
        book = await self.load_mangalib_book(session, id, True)
        if book:
            info = await book.info
            await self.send_manga(chat_id, 'mangalib', id, book.thumbnail, info['name'], info['rus_name'], book.lang, info['summary'].replace('
', ''), book.chapters, book.volumes)

    async def load_all_chapters(self, message, book, filename):
        msg = StatusMessage(self.bot, message, 'Loading...')
        chapter_count = len(book.chapters)
        await msg.next('Loading {} chapters...'.format(chapter_count))
        async def status_chapters(num):
            await msg.next('Chapters loaded: {}/{}'.format(chapter_count - num, chapter_count))
        pictures = await book.load_chapters(status_chapters)
        pic_count = len(pictures)
        await msg.next('Loading {} pictures...'.format(pic_count))
        async def status_pictures(num):
            await msg.next('Pictures loaded: {}/{}'.format(num, pic_count))
        callback_pic = Clock(status_pictures, self.callback_timesleep)
        filename = '{}_{}_{}'.format(msg.chat_id, msg.message_id, filename)
        await asyncio.gather(self.composer.pics2pdf(pictures, filename, callback_pic.set_args), callback_pic.start(lambda n: n < pic_count - 1))
        async def status_uploading(num, all_):
            await msg.next('Uploading: {} %'.format(int(num/all_*100)))
        await self.send_via_telethon(filename, msg.chat_id, status_uploading)
        await msg.next('Uploaded!!!')

    async def back_to_main(self, source, id, chat_id, message_id):
        await self.bot.editreplymarkup(chat_id=chat_id, message_id=message_id, reply_markup=markup(self.main_menu(source, id))).send()

    async def load_all_chapters_from_mangalib(self, message, id, session):
        book = await self.load_mangalib_book(session, id)
        if book:
            await self.load_all_chapters(message, book, id + '.pdf')

    async def send_via_telethon(self, file, chat_id, progress_callback):
        callback = Clock(progress_callback, self.callback_timesleep)
        token = str(random_token())
        async def send_file(telethon):
            self.set_file_to_send(token, chat_id)
            if not self.bot_name:
                await self.set_bot_name()
            await telethon.send_file(self.bot_name, file, caption=token, progress_callback=callback.set_args)
            callback.stop()
        async with TelegramClient(StringSession(self.telethon_session), self.telethon_api_id, self.telethon_api_hash) as telethon:
            await asyncio.gather(send_file(telethon), callback.start())

    def set_file_to_send(self, token, chat_id):
        self.files[token] = {'chat_id': chat_id}

    def get_file_to_send(self, token):
        return self.files.get(token)

    async def send_file_by_token(self, file_id, token):
        unfile = self.get_file_to_send(token)
        if unfile:
            await self.bot.document(file_id, unfile['chat_id']).send()

    async def content_list(self, chat_id, message_id, id, page, source, volumes, turn=None):
        lines_count = 0
        lines = []
        for vol in volumes:
            lines.append(vol)
            if vol.choosen:
                lines.extend(vol.chapters)
                lines_count += (2 + len(vol.chapters))
            else:
                lines_count += 1
        if turn:
            pages = lines_count // self.per_page + 1
            vector = self.nav_codes(turn)
            start_page = pages + page + vector if page + vector < 0 else page + vector if page + vector < pages else page + vector - pages
        else:
            start_page = page
        start = start_page * self.per_page
        end = None if start + self.per_page > lines_count else start + self.per_page
        buttons_lines = []
        for line in lines[start:end]:
            if line.type == 'chapter':
                buttons_lines.append([button('{} {}: {}'.format('✅' if line.choosen else '📑', line.id, line.title), callback_data='c{} {} {} {}'.format(0 if line.choosen else 1, source, id, line.id))])
            elif line.type == 'volume':
                buttons_lines.append([button('📁 Volume {}'.format(line.id), callback_data='v{} {} {} {}'.format(0 if line.choosen else 1, source, id, line.id))])
                if line.choosen:
                    buttons_lines.append([button('Choose all'.format(line.id), callback_data='c_all {} {} {}'.format(source, id, line.id))])
        buttons = [[button('< To main', callback_data='main mangalib {}'.format(id))]] + buttons_lines + [self.navbar(lines_count, 'vls mangalib {} {}'.format(id, start_page), [button(start_page + 1, callback_data='page {} {}'.format(start_page, self.per_page))])]
        await self.bot.editreplymarkup(chat_id=chat_id, message_id=message_id, reply_markup=markup(buttons)).send()

    async def volumes_mangalib(self, chat_id, message_id, id, page, turn, session):
        book = await self.load_mangalib_book(session, id)
        if book:
            await self.content_list(chat_id, message_id, id, page, 'mangalib', book.volumes_list, turn)

    #async def set_volume_choose(chat, i)

class Clock:
    def __init__(self, async_callback, delta):
        self.callback = async_callback
        self.delta = delta
        self.args = None
        self.is_running = True

    def set_args(self, *args):
        self.args = args

    async def start(self, while_=None):
        while self.is_running and not (while_ and self.args and not while_(*self.args)):
            if self.args:
                await self.callback(*self.args)
            await asyncio.sleep(self.delta)

    def stop(self):
        self.is_running = False


# -*- coding:UTF-8  -*-
"""
获取指定账号的全部重复库存内个人资料背景和表情
https://store.steampowered.com/
@author: hikaru
email: hikaru870806@hotmail.com
如有问题或建议请联系
"""
from common import output, crawler
from game.steam import steamCommon

CHECK_EXTRA_CARD = True  # 是否检测额外的交换卡（徽章等级大于5，并且还有卡牌）
CHECK_DUPLICATE_BACKGROUND = True  # 是否检测重复的个人背景图
CHECK_DUPLICATE_EMOTICON = False  # 是否检测重复的表情

# 获取当前account正在收集的徽章进度
def main(account_id):
    try:
        inventory_item_list = steamCommon.get_account_inventory(account_id)
    except crawler.CrawlerException as e:
        output.print_msg("库存解析失败，原因：%s" % e.message)
        raise
    badges_list = {}
    if CHECK_EXTRA_CARD:
        # 获取徽章等级
        try:
            badges_list = steamCommon.get_account_badges(account_id)
        except crawler.CrawlerException as e:
            output.print_msg("获取徽章等级失败，原因：%s" % e.message)
            raise
    for item_id, item_info in inventory_item_list.items():
        if item_info["type"] == steamCommon.INVENTORY_ITEM_TYPE_PROFILE_BACKGROUND:
            if CHECK_DUPLICATE_BACKGROUND and item_info["count"] > 1:
                output.print_msg(item_info)
        elif item_info["type"] == steamCommon.INVENTORY_ITEM_TYPE_EMOTICON:
            if CHECK_DUPLICATE_EMOTICON and item_info["count"] > 1:
                output.print_msg(item_info)
        elif item_info["type"] == steamCommon.INVENTORY_ITEM_TYPE_TRADE_CARD:
            if CHECK_EXTRA_CARD:
                # 闪卡，跳过
                if item_info['name'].find("(Foil)") >= 0:
                    continue
                if item_info["game_id"] in badges_list:
                    badge_level = badges_list[item_info["game_id"]]
                else:
                    badge_level = 0
                # 如果剩余卡牌数量 + 当前徽章等级 > 5
                if item_info["count"] + badge_level > steamCommon.MAX_BADGE_LEVEL:
                    output.print_msg(item_info)

if __name__ == "__main__":
    main(steamCommon.get_account_id_from_file())


import time
import utils

from autotest_lib.client.common_lib import error
from autotest_lib.client.cros.enterprise import enterprise_policy_base
from telemetry.core.exceptions import EvaluateException

class policy_DefaultJavaScriptSetting(
        enterprise_policy_base.EnterprisePolicyTest):
    """
    Tests the DefaultJavaScriptSetting policy in Chrome OS.

    If the DefaultJavaScriptSetting policy is set to Allow, going to the
    local test page should display YES. If the policy is set to Do Not Allow
    going to the local test page should display NO.

    """
    version = 1

    def initialize(self, **kwargs):
        """Initialize this test."""
        super(policy_DefaultJavaScriptSetting, self).initialize(**kwargs)
        self.TEST_FILE = 'js_test.html'
        self.TEST_URL = '%s/%s' % (self.WEB_HOST, self.TEST_FILE)
        self.POLICY_NAME = 'DefaultJavaScriptSetting'
        self.POLICIES = {}
        self.TEST_CASES = {
            'Allow': 1,
            'Do Not Allow': 2}
        self.start_webserver()

    def _can_execute_javascript(self):
        """
        Determine whether JavaScript is allowed to run on the given page.

        @param tab: browser tab containing JavaScript to run.

        Note: This test does not use self.navigate_to_url(), because it can
        not depend on methods that evaluate or execute JavaScript.

        """
        tab = self.cr.browser.tabs.New()
        tab.Activate()
        tab.Navigate(self.TEST_URL)
        time.sleep(1)

        utils.poll_for_condition(
            lambda: tab.url == self.TEST_URL,
            exception=error.TestError('Test page is not ready.'))

        try:
            utils.poll_for_condition(
                lambda: tab.EvaluateJavaScript('jsAllowed', timeout=2),
                exception=error.TestError('Test page is not ready.'))
            return True
        except (EvaluateException, utils.TimeoutError):
            return False

    def run_once(self, case):
        """
        Setup and run the test configured for the specified test case.

        @param case: Name of the test case to run.

        """
        case_value = self.TEST_CASES[case]
        self.POLICIES[self.POLICY_NAME] = case_value
        self.setup_case(user_policies=self.POLICIES)

        if case == 'Allow':
            if not self._can_execute_javascript():
                raise error.TestFail('JavaScript disabled, should be enabled.')
        else:
            if self._can_execute_javascript():
                raise error.TestFail('JavaScript enabled, should be disabled.')


import os
import sys

from get_sys import getSystemInfo

# ------------------------------------------------------- #

PROJECT_DIR = os.path.abspath(os.path.dirname(sys.argv[0]))
PROJECT_PATH_NAME = "KeyKeeperData"
PROJECT_PATH = os.path.join('C:', PROJECT_PATH_NAME)
USERS_PATH = os.path.join(PROJECT_PATH, 'users')

SYS = getSystemInfo()

BASE_FILTERS = [
    "private",
    "work",
    "family"
]

BASE_SETTINGS = {
    "user": {
        "uuid": ""
    },
    "color": {
        "theme": "Dark",
        "primary": "Red",
        "hue": "500"
    },
    "encryption": {
        "level": 5
    },
    "session": {
        "filter": ""
    }
}


from lexata import Lexata

def test_query_simple():
    lexata = Lexata()
    query = 'pollution'
    results = lexata.get_closest_to(query)
    print('Best: ', results)
    assert type(results) == dict
    assert len(results) == 1

def test_query_multiple_factors():
    lexata = Lexata()
    query = 'can changes in the global climate or extreme weather conditions affect my business'
    results = lexata.get_closest_to(query, 4, ['information Technology'])
    print('Best: ', results)
    assert type(results) == dict
    assert len(results) == 4


from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from PyQt5.QtGui import *

class MyWindow(QMainWindow):
    def __init__(self, *args, **kwargs):
        super(MyWindow, self).__init__(*args, **kwargs)
        self.setWindowTitle("Buttons Window")
        self.resize(500,500)

        # now lets create buttons
        btn1 = QPushButton("Button 1")
        btn2 = QPushButton("Button 2")

        # we can use the following two lines as labels but they will show output on command window.
        #label1 = QLabel("Click on button 1")
        #label2 = QLabel("Click on button 2")

        # we want output on application window after we clicked on buttons. to do so , we need to make label part of the class which is our application window class. we will do it by using self keyword
        self.label1 = QLabel("Click on button 1:")
        self.label2 = QLabel("Click on button 2")

        # change the font of these labels: pick font of the label in variable and change it
        font1 = self.label1.font()
        font1.setPointSize(40)

        # now apply this changing of point size of font on label
        self.label1.setFont(font1)

        # change the font of these labels: pick font of the label in variable and change it
        font2 = self.label2.font()
        font2.setPointSize(60)

        # now apply this changing of point size of font on label
        self.label2.setFont(font2)

        # align these labels to center
        self.label1.setAlignment(Qt.AlignCenter)
        self.label2.setAlignment(Qt.AlignCenter)

        # add actions to the button: the function will be called by clicking on button. must call that function with keyword 'self' as those functions are part of class
        btn1.clicked.connect(self.btn1_clicked)
        btn2.clicked.connect(self.btn2_clicked)

        # if you do not want to call to functions on clicking the buttons you can use the lambda functions
        # btn1.clicked.connect(lambda x: self.label.setText("Button 1 is clicked"))
        # btn2.clicked.connect(lambda x: self.label.setText("Button 2 is clicked"))

        # add the vertical layout
        layout = QVBoxLayout()

        # Add stuff to the layout using addwidget method
        # if we want to use labels without self, then add them in layout with the following line:
        #layout.addWidget(label1)
        # layout.addWidget(label2)

        # if you want to show label on the application window, then add them in layout with self keyword
        layout.addWidget(self.label1)
        layout.addWidget(btn1)
        layout.addWidget(self.label2)
        layout.addWidget(btn2)

        # Create a universal widget and add everything to that i.e. layouts, buttons etc.
        widget = QWidget()

        # First of all, add layout to this universal widget mtlb is widget ko aik layout di pehly
        widget.setLayout(layout)

        # make this widget central on the window: as we are making changes on application window, so use self

        self.setCentralWidget(widget)

        # must give self as parameter in both these functions
    def btn1_clicked(self):
        # this print output will be displayed on the command window, not on the application window
        #print("Button 1 is clicked")
        # to display output on the application window, use the following commands
        self.label1.setText("You clicked on button 1")
    def btn2_clicked(self):
        # this print output will be displayed on the command window, not on the application window
        #print("Button 2 is clicked")

        # to display output on the application window, use the following commands
        self.label2.setText("You clicked on button 2")

# Now lets make application and run it
app = QApplication([])
window = MyWindow()
window.show()
app.exec()


import os
import cv2
import torch
import torchvision
from torchvision import transforms, datasets
import torch.nn as nn
import time
from torchvision import models
import numpy as np
from torch.utils.data.sampler import WeightedRandomSampler
from skimage import io, transform

torch.cuda.set_device(0)

class TransferModel(nn.Module):
    def __init__(self,
                base_model : str = 'resnet50',
                pretrain : bool = True,
                n_class : int = 4):
        super(TransferModel, self).__init__()
        self.base_model = base_model
        self.pretrain = pretrain
        self.n_class = n_class
        if self.base_model == 'resnet50':
            self.model = torchvision.models.resnet50(pretrained=True)
            n_features = self.model.fc.in_features
            fc = torch.nn.Linear(n_features, n_class)
            self.model.fc = fc
        else:
            # Use other models you like, such as vgg or alexnet
            pass
        self.model.fc.weight.data.normal_(0, 0.005)
        self.model.fc.bias.data.fill_(0.1)

    def forward(self, x):
        return self.model(x)

    def predict(self, x):
        return self.forward(x)

class Rescale(object):
    def __init__(self, output_size):
        assert isinstance(output_size, (int, tuple))
        self.output_size = output_size

    def __call__(self, image):
        h, w = image.shape[:2]
        if isinstance(self.output_size, int):
            if h > w:
                new_h, new_w = self.output_size * h / w, self.output_size
            else:
                new_h, new_w = self.output_size, self.output_size * w / h
        else:
            new_h, new_w = self.output_size

        new_h, new_w = int(new_h), int(new_w)

        img = transform.resize(image, (new_h, new_w), mode='constant')

        return img

def get_video_details(video_array, model):
    # video_array(frame_num, w, h, channel)
    crop = Rescale((224, 224))
    to_tensor = transforms.ToTensor()
    norm = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                std=[0.229, 0.224, 0.225])
    print(video_array.shape)
    preprocess_array = None
    results = None
    batch_size = 10
    batch_num = int(video_array.shape[0]/batch_size)
    for i, frame in enumerate(video_array):
        crop_image = crop(frame)
        tensor_image = to_tensor(crop_image).cuda()
        # print(tensor_image.shape)
        norm_image = norm(tensor_image)
        norm_image = norm_image.float()
        norm_image = norm_image.unsqueeze(0)
        if(preprocess_array == None):
            # print("new!")
            preprocess_array = norm_image
        else:
            # print("append!")
            preprocess_array = torch.cat([preprocess_array, norm_image], dim=0)
    print(preprocess_array.shape)

    for i in range(batch_num):
        if(results==None):
            results = model(preprocess_array[i*batch_size:(i+1)*batch_size])
        else:
            result_now = model(preprocess_array[i*batch_size:(i+1)*batch_size])
            results = torch.cat([results,result_now],axis = 0)

    if(results==None):
        results = model(preprocess_array[batch_num*batch_size:])
    elif(video_array.shape[0]-batch_size*batch_num >0):
        # print(preprocess_array[batch_num*batch_size:].shape)
        result_now = model(preprocess_array[batch_num*batch_size:])
        results = torch.cat([results,result_now],axis = 0)

    return results

def get_image_details(np_image,model):
    # image_array(w,h,channel)

    crop = Rescale((224, 224))
    to_tensor = transforms.ToTensor()
    norm = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                std=[0.229, 0.224, 0.225])

    crop_image = crop(np_image)
    tensor_image = to_tensor(crop_image).cuda()
    # print(tensor_image.shape)
    norm_image = norm(tensor_image)
    norm_image = norm_image.float()
    norm_image = norm_image.unsqueeze(0)

    result = model(norm_image)

    return result


import eikon as ek
import asyncio as asyncio
from IPython.display import display
import pandas as pd
import twitter

print("Python will now attempt to submit tweets to twitter...")

print("Connecting to Twitter API")
#Twitter API https://github.com/bear/python-twitter
api = twitter.Api(consumer_key='HIDDEN',
                      consumer_secret='HIDDEN',
                      access_token_key='HIDDEN',
                      access_token_secret='HIDDEN')

print("Connecting to Refinitiv API")
ek.set_app_key('HIDDEN')

streaming_prices = ek.StreamingPrices(
    instruments = ['BTC=', '.SPX', 'SPY', 'VXX', 'TLT.O', 'GLD', 'SLV', 'BBY'],
    fields   = ['CF_LAST', 'CF_BID','CF_ASK','OPEN_PRC', 'CF_HIGH','CF_LOW', 'CF_CLOSE', 'TR.TSVWAP', 'OPINT_1', 'OPINTNC']
)

streaming_prices.open()

df = streaming_prices.get_snapshot(
    instruments = ['BTC=', '.SPX', 'SPY', 'VXX', 'BBY'],
    fields   = ['CF_LAST','OPINTNC','OPINT_1']
)

status = api.PostUpdate("{0}".format(df))

print("Tweets submitted successfully!")

streaming_prices.close()


# -*- coding: utf-8 -*-
"""
Created on Mon Mar 30 21:34:43 2020

@author: aarias
"""
import numpy as np

# def parse_ScanMetada(xyz_scan_txt, channel_txt, ending_txt):
def parse_ScanMetada(dataStructure):

    #Unpack Dictionary
    ending_txt = dataStructure["ending"]
    channel_txt = dataStructure["channel"]
    tileX_txt = dataStructure["tileX"]
    tileY_txt = dataStructure["tileY"]
    sliceZ_txt = dataStructure["sliceZ"]

    #Parse DataStructure
    [myRoot, tileX, x_index] = parse_String(tileX_txt)
    [myRoot, tileY, y_index] = parse_String(tileY_txt)
    [myRoot, sliceZ, z_index] = parse_String(sliceZ_txt)
    [myRoot, channel, index] = parse_String(channel_txt)
    [myRoot, ending, index]  = parse_String(ending_txt)
    xyz_index = np.array([x_index, y_index, z_index], dtype=object)

    # print()
    # print('x_index', x_index)
    # print('y_index', y_index)
    # print('z_index', z_index)
    # print('xyz_index', xyz_index)
    # jajaja

    #Pack Dictionary
    parsedStructure = {"parsed_ending": ending,
                      "parsed_channel": channel,
                      "parsed_coord": xyz_index,
                      }

    return parsedStructure
    # return  [xyz_index, channel, ending]

def parse_String(myStr):
    myStr  = myStr.split('_', 2)
    myRoot = myStr[0]
    data = None
    index = None
    if len(myStr)==3:
        data  = myStr[2]
        index = [int(myStr[1]), (int(myStr[1]) + len(data))]
    return [myRoot, data, index]

if __name__ == '__main__':
     ending = "File_0_Cam_Left_00000.h5"
     channel = "Folder_16_channel_0"
     scanX = "Folder_9_12"
     scanY = "Folder_13_00"
     scanZ = '_'
     # scanZ = None
# =============================================================================
#
# =============================================================================

     print (parse_String(scanX))
     [myRoot, data, index] = parse_String(scanX)
     myName = 'stack_0-x12-y00_channel_0'
     print(myName[index[0]:index[1]])

     a ='dfgdfgd'.split('_', 2)
     b = 'Folder_13_00'.split('_', 2)
     c = "File_0_Cam_Left_00000.h5".split('_', 2)
     print (parse_String(scanZ))

##     myName[x_index[0]:x_index[-1]]
#     v = list(np.arange(4))
#     myName[1:2]


# 상근이의 여행

# 가장 적은 종류의 비행기를 타고 모든 국가를 여행하려 함.
# 상근이가 한 국가에서 다른 국가로 이동할 때 다른 국가(이미 방문국가도 O)를 거쳐가도 가능

import sys
input = sys.stdin.readline

T=int(input())

# index 번째 나라 방문하도록 함 => result[index]=1
# index 번째 나라를 가는 비행기들 번호를
def dfs(index, cnt):
    result[index]=1
    for i in planes[index]:
        if result[i] == 0:
            cnt = dfs(i, cnt+1)
    return cnt

for _ in range(T):
    N,M = map(int, input().split())
    planes=[[] for _ in range(N)]
    result = [0]*N
    for _ in range(M):
        a, b = map(int, input().split())
        planes[a-1].append(b-1)
        planes[b-1].append(a-1)

    count = dfs(0,0)
    print(count)


import json
import os
import time
import traceback
from socket import gethostname
from threading import Thread

from framework.base.internal_msgQ import InternalMsgQ
from framework.base.module_thread import (ScheduledModuleThread, SensorThread,
    SensorThreadState)
from framework.base.sspl_constants import (OperatingSystem, cs_legacy_products,
    cs_products, enabled_products, DATA_PATH)
# Import modules to control
from framework.messaging.egress_processor import \
    EgressProcessor
from framework.messaging.ingress_processor import \
    IngressProcessor
from framework.utils.conf_utils import (
    GLOBAL_CONF, SSPL_CONF, Conf, SSPL_LL_SETTING, NODE_ID_KEY)
from framework.utils.service_logging import logger
from json_msgs.messages.actuators.thread_controller import ThreadControllerMsg
from json_msgs.messages.sensors.thread_monitor_msg import ThreadMonitorMsg
from message_handlers.disk_msg_handler import DiskMsgHandler
# Note that all threaded message handlers must have an import here to be controlled
from message_handlers.node_controller_msg_handler import \
    NodeControllerMsgHandler
from message_handlers.node_data_msg_handler import NodeDataMsgHandler
from message_handlers.real_stor_actuator_msg_handler import \
    RealStorActuatorMsgHandler
from message_handlers.real_stor_encl_msg_handler import RealStorEnclMsgHandler
from message_handlers.service_msg_handler import ServiceMsgHandler
from framework.utils.store_factory import store

module_persistent_data = {}
module_cache_dir = os.path.join(DATA_PATH, "modules")
node_id = Conf.get(GLOBAL_CONF, NODE_ID_KEY, "SN01")

def _get_recovery_config(module_name):
    """
    Read sspl config for corresponding module recovery configs.

    Common sensor recovery config will be overriden by individual
    module recovery config.
    """
    recovery_count = Conf.get(
        SSPL_CONF, f"{SSPL_LL_SETTING}>sensor_recovery_count", 0)
    recovery_interval = Conf.get(
        SSPL_CONF, f"{SSPL_LL_SETTING}>sensor_recovery_interval", 0)
    # Override common recovery config if individual module has it
    recovery_count = Conf.get(
        SSPL_CONF,
        f"{module_name.upper()}>sensor_recovery_count", recovery_count)
    recovery_interval = Conf.get(
        SSPL_CONF,
        f"{module_name.upper()}>sensor_recovery_interval", recovery_interval)
    return recovery_count, recovery_interval

# Global method used by Thread to capture and log errors.  This must be global.
def execute_thread(module, msgQlist, conf_reader, product, resume=True):
    """
    Run module as a thread. Recover the module if any error during
    initialization and run time of the module.

    If recovery count>0,
        module will be recovered from failure until the maximum recovery
        attempt. If not recoverable, corresponding module will be shutdown
        and failure alert will be raised due to its impact.
    If recovery count=0,
        no recovery attempt will be made.
    """
    module_name = module.name()
    # Suspend module threads
    if resume == False:
        module.suspend()

    # Initialize persistent cache for sensor status
    per_data_path = os.path.join(
        module_cache_dir, f"{module_name.upper()}_{node_id}")
    if not os.path.isfile(per_data_path):
        module_persistent_data[module_name] = {}
        store.put(module_persistent_data[module_name], per_data_path)

    is_sensor_thread = False
    recovery_count = recovery_interval = 0
    if isinstance(module, SensorThread):
        recovery_count, recovery_interval = _get_recovery_config(module_name)
        is_sensor_thread = True

    attempt = 0

    while attempt <= recovery_count:
        attempt += 1
        try:
            # Each module is passed a reference list to message queues so it
            # can transmit internal messages to other modules as desired
            module.start_thread(conf_reader, msgQlist, product)
        except Exception as err:
            curr_state = "fault"
            err_msg = f"{module_name}, {err}"
            logger.error(err_msg)
            if attempt > recovery_count:
                logger.debug(traceback.format_exc())
                description = f"{module_name} is stopped and unrecoverable. {err_msg}"
                impact = module.impact()
                recommendation = "Restart SSPL service"
                logger.critical(
                    f"{description}. Impact: {impact} Recommendation: {recommendation}")
                # Check previous state of the module and send fault alert
                if os.path.isfile(per_data_path):
                    module_persistent_data[module_name] = store.get(per_data_path)
                prev_state = module_persistent_data[module_name].get('prev_state')
                if is_sensor_thread and curr_state != prev_state:
                    module_persistent_data[module_name] = {"prev_state": curr_state}
                    store.put(module_persistent_data[module_name], per_data_path)
                    specific_info = Conf.get(SSPL_CONF, f"{module_name.upper()}")
                    info = {
                        "module_name": module_name,
                        "alert_type": curr_state,
                        "description": description,
                        "impact": impact,
                        "recommendation": recommendation,
                        "severity": "critical",
                        "specific_info": specific_info
                    }
                    jsonMsg = ThreadMonitorMsg(info).getJson()
                    module._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
            else:
                logger.debug(f"Recovering {module_name} from failure, "
                             f"attempt: {attempt}")
                time.sleep(recovery_interval)

            # Shutdown if no recovery attempt
            logger.info(f"Terminating monitoring thread {module_name}")
            module.shutdown()
            retry = 5
            while module.is_running():
                module.shutdown()
                retry -= 1
                if not retry:
                    break
                time.sleep(2)

def _check_module_recovered(module):
    """
    Once SSPL is restarted, check current status of the module after
    certain recovery cycle time. If module is running and its previous
    state is fault, raise fault_resolved alert and update cache.
    """
    module_name = module.name()
    # Wait till sensor module completes few run cycle. Then
    # raise module recovery fault_resolved alert.
    polling_cycle_time = Conf.get(
        SSPL_CONF, f"{SSPL_LL_SETTING}>sensor_polling_cycle_time", 60)
    time.sleep(polling_cycle_time)
    if not module.is_running():
        return

    curr_state = "fault_resolved"
    per_data_path = os.path.join(
        module_cache_dir, f'{module_name.upper()}_{node_id}')
    if not os.path.isfile(per_data_path):
        module_persistent_data[module_name] = {}
        store.put(module_persistent_data[module_name], per_data_path)
    # Check previous state before sending fault resolved alert
    module_persistent_data[module_name] = store.get(per_data_path)
    prev_state = module_persistent_data[module_name].get('prev_state')
    if prev_state and curr_state != prev_state:
        module_persistent_data[module_name] = {"prev_state": curr_state}
        store.put(module_persistent_data[module_name], per_data_path)
        specific_info = Conf.get(SSPL_CONF, f"{module_name.upper()}")
        info = {
            "module_name": module_name,
            "alert_type": curr_state,
            "description": f"{module_name} is recovered",
            "impact": "",
            "recommendation": "",
            "severity": "info",
            "specific_info": specific_info
        }
        jsonMsg = ThreadMonitorMsg(info).getJson()
        module._write_internal_msgQ(EgressProcessor.name(), jsonMsg)

class ThreadController(ScheduledModuleThread, InternalMsgQ):

    MODULE_NAME = "ThreadController"
    PRIORITY = 1

    # Section and keys in configuration file
    THREADCONTROLLER = MODULE_NAME.upper()
    ALWAYS_ACTIVE_MODULES = [
        "EgressProcessor", "IngressProcessor",
        "ThreadController"
    ]

    # Constats for keys to read from conf file
    SSPL_SETTING = 'SSPL_LL_SETTING'
    DEGRADED_STATE_MODULES = 'degraded_state_modules'

    @staticmethod
    def name():
        """ @return: name of the monitoring module."""
        return ThreadController.MODULE_NAME

    def __init__(self):
        super(ThreadController, self).__init__(self.MODULE_NAME,
                                                  self.PRIORITY)
        self._threads_initialized = False
        self._thread_response = "N/A"
        self.debug_section = None

        # Location of hpi data directory populated by dcs-collector
        self._hpi_base_dir = "/tmp/dcs/hpi"
        self._start_delay = 10
        self._systemd_support = True
        self._hostname = gethostname()
        self._modules_to_resume = []

    def get_sspl_module(self, module):
        try:
            return self._sspl_modules[module]
        except KeyError:
            return None

    def initialize(self, conf_reader, msgQlist, product):
        """initialize configuration reader and internal msg queues"""
        # Initialize ScheduledMonitorThread
        super(ThreadController, self).initialize(conf_reader)

        # Initialize internal message queues for this module
        super(ThreadController, self).initialize_msgQ(msgQlist)
        self._modules_to_resume = self._get_degraded_state_modules_list()

    def initialize_thread_list(self, sspl_modules, operating_system, product,
                               systemd_support):
        """initialize list of references to all modules"""
        self._sspl_modules = sspl_modules
        self._product = product
        self._operating_system = operating_system
        self._systemd_support = systemd_support

        if operating_system == OperatingSystem.CENTOS7.value or operating_system == OperatingSystem.RHEL7.value or \
            operating_system.lower() in OperatingSystem.CENTOS7.value.lower() or operating_system.lower() in OperatingSystem.RHEL7.value.lower():
            # Note that all threaded sensors and actuators must have an
            # import here to be controlled
            from sensors.impl.centos_7.disk_monitor import DiskMonitor
            from sensors.impl.centos_7.service_monitor import ServiceMonitor
            from sensors.impl.centos_7.drive_manager import DriveManager
            from sensors.impl.centos_7.hpi_monitor import HPIMonitor

        # Handle configurations for specific products
        if product.lower() == "cs-a":
            from sensors.impl.generic.SMR_drive_data import SMRdriveData
        if product.lower() in [x.lower() for x in enabled_products]:
            from sensors.impl.platforms.realstor.realstor_disk_sensor \
                import RealStorDiskSensor
            from sensors.impl.platforms.realstor.realstor_psu_sensor \
                import RealStorPSUSensor
            from sensors.impl.platforms.realstor.realstor_fan_sensor \
                import RealStorFanSensor
            from sensors.impl.platforms.realstor.realstor_controller_sensor \
                import RealStorControllerSensor
            from sensors.impl.platforms.realstor.realstor_sideplane_expander_sensor \
                import RealStorSideplaneExpanderSensor
            from sensors.impl.platforms.realstor.realstor_dg_volume_sensor \
                import RealStorLogicalVolumeSensor
            from sensors.impl.platforms.realstor.realstor_enclosure_sensor \
                import RealStorEnclosureSensor
            from sensors.impl.generic.raid import RAIDsensor
            from sensors.impl.generic.raid_integrity_data import RAIDIntegritySensor

    def run(self):
        """Run the module periodically on its own thread."""
        if (self._product.lower() in [x.lower() for x in enabled_products]) and \
           not self._threads_initialized:
            if self._product.lower() in [x.lower() for x in cs_products]:
                # Wait for the dcs-collector to populate the /tmp/dcs/hpi directory
                while not os.path.isdir(self._hpi_base_dir):
                    logger.info(
                        "ThreadController, dir not found: %s " % self._hpi_base_dir)
                    logger.info(
                        "ThreadController, rechecking in %s secs" % self._start_delay)
                    time.sleep(int(self._start_delay))

            logger.debug("ThreadController._sspl_modules is {}".format(
                self._sspl_modules))
            # Allow other threads to initialize
            continue_waiting = False
            for (n,m) in self._sspl_modules.items():
                if not isinstance(m, SensorThread):
                    continue
                thread_init_status = m.get_thread_init_status()
                logger.debug("Thread status for {} is {}".format(
                    m.__class__, thread_init_status))
                if thread_init_status == SensorThreadState.WAITING:
                    continue_waiting = True

            if continue_waiting:
                logger.debug("ThreadController, waiting for all modules to initialize")
                self._scheduler.enter(10, self._priority, self.run, ())
                return

            # Notify external applications that've started up successfully
            startup_msg = "SSPL-LL service has started successfully"
            json_msg = ThreadControllerMsg(ThreadController.name(), startup_msg).getJson()
            self._write_internal_msgQ(EgressProcessor.name(), json_msg)
            self._threads_initialized = True

            # Check sensor module is recovered from previous failure
            for module in self._sspl_modules.values():
                if not isinstance(module, SensorThread):
                    continue
                module_checker = Thread(target=_check_module_recovered,
                                        args=(module,))
                module_checker.start()

            #self._set_debug(True)
            #self._set_debug_persist(True)
            self._log_debug("Start accepting requests")
        try:
            # Block on message queue until it contains an entry
            jsonMsg, _ = self._read_my_msgQ()
            if jsonMsg is not None:
                self._process_msg(jsonMsg)

            # Keep processing until the message queue is empty
            while not self._is_my_msgQ_empty():
                jsonMsg, _ = self._read_my_msgQ()
                if jsonMsg is not None:
                    self._process_msg(jsonMsg)
        except Exception as ex:
            # Log it and restart the whole process when a failure occurs
            logger.exception("ThreadController restarting: %r" % ex)

        self._scheduler.enter(1, self._priority, self.run, ())
        self._log_debug("Finished processing successfully")

    def _process_msg(self, jsonMsg):
        """Parses the incoming message and calls the appropriate method"""
        self._log_debug("_process_msg, jsonMsg: %s" % jsonMsg)

        # Check to see if debug mode is being globally turned off on all modules
        if self._check_reset_all_modules(jsonMsg) is True:
            return

        # Parse out the module name and request
        module_name    = jsonMsg.get("actuator_request_type").get("thread_controller").get("module_name")
        thread_request = jsonMsg.get("actuator_request_type").get("thread_controller").get("thread_request")

        # Parse out the uuid so that it can be sent back in Ack message
        uuid = None
        if jsonMsg.get("sspl_ll_msg_header") is not None and \
           jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None:
            uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid")
            self._log_debug("_processMsg, uuid: %s" % uuid)

        # Pass along the debug section to the module
        if jsonMsg.get("sspl_ll_debug") is not None:
            self.debug_section = { "sspl_ll_debug": {}}
            self.debug_section["sspl_ll_debug"] = jsonMsg.get("sspl_ll_debug")
        else:
            self.debug_section = None

        self._log_debug("_process_msg, self.debug_section: %s" % self.debug_section)

        # Parse out thread request and call the appropriate method
        if thread_request == "restart":
            self._restart_module(module_name)
        elif thread_request == "start":
            self._start_module(module_name)
        elif thread_request == "stop":
            # Don't let the outside world stop us or shut down this thread
            if module_name == "EgressProcessor" or \
                module_name == "IngressProcessor" or \
                module_name == "ThreadController":
                    logger.warn("Attempt to stop message processors or ThreadController Processors, \
                                    ignoring. Please try 'restart' instead.")
                    return
            self._stop_module(module_name)
        elif thread_request == "status":
            self._status_module(module_name)
        elif thread_request == "degrade":
            if module_name.lower() != "all":
                logger.warn(
                    "Invalid module_name {0}. Need 'all' in module_name"
                    .format(module_name))
                return
            self._switch_to_degraded_state(self._sspl_modules)
        elif thread_request == "active":
            if module_name.lower() != "all":
                logger.warn(
                    "Invalid module_name {0}. Need 'all' in module_name"
                    .format(module_name))
                return
            self._switch_to_active_state(self._sspl_modules)
        else:
            self._thread_response = "Error, unrecognized thread request"

        node_id = []
        if jsonMsg.get("actuator_request_type").get("thread_controller").get("parameters") is not None and \
           jsonMsg.get("actuator_request_type").get("thread_controller").get("parameters").get("node_id"):
            node_id = jsonMsg.get("actuator_request_type").get("thread_controller").get("parameters").get("node_id")

        ack_type = {}
        ack_type["hostname"] = self._hostname
        ack_type["node_id"] = node_id

        # Populate an actuator response message and transmit
        threadControllerMsg = ThreadControllerMsg(module_name, self._thread_response, \
                                                  json.dumps(ack_type))

        if uuid is not None:
            threadControllerMsg.set_uuid(uuid)
        msgString = threadControllerMsg.getJson()
        logger.info("ThreadController, response: %s" % str(msgString))
        if self._product.lower() in [x.lower() for x in enabled_products]:
            self._write_internal_msgQ(EgressProcessor.name(), msgString)
        elif self._product.lower() in [x.lower() for x in cs_legacy_products]:
            self._write_internal_msgQ(EgressProcessor.name(), msgString)

    def _restart_module(self, module_name):
        """Restart a module"""
        self._log_debug("_restart_module, module_name: %s" % module_name)

        try:
            # Stop the module if it's running and let existing thread die gracefully
            if self._status_module(module_name) is True:
                self._stop_module(module_name)

            # Allow module a few seconds to shut down gracefully
            max_wait  = 10
            curr_wait = 1
            while self._status_module(module_name) is True:
                time.sleep(3)
                logger.info("Retrying: %s" % str(curr_wait))
                self._stop_module(module_name)
                curr_wait += 1
                if curr_wait > max_wait:
                    break

            # Start the module
            self._start_module(module_name)
        except Exception as ae:
            logger.warn("Restart thread failed: %s" % str(ae))
            self._thread_response = "Restart Failed"
        else:
            self._thread_response = "Restart Successful"

    def _switch_to_degraded_state(self, modules):
        """Shifts SSPL to degraded state. Essentially it calls a suspend of
           every module every running in an independent thread
        """
        self._log_debug("_switch_to_degraded_state, modules: %s" % modules)
        if not modules:
            raise TypeError("module parameter can't be None")
        try:
            for name, module_instance in modules.items():
                if not name in self._modules_to_resume:
                    module_instance.suspend()
            self._thread_response = "Degrade Successful"
        except Exception as e:
            logger.warn("Degrade operation failed: {0}".format(e))
            self._thread_response = "Degrade failed"

    def _switch_to_active_state(self, modules):
        """Shifts SSPL to active state. Essentially it calls a resume of
           every module every running in an independent thread
        """
        self._log_debug("_switch_to_active_state, modules: %s" % modules)
        if not modules:
            raise TypeError("module parameter can't be None")
        try:
            for name, module_instance in modules.items():
                module_instance.resume()
            self._thread_response = "Active Successful"
        except Exception as e:
            logger.error("Active operation failed: {0}".format(e))
            self._thread_response = "Active failed"

    def _stop_module(self, module_name):
        """Stop a module"""
        self._log_debug("_stop_module, module_name: %s" % module_name)

        try:
            if self._status_module(module_name) is False:
                self._log_debug("_stop_module, status: False")
                return

            self._thread_response = "Stop Successful"

            # Put a debug message on the module's queue before shutting it down
            if self.debug_section is not None:
                self._write_internal_msgQ(module_name, self.debug_section)

            # Call the module's shutdown method for a graceful halt
            self._sspl_modules[module_name].shutdown()
        except Exception as ae:
            logger.warn("Stop thread failed: %s" % str(ae))
            self._thread_response = "Stop Failed"

    def _start_module(self, module_name):
        """Start a module"""
        self._log_debug("_start_module, module_name: %s" % module_name)

        try:
            if self._status_module(module_name) is True:
                self._log_debug("_start_module, status: True")
                return

            self._thread_response = "Start Successful"

            # NOTE: This is internal code that is currently unused.
            # If this is brought into use again its interaction
            # with the init dependency code will need to be considered
            module_thread = Thread(target=execute_thread,
                                   args=(self._sspl_modules[module_name],
                                         self._msgQlist, self._conf_reader,
                                         self._product))

            # Put a configure debug message on the module's queue before starting it up
            if self.debug_section is not None:
                self._write_internal_msgQ(module_name, self.debug_section)

            module_thread.start()
        except Exception as ae:
            logger.warn("Start thread failed: %s" % str(ae))
            self._thread_response = "Start Failed"

    def _status_module(self, module_name):
        """Returns if the module is running or not"""
        if self._sspl_modules[module_name].is_running() is True:
            self._thread_response = "Status: Running"
        else:
            self._thread_response = "Status: Halted"

        self._log_debug("_status_module, module_name: %s, _thread_response: %s" %
                        (module_name, self._thread_response))
        return self._sspl_modules[module_name].is_running()

    def _check_reset_all_modules(self, jsonMsg):
        """Restarts all modules with debug mode off. Activated by internal_msgQ"""
        if jsonMsg.get("sspl_ll_debug") is not None and \
            jsonMsg.get("sspl_ll_debug").get("debug_component") is not None and \
            jsonMsg.get("sspl_ll_debug").get("debug_component") == "all":
                for module in self._sspl_modules:
                    self._log_debug("_check_reset_all_modules, module: %s" % module)
                    # Don't restart this thread or it won't complete the loop
                    if module != self.name():
                        self._restart_module(module)

                # Populate an actuator response message and transmit
                msgString = ThreadControllerMsg("All Modules", "Restarted with debug mode off").getJson()
                self._write_internal_msgQ(EgressProcessor.name(), msgString)
                return True

        return False

    def shutdown_all_modules(self):
        """Calls shutdown for all modules"""
        logger.info("Shutting down all modules")
        for name, other_module in list(self._sspl_modules.items()):
            other_module.shutdown()

    def shutdown(self):
        """Clean up scheduler queue and gracefully shutdown thread"""
        super(ThreadController, self).shutdown()

    def check_EgressProcessor_is_running(self):
        """Used by the shutdown_handler to allow queued egress msgs to complete"""
        if self._product.lower() in [x.lower() for x in enabled_products]:
            return self._sspl_modules[EgressProcessor.name()].is_running()
        elif self._product.lower() in [x.lower() for x in cs_legacy_products]:
            return self._sspl_modules[EgressProcessor.name()].is_running()

    def _get_degraded_state_modules_list(self):
        """Reads list of modules to run in degraded state and returns a list
           of those modules.
        """
        # List of modules to run in degraded mode
        modules_to_resume = []
        try:
            # Read list of modules from conf file to load in degraded mode
            modules_to_resume = Conf.get(SSPL_CONF,
                f"{self.SSPL_SETTING}>{self.DEGRADED_STATE_MODULES}", [])
        except Exception as e:
            logger.warn("ThreadController: Configuration not found,"
                "degraded_state_modules")
        return modules_to_resume


import os
from os import listdir
from os.path import isfile, join
import string
import collections
import math
import pickle
import json
import sys
sys.path.append('./function')
import tf_idf as tf_idf
import cosine_similarity as cs

def do_HAC():

    new_fileObject = open("../tf_idf.p",'rb')
    tf_idf = pickle.load(new_fileObject)
    # with open('../tf_idf.txt') as json_file:
    #     tf_idf = json.load(json_file)

    doc_num = len(tf_idf)
    print ("doc_num: " +str(doc_num))
    C = [[0 for x in range(doc_num)] for y in range(doc_num)]
    I = [1 for x in range(doc_num)]
    A = []
    cluster_dis = [[y] for y in range(1,doc_num+1)]
    cluster_num = doc_num

    # Initialize
    print("Initialize")
    # for n in range(0,doc_num):
    #     print(n)
    #     for i in range(n+1,doc_num):
    #         C[n][i] = cs.cosine(tf_idf[n],tf_idf[i])
    #     I[n] = 1
    #
    # #pkl file
    # fileObject = open("../cs_table.p",'wb')
    # pickle.dump(C,fileObject)
    # fileObject.close()

    new_fileObject = open("../cs_table.p",'rb')
    C = pickle.load(new_fileObject)

    print("Calculate")
    for k in range(0,doc_num-1):
        print(k)
        merge_index = [0,0]
        merge_index = find_argmax(C, doc_num, I)
        A.append(merge_index)
        # print(cluster_dis[merge_index[1]])
        cluster_dis[merge_index[0]].extend(cluster_dis[merge_index[1]])
        cluster_dis[merge_index[1]] = 0

        # print(cluster_dis[merge_index[0]])
        for j in range(0,doc_num):
            max_sim = min(C[j][merge_index[0]],C[j][merge_index[1]])
            C[merge_index[0]][j] = max_sim
            C[j][merge_index[0]] = max_sim
        I[merge_index[1]] = 0
        cluster_num -= 1

        if cluster_num == 8 or cluster_num == 13 or cluster_num == 20:
            print_txt_file(cluster_num,cluster_dis)
            convert_json(cluster_num,cluster_dis,doc_num)

def find_argmax(C, doc_num, I):
    max_value = 0
    merge_index = [0,0]
    for n in range(0,doc_num):
        for i in range(n+1,doc_num):
            if I[n] and I[i]:
                if C[n][i] >= max_value:
                    max_value = C[n][i]
                    if n > i:
                        merge_index[0] = i
                        merge_index[1] = n
                    else:
                        merge_index[0] = n
                        merge_index[1] = i
    return merge_index

def print_txt_file(cluster_num,cluster_dis):
    temp_file_name = str(cluster_num)+".txt"
    fwrite = open("../"+temp_file_name ,"w")
    for cluster_array in cluster_dis:
        if cluster_array != 0:
            temp_sort_array = sorted(cluster_array)
            for value in temp_sort_array:
                fwrite.write(str(value)+"\n")
            fwrite.write("\n")

def convert_json(cluster_num,cluster_dis,doc_num):
    cluster_data = [0 for x in range(doc_num)]
    cluster_count = 0
    for cluster_array in cluster_dis:
        if cluster_array != 0:
            for doc in cluster_array:
                cluster_data[doc-1] = cluster_count
            cluster_count += 1

    #pkl file
    temp_file_name = "clusters_"+str(cluster_num)+".p"
    fileObject = open("../"+temp_file_name,'wb')
    pickle.dump(cluster_data,fileObject)
    fileObject.close()

# do_HAC()


#-*- coding: utf-8 -*-

import os
import unittest
from math import pi
from quicknxs import qreduce
from numpy.testing import assert_array_equal

TEST_DATASET=os.path.join(os.path.dirname(os.path.abspath(__file__)), u'test1_histo.nxs')
TEST_EVENT=os.path.join(os.path.dirname(os.path.abspath(__file__)), u'test1_event.nxs')

class GeneralClassTest(unittest.TestCase):

  def test_wrong_keyword(self):
    # make sure the classes don't accept a wrong keyword argument
    self.assertRaises(ValueError, qreduce.NXSData, '', blabli=12)
    self.assertRaises(ValueError, qreduce.NXSMultiData, [''], blabli=12)
    self.assertRaises(ValueError, qreduce.Reflectivity, None, blabli=12)
    self.assertRaises(ValueError, qreduce.OffSpecular, None, blabli=12)
    self.assertRaises(ValueError, qreduce.GISANS, None, blabli=12)

  def test_no_file(self):
    self.assertTrue(qreduce.NXSData('nonexisting') is None)

  def test_read_file(self):
    obj=qreduce.NXSData(TEST_DATASET)
    self.assertTrue(isinstance(obj, qreduce.NXSData), 'NXS file readout')
    self.assertEqual(obj.origin, TEST_DATASET, 'make sure right file name is saved')
    self._attr_check(obj, 'measurement_type')
    self.assertEqual(len(obj), 4)
    repr(obj)

  def test_read_file_event(self):
    obj=qreduce.NXSData(TEST_EVENT, bins=40)
    self.assertTrue(isinstance(obj, qreduce.NXSData), 'NXS file readout')
    self.assertEqual(obj.origin, TEST_EVENT, 'make sure right file name is saved')
    self._attr_check(obj, 'measurement_type')
    self.assertEqual(len(obj), 4)
    self.assertEqual(len(obj[0].tof), 40)

  def test_caching(self):
    obj=qreduce.NXSData(TEST_DATASET, use_caching=True, bins=40)
    obj2=qreduce.NXSData(TEST_DATASET, use_caching=True, bins=40)
    # make sure changing options triggers reload
    obj3=qreduce.NXSData(TEST_DATASET, use_caching=True, bins=50)
    self.assertTrue(obj is obj2)
    self.assertFalse(obj is obj3)
    # make sure non cached reads dont lead to the same object
    obj=qreduce.NXSData(TEST_DATASET, use_caching=False, bins=50)
    obj2=qreduce.NXSData(TEST_DATASET, use_caching=False, bins=50)
    self.assertFalse(obj is obj2)

  def test_callback(self):
    self._progress=None
    qreduce.NXSData(TEST_DATASET, use_caching=False, bins=40, callback=self._cbtest)
    self.assertFalse(self._progress is None)
    self._progress=None
    qreduce.NXSData(TEST_EVENT, use_caching=False, bins=40, callback=self._cbtest)
    self.assertFalse(self._progress is None)
    self._progress=None
    obj=qreduce.NXSMultiData([TEST_DATASET, TEST_DATASET], use_caching=False,
                             bins=40, callback=self._cbtest)
    self.assertFalse(self._progress is None)
    repr(obj)

  def _cbtest(self, progress):
    self._progress=progress

  def _attr_check(self, obj, attr, msg=None):
    result=getattr(obj, attr, None)
    self.assertFalse(result is None, msg=msg)

  def test_functions(self):
    qreduce.time_from_header(TEST_DATASET)

class DataConsistencyChecks(unittest.TestCase):

  def setUp(self):
    self.data=qreduce.NXSData(TEST_DATASET)
    self.assertFalse(self.data is None, 'setting up data object')

  def test_origin(self):
    self.assertEqual(self.data.origin, self.data[0].origin[0])

  def test_options(self):
    self.assertEqual(self.data._options, self.data[0].read_options)

  def test_itemize(self):
    item0=self.data[0]
    ch0=self.data._channel_names[0]
    cho0=self.data._channel_origin[0]
    self.assertTrue(item0 is self.data[ch0])
    self.assertTrue(item0 is self.data[cho0])
    self.assertRaises(KeyError, self.data.__getitem__, 'not_there')
    self.assertRaises(IndexError, self.data.__getitem__, 100)
    self.data[0]=item0
    self.data[ch0]=item0
    self.data[cho0]=item0
    self.assertRaises(KeyError, self.data.__setitem__, 'not_there', item0)
    self.data.numitems()
    for ignore in self.data:
      pass

  def test_data_shape(self):
    d=self.data[0]
    x, y, tof=d.x, d.y, d.tof
    self.assertEqual(d.xdata.shape, x.shape)
    self.assertEqual(d.ydata.shape, y.shape)
    self.assertEqual(d.tofdata.shape, tof.shape)

    self.assertEqual(d.xydata.shape, (y.shape[0], x.shape[0]))
    self.assertEqual(d.xtofdata.shape, (x.shape[0], tof.shape[0]))
    self.assertEqual(d.data.shape, (x.shape[0], y.shape[0], tof.shape[0]))

  def test_counts(self):
    d=self.data[0]
    # compare total counts attribute with all three arrays
    self.assertEqual(d.total_counts, d.xydata.sum())
    self.assertEqual(d.total_counts, d.xtofdata.sum())
    self.assertEqual(d.total_counts, d.data.sum())
    # compare sum over full data axes with combined arrays
    assert_array_equal(d.data.sum(axis=1), d.xtofdata, verbose=True)
    assert_array_equal(d.data.sum(axis=2), d.xydata.transpose(), verbose=True)

  def test_properies(self):
    d=self.data[0]

    assert_array_equal((d.tof_edges[:-1]+d.tof_edges[1:])/2., d.tof, verbose=True)
    v_n=d.dist_mod_det/d.tof*1e6 #m/s
    lamda_n=qreduce.H_OVER_M_NEUTRON/v_n*1e10 #A
    assert_array_equal(lamda_n, d.lamda, verbose=True)
    for item in ['lambda_center', 'number', 'experiment', 'merge_warnings',
                 'dpix', 'dangle', 'dangle0', 'sangle']:
      self.assertEqual(getattr(self.data, item), getattr(d, item),
                       msg='Wrong property lookup for "%s"'%item)
    self.assertEqual(self.data.nbytes, sum([d.nbytes for d in self.data]),
                     msg='Wrong property calculation for "nbytes"')
    qreduce.NXSData.get_cachesize()

  def test_addition(self):
    d=self.data[0]
    d2=d+d
    self.assertEqual(d.total_counts*2, d2.total_counts)
    assert_array_equal(d.data*2, d2.data, verbose=True)

  def test_multi(self):
    d=self.data[0]
    d2=qreduce.NXSMultiData([TEST_DATASET])[0]
    self.assertEqual(d.origin, d2.origin)
    self.assertEqual(d.total_counts, d2.total_counts)
    assert_array_equal(d.data, d2.data, verbose=True)

class EventModeTests(unittest.TestCase):
  def test_data(self):
    full_ds=qreduce.NXSData(TEST_EVENT, event_split_bins=None)
    self.assertEqual(full_ds[0].total_counts, full_ds[0].xydata.sum())
    full_ds=qreduce.NXSData(TEST_EVENT, event_split_bins=None, bin_type=1)
    self.assertEqual(full_ds[0].total_counts, full_ds[0].xydata.sum())

  def test_splitting(self):
    full_ds=qreduce.NXSData(TEST_EVENT, event_split_bins=None)
    split_ds=[]
    for i in range(4):
      split_ds.append(qreduce.NXSData(TEST_EVENT, event_split_bins=4,
                                      event_split_index=i))
    self.assertEqual(full_ds[0].total_counts,
                     sum([d[0].total_counts for d in split_ds]))
    self.assertEqual(full_ds[0].proton_charge,
                     sum([d[0].proton_charge for d in split_ds]))

  def not_test_direct_compare(self):
    histo=qreduce.NXSData(TEST_DATASET)
    evnt=qreduce.NXSData(TEST_EVENT, event_tof_overwrite=histo[0].tof_edges)
    assert_array_equal(evnt[0].data, histo[0].data, verbose=True)

class DataReductionTests(unittest.TestCase):
  def setUp(self):
    self.data=qreduce.NXSData(TEST_DATASET)
    self.assertFalse(self.data is None, 'setting up data object')

  def test_1reflectivity(self):
    res=qreduce.Reflectivity(self.data[0])
    self.assertTrue(isinstance(res, qreduce.Reflectivity))

  def test_2reflectivity_fan(self):
    res=qreduce.Reflectivity(self.data[0], x_pos=206., tth=0., dpix=206.)
    res2=qreduce.Reflectivity(self.data[0], extract_fan=True, normalization=res,
                              tth=0.5, dpix=206.)
    self.assertTrue(isinstance(res2, qreduce.Reflectivity))
    repr(res2)

  def test_metadata(self):
    d=self.data[0]
    res=qreduce.Reflectivity(d)
    self.assertEqual(res.read_options, d.read_options)
    self.assertEqual(res.origin, d.origin)
    repr(res)

  def test_shapes(self):
    res=qreduce.Reflectivity(self.data[0])
    res=qreduce.Reflectivity(self.data[0], normalization=res, tth=0.5, x_pos=206., dpix=206.)
    rshape=len(self.data[0].tof_edges)-1
    # as Q and R are calculated from tof/lamda/I etc., there is no need to test each one
    self.assertEqual(res.Q.shape[0], rshape)
    self.assertEqual(res.R.shape[0], rshape)
    self.assertEqual(res.dR.shape[0], rshape)
    self.assertEqual(res.dQ.shape[0], rshape)

  def test_self_normalization(self):
    res=qreduce.Reflectivity(self.data[0], x_pos=206.)
    res2=qreduce.Reflectivity(self.data[0], x_pos=206., tth=0., normalization=res, scale=0.5)
    assert_array_equal(res2.R[res.Rraw>0], 0.5, verbose=True)

  def test_background(self):
    res=qreduce.Reflectivity(self.data[0], x_pos=206., x_width=10.,
                                            bg_pos=206., bg_width=10.)
    assert_array_equal(res.R, 0., verbose=True)

  def test_background_advanced(self):
    res=qreduce.Reflectivity(self.data[0], x_pos=206., tth=0., dpix=206.)
    ignore=qreduce.Reflectivity(self.data[0], x_pos=206., x_width=10.,
                                normalization=res, bg_pos=206., bg_width=10.,
                                bg_tof_constant=True,
                                bg_poly_regions=[[(2., 100), (2., 120), (4., 120), (4., 100)]])

  def test_angle_calculation(self):
    res=qreduce.Reflectivity(self.data[0], x_pos=206., x_width=10.,
                                            bg_pos=206., bg_width=10.,
                                            tth=0., dpix=206.)
    self.assertEqual(res.ai, 0.)
    res=qreduce.Reflectivity(self.data[0], x_pos=206., x_width=10.,
                                            bg_pos=206., bg_width=10.,
                                            tth=1., dpix=206.)
    self.assertEqual(res.ai, 0.5/180.*pi)

  def test_offspec(self):
    res=qreduce.Reflectivity(self.data[0], x_pos=206., tth=0., dpix=206.)
    res2=qreduce.OffSpecular(self.data[0], normalization=res)
    self.assertTrue(isinstance(res2, qreduce.OffSpecular))
    repr(res2)

  def test_gisans(self):
    res=qreduce.Reflectivity(self.data[0], x_pos=206., tth=0., dpix=206.)
    res2=qreduce.GISANS(self.data[0], normalization=res)
    self.assertTrue(isinstance(res2, qreduce.GISANS))
    repr(res2)

suite=unittest.TestLoader().loadTestsFromTestCase(GeneralClassTest)
suite.addTest(unittest.TestLoader().loadTestsFromTestCase(DataConsistencyChecks))
suite.addTest(unittest.TestLoader().loadTestsFromTestCase(DataReductionTests))
suite.addTest(unittest.TestLoader().loadTestsFromTestCase(EventModeTests))


from django.urls import path
from . import views

app_name = 'subjects'

urlpatterns = [
    path("subjects/", views.subjects, name="subjects"),
    path("subjects/<int:pk>/", views.subject_detail, name="subject_detail"),
    path("subjects/add/", views.add, name="add"),

]


from __future__ import print_function
import matplotlib.pyplot as plt
import argparse
import os
import numpy as np
from networks.skip import skip
from networks.fcn import fcn
import cv2
import torch
import torch.optim
import glob
from skimage.io import imread
from skimage.io import imsave
import warnings
from tqdm import tqdm
from torch.optim.lr_scheduler import MultiStepLR
from utils.common_utils import *
from SSIM import SSIM
from metric_utils import gen_noisy_image
from scipy.io import savemat
from skimage import feature

parser = argparse.ArgumentParser()
parser.add_argument('--num_iter', type=int, default=3000, help='number of epochs of training')
parser.add_argument('--img_size', type=int, default=[256, 256], help='size of each image dimension')
parser.add_argument('--kernel_size', type=int, default=[21, 21], help='size of blur kernel [height, width]')
parser.add_argument('--data_path', type=str, default="datasets/levin/", help='path to blurry image')
parser.add_argument('--save_path', type=str, default="results/levin/", help='path to save results')
parser.add_argument('--save_frequency', type=int, default=1000, help='lfrequency to save results')
parser.add_argument('--tv', type=bool, default=False, help='use total variation as loss regularizer')
parser.add_argument('--canny', type=bool, default=False, help='add canny edges to input noise')
parser.add_argument('--noise', type=float, default=0, help='sets noise variance')
parser.add_argument('--save_noise', type=bool, default=False, help='saves noisy image')
opt = parser.parse_args()
#print(opt)

torch.backends.cudnn.enabled = True
dtype = torch.cuda.FloatTensor

warnings.filterwarnings("ignore")

files_source = glob.glob(os.path.join(opt.data_path, '*.png'))
files_source.sort()
save_path = opt.save_path
os.makedirs(save_path, exist_ok=True)
def total_variation_loss(img, weight):
     bs_img, c_img, h_img, w_img = img.size()
     tv_h = torch.pow(img[:,:,1:,:]-img[:,:,:-1,:], 2).sum()
     tv_w = torch.pow(img[:,:,:,1:]-img[:,:,:,:-1], 2).sum()
     return weight*(tv_h+tv_w)/(bs_img*c_img*h_img*w_img)

def tv_kernel(kernel,weight):
    tv = torch.pow(kernel,2).sum()

    return weight*tv/tv.numel()
# start #image
for f in files_source:
    INPUT = 'noise'
    pad = 'reflection'
    LR = 0.01
    num_iter = opt.num_iter
    reg_noise_std = 0.001

    path_to_image = f
    imgname = os.path.basename(f)
    imgname = os.path.splitext(imgname)[0]

    _, imgs = get_image(path_to_image, -1) # load image and convert to np.
    #If noise variance > 0, then generate noisy image
    if opt.noise > 0:
        imgs = gen_noisy_image(imgs,[opt.noise])[0]
        if opt.save_noise:
            noisy = imgs.squeeze()
            if not os.path.exists(f"{opt.data_path}/noisy/"):
                os.mkdir(f"{opt.data_path}/noisy/")
            imsave(f"{opt.data_path}/noisy/{imgname[:-4]}_noisy.png",noisy)
    y = np_to_torch(imgs).type(dtype)

    img_size = imgs.shape
    print(imgname)
    # ######################################################################
    padh, padw = opt.kernel_size[0]-1, opt.kernel_size[1]-1
    opt.img_size[0], opt.img_size[1] = img_size[1]+padh, img_size[2]+padw

    '''
    x_net:
    '''
    input_depth = 8

    #Generate canny edges from padded image
    if opt.canny:
        edges = np.zeros((opt.img_size[0], opt.img_size[1]))
        edges[padh//2:-padh//2,padw//2:-padw//2] = (feature.canny(imgs.squeeze()))
        edges = torch.Tensor(edges).type(dtype)

    net_input = get_noise(input_depth, INPUT, (opt.img_size[0], opt.img_size[1]),).type(dtype)

   #For each depth add edges
    if opt.canny:
        for d in range(input_depth):
            net_input[0,d,:,:] += edges

    net = skip( input_depth, 1,
                num_channels_down = [128, 128, 128, 128, 128],
                num_channels_up   = [128, 128, 128, 128, 128],
                num_channels_skip = [16, 16, 16, 16, 16],
                upsample_mode='bilinear',
                need_sigmoid=True, need_bias=True, pad=pad, act_fun='LeakyReLU')

    net = net.type(dtype)

    '''
    k_net:
    '''
    n_k = 200
    net_input_kernel = get_noise(n_k, INPUT, (1, 1)).type(dtype)
    net_input_kernel.squeeze_()

    net_kernel = fcn(n_k, opt.kernel_size[0]*opt.kernel_size[1])
    net_kernel = net_kernel.type(dtype)

    # Losses
    mse = torch.nn.MSELoss().type(dtype)
    ssim = SSIM().type(dtype)

    # optimizer
    optimizer = torch.optim.Adam([{'params':net.parameters()},{'params':net_kernel.parameters(),'lr':1e-4}], lr=LR)
    scheduler = MultiStepLR(optimizer, milestones=[2000, 3000, 4000], gamma=0.5)  # learning rates

    # initilization inputs
    net_input_saved = net_input.detach().clone()
    net_input_kernel_saved = net_input_kernel.detach().clone()

    ### start SelfDeblur
    net_input = net_input_saved - total_variation_loss(net_input_saved, 0.5)

    net_input_kernel = net_input_kernel_saved - tv_kernel(net_input_kernel_saved, 0.5)

    ### start SelfDeblur
    for step in tqdm(range(num_iter)):

        # input regularization
        net_input = net_input_saved + reg_noise_std*torch.zeros(net_input_saved.shape).type_as(net_input_saved.data).normal_()

        # change the learning rate
        scheduler.step(step)
        optimizer.zero_grad()

        # get the network output
        out_x = net(net_input)
        out_k = net_kernel(net_input_kernel)

        out_k_m = out_k.view(-1,1,opt.kernel_size[0],opt.kernel_size[1])
        # print(out_k_m)
        out_y = nn.functional.conv2d(out_x, out_k_m, padding=0, bias=None)

        if step < 1000:
            total_loss = mse(out_y,y)
        else:
            if not opt.tv:
                total_loss = 1-ssim(out_y, y)
            else:
                total_loss = 1-ssim(out_y, y) + total_variation_loss(out_y,1)

        total_loss.backward()
        optimizer.step()

        if (step+1) % opt.save_frequency == 0:
            #print('Iteration %05d' %(step+1))

            save_path = os.path.join(opt.save_path, '%s_x.png'%imgname)
            out_x_np = torch_to_np(out_x)
            out_x_np = out_x_np.squeeze()
            out_x_np = out_x_np[padh//2:padh//2+img_size[1], padw//2:padw//2+img_size[2]]
            imsave(save_path, out_x_np)

            save_path = os.path.join(opt.save_path, '%s_k.png'%imgname)
            out_k_np = torch_to_np(out_k_m)
            out_k_np = out_k_np.squeeze()
            out_k_np /= np.max(out_k_np)
            imsave(save_path, out_k_np)

            torch.save(net, os.path.join(opt.save_path, "%s_xnet.pth" % imgname))
            torch.save(net_kernel, os.path.join(opt.save_path, "%s_knet.pth" % imgname))


from django.conf.urls import patterns, url

from ..core import TOKEN_PATTERN
from . import views

urlpatterns = [
    url(r'^%s/$' % (TOKEN_PATTERN,), views.details, name='details'),
    url(r'^%s/payment/$' % (TOKEN_PATTERN,),
        views.payment, name='payment'),
    url(r'^%s/payment/(?P<variant>[-\w]+)/$' % (TOKEN_PATTERN,),
        views.start_payment, name='payment'),
    url(r'^%s/cancel-payment/$' % (TOKEN_PATTERN,), views.cancel_payment,
        name='cancel-payment')
]


from gmshparser.mesh_format_parser import MeshFormatParser
from gmshparser.mesh import Mesh
import io

def test_mesh_format_parser():
    parser = MeshFormatParser()
    mesh = Mesh()
    data = io.StringIO("4.1 0 8")
    parser.parse(mesh, data)
    assert mesh.get_version() == 4.1
    assert mesh.get_ascii() is True
    assert mesh.get_precision() == 8


#get the lists of maori and english words

english_line = "red,white,black,blue,green,yellow"
maori_line = "whero,ma,panga,kahurangi,kakariki,kowhai"

english = english_line.split(",")
maori = maori_line.split(",")

#write a function to perform the quiz
def do_quiz():
  """
  main quiz code
  """
  score = 0
  print('starting quiz')
  #start a loop
  for i in range(6):
    print('Question ',i)
    question = "What is the maori name for " + english[i]
    #print(question)
    #ask the next/first question
    user_response = input(question + "\n")
    print(user_response)
  #compare response with the correct answer
    if user_response == maori[i]:
      print("Correct")
      score += 1
    else:
      print('incorrect')
      pass
  #amend  tte  score
  #report back
  #display total score.
  print(f"You scored {score}.")

if __name__ == '__main__':
  do_quiz()