import subprocess
import textwrap
import sys

import pytest

def run(args, **kwargs):
    kwargs.setdefault('check', True)
    return subprocess.run(args, **kwargs)

@pytest.fixture
def run_pytest(tmp_path):
    def _run_pytest(test_file_content, *args, should_fail=False, **kwargs):
        filename = tmp_path / 'test_a.py'
        filename.write_text(textwrap.dedent(test_file_content))
        args = [sys.executable, '-m', 'pytest', filename, *args]
        kwargs['check'] = True
        if should_fail:
            with pytest.raises(subprocess.CalledProcessError):
                subprocess.run(args, **kwargs)
        else:
            subprocess.run(args, **kwargs)
    return _run_pytest

def test_sanity_positive(run_pytest):
    run_pytest(
        """
            def test_yup():
                assert 1 + 1 == 2
        """
    )

def test_sanity_negative(run_pytest):
    run_pytest(
        """
            def test_nope():
                assert 1 + 1 == 0
        """,
        should_fail=True,
    )

def test_level_marker(run_pytest):
    run_pytest(
        """
            import pytest

            @pytest.mark.level(1)
            def test_basic_math():
                assert 1 + 1 == 2

            @pytest.mark.level(2)
            def test_intermediate_math():
                assert 10 / 2 == 5

            @pytest.mark.level(3)
            def test_complicated_math():
                assert 10 ** 3 == 1000
        """,
    )

TEST_FILE_CONTENT = """
    import pytest

    @pytest.mark.level(1)
    def test_1_ok():
        assert True

    @pytest.mark.level(2)
    def test_2_ok():
        assert True

    @pytest.mark.level(3)
    def test_3_fail():
        assert False

    @pytest.mark.level(3)
    def test_4_fail():
        assert False

    @pytest.mark.level(3)
    def test_5_ok():
        assert True
"""

def test_deselects_larger(run_pytest):
    """Tests marked with a larger level are deselected"""
    run_pytest(TEST_FILE_CONTENT, '--level', '2')

def test_selects_given(run_pytest):
    """Test with the given level is selected"""
    run_pytest(TEST_FILE_CONTENT, '--level', '3', should_fail=True)

def test_selects_previous(run_pytest):
    """Test with lower levels are selected"""
    run_pytest(TEST_FILE_CONTENT, '--level', '5', should_fail=True)

@pytest.mark.level(0)
def test_marker():
    """Test with a marker.

    This should detect non-fatal issues (warnings) with the level marker.
    It doesn't actually test anything.
    """
    pass


"""
parse mln program
"""

import sys
from antlr4 import *
from .impl.MLNParser import MLNParser
from .impl.MLNLexer import MLNLexer
from .ast_visitors import NetworkListener, EvidencesListener, QueriesListener

def parse_mln(input_stream):
    """
    parse_mln_program
    """
    input = InputStream(input_stream)
    lexer = MLNLexer(input)
    stream = CommonTokenStream(lexer)
    parser = MLNParser(stream)
    tree = parser.definitions()

    mln_builder = NetworkListener()
    walker = ParseTreeWalker()
    walker.walk(mln_builder, tree)

    return mln_builder.mln

def parse_evidences(mln_program, input_stream):
    """
    parse_mln_program
    """
    input = InputStream(input_stream)
    lexer = MLNLexer(input)
    stream = CommonTokenStream(lexer)
    parser = MLNParser(stream)
    tree = parser.evidenceList()

    evidence_builder = EvidencesListener(mln_program)
    walker = ParseTreeWalker()
    walker.walk(evidence_builder, tree)

    return evidence_builder.evidences

def parse_queries(mln_program, input_stream):
    """
    parse_mln_program
    """
    input = InputStream(input_stream)
    lexer = MLNLexer(input)
    stream = CommonTokenStream(lexer)
    parser = MLNParser(stream)
    tree = parser.queryList()

    query_builder = QueriesListener(mln_program)
    walker = ParseTreeWalker()
    walker.walk(query_builder, tree)

    return query_builder.queries


#!/usr/bin/env python3

import threading
import sys
import socket
import tkinter
import time

#The constants needed
HEADER = 10
IP = ""
PORT = 0
USERNAME = ""

##########################################
#set up the tkinter GUI window

root = tkinter.Tk()
root.title("Ludde_Chat")
#root.geometry("700x500")

def raise_frame(frame):
    frame.tkraise()

def connect(event=None):
    IP = ip_msg.get()
    print (IP)

    PORT = int(port_entry.get())
    print (PORT)
    USERNAME = usr_entry.get()

    client_socket.connect((IP, PORT))
    #client_socket.setblocking(False)

    usr_name = USERNAME.encode('utf-8')
    usr_header = f"{len(usr_name):<{HEADER}}".encode('utf-8')
    client_socket.send(usr_header + usr_name)

    receive_thread.start()
    raise_frame(chat_frame)

    messages.grid()
    scrollbar.grid()
    scrollbarx.grid()
    message_box.grid()

def send(my_msg):

    new_message = my_msg.get()

    if new_message:
        message_header = f"{len(new_message):<{HEADER}}".encode('utf-8')
        ready_message = new_message.encode('utf-8')

        client_socket.send(message_header + ready_message)

        messages.insert(tkinter.END, new_message)
        messages.see(tkinter.END)
        msg.set("")

def receive():

    while True:

        try:
            usr_header = client_socket.recv(HEADER)

            usrname_length = int(usr_header.decode('utf-8').strip())

            if usrname_length == 0:
                message_box.insert(tkinter.END, "Disconnected from server")
                sys.exit()

            print(usrname_length)
            usrname = client_socket.recv(usrname_length).decode('utf-8')

            msg_header = client_socket.recv(HEADER)
            msg_length = int(msg_header.decode('utf-8').strip())

            msg = client_socket.recv(msg_length).decode('utf-8')

            complete_msg = f"{usrname} >> {msg}"

            messages.insert(tkinter.END, complete_msg)
            messages.see(tkinter.END)
        except:
            break

def closing(event=None):

    client_socket.close()
    root.destroy()
    sys.exit()

#####Functions

root.protocol("WM_DELETE_WINDOW", closing)

chat_frame = tkinter.Frame(master=root)
start_frame = tkinter.Frame(master=root)

####START####
start_frame.grid(row=0, column=0)
#start_frame.place(relx=0.5, rely=0.5, anchor="center")

ip_msg = tkinter.StringVar()
port_msg = tkinter.StringVar()
usr_msg = tkinter.StringVar()

ip_entry = tkinter.Entry(start_frame, textvariable=ip_msg)
port_entry = tkinter.Entry(start_frame, textvariable=port_msg)
usr_entry = tkinter.Entry(start_frame, textvariable=usr_msg)

ip_label = tkinter.Label(start_frame, text="IP ADDRESS")
port_label = tkinter.Label(start_frame, text="PORT NUMBER")
usr_label = tkinter.Label(start_frame, text="USERNAME")

ip_label.grid(row=0, column=0)
port_label.grid(row=1, column=0)
usr_label.grid(row=2, column=0)

ip_entry.grid(row=0, column=1)
port_entry.grid(row=1, column=1)
usr_entry.grid(row=2, column=1)

connect_button = tkinter.Button(start_frame, text="Connect", command=connect)

connect_button.grid(row=2, column=2)

####CHAT####

chat_frame.grid(row=0, column=0)
chat_frame.grid_rowconfigure(0, weight=1)
chat_frame.grid_columnconfigure(0, weight=1)
chat_frame.grid_rowconfigure(1, weight=1)
chat_frame.grid_columnconfigure(1, weight=1)

#chat_frame.place(relx=0.5, rely=0.5, anchor="center")

scrollbar = tkinter.Scrollbar(master=chat_frame)
scrollbarx = tkinter.Scrollbar(master=chat_frame, orient=tkinter.HORIZONTAL)

messages = tkinter.Listbox(chat_frame, height=20, width=50, font=("Helvetica", 14), yscrollcommand=scrollbar.set, xscrollcommand=scrollbarx.set)
scrollbar.config(command=messages.yview)
scrollbar.grid(row=0, column=1, sticky=tkinter.N+tkinter.S)

scrollbarx.grid(row=1, column=0, sticky=tkinter.W+tkinter.E)
scrollbarx.config(command=messages.xview)

messages.grid(row=0, column=0, sticky=tkinter.E+tkinter.N+tkinter.S+tkinter.W)

msg = tkinter.StringVar()
message_box = tkinter.Entry(master=chat_frame, textvariable=msg)
message_box.grid(row=2, column=0, sticky=tkinter.E+tkinter.S+tkinter.W)
message_box.bind("<Return>", lambda event: send(msg))

messages.grid_remove()
scrollbar.grid_remove()
scrollbarx.grid_remove()
message_box.grid_remove()

#########################################Networking side'

client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

receive_thread = threading.Thread(target=receive)
receive_thread.daemon = True

tkinter.mainloop()


import time
import random
import queue
import grequests

from ipproxy.utils import (
    Parser, request, logger, cut_queue, hour_delta, day_delta,
    exception_handler as eh
)
from ipproxy.settings import (
    RE_IP, DATA5U, XS_START, XS_END,
    PROXY_SITES, COLLECTION, PAGES, Q, ZDAYE_START_PAGE_ID,
)

class Collector:

    def __init__(self):
        self.logger = logger
        self.q = queue.Queue()
        self.proxy_q = queue.Queue()
        self.put_to_queue()

    def storage(self, ips):
        if not ips:
            return
        for ip in ips:
            proxy = {'http': f'http://{ip}', 'https': f'https://{ip}'}
            Q.put(proxy)
            # self.proxy_q.put(proxy)
            COLLECTION.update_one(
                {'proxy': proxy},
                {'$set': {'proxy': proxy}},
                upsert=True
            )

    def put_to_queue(self):

        urls = list()

        # for api
        urls.extend(
            [(api, self.parse_regex) for api in PROXY_SITES]
        )

        # for data5u
        # urls.extend(
        #     [(f'http://www.data5u.com/dayip-{i}-10.html', self.parse_data5u)
        #      for i in range(1, sum((1, 290)))]
        # )

        # for ip3366
        # TODO: network firewall  (head/title -> network firewall)
        # urls.extend(
        #     [(f'http://www.ip3366.net/free/?stype={j}/page={i}', self.parse_regex_2)
        #      for i in range(1, sum((1, PAGES)))
        #      for j in range(1, 5)]
        # )

        # for mayi
        now = day_delta() - 18
        urls.extend(
            [(f'http://www.mayidaili.com/share/view/{i}/', self.parse_regex)
             # for i in range(now, now - PAGES, -1)]
             for i in range(now, now - 2, -1)]
        )

        # for kuaidaili todo: 1001-1890
        urls.extend(
            [(f'http://www.kuaidaili.com/free/inha/{i}', self.parse_regex)
             for i in range(1, sum((1, 1)))]  # +
            # [(f'http://www.kuaidaili.com/free/intr/{i}', self.parse_kuai)
            #  for i in range(1, sum((1, PAGES)))]
        )

        # for zdaye
        now = hour_delta() + ZDAYE_START_PAGE_ID
        urls.extend(
            [(f'http://ip.zdaye.com/dayProxy/ip/{i}.html', self.parse_regex)
             for i in range(now, now - 2, -1)]
        )

        # for ip181
        # urls.extend(
        #     [(f'http://www.ip181.com/daili/{i}.html', self.parse_regex)
        #      for i in range(1, sum((1, 690)))]
        # )
        #
        # # for xici
        urls.extend(
            [(f'http://www.xicidaili.com/nn/{i}', self.parse_regex)
             for i in range(1, sum((1, 2)))]
        )
        #
        # # for mimi
        # urls.extend(
        #     [(f'http://www.mimiip.com/gngao/{i}', self.parse_regex_2)
        #      for i in range(1, sum((1, 680)))] +
        #     [(f'http://www.mimiip.com/gnpu/{i}', self.parse_regex_2)
        #      for i in range(1, sum((1, 100)))]
        # )

        # for xiaoshu+
        now = day_delta(XS_START, XS_END) * 2 + 27
        urls.extend(
            [(f'http://www.xsdaili.com/dayProxy/ip/{i}.html', self.parse_regex)
             for i in range(now, now - 2, -1)]  # not contains 1
        )

        # for goubanjia
        # TODO: 504
        # 'http://www.goubanjia.com/free/'

        # for xiaoma
        # TODO: new regex
        # 'http://yun-daili.com/free.asp'

        # for pachong
        # TODO: not update
        # 'http://www.pcdaili.com/index.php'

        random.shuffle(urls)
        for url in urls:
            self.q.put(url)

    def parse_regex(self, response):
        ips = RE_IP.findall(response.text)
        if '</td>' not in ips[0] and ':' in ips[0]:
            self.storage(ips)
            return

        ips = (f'{ip.split("<")[0]}:{ip.split(">")[-1]}' for ip in ips)
        self.storage(ips)

    def parse_data5u(self, response):
        matches = DATA5U.findall(response.text)
        if not matches:
            self.logger.error('%s matches nothing', response.url)
            self.q.put((response.url, self.parse_data5u))
            return
        for url in matches:
            self.q.put((url, self.parse_regex))

    # def parse_kuai(self, response):
    #     parser = Parser(response,
    #                     '//*[@id="list"]/table/tbody/tr',
    #                     './td[@data-title="IP"]/text()',
    #                     './td[@data-title="PORT"]/text()'
    #                     )
    #     ips = parser.parse(self.q, self.parse_kuai)
    #     self.storage(ips, response.url)

    # def parse_xici(self, response):
    #     parser = Parser(response,
    #                     '//table[@id="ip_list"]/tr[position()>2]',
    #                     './td[2]/text()',
    #                     './td[3]/text()'
    #                     )

    #     ips = parser.parse(self.q, self.parse_xici)
    #     self.storage(ips, response.url)

    def collect(self, concurrent_num=3):
        while not self.q.empty():
            elements = cut_queue(self.q, concurrent_num)
            reqs = (request(ele[0], is_map=True) for ele in elements)
            resps = grequests.map(reqs, gtimeout=10, exception_handler=eh)
            for resp, element in zip(resps, elements):
                if not resp:
                    self.q.put(element)
                    self.logger.error('recycle %s', element[0])
                    continue
                self.logger.info('fetched %s', resp.url)
                element[1](resp)
            # time.sleep(3)


from Box2D import *
from contextlib import contextmanager
import math
from pyglet.gl import *

class CircleDisplayList(object):
    def __init__(self, vertex_count=64):
        self.display_list = glGenLists(1)
        glNewList(self.display_list, GL_COMPILE)
        for i in xrange(vertex_count):
            angle = 2.0 * math.pi * float(i) / float(vertex_count)
            glVertex2f(math.cos(angle), math.sin(angle))
        glEndList()

    def delete(self):
        if self.display_list:
            glDeleteLists(self.display_list, 1)
            self.display_list = 0

    def draw(self, center=(0.0, 0.0), radius=1.0, mode=GL_LINE_LOOP):
        glPushMatrix()
        glTranslatef(center[0], center[1], 0.0)
        glScalef(radius, radius, 1.0)
        glBegin(mode)
        glCallList(self.display_list)
        glEnd()
        glPopMatrix()

class MyDebugDraw(b2DebugDraw):
    def __init__(self, flags=None):
        super(MyDebugDraw, self).__init__()
        if flags is None:
            flags = (self.e_aabbBit | self.e_centerOfMassBit |
                     self.e_controllerBit | self.e_coreShapeBit |
                     self.e_jointBit | self.e_obbBit | self.e_pairBit |
                     self.e_shapeBit)
        self.SetFlags(flags)
        self.display_list = glGenLists(1)
        self.circle_display_list = CircleDisplayList()
        self.axis_color = b2Color(1.0, 0.0, 1.0)
        self.compiling = False

    def delete(self):
        assert not self.compiling
        if self.circle_display_list is not None:
            self.circle_display_list.delete()
            self.circle_display_list = None
        if self.display_list:
            glDeleteLists(self.display_list, 1)
            self.display_list = 0

    @contextmanager
    def compile(self):
        assert not self.compiling
        self.compiling = True
        glNewList(self.display_list, GL_COMPILE)
        glPushAttrib(GL_ALL_ATTRIB_BITS)
        glPushMatrix()
        yield None
        glPopMatrix()
        glPopAttrib()
        glEndList()
        self.compiling = False

    def draw(self):
        assert not self.compiling
        glCallList(self.display_list)

    def DrawCircle(self, center, radius, color):
        assert self.compiling
        glColor3f(color.r, color.g, color.b)
        self.circle_display_list.draw(center.tuple(), radius, GL_LINE_LOOP)

    def DrawSegment(self, p1, p2, color):
        assert self.compiling
        glColor3f(color.r, color.g, color.b)
        glBegin(GL_LINES)
        glVertex2f(p1.x, p1.y)
        glVertex2f(p2.x, p2.y)
        glEnd()

    def DrawXForm(self, xf):
        assert self.compiling
        glPopMatrix()
        glPushMatrix()
        glTranslatef(xf.position.x, xf.position.y, 0.0)
        glRotatef(xf.R.GetAngle() * 180.0 / math.pi, 0.0, 0.0, 1.0)

    def DrawSolidCircle(self, center, radius, axis, color):
        assert self.compiling
        glColor3f(color.r, color.g, color.b)
        self.circle_display_list.draw(center.tuple(), radius, GL_POLYGON)
        self.DrawSegment(center, center + radius * axis, self.axis_color)

    def DrawPolygon(self, vertices, vertexCount, color):
        assert self.compiling
        glColor3f(color.r, color.g, color.b)
        glBegin(GL_LINE_LOOP)
        for i in xrange(vertexCount):
            glVertex2f(*vertices[i])
        glEnd()

    def DrawSolidPolygon(self, vertices, vertexCount, color):
        assert self.compiling
        glColor3f(color.r, color.g, color.b)
        glBegin(GL_POLYGON)
        for i in xrange(vertexCount):
            glVertex2f(*vertices[i])
        glEnd()


def cropRustConsoleSeedImage(cvImg):
    """Handles uniform cropping of "patch" images for a Seed.
    It crops all non-UI content to reduce workload when Stitching."""

    # Decide crop based on image dimensions
    imgHeight, imgWidth, _ = cvImg.shape # reversed dims...
    topContentStart = 80 # just after top UI
    bottomContentEnd = 947 # just before bottom UI
    artifactingBuffer = 2 # artifacting minimum-buffer-zone; important

    # Set 2K standard here for possible use in 4K
    top = topContentStart + artifactingBuffer
    bottom = bottomContentEnd - artifactingBuffer

    # 2K Standard Resolution
    if (imgWidth == 1920) and (imgHeight == 1080):
        # Contentful Y-Range: (80, 947)
        # Artifact Buffer Zone: 2 pixels
        pass

    # 4K UHD Resolution
    elif (imgWidth, imgHeight) == [3840, 2160]:
        # Contentful Y-Range: (160, 1894)
        # Artifact Buffer Zone: 4 pixels
        top = top * 2; bottom = bottom * 2

    # Guard Clause
    else:
        print(f"Error: image dimensions {cvImg.shape} not supported.")
        exit()

    # Return the max contentful crop of the image to outer scope.
    return cvImg[top:bottom, 0:imgWidth] # contentful height, full width

def crop2KRustConsoleSeedImage(cvImg):
    return cvImg[82:945, 0:1920] # contentful height, full width

def crop4kRustConsoleSeedImage(cvImg):
    return cvImg[164:1890, 0:3840] # contentful height, full width

def prompt():
    # Enter image filename
    filename = input(
f"""\nYou are running the script (crop.py) directly.\n
Provide the filename of a 2K or 4K image to proceed.
(You may also enter nothing to exit this script.)\n
Image: {os.getcwd()}/"""
    )
    if filename == "": # empty
        exitPrompt()
    elif not os.path.isfile(filename):
        print(f"{os.getcwd()}/{filename} does not exist...")
        exitPrompt()

    # Perform the crop, and show the results
    croppedImage = cropRustConsoleSeedImage(cv2.imread(filename))
    #print("\nImage was cropped to:", croppedImage.shape)
    cv2.imshow("Cropped from: {filename}", croppedImage)
    cv2.waitKey(2500) # milliseconds; 0 = wait till key press
    cv2.destroyAllWindows()

    # Enter image savename
    savename = input(
f"""\nYou may enter nothing to skip saving the cropped image.
Otherwise, enter a filename to save the cropped image.\n
Crop: """
    )
    if savename == "": # empty
        exitPrompt()

    # Save the cropped image file
    cv2.imwrite(savename, croppedImage)

def exitPrompt():
    print("\n(crop.py) will now exit.")
    exit()

# If script is run, prompt for image filename w/ relative path
if __name__ == "__main__":
    import cv2
    import os
    prompt()
    exitPrompt()


from quantum import *
n = 15
O1 = Hadamard(4) * Identity(4)
a = 7
L = 2 ** 4

preO2 = [[1 if j==pow(a, i, n) else 0 for i in range(L)] for j in range(L)]
O2 = Identity(4) * operator(preO2)

phi0 = QDit(('|0>', 8))
phi1 = O1(phi0)
phi2 = O2(phi1)
phi3 = QFT(8)(phi2)

obs = observer()
obs.plot(phi3)


#!/usr/bin/env python3
#
# etl_migrate.py:
# Migrate an input database file to an output file
# Based on etl_ipums.py
# Note coding around Pandas memory leak...

import gc
import gzip
import json
import logging
import logging.handlers
import os
import os.path
import psutil
import re
import sqlite3
import subprocess
import math
import sys
import time
import gzip
import sqlite3

from schema import Schema,Table,Variable,Recode,TYPE_VARCHAR,TYPE_INTEGER
from configparser import ConfigParser
from census_spec_scanner import CensusSpec

from migrate import Migrate

import ctools.cspark  as cspark
import ctools.dconfig as dconfig
import ctools.clogging as clogging

PROCESS = psutil.Process(os.getpid())

SETUP_SECTION = "setup"
CONFIGFILE_TYPE = "configfile_type"
CONFIGFILE_TYPE_WANTED = "etl_migrate"
OUTDIR = 'outdir'
RANDOM_RECORDS = 'random_records'
LIMIT = 'limit'
DEBUG_OPTION = 'debug'
SPARK_SECTION= 'spark'
SPARK_OPTION = 'spark'
NUM_EXECUTORS = 'num_executors'
NUM_EXECUTORS_DEFAULT = '2'
WRITE_DELIMITER = "write_delimiter"
SETUP_OPTIONS = set([CONFIGFILE_TYPE,CONFIGFILE_TYPE_WANTED,OUTDIR,RANDOM_RECORDS,LIMIT,WRITE_DELIMITER,DEBUG_OPTION])

RECODES_SECTION = "recodes"
RUN_SECTION = "run"
GLOBALS_SECTION = "globals"
GLOBALS = "globals"
COMPLETED_FLAG = '.completed'
BULK_LOAD_REPORTING_INTERVAL=100000

def fatal(msg):
    """Send msg to the error log and then raise a runtime error"""
    logging.error(msg)
    raise RuntimeError(msg)

def squash_floats(d):
    """Transform floats into ints; used in the output routines"""
    for k in d.keys():
        try:
            if math.isnan(d[k]):
                d[k] = ''
                continue
        except TypeError:
            continue

        try:
            if (type(d[k])==float and float(d[k])==int(d[k])):
                d[k]=int(d[k])
        except ValueError:
            pass

def configFileHasDuplicateKeys(f):
    # return true if any section in the config file has multiple keys
    import configparser
    config = ConfigParser()
    try:
        config.readfp(f)
        return False
    except configparser.DuplicateOptionError as e:
        return True

def configFileSectionOutOfOrder(f,section):
    # return true if any of the labels in the section of the config file are out of order
    section_re = re.compile(r"^\[(\S+)\]")
    label_re   = re.compile(r"^([^# \t]\S*):")
    in_section = ''
    last_label = ''
    this_label = ''
    for line in f:
        m = section_re.search(line)
        if m:
            in_section = m.group(1)
            continue
        m = label_re.search(line)
        if m:
            last_label = this_label
            this_label = m.group(1)
            if last_label and this_label < last_label:
                return (last_label,this_label)
    return None

def infodb(path):
    conn = sqlite3.connect(path)
    c = conn.cursor()
    print("Database: {} Metadata".format(path))
    print("Key: value")
    print("==========")
    for (key,value) in c.execute("select key,value from metadata"):
        print("{}: {}".format(key,value))
    print("Database: {} Data (limit 10)".format(path))
    print("Key: value")
    print("==========")
    for (key,value) in c.execute("select key,value from data limit 10"):
        print("{}: {}".format(key,value))
    exit(0)

def infosas(path):
    s = Schema()
    print("Loading schema from {}".format(path))
    s.load_schema_from_file( dconfig.dopen(path))
    s.dump(path)
    exit(0)

RUN_COMMAND_RE = re.compile(r"\s*((\w+)\s*=)?\s*(\w+)\((.*)\)")
def parse_run_step_command(line):
    m = RUN_COMMAND_RE.search(line.strip())
    if not m:
        fatal("Cannot parse: {}".format(line))
    return m.group(2,3,4)

def process_setup_section(config, sc=None):
    """Process the setup section of the config file. Return the mig variable"""
    setup = config[SETUP_SECTION]

    # Check for configuration file variables that set options

    mig = Migrate( outdir=os.path.expandvars(config[SETUP_SECTION][OUTDIR]),
                   dry_run = args.dry_run ,
                   limit   = args.limit,
                   dumpdb  = args.dumpdb,
                   spark_context = sc)

    # Get the input and output schemas from the input and output options
    db = CensusSpec(name='etl_db')
    for (key,val) in config[SETUP_SECTION].items():
        # inputNN: specify the schema of an input table.
        # For some tables, also specifies the filenames containing the data
        if key.startswith("input"):
            val = os.path.expandvars(val)
            logging.info("{}: {}".format(key,val))
            db.load_schema_from_file( val )

        # outputNN: or formatNN: specify the schema of an output table
        elif key.startswith("output"):
            val = os.path.expandvars(val)
            logging.info("{}: {}".format(key,val))
            db.load_schema_from_file( val )

        # sqlNN: Specify the schema of a table with SQL
        elif key.startswith("sql"):
            db.add_sql_table(val)

        elif key == RANDOM_RECORDS:
            args.random = int(val)
        elif key == LIMIT:
            args.limit = int(val)
        elif key == WRITE_DELIMITER:
            args.write_delimiter = val

        elif key not in SETUP_OPTIONS: # ignore these
            fatal("Unknown option in secton {}: '{}'".format(SETUP_SECTION,key))

    return (mig,db)

def process_recodes_section(config,mig,db):
    if RECODES_SECTION not in config.sections():
        return
    logging.info("Adding recodes")
    recode_re = re.compile("([^(]+)[(](.+)[)]")
    for option in sorted(config.options(RECODES_SECTION)):
        recode_desc = config[RECODES_SECTION][option]
        m = recode_re.search(option)
        if m:
            (option_without_vtype,vtype) = m.group(1,2)
            vtype = vtype.upper()
            logging.info("{}:  {} ({})".format(option_without_vtype,recode_desc,vtype))
            db.add_recode(option_without_vtype,vtype,recode_desc)
        else:
            fatal("Cannot code recode {}:{}".format(name,recode_desc))
    db.compile_recodes()

def process_run_section(config,mig,db):
    # Process the steps
    for option in sorted(config.options(RUN_SECTION)):
        if option==GLOBALS:     # ignore the globals
            continue
        logging.info("processing [{}] {}".format(RUN_SECTION,option))
        if not option.startswith("step"):
            logging.info("Ignoring option '{}'".format(option))
            continue
        lines = config[RUN_SECTION][option].split("\n")

        (dest,func_name,func_args) = parse_run_step_command(lines[0])
        logging.info("{} dest:{} func_name:{}  func_args:{}".format(option,dest,func_name,func_args))
        if func_args:
            try:
                kvargs = dict([kv.strip().split("=") for kv in func_args.split(",")])
            except ValueError as e:
                raise RuntimeError("Parsing error: option:{} argument:{} missing '=' for key=value assignment".format(
                    option,func_args))
        else:
            kvargs = {}
        # If any of the kvargs are quoted strings, unquote them
        for (key,value) in kvargs.items():
            while (value[0]==value[-1]) and (value[0] in "\"'"):
                value = value[1:-1]
                kvargs[key] = value

        if mig.command(db=db, cmd=func_name, dest=dest, kvargs=kvargs, lines=lines):
            continue
        if func_name=='DUMP':
            db.dump(logging.info)
        elif func_name=="COLLECT_ACCUM":
            mig.compile_and_exec("euid_counts = euid_counts.value")
        else:
            fatal("Unknown command in config file option '{}': func_name={} func_args={} dest={} ".
                               format(option,func_name,func_args,dest))

if __name__=="__main__":
    from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
    parser = ArgumentParser( formatter_class = ArgumentDefaultsHelpFormatter,
                             description="Migrate an input dataset to an output dataset" )
    parser.add_argument('--debug', action='store_true')
    parser.add_argument('--dump',  action='store_true', help='Dump schema and internal functions; do not run the transfer')
    parser.add_argument('--dry-run', action='store_true', help='Print what would be done only')
    parser.add_argument('--schema', action='store_true', help='display the SQL schema')
    parser.add_argument('--limit', help='Only copy this many records from each file (for testing)', type=int)
    parser.add_argument('--random', action='store_true', help='generate random data')
    parser.add_argument('--loglevel', help='Set logging level',
                        choices=['CRITICAL','ERROR','WARNING','INFO','DEBUG'],
                        default='INFO')
    parser.add_argument('--write_delimiter', help='delimiter for output csv files',default='|')
    parser.add_argument("--justglobals", help="Just run the globals setup, dump what's in the environment, and then stop",
                        action='store_true')
    parser.add_argument('--infosas',help='Print information about SAS7BDAT file and exit; config is ignored')
    parser.add_argument('--loaddb', action='store_true', help='Only load join databases')
    parser.add_argument('--infodb', help='Print information about the SQlite3 database file and exit; config is ignored')
    parser.add_argument('--dumpdb', help='If specified, dump this many records of the SQLite3 database',type=int)
    parser.add_argument("--detach", action="store_true", help="disconnection from console and write output to logfiles")
    parser.add_argument("--spark",  action="store_true", help="Run under spark")
    parser.add_argument('config', help='config file')

    args   = parser.parse_args()

    # Check the args
    if args.random and not args.limit:
        parser.error("--limit must be specified with --random")

    # See if simple infos are specified
    if args.infodb:  infodb(args.infodb); exit(0)
    if args.infosas: infosas(args.infosas); exit(0)

    ###
    ### VALIDATE AND THEN READ CONFIG FILE
    ###

    if configFileHasDuplicateKeys(open(args.config)):
        fatal("{} has duplicate keys".format(args.config))
    os.environ["CONFIGDIR"] = os.path.split(os.path.abspath(args.config))[0]
    config = dconfig.get_config(pathname=args.config)
    if CONFIGFILE_TYPE not in config[SETUP_SECTION]:
        fatal("{} not in config file {}".format(CONFIGFILE_TYPE,args.config))

    if config[SETUP_SECTION][CONFIGFILE_TYPE] != CONFIGFILE_TYPE_WANTED:
        fatal("{} is not {} in config file {}".
                           format(CONFIGFILE_TYPE,CONFIGFILE_TYPE_WANTED,args.config))
    order = configFileSectionOutOfOrder(open(args.config),"run")
    if order:
        fatal("[run] section has steps that are out of order: {} comes before {}".format(order[0],order[1]))

    if config[SETUP_SECTION].get("DEBUG","").lower()[0:1] in ["1","t","y"]:
        args.debug = True

    if SPARK_SECTION in config and config[SPARK_SECTION].get("SPARK","").lower()[0:1] in ["1","t","y"]:
        args.spark = True

    # Detach if requested
    if args.detach:
        print("Detaching...")
        cspark.detach()

    ###
    ### ENABLE SPARK IF REQUESTED
    ###

    sc = None
    if args.spark:
        num_executors = config[SPARK_SECTION].get(NUM_EXECUTORS, NUM_EXECUTORS_DEFAULT)
        sc = cspark.spark_context(num_executors=int(num_executors),
                                  pydirs=[os.path.dirname(__file__),
                                          os.path.join(os.path.dirname(__file__),'ctools'),
                                          os.path.join(os.path.dirname(__file__),'dfxml/python')])

    ###
    ### ENABLE LOGGING.
    ### Logging must be set up before any logging is done
    ### Note that we log to both a file and to syslog facility 1
    ###

    clogging.setup(args.loglevel)
    logging.info("START {} ".format(os.path.abspath(__file__)))

    (mig,db) = process_setup_section(config, sc)
    process_recodes_section(config,mig,db)

    # Dump or print schema, as necessary
    if args.dump:       db.dump()
    if args.schema:     print(db.sql_schema())
    #if args.spark:      mig.add_spark_to_environment()

    # run steps are implemented as compiled code.
    mig.add_tables_to_environment(db)

    # Add globals to environment if we have any
    if GLOBALS in config[RUN_SECTION]:
        fatal("globals must now be  [globals] section")

    if GLOBALS_SECTION in config:
        logging.info("Adding GLOBALs")
        for option in config[GLOBALS_SECTION]:
            logging.info("Adding [{}]{}".format(GLOBALS_SECTION,option))
            lines = config[GLOBALS_SECTION][option]
            mig.compile_and_exec(lines)
    else:
        logging.info("No [{}] section".format(GLOBALS_SECTION))

    #mig.undo_spark()

    if args.justglobals:
        mig.dump_context()
    else:
        process_run_section(config, mig, db)

    logging.info("DONE")
    exit(0)


import os
import subprocess

def clip(input_path, clip_path, output_path):
    """Clip a feature dataset using ogr2ogr

    Parameters
    ----------
    input_path : str
    clip_path : str
    output_path : str

    """
    if os.name == 'posix':
        shell_flag = False
    else:
        shell_flag = True

    # subprocess.run(
    subprocess.check_output(
        ['ogr2ogr', '-f', 'ESRI Shapefile', '-overwrite',
            '-preserve_fid', '-unsetFieldWidth',
            '-clipsrc', clip_path, output_path, input_path],
        shell=shell_flag)

def copy(input_path, output_path):
    """Copy a feature or raster dataset

    Parameters
    ----------
    input_path : str
    output_path : str

    """
    if os.name == 'posix':
        shell_flag = False
    else:
        shell_flag = True

    # if not is_shapefile(input_path):
    #     raise Exception('input must be a .shp type')

    # DEADBEEF - Extra command(s) remove the field type warning but make
    # the DBF quite a bit larger since all the field types are doubles.
    # subprocess.run(
    subprocess.check_output(
        ['ogr2ogr', '-f', 'ESRI Shapefile', '-overwrite', '-preserve_fid',
            '-unsetFieldWidth', output_path, input_path],
        shell=shell_flag)

    # # DEADBEEF - Only runs on Python 3 but suppresses field width warning
    # subprocess.run(
    #     ['ogr2ogr', '-f', 'ESRI Shapefile', output_path, input_path],
    #     shell=shell_flag, stderr=subprocess.DEVNULL)

def delete(input_path):
    """Delete a raster dataset

    Parameters
    ----------
    input_path : str

    """
    if os.name == 'posix':
        shell_flag = False
    else:
        shell_flag = True

    # TODO: Add a format type function to gdal_common
    if input_path.upper().endswith('.IMG'):
        input_format = 'HFA'
        input_type = 'raster'
    elif input_path.upper().endswith('.TIF'):
        input_format = 'GTiff'
        input_type = 'raster'
    elif input_path.upper().endswith('.TIFF'):
        input_format = 'GTiff'
        input_type = 'raster'

    if input_type == 'raster':
        # subprocess.run(
        subprocess.check_output(
            ['gdalmanage', 'delete', '-f', input_format, input_path],
            shell=shell_flag)
    # elif input_type == 'vector':
    else:
        raise ValueError('unsupported format for delete')

def project(input_path, output_path, output_wkt):
    """Project a feature dataset to a new projection

    Parameters
    ----------
    input_path : str
    output_path : str
    output_wkt : str

    """
    if os.name == 'posix':
        shell_flag = False
    else:
        shell_flag = True

    # subprocess.run(
    subprocess.check_output(
        ['ogr2ogr', '-f', 'ESRI Shapefile', '-overwrite',
            '-preserve_fid', '-unsetFieldWidth', '-t_srs', output_wkt,
            output_path, input_path],
        shell=shell_flag)


#------------------------------------------------------------#
# This is a script to perform evaluations                    #
#                                                            #
# Author: Saad Hossain (s42hossa@uwaterloo.ca)               #
#------------------------------------------------------------#

import os
import yaml
import pandas as pd
from datetime import datetime
from prettytable import PrettyTable

from src.utils.starter import CircleParams, iou
from src.utils.vis import plot_and_save_circles

THRESHOLDS = [0.5, 0.75, 0.9, 0.95]
RADIUS_RANGES = [(5, 10), (10, 25), (25, 50), (50, 100)]

def calculate_accuracy(
        df: pd.DataFrame, threshold: float
    ) -> float:
    """
    Calculates accuracy given a threshold and dataframe of predicted
    circle parameters.

    Args:
        df: dataframe with ground truth and predicted circle parameters
        threshold: the IoU threshold upon which to inspect detections
    Returns:
        accuracy: the accuracy based upon the  defined threshold
    """

    ious = df.apply(
        lambda x: iou(CircleParams(row=x['row'],
                                   col=x['col'],
                                   radius=x['radius']),
                      CircleParams(row=x['predicted_row'],
                                   col=x['predicted_col'],
                                   radius=x['predicted_radius'])),
                      axis=1)

    return (ious >= threshold).mean()

def evaluate_accuracy(df: pd.DataFrame) -> None:
    """
    Calculates accuracies by IoU threshold and report results for different
    sizes of radii to understand model performance.

    Args:
        df: dataframe with ground truth and predicted circle parameters
    Returns:
        does not return anything, prints out results of evaluation
    """

    thresholds = THRESHOLDS
    radius_ranges = RADIUS_RANGES  + [('All', None)]

    table = PrettyTable()
    table.field_names = ["Radius Range/IoU"] + [
        f"IoU ≥ {thres}" for thres in thresholds]

    for min_radius, max_radius in radius_ranges:
        radius_range = f"{min_radius}-{max_radius}" \
            if max_radius else 'All'
        row = [radius_range]

        for threshold in thresholds:
            radius_df = df[
                (df['radius'] >= min_radius)
                & (df['radius'] < max_radius)] if max_radius else df

            if len(radius_df) > 0:
                accuracy = calculate_accuracy(radius_df, threshold)
                row.append(f"{accuracy * 100:.2f}%")
            else:
                row.append(f"N/A")

        table.add_row(row)

    print(table)

    return

if __name__ == "__main__":
    cfg = yaml.full_load(
        open(os.path.join(os.getcwd(), 'config.yml'), 'r'))
    df = pd.read_csv(cfg['EVAL']['PREDICTIONS_CSV'])

    # Evaluate and final model fail/success cases
    evaluate_accuracy(df)

    # Visualize model performance and explainability
    plot_and_save_circles(df=df,
                          directory=os.path.join(
                              cfg['EVAL']['VIS_DIR'],
                              datetime.now().strftime("%Y%m%d_%H%M%S")),
                          radius_ranges=RADIUS_RANGES,
                          model_path=cfg['EVAL']['CHECKPOINT'],
                          backbone=cfg['EVAL']['BACKBONE'],
                          class_idx=cfg['EVAL']['SALIENCY_FOCUS'],
                          dimensions=cfg['EVAL']['INPUT_DIMS'])


import oss2
import posixpath
import os
from django.core.files.storage import Storage
from django.conf import settings
from urllib.parse import urljoin
from django.utils.encoding import force_str, force_bytes
from django.core.files import File
from django.utils.module_loading import import_string
from io import BytesIO
from pypinyin import lazy_pinyin, Style

"""
    阿里云OSS集成
    1. settings.py内新增配置
        1>. Storage 配置
        DEFAULT_FILE_STORAGE = 'utils.AliYunOSS2Storage'
        STATICFILES_STORAGE = 'utils.AliYunOSS2Storage'
        2>. OSS Access key 配置
        OSS2_ACCESS_KEY_ID = "oss2_access_key_id"
        OSS2_ACCESS_KEY_SECRET = "oss2_access_key_secret"
        OSS2_END_POINT = "example.aliyuncs.com"
        OSS2_BUCKET_NAME = "oss2_bucket_name"
    2. Django Model Field写法

        from utils.aliyun_oss import get_image_path

        class DemoModel(models.Model):
            ...
            avatar = models.ImageField(u"用户头像", upload_to=get_image_path, null=True, blank=True)
            ...

        * FileField类型的字段的 upload_to 参数配置为 get_image_path方法
"""

class AliYunOSS2Storage(Storage):

    def __init__(self):
        self.location = settings.MEDIA_URL
        self.access_key_id = settings.OSS2_ACCESS_KEY_ID
        self.access_key_secret = settings.OSS2_ACCESS_KEY_SECRET
        self.end_point = settings.OSS2_END_POINT
        self.bucket_name = settings.OSS2_BUCKET_NAME

        self.auth = oss2.Auth(self.access_key_id, self.access_key_secret)
        self.service = oss2.Service(self.auth, self.end_point, connect_timeout=30)
        self.bucket = oss2.Bucket(self.auth, self.end_point, self.bucket_name)

        self.FileMode = AliYunOSS2File

    @staticmethod
    def _clean_name(name):
        clean_name = posixpath.normpath(name).replace('\\', '/')

        if name.endswith('/') and not clean_name.endswith('/'):
            return clean_name + '/'
        else:
            return clean_name

    def _normalize_name(self, name):

        base_path = force_str(self.location)
        base_path = base_path.rstrip('/')

        final_path = urljoin(base_path.rstrip('/') + "/", name)

        base_path_len = len(base_path)
        if not final_path.startswith(base_path) or final_path[base_path_len:base_path_len + 1] not in ('', '/'):
            raise Exception("Attempted access to '%s' denied." % name)
        return final_path.lstrip('/')

    def _get_target_name(self, name):
        return self._normalize_name(self._clean_name(name))

    def _open(self, name, mode='rb'):
        return self.FileMode(name)

    def save(self, name, content, max_length=None):
        return self._save(name, content)

    def _save(self, name, content):
        file_name = self._get_target_name(name)
        content.open()
        content_str = b''.join(chunk for chunk in content.chunks())
        self.bucket.put_object(file_name, content_str)
        content.close()
        return file_name

    def path(self, name):
        return self._get_target_name(name)

    def delete(self, name):
        name = self._get_target_name(name)
        self.bucket.delete_object(name)

    def exists(self, name):
        return self.bucket.object_exists(name)

    def listdir(self, path):
        name = self._get_target_name(path)
        if name and name.endswith('/'):
            name = name[:-1]

        files = []
        dirs = set()
        for obj in oss2.ObjectIterator(self.bucket, prefix=name, delimiter='/'):
            if obj.is_prefix():
                dirs.add(obj.key)
            else:
                files.append(obj.key)
        return list(dirs), files

    def size(self, name):
        obj = self.bucket.get_object(name)
        raise obj.size()

    def url(self, name):
        url = getattr(self.bucket, 'sign_url')('GET', name, 24*3600)
        return url

    def get_accessed_time(self, name):
        pass

    def get_created_time(self, name):
        pass

    def get_modified_time(self, name):
        pass

class AliYunOSS2File(File):

    def __init__(self, name):
        self._storage = AliYunOSS2Storage()
        self._name = name[len(self._storage.location):]
        self.file = BytesIO()
        self._reading = False
        self._dirty = False
        super(AliYunOSS2File, self).__init__(self.file, self._name)

    def read(self):
        if not self._reading:
            self.file = BytesIO(self._storage.bucket.get_object(self._name).read())
            self._reading = True
        return self.file.read()

    def write(self, content):
        self.file.write(force_bytes(content))
        self._dirty = True
        self._reading = True

    def close(self):
        if self._dirty:
            self.file.seek(0)
            self._storage.save(self._name, self.file)
        self.file.close()

def get_image_path(instance, filename):
    """
    获取图片的保存位置
    暂定以company.cid为单位
    """
    module_name = instance.__class__.__name__.lower()

    suffix = filename.split(".")[-1]

    if module_name == 'company':
        return f'{instance.domain_name}/{filename}'
    elif module_name == 'application':
        return f'{instance.company.domain_name}/app_logo/{instance.uuid}.{suffix}'
    elif module_name == 'user':
        return f'user_avatar/{instance.uuid}.{suffix}'
    elif module_name == 'app_store':
        return f'software_icon/{instance.name}.{suffix}'

def image_storage(field, file, object_id, company_id, call_path):
    """
    将图片保存到数据库对应的字段
    :param field: 待更新字段名
    :param file: 文件
    :param object_id: 待更新的数据ID
    :param company_id: 调用者所属企业ID
    :param call_path: 调用来源，一般传入__file__
    :return: dict
    """
    module_name = os.path.basename(os.path.dirname(call_path))
    module = import_string(f'apps.{module_name}.models.{module_name.capitalize()}')
    if module_name == "user":
        instance = module.objects.filter(id=object_id).first()
    else:
        instance = module.objects.filter(id=object_id, company_id=company_id).first()

    if not instance:
        raise ModuleNotFoundError()

    if hasattr(instance, field):
        setattr(instance, field, file)
        instance.save()  # TODO: 过期图片清理方式待定
        return {'img_url': getattr(instance, field).url}
    else:
        raise AttributeError()

def get_letters(text):
    """
        获取文本拼音首字母组成的字符串
        Args:
            text: 文本
        Returns:
            str: 拼音首字母组成的字符串
    """
    letters = ''
    letters_list = lazy_pinyin(text, style=Style.FIRST_LETTER)
    for letter in letters_list:
        letters += letter
    return letters.upper()


class Shape:
    store_fg = {
        'плоскость': ('круг', 'квадрат', 'прямоугольник', 'треугольник', 'трапеция', 'ромб'),
        'объём': ('сфера', 'куб', 'параллелепипед', 'пирамида', 'цилиндр', 'конус')
    }

    def __init__(self):

#        self.show_store()
        pass

    @staticmethod
    def show_store():

#        for el in Shape.store_fg:
#            print(el, Shape.store_fg[el])
        return Shape.store_fg

    @staticmethod
    def random_figure():
        import random
        xs = list(range(random.randint(1, 100)))
        ys = [random.randint(1, 50) for x in xs]
        # xs = [2, 4, 3, 7]
        # ys = [3, 7, 9, 9]
        xs.append(xs[0])
        ys.append(ys[0])
        return xs, ys

    @staticmethod
    def create_pfigure(xs, ys):
        from matplotlib.figure import Figure
        fig = Figure()
        axis = fig.add_subplot(1, 1, 1)
        axis.plot(xs, ys)
        # установить названия осей
        axis.set_xlabel('X')
        axis.set_ylabel('Y')
        return fig

    @staticmethod
    def create_vfigure():
        from matplotlib.figure import Figure
        fig = Figure()
        axis = fig.add_subplot(111, projection='3d')
        # установить названия осей
        axis.set_xlabel('X')
        axis.set_ylabel('Y')
        axis.set_zlabel('Z')
        return fig, axis;


# !/usr/bin/env python3
# -*- coding:utf-8 -*-
###
# File: e:\code\python\spider\.vscode\music_spider\main.py
# Project: e:\code\python\spider\.vscode\music_spider
# Created Date: Saturday, September 12th 2020, 10:10:32 pm
# Author: rz
# -----
# Last Modified:
# Modified By:
# -----
# Copyright (c) 2020 rz WTI bupt
#
# All shall be well and all shall be well and all manner of things shall be well.
# Nope...we're doomed!
# -----
# HISTORY:
# Date      	By	Comments
# ----------	---	----------------------------------------------------------
###

import requests
import json
import os
import random
import time

def download(url, name):
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36'}  # 设置头文件，简单的爬虫伪装
    path = 'E:/娱乐/music/' + name.replace('/', '').replace('\\', '').replace(':', '').replace(
        '*', '').replace('"', '').replace('<', '').replace('|', '').replace('?', '').replace(',', '')  # 设置路径和文件名，并去除文件名中不允许的字符

    if not os.path.exists(path) or os.path.getsize(path) == 0:  # 文件不存在或大小为0时开始下载
        # 请求时需关闭verify属性，否则会报SSL认证错误
        r = requests.get(url, headers=header, verify=False)
        # r.encoding = r.apparent_encoding
        with open(path, 'wb') as f:
            f.write(r.content)
            f.close()
            print(name + '保存成功')
        # time.sleep(random.random()) # 下载完成后随机休眠，对服务器也好
    else:
        print("文件已存在")

def parse(lists):
    for lis in lists:
        name = lis['name']

        if 'lrc' in lis:
            download(lis['lrc'], name+'.lrc')
        if 'url_flac' in lis:
            download(lis['url_flac'], name+'.flac')
        elif 'url_320' in lis:
            download(lis['url_320'], name + '.mp3')
        elif 'url_128' in lis:
            download(lis['url_128'], name + '.mp3')
        elif 'url' in lis:
            download(lis['url'], name + '.mp3')
        else:
            print('暂无资源')

def main():
    if not os.path.exists('E:/娱乐/music'):  # 创建文件夹
        os.mkdir('E:/娱乐/music')
    with open('music_list.json', 'r', encoding='utf-8') as raw:
        music_lists = json.load(raw)['data']['list']
        parse(music_lists)

if __name__ == '__main__':
    main()


# Use sentence transformers to get embeddings for the utterances.
# Train a linear model on the embeddings to predict the intent.

import json
import numpy as np
import torch
from sentence_transformers import SentenceTransformer

# Load data
with open('data_final.json', 'r') as data_file:
    data = json.load(data_file)

transformer = SentenceTransformer('llmrails/ember-v1')

text_embeddings = transformer.encode(data["utt"])

from sklearn.preprocessing import LabelEncoder

encoder = LabelEncoder()
encoder.fit(data["intent"])

labels = encoder.transform(data["intent"])

model = torch.nn.Sequential(
    torch.nn.GELU(),
    torch.nn.Dropout(0.1),
    torch.nn.Linear(768, 768),
    torch.nn.GELU(),
    torch.nn.Linear(768, 150)
)

loss_fn = torch.nn.CrossEntropyLoss()

optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)

dataset = torch.utils.data.TensorDataset(torch.tensor(text_embeddings), torch.tensor(labels))
dataloader = torch.utils.data.DataLoader(dataset, batch_size=64, shuffle=True)

for epoch in range(100):
    for batch in dataloader:
        optimizer.zero_grad()
        pred = model(batch[0])
        loss = loss_fn(pred, batch[1])
        loss.backward()
        optimizer.step()
    print(epoch, loss.item())

# Open massive_test.data and get the utterances
with open('massive_test.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

utt = [item['utt'] for item in data]

# Get embeddings for the utterances
text_embeddings = transformer.encode(utt)

# Predict the intent for the utterances
pred = model(torch.tensor(text_embeddings))

# Get the predicted intent
pred = torch.argmax(pred, dim=1)

# Get the intent from the encoder
intent = encoder.inverse_transform(pred)

# Write to file
with open('massive_test.solution', 'w') as outfile:
    for i in range(1,len(intent)+1):
        outfile.write(json.dumps({'indoml_id': data[i]['indoml_id']
                                  , 'intent': intent[i]}) + '\n')
        print(utt[i])
        print(intent[i])


import time

def get_integer():
    return int(input("enter a number: "))

start = time.time_ns()
num = get_integer()

for i in range(2, round(num / 2) + 1):
    if num % i == 0:
        print("not prime")
        # isPrime = False
        print("total time is ", (time.time_ns() - start) / 1000000)
        exit()

# print("Prime = " isPrime)
print("prime")

print("total time is ", (time.time_ns() - start) / 1000000)


# 백준 16236 아기상어
from collections import deque
import sys
input = sys.stdin.readline

N = int(input())
field = []
dr = [-1, 0, 0, 1]
dc = [0, -1, 1, 0]
fish_cnt = 0
answer = 0
level = 2
result = 1
cnt = 0

for i in range(N):
    s = list(map(int, input().split()))
    for j in range(N):
        if s[j] == 9:
            pos = [i, j]
        elif 0 < s[j] <= 6:
            fish_cnt += 1
    field.append(list(map(int, s)))

def bfs(pos, level):
    visited = [[0]*(N) for _ in range(N)]
    q = deque([])
    q.append(pos)
    fishes = []
    min_dist = 10**6
    while q:
        r, c = q.popleft()
        for i in range(4):
            R = r+dr[i]
            C = c+dc[i]
            if 0 <= R < N and 0 <= C < N and not visited[R][C]:
                visited[R][C] = visited[r][c]+1
                if 0 < field[R][C] < level:
                    min_dist = visited[R][C]
                    fishes.append([visited[R][C], R, C])
                elif field[R][C] == level and min_dist >= visited[R][C]:
                    q.append([R, C])
                elif field[R][C] == 0 and min_dist >= visited[R][C]:
                    q.append([R, C])
    if fishes:
        fishes.sort()
        return fishes[0]
    else:
        return False

while fish_cnt:
    result = bfs(pos, level)
    if result:
        cnt += 1
        pos_r, pos_c = pos
        field[pos_r][pos_c] = 0
        dist, result_r, result_c = result
        answer += dist
        field[result_r][result_c] = 999
        pos = [result_r, result_c]
        fish_cnt -= 1
        if cnt == level:
            cnt = 0
            level += 1
    else:
        break

print(answer)


import csv

def write_predictions_to_csv(predicted_data, test_data):
    # print predicted_data
    # print test_data.shape

    with open('Resources/csv/testSet_categories.csv', mode='w', encoding="utf8", newline='') as csv_file:
        csv_writer = csv.writer(csv_file, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL)
        csv_writer.writerow(['ID'] + ['Predicted_Category'])  # Write headers.
        for x in range(len(test_data)):
            csv_writer.writerow([test_data['Id'][x]] + [predicted_data[x]])  # Write id-category pairs.


from django.db import models
from users.models import Account
from uuid import uuid4

class Reaction(models.Model):
    id = models.UUIDField(primary_key=True, default=uuid4)
    owner = models.ForeignKey(
        Account,
        on_delete=models.PROTECT,
        verbose_name="Кто поставил",
        related_name="reactions",
    )
    reaction_option = models.ForeignKey(
        "ReactionItem",
        on_delete=models.CASCADE,
        verbose_name="Варианты реакций из определенного списка",
    )

    class Meta:
        verbose_name = "Реакция пользователя на пост/сообщение"
        verbose_name_plural = "Реакции пользователя на пост/сообщение"

class ReactionItem(models.Model):
    id = models.UUIDField(primary_key=True, default=uuid4)
    image = models.ImageField(upload_to="uploads/reaction_images/")

    class Meta:
        verbose_name = "Изображение реакции"
        verbose_name_plural = "Изображения реакций"


import ast
import boto3
import botocore
import datetime
import time

from botocore.exceptions import ClientError

codecommit_client = boto3.client('codecommit')
codepipeline_client = boto3.client('codepipeline')
codebuild_client = boto3.client('codebuild')

# Repository name from codecommit
repository_name = 'Android'

# AWS account details
account_number = ''
role = 'codepipeline-android-service-role'
region = 'ap-south-1'

# S3 bucket to store Source and Build Artifacts
s3_android_bucket_development = 'android-source'
s3_android_bucket_builds = 'android-builds'
s3_android_bucket_release = 'android-release'

# DeviceFarm Project and DevicePool arn to test the application
device_farm_project_id = ''
device_farm_device_pool_arn = ''

pipeline_configuration = {
    "pipeline": {
        "stages": [
            {
                "actions": [
                    {
                        "runOrder": 1,
                        "actionTypeId": {
                            "category": "Source",
                            "provider": "CodeCommit",
                            "version": "1",
                            "owner": "AWS"
                        },
                        "name": "Source",
                        "outputArtifacts": [
                            {
                                "name": "SourceArtifact"
                            }
                        ],
                        "configuration": {
                            "BranchName": "",
                            "OutputArtifactFormat": "CODE_ZIP",
                            "PollForSourceChanges": "false",
                            "RepositoryName": repository_name
                        },
                        "inputArtifacts": []
                    }
                ],
                "name": "Source"
            },
            {
                "actions": [
                    {
                        "runOrder": 1,
                        "actionTypeId": {
                            "category": "Build",
                            "provider": "CodeBuild",
                            "version": "1",
                            "owner": "AWS"
                        },
                        "name": "CodeBuild",
                        "outputArtifacts": [
                            {
                                "name": "BuildArtifact"
                            }
                        ],
                        "configuration": {
                            "BatchEnabled": "false",
                            "ProjectName": ""
                        },
                        "inputArtifacts": [
                            {
                                "name": "SourceArtifact"
                            }
                        ]
                    }
                ],
                "name": "Build"
            },
            {
                "actions": [
                    {
                        "runOrder": 1,
                        "actionTypeId": {
                            "category": "Deploy",
                            "provider": "S3",
                            "version": "1",
                            "owner": "AWS"
                        },
                        "name": "Deploy",
                        "configuration": {
                            "BucketName": s3_android_bucket_builds,
                            "ObjectKey": "MyWebsite",
                            "Extract": "true"
                            },
                        "outputArtifacts": [],
                        "region": region,
                        "inputArtifacts": [
                            {
                                "name": "BuildArtifact"
                            }
                        ]
                    }
                ],
                "name": "Deploy"
            },
            {
                "name": "Test",
                "actions": [
                    {
                        "name": "TestDeviceFarm",
                        "actionTypeId": {
                            "category": "Test",
                            "owner": "AWS",
                            "provider": "DeviceFarm",
                            "version": "1"
                        },
                        "runOrder": 1,
                        "configuration": {
                            "App": "app-debug.apk",
                            "AppType": "Android",
                            "DevicePoolArn": device_farm_device_pool_arn,
                            "FuzzEventCount": "6000",
                            "FuzzEventThrottle": "50",
                            "ProjectId": device_farm_project_id,
                            "TestType": "BUILTIN_FUZZ"
                        },
                        "outputArtifacts": [],
                        "inputArtifacts": [
                            {
                                "name": "BuildArtifact"
                            }
                        ],
                        "region": region
                    }
                ]
            },
        ],
        "artifactStore": {
            "type": "S3",
            "location": s3_android_bucket_development
        },
        "name": "ci-pipeline",
        "version": 1,
        "roleArn": 'arn:aws:iam::'+account_number+':role/service-role/'+role
    }
}

def get_status(pipeline_name):
    pipeline_status = codepipeline_client.get_pipeline_state(name=pipeline_name)
    return pipeline_status

def post_comment(pr_id, repository_name, source_commit,
                 destination_commit, content):
    codecommit_client.post_comment_for_pull_request(
            pullRequestId=pr_id,
            repositoryName=repository_name,
            beforeCommitId=source_commit,
            afterCommitId=destination_commit,
            content=content
        )

def delete_pipeline(pipeline_name):
    try:
        print("Cleaning up, deleting pipeline %s" % pipeline_name)
        codepipeline_client.delete_pipeline(name=pipeline_name)
    except ClientError as e:
        if e.response['Error']['Code'] == 'PipelineNotFoundException':
            print("Pipeline %s does not exist, nothing to do." % pipeline_name)
        else:
            print("Unexpected error: %s" % e)

def update_pipeline(code_pipeline_configuration, branch_ref, destination_branch=None):
    for stage in code_pipeline_configuration['pipeline']['stages']:
        if stage['name'] == 'Source':
            stage['actions'][0]['configuration']['BranchName'] = branch_ref
        elif stage['name'] == 'Build':
            if destination_branch == 'master':
                stage['actions'][0]['configuration']['ProjectName'] = 'android-master-apk-build'
            else:
                stage['actions'][0]['configuration']['ProjectName'] = 'android-develop-apk-build'

    return code_pipeline_configuration['pipeline']

def create_pipeline(modified_pipeline_json):
    new_pipeline_response = codepipeline_client.create_pipeline(pipeline=modified_pipeline_json)
    return new_pipeline_response

def get_pipeline_artifact(build_id):
    response = codebuild_client.batch_get_builds(ids=[build_id])
    pipeline_artifact = '/'.join(response['builds'][0]['artifacts']['location'].split('/')[1:])
    return pipeline_artifact

# Beanch Events Ex: referenceCreated, referenceDeleted, referenceUpdated
def branch_events(message, event_ytpe):
    print(message)
    #reference_type = message['detail']['referenceType'] == 'branch' or 'tag'
    commit_id = message['detail']['commitId']
    print("Commit ID :: %s" % commit_id)
    #branch_ref = '/'.join(message['detail']['referenceFullName'].split('/')[2:])
    branch_ref = message['detail']['referenceName']
    branch_name = '-'.join(message['detail']['referenceName'].split('/'))
    print("Branch Name :: %s" % branch_ref)

    code_pipeline_configuration = pipeline_configuration
    pipeline_name = branch_name + '-' + commit_id + '-pipeline'
    code_pipeline_configuration['pipeline']['name'] = pipeline_name
    code_pipeline_configuration['pipeline']['stages'][2]['actions'][0]['configuration']['ObjectKey'] = commit_id

    if message['detail']['referenceType'] == 'tag':
        #code_pipeline_configuration['pipeline']['artifactStore']['location'] = s3_android_bucket_release
        code_pipeline_configuration['pipeline']['stages'][2]['actions'][0]['configuration']['BucketName'] = s3_android_bucket_release
        # Use master branch for tags
        branch_ref = 'master'
        s3_android_bucket_builds = s3_android_bucket_release

    modified_pipeline_json = update_pipeline(code_pipeline_configuration, branch_ref)
    print(modified_pipeline_json)
    delete_pipeline(pipeline_name)

    time.sleep(5)

    new_pipeline_response = create_pipeline(modified_pipeline_json)

    time.sleep(10)

    pipeline_name = new_pipeline_response['pipeline']['name']
    pipeline_stages = get_status(pipeline_name)

    for stage in pipeline_stages['stageStates']:

        if stage['stageName'] == 'Source':
            source_execution_status = stage['actionStates'][0]['latestExecution']['status']

            if source_execution_status == 'Succeeded':
                print('Stage Source Succeeded.')
            else:
                print('Stage Source Failed.')
                break

        elif stage['stageName'] == 'Build':
            build_url = stage['actionStates'][0]['latestExecution']['externalExecutionUrl']
            count = 0
            while count < 2000:
                pipeline_build_status = get_status(pipeline_name)
                build_execution_status = pipeline_build_status['stageStates'][1]['latestExecution']['status']
                #print('Build status :: ', build_execution_status)

                if build_execution_status == 'Succeeded':
                    print('Stage Build Succeeded.')
                    print(build_url)
                    #delete_pipeline(pipeline_name)
                    break

                elif build_execution_status == 'Failed':
                    print(pipeline_build_status['stageStates'][1]['actionStates'][0]['latestExecution']['errorDetails']['message'])
                    print(build_url)
                    #delete_pipeline(pipeline_name)
                    break

                else:
                    count += 1
                    time.sleep(2)

        elif stage['stageName'] == 'Deploy':
            count = 0
            while count < 500:
                #try:
                pipeline_deploy_status = get_status(pipeline_name)
                deploy_execution_status = pipeline_deploy_status['stageStates'][2]['latestExecution']['status']
                if deploy_execution_status == 'Succeeded':
                    print('Stage '+pipeline_deploy_status['stageStates'][2]['actionStates'][0]['latestExecution']['summary'])
                    build_path = pipeline_deploy_status['stageStates'][2]['actionStates'][0]['latestExecution']['externalExecutionId']
                    print(build_path)
                    apk_s3_location = 'https://'+region+'.console.aws.amazon.com/s3/buckets/android-builds?region='+region+'&prefix='+commit_id+'/&showversions=false'
                    print('Download APK file from :: {}'.format(apk_s3_location))
                    break
                elif deploy_execution_status == 'Failed':
                    print('Stage Deploy failed')
                    break
                else:
                    count += 1
                    time.sleep(2)

        elif stage['stageName'] == 'Test':
            count = 0
            while count < 500:
                pipeline_test_status = get_status(pipeline_name)
                test_execution_status = pipeline_test_status['stageStates'][3]['latestExecution']['status']
                if test_execution_status == 'Succeeded':
                    print('Stage '+pipeline_test_status['stageStates'][3]['actionStates'][0]['latestExecution']['summary'])
                    devicefarm_url = pipeline_test_status['stageStates'][3]['actionStates'][0]['latestExecution']['externalExecutionUrl']
                    print(devicefarm_url)
                    break
                elif test_execution_status == 'Failed':
                    print('Stage Test failed')
                    break
                else:
                    count += 1
                    time.sleep(2)

# Pull Request Events Ex: pullRequestCreated, pullRequestSourceBranchUpdated
def pr_events(message, event_ytpe):
    print(message)
    pr_id = message['detail']['pullRequestId']
    print("Pull Request ID :: %s" % pr_id)
    source_commit = message['detail']['sourceCommit']
    print("Source Commit ID :: %s" % source_commit)
    destination_commit = message['detail']['destinationCommit']
    print("Destination Commit ID :: %s" % destination_commit)
    pipeline_name = str(repository_name+'-'+'PullRequest'+'-'+pr_id+'-'+source_commit)
    print("Pipeline Name :: %s" % pipeline_name)
    destination_branch = message['detail']['destinationReference'].split('/')[-1]
    print("Destination Branch Reference :: %s" % destination_branch)

    try:
        pipeline_response = codepipeline_client.get_pipeline(name=pipeline_name)
    except ClientError as e:
        if e.response['Error']['Code'] == 'PipelineNotFoundException':
            print("Pipeline %s does not exist, creating a new one." % pipeline_name)
            branch_ref = '/'.join(message['detail']['sourceReference'].split('/')[2:])
            print("Branch Name :: %s" % branch_ref)

            code_pipeline_configuration = pipeline_configuration
            code_pipeline_configuration['pipeline']['name'] = pipeline_name
            code_pipeline_configuration['pipeline']['stages'][2]['actions'][0]['configuration']['ObjectKey'] = source_commit
            code_pipeline_configuration['pipeline']['stages'][3]['actions'][0]['configuration']['App'] = source_commit+'/app-debug.apk'

            modified_pipeline_json = update_pipeline(code_pipeline_configuration, branch_ref, destination_branch)
            new_pipeline_response = create_pipeline(modified_pipeline_json)

            pipeline_url = 'https://'+region+'.console.aws.amazon.com/codesuite/codepipeline/pipelines/'+pipeline_name+'/view?region='+region
            content = u'\u23F3'+' Pipeline started at {}'.format(datetime.datetime.utcnow().time())+' '+'- See the [Pipeline]({0})'.format(pipeline_url)
            post_comment(pr_id, repository_name, source_commit, destination_commit, content)

            # TO-DO get the comment ID and use it to post all other status updates
            time.sleep(10)

            pipeline_name = new_pipeline_response['pipeline']['name']
            pipeline_stages = get_status(pipeline_name)

            for stage in pipeline_stages['stageStates']:

                if stage['stageName'] == 'Source':
                    source_execution_status = stage['actionStates'][0]['latestExecution']['status']
                    if source_execution_status == 'Succeeded':
                        print('Stage Source Passed.')
                    else:
                        print('Stage Source Failed.')
                        break

                elif stage['stageName'] == 'Build':
                    build_url = stage['actionStates'][0]['latestExecution']['externalExecutionUrl']
                    count = 0
                    while count < 500:
                        pipeline_build_status = get_status(pipeline_name)
                        build_execution_status = pipeline_build_status['stageStates'][1]['latestExecution']['status']
                        print('Build status :: ', build_execution_status)
                        if build_execution_status == 'Succeeded':
                            print('Stage Build Passed.')
                            content = u'\u2705'+' Stage Build Passed - See the [Logs]({0})'.format(build_url)
                            post_comment(pr_id, repository_name,
                                         source_commit, destination_commit,
                                         content)

                            #if destination_branch == 'master':
                            #    print('Keeping the Master Branch Pipelines for reference.')
                            #else:
                            #    delete_pipeline(pipeline_name)
                            break

                        elif build_execution_status == 'Failed':
                            content = u'\u274C'+' Build Failed - See the [Logs]({0})'.format(build_url)
                            post_comment(pr_id, repository_name,
                                         source_commit, destination_commit,
                                         content)
                            #if destination_branch == 'master':
                            #    print('Keeping the Master Branch Pipelines for reference.')
                            #else:
                            #    delete_pipeline(pipeline_name)
                            break

                        else:
                            count += 1
                            time.sleep(10)

                elif stage['stageName'] == 'Deploy':
                    count = 0
                    while count < 500:
                        pipeline_deploy_status = get_status(pipeline_name)
                        deploy_execution_status = pipeline_deploy_status['stageStates'][2]['latestExecution']['status']
                        if deploy_execution_status == 'Succeeded':
                            print('Stage '+pipeline_deploy_status['stageStates'][2]['actionStates'][0]['latestExecution']['summary'])
                            build_path = pipeline_deploy_status['stageStates'][2]['actionStates'][0]['latestExecution']['externalExecutionId']
                            print(build_path)
                            apk_s3_location = 'https://'+region+'.console.aws.amazon.com/s3/buckets/android-builds?region='+region+'&prefix='+commit_id+'/&showversions=false'
                            print('Download APK file from :: {}'.format(apk_s3_location))
                            content = u'\u2705'+' Stage Deploy Passed - See the [Artifactory]({0})'.format(apk_s3_location)
                            post_comment(pr_id, repository_name,
                                         source_commit, destination_commit,
                                         content)
                            break
                        elif deploy_execution_status == 'Failed':
                            print('Stage Deploy failed')
                            break
                        else:
                            count += 1
                            time.sleep(2)

                elif stage['stageName'] == 'Test':
                    count = 0
                    while count < 500:
                        pipeline_test_status = get_status(pipeline_name)
                        test_execution_status = pipeline_test_status['stageStates'][3]['latestExecution']['status']
                        if test_execution_status == 'Succeeded':
                            summary = pipeline_test_status['stageStates'][3]['actionStates'][0]['latestExecution']['summary']
                            print('Stage '+summary)
                            devicefarm_url = pipeline_test_status['stageStates'][3]['actionStates'][0]['latestExecution']['externalExecutionUrl']
                            print(devicefarm_url)
                            content = u'\u2705'+' '+summary+' - See the logs [DeviceFarm]({0})'.format(devicefarm_url)
                            post_comment(pr_id, repository_name,
                                         source_commit, destination_commit,
                                         content)
                            break
                        elif test_execution_status == 'Failed':
                            print('Stage Deploy failed')
                            break
                        else:
                            count += 1
                            time.sleep(2)

def lambda_handler(event, context):

    message = ast.literal_eval(event['Records'][0]['Sns']['Message'])
    event_type = message['detail']['event']

    if event_type == 'referenceCreated' or event_type == 'referenceUpdated':
        branch_events(message, event_type)
    elif event_type == 'pullRequestCreated' or event_type == 'pullRequestSourceBranchUpdated':
        pr_events(message, event_type)


import MapReduce
import sys

"""
Inverted Index using Python MapReduce Framework
"""

mr = MapReduce.MapReduce()

# =============================
# Do not modify above this line

def mapper(record):
    # key: document identifier
    # value: document contents
    key = record[0]
    value = record[1]
    words = value.split()
    #for k in key:
    for w in words:
      mr.emit_intermediate(w, key)

def reducer(key, list_of_values):
    # key: word
    # value: list of occurrence counts
    doc_list = []
    for v in list_of_values:
      doc_list.append(v)
    # removing duplicates
    doc_set = set(doc_list)
    doc_list = list(doc_set)
    mr.emit((key, doc_list))

# Do not modify below this line
# =============================
if __name__ == '__main__':
  inputdata = open(sys.argv[1])
  mr.execute(inputdata, mapper, reducer)

Introduction to Data Science/assignment3/inverted_index.py


"""
@brief Unit tests for image_utils.py module.

@author J. Chiang <jchiang@slac.stanford.edu>
"""
import os
import unittest
import numpy as np
import lsst.afw
import lsst.eotest.image_utils as imutils
from lsst.eotest.sensor import MaskedCCD, AmplifierGeometry
import lsst.eotest.sensor.sim_tools as sim_tools
import lsst.afw.image as afwImage

class BiasFunc(object):
    def __init__(self, slope, intercept):
        self.pars = slope, intercept

    def __call__(self, x):
        return x*self.pars[0] + self.pars[1]

class BiasHandlingTestCase(unittest.TestCase):
    bias_slope = 1e-3
    bias_intercept = 0.5
    exptime = 1
    gain = 1
    image_file = 'test_image.fits'

    @classmethod
    def setUpClass(cls):
        cls.amp_geom = AmplifierGeometry()
        cls.bias_image = afwImage.ImageF(cls.amp_geom.full_segment)
        imarr = cls.bias_image.getArray()
        ny, nx = imarr.shape
        yvals = np.arange(0, ny, dtype=np.float)
        bias_func = BiasFunc(cls.bias_slope, cls.bias_intercept)
        for x in range(nx):
            imarr[:, x] += bias_func(yvals)
        ccd = sim_tools.CCD(exptime=cls.exptime, gain=cls.gain,
                            geometry=cls.amp_geom)
        for amp in ccd.segments:
            ccd.segments[amp].image += cls.bias_image
        ccd.writeto(cls.image_file)

    @classmethod
    def tearDownClass(cls):
        os.remove(cls.image_file)

    def test_bias_func(self):
        bias_func = BiasFunc(self.bias_slope, self.bias_intercept)
        ccd = MaskedCCD(self.image_file)
        for amp in ccd:
            bf_i = imutils.bias_func(ccd[amp].getImage(),
                                     self.amp_geom.serial_overscan)
            bf_m = imutils.bias_func(ccd[amp], self.amp_geom.serial_overscan)
            for y in range(2022):
                self.assertEqual(bf_i(y), bf_m(y))
                self.assertAlmostEqual(bias_func(y), bf_m(y))
            # Test that row-by-row median operates.
            row_bias = imutils.bias_func(ccd[amp],
                                         self.amp_geom.serial_overscan,
                                         fit_order=-1)
            for y in range(2022):
                self.assertAlmostEqual(bf_i(y), row_bias(y), places=6)

    def test_bias_image(self):
        ccd = MaskedCCD(self.image_file)
        for amp in ccd:
            my_bias_image = imutils.bias_image(ccd[amp],
                                               self.amp_geom.serial_overscan)
            fracdiff = ((self.bias_image.getArray() - my_bias_image.getArray())
                        / self.bias_image.getArray())
            self.assertTrue(max(np.abs(fracdiff.flat)) < 1e-6)

    def test_unbias_and_trim(self):
        ccd = MaskedCCD(self.image_file)
        for amp in ccd:
            image = imutils.unbias_and_trim(ccd[amp],
                                            self.amp_geom.serial_overscan,
                                            self.amp_geom.imaging)
            imarr = image.getImage().getArray()
            self.assertTrue(max(np.abs(imarr.flat)) < 1e-6)
            #
            # Test of corresponding MaskedCCD method.
            #
            image = ccd.unbiased_and_trimmed_image(amp)
            imarr = image.getImage().getArray()
            self.assertTrue(max(np.abs(imarr.flat)) < 1e-6)

class FitsMedianTestCase(unittest.TestCase):
    def setUp(self):
        self.values = (0, 1, 2, 3, 4)
        self.files = []
        for i in self.values:
            self.files.append('test_fits_median_image_%02i.fits' % i)
            ccd = sim_tools.CCD(exptime=1)
            for amp in ccd.segments:
                ccd.segments[amp].image += i
            ccd.writeto(self.files[-1])

    def tearDown(self):
        for item in self.files:
            os.remove(item)

    def test_fits_median(self):
        hdu = 2 if lsst.afw.__version__.startswith('12.0') else 1
        median_image = imutils.fits_median(self.files, hdu=hdu, fix=True)
        imarr = median_image.getArray()
        for x in imarr.flat:
            self.assertEqual(self.values[len(self.values)//2], x)

if __name__ == '__main__':
    unittest.main()


import operator
import math
import numpy as np
from copy import copy, deepcopy
import random
import matplotlib.pyplot as plt

class NeuralNetwork():

	def __init__(self,sizes,squashFunction,learningRate,weights=None,biases=None):
		self.network = self.createMatrix(sizes)
		self.squashFunction = squashFunction
		self.errorList = []
		if(weights == None):
			self.weights = self.createMatrix(sizes)
			self.randomizeWeights()
		else:
			self.weights = weights
			if(self.compareSizes(self.network,weights) == False):
				raise Error("Network matrix and first 2 dimensions of weight matrix are not of the same dimensions.")
		#self.weights = weights
		if(biases == None):
			self.biases = self.createMatrix(sizes)
			self.randomizeBiases()
		else:
			self.biases = biases
			if(self.compareSizes(self.network,biases) == False):
				raise Error("Network and bias matrices are not of the same dimensions.")

		self.MLTools.learningRate = learningRate
		# if(len(expectedValues) != len(weights[-1])):
		# 	throw Error("Number of expected outputs does not match number of output nodes")
		# self.expectedValues = expectedValues

	# def squash(value,function):
	# 	if function == "sigmoid":
	# 		return sigmoid(value)
	# 	else:
	# 		return None

	def randomizeBiases(self):
		for i in range(len(self.biases)):
			if(i == 0):
				continue
			else:
				for j in range(len(self.biases[i])):
					self.biases[i][j] = random.random()

	def randomizeWeights(self):
		for i in range(len(self.weights)):
			if(i == len(self.weights)-1):
				for j in range(len(self.weights[i])):
					self.weights[i][j] = [1]
			else:
				for j in range(len(self.weights[i])):
					self.weights[i][j] = [0] * len(self.weights[i+1])
					for k in range(len(self.weights[i+1])):
						self.weights[i][j][k] = random.random()

	def compareSizes(self,m1,m2):
		if(len(m1) != len(m2)):
			return False
		else:
			for i in range(len(m1)):
				if(len(m1[i]) != len(m2[i])):
					return False
		return True

	def setInput(self,inputVector):
		if(self.network == None):
			raise Exception("Attempted to assign input vector to empty network")
		elif(len(self.network[0]) != len(inputVector)):
			raise IndexError("Attempted to assign vector of size " + str(len(inputVector))
				+ " to matrix column with size " + str(len(self.network[0])))
		else:
			self.network[0] = inputVector

	def flush(self):
		for i in range(len(self.network)):
			if(i == 0):
				continue
			else:
				for j in range(len(self.network[i])):
					self.network[i][j] = 0

	class MLTools(object):

		#sigmoid squash function
		def sigmoid(value):
			exp = math.exp(-value)
			return 1/(1+exp)

		def sigmoidDerivative(value):
			return (value)*(1-value)

		def relu(netInput):
			return max(0, netInput)

		def reluDerivative(value):
			if(value > 0):
				return 1
			else:
				return 0

		@staticmethod
		def squaredError(expected,actual):
			total = 0
			#errorVector = []
			for e,a in zip(expected,actual):
				error = (.5) * ((e-a)**2)  #Σ (1/2 (expected−actual)^2)
				total += error
				#errorVector.append(error)
			return total #,errorVector

		#Dictionary containing refs to functions
		squash = {"sigmoid":sigmoid,"relu":relu}

		#Dictionary containing refs to functions derivatives
		derivatives = {"sigmoid":sigmoidDerivative,"relu":reluDerivative}

		#initial learning rate
		#learningRate = .5

	#Input size is the size of the input layer
	#Output size is the size of the output layer
	#sizes is a vector containing the sizes of each
	#of the hidden layers
	def createMatrix(self,sizes,inputSize=None,outputSize=None):
		if inputSize == None:
		 #creates an empty vector the size of all the
		 #layers (including input and output)
			newNetwork = [0] * (len(sizes))

			#populates the vector with vectors for each layer
			for i in range(len(newNetwork)):
				newNetwork[i] = [0] * sizes[i]

		else:
			 #creates an empty vector the size of all the
			 #layers (including input and output)
			newNetwork = [0] * (len(sizes)+2)

			newNetwork[0] = [0] * inputSize #input layer
			newNetwork[-1] = [0] * outputSize #output layer
			for i,j in zip(range(1,len(newNetwork)-1),range(len(sizes))): #skip the input and output layer
				newNetwork[i] = [0] * sizes[j]

			#network = np.matrix(network)

		return newNetwork

	def addMatrices(self,m1,m2):
		for i in range(len(m1)):
			for j in range(len(m1[i])):
				m1[i][j] += m2[i][j]
		return m1

	def feedForward(self,inputs):

		self.setInput(inputs) #sets the inputs
		self.flush() #flush the previous data in the network feed
		#self.network = (np.matrix(self.network) + np.matrix(self.biases)).tolist()  #add biases to network
		self.network = self.addMatrices(self.network,self.biases)

		for i in range(len(self.network)): #layer
			if i == len(self.network)-1: #when calculating outputs
				break
			for j in range(len(self.network[i])): #neurons in current layer
				for k in range(len(self.network[i+1])): #neurons in next layer
					self.network[i+1][k] += (self.network[i][j])*(self.weights[i][j][k]) #add the value from layer times the weight
			for m in range(len(self.network[i+1])):
				self.network[i+1][m] = self.MLTools.squash[self.squashFunction](self.network[i+1][m])

		return self.network[-1] #output

	def backpropagation(self,expectedValues):
		error = self.MLTools.squaredError(self.network[-1],expectedValues)
		#errorVector = [] #Vector containing the dError/dOut values
		#outVector = [] #Vector containing the dOut/dNet values
		originalWeights = deepcopy(self.weights) #deepcopy prevents the references from being copied rather than values

		for i in range(len(self.network)-1,-1,-1): #layer
			for j in range(len(self.network[i])): #current node
				if i == len(self.network)-1: #for the output layer
					dError_dOut = -(expectedValues[j] - self.network[i][j])
					dOut_dNet = self.MLTools.derivatives[self.squashFunction](self.network[i][j])
					#errorVector.append(dError_dOut)
					#outVector.append(dOut_dNet)
					for k in range(len(self.network[i-1])):
						dNet_dW = self.network[i-1][k]
						gradient = (dError_dOut*dOut_dNet*dNet_dW)

						#print(str(gradient))

						# 	#i-1 = prev layer, k = which node in that layer, j = weight pointing to our current node
						self.weights[i-1][k][j] = originalWeights[i-1][k][j] - (self.MLTools.learningRate * gradient)

						#print(weights[i-1][k][j])

				elif i == 0:
					break
				else:
					dError_dOut = 0
					for l in range(len(self.network[i+1])): #the layer ahead, analogous to j
						#these derivatives are relative to the next layer
						dE_dNet = -(expectedValues[l] - self.network[i+1][l])
						dOut_dNet = self.MLTools.derivatives[self.squashFunction](self.network[i+1][l])
						dEl_doutk = dE_dNet * dOut_dNet * originalWeights[i][j][l]     #the error of the next node/output of this node
						dError_dOut += dEl_doutk
					#these derivatives are for the current layer
					dOut_dNet = self.MLTools.derivatives[self.squashFunction](self.network[i][j])
					for k in range(len(self.network[i-1])): #the previous layer
						dNet_dW = self.network[i-1][k]
						gradient = (dError_dOut * dOut_dNet * dNet_dW)
						self.weights[i-1][k][j] = originalWeights[i-1][k][j] - (self.MLTools.learningRate * gradient)

						#print(weights[i-1][k][j])

		return error

	def train(self,inputs,expected,showErrors=False,showOutput=False):
		#print(self.weights)
		output = self.feedForward(inputs)
		#print("output",output)
		errors = self.backpropagation(expected)
		self.errorList.append(errors)
		if(showErrors and not showOutput):
			return errors
		elif(showOutput and not showErrors):
			return output
		else:
			return errors,output

def createAdd(lower,upper,size):
	inputs = []
	outputs = []
	for i in range(size):
		n1 = random.randint(lower,upper)
		n2 = random.randint(lower,upper)
		inputs.append([lower+n1,upper-n2])
		outputs.append((lower+n1)+(upper-n2))
	return inputs,outputs

# networkSizes = [2,2,2] #The vector containing the sizes of the layers
# #network = createMatrix(networkSizes)
# inputs = [.05,.10] #initialize the input layer

# weights = []
# weights.append([[.15,.25],[.20,.30]])
# weights.append([[.40,.50],[.45,.55]])
# weights.append([[1],[1]])#no weights
# #weights = np.matrix(weights)

# biases = []
# biases.append([0,0])
# biases.append([.35,.35])
# biases.append([.60,.60])
# #biases = np.matrix(biases)

# targets = [.01,.99]
# squashFunction = "sigmoid"  #a string with the name of the squash function
# learningRate = .5
# #Initialize Netword
# network = NeuralNetwork(networkSizes,squashFunction,learningRate,weights=weights,biases=biases)
# # network.train(inputs,targets)
# # print(network.network)
# for i in range(10000): #num epochs
# 	network.train(inputs,targets)
# 	#print(network.weights,"\n","-"*10)
# print(network.feedForward(inputs),"\n","-"*10)

#XOR
networkSizes = [2,2,1] #The vector containing the sizes of the layers
inputs = [[0,0],[0,1],[1,0],[1,1]]
targets = [0,1,1,0]
squashFunction = "sigmoid"
learningRate = .1
network = NeuralNetwork(networkSizes,squashFunction,learningRate)
for i in range(50000): #num epochs
	for j in range(len(inputs)):
		network.train(inputs[j],[targets[j]])
for i in range(len(inputs)):
	print(network.feedForward(inputs[i]))
print(network.weights)

#plt.plot(network.errorList)
#plt.show()

# #addition: doesnt work
# networkSizes = [2,2,1] #The vector containing the sizes of the layers
# inputs,targets = createAdd(0,50,10)
# print(targets)
# validation_inputs,validation_targets = createAdd(50,100,10)
# squashFunction = "relu"
# learningRate = 1
# network = NeuralNetwork(networkSizes,squashFunction,learningRate)
# for i in range(100000): #num epochs
# 	for j in range(len(inputs)):
# 		network.train(inputs[j],[targets[j]])
# for i in range(len(validation_inputs)):
# 	print(network.feedForward(validation_inputs[i])," | ",validation_targets[i])