from flask_restful import Resource, fields, marshal_with
from ..backend import database, auth

class Players(Resource):

    decorators = [auth.login_required]

    types = {
        'uuid': fields.String,
        'name': fields.String,
        'prefix': fields.String,
        'suffix': fields.String
    }

    def __init__(self):
        self.db = database

    @marshal_with(types)
    def get(self, playerUuid=None):
        if playerUuid is None:
            return self.db.getPlayers()
        else:
            player = self.db.getPlayers(playerUuid)
            if player is None:
                return None, 404
            else:
                return player


from uuid import uuid4
import time
#
class Worker(object):
    def __init__(self,**kws):
        self.workerId     = kws.get("workerId",str(uuid4))
        self.port      = kws.get("port")
        self.balls     = kws.get("balls", [])
        self.isStarted = kws.get("isStarted",False)
        self.createdAt = kws.get("createdAt",time.time())


"""
Tcpdump parser

Source:
 * libpcap source code (file savefile.c)
 * RFC 791 (IPv4)
 * RFC 792 (ICMP)
 * RFC 793 (TCP)
 * RFC 1122 (Requirements for Internet Hosts)

Author: Victor Stinner
Creation: 23 march 2006
"""

from lib.hachoir_parser import Parser
from lib.hachoir_core.field import (FieldSet, ParserError,
    Enum, Bytes, NullBytes, RawBytes,
    UInt8, UInt16, UInt32, Int32, TimestampUnix32,
    Bit, Bits, NullBits)
from lib.hachoir_core.endian import NETWORK_ENDIAN, LITTLE_ENDIAN
from lib.hachoir_core.tools import humanDuration
from lib.hachoir_core.text_handler import textHandler, hexadecimal
from lib.hachoir_core.tools import createDict
from lib.hachoir_parser.network.common import MAC48_Address, IPv4_Address, IPv6_Address

def diff(field):
    return humanDuration(field.value*1000)

class Layer(FieldSet):
    endian = NETWORK_ENDIAN
    def parseNext(self, parent):
        return None

class ARP(Layer):
    opcode_name = {
        1: "request",
        2: "reply"
    }
    endian = NETWORK_ENDIAN

    def createFields(self):
        yield UInt16(self, "hw_type")
        yield UInt16(self, "proto_type")
        yield UInt8(self, "hw_size")
        yield UInt8(self, "proto_size")
        yield Enum(UInt16(self, "opcode"), ARP.opcode_name)
        yield MAC48_Address(self, "src_mac")
        yield IPv4_Address(self, "src_ip")
        yield MAC48_Address(self, "dst_mac")
        yield IPv4_Address(self, "dst_ip")

    def createDescription(self):
        desc = "ARP: %s" % self["opcode"].display
        opcode = self["opcode"].value
        src_ip = self["src_ip"].display
        dst_ip = self["dst_ip"].display
        if opcode == 1:
            desc += ", %s ask %s" % (dst_ip, src_ip)
        elif opcode == 2:
            desc += " from %s" % src_ip
        return desc

class TCP_Option(FieldSet):
    NOP = 1
    MAX_SEGMENT = 2
    WINDOW_SCALE = 3
    SACK = 4
    TIMESTAMP = 8

    code_name = {
        NOP: "NOP",
        MAX_SEGMENT: "Max segment size",
        WINDOW_SCALE: "Window scale",
        SACK: "SACK permitted",
        TIMESTAMP: "Timestamp"
    }

    def __init__(self, *args):
        FieldSet.__init__(self, *args)
        if self["code"].value != self.NOP:
            self._size = self["length"].value * 8
        else:
            self._size = 8

    def createFields(self):
        yield Enum(UInt8(self, "code", "Code"), self.code_name)
        code = self["code"].value
        if code == self.NOP:
            return
        yield UInt8(self, "length", "Option size in bytes")
        if code == self.MAX_SEGMENT:
            yield UInt16(self, "max_seg", "Maximum segment size")
        elif code == self.WINDOW_SCALE:
            yield UInt8(self, "win_scale", "Window scale")
        elif code == self.TIMESTAMP:
            yield UInt32(self, "ts_val", "Timestamp value")
            yield UInt32(self, "ts_ecr", "Timestamp echo reply")
        else:
            size = (self.size - self.current_size) // 8
            if size:
                yield RawBytes(self, "data", size)

    def createDescription(self):
        return "TCP option: %s" % self["code"].display

class TCP(Layer):
    port_name = {
        13: "daytime",
        20: "ftp data",
        21: "ftp",
        23: "telnet",
        25: "smtp",
        53: "dns",
        63: "dhcp/bootp",
        80: "HTTP",
        110: "pop3",
        119: "nntp",
        123: "ntp",
        139: "netbios session service",
        1863: "MSNMS",
        6667: "IRC"
    }

    def createFields(self):
        yield Enum(UInt16(self, "src"), self.port_name)
        yield Enum(UInt16(self, "dst"), self.port_name)
        yield UInt32(self, "seq_num")
        yield UInt32(self, "ack_num")

        yield Bits(self, "hdrlen", 6, "Header lenght")
        yield NullBits(self, "reserved", 2, "Reserved")

        yield Bit(self, "cgst", "Congestion Window Reduced")
        yield Bit(self, "ecn-echo", "ECN-echo")
        yield Bit(self, "urg", "Urgent")
        yield Bit(self, "ack", "Acknowledge")
        yield Bit(self, "psh", "Push mmode")
        yield Bit(self, "rst", "Reset connection")
        yield Bit(self, "syn", "Synchronize")
        yield Bit(self, "fin", "Stop the connection")

        yield UInt16(self, "winsize", "Windows size")
        yield textHandler(UInt16(self, "checksum"), hexadecimal)
        yield UInt16(self, "urgent")

        size = self["hdrlen"].value*8 - self.current_size
        while 0 < size:
            option = TCP_Option(self, "option[]")
            yield option
            size -= option.size

    def parseNext(self, parent):
        return None

    def createDescription(self):
        src = self["src"].value
        dst = self["dst"].value
        if src < 32768:
            src = self["src"].display
        else:
            src = None
        if dst < 32768:
            dst = self["dst"].display
        else:
            dst = None
        desc = "TCP"
        if src != None and dst != None:
            desc += " (%s->%s)" % (src, dst)
        elif src != None:
            desc += " (%s->)" % (src)
        elif dst != None:
            desc += " (->%s)" % (dst)

        # Get flags
        flags = []
        if self["syn"].value:
            flags.append("SYN")
        if self["ack"].value:
            flags.append("ACK")
        if self["fin"].value:
            flags.append("FIN")
        if self["rst"].value:
            flags.append("RST")
        if flags:
            desc += " [%s]" % (",".join(flags))
        return desc

class UDP(Layer):
    port_name = {
        12: "daytime",
        22: "ssh",
        53: "DNS",
        67: "dhcp/bootp",
        80: "http",
        110: "pop3",
        123: "ntp",
        137: "netbios name service",
        138: "netbios datagram service"
    }

    def createFields(self):
        yield Enum(UInt16(self, "src"), UDP.port_name)
        yield Enum(UInt16(self, "dst"), UDP.port_name)
        yield UInt16(self, "length")
        yield textHandler(UInt16(self, "checksum"), hexadecimal)

    def createDescription(self):
        return "UDP (%s->%s)" % (self["src"].display, self["dst"].display)

class ICMP(Layer):
    REJECT = 3
    PONG = 0
    PING = 8
    type_desc = {
        PONG: "Pong",
        REJECT: "Reject",
        PING: "Ping"
    }
    reject_reason = {
        0: "net unreachable",
        1: "host unreachable",
        2: "protocol unreachable",
        3: "port unreachable",
        4: "fragmentation needed and DF set",
        5: "source route failed",
        6: "Destination network unknown error",
        7: "Destination host unknown error",
        8: "Source host isolated error",
        9: "Destination network administratively prohibited",
        10: "Destination host administratively prohibited",
        11: "Unreachable network for Type Of Service",
        12: "Unreachable host for Type Of Service.",
        13: "Communication administratively prohibited",
        14: "Host precedence violation",
        15: "Precedence cutoff in effect"
    }

    def createFields(self):
        # Type
        yield Enum(UInt8(self, "type"), self.type_desc)
        type = self["type"].value

        # Code
        field = UInt8(self, "code")
        if type == 3:
            field = Enum(field, self.reject_reason)
        yield field

        # Options
        yield textHandler(UInt16(self, "checksum"), hexadecimal)
        if type in (self.PING, self.PONG): # and self["code"].value == 0:
            yield UInt16(self, "id")
            yield UInt16(self, "seq_num")
            # follow: ping data
        elif type == self.REJECT:
            yield NullBytes(self, "empty", 2)
            yield UInt16(self, "hop_mtu", "Next-Hop MTU")

    def createDescription(self):
        type = self["type"].value
        if type in (self.PING, self.PONG):
            return "%s (num=%s)" % (self["type"].display, self["seq_num"].value)
        else:
            return "ICMP (%s)" % self["type"].display

    def parseNext(self, parent):
        if self["type"].value == self.REJECT:
            return IPv4(parent, "rejected_ipv4")
        else:
            return None

class ICMPv6(Layer):
    ECHO_REQUEST = 128
    ECHO_REPLY = 129
    TYPE_DESC = {
        128: "Echo request",
        129: "Echo reply",
    }

    def createFields(self):
        yield Enum(UInt8(self, "type"), self.TYPE_DESC)
        yield UInt8(self, "code")
        yield textHandler(UInt16(self, "checksum"), hexadecimal)

        if self['type'].value in (self.ECHO_REQUEST, self.ECHO_REPLY):
            yield UInt16(self, "id")
            yield UInt16(self, "sequence")

    def createDescription(self):
        if self['type'].value in (self.ECHO_REQUEST, self.ECHO_REPLY):
            return "%s (num=%s)" % (self["type"].display, self["sequence"].value)
        else:
            return "ICMPv6 (%s)" % self["type"].display

class IP(Layer):
    PROTOCOL_INFO = {
         1: ("icmp", ICMP, "ICMP"),
        6: ("tcp",  TCP, "TCP"),
        17: ("udp",  UDP, "UDP"),
        58: ("icmpv6",  ICMPv6, "ICMPv6"),
        60: ("ipv6_opts", None, "IPv6 destination option"),
    }
    PROTOCOL_NAME = createDict(PROTOCOL_INFO, 2)

    def parseNext(self, parent):
        proto = self["protocol"].value
        if proto not in self.PROTOCOL_INFO:
            return None
        name, parser, desc = self.PROTOCOL_INFO[proto]
        if not parser:
            return None
        return parser(parent, name)

class IPv4(IP):
    precedence_name = {
        7: "Network Control",
        6: "Internetwork Control",
        5: "CRITIC/ECP",
        4: "Flash Override",
        3: "Flash",
        2: "Immediate",
        1: "Priority",
        0: "Routine",
    }

    def __init__(self, *args):
        FieldSet.__init__(self, *args)
        self._size = self["hdr_size"].value * 32

    def createFields(self):
        yield Bits(self, "version", 4, "Version")
        yield Bits(self, "hdr_size", 4, "Header size divided by 5")

        # Type of service
        yield Enum(Bits(self, "precedence", 3, "Precedence"), self.precedence_name)
        yield Bit(self, "low_delay", "If set, low delay, else normal delay")
        yield Bit(self, "high_throu", "If set, high throughput, else normal throughput")
        yield Bit(self, "high_rel", "If set, high relibility, else normal")
        yield NullBits(self, "reserved[]", 2, "(reserved for future use)")

        yield UInt16(self, "length")
        yield UInt16(self, "id")

        yield NullBits(self, "reserved[]", 1)
        yield Bit(self, "df", "Don't fragment")
        yield Bit(self, "more_frag", "There are more fragments? if not set, it's the last one")
        yield Bits(self, "frag_ofst_lo", 5)
        yield UInt8(self, "frag_ofst_hi")
        yield UInt8(self, "ttl", "Type to live")
        yield Enum(UInt8(self, "protocol"), self.PROTOCOL_NAME)
        yield textHandler(UInt16(self, "checksum"), hexadecimal)
        yield IPv4_Address(self, "src")
        yield IPv4_Address(self, "dst")

        size = (self.size - self.current_size) // 8
        if size:
            yield RawBytes(self, "options", size)

    def createDescription(self):
        return "IPv4 (%s>%s)" % (self["src"].display, self["dst"].display)

class IPv6(IP):
    static_size = 40 * 8
    endian = NETWORK_ENDIAN

    def createFields(self):
        yield Bits(self, "version", 4, "Version (6)")
        yield Bits(self, "traffic", 8, "Traffic class")
        yield Bits(self, "flow", 20, "Flow label")
        yield Bits(self, "length", 16, "Payload length")
        yield Enum(Bits(self, "protocol", 8, "Next header"), self.PROTOCOL_NAME)
        yield Bits(self, "hop_limit", 8, "Hop limit")
        yield IPv6_Address(self, "src")
        yield IPv6_Address(self, "dst")

    def createDescription(self):
        return "IPv6 (%s>%s)" % (self["src"].display, self["dst"].display)

class Layer2(Layer):
    PROTO_INFO = {
        0x0800: ("ipv4", IPv4, "IPv4"),
        0x0806: ("arp",  ARP,  "ARP"),
        0x86dd: ("ipv6", IPv6, "IPv6"),
    }
    PROTO_DESC = createDict(PROTO_INFO, 2)

    def parseNext(self, parent):
        try:
            name, parser, desc = self.PROTO_INFO[ self["protocol"].value ]
            return parser(parent, name)
        except KeyError:
            return None

class Unicast(Layer2):
    packet_type_name = {
        0: "Unicast to us"
    }
    def createFields(self):
        yield Enum(UInt16(self, "packet_type"), self.packet_type_name)
        yield UInt16(self, "addr_type", "Link-layer address type")
        yield UInt16(self, "addr_length", "Link-layer address length")
        length = self["addr_length"].value
        length = 8   # FIXME: Should we use addr_length or not?
        if length:
            yield RawBytes(self, "source", length)
        yield Enum(UInt16(self, "protocol"), self.PROTO_DESC)

class Ethernet(Layer2):
    static_size = 14*8
    def createFields(self):
        yield MAC48_Address(self, "dst")
        yield MAC48_Address(self, "src")
        yield Enum(UInt16(self, "protocol"), self.PROTO_DESC)

    def createDescription(self):
        return "Ethernet: %s>%s (%s)" % \
            (self["src"].display, self["dst"].display, self["protocol"].display)

class Packet(FieldSet):
    endian = LITTLE_ENDIAN

    def __init__(self, parent, name, parser, first_name):
        FieldSet.__init__(self, parent, name)
        self._size = (16 + self["caplen"].value) * 8
        self._first_parser = parser
        self._first_name = first_name

    def createFields(self):
        yield TimestampUnix32(self, "ts_epoch", "Timestamp (Epoch)")
        yield UInt32(self, "ts_nanosec", "Timestamp (nano second)")
        yield UInt32(self, "caplen", "length of portion present")
        yield UInt32(self, "len", "length this packet (off wire)")

        # Read different layers
        field = self._first_parser(self, self._first_name)
        while field:
            yield field
            field = field.parseNext(self)

        # Read data if any
        size = (self.size - self.current_size) // 8
        if size:
            yield RawBytes(self, "data", size)

    def getTimestamp(self):
        nano_sec = float(self["ts_nanosec"].value) / 100
        from datetime import timedelta
        return self["ts_epoch"].value + timedelta(microseconds=nano_sec)

    def createDescription(self):
        t0 = self["/packet[0]"].getTimestamp()
#        ts = max(self.getTimestamp() - t0, t0)
        ts = self.getTimestamp() - t0
        #text = ["%1.6f: " % ts]
        text = ["%s: " % ts]
        if "icmp" in self:
            text.append(self["icmp"].description)
        elif "tcp" in self:
            text.append(self["tcp"].description)
        elif "udp" in self:
            text.append(self["udp"].description)
        elif "arp" in self:
            text.append(self["arp"].description)
        else:
            text.append("Packet")
        return "".join(text)

class TcpdumpFile(Parser):
    PARSER_TAGS = {
        "id": "tcpdump",
        "category": "misc",
        "min_size": 24*8,
        "description": "Tcpdump file (network)",
        "magic": (("\xd4\xc3\xb2\xa1", 0),),
    }
    endian = LITTLE_ENDIAN

    LINK_TYPE = {
          1: ("ethernet", Ethernet),
        113: ("unicast", Unicast),
    }
    LINK_TYPE_DESC = createDict(LINK_TYPE, 0)

    def validate(self):
        if self["id"].value != "\xd4\xc3\xb2\xa1":
            return "Wrong file signature"
        if self["link_type"].value not in self.LINK_TYPE:
            return "Unknown link type"
        return True

    def createFields(self):
        yield Bytes(self, "id", 4, "Tcpdump identifier")
        yield UInt16(self, "maj_ver", "Major version")
        yield UInt16(self, "min_ver", "Minor version")
        yield Int32(self, "this_zone", "GMT to local time zone correction")
        yield Int32(self, "sigfigs", "accuracy of timestamps")
        yield UInt32(self, "snap_len", "max length saved portion of each pkt")
        yield Enum(UInt32(self, "link_type", "data link type"), self.LINK_TYPE_DESC)
        link = self["link_type"].value
        if link not in self.LINK_TYPE:
            raise ParserError("Unknown link type: %s" % link)
        name, parser = self.LINK_TYPE[link]
        while self.current_size < self.size:
            yield Packet(self, "packet[]", parser, name)


class Solution03:
    def is_isomorphic(self, s1, s2):
        result = True
        sv1, sv2 = {}, {}

        if len(s1) == len(s2):
            for i in range(len(s1)):
                c1, c2 = s1[i], s2[i]

                if c1 not in sv1:
                    sv1[c1] = c2

                if c2 not in sv2:
                    sv2[c2] = c1

                if sv1[c1] != c2 or sv2[c2] != c1:
                    result = False
        else:
            result = False

        return result

if __name__ == "__main__":
    input_s1 = "paper"
    input_s2 = "title"

    s = Solution03()
    print("Input: (%s, %s)" % (input_s1, input_s2))
    print("Result: %s" % s.is_isomorphic(input_s1, input_s2))


numlines = input()

valsdict = {}

for i in range(int(numlines)):
    line = input()
    if (line in valsdict):
        valsdict[line]+=1
    else:
        valsdict[line]=1
print(len(valsdict))

summary = ''
for count in valsdict.values():
    summary += str(count) + ' '
print(summary[:-1])


# dump roms in binary and octal format
# import io

# rom_in: str = input('Select rom binary: ')
rom_in = "1816-0420.bin"

print(rom_in)

# raw = io.BytesIO()
with open("rom_in","rb") as rom_bin:
    rom_out = rom_bin.read()
    print(rom_out)


import pygame
from pygame import Surface
import pyautogui
WIN_WIDTH, WIN_HEIGHT = pyautogui.size()[0], pyautogui.size()[1]

def write(screen, text, pos_x, pos_y, color, size):
    font = pygame.font.SysFont('data/qhyts___.ttf', size)
    text = font.render(text, 1, color)
    text_y = pos_y - text.get_height() // 2
    screen.blit(text, (pos_x, text_y))
    return

def create_button(screen, size, color, pos):
    new_button = pygame.Rect(pos[0], pos[1], size[0], size[1])
    pygame.draw.rect(screen, color, new_button, 3)

def blurSurf(surface, amt):
    """
    Blur the given surface by the given 'amount'.  Only values 1 and greater
    are valid.  Value 1 = no blur.
    """
    if amt < 1.0:
        raise ValueError("Arg 'amt' must be greater than 1.0, passed in value is %s"%amt)
    scale = 1.0/float(amt)
    surf_size = surface.get_size()
    scale_size = (int(surf_size[0]*scale), int(surf_size[1]*scale))
    surf = pygame.transform.smoothscale(surface, scale_size)
    surf = pygame.transform.smoothscale(surf, surf_size)
    return surf

def menu_pause(screen, screenshot):

    half_w = WIN_WIDTH // 2
    pause_text = ["Pause", "Continue", "Quit to menu"]

    blur_surf = Surface((WIN_WIDTH, WIN_HEIGHT), pygame.SRCALPHA)
    blur_surf.blit(screenshot, (0, 0))
    new_serf = blurSurf(blur_surf, 20)
    screen.blit(new_serf, (0, 0))

    pygame.draw.line(screen, (78, 37, 245), [0, 110], [WIN_WIDTH, 110], 3)

    # image = pygame.image.load('data/pause_gradient.png').convert_alpha()
    # screen.blit(image, (0, 0))

    wasd = pygame.image.load('data/pause/wasd.png').convert_alpha()
    wasd = pygame.transform.scale(wasd, (100, 100))
    screen.blit(wasd, (WIN_WIDTH - 450, WIN_HEIGHT - 100))
    write(screen, '- move', WIN_WIDTH - 330, WIN_HEIGHT - 50, (255, 255, 255), 50)

    shoot = pygame.image.load('data/pause/лкм.png').convert_alpha()
    shoot = pygame.transform.scale(shoot, (110, 80))
    screen.blit(shoot, (WIN_WIDTH - 230, WIN_HEIGHT - 100))
    write(screen, '- shoot', WIN_WIDTH - 130, WIN_HEIGHT - 50, (255, 255, 255), 50)

    pos_x = 30

    create_button(screen, (250, 50), (78, 37, 245), (pos_x, 30))
    write(screen, pause_text[2], 43, 55, (255, 255, 255), 50)

    pos_x += 260

    create_button(screen, (200, 50), (78, 37, 245), (pos_x, 30))
    write(screen, pause_text[1], pos_x + 22, 55, (255, 255, 255), 50)

def main_menu(screen, bg):
    # blur_surf = Surface((WIN_WIDTH, WIN_HEIGHT), pygame.SRCALPHA)
    # blur_surf.blit(bg, (0, 0))
    # new_serf = blurSurf(blur_surf, 200)
    # screen.blit(new_serf, (0, 0))

    main_serf_width = 500
    main_serf_height = 100

    menu_text = ['New game', 'Load', 'Quit', 'v1.36']

    pos_x = 200
    pos_y = 400
    create_button(screen, (300, 50), (78, 37, 245), (pos_x, pos_y))
    write(screen, menu_text[0], pos_x + 13, pos_y + 27, (78, 37, 245), 50)

    pos_y += 60
    create_button(screen, (300, 50), (78, 37, 245), (pos_x, pos_y))
    write(screen, menu_text[1], pos_x + 13, pos_y + 27, (78, 37, 245), 50)

    pos_y += 60
    create_button(screen, (300, 50), (78, 37, 245), (pos_x, pos_y))
    write(screen, menu_text[2], pos_x + 13, pos_y + 27, (78, 37, 245), 50)

    write(screen, menu_text[3], pos_x + 10, pos_y + 220, (78, 37, 245), 30)


from datasets import load_dataset
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoTokenizer, TrainingArguments
from peft import LoraConfig, get_peft_model

dataset_name = 'LinhDuong/chatdoctor-200k'
dataset = load_dataset(dataset_name, split="train")

model_name = "/home/asif/llm/Llama-2-7b-chat-hf"

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.float16,
)

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    quantization_config=bnb_config,
    # load_in_8bit=True,
    trust_remote_code=True
)
model.config.use_cache = False

tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
tokenizer.pad_token = tokenizer.eos_token

lora_alpha = 16
lora_dropout = 0.5
lora_r = 32

peft_config = LoraConfig(
    lora_alpha=lora_alpha,
    lora_dropout=lora_dropout,
    r=lora_r,
    bias="none",
    task_type="CAUSAL_LM"
)

def formatting_prompts_func(example):
    output_texts = []
    for i in range(len(example['input'])):
        text = f"### User: {example['input'][i]}\n ### Chatbot: {example['output'][i]}"
        output_texts.append(text)
    return output_texts

output_dir = "./results_llama2_chat"
per_device_train_batch_size = 4
per_device_eval_batch_size = 4
gradient_accumulation_steps = 8
optim = "paged_adamw_32bit"
# optim="adamw_torch"
save_strategy = "steps"
save_steps = 500
logging_steps = 1
learning_rate = 2e-5
weight_decay = 0.
max_grad_norm = 0.3
max_steps = 10000
warmup_ratio = 0.03
lr_scheduler_type = "cosine"

training_arguments = TrainingArguments(
    output_dir=output_dir,
    per_device_train_batch_size=per_device_train_batch_size,
    gradient_accumulation_steps=gradient_accumulation_steps,
    optim=optim,
    save_strategy=save_strategy,
    weight_decay=weight_decay,
    save_steps=save_steps,
    logging_steps=logging_steps,
    learning_rate=learning_rate,
    fp16=True,
    max_grad_norm=max_grad_norm,
    max_steps=max_steps,
    warmup_ratio=warmup_ratio,
    group_by_length=True,
    lr_scheduler_type=lr_scheduler_type,
)

from trl import SFTTrainer

max_seq_length = 512

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    peft_config=peft_config,
    # dataset_text_field="text",
    formatting_func=formatting_prompts_func,
    max_seq_length=max_seq_length,
    tokenizer=tokenizer,
    args=training_arguments,
)

for name, module in trainer.model.named_modules():
    if "norm" in name:
        module = module.to(torch.float32)

trainer.train()

model_to_save = trainer.model.module if hasattr(trainer.model, 'module') else trainer.model
model_to_save.save_pretrained("llama-2-chat-medical")


#!/usr/bin/env python3

import arrow
import xml.etree.ElementTree as ET
import os
import sys

for directory in sys.argv[1:]:

    for file in os.listdir(os.fsencode(directory)):
        filename = os.path.join(directory, os.fsdecode(file))
        if not filename.endswith(".xml"):
            continue

        # Reciept timestamp, from filename
        basename = os.path.basename(filename)
        timestamp = basename[:10]
        our_timestamp = arrow.get(timestamp)

        tree = ET.parse(filename)
        root=tree.getroot()

        service_delivery = root.find('{http://www.siri.org.uk/siri}ServiceDelivery')
        if service_delivery is None:
            continue
        vehicle_monitoring_delivery = service_delivery.find('{http://www.siri.org.uk/siri}VehicleMonitoringDelivery')
        if vehicle_monitoring_delivery is None:
            continue
        for element in vehicle_monitoring_delivery.findall(
            '{http://www.siri.org.uk/siri}VehicleActivity/' +
            '{http://www.siri.org.uk/siri}RecordedAtTime'):
            timestamp = element.text
            vehicle_activity_timestamp = arrow.get(timestamp)
            delta = (our_timestamp - vehicle_activity_timestamp).total_seconds()
            print(delta)


from django.shortcuts import render
from .models import Random_Category, Random_Detail, Quote
import random

def index(request):
    categorys = Random_Category.objects.all()
    details = Random_Detail.objects.order_by('category','?').distinct('category') #shuffle and one of each category

    total_random_query = Random_Detail.objects.order_by('?')
    random_number = random.randint(0, len(total_random_query)-1)
    total_random_detail = total_random_query[random_number]
    print(total_random_detail)

    quote_random = Quote.objects.order_by('?')
    random_quote_number = random.randint(0, len(quote_random)-1)
    quote = quote_random[random_quote_number]

    context = {
        'categorys':categorys,
        'details':details,
        'total_random_detail':total_random_detail ,
        'quote': quote
    }
    return render(request, 'random_category/index.html' ,context)


import numpy as np

def __read_dep_graph(string):
    parts = string.strip().split(' ')
    return np.array([float(x) for x in parts])

def __write_dep_graph(graph):
    return ' '.join([str(x) for x in graph])

'''
Defines the stanford and universal POS-tag sets:
'''

stanford_pos=['TO', 'VBG', 'HYPH', 'VBP', "\'\'", 'JJR', 'NNPS', 'AFX', 'POS', ',', 'VB', 'RBR', 'WDT', 'WRB', 'RBS', 'NFP', 'WP', 'WP$', 'XX', 'MD', '-RRB-', 'CC', 'FW', 'JJ', 'VBZ', 'GW', 'VBD', 'UH', '-LRB-', 'RP', 'LS', '.', '``', 'ADD', 'NNP', '$', 'JJS', 'IN', ':', 'CD', 'NNS', 'VBN', 'DT', 'PDT', 'EX', 'PRP$', 'SYM', 'RB', 'PRP', 'NN']

universal_pos=['NUM', 'DET', 'PUNCT', 'ADJ', 'X', 'SYM', 'AUX', 'NOUN', 'VERB', 'PRON', 'ADV', 'ADP', 'CONJ', 'INTJ', 'SCONJ', 'PROPN', 'PART']

'''
Defines the conll sentence file structure.
Terminology is (header name, function to read element, function to write element).
'''
conll_column_headers = [('id', int, str),
                        ('token', str, str),
                        ('lemma', str, str),
                        ('universal_pos', str, str),
                        ('stanford_pos', str, str),
                        ('features', str, str),
                        ('dependency_head_id', int, str),
                        ('dependency_head_relation', str, str),
                        ('weird_underscore', str, str),
                        ('space_after', str, str),
                        ('dependency_graph', __read_dep_graph, __write_dep_graph)]


import os,sqlite3,discord
from discord.ext import commands
from model import model
model.load()

intents = discord.Intents().all()
bot = commands.Bot(command_prefix='?',intents=intents)

global maxWarnings
global punishment
maxWarnings = 5
punishment = "kick"

#events
@bot.event
async def on_ready():
  print("Bot is ready")

@bot.event
async def on_guild_join(guild):
  print (f"Bot deployed to server:{guild} ")
  conn = sqlite3.connect(guild.name+"-database.db")
  cursor = conn.cursor()
  cursor.execute("CREATE TABLE IF NOT EXISTS warnings (name TEXT, amount_of_warnings INTEGER)")

  params = [(member.name+"#"+member.discriminator, 0) for member in guild.members]
  cursor.executemany("INSERT INTO warnings VALUES(?,?)", params)
  conn.commit()
  conn.close()

@bot.event
async def on_message(message):
  if message.author == bot.user:
    return

  await bot.process_commands(message)

  global maxWarnings
  global punishment

  if model.predict(message.content)=="__label__0":
    author = str(message.author)
    await message.delete()
    await message.channel.send("Be nice "+message.author.mention)

    # database update stuff
    conn = sqlite3.connect(message.guild.name+"-database.db")
    cursor = conn.cursor()
    warnings = cursor.execute("SELECT amount_of_warnings FROM warnings WHERE name = ?",(author,)).fetchall()[0][0]
    if warnings >= maxWarnings-1:

      if punishment =="ban":
        await message.author.ban(reason="Be nice, if you think this is a mistake, contact the moderator team")
        cursor.execute("UPDATE warnings SET amount_of_warnings = 0 WHERE name = ?",(author,))

      else:
        await message.author.kick()
        cursor.execute("UPDATE warnings SET amount_of_warnings = 0 WHERE name = ?",(author,))

    else:
      cursor.execute("UPDATE warnings SET amount_of_warnings = ? WHERE name = ?",(warnings+1, author))
      await message.channel.send(message.author.mention+", you've been warned. Amount of warnings:"+str(warnings+1))
    conn.commit()
    conn.close()

#commands
@bot.command()
async def viewWarnings(ctx):
  print(f"User {ctx.author} has requested to view warnings.")

  conn = sqlite3.connect(ctx.guild.name+"-database.db")
  cursor = conn.cursor()
  amountOfWarnings = cursor.execute("SELECT amount_of_warnings FROM warnings WHERE name = ?",(str(ctx.author),)).fetchall()[0][0]

  print(f"User {ctx.author} request to view warnings was fulfilled.")

  await ctx.channel.send(ctx.author.mention+", Your amount of warnings is: "+str(amountOfWarnings))

  conn.close()

@bot.command()
async def toggle(ctx, newMaxWarnings=5, newPunishment="kick"):
  global maxWarnings
  global punishment

  if type(newMaxWarnings)!=int:
    await ctx.channel.send(ctx.author.mention+", You cannot set the max warnings to anything other than an integer!")
    return

  if newPunishment!="kick" and newPunishment!="ban":
    await ctx.channel.send(ctx.author.mention+", You cannot set the punishment to anything other than ban or kick!")
    return

  maxWarnings = newMaxWarnings
  punishment = newPunishment

  await ctx.channel.send(ctx.author.mention+", You've changed my mind.")

bot.run(os.environ['DISCORD TOKEN'])


T = int(input())
for t in range(1, T+1):
    p, q = map(float, input().split())
    answer = 'NO'

    s1, s2 = (1-p) * q, p * (1-q) * q

    if s1 < s2:
        answer = 'YES'

    print(f'#{t} {answer}')

SW Expert Academy/Python/Python D3/9700. USB 꽂기의 미스터리.py


import os
import json
# Imports the Google Cloud client library
from google.cloud import vision

def run_quickstart():

    # Instantiates a client
    client = vision.ImageAnnotatorClient()

    # The name of the image file to annotate
    file_name = os.path.abspath("files/compressed-gas.jpg")

    # Loads the image into memory
    with open(file_name, "rb") as image_file:
        content = image_file.read()

    image = vision.Image(content=content)

    # Performs label detection on the image file
    response = client.text_detection(image=image)
    text = json.loads(type(response).to_json(response))
    # text = MessageToJson(data)
    print(text["textAnnotations"][0]["description"])

if __name__ == "__main__":
    run_quickstart()


from __future__ import annotations

import hikari
from typing_extensions import Annotated as Atd

import crescent

bot = hikari.GatewayBot(token="...")
client = crescent.Client(bot)

class RandomError(Exception):
    pass

class UnhandledError(Exception):
    pass

# error handling
@client.include
@crescent.catch_command(RandomError)
async def on_cmd_random_error(exc: RandomError, ctx: crescent.Context) -> None:
    await ctx.respond(f"{exc} raised in {ctx.command}!")

@client.include
@crescent.catch_event(RandomError)
async def on_event_random_error(exc: RandomError, event: hikari.Event) -> None:
    print(f"{exc} raised in {event}!")

@client.include
@crescent.catch_autocomplete(RandomError)
async def on_autocomplete_random_error(
    exc: RandomError,
    ctx: crescent.AutocompleteContext,
    inter: hikari.AutocompleteInteractionOption,
) -> None:
    print(f"{exc} raised in autocomplete for {ctx.command}!")

# buggy command/event/autocompletes
@client.include
@crescent.command
async def raise_error_cmd(ctx: crescent.Context, unhandled: bool) -> None:
    if unhandled:
        raise UnhandledError("Unhandled error!")
    raise RandomError("Handled error")

@client.include
@crescent.event
async def raise_error_event(event: hikari.MessageCreateEvent) -> None:
    if event.author.is_bot:
        return
    if event.message.content is None:
        return

    if event.message.content == "!unhandled":
        raise UnhandledError("Unhandled error!")
    elif event.message.content == "!error":
        raise RandomError("Handled error")
    elif event.message.content.startswith("!"):
        await event.message.respond("Use !unhandled or !error")

async def autocomplete(
    ctx: crescent.AutocompleteContext, option: hikari.AutocompleteInteractionOption
) -> list[tuple[str, str]]:
    assert isinstance(option.value, str)
    if option.value == "unhandled":
        raise UnhandledError("Unhandled error!")
    elif option.value == "error":
        raise RandomError("Handled error")

    # returns a list of tuples of (option name, option value).
    return [("error", "error"), ("unhandled", "unhandled")]

@client.include
@crescent.command
async def autocomplete_error(
    ctx: crescent.Context,
    option: Atd[str, "Type error to error out", crescent.Autocomplete(autocomplete)],
) -> None:
    await ctx.respond(f"{option} (type unhandled or error inside option)")

bot.run()


from django.http import JsonResponse
from django.utils.decorators import method_decorator
from django.views import View
from django.views.decorators.csrf import csrf_exempt
from .import models
import json

# Create your views here.

class CompanyView(View):

    @method_decorator(csrf_exempt)
    def dispatch(self, request, *args, **kwargs):
        return super().dispatch(request, *args, **kwargs)

    def get(self, request , id=0):
        if id > 0:
            companies = list(models.Company.objects.filter(id = id).values())
            company = companies[0]
            context = {
                    'message':'success',
                    'data' : company
                }
            return JsonResponse(context)

        else:
            companies = list(models.Company.objects.values())
            if len(companies)>0:
                context = {
                    'message':'success',
                    'companies' : companies
                }
            else:
                context = {
                    'message':'not found'
                }
            return JsonResponse(context)

    def post(self, request):
        jd= json.loads(request.body)
        # print(jd)
        models.Company.objects.create(name=jd['name'], website=jd['website'], foundation=jd['foundation'])
        context = {
            'message': 'success'
        }
        return JsonResponse(context)

    def put(self, request ,id ):
        jd= json.loads(request.body)
        c = list(models.Company.objects.filter(id = id).values())
        if len(c) > 0:
            c = models.Company.objects.get(id= id)
            c.name=jd['name']
            c.website=jd['website']
            c.foundation=jd['foundation']
            c.save()
            context = {
                    'message':'Success'
                }
            return JsonResponse(context)
        else:
            context = {
                    'message':'not fail'
                }
        return JsonResponse(context)

    def delete(self, request ,id):
        compani = models.Company.objects.get(id= id)
        compani.delete()
        context = {
                    'message':'deleted'
                }

        return JsonResponse(context)


def main(m, n, u, l, r, d):
    out = ""
    for i in range(u):
        out += "#." * (l + n + r) + "#\n"
    for i in range(m):
        out += "#." + ".".join(input()) + ".#\n"
    for i in range(d):
        out += "#." * (l + n + r) + "#\n"
    print(out)

if __name__ == "__main__":
    m, n = map(int, input().split())
    u, l, r, d = map(int, input().split())
    main(m, n, u, l, r, d)


#!/usr/bin/python3
"""my github id haha"""

import sys
import requests

if __name__ == '__main__':
    user = sys.argv[1]
    token = sys.argv[2]
    url = 'https://api.github.com/user'

    login = requests.get(url, auth=(user, token))
    login_data = login.json()
    print(login_data.get('id'))


from ..serialiser import SerialiserBase

import os

try:
    # noinspection PyPep8Naming
    import cPickle as pickle

except ImportError:
    import pickle

# ------------------------------------------------------------------------------
class PickleSerialiser(SerialiserBase):
    """
    This will serialise the memento object to a json file and store it in
    an application data location (varies depending on platform). You can
    utilise the identifier as a sub-path to create namespaced serialisation
    objects.

    For instance:
        serialiser.serialise(memento, 'foo/bar'

    would (on windows) result in:
        %APPDATA%/memento/foo/bar.json

    """

    # --------------------------------------------------------------------------
    @staticmethod
    def serialise(data, identifier):
        """
        The identifier for this serialiser should be the absolute path
        to the file you want to write to

        :param data: dict
        :param identifier: str

        :return:
        """
        directory = os.path.dirname(identifier)

        if not os.path.exists(directory):
            os.makedirs(directory)

        with open(identifier, 'wb') as f:
            pickle.dump(
                data,
                f,
            )

    # --------------------------------------------------------------------------
    @staticmethod
    def deserialise(identifier):
        """
        The identifier for this serialiser should be the absolute path
        to the file you want to write to

        :param identifier: str

        :return:
        """
        if not os.path.exists(identifier):
            return None

        with open(identifier, 'rb') as f:
            return pickle.load(f)

    # --------------------------------------------------------------------------
    @staticmethod
    def locator(identifier):
        """
        With the given identifier this should return the resolved location
        of the data

        :param identifier: This is very dependent on the requirements of
            each individual serialiser. This could be a filepath, a url,
            a name etc.
        :type identifier: str

        :return: str
        """
        return identifier


from django.shortcuts import render
from django.http import HttpResponse
from .models import User

# Create your views here.
def showUsers(request):
    user = User.objects.all()
    context = {
        'user':user,
        'count':user.count,
    }

    return render(request, 'myapp/users.html', context)


# Thomas Karr
# Wesley Benica
# Lab 4 - CSC369 - Spring 2020

import argparse

from config import Configuration
from output_graph import create_graph
from output_html import get_header
from output_table import get_table, create_table, Query
from pipeline import create_pipeline
from update_data import get_db_connection, update_collections

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-auth', type=str, required=False, default='credentials.json')
    parser.add_argument('-config', type=str, required=False, default='trackerConfig.json')
    args = parser.parse_args()

    try:
        test_config = Configuration(args.config)
    except ValueError as err:
        print(f'{type(err).__name__}: {err}')
        return

    db = get_db_connection(args.auth)
    update_collections(db, test_config.refresh)

    collection = db[test_config.collection]
    pipeline = create_pipeline(test_config)

    with open('pipeline.json', 'w') as f:
        print(pipeline, file=f)

    result = collection.aggregate(pipeline).next()

    page = get_header()
    for n, t in enumerate(result):
        if test_config.analysis[n]['task'].get('stats') is not None:
            test_config.analysis[n]['task'].update({"aggregation": test_config.aggregation})
            q = Query(
                task=test_config.analysis[n]['task'],
                output=test_config.analysis[n]['output'],
                data={"data": result[t]})
            page += create_table(q)
        else:
            q = Query(
                task=test_config.analysis[n]['task'],
                output=test_config.analysis[n]['output'],
                data=result[t][0])

        for key in q.output:
            if ('track' in q.task or 'ratio' in q.task) and key == 'table':
                page += get_table(q)
            if ('track' in q.task or 'ratio' in q.task) and key == 'graph':
                create_graph(q, n)
                page += f''

    with open(test_config.output_file, 'w') as f:
        f.write(page)

    print("done")

if __name__ == '__main__':
    main()


from . import WallpaperProvider as _WProv
from os import environ as _environ, system as _system
from shutil import which as _which
from sys import exit as _exit

class I3wmAdapter(_WProv):
    def change_wallpaper(path: str):
        file = _which("feh")
        if file is None:
            # I did the checking here not to seem we don't support i3
            print("feh not found, you need it to set the background in i3.")
            _exit(0)
        _system(f'feh --bg-fill "{path}"')

    def is_compatible() -> bool:
        return 'i3' in _environ.get('XDG_CURRENT_DESKTOP', '') and \
            'i3' in _environ.get('XDG_SESSION_DESKTOP', '')


import math

def is_bissextile(y):
    return ( y % 4 == 0 and y % 100 != 0 ) or ( y % 400 == 0 )

assert is_bissextile(2004)

def number_of_days(year, month):
    """Returns the number of days of the specified month"""
    if month in (1, 3, 5, 7, 8, 10, 12):
        return 31
    elif month == 2:
        return 29 if is_bissextile(year) else 28
    else:
        return 30

assert number_of_days(2020, 2) == 29
assert number_of_days(2019, 2) == 28
assert number_of_days(2019, 9) == 30
assert number_of_days(2019, 12) == 31

def is_valid_date(y, m, d):
    return y >= 0 \
        and ( 1 <= m <= 12 ) \
        and ( 1 <= d <= number_of_days(y, m) )

assert is_valid_date(*(2020, 9, 23))
assert not is_valid_date(*(2015, 29, 2))
assert not is_valid_date(*(2015, 2, 29))

"""
# lazy version
def days_between(y, m, d, Y, M, D):
    # calulate number of days from 0000/00/00 to y/m/d
    days1 = sum([366 if is_bissextile(yy) else 365 for yy in range(y)]) \
        + sum([number_of_days(y, mm) for mm in range(1, m)]) + d

    # calulate number of days from 0000/00/00 to Y/M/D
    days2 = sum([366 if is_bissextile(YY) else 365 for YY in range(Y)]) \
        + sum([number_of_days(Y, MM) for MM in range(1, M)]) + D

    # then substract
    return days2 - days1
"""

def days_between(y, m, d, Y, M, D):
    def nb_days_of_year(y):
        return 366 if is_bissextile(y) else 365

    def nb_days_from_new_year(y, m, d):
        return sum([number_of_days(y, mm) for mm in range(1, m)]) + d

    def days_between_positive_year(y, m, d, Y, M, D):
        # y must be < or = Y
        return sum([nb_days_of_year(yy) for yy in range(y, Y)]) \
            - nb_days_from_new_year(y, m, d) \
            + nb_days_from_new_year(Y, M, D)

    if y <= Y:
        return days_between_positive_year(y, m, d, Y, M, D)
    else:
        return -days_between_positive_year(Y, M, D, y, m, d)

assert (days_between(1985,10,21, 1985,10,21) == 0)
assert (days_between(1985,10,20, 1985,10,21) == 1)
assert (days_between(1985,10,21, 1985,10,20) == -1)
assert (days_between(1985,10,21, 2017,9,19) == 11656)
assert (days_between(2017,9,19, 1985,10,21) == -11656)
assert (days_between(1999,12, 5, 2000,3,1) == 87)

def weekday(y, m, d):
    days = days_between(1900,1,1, y,m,d)
    return (days+1) % 7

assert weekday(1900,1,1) == 1
assert weekday(1985,10,21) == 1
assert weekday(2017,9,19) == 2
assert weekday(1899,12,31) == 0
assert weekday(1700,1,1) == 5
assert weekday(2019,9,14) == 6

def cal(y, m, weekday=weekday):
    def print_cell(s):
        print('{0: >3}'.format(s), end='')

    months = [
        '  Janvier',
        '  Février',
        '     Mars',
        '    Avril',
        '      Mai',
        '     Juin',
        '  Juillet',
        '     Août',
        'Septembre',
        '  Octobre',
        ' Novembre',
        ' Décembre'
    ]

    print(' '*3, months[m-1], y)
    print(' di lu ma me je ve sa')

    first_day_of_week = weekday(y, m, 1)
    nb_days = number_of_days(y, m)
    current_day = 1
    for y in range(6):
        for x in range(7):
            if (y == 0 and x < first_day_of_week) or current_day > nb_days:
                print_cell('')
            else:
                print_cell(current_day)
                current_day = current_day + 1
        print()

cal(2020,9)

def weekday_delambre(y, m, d):
    def m_bar(m, y):
        m_year_norm = [4, 0, 0, 3, 5, 1, 3, 6, 2, 4, 0, 2]
        m_year_leap = [3, 6, 0, 3, 5, 1, 3, 6, 2, 4, 0, 2]
        return (m_year_leap if is_bissextile(y) else m_year_norm)[m-1]

    def y_pp(y):
        return y // 100

    def y_p(y):
        return y % 100

    s = d + m_bar(m, y) + (21 * y_pp(y)) // 4 + (5 * y_p(y)) // 4 + 2
    return s % 7

assert weekday_delambre(1985, 10, 21) == 1
assert weekday_delambre(2020, 9, 29) == 2

cal(2020,9,weekday=weekday_delambre)

APPROX_DAYS_PER_YEAR = 365.2425
def days_between_approx(y,m,d,Y,M,D):
    #return (D + M*APPROX_DAYS_PER_YEAR/12 + Y*APPROX_DAYS_PER_YEAR) - (d + m*APPROX_DAYS_PER_YEAR/12 + y*APPROX_DAYS_PER_YEAR)
    return (D-d) + (M-m)*APPROX_DAYS_PER_YEAR/12 + (Y-y)*APPROX_DAYS_PER_YEAR

def isalmost(n, m, d=1):
	return abs(n - m) <= d

def approxrat(*p):
    ex = days_between(*p)
    ap = days_between_approx(*p)
    return 0 if ex == 0 else ap/ex

assert isalmost(approxrat(1985,10,21,2020,9,19) , 1 , 0.0001)
assert isalmost(approxrat(1800,1,1, 2100,1,1)   , 1 , 0.000005)
print(approxrat(2019,2,28,2019,3,1))

'''
36.10 Effects of approximations onweekday
'''

def weekday_approx(y, m, d):
    days = int(days_between_approx(1900,1,1,y,m,d))
    return (days+1) % 7

'''assert weekday_approx(1900,1,1) == 1
assert weekday_approx(1985,10,21) == 1
assert weekday_approx(2017,9,19) == 2
assert weekday_approx(1899,12,31) == 0
assert weekday_approx(1700,1,1) == 5
assert weekday_approx(2019,9,14) == 6'''

def daysgen(y,m,d,Y,M,D,show_weekday=False):
    date = (y,m,d)
    while date != (Y,M,D):
        yield (date, weekday(*date)) if show_weekday else date
        (_y,_m,_d) = date
        if _m == 12 and _d == 31:
            date = (_y+1,1,1)
        elif _d == number_of_days(_y, _m):
            date = (_y,_m+1,1)
        else:
            date = (_y,_m,_d+1)

assert next(daysgen(1899,12,31,1900,1,4)) == (1899, 12, 31)
assert list(daysgen(1899,12,31,1900,1,4)) == \
    [(1899, 12, 31), (1900, 1, 1), (1900, 1, 2), (1900, 1, 3)]
assert list(daysgen(2020,2,28,2020,3,2)) == \
    [(2020, 2, 28), (2020, 2, 29), (2020, 3, 1)]
assert list(daysgen(2019,2,28,2019,3,2)) == \
    [(2019, 2, 28), (2019, 3, 1)]
assert sum( 1 for _ in daysgen(1985,10,21, 2017,9,19)) == 11656

from datetime import date
#assert all( (date(*t).weekday()+1)%7 == weekday(*t)for t in daysgen(1800,1,1, 2100,1,1) )

assert list(daysgen(1899,12,31,1900,1,4,True)) == \
    [((1899, 12, 31), 0), ((1900, 1, 1), 1),((1900, 1, 2), 2), ((1900, 1, 3), 3)]

assert list(daysgen(1899,12,31,1905,1,4,True)) == \
    [ (t,weekday(*t))for t in daysgen(1899,12,31,1905,1,4) ]

def approxdayrat(*p):
    N = days_between(*p)
    n = sum( 1 for (t,d) in daysgen(*p,True) if d == weekday_approx(*t) )
    #print(n,N,n/N)
    return n/N

assert isalmost( approxdayrat(1985,10,21, 2017,9,19), .49, .001 )
assert isalmost( approxdayrat(1800,1,1, 2100,1,1),     .35, .01 )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Nov  6 20:04:30 2020

@author: TH
"""
A = [int(i) for i in input().split(' ')]
B = [int(i) for i in input().split(' ')]

A_n = A[0]
B_n = B[0]

A = A[1:]
B = B[1:]

A = sorted(A)
B = sorted(B)
ia = 0
ib = 0
diff = abs(A[ia] - B[ib])
while True:
    if A[ia] > B[ib]:
        ib += 1
        diff = min(diff, abs(A[ia] - B[ib]))
        if ib == B_n - 1:
            break
        else:
            continue
    elif A[ia] < B[ib]:
        ia += 1
        diff = min(diff, abs(A[ia] - B[ib]))
        if ia == A_n - 1:
            break
        else:
            continue
    else:
        diff = 0
        break

print (diff)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Author: Zhichao Ouyang
# Time: 2021/3/11 16:19

import javalang
import json
from tqdm import tqdm
import collections
from graphviz import Digraph
import sys
import lib
import argparse
import codecs
import ast, asttokens
import optparse
import sys
import os
import copy

NODE_FIX = '1*NODEFIX'#'1*NODEFIX'

def python2tree(line):
    atok = asttokens.ASTTokens(line, parse=True)
    return atok, atok.tree

def traverse_python_tree(atok, root):
    iter_children = asttokens.util.iter_children_func(root)
    node_json = {}
    current_global = {}
    current_idx, global_idx = 1, 1
    for node in asttokens.util.walk(root):
        if not next(iter_children(node), None) is None:
            child_num = 0
            for child in iter_children(node):
                child_num += 1
            global_idx = global_idx + child_num
            current_global[current_idx] = global_idx
        current_idx += 1
    # print current_global
    current_idx = 1
    for node in asttokens.util.walk(root):
        # print current_idx
        # idx_upper = current_idx
        node_json["%s%s" % (NODE_FIX, current_idx)] = {"node": type(node).__name__, "children": [],
                                                                 "parent": None}
        # idx_upper = len(node_json)
        if not next(iter_children(node), None) is None:
            child_idx = 0
            for child in iter_children(node):
                child_idx += 1
                node_json["%s%s" % (NODE_FIX, current_idx)]['children'].insert(0, "%s%s" % (
                NODE_FIX, current_global[current_idx] - child_idx + 1))
        else:  # leaf node
            node_json["%s%s" % (NODE_FIX, current_idx)]['children'].append(atok.get_text(node))

        current_idx += 1

    # update_parent
    for k, node in node_json.items():
        children = [c for c in node['children'] if c.startswith(NODE_FIX)]
        if len(children):
            for c in children:
                node_json[c]['parent'] = k

    return node_json

def process_source(file_name, save_file):  # referred from EMSE-DeepCom
    with open(file_name, 'r', encoding='utf-8') as source:
        lines = source.readlines()
    with open(save_file, 'w+', encoding='utf-8') as save:
        for line in lines:
            code = line.strip()
            tokens = list(javalang.tokenizer.tokenize(code))
            tks = []
            for tk in tokens:
                if tk.__class__.__name__ == 'String' or tk.__class__.__name__ == 'Character':
                    tks.append('STR_')
                elif 'Integer' in tk.__class__.__name__ or 'FloatingPoint' in tk.__class__.__name__:
                    tks.append('NUM_')
                elif tk.__class__.__name__ == 'Boolean':
                    tks.append('BOOL_')
                else:
                    tks.append(tk.value)
            save.write(" ".join(tks) + '\n')

def get_ast(file_name, w):  # referred from EMSE-DeepCom
    with open(file_name, 'r', encoding='utf-8') as f:
        lines = f.readlines()
    with open(w, 'w+', encoding='utf-8') as wf:
        ign_cnt = 0
        for line in tqdm(lines):
            code = line.strip()
            tokens = javalang.tokenizer.tokenize(code)
            token_list = list(javalang.tokenizer.tokenize(code))
            length = len(token_list)
            parser = javalang.parser.Parser(tokens)
            try:
                tree = parser.parse_member_declaration()
            except (javalang.parser.JavaSyntaxError, IndexError, StopIteration, TypeError):
                print('Error')
                continue
            flatten = []
            for path, node in tree:
                flatten.append({'path': path, 'node': node})

            ign = False
            outputs = []
            stop = False
            for i, Node in enumerate(flatten):
                d = collections.OrderedDict()
                path = Node['path']
                node = Node['node']
                children = []
                for child in node.children:
                    child_path = None
                    if isinstance(child, javalang.ast.Node):
                        child_path = path + tuple((node,))
                        for j in range(i + 1, len(flatten)):
                            if child_path == flatten[j]['path'] and child == flatten[j]['node']:
                                children.append(j)
                    if isinstance(child, list) and child:
                        child_path = path + (node, child)
                        for j in range(i + 1, len(flatten)):
                            if child_path == flatten[j]['path']:
                                children.append(j)
                d["id"] = i
                d["type"] = str(node.__class__.__name__)
                if children:
                    d["children"] = children
                value = None
                if hasattr(node, 'name'):
                    value = node.name
                elif hasattr(node, 'value'):
                    value = node.value
                elif hasattr(node, 'position') and node.position:
                    for i, token in enumerate(token_list):
                        if node.position == token.position:
                            pos = i + 1
                            value = str(token.value)
                            while (pos < length and token_list[pos].value == '.'):
                                value = value + '.' + token_list[pos + 1].value
                                pos += 2
                            break
                elif type(node) is javalang.tree.This \
                        or type(node) is javalang.tree.ExplicitConstructorInvocation:
                    value = 'this'
                elif type(node) is javalang.tree.BreakStatement:
                    value = 'break'
                elif type(node) is javalang.tree.ContinueStatement:
                    value = 'continue'
                elif type(node) is javalang.tree.TypeArgument:
                    value = str(node.pattern_type)
                elif type(node) is javalang.tree.SuperMethodInvocation \
                        or type(node) is javalang.tree.SuperMemberReference:
                    value = 'super.' + str(node.member)
                elif type(node) is javalang.tree.Statement \
                        or type(node) is javalang.tree.BlockStatement \
                        or type(node) is javalang.tree.ForControl \
                        or type(node) is javalang.tree.ArrayInitializer \
                        or type(node) is javalang.tree.SwitchStatementCase:
                    value = 'None'
                elif type(node) is javalang.tree.VoidClassReference:
                    value = 'void.class'
                elif type(node) is javalang.tree.SuperConstructorInvocation:
                    value = 'super'

                if value is not None and type(value) is type('str'):
                    d['value'] = value
                if not children and not value:
                    # print('Leaf has no value!')
                    print(type(node))
                    print(code)
                    ign = True
                    ign_cnt += 1
                    # break
                outputs.append(d)
            if not ign:
                wf.write(json.dumps(outputs))
                wf.write('\n')
    print(ign_cnt)

def build_ast(line):
    code = line.strip()
    tokens = list(javalang.tokenizer.tokenize(code))
    tks = []
    for tk in tokens:
        if tk.__class__.__name__ == 'String' or tk.__class__.__name__ == 'Character':
            tks.append('STR_')
        elif 'Integer' in tk.__class__.__name__ or 'FloatingPoint' in tk.__class__.__name__:
            tks.append('NUM_')
        elif tk.__class__.__name__ == 'Boolean':
            tks.append('BOOL_')
        else:
            tks.append(tk.value)
    line = " ".join(tks)

    code = line.strip()
    tokens = javalang.tokenizer.tokenize(code)
    token_list = list(javalang.tokenizer.tokenize(code))
    length = len(token_list)
    parser = javalang.parser.Parser(tokens)
    try:
        tree = parser.parse_member_declaration()
    except (javalang.parser.JavaSyntaxError, IndexError, StopIteration, TypeError):
        print('Error: ' + line)
        return None
    flatten = []
    for path, node in tree:
        flatten.append({'path': path, 'node': node})
    outputs = []
    for i, Node in enumerate(flatten):
        d = collections.OrderedDict()
        path = Node['path']
        node = Node['node']
        children = []
        for child in node.children:
            if isinstance(child, javalang.ast.Node):
                child_path = path + tuple((node,))
                for j in range(i + 1, len(flatten)):
                    if child_path == flatten[j]['path'] and child == flatten[j]['node']:
                        children.append(j)
            if isinstance(child, list) and child:
                child_path = path + (node, child)
                for j in range(i + 1, len(flatten)):
                    if child_path == flatten[j]['path']:
                        children.append(j)
        d["id"] = i
        d["type"] = str(node.__class__.__name__)
        if children:
            d["children"] = children
        value = None
        if hasattr(node, 'name'):
            value = node.name
        elif hasattr(node, 'value'):
            value = node.value
        elif hasattr(node, 'position') and node.position:
            for i, token in enumerate(token_list):
                if node.position == token.position:
                    pos = i + 1
                    value = str(token.value)
                    while (pos < length and token_list[pos].value == '.'):
                        value = value + '.' + token_list[pos + 1].value
                        pos += 2
                    break
        elif type(node) is javalang.tree.This \
                or type(node) is javalang.tree.ExplicitConstructorInvocation:
            value = 'this'
        elif type(node) is javalang.tree.BreakStatement:
            value = 'break'
        elif type(node) is javalang.tree.ContinueStatement:
            value = 'continue'
        elif type(node) is javalang.tree.TypeArgument:
            value = str(node.pattern_type)
        elif type(node) is javalang.tree.SuperMethodInvocation \
                or type(node) is javalang.tree.SuperMemberReference:
            value = 'super.' + str(node.member)
        elif type(node) is javalang.tree.Statement \
                or type(node) is javalang.tree.BlockStatement \
                or type(node) is javalang.tree.ForControl \
                or type(node) is javalang.tree.ArrayInitializer \
                or type(node) is javalang.tree.SwitchStatementCase:
            value = 'None'
        elif type(node) is javalang.tree.VoidClassReference:
            value = 'void.class'
        elif type(node) is javalang.tree.SuperConstructorInvocation:
            value = 'super'

        if value is not None and type(value) is type('str'):
            d['value'] = value
        if not children and not value:
            # print('Leaf has no value!')
            print(type(node))
            print(code)
            # break
        outputs.append(d)
    return outputs

def gen_all_ast_py(code_split_dir, ast_file_path):

    code_split_list = os.listdir(code_split_dir)
    os.chdir(code_split_dir)
    syntax_error_count = 0
    empty_count = 0
    # ast_file = open(ast_file_path, 'w', encoding='utf-8')

    for f in tqdm(code_split_list):
        try:
            ast_list = []
            file = open(f, encoding='utf-8')
            idx = f.replace('.txt', '')
            # print('当前id:', idx)
            lines = file.readlines()
            if len(lines) == 0:
                line = ''
            else:
                line = "".join(lines)
            code_list = line.split('<sep>')
            head = code_list[0]
            if len(code_list) > 1:
                code_list = code_list[1:]
                # Add method headers to each code segment to generate AST
                for code in code_list:
                    if code.find('except') >= 0:
                        continue
                    code = head + "\n" + code
                    # print(code)
                    code = clean_code_py(code)
                    # print('code2:' + code)
                    atok, tree = python2tree(code)
                    tree_json = traverse_python_tree(atok, tree)
                    # print(tree_json)
                    if tree_json is not None:
                        ast_list.append(tree_json)
                    else:
                        syntax_error_count += 1
            if len(ast_list) == 0:
                empty_count += 1
                continue
            ast_file = open(ast_file_path + idx + '.json', 'w', encoding='utf-8')
            ast_file.write(json.dumps(ast_list))
            ast_file.close()

            file.close()
        except Exception:
            print(f)
            pass
        continue
        print('syntax_error_count: ' + str(syntax_error_count))
        print('empty_count: ' + str(empty_count))

def clean_code(code):
    code = code.replace('\n', '')
    code = code.replace('default:', '')
    if code.find('throw') >= 0:
        return ''
    while code.find('case') >= 0:
        # code = code[code.rfind(':') + 1:]
        code = code[code.find(':') + 1:]
        if code.find(':') < 0:
            break
    if code.find('switch (') == 0 and code.find('{'):
        code = code[code.find('{')+1:code.find('}')]
    if code.find('for') >= 0 or code.find('while') >= 0:
        code += '{}'
    return code

def get_real_arr(arr):
    """
    Return arr after removing all null values
    """
    arr_copy = copy.deepcopy(arr)
    arr_copy = list(filter(None, arr_copy))
    while '' in arr_copy:
        arr_copy.remove('')
    return arr_copy

def clean_code_py(code):
    # code = code.replace('\n', '')
    # code = code.replace('default:', '')
    # if code.find('throw') >= 0:
    #     return ''
    # while code.find('case') >= 0:
    #     # code = code[code.rfind(':') + 1:]
    #     code = code[code.find(':') + 1:]
    #     if code.find(':') < 0:
    #         break
    # if code.find('switch (') == 0 and code.find('{'):
    #     code = code[code.find('{')+1:code.find('}')]
    num = 0
    # print('code:' + code)
    code_tmp2 = code.strip()
    tmp = code.split(':')[0] + ':'
    code_tmp = code.replace(tmp, '', 1)
    code_tmp = code_tmp.split('\n')
    code_tmp3 = get_real_arr(code_tmp)
    for c in code_tmp3:
        if c.find('def') >= 0:
            if (c.strip().index('def') == 0) or (c.strip().index('def') == 6):
                code = code.replace(c, '')

    if (code.find('for ') >= 0 and code_tmp2.find(':', code_tmp2.index('for ')) != -1) or (code.find('while ') >= 0 and code_tmp2.find(':', code_tmp2.index('while ')) != -1):

        # Determine if for or while is the last line
        i = 0
        for j in range(len(code_tmp3)):
            i += 1
            if code_tmp3[j].find('for ') >= 0:
                if code_tmp3[j].strip().index('for ') == 0 and j == len(code_tmp3)-1:
                    code_tmp = code_tmp3[j]
                    num = i
                elif code_tmp3[j].strip().index('for ') == 0 and j == len(code_tmp3)-2 and code_tmp3[j+1].find('if') <0 and code_tmp3[j+1].find(':') >= 0 :
                    code_tmp = code_tmp3[j]
                    num = i+1
            if code_tmp3[j].find('while ') >= 0:
                if code_tmp3[j].strip().index('while ') == 0 and j == len(code_tmp3)-1:
                    code_tmp = code_tmp3[j]
                    num = i
                elif code_tmp3[j].strip().index('while ') == 0 and j == len(code_tmp3)-2 and code_tmp3[j+1].find('if') <0 and code_tmp3[j+1].find(':') >= 0:
                    code_tmp = code_tmp3[j]
                    num = i+1

        # print(num)
        # print('after:\n', code)
        # print(len(code_tmp3))
        if num == len(code_tmp3):
            num = 0
            while code_tmp.find('    ') >= 0:
                code_tmp = code_tmp.replace('    ', '', 1)
                num += 1
            code += "    " * (num+1) + 'continue'

    if code_tmp2.find('if') >= 0 and code_tmp2.find(':', code_tmp2.index('if')) != -1:
        # if code_tmp2.index(':', code_tmp2.index('if')) == len(code_tmp2)-1:

        # Determine if is the last line
        i = 0
        for c in code_tmp3:
            i += 1
            if c.find('if') >= 0:
                if c.strip().index('if') == 0:
                    code_tmp = c
                    num = i

        # print('num:', num)
        # print(len(code_tmp3))
        if num == len(code_tmp3):
            num = 0
            while code_tmp.find('    ') >= 0:
                code_tmp = code_tmp.replace('    ', '', 1)
                num += 1
            code += "    " * (num+1) + 'print("")'

    if code_tmp2.find('elif') >= 0 and code_tmp2.find(':', code_tmp2.index('elif')) != -1:
        # if code_tmp2.index(':', code_tmp2.index('elif')) == len(code_tmp2)-1:

        i = 0
        for c in code_tmp3:
            i += 1
            if c.find('elif') >= 0:
                if c.strip().index('elif') == 0:
                    code_tmp = c
                    num = i

        if num == len(code_tmp3):
            num = 0
            while code_tmp.find('    ') >= 0:
                code_tmp = code_tmp.replace('    ', '', 1)
                num += 1

            code += "    " * (num+1) + 'print("")'
        code = code.replace('elif', 'if')
    # print('code3:' + code)

    if code_tmp2.find('else') >= 0 and code_tmp2.find(':', code_tmp2.index('else')) != -1:
        # for c in code_tmp3:
        #     if c.find('else') >= 0:
        #         if c.strip().index('else') == 0:
        #             code_tmp = c
        #
        # if num == len(code_tmp3):
        #     num = 0
        #     while code_tmp.find('    ') >= 0:
        #         code_tmp = code_tmp.replace('    ', '', 1)
        #         num += 1
        #
        #     code += "    " * num + 'print("")'
        code = code.replace('else:', '')
    # print('code3:' + code)

    return code

def clean_head(head):
    first = head[0:head.find('(')-1]
    second = head[head.find('(')-1:]
    words = first.split(' ')
    name = words[-1]
    head = 'void ' + name + second
    return head

def read_ast(ast_file_path):
    ast_file = open(ast_file_path, encoding='utf-8')
    for line in ast_file.readlines():
        ast_list = json.loads(line)
        for ast in ast_list:
            draw_ast(ast)

def draw_ast(ast):
    g = Digraph('AST')
    for node in ast:
        g.node(name=str(node['id']), label=node['type'])
        if 'children' in node.keys():
            for c in node['children']:
                g.edge(str(node['id']), str(c))
    g.view(directory='img/')

def clean_dataset_2(origin_dir, origin_clean_dir, code_split_dir, batch_type):
    code_split_list = os.listdir(code_split_dir)
    print(code_split_list)
    source_file = open(origin_dir + batch_type + '.source', encoding='utf-8')
    code_file = open(origin_dir + batch_type + '.token.code', encoding='utf-8')
    nl_file = open(origin_dir + batch_type + '.token.nl', encoding='utf-8')
    # sbt_file = open(origin_dir + batch_type + '.token.sbt', encoding='utf-8')
    # ast_file = open(origin_dir + batch_type + '_ast.json', encoding='utf-8')
    source_clean_file = open(origin_clean_dir + 'clean_' + batch_type + '.source', 'w', encoding='utf-8')
    code_clean_file = open(origin_clean_dir + 'clean_' + batch_type + '.token.code', 'w', encoding='utf-8')
    nl_clean_file = open(origin_clean_dir + 'clean_' + batch_type + '.token.nl', 'w', encoding='utf-8')
    split_ast_clean_file = open(origin_clean_dir + 'clean_' + batch_type + '.split.ast', 'w', encoding='utf-8')
    # sbt_clean_file = open(origin_clean_dir + 'clean_' + batch_type + '.token.sbt', 'w', encoding='utf-8')
    # ast_clean_file = open(origin_clean_dir + 'clean_' + batch_type + '_ast.json', 'w', encoding='utf-8')

    source_lines = source_file.readlines()
    code_lines = code_file.readlines()
    nl_lines = nl_file.readlines()
    # sbt_lines = sbt_file.readlines()
    # ast_lines = ast_file.readlines()
    for path in tqdm(code_split_list):
        print(path)
        idx = path.replace('.json', '')
        source_line = source_lines[int(idx)]
        source_clean_file.write(source_line)
        code_line = code_lines[int(idx)]
        code_clean_file.write(code_line)
        nl_line = nl_lines[int(idx)]
        nl_clean_file.write(nl_line)
        split_ast = open(code_split_dir+path, encoding='utf-8')
        split_ast_json = split_ast.readlines()[0].strip()
        split_ast_clean_file.write(split_ast_json + '\n')
        split_ast.close()
        # sbt_line = sbt_lines[int(idx) - 1]
        # sbt_clean_file.write(sbt_line)
        # ast_line = ast_lines[int(idx) - 1]
        # ast_clean_file.write(ast_line)

    # ast_clean_file.close()
    # ast_file.close()
    # sbt_clean_file.close()
    # sbt_file.close()
    nl_clean_file.close()
    code_clean_file.close()
    source_clean_file.close()
    nl_file.close()
    code_file.close()
    source_file.close()
    split_ast_clean_file.close()

if __name__ == '__main__':
    # code for test
    # process_source('ast_files/8.java', 'ast_files/source.code')
    # get_ast('ast_files/source.code', 'ast_files/8.json')

    gen_all_ast_py(code_split_dir='E:\\chao\\codeSum\\code_process\\train\\code_split_addhead2\\',
                ast_file_path='E:\\chao\\codeSum\\code_process\\train\\split_ast\\')

    # clean_dataset_2(origin_dir='E:\\chao\\codeSum\\ASE2020\\transformer-ast\\python_datautils\\python_datautils\\data\\train\\origin\\',
    #                 origin_clean_dir='E:\\chao\\codeSum\\ASE2020\\transformer-ast\\python_datautils\\python_datautils\\data\\train\\final_clean\\',
    #                 code_split_dir='E:\\chao\\codeSum\\code_process\\train\\split_ast\\',
    #                 batch_type='train')


def SumOfOddNumbers(List):
    summa = 0
    for number in List:
        if number % 2:
            summa +=number
    return summa

def OrimeNumber(x):
    if x ==1:
        return 0
    for i in rage(2, x):
        if x % i ==0:
            return 0
        return x
List = [4,6,2,1,19,43,21,3,112,7]
print ('Сумма нечетных чисел - ', SumOfOddNumbers(List))


from flask import Flask, render_template, request
import nltk

app = Flask(__name__)

@app.route("/")
def index(name=None):
    return render_template('index.html', name=name)

@app.route('/process', methods=['POST'])
def process():
    # Extract the text from the request
    text = request.form['text']

    # Perform NLTK operations on the text
    # Example: Tokenization
    tokens = nltk.tokenize.word_tokenize(text)

    # Print the tokenized text in the console
    print("Tokenized Text:")
    for token in tokens:
        print(token)

    # Return the results
    return render_template('result.html', tokens=tokens)


'''
Author: Eric Reschke
Cite: https://metricsnavigator.org/housing-cost-preprocessing/
Last Reviewed: 2022-11-25
License: Open to all
'''

import pandas as pd
import numpy as np
import sklearn.linear_model
from sklearn.metrics import mean_squared_error
import matplotlib.pyplot as plt
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import LabelEncoder

housing_train_import = pd.read_csv(wd+'kaggle_train.csv')
housing_test_import = pd.read_csv(wd+'kaggle_test.csv')

df_training = housing_train_import.copy()
df_testing = housing_test_import.copy()

salePrice = df_training['SalePrice']
df_training = df_training.drop('SalePrice',axis=1)
df_training = pd.concat([df_training,df_testing],sort=False).reset_index()

'''
# confirming null values across columns
for col in df_training:
    # this reveals null values; I cut out anything missing over 50% of useable data next
    print(col,': Missing Percent =',round(df_training[col].isna().sum()/len(df_training),2))
'''

# remove features that have too many missing elements
droppedCols = ['Alley','PoolQC','Fence','MiscFeature']
df_training = df_training.drop(droppedCols,axis=1)

# impute (median) missing numerical values
# Note: I let GarageYrBlt be included in this impute-exercise for now
for col in df_training:
    if df_training[col].dtypes!='object':
        median = df_training[col].median()
        df_training[col].fillna(median,inplace=True)

# convert all float data to integers
for col in df_training:
    if df_training[col].dtypes!='object':
        df_training[col] = df_training[col].apply(np.int64)

'''
# code to confirm conversion went as expected
print(df_training.dtypes)
'''

# replace null values for categorical features with 'unknown'
df_training.fillna('unknown',inplace=True)

# add labels for categorical data tracking
labelencoder = LabelEncoder()
enc = OneHotEncoder()

# loop that automatically transforms data to binary and creates unique column names
for col in df_training:
    colNames = []
    if df_training[col].dtypes=='object':
      temp = pd.DataFrame(enc.fit_transform(df_training[[col]]).toarray())
      curCol = col
      for col in temp:
        tCol = (curCol+str(col))
        colNames.append(tCol)
      temp.columns = colNames
      df_training = df_training.join(temp)

# creating a Python list of columns to keep for the final setup
finalCols = []
for col in df_training:
    if df_training[col].dtypes!='object':
      finalCols.append(col)

df_training = df_training[finalCols]

# split the testing from the training dataset
df_testing = df_training.iloc[1460:,:].drop('index',axis=1)
df_training = df_training.iloc[0:1460,:].drop('index',axis=1)

# removing the ids from the datasets
trainingIDs = df_training['Id']
df_training = df_training.drop('Id',axis=1)

testingIDs = df_testing['Id']
df_testing = df_testing.drop('Id',axis=1)

# add back saleprice for the regression model
df_training['SalePrice'] = salePrice

## end of script


from django.urls import path

from .views import (
    CourseInstructorProfileAPIView,
    MyProfileAPIView,
    UpdateMyProfileAPIView,
)

urlpatterns = [
    path("me/", MyProfileAPIView.as_view(), name="my-profile"),
    path("<int:pk>/update/", UpdateMyProfileAPIView.as_view(), name="update-profile"),
    path("<int:pk>/", CourseInstructorProfileAPIView.as_view(), name="instructor-profile"),
]


'''
Udacity machine learning devops project 1
Encapsulate a model fitting pipeline in production worthy script

author: Jeremy Smith
date: 12/21/2021
'''

from pathlib import Path
from constants import *
from sklearn.exceptions import NotFittedError
from sklearn.utils.validation import check_is_fitted
from sklearn.metrics import plot_roc_curve, classification_report
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from numpy.core.records import array
from pandas.core.frame import DataFrame, Series
from pandas.api.types import is_numeric_dtype
import shap
import joblib
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

def import_data(pth: str):
    '''
    returns dataframe for the csv found at pth

    input:
        pth: str - a path to the csv
    output:
        customer_churn: df - pandas dataframe containing X,y for customer churn
    '''
    try:
        customer_churn = pd.read_csv(pth).drop(columns=['Unnamed: 0'])
        customer_churn['Churn'] = customer_churn['Attrition_Flag'].apply(
            lambda val: 0 if val == "Existing Customer" else 1)
        customer_churn = customer_churn.drop(
            columns=['CLIENTNUM', 'Attrition_Flag'])
    except FileNotFoundError as err:
        print('ERROR:%s, does not exist; check path...', pth)
        raise err
    return customer_churn

def perform_eda(customer_churn: DataFrame, impth: str):
    '''
    perform eda on df and save figures to images folder

    input:
        customer_churn: df - pandas dataframe containing X,y for customer churn
        impth:str - location of folder to save image in
    output:
        None
    '''
    try:
        assert Path(impth).is_dir()
    except AssertionError as err:
        print('ERROR: %s image save path does not exist!', impth)
        raise FileNotFoundError from err
    try:
        for col in customer_churn.columns:
            if is_numeric_dtype(customer_churn[col]):
                plot_histogram(customer_churn, col, impth)
                plot_distogram(customer_churn, col, impth)
            else:
                plot_bar(customer_churn, col, impth)

        plot_heatmap(customer_churn, impth)
    except AssertionError as err:
        print(
            'ERROR: Plots failed to render, check dataframe is well formed!')
        raise err
    pass

def plot_bar(customer_churn: DataFrame, col: str, impth: str):
    '''
    plot and save a bar chart of value counts of a column in a dataframe

    input:
        customer_churn: df - pandas dataframe containing X,y for customer churn
        col:str - one of the columns in customer_churn, chosen for plotting
        impth:str - location of folder to save image in
    output:
        None
    '''
    try:
        assert Path(impth).is_dir()
    except AssertionError as err:
        print('ERROR: %s image save path does not exist!', impth)
        raise err
    try:
        assert not customer_churn[col].empty
    except AssertionError as err:
        print('ERROR: No data for plotting!')
        raise err
    try:
        assert len(customer_churn[col].unique()) < 10
    except AssertionError as err:
        print(
            'ERROR: Categorical column contains too many categories, max 10')
        raise err
    plt.figure(figsize=(20, 10))
    customer_churn[col].value_counts('normalize').plot(kind='bar')
    plt.savefig(impth + '%s_bar.png' % col)
    plt.close();

def plot_histogram(customer_churn: DataFrame, col: str, impth: str):
    '''
    plot and save a histogram of a column in a dataframe

    input:
        customer_churn: df - pandas dataframe containing X,y for customer churn
        col:str - one of the columns in customer_churn, chosen for plotting
        impth:str - location of folder to save image in
    output:
        None
    '''
    try:
        assert Path(impth).is_dir()
    except AssertionError as err:
        print('ERROR: %s image save path does not exist!', impth)
        raise err
    try:
        assert not customer_churn[col].empty
    except AssertionError as err:
        print('ERROR: No data for plotting!')
        raise err
    plt.figure(figsize=(20, 10))
    customer_churn[col].hist()
    plt.savefig(impth + '%s_histogram.png' % col)
    plt.close();

def plot_distogram(customer_churn: DataFrame, col: str, impth: str):
    '''
    plot and save a distogram of a column in a dataframe
    input:
        customer_churn: df - pandas dataframe containing X,y for customer churn
        col:str - one of the columns in customer_churn, chosen for plotting
        impth:str - location of folder to save image in
    output:
        None
    '''
    try:
        assert Path(impth).is_dir()
    except AssertionError as err:
        print('ERROR: %s image save path does not exist!', impth)
        raise err
    try:
        assert not customer_churn[col].empty
    except AssertionError as err:
        print('ERROR: No data for plotting!')
        raise err
    plt.figure(figsize=(20, 10))
    sns.distplot(customer_churn[col])
    plt.savefig(impth + '%s_distogram.png' % col)
    plt.close();

def plot_heatmap(customer_churn: DataFrame, impth: str):
    '''
    plot and save a histogram of a column in a dataframe
    input:
        customer_churn: df - pandas dataframe containing X,y for customer churn
        col:str - one of the columns in customer_churn, chosen for plotting
        impth:str - location of folder to save image in
    output:
        None
    '''
    try:
        assert Path(impth).is_dir()
    except AssertionError as err:
        print('ERROR: %s image save path does not exist!', impth)
        raise err
    try:
        assert not customer_churn.empty
    except AssertionError as err:
        print('ERROR: No data for plotting!')
        raise err
    plt.figure(figsize=(20, 10))
    sns.heatmap(
        customer_churn.corr(),
        annot=False,
        cmap='Dark2_r',
        linewidths=2)
    plt.savefig(impth + 'correlation_heatmap.png')
    plt.close();

def filter_encode_columns(
        customer_churn: DataFrame,
        category_lst: list,
        quant_lst: list,
        response: str):
    '''
    split dataframe into valid inputs and outputs for fitting

    input:
        customer_churn: df - pandas dataframe containing X,y for customer churn
        category_lst: list - a list of column names for categorical variables
        qunat_lst: list - a list of column names for quantitative variables
        response: str - name of column containg y
    output:
        X: DataFrame - filtered and encoded input variables
        y: Series - reponse variable series
    '''
    try:
        X = customer_churn[quant_lst + category_lst]
        X = encoder_helper(customer_churn, category_lst)
    except KeyError as err:
        print(
            'ERROR: Some of columns defined in constants are not in data!')
        raise err
    try:
        y = customer_churn[response]
    except KeyError as err:
        print('ERROR: %s, response column, not in data!', response)
        raise err
    X = X.drop(columns=response)
    return X, y

def encoder_helper(customer_churn: DataFrame, category_lst: list):
    '''
    helper function to turn each categorical column into a new column with
    propotion of churn for each category

    input:
        df: pandas dataframe
        category_lst: list of columns that contain categorical features

    output:
            df: pandas dataframe with new columns for
    '''
    try:
        one_hot_variables = pd.get_dummies(
            customer_churn[category_lst], drop_first=True)
    except KeyError as err:
        print(
            'ERROR: Some of columns defined in constants are not in data!')
        raise err
    encoded_customer_churn = customer_churn.drop(
        columns=category_lst).join(one_hot_variables)
    return encoded_customer_churn

def perform_feature_engineering(
        customer_churn: DataFrame,
        category_lst: list,
        quant_lst: list,
        response: str):
    '''
    input:
        customer_churn: df - pandas dataframe containing X,y for customer churn
        category_lst: list - a list of column names for categorical variables
        qunat_lst: list - a list of column names for quantitative variables
        response: str - name of column containg y

    output:
        X_train: X training data
        X_test: X testing data
        y_train: y training data
        y_test: y testing data
    '''
    try:
        X, y = filter_encode_columns(
            customer_churn, category_lst, quant_lst, response)
    except KeyError as err:
        print('ERROR: Could not filter data and engineer columns!')
        raise err
    try:
        X_train, X_test, y_train, y_test = train_test_split(
            X, y, test_size=0.3, random_state=42)
    except ValueError as err:
        # Means that dataset passed was too small probably
        print(
            'ERROR: Could not split data, with input shapes of X:%s,%s y:%s' %
            (X.shape[0], X.shape[1], len(y)))
        raise err
    return X_train, X_test, y_train, y_test

def plot_classification_report(
        report_str_test: str,
        report_str_train: str,
        report_name: str,
        impth: str):
    '''
    given plotting DataFrame, generate a heatmap plot, and save

    input:
        report_str_test: str - output of sklearn class report on test data
        report_str_train: str - output of sklearn class report on train data
        report_name: str - name to be applied to report as a label
        impth:str - path to save classification report image
    output:
        None
    '''

    try:
        assert len(report_str_test) > 0
    except AssertionError as err:
        print('ERROR: Empty test string %s', report_str_test)
        raise err
    try:
        assert len(report_str_train) > 0
    except AssertionError as err:
        print('ERROR: Empty train string %s', report_str_train)
        raise err
    try:
        assert len(report_name) > 0
    except AssertionError as err:
        print('ERROR: Empty name string %s', report_name)
        raise err
    try:
        assert Path(impth).is_dir()
    except AssertionError as err:
        print('ERROR: %s image save path does not exist!', impth)
        raise err

    plt.figure(figsize=(5, 5))
    plt.text(
        0.01, 1.25, str(
            '%s Train' %
            report_name), {
            'fontsize': 10}, fontproperties='monospace')
    plt.text(0.01, 0.05, str(report_str_train), {
             'fontsize': 10}, fontproperties='monospace')
    plt.text(
        0.01, 0.6, str(
            '%s Test' %
            report_name), {
            'fontsize': 10}, fontproperties='monospace')
    plt.text(0.01, 0.7, str(report_str_test), {
             'fontsize': 10}, fontproperties='monospace')
    plt.axis('off')
    plt.savefig(impth + '%s_class_report.png' % report_name)
    plt.close();

def classification_report_image(y_train: Series,
                                y_test: Series,
                                y_train_preds_lr: array,
                                y_train_preds_rf: array,
                                y_test_preds_lr: array,
                                y_test_preds_rf: array,
                                impth: str):
    '''
    classification report based on results and stores report as image
    input:
            y_train: Series - training response values
            y_test: Series - test response values
            y_train_preds_lr: array - training predictions, logistic regression
            y_train_preds_rf: array - training predictions, random forest
            y_test_preds_lr: array - test predictions from logistic regression
            y_test_preds_rf: array - test predictions from random forest
            impth: str - location of folder to save image in

    output:
             None
    '''

    try:
        assert not y_train.empty or y_test.empty
    except AssertionError as err:
        print(
            'ERROR: Empty y reference data passed to classification report')
        raise err
    try:
        assert len(y_train_preds_lr) == len(
            y_train) and len(y_test_preds_lr) == len(y_test)
    except AssertionError as err:
        print(
            'ERROR: Length of ref data does not match prediction data for logistic regression!')
        raise err
    try:
        assert len(y_train_preds_rf) == len(
            y_train) and len(y_test_preds_rf) == len(y_test)
    except AssertionError as err:
        print(
            'ERROR: Length of ref data does not match prediction data for random forest!')
        raise err
    report_str_rf_test = classification_report(y_test, y_test_preds_rf)
    report_str_rf_train = classification_report(y_train, y_train_preds_rf)
    report_str_lr_test = classification_report(y_test, y_test_preds_lr)
    report_str_lr_train = classification_report(y_train, y_train_preds_lr)

    plot_classification_report(
        report_str_rf_test,
        report_str_rf_train,
        'RandomForest',
        impth)
    plot_classification_report(
        report_str_lr_test,
        report_str_lr_train,
        'LogisticRegression',
        impth)

def roc_curve_plot(
        lrc: LogisticRegression,
        rfc: RandomForestClassifier,
        X_test: DataFrame,
        y_test: Series,
        impth: str):
    '''
    given trained logistic regressor and random forest classifier, make roc plot

    input:
        lrc: LogisticRegression - tranined logistic regressor
        rfc: RandomForestClassifier - trained random forest classifier
        X_test: DataFrame - test predictors
        y_test: Series - test labels
        impth:str - location of folder to save image in
    output:
        None
    '''

    try:
        check_is_fitted(lrc)
    except NotFittedError as err:
        print('ERROR: Logistic regressor is not fitted!')
        raise err
    try:
        check_is_fitted(rfc)
    except NotFittedError as err:
        print('ERROR: Random forest is not fitted!')
        raise err
    try:
        assert len(X_test) == len(y_test)
    except AssertionError as err:
        print(
            'ERROR: X_train and y_train are different length for roc plot!')
        raise err
    try:
        assert Path(impth).is_dir()
    except AssertionError as err:
        print('ERROR: %s image save path does not exist!', impth)
        raise FileNotFoundError from err

    lrc_plot = plot_roc_curve(lrc, X_test, y_test)
    plt.figure(figsize=(15, 8))
    ax = plt.gca()
    rfc_disp = plot_roc_curve(
        rfc.best_estimator_,
        X_test,
        y_test,
        ax=ax,
        alpha=0.8)
    lrc_plot.plot(ax=ax, alpha=0.8)
    plt.savefig(impth + 'roc_curve.png')
    plt.close();

def feature_importance_plot(
        rfc: RandomForestClassifier,
        X_data: DataFrame,
        impth: str):
    '''
    creates and stores the feature importances in pth
    input:
        rfc: RandomForestClassifier - trained random forest classifier
        X_test: DataFrame - test predictors
        impth:str - location of folder to save image in

    output:
             None
    '''

    try:
        check_is_fitted(rfc)
    except NotFittedError as err:
        print('ERROR: Random forest is not fitted!')
        raise err
    try:
        assert not X_data.empty
    except AssertionError as err:
        print('ERROR: Empty X_data passed to feature importance!')
        raise err
    try:
        assert Path(impth).is_dir()
    except AssertionError as err:
        print('ERROR: %s image save path does not exist!', impth)
        raise err

    # Calculate feature importances
    importances = rfc.best_estimator_.feature_importances_
    # Sort feature importances in descending order
    indices = np.argsort(importances)[::-1]
    # Rearrange feature names so they match the sorted feature importances
    names = [X_data.columns[i] for i in indices]

    plt.figure(figsize=(20, 5))
    plt.title("Feature Importance")
    plt.ylabel('Importance')
    plt.bar(range(X_data.shape[1]), importances[indices])
    plt.xticks(range(X_data.shape[1]), names, rotation=90)
    plt.savefig(impth + 'feature_importance.png')
    plt.close();

def explainer_plot(rfc: RandomForestClassifier, X_test: DataFrame, impth: str):
    '''
    generate feature importance plot using shap

    inputs:
        rfc: RandomForestClassifier - trained random forest classifier
        X_test: DataFrame - test predictors
        impth:str - location of folder to save image in
    outputs:
        None
    '''

    try:
        check_is_fitted(rfc)
    except NotFittedError as err:
        print('ERROR: Random forest is not fitted!')
        raise err
    try:
        assert not X_test.empty
    except AssertionError as err:
        print('ERROR: Empty X_data passed to feature importance!')
        raise err
    try:
        rfc.predict(X_test)
    except ValueError as err:
        print(
            'ERROR: X_test is malformed for random forest classifier!')
        raise err
    try:
        assert Path(impth).is_dir()
    except AssertionError as err:
        print('ERROR: %s image save path does not exist!', impth)
        raise err

    plt.figure(figsize=(8, 12))
    explainer = shap.TreeExplainer(rfc.best_estimator_)
    shap_values = explainer.shap_values(X_test)
    shap.summary_plot(shap_values, X_test, plot_type="bar", show=False)
    plt.tight_layout()
    plt.savefig(impth + 'shap_plot.png')
    plt.close();

def train_models(
        X_train: DataFrame,
        X_test: DataFrame,
        y_train: DataFrame,
        y_test: DataFrame,
        param_grid: dict,
        impth: str,
        modelpth: str):
    '''
    train, store model results: images + scores, and store models
    input:
              X_train: X training data
              X_test: X testing data
              y_train: y training data
              y_test: y testing data
              param_grid: dict - specifies grid search parameters
              impth:str - location of folder to save image in
              modelpth:str - location of folder to save models in
    output:
              None
    '''

    try:
        assert len(X_train) == len(y_train)
    except AssertionError as err:
        print('ERROR: X_train and y_train have different lengths, cannot train!')
        raise err
    try:
        assert len(X_test) == len(y_test)
    except AssertionError as err:
        print('ERROR: X_test and y_test have different lengths, cannot test!')
        raise err
    try:
        assert len(param_grid) > 0
    except AssertionError as err:
        print('ERROR: Empty parameter grid passed to grid search!')
        raise err
    try:
        assert Path(modelpth).is_dir()
    except AssertionError as err:
        print('ERROR: %s model save path does not exist!', modelpth)
        raise FileNotFoundError from err
    try:
        assert Path(impth).is_dir()
    except AssertionError as err:
        print('ERROR: %s image save path does not exist!', impth)
        raise FileNotFoundError from err

    # Fit the models
    rfc = RandomForestClassifier(random_state=42)
    lrc = LogisticRegression()
    cv_rfc = GridSearchCV(estimator=rfc, param_grid=param_grid, cv=5)
    cv_rfc.fit(X_train, y_train)
    lrc.fit(X_train, y_train)

    # Save trained models
    joblib.dump(cv_rfc, modelpth + 'rfc_model.pkl')
    joblib.dump(lrc, modelpth + 'logistic_model.pkl')

    # Make predictions, to evaluate performance
    y_train_preds_rf = cv_rfc.best_estimator_.predict(X_train)
    y_test_preds_rf = cv_rfc.best_estimator_.predict(X_test)

    y_train_preds_lr = lrc.predict(X_train)
    y_test_preds_lr = lrc.predict(X_test)

    # Generate plots to summarize performance
    classification_report_image(
        y_train,
        y_test,
        y_train_preds_lr,
        y_train_preds_rf,
        y_test_preds_lr,
        y_test_preds_rf,
        impth)
    roc_curve_plot(lrc, cv_rfc, X_test, y_test, impth)
    feature_importance_plot(cv_rfc, X_train.append(X_test), impth)
    explainer_plot(cv_rfc, X_train.append(X_test), impth)

if __name__ == '__main__':

    churn_dataframe = import_data(path_to_data)
    perform_eda(churn_dataframe, image_save_path)
    train_x, test_x, train_y, test_y = perform_feature_engineering(
        churn_dataframe, cat_columns, quant_columns, resp_col)
    train_models(
        train_x,
        test_x,
        train_y,
        test_y,
        grid_search_parameters,
        image_save_path,
        model_save_path)


import json
import string
import random
import glob, os

used_names = []
activity_mangle_setting = {}

def process_file_content(path, file_name):
    print("processing content: %s..."%path)
    with open(path, "rt") as fin:
        file_content = fin.read()

        # replace the file content based on mangle settings
        for (src, dst) in activity_mangle_setting.items():
            file_content = file_content.replace(src, dst)

        ext = os.path.splitext(file_name)[1]
        if ext == '.java':
            comment = "// machine renamed: %s\n"%file_name
            file_content = "%s%s"%(comment, file_content)
        with open(path, "wt") as fout:
            fout.write(file_content)
        print("done\n")
    return

# rand string for class names
def gen_rand_str():
    rand_str = ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase, k=8))
    while rand_str in used_names:
        rand_str = ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase, k=8))

    used_names.append(rand_str)
    return rand_str

# read configs
with open('obact.json', "rt") as f:
    json_root = json.load(f)
    activities = json_root["activities"]
    source_root = json_root["root"]
    manifest = json_root["manifest"]

# mangle activity names
for act_name in activities:
    activity_mangle_setting[act_name] = gen_rand_str()

print("activity mapping %s"%activity_mangle_setting)

root_path = os.path.join(os.getcwd(), source_root)
# perform content replace for the files
print("processing file content...")
for root, dirs, files in os.walk(source_root):
    directory = os.path.join(os.getcwd(), root)
    for file in files:
        if file.endswith(".java"):
            # get the path of the java file
            path = os.path.join(directory, file)

            process_file_content(path, file)
print("processing file content done")

print("renaming files")
for root, dirs, files in os.walk(source_root):
    directory = os.path.join(os.getcwd(), root)
    for file in files:
        if file.endswith(".java"):
            name = os.path.splitext(file)[0]
            if name in activity_mangle_setting:
                # get the path of the java file
                src = os.path.join(directory, file)
                dst = os.path.join(directory, "%s.java"%activity_mangle_setting[name])

                print("%s -> %s"%(src, dst))
                os.rename(src, dst)
print("renaming files done")

manifest_path = os.path.join(os.getcwd(), manifest)
process_file_content(manifest_path, "AndroidManifest.xml")