#-------------------------------------------------------------------------
# AUTHOR: Isaiah Hessler
# FILENAME: Assignment 2 - Part 1
# SPECIFICATION: Complete the Python program (decision_tree_2.py) that will read the files
# contact_lens_training_1.csv, contact_lens_training_2.csv, and contact_lens_training_3.csv. Each of
# those training sets has a different number of instances. You will observe that now the trees are being
# created setting the parameter max_depth = 3, which it is used to define the maximum depth of the tree
# (pre-pruning strategy) in sklearn. Your goal is to train, test, and output the performance of the 3 models
# created by using each training set on the test set provided (contact_lens_test.csv). You must repeat
# this process 10 times (train and test by using a different training set), choosing the average accuracy
# as the final classification performance of each model.
# FOR: CS 4210- Assignment #2
# TIME SPENT: 4 Hours
#-----------------------------------------------------------*/

#IMPORTANT NOTE: DO NOT USE ANY ADVANCED PYTHON LIBRARY TO COMPLETE THIS CODE SUCH AS numpy OR pandas. You have to work here only with standard
# dictionaries, lists, and arrays

#importing some Python libraries
from sklearn import tree
import csv

dataSets = ['contact_lens_training_1.csv', 'contact_lens_training_2.csv', 'contact_lens_training_3.csv']
testData = 'contact_lens_test.csv'
counter = 1

for ds in dataSets:

    dbTraining = []
    X = []
    Y = []
    XTest = []
    YTest = []
    sum = 0

    #reading the training data in a csv file
    with open(ds, 'r') as csvfile:
         reader = csv.reader(csvfile)
         for i, row in enumerate(reader):
             if i > 0: #skipping the header
                dbTraining.append (row)

    #transform the original categorical training features to numbers and add to the 4D array X. For instance Young = 1, Prepresbyopic = 2, Presbyopic = 3
    # so X = [[1, 1, 1, 1], [2, 2, 2, 2], ...]]
    for row in dbTraining:
        temp = []
        for j, value in enumerate(row):
            if j == 0:
                if value == 'Young':
                    temp.append(1)
                elif value == 'Prepresbyopic':
                    temp.append(2)
                else:
                    temp.append(3)
            if j == 1:
                if value == 'Myope':
                    temp.append(1)
                else:
                    temp.append(2)
            if j == 2:
                if value == 'No':
                    temp.append(1)
                else:
                    temp.append(2)
            if j == 3:
                if value == 'Reduced':
                    temp.append(1)
                else:
                    temp.append(2)
        X.append(temp)

    #transform the original categorical training classes to numbers and add to the vector Y. For instance Yes = 1, No = 2, so Y = [1, 1, 2, 2, ...]
    for row in dbTraining:
        if row[-1] == 'Yes':
            Y.append(1)
        else:
            Y.append(2)

    #loop your training and test tasks 10 times here
    for i in range (10):

       #fitting the decision tree to the data setting max_depth=3
        clf = tree.DecisionTreeClassifier(criterion = 'entropy', max_depth=3)
        clf = clf.fit(X, Y)

       #read the test data and add this data to dbTest
        dbTest = []
        with open(testData, 'r') as csvfile:
            reader = csv.reader(csvfile)
            for i, row in enumerate(reader):
                if i > 0: #skipping the header
                    dbTest.append (row)

        #transform the features of the test instances to numbers following the same strategy done during training,
        #and then use the decision tree to make the class prediction. For instance: class_predicted = clf.predict([[3, 1, 2, 1]])[0]
        #where [0] is used to get an integer as the predicted class label so that you can compare it with the true label
        #--> add your Python code here
        for row in dbTest:
            temp = []
            for j, value in enumerate(row):
                if j == 0:
                    if value == 'Young':
                        temp.append(1)
                    elif value == 'Prepresbyopic':
                        temp.append(2)
                    else:
                        temp.append(3)
                if j == 1:
                    if value == 'Myope':
                        temp.append(1)
                    else:
                        temp.append(2)
                if j == 2:
                    if value == 'No':
                        temp.append(1)
                    else:
                        temp.append(2)
                if j == 3:
                    if value == 'Reduced':
                        temp.append(1)
                    else:
                        temp.append(2)
            XTest.append(temp)

        for row in dbTest:
            if row[-1] == 'Yes':
                YTest.append(1)
            else:
                YTest.append(2)

        class_predicted = clf.predict(XTest)
           #compare the prediction with the true label (located at data[4]) of the test instance to start calculating the accuracy.
           #--> add your Python code here

        TP = 0
        TN = 0
        FP = 0
        FN = 0
        accuracy = 0
        x = 0
        for class_row in YTest:
            if class_row == 1:
                if class_predicted[x] == 1:
                    TP += 1
                else:
                    FN += 1
            elif class_row == 2:
                if class_predicted[x] == 2:
                    TN += 1
                else:
                    FP += 1
            x += 1

        accuracy = (TP + TN) / (TP + TN + FP + FN)
        XTest.clear()
        YTest.clear()

        #find the average accuracy of this model during the 10 runs (training and test set)
        sum += accuracy

    #print the average accuracy of this model during the 10 runs (training and test set).
    #your output should be something like that: final accuracy when training on contact_lens_training_1.csv: 0.2
    sum = sum / 10
    print(f'Final accuracy when training on contact_lens_training_{counter}.csv: {sum}')
    counter += 1


# -*- coding: utf-8 -*-
"""
Created on Sat Mar  4 18:25:42 2017
@author: Ignasi Dev
"""

# VERSION WITHOUT THE EVERIS-API PROCESSING

import json

# Requests needed for web requests with the Telegram API:
import requests

# To force a small delay between requests:
import time

# To accept special characters:
import urllib

from dbhelper import DBHelper
from random import randint

db = DBHelper()

TOKEN = "insertTokenHere"
URL = "https://api.telegram.org/bot{}/".format(TOKEN)

# Download the content from an url and returns a string:
def get_url(url):
    response = requests.get(url)
    content = response.content.decode("utf8")
    return content

# Parse the string response into the py library:
def get_json_from_url(url):
    content = get_url(url)
    js = json.loads(content)
    return js

# Use Long Polling: keep the connection open until there are updates, when
# sends it down the open pipe. Also, specify the time the connection
# stays opened with timeout:
def get_updates(offset=None):
    url = URL + "getUpdates"
    if offset:
        url += "?offset={}".format(offset)
    js = get_json_from_url(url)
    return js

def get_last_update_id(updates):
    update_ids = []
    for update in updates["result"]:
        update_ids.append(int(update["update_id"]))
    return max(update_ids)

# Hw to react given the user text entry:
def handle_updates(updates):
    for update in updates["result"]:
        text = update["message"]["text"]
        chat = update["message"]["chat"]["id"]
        items = db.get_items(chat)
        if text == "/delete" or text == "delete":
            keyboard = build_keyboard(items)
            send_message("Delete mode entered.\nSelect an item to be deleted. If you want to add something instead, just type the new task anytime", chat, keyboard)
        elif text == "/help" or text == "help" or text == "Help" or text == "HELP":
            send_message("HELP INFORMATION: \n- To see the whole task list, send '\list'\n- To enter delete mode, send me '\delete', and I will erase whatever task you then select. From this mode, if you want to add more tasks just type'em!\n- To restart but with the same list, type '/start'\n- For further info just hit up my creator at Slack! (@ignasi)", chat)
            send_message("So, what is something you want to add to the task list?", chat)
        elif text == "/list" or text == "list" or text == "List" or text == "LIST":
            send_message("Task List (updated):", chat)
            message = "\n".join(items)
            send_message(message, chat)
        elif text == "/start" or text == "start" or text == "Start" or text == "START":
            send_message("Hi! I am your task manager. Add items and remove them via /delete. Type /help anytime you want.", chat)
        elif text == "/list" or text == "list":
            message = "\n".join(items)
            send_message(message, chat)
        elif text.startswith("/"):
            continue
        elif text in items:
            db.delete_item(text, chat)
            items = db.get_items(chat)
            keyboard = build_keyboard(items)
            send_message("Select an item to be deleted", chat, keyboard)
            message = "\n- ".join(items)
            send_message(message, chat)
        else:
            db.add_item(text, chat)
            items = db.get_items(chat)
            send_message("Task List (updated):", chat)
            message = "\n- ".join(items)
            send_message(message, chat)
            question = randint(1, 9)
            if question == 1:
              send_message("What else?", chat)
            if question == 2:
              send_message("Anything else?", chat)
            if question == 3:
              send_message("Any other task?", chat)
            if question == 4:
              send_message("What else do you want to add?", chat)
            if question == 5:
              send_message("Tell me more tasks!", chat)
            if question == 6:
              send_message("Keep telling me tasks!", chat)
            if question == 7:
              send_message("Anything you would like to add?", chat)
            if question == 8:
              send_message("What else do you have for me?", chat)

def get_last_chat_id_and_text(updates):
    num_updates = len(updates["result"])
    last_update = num_updates - 1
    text = updates["result"][last_update]["message"]["text"]
    chat_id = updates["result"][last_update]["message"]["chat"]["id"]
    return (text, chat_id)

# Pass the special keyboard and let it open when called:
def build_keyboard(items):
    keyboard = [[item] for item in items]
    reply_markup = {"keyboard":keyboard, "one_time_keyboard": True}
    return json.dumps(reply_markup)
# "reply_markup" does not only reference the Keyboard: it is indeed an object
# that includes various features of the keyboard.

def send_message(text, chat_id, reply_markup = None):
    text = urllib.parse.quote_plus(text)
    url = URL + "sendMessage?text = {}&chat_id = {}&parse_mode = Markdown".format(text, chat_id)
    if reply_markup:
        url += "&reply_markup = {}".format(reply_markup)
    get_url(url)

def main():
    db.setup()
    last_update_id = None
    while True:
        updates = get_updates(last_update_id)
        if len(updates["result"]) > 0:
            last_update_id = get_last_update_id(updates) + 1
            handle_updates(updates)
        time.sleep(0.5)

if __name__ == '__main__':
  main()


import bpy
import os
from pathlib import Path
import math
import sys
import json

PATH_TO_OUTPUT_IMAGES = os.path.abspath('./imgs')
OBJ_PATH = Path("./plane_generation") # TODO: change path
TEXTURE_PATH = Path("./textures")

def reset_blend():
    bpy.ops.wm.read_factory_settings()
    bpy.context.scene.cursor.location = (0.0, 0.0, 0.0)

    # only worry about data in the startup scene
    for bpy_data_iter in (
            bpy.data.meshes,
            bpy.data.lights,
            bpy.data.images,
            bpy.data.materials
            ):
        for id_data in bpy_data_iter:
            bpy_data_iter.remove(id_data, do_unlink=True)

def select_object(ob):
    bpy.ops.object.select_all(action='DESELECT')
    bpy.context.view_layer.objects.active = ob
    ob.select_set(True)

def page_texturing(mesh, texpath):
    select_object(mesh)
    bpy.ops.object.mode_set(mode="OBJECT")
    bpy.ops.object.material_slot_add()
    bpy.data.materials.new('Material.001')
    mesh.material_slots[0].material = bpy.data.materials['Material.001']
    material = bpy.data.materials['Material.001']

    material.use_nodes = True
    nodes = material.node_tree.nodes
    for n in nodes:
        nodes.remove(n)

    # node_principled = nodes.new(type='ShaderNodeBsdfPrincipled')
    # node_principled.location = 0,0

    # node_tex = nodes.new('ShaderNodeTexImage')
    # node_tex.image = bpy.data.images.load(texpath)
    # node_tex.extension='EXTEND'
    # node_tex.location = -400,0

    # node_output = nodes.new(type='ShaderNodeOutputMaterial')
    # node_output.location = 400,0

    # links = material.node_tree.links
    # links.new(node_tex.outputs["Color"], node_principled.inputs["Base Color"])
    # links.new(node_principled.outputs["BSDF"], node_output.inputs["Surface"])

    # # mesh.data.materials[0] = material (does not work ?)
    # mesh.active_material = material # works

    out_node=nodes.new(type='ShaderNodeOutputMaterial')
    bsdf_node=nodes.new(type='ShaderNodeBsdfDiffuse')
    texture_node=nodes.new(type='ShaderNodeTexImage')

    texture_node.image=bpy.data.images.load(texpath)

    links=material.node_tree.links
    links.new(bsdf_node.outputs[0],out_node.inputs[0])
    links.new(texture_node.outputs[0],bsdf_node.inputs[0])

    bsdf_node.inputs[0].show_expanded=True
    texture_node.extension='EXTEND'
    texturecoord_node=nodes.new(type='ShaderNodeTexCoord')
    links.new(texture_node.inputs[0],texturecoord_node.outputs[2])

def uv_unwrap(mesh):
    select_object(mesh)
    bpy.ops.object.mode_set(mode="EDIT")
    bpy.ops.uv.unwrap(method='ANGLE_BASED')
    bpy.ops.object.mode_set(mode="OBJECT")

def position_object(mesh_name):
    mesh=bpy.data.objects[mesh_name]
    select_object(mesh)
    mesh.rotation_euler=[0.0,0.0,0.0]
    return mesh

def scaleUV(obj, pivot, scale):
    uvMap = obj.data.uv_layers[0]
    for d in uvMap.data:
        d.uv[0] = pivot[0] + scale[0]*(d.uv[0] - pivot[0])
        d.uv[1] = pivot[1] + scale[1]*(d.uv[1] - pivot[1])

def prepare_rendersettings(resolution: tuple):
    bpy.ops.object.select_all(action='DESELECT')
    bpy.data.scenes['Scene'].render.engine = "CYCLES"
    bpy.data.scenes['Scene'].cycles.device='GPU'
    bpy.context.preferences.addons[
        'cycles'
    ].preferences.compute_device_type = 'CUDA'
    bpy.context.preferences.addons['cycles'].preferences.get_devices()
    print(bpy.context.preferences.addons["cycles"].preferences.compute_device_type)
    for d in bpy.context.preferences.addons["cycles"].preferences.devices:
        d["use"] = 0
        if d["name"][:6] == 'NVIDIA':
            d["use"] = 1
        print(d["name"], d["use"])
    bpy.data.scenes['Scene'].render.resolution_x=resolution[0]
    bpy.data.scenes['Scene'].render.resolution_y=resolution[1]
    bpy.data.scenes['Scene'].render.resolution_percentage=100

def place_camera(object, eul):
    pass

def render_pass(file_name: str):
    file_name = file_name + '.png'
    scene=bpy.data.scenes['Scene']
    bpy.context.scene.use_nodes = True
    tree = bpy.context.scene.node_tree
    links = tree.links

    for n in tree.nodes:
        tree.nodes.remove(n)

    render_layers = tree.nodes.new('CompositorNodeRLayers')

    file_output_node_img = tree.nodes.new('CompositorNodeOutputFile')
    file_output_node_img.format.file_format = 'PNG'
    file_output_node_img.base_path = PATH_TO_OUTPUT_IMAGES
    file_output_node_img.file_slots[0].path = file_name
    imglk = links.new(render_layers.outputs[0], file_output_node_img.inputs[0])
    scene.cycles.samples=128
    bpy.ops.render.render(write_still=False)

def add_camera_active(location: tuple[float], rotation: tuple[float]):
    bpy.ops.object.camera_add(
        align='VIEW',
        enter_editmode=False,
        location=location,
        rotation=rotation)

    newCams = []
    for objName in bpy.data.objects.keys():
        if(len(objName.split("amera")) > 1):
            newCams.append(objName)

    myCam = bpy.data.objects[newCams[len(newCams) - 1]]
    bpy.context.view_layer.objects.active = myCam
    scene = bpy.data.scenes['Scene']
    scene.camera = myCam

def add_light(location: tuple[float], energy, name: str, type: str = 'POINT'):
    # create light datablock, set attributes
    light_data = bpy.data.lights.new(name=name, type=type)
    light_data.energy = energy

    # create new object with our light datablock
    light_object = bpy.data.objects.new(name=name, object_data=light_data)

    # link light object
    bpy.context.collection.objects.link(light_object)

    # make it active
    bpy.context.view_layer.objects.active = light_object

    #change location
    light_object.location = location

def restruct_uv():
    me = bpy.context.object.data
    uv_layer = me.uv_layers.active.data

    u_max = 0
    v_max = 0
    for poly in me.polygons:
        for loop_index in poly.loop_indices:
            if u_max < uv_layer[loop_index].uv[0]:
                u_max = uv_layer[loop_index].uv[0]
            if v_max < uv_layer[loop_index].uv[1]:
                v_max = uv_layer[loop_index].uv[1]

    for poly in me.polygons:
        for loop_index in range(poly.loop_start, poly.loop_start + poly.loop_total):
            uv_layer[loop_index].uv = (uv_layer[loop_index].uv[0] / u_max,
                                    uv_layer[loop_index].uv[1] / v_max)

def main(file_name):
    if not os.path.exists(PATH_TO_OUTPUT_IMAGES):
        os.makedirs(PATH_TO_OUTPUT_IMAGES)

    reset_blend()

    bpy.ops.import_scene.obj(filepath=str(OBJ_PATH.joinpath(file_name).with_suffix('.obj')), axis_forward='Y', axis_up='Z')
    mesh_name=bpy.data.meshes[-1].name
    mesh = bpy.data.objects[mesh_name]

    uv_unwrap(mesh)
    bpy.context.view_layer.objects.active = bpy.context.window.scene.objects[1]
    restruct_uv()

    page_texturing(mesh, str(TEXTURE_PATH.joinpath(file_name).with_suffix('.png')))
    # uv_unwrap(mesh)
    add_camera_active((0, -6, 2), (math.radians(80), 0, math.radians(-15)))
    add_light((0, 0, 5), 800, 'light')
    # scaleUV(mesh, (0.7, 0.4), (4, 4))

    prepare_rendersettings(resolution=(1920, 1080))
    render_pass(file_name+'_render')

if __name__ == '__main__':
    if not os.path.exists(PATH_TO_OUTPUT_IMAGES):
        os.makedirs(PATH_TO_OUTPUT_IMAGES)

    reset_blend()

    json_path = sys.argv[-1]
    print(json_path)
    with open(json_path, "r") as file:
        data = json.load(file)

    bpy.ops.import_scene.obj(filepath=str(OBJ_PATH.joinpath(data["file_name"]).with_suffix('.obj')), axis_forward='Y', axis_up='Z')
    mesh_name=bpy.data.meshes[-1].name
    mesh = bpy.data.objects[mesh_name]

    uv_unwrap(mesh)
    bpy.context.view_layer.objects.active = bpy.context.window.scene.objects[1]
    restruct_uv()

    page_texturing(mesh, str(TEXTURE_PATH.joinpath(data["file_name"]).with_suffix('.png')))
    # uv_unwrap(mesh)
    add_camera_active((0, -6, 2), (math.radians(80), 0, math.radians(-15)))
    add_light((0, 0, 5), 800, 'light')
    # scaleUV(mesh, (0.7, 0.4), (4, 4))

    prepare_rendersettings(resolution=(1920, 1080))
    render_pass(data["file_name"]+'_render')


import sqlite3
import openpyxl

class DBPortContainer:
    table = 'container'
    _sql_create_container_table = """ CREATE TABLE IF NOT EXISTS container (
                            port text,
                            date integer,
                            TEU real,
                            year integer,
                            month integer,
                            primary key(port, date)
                            ); """

    def __init__(self, conn):
        self.conn = conn

        if self.conn is not None:
            # create table
            self._create_table(self.conn, self._sql_create_container_table)
        else:
            print("E : cannot create the DB connection")

            # drop and create

    def reset_table(self):
        cursor = self.conn.cursor()
        cursor.execute('''DROP TABLE ''' + self.table)
        self.conn.commit()

        self._create_table(self.conn, self._sql_create_container_table)

    _sql_insert_container_table = """insert into container(
                                    port,
                                    date,
                                    TEU,
                                    year,
                                    month
                                    ) values (?, ?, ?, ?, ?)"""

    def insert_from_xls(self, xls_files, years, port):
        for year_index, file_name in enumerate(xls_files):
            wb = openpyxl.load_workbook(file_name)
            ws = wb.active

            year = years[year_index]
            cells = ws['B4':'C15']

            for row in cells:
                values = []

                month = row[0].value
                teu = row[1].value

                date = int(str(year) + month)

                values.append(port)
                values.append(date)
                values.append(teu)
                values.append(year)
                values.append(int(month))

                self._insert_table(self.conn, self._sql_insert_container_table, values)

            self.conn.commit()
            wb.close()
            print("db ship container file", file_name, "insert finish")

    @staticmethod
    def _create_connection(db_file):
        try:
            conn = sqlite3.connect(db_file)
            return conn
        except Exception as e:
            print("error : ", e)

        return None

    @staticmethod
    def _create_table(conn, create_table_sql):
        try:
            cursor = conn.cursor()
            cursor.execute(create_table_sql)
        except Exception as e:
            print("create table error : ", e)

    @staticmethod
    def _insert_table(conn, insert_table_sql, values):
        try:
            cursor = conn.cursor()
            cursor.execute(insert_table_sql, values)
        except Exception as e:
            print("insert error : ", e)

    def get_all(self):
        cur = self.conn.cursor()
        sql = """select * from container"""
        cur.execute(sql)
        try:
            return cur.fetchall()
        except Exception:
            return None

    def get_container_data(self, port):
        cur = self.conn.cursor()
        sql = """select port, year, month, TEU
                    from container
                    where port = :port"""
        cur.execute(sql, {'port': port})
        try:
            return cur.fetchall()
        except Exception:
            return None

    def get_container_data_in_year(self, port, year):
        cur = self.conn.cursor()
        sql = """select port, year, month, TEU
                    from container
                    where port = ? and
                        year = ?"""
        cur.execute(sql, (port, year))
        try:
            return cur.fetchall()
        except Exception:
            return None


import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import io
import csv
from tensorflow.keras.layers import Embedding, Conv1D, GlobalMaxPooling1D, Dense, Dropout

dataset = pd.read_csv('data.csv')

sentences = dataset['text'].tolist()
labels = dataset['sentiment'].tolist()
NUM_WORDS = 10000
EMBEDDING_DIM = 100
MAXLEN = 100
PADDING = 'post'
OOV_TOKEN = "<OOV>"
TRAINING_SPLIT = .96

def remove_stopwords(sentence):
   # List of stopwords
   stopwords = ["a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any", "are", "as", "at",
                "be", "because", "been", "before", "being", "below", "between", "both", "but", "by", "could", "did",
                "do", "does", "doing", "down", "during", "each", "few", "for", "from", "further", "had", "has", "have",
                "having", "he", "he'd", "he'll", "he's", "her", "here", "here's", "hers", "herself", "him", "himself",
                "his", "how", "how's", "i", "i'd", "i'll", "i'm", "i've", "if", "in", "into", "is", "it", "it's", "its",
                "itself", "let's", "me", "more", "most", "my", "myself", "nor", "of", "on", "once", "only", "or",
                "other", "ought", "our", "ours", "ourselves", "out", "over", "own", "same", "she", "she'd", "she'll",
                "she's", "should", "so", "some", "such", "than", "that", "that's", "the", "their", "theirs", "them",
                "themselves", "then", "there", "there's", "these", "they", "they'd", "they'll", "they're", "they've",
                "this", "those", "through", "to", "too", "under", "until", "up", "very", "was", "we", "we'd", "we'll",
                "we're", "we've", "were", "what", "what's", "when", "when's", "where", "where's", "which", "while",
                "who", "who's", "whom", "why", "why's", "with", "would", "you", "you'd", "you'll", "you're", "you've",
                "your", "yours", "yourself", "yourselves"]

   # Sentence converted to lowercase-only
   sentence = sentence.lower()

   words = sentence.split()
   no_words = [w for w in words if w not in stopwords]
   sentence = " ".join(no_words)

   return sentence

def parse_data_from_file(filename):
   sentences = []
   labels = []
   with open(filename, 'r') as csvfile:
      reader = csv.reader(csvfile, delimiter=',')
      next(reader)
      for row in reader:
         labels.append(row[2])
         sentence = row[1]
         sentence = remove_stopwords(sentence)
         sentences.append(sentence)

   return sentences, labels
sentences, labels = parse_data_from_file("data.csv")

# GRADED FUNCTIONS: train_val_split
def train_val_split(sentences, labels, training_split):
   ### START CODE HERE

   # Compute the number of sentences that will be used for training (should be an integer)
   train_size = int(len(sentences) * training_split)

   # Split the sentences and labels into train/validation splits
   train_sentences = sentences[:train_size]
   train_labels = labels[:train_size]

   validation_sentences = sentences[train_size:]
   validation_labels = labels[train_size:]

   ### END CODE HERE

   return train_sentences, validation_sentences, train_labels, validation_labels
train_sentences, val_sentences, train_labels, val_labels = train_val_split(sentences, labels, TRAINING_SPLIT)

# GRADED FUNCTION: fit_tokenizer
def fit_tokenizer(train_sentences, num_words, oov_token):
   ### START CODE HERE

   # Instantiate the Tokenizer class, passing in the correct values for num_words and oov_token
   tokenizer = Tokenizer(num_words=num_words, oov_token=oov_token)

   # Fit the tokenizer to the training sentences
   tokenizer.fit_on_texts(train_sentences)

   ### END CODE HERE

   return tokenizer
tokenizer = fit_tokenizer(train_sentences, NUM_WORDS, OOV_TOKEN)
word_index = tokenizer.word_index

# GRADED FUNCTION: seq_and_pad
def seq_and_pad(sentences, tokenizer, padding, maxlen):
   ### START CODE HERE

   # Convert sentences to sequences
   sequences = tokenizer.texts_to_sequences(sentences)

   # Pad the sequences using the correct padding and maxlen
   padded_sequences = pad_sequences(sequences, maxlen=maxlen, padding=padding)

   ### END CODE HERE

   return padded_sequences
train_padded_seq = seq_and_pad(train_sentences, tokenizer, PADDING, MAXLEN)
val_padded_seq = seq_and_pad(val_sentences, tokenizer, PADDING, MAXLEN)

# GRADED FUNCTION: tokenize_labels
def tokenize_labels(all_labels, split_labels):
   ### START CODE HERE

   # Instantiate the Tokenizer (no additional arguments needed)
   label_tokenizer = Tokenizer()

   # Fit the tokenizer on all the labels
   label_tokenizer.fit_on_texts(all_labels)

   # Convert labels to sequences
   label_seq = label_tokenizer.texts_to_sequences(split_labels)

   # Convert sequences to a numpy array. Don't forget to substact 1 from every entry in the array!
   label_seq_np = np.array(label_seq) - 1

   ### END CODE HERE

   return label_seq_np
train_label_seq = tokenize_labels(labels, train_labels)
val_label_seq = tokenize_labels(labels, val_labels)

# GRADED FUNCTION: create_model
def create_model(num_words, embedding_dim, maxlen):
   tf.random.set_seed(123)

   ### START CODE HERE

   model = tf.keras.Sequential([
      tf.keras.layers.Embedding(num_words, embedding_dim, input_length=maxlen),
      Conv1D(128, 5, activation='relu'),
      GlobalMaxPooling1D(),
      Dense(64, activation='relu'),
      Dropout(0.5),
      Dense(1, activation='sigmoid')
   ])

   model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
   loss, accuracy = model.evaluate(val_padded_seq, val_label_seq, verbose=False)
   print(f'Validation accuracy: {accuracy:.4f}')
   model.summary()

   ### END CODE HERE

   return model

model = create_model(NUM_WORDS, EMBEDDING_DIM, MAXLEN)

history = model.fit(train_padded_seq, train_label_seq, epochs=4, validation_data=(val_padded_seq, val_label_seq))

# Epoch 4/4
# 60/60 [==============================] - 0s 5ms/step - loss: 0.0465 - accuracy: 0.9864 - val_loss: 0.2067 - val_accuracy: 0.9375

def plot_graphs(history, metric):
   plt.plot(history.history[metric])
   plt.plot(history.history[f'val_{metric}'])
   plt.xlabel("Epochs")
   plt.ylabel(metric)
   plt.legend([metric, f'val_{metric}'])
   plt.show()

plot_graphs(history, "accuracy")
plot_graphs(history, "loss")


import numpy as np
from .sigmoid import sigm

def getCost(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lambd):
    # Reshape nn_params back into the parameters Theta1 and Theta2, the weight matrices
    Theta1 = nn_params[0:( ( input_layer_size + 1 ) * hidden_layer_size),]
    Theta1 = Theta1.reshape((hidden_layer_size, input_layer_size + 1))
    Theta2 = nn_params[( ( input_layer_size + 1 ) * hidden_layer_size):,]
    Theta2 = Theta2.reshape((num_labels, hidden_layer_size + 1))

    #print('Theta1[24][356]',  Theta1[24][356])
    #print('Theta2[7][23]', Theta2[7][23])
    #print('sum ',np.sum(Theta1 * Theta1) + np.sum(Theta2 * Theta2))
    # input_layer_size 400
    # hidden_layer_size 25
    # num_labels 10
    # Theta1.shape (25, 401)
    # Theta2.shape (10, 26)
    # X.shape (5000, 400)
    # y (5000, 1)

    #print('theta1' , Theta1)
    #print('theta2', Theta2)

    m, n = X.shape # n is similar to input_layer_size
    # First Layer#
    ones = np.ones((m ,1))
    a1 = np.concatenate((ones, X), axis=1) # (5000, 401)

    z1 = a1.dot(Theta1.T) # (5000, 401) X (401, 25) = (5000, 25)
    h1 = sigm(z1) # (5000, 25)

    # Second Layer
    a2 = np.concatenate((ones, h1), axis=1) # (5000, 26)
    z2 = a2.dot(Theta2.T) # (5000, 26) X (26, 10) = (5000, 10)
    h2 = sigm(z2) # (5000, 10)

    log_h2  = np.log(h2) # (5000, 10)
    log_1_h2 = np.log(1 - h2) # (5000, 10)

    # Setting initial cost to 0
    J = 0

    #print(log_h2[1,:])

    for i in range(m):
        val = np.asscalar(y[i]) # value ranging from 1 to 10
        # Reinitializing to zeros column vector
        term_z = np.zeros((num_labels + 1, 1)) # (10, 1)
        term_z[int(val)] = 1 # Setting the particular index to 1, rest all zeros
        lh2 = log_h2[i, :].reshape(1, num_labels) # (1, 10)
        l_1_h2 = log_1_h2[i, :].reshape(1, num_labels)
        cst = lh2.dot(term_z[1:,]) + l_1_h2.dot(1 - term_z[1:,])

        J += (-1/m) * np.asscalar(cst)
        #if(i < 10): print('J', J)

    #print('First J', J)
    # For regualrization, we should not regularize for theta0 term for all Theta values
    Th = Theta1[:]
    Th[:, 0] = 0
    Th2 = Theta2[:]
    Th2[:, 0] = 0

    J_reg = (lambd / ( 2 * m )) * (np.sum(Th * Th) + np.sum(Th2 * Th2))

    #print('J reg', J_reg)
    J += J_reg
    return J

def getGrad(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lambd):
    '''
    Returns the partial derivative or Gradient
    '''
    m, n = X.shape

    # Reshape nn_params back into the parameters Theta1 and Theta2, the weight matrices
    Theta1 = nn_params[0:( ( input_layer_size + 1 ) * hidden_layer_size),]
    Theta1 = Theta1.reshape((hidden_layer_size, input_layer_size + 1))
    Theta2 = nn_params[( ( input_layer_size + 1 ) * hidden_layer_size):,]
    Theta2 = Theta2.reshape((num_labels, hidden_layer_size + 1))

    ones = np.ones((1 ,1))
    for i in range(m):
        # First Layer#
        x1 = X[i, :] # (1, 400)

        # Second Layer
        a1 = np.concatenate((ones, x1), axis=1) # (1, 401)
        z1 = a1.dot(Theta1.T) # (1, 401) X (401, 25) = (1, 25)
        h1 = sigm(z1) # (1, 25)

        # Third Layer
        a2 = np.concatenate((ones, h1), axis=1) # (1, 26)
        z2 = a2.dot(Theta2.T) # (1, 26) X (26, 10) = (1, 10)
        h2 = sigm(z2) # (1, 10)
    pass


# Question: https://leetcode.com/problems/rotating-the-box/
# Medium
# Can be optimized
from typing import Optional, List

class Solution:
    # O(HEIGHT * (WIDTH^2)) Time and O(1) Space
    def rotateTheBox(self, box: List[List[str]]) -> List[List[str]]:
        HEIGHT, WIDTH = len(box), len(box[0])

        for row in range(HEIGHT):
            for col in reversed(range(WIDTH)):
                if box[row][col] == '#':
                    idx = col + 1

                    while idx < WIDTH and box[row][idx] == '.':
                        idx += 1

                    idx -= 1
                    box[row][col] = '.'
                    box[row][idx] = '#'

        return zip(*box[::-1])
'''

# Kunal Wadhwa

'''


import numpy as np

n = int(input())
a = [list(map(int, input().split())) for _ in range(n)]
solo = 0
s = []

for i, x in enumerate(a):
  flag = False

  if [x[0]-1, x[1]-1] in a:
    flag = True
    s.append([i, a.index([x[0]-1, x[1]])])

  if [x[0]-1, x[1]] in a:
    flag = True
    s.append([i, a.index([x[0]-1, x[1]])])

  if [x[0], x[1]-1] in a:
    flag = True
    s.append([i, a.index([x[0], x[1]-1])])

  if [x[0], x[1]+1] in a:
    flag = True
    s.append([i, a.index([x[0], x[1]+1])])

  if [x[0]+1, x[1]] in a:
    flag = True
    s.append([i, a.index([x[0]+1, x[1]])])

  if [x[0]+1, x[1]+1] in a:
    flag = True
    s.append([i, a.index([x[0], x[1]+1])])

  if not flag:
    solo +=1

print(solo)
s =  np.sort(s).tolist()
s = list(map(list, set(map(tuple, s))))
print(s)

for x in s:
  for y in x:
    print(y)


"""
    Minimizing autocorrelation of bitarray in CPMpy

    Problem 005 on CSPlib

    Model created by Ignace Bleukx, ignace.bleukx@kuleuven.be
"""
import argparse

import numpy as np

from cpmpy import *

def auto_correlation(n=16):

    # bitarray of length n
    arr = intvar(-1,1,shape=n, name="arr")

    model = Model()

    # exclude 0
    model += arr != 0

    # minimize sum of squares
    model.minimize(
        sum([PAF(arr,s) ** 2 for s in range(1,n)])
    )

    return model, (arr,)

# periodic auto correlation
def PAF(arr, s):
    # roll the array 's' indices
    return sum(arr * np.roll(arr,-s))

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("-length", nargs='?', type=int, default=16, help="Length of bitarray")

    length = parser.parse_args().length

    model, (arr,) = auto_correlation(length)

    if model.solve():
        # print using runlength notation
        arr = arr.value()
        pieces = np.split(arr, np.where(arr == -1)[0])

        print("Run length encoding of solution:")
        print("".join([str(len(p)) for p in pieces if len(p) != 0]))

    else:
        raise ValueError("Model is unsatisfiable")


import os
import requests
from bs4 import BeautifulSoup
import pandas as pd
from requests import get

## Blog Articles

def get_blog_article_urls():
    '''
    This function makes a request to the Codeup blog site and gathers the urls by analyzing the returned html text. Returns the list of urls.
    '''
    headers = {'User-Agent': 'Codeup Data Science'}
    response = requests.get('https://codeup.com/blog/', headers= headers)
    soup = BeautifulSoup(response.text)
    # Getting the url by the a class subcategory 'more-link'
    urls = [a.attrs['href'] for a in soup.select('a.more-link')]
    return urls

def parse_blog_article(soup):
    '''
    This function parses each individual article to gather the title, content, and published date.
    '''
    return {
        'title': soup.select_one('h1.entry-title').text,
        'content': soup.select_one('.entry-content').text.strip(),
        'published': soup.select_one('.published').text
    }

def get_blog_articles():
    '''
    This function returns a dataframe of the articles from the Codeup blog site.
    Inside the function the list of urls is used to gather the article's html and then parse it for title, content, and date published.
    '''
    urls = get_blog_article_urls()
    articles = []
    headers = {'User-Agent': 'Codeup Data Science'}

    for url in urls:
        response = requests.get(url, headers= headers)
        soup = BeautifulSoup(response.text)
        articles.append(parse_blog_article(soup))

    df = pd.DataFrame(articles)
    return df

## Inshort News articles

def get_news_articles():
    '''
    This function returns a dataframe after acquiring the selected categories of news from Inshorts. It parses the html to produce dictionaries that are then
    converted to a larger dataframe containing the category, title, content, author, and published date of the article.
    '''
    categories = ['business', 'sports', 'entertainment', 'technology']
    articles = []

    for category in categories:
        print(f'getting articles for {category}')
        category_articles = parse_inshorts_page(category)
        articles.extend(category_articles)

    return pd.DataFrame(articles)

def parse_inshorts_page(category):
    '''
    This function parses the html from Inshorts using an input category. It creates a list of dictionaries for each article and returns the list.
    '''
    url = 'http://www.inshorts.com/en/read/' + category
    response = get(url)
    soup = BeautifulSoup(response.text)
    articles = [parse_news_card(card, category) for card in soup.select('.news-card')]
    return articles

def parse_news_card(card, category):
    '''
    Take in a news card and return a dictionary.
    '''
    output = {}
    output['category'] = category
    output['title'] = card.select_one('a.clickable').text.strip()
    card_content = card.select_one('div.news-card-content')
    output['content'] = card_content.select_one('div').text
    output['author'] = card_content.select_one('.author').text
    output['published'] = card_content.select_one('.time').attrs['content']

    return output


from ged import compute_geds
from distances import sum_squares, euclid_d
import numpy as np
# from tqdm import tqdm

import sys

# sys.path.insert(0, "../")

def optimize_costs_unlabeled(nb_cost_mat, dis_k_vec):
	"""
	Optimize edit costs to fit dis_k_vec according to edit operations in nb_cost_mat
	! take care that nb_cost_mat do not contains 0 lines
	:param nb_cost_mat: \in \mathbb{N}^{N x 6} encoding the number of edit operations for each pair of graph
	:param dis_k_vec: The N distances to fit
	"""
	import cvxpy as cp
	import numpy as np
	MAX_SAMPLE = 1000
	nb_cost_mat_m = np.array([[x[0], x[1], x[3], x[4]] for x in nb_cost_mat])
	dis_k_vec = np.array(dis_k_vec)
	# dis_k_vec_norm = dis_k_vec/np.max(dis_k_vec)

	# import pickle
	# pickle.dump([nb_cost_mat, dis_k_vec], open('debug', 'wb'))
	N = nb_cost_mat_m.shape[0]
	sub_sample = np.random.permutation(np.arange(N))
	sub_sample = sub_sample[:MAX_SAMPLE]

	x = cp.Variable(nb_cost_mat_m.shape[1])
	cost = cp.sum_squares(
		(nb_cost_mat_m[sub_sample, :] @ x) - dis_k_vec[sub_sample]
	)
	prob = cp.Problem(cp.Minimize(cost), [x >= 0])
	prob.solve()
	edit_costs_new = [x.value[0], x.value[1], 0, x.value[2], x.value[3], 0]
	edit_costs_new = [xi if xi > 0 else 0 for xi in edit_costs_new]
	residual = prob.value
	return edit_costs_new, residual

def optimize_costs_classif_unlabeled(nb_cost_mat, Y):
	"""
	Optimize edit costs to fit dis_k_vec according to edit operations in
	nb_cost_mat
	! take care that nb_cost_mat do not contains 0 lines
	:param nb_cost_mat: \in \mathbb{N}^{N x 6} encoding the number of edit
	operations for each pair of graph
	:param dis_k_vec: {-1,1}^N vector of common classes
	"""
	# import cvxpy as cp
	from ml import reg_log
	# import pickle
	# pickle.dump([nb_cost_mat, Y], open('debug', 'wb'))
	nb_cost_mat_m = np.array(
		[[x[0], x[1], x[3], x[4]]
		 for x in nb_cost_mat]
	)
	w, J, _ = reg_log(nb_cost_mat_m, Y, pos_contraint=True)
	edit_costs_new = [w[0], w[1], 0, w[2], w[3], 0]
	residual = J[-1]

	return edit_costs_new, residual

def optimize_costs_classif(nb_cost_mat, Y):
	"""
		Optimize edit costs to fit dis_k_vec according to edit operations in nb_cost_mat
		! take care that nb_cost_mat do not contains 0 lines
		:param nb_cost_mat: \in \mathbb{N}^{N x 6} encoding the number of edit operations for each pair of graph
		:param dis_k_vec: {-1,1}^N vector of common classes
	"""
	# import pickle
	# pickle.dump([nb_cost_mat, Y], open("test.pickle", "wb"))
	from ml import reg_log
	w, J, _ = reg_log(nb_cost_mat, Y, pos_contraint=True)
	return w, J[-1]

def optimize_costs(nb_cost_mat, dis_k_vec):
	"""
	Optimize edit costs to fit dis_k_vec according to edit operations in nb_cost_mat
	! take care that nb_cost_mat do not contains 0 lines
	:param nb_cost_mat: \in \mathbb{N}^{N x 6} encoding the number of edit operations for each pair of graph
	:param dis_k_vec: The N distances to fit
	"""
	import cvxpy as cp
	x = cp.Variable(nb_cost_mat.shape[1])
	cost = cp.sum_squares((nb_cost_mat @ x) - dis_k_vec)
	constraints = [
		x >= [0.01 for i in range(nb_cost_mat.shape[1])],
		np.array([1.0, 1.0, -1.0, 0.0, 0.0, 0.0]).T @ x >= 0.0,
		np.array([0.0, 0.0, 0.0, 1.0, 1.0, -1.0]).T @ x >= 0.0
	]
	prob = cp.Problem(cp.Minimize(cost), constraints)
	prob.solve()
	edit_costs_new = x.value
	residual = prob.value

	return edit_costs_new, residual

def compute_optimal_costs(
		G, y, init_costs=[3, 3, 1, 3, 3, 1],
		y_distance=euclid_d,
		mode='reg', unlabeled=False,
		ed_method='BIPARTITE',
		**kwargs
):
	N = len(y)

	G_pairs = []
	distances_vec = []

	for i in range(N):
		for j in range(i + 1, N):
			G_pairs.append([i, j])
			distances_vec.append(y_distance(y[i], y[j]))
	ged_vec_init, n_edit_operations = compute_geds(
		G_pairs, G, init_costs, ed_method, **kwargs
	)

	residual_list = [sum_squares(ged_vec_init, distances_vec)]

	if (mode == 'reg'):
		if unlabeled:
			method_optim = optimize_costs_unlabeled
		else:
			method_optim = optimize_costs

	elif (mode == 'classif'):
		if unlabeled:
			method_optim = optimize_costs_classif_unlabeled
		else:
			method_optim = optimize_costs_classif

	ite_max = 5
	for i in range(ite_max):
		print('ite', i + 1, '/', ite_max, ':')
		# compute GEDs and numbers of edit operations.
		edit_costs_new, residual = method_optim(
			np.array(n_edit_operations), distances_vec
		)
		ged_vec, n_edit_operations = compute_geds(
			G_pairs, G, edit_costs_new, ed_method, **kwargs
		)
		residual_list.append(sum_squares(ged_vec, distances_vec))

	return edit_costs_new


class Employee:
    _company="Futurense"
    __salary=19191

    def infoData():
        print(Employee.__salary)

class HRDepartment(Employee):
    department="HR"

#  name mangling => concept where the python the variable name if you put __ in the starting
e1=Employee()

#  __salary => _ClassName__salary

print(e1._company, Employee._Employee__salary)
Employee.infoData()

h1 = HRDepartment()
print(h1.department, h1._Employee__salary, "HR department")


import setuptools

with open('README.md', 'r') as fh:
    long_description = fh.read()

with open('requirements.txt', 'r') as f:
    requirements = f.read().splitlines()
    requirements = [l for l in requirements if not l.startswith('#')]

setuptools.setup(
    name='fanc-fly',
    version='3.1.0',
    author='Jasper Phelps',
    author_email='jasper.s.phelps@gmail.com',
    description='Tools for the Female Adult Nerve Cord Drosophila EM dataset',
    long_description=long_description,
    long_description_content_type='text/markdown',
    url='https://github.com/htem/FANC_auto_recon',
    packages=setuptools.find_packages(),
    package_data={'fanc.transforms': ['transform_parameters/*.txt']},
    classifiers=[
        'Programming Language :: Python :: 3',
        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
        'Operating System :: OS Independent',
    ],
    python_requires='>=3.6',
    install_requires=requirements
)


import logging
import os
import sys
import warnings
from datetime import datetime
from os import listdir
from os.path import isfile, join

import numpy as np
import pandas as pd

from util_functions.csv_export.csv_export import csv_export
from util_functions.fixes.fix_col_dates import check_na_col_dates
from util_functions.fixes.fix_enum import fix_enum
from util_functions.fixes.fix_lot_productCode import fix_lot_product_code
from util_functions.fixes.fix_urls import check_urls
from util_functions.fixes.masterid_modification import masterid_modification
from util_functions.importing.clean_text_columns import clean_text_columns
from util_functions.importing.import_file import import_data_file, reading_mapper
from util_functions.mini_jsons.indicators import indicators_mini_json
from util_functions.mini_jsons.sanctions import sanctions
from util_functions.set_up_logging import set_up_logging
from util_functions.util.move_util_files import move_util_files

def generate_indicators(filename_csv, country, chunk_it=None, chunk_size=10):
    """
    Takes in a data frame and generates indicators
    :param chunk_size: The number of chunks to split the data to using IDs
    :param chunk_it: chunk the data
    :param filename_csv: The name of the starting CSV file
    :param country: ISO-2 country code
    :return: Main data frame with indicators
    """
    print(os.getcwd())

    df, column_names = import_data_file(filename_csv=filename_csv, country=country)
    print(f"processing {country} with {len(df)} rows")
    # rename variables
    logging.info(f"Renaming columns...")
    # rename_mapper = reading_mapper(filename=f'../country_codes/{country}/name_mapper.txt')
    rename_mapper = reading_mapper(country=country, sheet_name="columns")
    df = df.rename(columns=rename_mapper)
    # PT FIX for
    df.loc[df.tender_id == "64c85640-8297-4ab7-b0c3-866880f0bd9a", "tender_addressofimplementation_country"] = "AT"
    # FIX CO master IDs
    if country == "CO":
        df = df.loc[df.buyer_masterid.notnull()]
        df = df.loc[df.bidder_masterid.notnull()]
        print(f"processing {country} after dropping missing master IDs with {len(df)} rows")

    col_text = ['buyer_name',
                'buyer_id',
                'buyer_masterid',
                'bidder_name',
                'bidder_id',
                'bidder_masterid',
                'lot_title']

    df = clean_text_columns(df, col_text)
    df = masterid_modification(df=df, country=country)
    # lot_productCode -- > lot_productcode
    df = fix_lot_product_code(df=df, country=country)
    # col_dates
    col_dates = ['tender_awarddecisiondate',
                 'tender_contractsignaturedate',
                 'tender_publications_firstdcontractawarddate']
    df = check_na_col_dates(df=df, col_dates=col_dates, country=country)

    if country in ['UK', 'DE', 'ES']:
        logging.info(f"special fix for {country}")
        logging.info(f"current number of rows {len(df)}")
        df = df.dropna(subset=['bidder_name']).reset_index()
        logging.info(f"dropping missing bidder_name...")
        logging.info(f"current number of rows {len(df)}")

    # logging.info(f'setting [""] in {col_name} no NA')
    # df.loc[df['buyer_mainactivities'] == '[""]', 'buyer_mainactivities'] = np.nan
    df = fix_enum(df=df, country=country)
    url_values = []
    url_list = ["tender_publications_lastcontractawardurl", "notice_url", "source"]
    df = check_urls(df=df, url_list=url_list)
    prices = ['bid_priceusd']
    logging.info(f"Finished renaming and cleaning columns")
    logging.info(f"Generating indicators' JSONs starting...")
    for entity_country in ['buyer_country', 'bidder_country', 'tender_country']:
        print(f"Fixing country codes in {entity_country} that are GB to UK")
        logging.info(f"Fixing country codes in {entity_country} that are GB to UK")
        try:
            df.loc[df[entity_country] == "GB", entity_country] = "UK"
            df.loc[df[entity_country] == 'British lndian Ocean Territory', entity_country] = "IO"

        except KeyError:
            print(f"{entity_country} is not available")
            pass

    # generate CSV export
    print("generate CSV export..")
    df_csv = csv_export(df=df, country=country)
    df_csv.to_csv(f"csv_export/{country}_export.csv.gz",
                  index=False, encoding='utf-8')
    del df_csv
    print(f"generate CSV export done. File in csv_export/{country}_export.csv")
    # generate indicators data
    if chunk_it:
        print(f"Chunking data for {country} into {chunk_size} chunks")
        logging.info(f"Chunking data for {country} into {chunk_size} chunks")
        df_groups = df.groupby('tender_id').ngroups
        range_groups_df = range(df_groups + 1)
        range_groups_df = np.array_split(np.array(range_groups_df), chunk_size)
        df['id_group'] = df.groupby('tender_id').ngroup()
        df_full = pd.DataFrame()
        for index_df, range_out_df in enumerate(range_groups_df):
            df_chunk = df[df['id_group'].isin(range_out_df)]
            print(f"processing range {index_df+1}, with chunk size of {len(df_chunk)}")
            logging.info(f"processing range {index_df}, with chunk size of {len(df_chunk)}")
            df_chunk = indicators_mini_json(df=df_chunk)
            logging.info(f"Selecting columns....")
            final_cols = list(rename_mapper.values()) + ["indicators"]
            for col_to_drop in ['bidder_previousSanction',
                                'bidder_hasSanction', 'sanct_startdate',
                                'sanct_enddate', 'sanct_name',
                                'lot_bidscount']:
                try:
                    final_cols.remove(col_to_drop)
                except ValueError:
                    continue
            df_chunk = df_chunk[final_cols]
            df_full = pd.concat([df_full, df_chunk])
            df_full = df_full.replace({'[""]': np.nan})  # temp
        logging.info(f"Processing sanctions...")
        df_full = sanctions(df=df_full, country=country)
        return df_full

    else:
        df = indicators_mini_json(df=df)
        logging.info(f"Selecting columns....")
        final_cols = list(rename_mapper.values()) + ["indicators"]
        for col_to_drop in ['bidder_previousSanction',
                            'bidder_hasSanction', 'sanct_startdate',
                            'sanct_enddate', 'sanct_name',
                            'lot_bidscount']:
            try:
                final_cols.remove(col_to_drop)
            except ValueError:
                continue
        df = df[final_cols]
        df = df.replace({'[""]': np.nan})  # temp
        logging.info(f"Processing sanctions...")
        df = sanctions(df=df, country=country)
        return df

    # processing sanctions output from matching in R

if __name__ == '__main__':
    warnings.simplefilter(action='ignore', category=FutureWarning)
    time_in = datetime.now()
    logging.info("making directories")
    os.makedirs("mod_ind_df", exist_ok=True)
    os.makedirs("csv_export", exist_ok=True)
    os.makedirs("validation_node/datasets", exist_ok=True)
    os.makedirs("validation_node/logs", exist_ok=True)

    country = str(sys.argv[1].split('/')[-1]).upper()
    try:
        chunk = str(sys.argv[2].split('/')[-1])
    except IndexError:
        chunk = False
        print("processing data as whole")
        pass
    try:
        chunk_it = str(sys.argv[3].split('/')[-1])
        chunk_it = True
        chunk_size = int(sys.argv[4].split('/')[-1])
    except IndexError:
        chunk_it = False
        chunk_size = 0
        print("processing data as whole with no chunks")
        pass
    set_up_logging(sys.argv[1].split('/')[-1], time_in)
    fileName_CSV = f"../utility_data/country/{country}/{country}_mod.csv"
    logging.info(f"Starting process...")

    current_path = f"../utility_data/country/{country}/"
    country_files = [f for f in listdir(current_path) if isfile(join(current_path, f)) and f[-3:] == 'zip']

    df_final = generate_indicators(filename_csv=fileName_CSV, country=country,
                                   chunk_it=chunk_it, chunk_size=chunk_size)

    if chunk == "chunk":
        df_final_groups = df_final.groupby('tender_id').ngroups
        range_groups = range(df_final_groups + 1)
        range_groups = np.array_split(np.array(range_groups), chunk_size)
        df_final['id_group'] = df_final.groupby('tender_id').ngroup()
        os.makedirs(f"mod_ind_df/{country}", exist_ok=True)
        for index, range_out in enumerate(range_groups):
            # df_final_chunk = df_final.loc[range_out, :]
            df_final_chunk = df_final[df_final.id_group.isin(range_out)]
            os.makedirs(f"mod_ind_df/{country}/{country}_{index + 1}/{country}", exist_ok=True)
            outfile = f"mod_ind_df/{country}/{country}_{index + 1}/{country}/{country}_mod_ind.csv"
            move_util_files(country=country, des_file=f"mod_ind_df/{country}/{country}_{index + 1}/{country}")
            print(f"copying tool to {outfile} dir")
            source_file = "reverse-flatten-tool"
            des_file = f"mod_ind_df/{country}/{country}_{index + 1}/reverse-flatten-tool-{country.lower()}-{index + 1}"
            os.system(f"\cp -fr {source_file} -T {des_file}")
            # shutil.copytree(source_file, des_file,)
            os.system(f"bash run_country_chunks.sh -c {country} -x {index+1}")
            logging.info(f"Exporting {outfile}...")
            df_final_chunk.to_csv(outfile, sep=';', index=False, encoding='utf-8')
            print(f"File saved in {outfile}")
        # move_util_files(country=country, des_file=f"mod_ind_df/{country}")
    else:
        outfile = f"../country_codes/{country}/{country}_mod_ind.csv"
        logging.info(f"Exporting {outfile}...")

        df_final.to_csv(outfile, sep=';', index=False, encoding='utf-8'
                        # , quoting=csv.QUOTE_NONNUMERIC
                        )
        print(f"File saved in {outfile}")

        move_util_files(country=country)
    time_out_temp = datetime.now()
    logging.info(f"Finished")
    logging.info(f"Run Time of Processing {time_out_temp - time_in}")
    print(f"Run Time of Processing for {country} {time_out_temp - time_in}")


import numpy as np
import matplotlib.pyplot as plt
from scipy import signal
from bandit import Bandits

import glob
import os
import imageio

class UCB_Single_Freq():
    """
    The upper confidence bound algorithm
    (one in RL course slides)
    N      - Number of experiment
    l_na   - number of times a played up to t
    theta  - empirical reward of a up to t
    bandit - bandit class that store info about this
    """
    def __init__(self,bandit,sigma,N,name,tag):
        self.name      = name
        self.bandit    = bandit
        self.N         = N
        self.l_na      = np.zeros((self.bandit.K,1))
        self.sigma     = sigma
        self.regret    = [0]
        self.emp_rewards = np.random.random((self.bandit.K,1)) ### empirical rewards
        self.MSE         = []
        self.MSE1        = []
        self.tag = tag
        self.current_arm = 0
#         self.logregret   = []
    def compute_regret(self,arms):

        current_regret   = self.regret[-1] + self.bandit.maxnorm - self.bandit.norm[arms]

        self.regret.append(current_regret)

    def training(self,SHOWED = True,savefig=False,frame = 100):
        #### clear the previous values // re-initialize the values

        self.regret      = [0]
        self.l_na        = np.zeros((self.bandit.K,1))
        self.regret      = [0]
        self.MSE         = []
        self.MSE1        = []

        mse              = 0
        #self.theta       = np.zeros((bandit.K,2))
        self.emp_rewards = np.random.random((self.bandit.K,1))
        self.SUM_PX      = np.zeros((self.bandit.K,2))
        if savefig:
            if not os.path.exists(os.path.join('./Resultfig',"Result_GIF")):
                os.makedirs(os.path.join('./Resultfig',"Result_GIF"))
            filename = os.path.join('./Resultfig',self.name + "_"+self.tag+".gif")
            writer = imageio.get_writer(filename, format='GIF-PIL', mode='I', fps= 10)

        for t in range(self.N):
            ### SELECT ARMS
            if t < self.bandit.K:
                self.current_arm = t
            else:
                self.current_arm = np.argmax(self.emp_rewards +  self.sigma*np.sqrt(2*np.log(t+2)/self.l_na))

            self.l_na[self.current_arm] +=1

            ### perform the experiment and obtain X
            mean = [self.bandit.hnorm[self.current_arm].real,self.bandit.hnorm[self.current_arm].imag]
            cov  = np.identity(2) * self.sigma**2 / 2
            x_t  = np.random.multivariate_normal(mean,cov,1) ### rewards

            self.SUM_PX[self.current_arm][:] += x_t[0]

            ### update the emp_rewards
            self.emp_rewards[self.current_arm]    = (self.SUM_PX[self.current_arm][0]**2 + self.SUM_PX[self.current_arm][1]**2) / self.l_na[self.current_arm]**2

            ### update regret and rewards
            mse  = (np.nanmax(self.emp_rewards) - self.bandit.real_norm)**2
            self.MSE.append(mse)
            mse1  = (self.emp_rewards[np.nanargmax(self.l_na)] - self.bandit.real_norm)**2
            self.MSE1.append(mse1)
            self.compute_regret(self.current_arm)
            if savefig and t < frame:
                files = self.__create_png(t)
                image = imageio.imread(files)
                writer.append_data(image)
                os.remove(files)

    def __create_png(self,i):
        if not os.path.exists(os.path.join("./Resultfig","Result_png")):
            os.makedirs(os.path.join("./Resultfig","Result_png"))
        filename = os.path.join("./Resultfig","Result_png","empirical_"+str(i)+".png")
        plt.figure()
        plt.plot(self.bandit.arms,self.bandit.norm, label = '|G(e^{j$\omega_k$})|')
        dy  = self.sigma*np.sqrt(2*np.log(i+2)/(self.l_na+1e-10))
        y1  = [float(self.emp_rewards[inn] - dy[inn]) for inn in range(self.bandit.K)]
        y2  = [float(self.emp_rewards[inn] + dy[inn]) for inn in range(self.bandit.K)]
        plt.fill_between(self.bandit.arms, y1, y2,color='gray', alpha=0.2)
        plt.scatter(self.bandit.arms,self.emp_rewards,color = 'orange',label = r'|$\^G$(e^{j$\omega$})|')
        plt.scatter(self.bandit.arms[self.current_arm],self.emp_rewards[self.current_arm],color = 'red',label = r'|$\^G$(e^{j$\omega^*$})|')
        plt.legend()
        plt.xlabel('Frequency $\omega$ [rad/s]')
        plt.ylabel(r'|$\hat{G(j\omega)}$)|')
        plt.title("The empirical reward in "+self.name)
        plt.ylim([-1,2])
        plt.savefig(filename)
        plt.close()
        return filename

class UCB_Power_Allocation :
    def __init__(self,bandit,sigma,N,prior,name,beta,tag):
        self.name        = name
        self.bandit      = bandit
        self.N           = N
        self.pho         = prior
        self.prior       = prior
        self.emp_rewards = np.random.random((self.bandit.K,1)) ### empirical rewards
        self.l_na        = np.zeros((self.bandit.K,1))
        self.SUM_PX      = np.zeros((self.bandit.K,2))
        self.sigma       = sigma
        self.regret      = [0]
        self.MSE         = []
        self.MSE1        = []
        self.run_list    = [i for i in range(self.bandit.K)]
        self.beta        = beta
        self.tag         = tag

    def Calculate_Bounds(self,t):

        self.bounds = [self.emp_rewards[arms] + np.sqrt(2*self.sigma**2)*np.sqrt((np.log(t+1)/self.l_na[arms])) for arms in range(self.bandit.K)]
        self.bounds = np.array(self.bounds)

    def compute_regret(self,t):
        delta_t         = [self.pho[i] * (self.bandit.maxnorm - self.bandit.norm[i]) for i in range(self.bandit.K)]
        current_regret  = self.regret[-1] + np.sum(delta_t)
        self.regret.append(np.copy(current_regret))
    def reset_prior(self):
        self.prior            = [1/self.bandit.K] * self.bandit.K
        self.prior           /= np.sum(self.prior)

    def training(self, SHOWED = True,savefig=False,frame=0):
        #### clear the previous values // re-initialize the values
        self.reset_prior
        self.regret      = [0]
        self.l_na        = np.zeros((self.bandit.K,1))
        self.SUM_PX      = np.zeros((self.bandit.K,2))
        self.emp_rewards = np.zeros((self.bandit.K,1))
        self.pho         = self.prior
        self.bounds      = np.zeros((self.bandit.K,1))
        self.MSE         = []
        self.run_list    = [i for i in range(self.bandit.K)]
        if savefig:
            if not os.path.exists(os.path.join('./Resultfig',"Result_GIF")):
                os.makedirs(os.path.join('./Resultfig',"Result_GIF"))
            filename = os.path.join('./Resultfig',"Result_GIF",self.name + "_Power_Allocation_"+self.tag+".gif")
            writer = imageio.get_writer(filename, format='GIF-PIL', mode='I', fps= 10)

        for t in range(self.N):
            ### SELECT ARMS
            for arms in self.run_list :

                ### perform the experiment and obtain X
                if self.pho[arms] != 0:
                    mean = [self.bandit.hnorm[arms].real,self.bandit.hnorm[arms].imag]
                    cov  = np.identity(2) * self.sigma**2 / (2*self.pho[arms])
                    x_t  = np.random.multivariate_normal(mean,cov,1) ### rewards

                    ### calculate rewards
                    self.SUM_PX[arms][0] += self.pho[arms] * x_t[0][0]
                    self.SUM_PX[arms][1] += self.pho[arms] * x_t[0][1]
                    self.l_na[arms]      += self.pho[arms]

                    ### update the emp_rewards
                    self.emp_rewards[arms]    = (self.SUM_PX[arms][0]**2 + self.SUM_PX[arms][1]**2) / self.l_na[arms]**2

            self.Calculate_Bounds(t)
            if savefig and t < frame:
                files = self.__create_png(t)
                image = imageio.imread(files)
                writer.append_data(image)
                os.remove(files)
            self.Update_Pho(t,SHOWED)
            mse  = (np.nanmax(self.emp_rewards) - self.bandit.real_norm)**2
            self.MSE.append(mse)
            mse  = (self.emp_rewards[np.nanargmax(self.l_na)] - self.bandit.real_norm)**2
            self.MSE1.append(mse)
            ### update regret and reward

            self.compute_regret(t)

    def Update_Pho(self,t,SHOWED = False):
        r             = self.beta*np.sqrt(2*self.sigma**2)*(np.log(t+2))/(2+t)
        z             = 0
        T             = 0
        bar           = list(self.bounds.T[0])
        a             = np.nanmax(bar)
        s             = [i  for i in bar if i != a]
        N             = self.bandit.K - len(s)
        b             = 0
        self.run_list = []
        while (T < r) and (N < self.bandit.K) :
            b         = np.max(s)
            T         = T + N * (a - b)
            if T > r:
                break
            a          = b
            s          = [i  for i in s if i!= b]
            N          = self.bandit.K - len(s)
            # print('b = ',b,'T = ',T,'a = ',a,'N = ',N,'s = ',s)
        if t == 0:
            self.pho = [1/self.bandit.K] *self.bandit.K
        else:
            if N >= self.bandit.K:
                self.pho       = [self.bounds[i] for i in range(self.bandit.K)]
                self.run_list  = np.nonzero(self.pho)[0]
                self.pho      /= np.sum(self.pho)
            else:
                z              = b + (T-r)/N
                self.pho       = [a - z if a > z else 0 for a in self.bounds.T[0] ]
                self.run_list  = np.nonzero(self.pho)[0]
                self.pho      /= np.sum(self.pho)

        if SHOWED == True:
            print("t = ",t," pho = ",self.pho)
            plt.figure()
            plt.scatter(range(self.bandit.K),self.bounds,color = 'blue')
            plt.plot(range(self.bandit.K),self.bounds,color = 'blue')
            plt.bar(range(self.bandit.K),self.pho,color = 'r')
            plt.show()
            input("Press enter to continue...")

    def __create_png(self,i):
        if not os.path.exists(os.path.join("./Resultfig","Result_png")):
            os.makedirs(os.path.join("./Resultfig","Result_png"))
        filename = os.path.join("./Resultfig","Result_png","empirical_"+str(i)+".png")
        plt.figure()
        plt.plot(self.bandit.arms,self.bandit.norm, label = '|G(e^{j$\omega_k$})|')
        dy  = self.sigma*np.sqrt(2*np.log(i+2)/(self.l_na+1e-10))
        y1  = [float(self.emp_rewards[inn] - dy[inn]) for inn in range(self.bandit.K)]
        y2  = [float(self.emp_rewards[inn] + dy[inn]) for inn in range(self.bandit.K)]
        plt.fill_between(self.bandit.arms, y1, y2,color='gray', alpha=0.2)
        for current_arm,i_pho in enumerate(self.pho):

            if float(i_pho) > 0.0:
                plt.scatter(self.bandit.arms[current_arm],self.emp_rewards[current_arm],color = 'red',label = r'|$\^G$(e^{j$\omega^*$})|')
            else:
                plt.scatter(self.bandit.arms,self.emp_rewards,color = 'orange',label = r'|$\^G$(e^{j$\omega$})|')

        plt.xlabel('Frequency $\omega$ [rad/s]')
        plt.ylabel(r'|$\hat{G(j\omega)}$)|')
        plt.title("The empirical reward in "+self.name)
        plt.ylim([-1,2])
        plt.savefig(filename)
        plt.close()
        return filename


#!/usr/bin/python
# -*- coding: utf-8 -*-

from flask import Flask, jsonify, request
from tinydb import Query, TinyDB
from crypt import AESCipher, bcrypt_checkpw, bcrypt_encrypt
import time
from totp import Totp
from expiringdict import ExpiringDict

app = Flask(__name__)
app.secret_key = '19#^f8kV8D55h!TXruYum%^VQh$zsCeQ'

DB = TinyDB('./db.json')
query = Query()

passphrase = "6V5#645L5FJn#Zc9i^i$b^3*38dShnk$"
aes = AESCipher(passphrase)

CACHE = ExpiringDict(max_len=10000, max_age_seconds=60)

def response(status, message, result=None):
    return jsonify({
        "status": status,
        "message": message,
        "result": result
    })

def response_success(result=None):
    return response(2000, "success", result)

@app.route('/api/create_seed', methods=['POST'])
def create_seed():
    """生成一个seed和对应的url"""
    username = request.form.get('username')
    tp = Totp()
    tp.create_seed()
    return response_success(result={"url": tp.create_url(username), "seed": tp.seed})

@app.route('/api/save_seed', methods=['POST'])
def save_seed():
    """提供用户名、seed和最近两次token，保存数据"""
    username = request.form.get('username')
    seed = request.form.get('seed')
    pin = request.form.get('pin')
    first_token = request.form.get("first_token")
    second_token = request.form.get("second_token")
    # first_token和second_token不能重复
    if first_token == second_token:
        return response(status=4000, message="verify seed fail")
    # 已存在的用户不允许再添加seed
    if DB.search(query.username == username):
        return response(status=4004, message="username exists")

    tp = Totp(seed=seed)
    # 计算服务器和本地的时间差
    jet_lag_unit = tp.calc_jet_lag_unit(first_token, second_token)
    if jet_lag_unit is None:
        # 如果超过了允许的时间差，则验证不通过
        return response(status=4000, message="verify seed fail")
    else:
        # seed aes256加密
        seed_aes = aes.encrypt(seed)
        # 不允许出现两个一样的seed，提高安全性
        if DB.search(query.seed == seed_aes):
            return response(status=4001, message="seed repeat")
        # 保存到db
        DB.insert({'username': username, 'seed': seed_aes, 'created': int(time.time()), "jet_lag_unit": jet_lag_unit, "pin": bcrypt_encrypt(pin)})
        return response_success()

@app.route('/api/resync', methods=['POST'])
def resync():
    """时间同步"""
    username = request.form.get('username')
    first_token = request.form.get("first_token")
    second_token = request.form.get("second_token")
    # first_token和second_token不能重复
    if first_token == second_token:
        return response(status=4000, message="verify seed fail")
    # 获取当前用户
    users = DB.search(query.username == username)
    if not users:
        return response(status=4002, message="username not exists")

    user = users[0]
    seed = aes.decrypt(user.get("seed"))
    tp = Totp(seed=seed)
    # 计算服务器和本地的时间差
    jet_lag_unit = tp.calc_jet_lag_unit(first_token, second_token)
    if jet_lag_unit is None:
        # 如果超过了允许的时间差，则验证不通过
        return response(status=4000, message="verify seed fail")
    else:
        # 更新时间差
        DB.update({'jet_lag_unit': jet_lag_unit}, query.username == username)
        return response_success()

@app.route('/api/verify_token', methods=['POST'])
def verify_token():
    """验证用户的token是否正确"""
    username = request.form.get('username')
    ptoken = request.form.get("ptoken")

    pin = ptoken[:-6]
    token = ptoken[-6:]

    # 如果用户不存在，抛出错误
    users = DB.search(query.username == username)
    if not users:
        return response(status=4002, message="username not exists")

    user = users[0]
    seed = aes.decrypt(user.get("seed"))
    jet_lag_unit = user.get("jet_lag_unit", 0)
    pin_hashed = user.get("pin")
    tp = Totp(seed=seed)
    if tp.token_verify(token, jet_lag_unit=jet_lag_unit) and bcrypt_checkpw(pin, pin_hashed):
        # 不能直接使用token缓存，防止多个人的token在一个时间内一致导致token不可用的情况。
        cache_key = username + token
        # 确保token只能被成功使用一次
        if CACHE.get(cache_key):
            return response(status=4003, message="token verify fail")
        else:
            CACHE[cache_key] = 1
        return response_success()
    else:
        return response(status=4003, message="token verify fail")

if __name__ == '__main__':
    app.run(port=8080)


from cProfile import label
from tkinter import *
from tkinter import ttk

root = Tk()
root.title("Les widgets de Tkinter")
root.columnconfigure(0, weight=1)
root.rowconfigure(0, weight=1)

# cadre principal
cadrePrincipal = ttk.Frame(root, padding=10, width=400,
                           height=400, borderwidth=2, relief='sunken')
cadrePrincipal.grid(column=0, row=0, sticky=(N, W, E, S))

ttk.Label(cadrePrincipal, width=32,
          text="Revue des widgets (cadre principal)").grid()

# 2 frames à mettre côte à côte
#   frame a
cadre2 = ttk.Frame(cadrePrincipal, padding=5, width=100,
                   height=150, borderwidth=2, relief='sunken')
cadre2.grid(column=0, row=1)
img_fox = PhotoImage(file='fox.gif')
lbl_cadre2 = ttk.Label(cadre2, image=img_fox).grid()
#   frame b
cadre3 = ttk.Frame(cadrePrincipal, padding=5, width=150,
                   height=100, borderwidth=2, relief='sunken')
cadre3.grid(column=1, row=1)
lbl_cadre3 = ttk.Label(cadre3, text='Cadre 3').grid()

cadre4 = ttk.Frame(cadrePrincipal, padding=10, width=200,
                   height=30)
cadre4.grid(column=0, row=2)
lbl_cadre4 = ttk.Label(cadre4, text='cadre 4 sous les autres').grid()

root.mainloop()


from typing import Tuple

TOL = 1e-9

class Node2D(object):

    def __init__(self, X: float, Y: float, support: str) -> None:
        """Constructor of the class.

        Args:
            X (float): x-coordinate of the node
            Y (float): y-coordinate of the node
            support (str): the support on the node
        """
        super().__init__()
        self.x = X
        self.y = Y
        self.support = support

        self.index = 0
        self.Fx = 0
        self.Fy = 0
        self.ux = 0
        self.uy = 0

    def __eq__(self, node: "Node2D") -> bool:
        return self.x == node.x and self.y == node.y

    @property
    def freedom(self) -> Tuple[bool, bool]:
        if self.support == "pin":
            return (False, False)
        if self.support == "h_roller":
            return (True, False)
        if self.support == "v_roller":
            return (False, True)
        return (True, True)

    @property
    def Fx(self):
        return self._Fx

    @Fx.setter
    def Fx(self, force: float):
        if abs(force) < TOL:
            self._Fx = 0
        else:
            self._Fx = force

    @property
    def Fy(self):
        return self._Fy

    @Fy.setter
    def Fy(self, force: float):
        if abs(force) < TOL:
            self._Fy = 0
        else:
            self._Fy = force


import numpy as np
import utils
import Model
import matplotlib.pyplot as plt
from matplotlib.legend_handler import HandlerLine2D

'''
a = '12 ,  34   '
re_splitA = re.compile(r'[\s\,]+')
print(re_splitA.split(a))
print(re.split(r'[\s\,]+', a))
print(re_splitA.split(a)[2] == '')      # True

b = [a for a in re_splitA.split(a) if a.isdigit()]
print(b)                   # work ! ! !
'''

print('Test...')
X, y = utils.readData('datas.xlsx')

X = np.atleast_2d(X).T
y = np.atleast_2d(y).T

print('Data info\n')
print('X = ',X.dtype, X.shape)
print('y = ',y.dtype, y.shape)

# Test datas
x = np.atleast_2d(np.linspace(X.min(), X.max(), 2000)).T

y_pred, sigma_pred = Model.gpSklearn(X, y, x)

fig = plt.figure(1)
plotf, = plt.plot(X, y, label='Origin')
plotfi, = plt.plot(x, y_pred, label='Predicted')
plotfi, = plt.plot(x, sigma_pred, label='predict_var')

plt.legend(handler_map={plotf: HandlerLine2D(numpoints=4)})
plt.scatter(X.reshape((-1)), y, marker='x', color='g')

plt.show()


import pygame,sys
from pygame.locals import *

pygame.init()

#setting up the window
DISPLAYSURF = pygame.display.set_mode((500,400), 0, 32)
pygame.display.set_caption("Drawing")

#setting up the colors
BLACK = (  0,   0,   0)
WHITE = (255, 255, 255)
RED   = (255,   0,   0)
GREEN = (  0, 255,   0)
BLUE  = (  0,   0, 255)

#drawing on the surface
DISPLAYSURF.fill(WHITE)
pygame.draw.polygon(DISPLAYSURF, GREEN, ((146,0), (291, 106), (236, 277), (56, 277), (0, 106)))
pygame.draw.line(DISPLAYSURF, BLUE, (60,60), (120,60), 4)
pygame.draw.line(DISPLAYSURF, BLUE, (120,60), (60,120))
pygame.draw.aaline(DISPLAYSURF, BLACK, (240,60), (60,120))
pygame.draw.line(DISPLAYSURF, BLUE, (60, 120), (120,120), 4)
pygame.draw.circle(DISPLAYSURF, BLUE, (300,50), 20, 0)
pygame.draw.ellipse(DISPLAYSURF, RED, (300,250,40,80), 1)
pygame.draw.rect(DISPLAYSURF, RED, (200,150,100,50))

pixObj = pygame.PixelArray(DISPLAYSURF)
pixObj[480][380] = BLACK
pixObj[482][382] = BLACK
pixObj[484][384] = BLACK
pixObj[486][386] = BLACK
pixObj[488][388] = BLACK
del pixObj #Deleting the PixelArrayobjectwill ―unlock‖ the Surfaceobject so that you can once again draw images on it

#run the game loop
while True:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
            sys.exit()
    pygame.display.update()


import tvm
from ..hw_abs_base import (
    HardwareAbstraction,
    register_abstraction,
    MemoryAbstraction,
    ComputeAbstraction,
    ElementwiseComputeAbstraction,
    ElementwiseMemoryAbstraction,
)
from .wmma_base import *

@register_abstraction("cuda", "nvcuda::wmma::load_matrix_sync::tf32")
class WMMALoadMatrixSyncTf32(MemoryAbstraction):
    def get_params_usage(self):
        """
        ---
        Returns:
        usage string: str
            help to understand the instruction this hardware abstraction contains
        """
        usage = (
            "nvcuda::wmma::load_matrix_sync::tf32" "Args:",
            "---",
            "fragment: dst fragment",
            "ptr: src memory pointer",
            "ldm: leading dimension length",
            "optional layout: layout for accumulator",
        )
        return usage

    get_buffer_memory_scope_info = WMMALoadMatrixSync.get_buffer_memory_scope_info
    get_instruction_prefix = WMMALoadMatrixSync.get_instruction_prefix
    def get_intrinsic(
        self,
        input_shapes,
        output_shapes,
        input_dtypes,
        output_dtypes,
        problem_size,
        ldm,
        layout,
        scope="shared",
    ):
        """
        input_shapes: list of tuple/list of int
        output_shapes: list of tuple/list of int
        input_dtypes: list of str
        output_dtypes: list of str
        problem_size: list of int
        ldm: int
        layout: str
        ---
        Returns:
        intrin: tvm.te.TensorIntrin
        """
        input_shapes = [[int(x) for x in y] for y in input_shapes]
        output_shapes = [[int(x) for x in y] for y in output_shapes]
        dtypes = [str(x) for x in input_dtypes]
        assert len(input_shapes) == 1
        assert len(output_shapes) == 1
        assert len(dtypes) == 1
        assert len(problem_size) == 3
        assert len(input_shapes[0]) == 2
        assert len(output_shapes[0]) == 2
        assert layout in [
            "nvcuda::wmma::row_major",
            "nvcuda::wmma::col_major",
            "nvcuda::wmma::mem_row_major",
            "nvcuda::wmma::mem_col_major",
        ]
        m, n, k = problem_size
        inputs, outputs = self.get_compute_expression(
            input_shapes, output_shapes, input_dtypes, output_dtypes, problem_size
        )
        elem_bytes = tvm.runtime.DataType(input_dtypes[0]).bits / 8
        load_elems = int(input_shapes[0][0]) * int(input_shapes[0][1])
        data_alignments = [int(elem_bytes * x[1]) for x in input_shapes]
        offset_factor = int(16 / elem_bytes)
        input_buffers = [
            tvm.tir.decl_buffer(
                x.shape, x.dtype, scope=scope, data_alignment=y, offset_factor=offset_factor
            )
            for x, y in zip(inputs, data_alignments)
        ]
        elem_bytes = tvm.runtime.DataType(output_dtypes[0]).bits / 8
        data_alignments = [int(elem_bytes * x[1]) for x in output_shapes]
        offset_factor = int(16 / elem_bytes)
        output_buffers = [
            tvm.tir.decl_buffer(
                x.shape, x.dtype, scope="local", data_alignment=y, offset_factor=offset_factor
            )
            for x, y in zip(outputs, data_alignments)
        ]
        bind_map = {x: y for x, y in zip(inputs + outputs, input_buffers + output_buffers)}

        def intrin_func(ins, outs):
            ib = tvm.tir.ir_builder.create()
            BA = ins[0]
            BC = outs[0]
            ib.emit(
                tvm.tir.call_intrin(
                    "handle",
                    self.get_tir_intrin_name(),
                    "cuda",
                    self.belong_hw_abs_dag_name,
                    "nvcuda::wmma::load_matrix_sync::tf32",
                    BC.data,
                    problem_size[0],
                    problem_size[1],
                    problem_size[2],
                    BC.elem_offset // load_elems,
                    BA.access_ptr("r"),
                    ldm,
                    layout,
                )
            )
            return ib.get()

        return tvm.te.decl_tensor_intrin(outputs[0].op, intrin_func, binds=bind_map)

    def _assemble_epilogue(self, *args):
        frag_arr_name, frag_index = args
        frag = f"{frag_arr_name}[{frag_index}]"
        return "\n".join(
            [
                f";",
                f"for (int t = 0; t < {frag}.num_elements; t++) {{",
                f"    {frag}.x[t] = nvcuda::wmma::__float_to_tf32({frag}.x[t]);",
                f"}}",
            ]
        )

    _assemble_instruction_body = WMMALoadMatrixSync.assemble_instruction

    def assemble_instruction(self, args):
        inst = self._assemble_instruction_body(args)
        epilogue = self._assemble_epilogue(args[0], args[4])
        full_inst = "\n".join([inst, epilogue])
        return full_inst


from mss import mss
import cv2
import threading
import numpy as np
import mouse
import pyautogui
import time
# SkLearn
import sklearn
from sklearn.metrics import plot_confusion_matrix
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import SGDClassifier
from sklearn.svm import LinearSVC
from sklearn.neural_network import MLPClassifier
import glob
import pickle
# matplotlib 3.3.1
from matplotlib import pyplot

class TrainModel:
    def __init__(self, data_path = 'character_identifier_data/', train_test_split = 0.8, max_each_class = 3000):
        self.model = None
        self.model_path = 'character_identifier.pkl'

        # Load labels and data locations
        self.train_data = []
        self.train_labels = []

        self.test_data = []
        self.test_labels = []

        # Read Data Locations
        self.train_data_read = []
        self.test_data_read = []

        # Data Path
        self.data_path = data_path

        # Max Each Class
        self.max_each_class = max_each_class

        # Train Test Split
        self.train_test_split = train_test_split

        self.classes = [0, 1, 2]
        self.label_dict = {'tracer': 0, 'ashe': 1, 'none': 2}

        # Load Data
        self.load_data()

        # Shuffle Data
        self.shuffle_data()

        # Test
        self.test_data = self.train_data[round(len(self.train_data)*self.train_test_split):]
        self.test_labels = self.train_labels[round(len(self.train_labels)*self.train_test_split):]
        # Train
        self.train_data = self.train_data[:round(len(self.train_data)*self.train_test_split)]
        self.train_labels = self.train_labels[:round(len(self.train_labels)*self.train_test_split)]

        # Read Data
        self.read_data()

        # Read Test Data
        self.read_test_data()

        # Train Model
        self.train()

    def apply_transformation(self, img):
        # Drop Alpha Channel
        img = img[:, :, :3]
        # Get Center third of image
        img = img[round(img.shape[0]/3):round(img.shape[0]*2/3), round(img.shape[1]/3):round(img.shape[1]*2/3)]
        # Convert to Grayscale
        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        # cv2.imshow('img', img)
        # cv2.waitKey(0)
        # Resize
        img = img.reshape(-1)
        return img

    # Loads Labels and X Data Locations
    def load_data(self):
        # Glob folders in train folder
        folders = glob.glob(self.data_path + '/*')

        for folder in folders:
            # Get label
            label = folder.split('\\')[-1]
            print(label)
            label_id = self.label_dict[label]

            # Get image files
            image_files = glob.glob(folder + '/*.png')

            # Add data to x_data and y_labels
            added = 0
            for image_file in image_files:
                self.train_data.append(image_file)
                self.train_labels.append(label_id)
                added += 1
                if added>=self.max_each_class:
                    break

    # Shuffles data locations and labels together
    def shuffle_data(self):
        # Shuffle data
        # Zip x locations and y labels together and turn into a list
        combined = list(zip(self.train_data, self.train_labels))
        # Shuffle this list
        np.random.shuffle(combined)
        # Unzip the list after zipping it again
        self.train_data, self.train_labels = zip(*combined)

    def read_data(self):
        # Read Data
        counter = 0
        for data in self.train_data:
            img = cv2.imread(data)
            img = self.apply_transformation(img)
            self.train_data_read.append(img)
            counter += 1
            if counter % 100 == 0:
                print(counter)

    def read_test_data(self):
        counter = 0
        for data in self.test_data:
            img = cv2.imread(data)
            img = self.apply_transformation(img)
            self.test_data_read.append(img)
            counter += 1
            if counter % 100 == 0:
                print(counter)

    def train(self):
        # Model
        print('Training Model')
        # Shape of Data
        print(f'Data Shape: {np.array(self.train_data_read).shape}')
        print(f'Labels Shape: {np.array(self.train_labels).shape}')
        print(self.train_labels[:10])
        # MLP Classifier
        self.model = MLPClassifier(max_iter=7, verbose=1, activation='relu', learning_rate_init=0.01) # Will change learning_rate_init to 0.001 later
        # Fit Data
        self.model.fit(self.train_data_read, self.train_labels)
        # Save model
        self.save_model()
        # Print Accuracy
        plot_confusion_matrix(self.model, self.test_data_read, self.test_labels)
        pyplot.show()
        self.test(self.model, self.test_data_read, self.test_labels)

    def test(self, model, test_x_final, test_y_final):
        preds = model.predict(test_x_final)
        correct = 0
        incorrect = 0
        for pred, gt in zip(preds, test_y_final):
            if pred == gt: correct += 1
            else: incorrect += 1
        print(f"Correct: {correct}, Incorrect: {incorrect}, % Correct: {correct/(correct + incorrect): 5.2}")
        pyplot.show()

    def save_model(self):
        pickle.dump(self.model, open(self.model_path, 'wb'))

    def load_model(self):
        self.model = pickle.load(open(self.model_path, 'rb'))

class GatherData:
    def __init__(self, save_folder = 'data', character_name = 'other', resolution = (1920, 1080), time_between_frames = 0.1, start_file_number = 0):
        # Member variables
        self.character_name = character_name
        self.save_folder = save_folder
        # Resolution
        self.x_res = resolution[0]
        self.y_res = resolution[1]

        # Time Between Frames
        self.time_between_frames = time_between_frames

        # BGR Separators for Red
        self.red_min = 180
        self.red_max = 255
        self.green_max = 100
        self.blue_max = 100

        # Initialize save location
        self.save_location = self.save_folder + '/' + self.character_name + '/'
        self.file_number = start_file_number

        # Initialize screen capture
        self.monitor = {'top': 0, 'left': 0, 'width': self.x_res, 'height': self.y_res}

        # Initialize Capture Thread
        self.capture_thread = threading.Thread(target=self.capture_loop).start()

    def get_frame(self):
        with mss() as sct:
            # Get Screen Frame
            frame = np.array(sct.grab(self.monitor))
        return frame

    def process_frame_for_save(self, frame):
        # Drop Alpha Channel
        frame = frame[:, :, :3]
        # Filtered Red Locations
        frame = cv2.inRange(frame, (0, 0, self.red_min), (self.blue_max, self.green_max, self.red_max))
        return frame

    def save_frame(self, frame):
        # Save Frame
        cv2.imwrite(f'{self.save_location}{self.character_name}{self.file_number}.png', frame)
        self.file_number += 1

    # Main Data Capture Loop
    def capture_loop(self):
        while True:
            # Get Frame
            frame = self.get_frame()
            # Process Frame
            frame = self.process_frame_for_save(frame)
            # Save Frame
            self.save_frame(frame)
            # Wait
            time.sleep(self.time_between_frames)

# class CharacterIdentifier:
#     def __init__(self, model_path):
#         self.model = self.load_model()
#         self.label_dict_reverse = {0: 'tracer', 1: 'ashe', 2: 'none'}

#         # Model Path
#         self.model_path = 'character_identifier.pkl'

#         # BGR Separators for Red
#         self.red_min = 180
#         self.red_max = 255
#         self.green_max = 100
#         self.blue_max = 100

#         # Downsize Proportion
#         self.downsize_proportion = 0.3

#     # Apply Transformations before Network Input
#     def apply_transformation(self, img):
#         # Drop Alpha Channel
#         img = img[:, :, :3]
#         # Filtered Red Locations
#         img = cv2.inRange(img, (0, 0, self.red_min), (self.blue_max, self.green_max, self.red_max))
#         # Get Center third of image
#         img = img[round(img.shape[0]/3):round(img.shape[0]*2/3), round(img.shape[1]/3):round(img.shape[1]*2/3)]
#         # Resize for Network Input
#         img = img.reshape(-1)
#         # Return Processed Image for Network
#         return img

#     def predict(self, screenshot):
#         processed_image = self.apply_transformation(screenshot)
#         return self.label_dict_reverse[self.model.predict(processed_image.reshape(1, -1))[0]]

#     def load_model(self):
#         with open(self.model_path, 'rb') as f:
#             model = pickle.load(f)
#         return model


from flask import Flask, render_template, jsonify
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func, desc
import sqlalchemy.dialects.sqlite
import datetime as dt
import pandas as pd
import os

app = Flask(__name__,
            static_folder='static',
            template_folder='templates')

@app.route('/')
def home():
    return render_template('viz1.html')

@app.route('/covid_data', methods=['GET'])
def covid_data():
    #create engine to connect to SQL database
    engine = create_engine("postgresql+psycopg2://llnvzqitayvosz:0c1511137a621fdcc484c0a1afdccf84d1b10e28b5370f0af994ca3113fad4d8@ec2-3-213-85-90.compute-1.amazonaws.com:5432/d3livmv6adnqas")
    #connect to SQL database
    connection = engine.connect()

    # creat dataframe of wins per country from database
    covid_data = pd.read_sql('SELECT * FROM infected;', connection)
    print(covid_data)
    # convert dataframe to list of lists with header
    covid_data_list=[covid_data.columns.values.tolist()] \
        + covid_data.values.tolist()

    response = jsonify(covid_data_list)
    response.headers.add("Access-Control-Allow-Origin", "*")
    return response

@app.route('/state_data', methods=['GET'])
def state_data():
    #create engine to connect to SQL database
    engine = create_engine("postgresql+psycopg2://llnvzqitayvosz:0c1511137a621fdcc484c0a1afdccf84d1b10e28b5370f0af994ca3113fad4d8@ec2-3-213-85-90.compute-1.amazonaws.com:5432/d3livmv6adnqas")
    #connect to SQL database
    connection = engine.connect()

    # creat dataframe of wins per country from database
    state_data = pd.read_sql('SELECT * FROM percent6;', connection)

    #print(state_data)

    state_data_list=[state_data.columns.values.tolist()] \
        + state_data.values.tolist()
    response = jsonify(state_data_list)
    response.headers.add("Access-Control-Allow-Origin", "*")
    return response

@app.route('/filtered_data', methods=['GET'])
def filtered_data():
    #create engine to connect to SQL database
    engine = create_engine("postgresql+psycopg2://llnvzqitayvosz:0c1511137a621fdcc484c0a1afdccf84d1b10e28b5370f0af994ca3113fad4d8@ec2-3-213-85-90.compute-1.amazonaws.com:5432/d3livmv6adnqas")
    #connect to SQL database
    connection = engine.connect()

    # creat dataframe of wins per country from database
    filtered_data = pd.read_sql('SELECT * FROM filtered_data4;', connection)

    #print(filtered_data)

    filtered_data_list=[filtered_data.columns.values.tolist()] \
        + filtered_data.values.tolist()
    response = jsonify(filtered_data_list)
    response.headers.add("Access-Control-Allow-Origin", "*")
    return response

if __name__ == '__main__':
    port = int(os.environ.get("PORT", 5000))
    app.run(host='0.0.0.0', port=port)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
一个机器人位于一个 m x n网格的左上角 （起始点在下图中标记为 “Start” ）。

机器人每次只能向下或者向右移动一步。机器人试图达到网格的右下角（在下图中标记为 “Finish” ）。

问总共有多少条不同的路径？

示例 1：

输入：m = 3, n = 7
输出：28
示例 2：

输入：m = 3, n = 2
输出：3
解释：
从左上角开始，总共有 3 条路径可以到达右下角。
1. 向右 -> 向下 -> 向下
2. 向下 -> 向下 -> 向右
3. 向下 -> 向右 -> 向下
示例 3：

输入：m = 7, n = 3
输出：28
示例 4：

输入：m = 3, n = 3
输出：6

提示：

1 <= m, n <= 100
题目数据保证答案小于等于 2 * 109

来源：力扣（LeetCode）
链接：https://leetcode-cn.com/problems/unique-paths
著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。

"""

class Solution:
    def uniquePaths(self, m: int, n: int) -> int:
        dp = [[0]*n for _ in range(m)]
        for i in range(m):
            dp[i][0] = 1

        for j in range(n):
            dp[0][j] = 1

        for i in range(1,m):
            for j in range(1,n):
                dp[i][j] = dp[i-1][j]+dp[i][j-1]
        return dp[-1][-1]

算法分析和归类/动态规划/不同路径.py


from bs4 import BeautifulSoup
import requests
try:
    from crawlers.utils import clean
    from crawlers.utils import save_preview
except:
    from utils import clean
    from utils import save_preview

def url_scheme():
    return "https://chapmanganato.com/manga-"

def type():
    return "bs4"

def get_preview(RAW_URL, title):
    temp = RAW_URL.split("/")
    URL = "/".join(temp[:-1]) +"/"

    r = requests.get(URL)
    if r.status_code == 200:
        soup = BeautifulSoup(r.text, "html.parser")
        temp = soup.find("span", class_="info-image")
        url = clean(temp.contents)[0].attrs['src']

        return save_preview(title, url)

def get_chapter_list(RAW_URL):
    temp = RAW_URL.split("/")
    URL = "/".join(temp[:-1]) +"/"

    r = requests.get(URL)
    if r.status_code == 200:
        soup = BeautifulSoup(r.text, "html.parser")
        chapterList = soup.find_all("li", class_="a-h")
        chapList = []
        for li in chapterList:
            a = clean(li.contents)[0]
            url = a.attrs['href']
            chapter_name = a.contents[0]
            chapList.append((chapter_name, url))
        return chapList[::-1]

def get_title(RAW_URL):
    temp = RAW_URL.split("/")
    URL = "/".join(temp[:-1]) +"/"

    r = requests.get(URL)
    if r.status_code == 200:
        soup = BeautifulSoup(r.text, "html.parser")
        name = clean(soup.find('div', class_ = "panel-breadcrumb").contents)
        title = name[2].attrs['title']
        return title


import pandas
import requests
import json
import streamlit as st
url = "http://apis.data.go.kr/B552584/ArpltnInforInqireSvc/getMsrstnAcctoRltmMesureDnsty?stationName=주엽동&dataTerm=DAILY&pageNo=1&numOfRows=100&returnType=json&serviceKey=KQRR%2BJLPRITcRv6CvRB1QUxmDQ%2BKmcKWMjK1A19g%2BiHLEbXTpqjWmut5pwHfKkH6O7KfqLSXxEmrLt6Ctooliw%3D%3D"

geturl = requests.get(url)
rawData = geturl.text

json_ob = json.loads(rawData)

jsonData = json_ob['response']['body']['items']
data = pandas.DataFrame(jsonData)

for i in range(len(data['khaiValue'])):
    if data['khaiValue'][i] == '-':
        data['khaiValue'][i] = '0'
for i in range(len(data['pm10Value'])):
    if data['pm10Value'][i] == '-':
        data['pm10Value'][i] = '0'
for i in range(len(data['coValue'])):
    if data['coValue'][i] == '-':
        data['coValue'][i] = '0'
for i in range(len(data['so2Value'])):
    if data['so2Value'][i] == '-':
        data['so2Value'][i] = '0'
for i in range(len(data['o3Value'])):
    if data['o3Value'][i] == '-':
        data['o3Value'][i] = '0'
for i in range(len(data['no2Value'])):
    if data['no2Value'][i] == '-':
        data['no2Value'][i] = '0'

data['환경 수치'] = pandas.to_numeric(data['khaiValue'])
data['미세먼지'] = pandas.to_numeric(data['pm10Value'])
data['일산화탄소'] = pandas.to_numeric(data['coValue'])
data['아황산가스'] = pandas.to_numeric(data['so2Value'])
data['오존'] = pandas.to_numeric(data['o3Value'])
data['이산화질소'] = pandas.to_numeric(data['no2Value'])

time = data['dataTime']
figure = data['환경 수치']
fineDust = data['미세먼지']
co = data['일산화탄소']
so2 = data['아황산가스']
o3 = data['오존']
no2 = data['이산화질소']

st.write('# 일산 주엽동 대기질')
st.write('***')
st.write('#### 선 그래프')
st.line_chart(figure)
st.line_chart(fineDust)
st.line_chart(co)
st.line_chart(so2)
st.line_chart(o3)
st.line_chart(no2)
st.write('***')
st.write('#### 막대 그래프')
st.bar_chart(figure)
st.bar_chart(fineDust)
st.bar_chart(co)
st.bar_chart(so2)
st.bar_chart(o3)
st.bar_chart(no2)