import plot_helper

import numpy as np
import pandas as pd
import random
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split

class data:

    def __init__(self,
                 image_features_file="data/2018-12-17_features_resnet_model_dont_include_top.npy",
                 ratings_train_file = "data/ratings-train.csv",
                 ratings_test_file = "data/ratings-test.csv",
                 user_features_file = "data/2018-12-17_collaborative_filtering_user_preferences.npz",
                 original_images_file = "data/data_occasi-images.npz",
                 image_input_type = 'pretrained_feature', # can be 'pretrained_feature' OR 'pixel_image'
                 feature_scaling=True):
        self.image_input_type = image_input_type

        print("init image features from " + image_features_file)
        # import images
        self.image_features = np.load(image_features_file)
        #Reshape, Da Densenet 1314,7,7,1920 Shape hat
        if(feature_scaling and 'DenseNet' in image_features_file): # TODO reshape würde ich bereits vor dem Speichern in ein File machen.
            print(self.image_features.shape)
            nsamples, nx, ny, nz = self.image_features.shape
            self.image_features = self.image_features.reshape((nsamples,nx*ny*nz))

        if feature_scaling:
            print("feature scaling")
            self.image_featuers_scaler = StandardScaler().fit(self.image_features)
            self.image_features = self.image_featuers_scaler.transform(self.image_features)
        print("loadad image features with shape " + str(self.image_features.shape))

        print("load ratings from " + ratings_train_file)
        ratings = pd.read_csv(ratings_train_file,sep=',',names="user_id,item_id,rating,user_ext_id,item_ext_id".split(","))
        self.train_ratings, self.cross_validation_ratings= train_test_split(ratings, test_size=0.1)
        print("loaded " + str(len(self.train_ratings)) + " train ratings and " + str(len(self.cross_validation_ratings)) + " cross validation ratings")

        print("load test ratings from " + ratings_test_file)
        self.test_ratings = pd.read_csv(ratings_test_file,sep=',',names="user_id,item_id,rating,user_ext_id,item_ext_id".split(","))
        print("loaded " + str(len(self.test_ratings)) + " test ratings")

        print("load user features from " + user_features_file)
        loaded_file = np.load(user_features_file)
        self.user_features = loaded_file['user_preferences']
        if feature_scaling:
            self.user_features_scaler = StandardScaler().fit(self.user_features)
            self.user_features = self.user_features_scaler.transform(self.user_features)
        print("loaded user features with shape " + str(self.user_features.shape))

        print("load original images from " + original_images_file)
        imported_file = np.load(original_images_file)
        self.original_images = imported_file['imgs']
        print("loaded " + str(self.original_images.shape[0]) + " images")

        self.model_summary = pd.Series({'user_vector':user_features_file,
                                        'img_vector':image_features_file,
                                        'image_input_type':image_input_type})

    def image_features(self):
        return self.image_features

    def image_inputs(self):
        return self.image_features if self.image_input_type == 'pretrained_feature' else self.original_images

    def train_triplets_with_imageids(self):
        return ratings_to_triplets(self.train_ratings, self.train_ratings.user_id.unique())

    def steps_for_all_inputs(self, batch_size):
        # Returns the number of steps it need with the given batch_size to see each input at least once
        train_triplets = ratings_to_triplets(self.train_ratings, self.train_ratings.user_id.unique())
        return int(len(train_triplets) / batch_size)

    def train_triplets_generator(self, batch_size):
        while True:
            train_triplets = ratings_to_triplets(self.train_ratings, self.train_ratings.user_id.unique())

            for i in range(0, len(train_triplets), batch_size):
                end_index = i + batch_size
                end_index = min(end_index, len(train_triplets))

                user_features, img1_features, img2_features, y = triplets_to_inputs(train_triplets[i:end_index], self.image_inputs(), self.user_features)

                yield ([np.array(user_features), np.array(img1_features), np.array(img2_features)], np.array(y))

    def test_triplets_generator(self, nr_of_steps):
        user_ids = self.test_ratings.user_id.values
        img_ids = self.test_ratings.item_id.values
        y_ratings = self.test_ratings.rating.values

        nr_examples = len(user_ids)

        batch_size = int(nr_examples / (nr_of_steps-1))

        for i in range(0,nr_examples,batch_size):
            end_index = min(i + batch_size, nr_examples)
            yield [
                [self.get_user_features(user_id) for user_id in user_ids[i:end_index]],
                [self.get_image_features(img_id) for img_id in img_ids[i:end_index]]
            ]

    def cross_validation_triplets(self):
        cv_triplets = ratings_to_triplets(self.cross_validation_ratings, self.train_ratings.user_id.unique())
        user_features, img1_features, img2_features, y = triplets_to_inputs(cv_triplets, self.image_inputs(), self.user_features)
        return user_features, img1_features, img2_features, y

    def user_vector_input_size(self):
        return self.user_features.shape[1]

    def image_vector_input_size(self):
        return [self.image_features.shape[1]] # TODO remove array or add array on user_vector_input_size method

    def get_user_features(self, user_id):
        return self.user_features[user_id]

    def get_image_features(self, item_id):
        return self.image_inputs()[item_id]

    def user_id_of(self, user_ext_id):
        user_ratings = self.train_ratings[self.train_ratings['user_ext_id'] == user_ext_id].user_id
        if len(user_ratings) == 0:
            print("could not find user " + user_ext_id)
            return

        return user_ratings.iloc[0]

    def plot_original_image(self, img_id):
        img = self.original_images[img_id]
        plot_helper.plot_image(img)

def triplets_to_inputs(triplets, image_features, user_features):
    #print("start 1 " + str(len(triplets)))
    user_vector = []
    img1_vector = []
    img2_vector = []
    y = []

    #print("start " + str(user_vector.shape))
    for idx, triplet in enumerate(triplets):
        if not user_features[triplet['user']].any():
            continue # ignore triplet if all values are 0 for user

        user_vector.append(user_features[triplet['user']])

        if round(random.uniform(0, 1)) == 1:
            # img1 as positiv image => y equals 0 TODO correct?
            img1_vector.append(image_features[triplet['image_pos']])
            img2_vector.append(image_features[triplet['image_neg']])
            y.append(0)
        else:
            img1_vector.append(image_features[triplet['image_neg']])
            img2_vector.append(image_features[triplet['image_pos']])
            y.append(1)

    #print("end " + str(user_vector.shape))
    return np.array(user_vector), np.array(img1_vector), np.array(img2_vector), np.array(y)

def ratings_to_triplets(ratings, all_user_ids):
    triplets = []

    for user_id in all_user_ids:
        user_ratings = ratings[ratings['user_id'] == user_id]

        rat_0 = user_ratings[user_ratings['rating'] == 0].item_id.values
        rat_1 = user_ratings[user_ratings['rating'] == 1].item_id.values
        rat_2 = user_ratings[user_ratings['rating'] == 2].item_id.values

        for img_id in rat_2:
            triplets.append({
              'user': user_id,
              'image_pos': img_id,
              'image_neg': random.choice(rat_1) if len(rat_1) > 0 else random.choice(rat_0)
            })

        # for each rating 1 add a rating 0 image as negativ example
        for img_id in rat_1:
            if (len(rat_0) < 1):
                break
            triplets.append({
              'user': user_id,
              'image_pos': img_id,
              'image_neg': random.choice(rat_0)
          })

    np.random.shuffle(triplets)
    return triplets


# Definition for a binary tree node.
class TreeNode:
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Solution:
    # @param {TreeNode} root
    # @return {TreeNode}
    def invertTree(self, root):
        if root is None:
            return
        root.left = self.invertTree(root.left)
        root.right = self.invertTree(root.right)
        temp = root.left
        root.left = root.right
        root.right = temp
        return root

# def testNoChildren():
#     root = TreeNode(1)
#     sol = Solution()
#     root = sol.invertTree(root)
#     assertEqual(root.left, None)
#     assertEqual(root.right, None)
#     assertEqual(root.val, 1)
#
# def testNormalCase():
#     root = TreeNode(1)
#     n1 = TreeNode(2)
#     n1_1 = TreeNode(3)
#     n1_2 = TreeNode(4)
#     n2 = TreeNode(5)
#     n2_1 = TreeNode(6)
#     n2_2 = TreeNode(7)
#     root.left = n1
#     root.right = n2
#     n1.left = n1_1
#     n1.right = n1_2
#     n2.left = n2_1
#     n2.right = n2_2
#
#     sol = Solution()
#     root = sol.invertTree(root)
#
#     assertEqual(root.left.val, 5)
#     assertEqual(root.right.val, 2)
#     assertEqual(root.left.left.val, 7)
#     assertEqual(root.left.right.val, 6)
#     assertEqual(root.right.left.val, 4)
#     assertEqual(root.right.right.val, 3)
#
# def testHalfNone():
#     root = TreeNode(1)
#     root.left = TreeNode(2)
#     root.left.left = TreeNode(3)
#
#     sol = Solution()
#     root = sol.invertTree(root)
#     assertEqual(root.left, None)
#     assertEqual(root.right.val, 2)
#     assertEqual(root.right.right.val, 3)
#
# def testNone():
#     root = None
#     sol = Solution()
#     root = sol.invertTree(root)
#     assertEqual(root, None)


# Goal: Find first round where no elf moves

# inputfile = "input_test.txt"    # expected result: 20
inputfile = "input.txt"

# process input:

elves=[]
checking_order=['N','S','W','E']
#elf={position:(), proposed_position:()}

y=0
with open(inputfile) as f:
    for line in f:
        x=0
        for tile in line.strip():
            if tile=='#':
                elf={'position': (x,y), 'proposed_position': ()}
                elves.append(elf)
            x+=1    # increase x coord for next column
        y+=1        # increase y coord for next line

elf_count=len(elves)

# for elf in elves:
#     print(elf)

# define required functions:

def check_surroundings(current_position):
    # print(current_position)
    x = current_position[0]
    y = current_position[1]
    N_clear=True    # N: (x,y-1)
    NE_clear=True   # NE: (x+1,y-1)
    E_clear=True    # E: (x+1,y)
    SE_clear=True   # SE: (x+1,y+1)
    S_clear=True    # S: (x,y+1)
    SW_clear=True   # SW: (x-1,y+1)
    W_clear=True    # W: (x-1,y)
    NW_clear=True   # NW: (x-1,y-1)
    for elf in elves:
        if elf['position']==(x,y-1):
            N_clear=False
        elif elf['position']==(x+1,y-1):
            NE_clear=False
        elif elf['position']==(x+1,y):
            E_clear=False
        elif elf['position']==(x+1,y+1):
            SE_clear=False
        elif elf['position']==(x,y+1):
            S_clear=False
        elif elf['position']==(x-1,y+1):
            SW_clear=False
        elif elf['position']==(x-1,y):
            W_clear=False
        elif elf['position']==(x-1,y-1):
            NW_clear=False
    N_sector_clear=(N_clear and NE_clear and NW_clear)
    E_sector_clear=(NE_clear and E_clear and SE_clear)
    S_sector_clear=(SE_clear and S_clear and SW_clear)
    W_sector_clear=(SW_clear and W_clear and NW_clear)
    return(N_sector_clear, E_sector_clear, S_sector_clear, W_sector_clear)

# process rounds until first round where no elf moves

stop=False
i=1
while not stop:
    # print()
    # print()
    # print("round",i, 'part 1')
    elves_moved=0
    proposed_positions=[]
    # part 1 (decide where to move):
    for elf in elves:
        position=elf['position']
        # print('checking elf in position',position)
        # 1a. If all 8 surrounding positions are empty, DO NOTHING
        position_info=check_surroundings(position)
        # print(position_info)
        N_sector_clear=position_info[0]
        E_sector_clear=position_info[1]
        S_sector_clear=position_info[2]
        W_sector_clear=position_info[3]
        if not (N_sector_clear and E_sector_clear and S_sector_clear and W_sector_clear):
            # 1b. Check each sector in turn and decide where to move
            for sector in checking_order:
                # print('checking sector',sector)
                if elf['proposed_position']==():        # only check and propose a move if we haven't already proposed a move
                    if sector=='N':
                        # print('North sector clear:',N_sector_clear)
                        # check North sector and propose moving one setp North if no elf there
                        if N_sector_clear:
                            proposed_position=(position[0],position[1]-1)
                            # print('propose move N to',proposed_position)
                            elf['proposed_position']=proposed_position
                            proposed_positions.append(proposed_position)
                    elif sector=='S':
                        # print('South sector clear:',S_sector_clear)
                        # check South sector and propose moving one setp South if no elf there
                        if S_sector_clear:
                            proposed_position=(position[0],position[1]+1)
                            # print('propose move S to',proposed_position)
                            elf['proposed_position']=proposed_position
                            proposed_positions.append(proposed_position)
                    elif sector=='W':
                        # print('West sector clear:',W_sector_clear)
                        # check West sector and propose moving one setp West if no elf there
                        if W_sector_clear:
                            proposed_position=(position[0]-1,position[1])
                            # print('propose move W to',proposed_position)
                            elf['proposed_position']=proposed_position
                            proposed_positions.append(proposed_position)
                    elif sector=='E':
                        # print('East sector clear:',E_sector_clear)
                        # check East sector and propose moving one setp East if no elf there
                        if E_sector_clear:
                            proposed_position=(position[0]+1,position[1])
                            # print('propose move E to',proposed_position)
                            elf['proposed_position']=proposed_position
                            proposed_positions.append(proposed_position)

    # print('elves have proposed moves to the following tiles:')
    # print(proposed_positions)

    # part 2 (move):
    # print("round",i, 'part 2')
    for elf in elves:
        # if proposed_position is not empty and no other elves have proposed moving to the same tile:
        proposed_position=elf['proposed_position']
        # print('Elf in position',elf['position'],'has proposed moving to',elf['proposed_position'])
        if proposed_position!=():
            num_proposed=0
            for p in proposed_positions:
                if p==proposed_position:
                    num_proposed+=1
            # print(num_proposed,'elves have proposed moving to position',proposed_position)
            if num_proposed<2:
                elf['position']=proposed_position
                # print('elf moved to',elf['position'])
                elves_moved+=1
            elf['proposed_position']=()
    # change checking order - move first element to the end
    a=checking_order.pop(0)
    checking_order.append(a)
    print("number of elves moved this round:",elves_moved)
    if elves_moved==0:
        print("First round where no elf moved = ",i)
        stop=True
    i+=1
    # for elf in elves:
    #     print(elf['position'])


def gcdlcm(a, b):
    m = a
    n = b
    while True:
        r = m % n
        if r == 0:
            gcd = n
            break
        else:
            m = n
            n = r
    lcm = int(a * b / gcd)
    answer = [gcd, lcm]
    return answer

# 아래는 테스트로 출력해 보기 위한 코드입니다.
print(gcdlcm(3,12))


import tkinter as tk
from tkinter import filedialog
import tkinter.messagebox as msgbox

import managedfolders, driveids, lastsynced

import os
from os.path import expanduser

class FolderFrame(tk.Frame):

    def __init__(self, master, callback, syncserver):
        tk.Frame.__init__(self, master)
        self['bd'] = 3
        self.callback = callback
        self.sync_server = syncserver

        self.bttn_selected_bg = 'cyan'
        self.bttn_common_bg = 'white'

        self.managed_folders = managedfolders.ManagedFolders()

        self.create_widgets()

    def on_click(self, event):
        widget = event.widget
        if self.currently_selected == widget:
            return

        self.set_selected(widget, True)
        self.set_selected(self.currently_selected, False)
        self.currently_selected = widget
        self.callback(widget['text'])

    def handle_add_folder(self):
        os.chdir(expanduser('~'))#start at home directory
        folder_path = filedialog.askdirectory()
        if folder_path == () or folder_path == '':
            pass
        else:
            add_all = msgbox.askyesno(title='Add folder ' + folder_path,
            message='Do you want to sync every file in this folder?')

            if self.sync_server == None:
                return
            print('adding folder: ' + folder_path)
            self.sync_server.add_new_dir(folder_path, add_all)
            self.refresh_widgets()

    def handle_remove_folder(self):
        folder = self.currently_selected['text']
        if folder == '<All>':
            remove_all = msgbox.askyesno(title='Remove every folder',
            message='Do you really want to remove every listed folder from sync?')

            if not remove_all:
                return

            self.managed_folders.clear()
            driveids.DriveIds().clear()
            lastsynced.LastSynced().clear()
            self.refresh_widgets()

        else:
            remove_folder = msgbox.askyesno(title='Remove folder ' + folder,
            message='Are you sure you want to remove\n%s\n and every file in it from being synced?' % folder)

            if not remove_folder:
                return

            do_ask_nested = False
            for path in self.get_nested_folders(folder):
                do_ask_nested = True
                break
                path#get rid of that warning

            remove_nested = False
            if do_ask_nested:
                remove_nested = msgbox.askyesno(title='Remove contained folders',
                message='%s\ncontains folders that are marked for syncing. Do you ' +
                'want to remove those folders as well?')

            self.remove_files_in_folder(folder, remove_nested)
            self.refresh_widgets()

    def get_nested_folders(self, base):
        for other_path in self.managed_folders:
            if base != other_path and self.is_folder_contained(base, other_path):
                yield other_path

    def is_folder_contained(self, base, child):
        return child[:len(base)] == base

    def remove_files_in_folder(self, folder, remove_nested):
        self.managed_folders.remove(folder)
        drive_ids = driveids.DriveIds()
        last_synced = lastsynced.LastSynced()

        for name in os.listdir(folder):
            path = folder + '/' + name
            if os.path.isdir(path):
                if remove_nested and path in self.managed_folders:
                    self.remove_files_in_folder(path, remove_nested)
            else:
                drive_ids.pop(path)
                last_synced.pop(path)

        #remove nested folders not directly in folder
        if remove_nested:
            for managed_folder in self.managed_folders:
                if self.is_folder_contained(folder, managed_folder):
                    self.remove_files_in_folder(managed_folder, remove_nested)

    def set_selected(self, widget, selected):
        if not widget:
            return

        if selected:
            widget['background'] = self.bttn_selected_bg
        else:
            widget['background'] = self.bttn_common_bg

    def create_widgets(self):
        button_frame = tk.Frame(self)
        tk.Button(button_frame, text='Add folder', background='#b2eef4', font=8,
                  command=self.handle_add_folder).grid(row=0, column=0, pady=20)
        tk.Button(button_frame, text='Remove selected folder', background='#b2eef4', font=8,
                  command=self.handle_remove_folder).grid(row=0, column=1)
        button_frame.grid(row=0)
        folders_label = tk.Label(self, text='Folders', font=4)
        folders_label.grid(row=1, sticky='W')
        folders_label.focus_set()
        self.add_disposable_widgets()

    def add_disposable_widgets(self):
        self.container_frame = tk.Frame(self)

        label = tk.Label(self.container_frame, text='<All>', bg=self.bttn_selected_bg)
        label.grid(row=0, column=0, sticky='W')
        label.bind("<Button-1>", self.on_click)
        self.currently_selected = label

        i = 0
        for path in self.managed_folders:
            i += 1
            label = tk.Label(self.container_frame, text=path, bg=self.bttn_common_bg)
            label.grid(row=i, column=0, sticky='W')
            label.bind("<Button-1>", self.on_click)

        self.container_frame.grid(row=2, sticky='W')

    def refresh_widgets(self):
        self.container_frame.destroy()
        self.add_disposable_widgets()


import numpy as np

class kNN:
    def __init__(self, neighbours):
        self.neighbours = neighbours

    def euclidean_distance(self, row_1, row_2):
        distance = 0.0
        for i in range(len(row_1)):
            distance += (row_1[i] - row_2[i])**2

        return np.sqrt(distance)

    def k_NN(self, train, test_row):
        distances = []

        for i in range(len(train)):
            dist = self.euclidean_distance(test_row, train[i])
            distances.append((train[i], dist, i))
        distances.sort(key=lambda tup: tup[1])

        neighbors = []
        distan = []
        index = []

        for i in range(self.neighbours):
            neighbors.append(distances[i][0])
            distan.append(distances[i][1])
            index.append(distances[i][2])

        return [neighbors, distan, index]


import logging
from odoo import api, fields, models
from odoo.exceptions import UserError

logger = logging.getLogger(__name__)

class HelpdeskTicket(models.Model):
    _inherit = 'helpdesk.ticket'

    active = fields.Boolean(default=True)

    @api.model
    def create(self, vals):
        partner_id = vals.get('partner_id')
        if partner_id:
            has_tickets = self.env['res.partner'].search([('id', '=', partner_id)]).has_available_tickets
            if not has_tickets:
                self.active = False
                vals['stage_id'] = 3 # En espera
            else:
                self.active = True
                vals['stage_id'] = 1 # Nuevo
                partner = self.env['res.partner'].search([('id', '=', partner_id)])
                partner.active_tickets -= 1

        return super(HelpdeskTicket, self).create(vals)

    def write(self, vals):
        ticket = self.env['helpdesk.ticket'].search([('id', '=', self.id)])
        partner = self.env['res.partner'].search([('id', '=', ticket.partner_id.id)])

        if not partner.has_available_tickets and self.stage_id.id == 3:
            vals['stage_id'] = 3 # En espera
            raise UserError('El usuario no tiene tickets disponibles')

        elif partner.has_available_tickets and self.stage_id.id == 3:
            partner.active_tickets -= 1

        return super().write(vals)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('spl', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='pill',
            name='spl_inactive_ing',
            field=models.TextField(null=True, verbose_name=b'SPL_INACTIVE_ING', blank=True),
            preserve_default=True,
        ),
        migrations.AddField(
            model_name='pill',
            name='spl_ingredients',
            field=models.TextField(null=True, verbose_name=b'SPL_INGREDIENTS', blank=True),
            preserve_default=True,
        ),
        migrations.AddField(
            model_name='pill',
            name='spl_strength',
            field=models.TextField(null=True, verbose_name=b'SPL_STRENGTH', blank=True),
            preserve_default=True,
        ),
    ]


import matplotlib.pyplot as plt
import numpy as np
import os, sys, math
from pathlib import Path
from timeit import default_timer as timer

MAX_ITERATIONS = 1200

def plot_graph_single_run(res_path: Path, in_path: Path, nodes, display=False):

    # loading logged data
    res_data = np.loadtxt(res_path.absolute(), delimiter=',')
    in_data = np.loadtxt(in_path.absolute(), delimiter=',', usecols=range(0,np.shape(res_data)[1]))
    in_data_k = np.repeat(in_data, np.append(np.repeat(5, np.shape(in_data)[0]-1), 205), axis = 0)
    mean = np.mean(in_data, axis=1)
    mean_k = np.repeat(mean, np.append(np.repeat(5, np.shape(mean)[0]-1), 205), axis = 0)
    # max_error = np.dot(np.abs(np.max(np.abs(res_data - np.resize(mean_k, np.shape(res_data))),axis=1)/mean_k), 100)

    nodes = np.shape(res_data)[1]

    # % Difference
    # max_rel_difference= np.dot(np.ptp(res_data, axis=1)/(np.abs(max_value_at_k) + np.abs(min_value_at_k)),100)
    max_rel_difference = np.abs(np.ptp(res_data, axis=1)/(np.max(res_data, axis=1) + np.min(res_data, axis=1)))
    for r , val in np.ndenumerate(max_rel_difference):
        if val > 1:
            max_rel_difference[r] = np.reciprocal(val)

    max_rel_difference = np.dot(max_rel_difference, 200)

    # %Error
    state_mean_vs_real_mean = np.dot(np.abs((np.mean(res_data, axis=1) - mean_k)/mean_k), 100)

    convergence_rates = np.full(np.shape(res_data), np.nan)
    # Convergence Rate
    for index, state in np.ndenumerate(res_data):
        row_index = index[0]
        column_index = index[1]
        if row_index%5<2 or row_index%5 > 3:
            continue
        if not (res_data[row_index-2, column_index] == res_data[row_index-1, column_index]
        or res_data[row_index-1, column_index] == res_data[row_index, column_index]):
            convergence_rates[row_index, column_index] = (
                np.log(
                    np.abs(
                        (res_data[row_index+1, column_index] - state) / (state - res_data[row_index-1, column_index])
                    ))
                / np.log(
                    np.abs(
                        (state - res_data[row_index-1, column_index]) / (res_data[row_index-1, column_index] - res_data[row_index-2, column_index])
                    )
                )
            )

    convergence_rates = np.nanmean(convergence_rates, axis=1)

    convergence_means = {}

    for index, mean in enumerate(convergence_rates):
        if(not np.isnan(mean)):
            convergence_means[index] = mean

    # percentage_diff_state_diff_input = np.dot(np.divide(np.ptp(res_data, axis=1), np.ptp(in_data_k, axis=1)), 100.0)
    print(np.ptp(in_data_k[:12,:], axis=1))
    print(np.ptp(res_data[:12,:], axis=1))

    fig, axes = plt.subplots(3,1, sharex=True, figsize=(16,12))

    # for x-axis : k
    k = np.arange(start=0, stop=np.shape(res_data)[0])
    # for in_data ( only 200 lines, new data at each k%5==0)
    k_in = np.arange(start=0, stop=np.shape(in_data)[0]*5, step=5)

    # generate parameters for plot
    for i in range(nodes):
        # adding all states to plot
        if i ==0:
            axes[0].plot(k, res_data[:,i], color='blue', linewidth=.1, label='State of node')
        else:
            axes[0].plot(k, res_data[:,i], color='blue', linewidth=.1)
        axes[1].plot(k, res_data[:,i], color='blue', linewidth=.2)
        # adding input signals to plot
        if i == 0:
            axes[0].plot(k_in, in_data[:,i], color='red', marker='x', markersize=2, linewidth=0.1, label='Reference signals')
        else:
            axes[0].plot(k_in, in_data[:,i], color='red', marker='x', markersize=2, linewidth=0.1)

    axes[1].plot(k, mean_k, 'g_', markersize=6, label='real mean')

    axes[2].plot(k, max_rel_difference, 'k', marker='x', markersize=3, label='Max % difference (states)', linewidth=0.3)
    axes[2].plot(k, state_mean_vs_real_mean, color='orange', label='%Error states mean', linewidth=1)
    # axes[2].plot(k, percentage_diff_state_diff_input, color='brown', label='Diff States / Diff Input', linewidth=1)
    axes[2].plot(list(convergence_means.keys()), list(convergence_means.values()), color='brown', label='Convergence', linewidth=1)

    # generate graph
    fig.suptitle(f"{res_path.parent.name}: {in_path.stem}")
    axes[0].grid(True)
    axes[2].grid(True)
    axes[0].set_ylabel('value', fontsize=10)
    axes[0].set_title('States and reference signals vs k', fontsize=12)
    axes[1].set_ylabel('value', fontsize=10)
    axes[1].set_title('Reference signals and true average vs k', fontsize=12)
    axes[2].set_xlabel('k', fontsize=10)
    axes[2].set_ylabel('%', fontsize=10)
    axes[2].set_title('Deviations', fontsize=12)
    axes[2].set_ylim(-10,110)

    legend = fig.legend()
    os.makedirs(f"graphics/{res_path.parent.name}", exist_ok=True)
    fig.savefig(f"graphics/{res_path.parent.name}/{res_path.stem}.png", dpi=500)
    axes[2].set_xlim(-5,120)
    fig.set_size_inches(8,8)
    legend.remove()
    # fig.savefig(f"graphics/{res_path.parent.name}/{res_path.stem}.start.png", dpi=300)
    if display:
        fig.show()
        plt.show()

if __name__ == "__main__":
    ''' The time recording was performed by the multi_visualizer! '''
    NODES = 15
    RUN = 2
    SET = 4
    result_path = Path(f"../results/res-run-{RUN}-{SET}-n{50}.csv")
    input_path = Path(f"../data/run-{RUN}-{SET}-n{50}.csv")
    # print(plt.style.available)
    plt.style.use('fivethirtyeight')
    if not result_path.exists() or not input_path.exists():
        print('File does not exist.')
        sys.exit(-1)
    plot_graph_single_run(result_path, input_path, NODES, False)


import os
import asyncio
from glob import glob
import json
import pandas as pd

import tensorflow as tf
from google.cloud import storage

# from dataaccess.session import database

gcp_project = os.environ["GCP_PROJECT"]
bucket_name = "vqa-dataset-bucket"
local_experiments_path = "/persistent/experiments"

# Setup experiments folder
if not os.path.exists(local_experiments_path):
    os.mkdir(local_experiments_path)

def download_blob(bucket_name, source_blob_name, destination_file_name):
    """Downloads a blob from the bucket."""

    storage_client = storage.Client(project=gcp_project)

    bucket = storage_client.bucket(bucket_name)
    blob = bucket.blob(source_blob_name)
    blob.download_to_filename(destination_file_name)

def download_experiment_metrics():
    # Get all model metrics
    models_list = tf.io.gfile.glob(
        "gs://vqa-dataset-bucket/model/*/*")

    nested_folders_list = tf.io.gfile.glob(
        "gs://vqa-dataset-bucket/model/*/*/*")

    models_list = models_list + nested_folders_list

    timestamp = 0

    for model_file in models_list:
        print("Metrics file:", model_file)

        if model_file.endswith("/"):
            print("Skipping since its a folder")
            continue

        relative_path = model_file.split("/model/")[1]

        local_model_file = os.path.join(local_experiments_path, relative_path)

        if not os.path.exists(local_model_file):
            print("Copying:", model_file, local_model_file)

            # Ensure user directory exists
            os.makedirs(os.path.dirname(local_model_file), exist_ok=True)

            model_file = model_file.replace(
                "gs://vqa-dataset-bucket/", "")
            # Download the metric json file
            download_blob(bucket_name, model_file, local_model_file)

            file_timestamp = os.path.getmtime(local_model_file)
            if file_timestamp > timestamp:
                timestamp = file_timestamp

    return timestamp

def download_best_models():
    print("Download leaderboard models and artifacts")
    try:

        df = pd.read_csv(local_experiments_path+"/leaderboard.csv")
        print("Shape:", df.shape)
        print(df.head())

        for index, row in df.iterrows():
            print(row["user"], row["experiment"], row["model_name"])

            download_file = os.path.join(
                row["user"], row["experiment"], row["model_name"]+".hdf5")
            download_blob(bucket_name, download_file,
                          os.path.join(local_experiments_path, download_file))

            download_file = os.path.join(
                row["user"], row["experiment"], row["model_name"]+"_train_history.json")
            download_blob(bucket_name, download_file,
                          os.path.join(local_experiments_path, download_file))

            # Data details
            download_file = os.path.join(
                row["user"], row["experiment"], "data_details.json")
            download_blob(bucket_name, download_file,
                          os.path.join(local_experiments_path, download_file))
    except:
        print("Error in download_best_models")

async def save_leaderboard_db():
    # read leaderboard
    df = pd.read_csv(local_experiments_path+"/leaderboard.csv")
    print("Shape:", df.shape)
    print(df.head())

    # Delete current data in table
    query = "delete from leaderboard;"
    print("query:", query)
    await database.execute(query)

    # Insert rows
    query = f"""
        INSERT INTO leaderboard (
                trainable_parameters ,
                execution_time ,
                loss ,
                accuracy ,
                model_size ,
                learning_rate ,
                batch_size ,
                epochs ,
                optimizer ,
                email ,
                experiment ,
                model_name
            ) VALUES (
                :trainable_parameters ,
                :execution_time ,
                :loss ,
                :accuracy ,
                :model_size ,
                :learning_rate ,
                :batch_size ,
                :epochs ,
                :optimizer ,
                :email ,
                :experiment ,
                :model_name
            );
       """
    for index, row in df.iterrows():
        await database.execute(query, {
            "trainable_parameters": row["trainable_parameters"],
            "execution_time": row["execution_time"],
            "loss": row["loss"],
            "accuracy": row["accuracy"],
            "model_size": row["model_size"],
            "learning_rate": row["learning_rate"],
            "batch_size": row["batch_size"],
            "epochs": row["epochs"],
            "optimizer": row["optimizer"],
            "email": row["user"],
            "experiment": row["experiment"],
            "model_name": row["model_name"]
        })

class TrackerService:
    def __init__(self):
        self.timestamp = 0

    async def track(self):
        while True:
            await asyncio.sleep(60)
            print("Tracking experiments...")

            # Download new model metrics
            timestamp = download_experiment_metrics()


import logging
import ntpath
import os
import pickle
import shutil
import time

import pywren_ibm_cloud.sort.config
from pywren_ibm_cloud.sort.asynchronous.sort import sort_async
from pywren_ibm_cloud.sort.config import INTERMEDIATE_PATH_PREFIX, OUTPUT_PATH_PREFIX, \
    DEFAULT_GRANULE_SIZE_SHUFFLE, PARTITION_FILE_PATH_PREFIX, CHUNK_RANGE_FILE_NAME, \
    PARSER_FATA_FILE_NAME, SAMPLE_PATH_PREFIX
from pywren_ibm_cloud.sort.parse import parse
from pywren_ibm_cloud.sort.phase1 import partitions_into_segments_csv
from pywren_ibm_cloud.sort.predict.search_optimal import search_optimal
from pywren_ibm_cloud.sort.tests.phase1_alternatives.phase1_imzml import map_chunks_into_segments_imzml_v3
from pywren_ibm_cloud.sort.phase2 import reduce_partition_segments
from pywren_ibm_cloud.sort.sample import sample_dataset_imzml, sample_dataset_csv

# Using enum class create enumerations
from pywren_ibm_cloud.sort.utils import get_dataset_size

SORT_DATA_FORMATS = {"imzml", "csv"}
DEFAULT_NUM_WORKERS = 5

logger = logging.getLogger(__name__)

# TODO: delete sleep time argument, only for straggler sensitivity testing.
def create_sort_csv(pw, input_data_path, format_of_data, num_segm, primary_key_column,
                    num_workers_p1=None, granularity=None, delimiter=None, dtypes=None,
                    speculative_map=False, speculative_reduce=False, asynchronous=False,
                    output_path=None, output_bucket=None, segment_ranges=None):
    print("Sort dataset in {} segments".format(num_segm))
    print("Key column: {}".format(primary_key_column))
    print("Asynchronous: {}".format(asynchronous))
    print("Speculative execution: map {} reduce {}".format(speculative_map, speculative_reduce))

    # Manage path format
    if input_data_path.startswith("cos"):
        bucket = input_data_path.split("/")
        bucket = bucket[3]
        path = '/'.join(input_data_path.split("/")[4:])
    else:
        path = input_data_path
    if segment_ranges is not None:
        if isinstance(segment_ranges, str):
            segment_ranges = pickle.load(open(segment_ranges, 'rb'))
            try:
                iter(segment_ranges)
            except Exception:
                raise ValueError("segment_ranges object must be iterable")
            segment_ranges = list(segment_ranges)
        if not isinstance(segment_ranges, list):
            raise ValueError("segment_ranges param must be a list")
    file_name = ntpath.basename(path)
    csv_file_path = path

    my_executor_id = pw.executor_id.replace("/", "_")
    intermediate_path = "{}_{}_{}".format(INTERMEDIATE_PATH_PREFIX, file_name, my_executor_id)
    if output_path is None:
        output_path = "{}_{}_{}".format(OUTPUT_PATH_PREFIX, file_name, my_executor_id)
    if output_bucket is None:
        output_bucket = bucket
    if num_workers_p1 is None:
        num_workers_p1 = num_segm
    if granularity is None:
        granularity = DEFAULT_GRANULE_SIZE_SHUFFLE
    partition_file_path = "{}_{}_{}".format(PARTITION_FILE_PATH_PREFIX, file_name, my_executor_id)
    chunk_range_file_path = "{}/{}.pickle".format(intermediate_path, CHUNK_RANGE_FILE_NAME)
    parser_file_path = "{}/{}.pickle".format(intermediate_path, PARSER_FATA_FILE_NAME)
    sample_info_path = "{}_{}_{}".format(SAMPLE_PATH_PREFIX, file_name, my_executor_id)

    if num_segm is None:
        dataset_size = get_dataset_size(pw, bucket, csv_file_path)
        print("Dataset size {}".format(dataset_size))
        num_segm = search_optimal(pw.config, dataset_size, granularity)
        num_workers_p1 = num_segm
        print("Inferenced {} mappers and {} reducers".format(num_workers_p1, num_segm))

    print("Parsing")
    parse(pw,
          input_bucket=bucket,
          output_bucket=bucket,
          input_path=csv_file_path,
          intermediate_path=intermediate_path,
          output_path=output_path,
          num_workers_phase1=num_workers_p1,
          num_workers_phase2=num_segm,
          delimiter=delimiter,
          granularity=granularity,
          partition_file_path=partition_file_path,
          parser_file_path=parser_file_path,
          sample_info_path=sample_info_path,
          partition_bucket=bucket,
          chunk_range_file_path=chunk_range_file_path,
          chunk_range_bucket=bucket,
          sort_column_numbers=[primary_key_column],
          runtime_memory=pw.config['pywren']['runtime_memory'],
          sample=True,
          dtypes=dtypes,
          sampling_mode="random",
          canary=False,
          segment_ranges=segment_ranges)

    if segment_ranges is None:
        print("Sampling")
        sample_dataset_csv(pw,
                           parser_file_path,
                           output_bucket,
                           num_workers_p1,
                           sample_type="random",
                           canary=False)

    print("Mapping chunks into segments")

    # Create straggler info
    # random.seed()
    # straggler_ids_p1 = random.sample(list(range(num_workers_p1)), max(int(num_workers_p1*DEFAULT_SPECULATIVECAP_TOTAL_TASKS), 1))
    # for id in straggler_ids_p1:
    #       ibm_cos.put_object(Bucket=bucket, Key="straggler_{}".format(id), Body=pickle.dumps(id))
    # print(straggler_ids_p1)
    # straggler_ids_p2 = random.sample(list(range(num_segm)),
    #                                   max(int(num_segm * DEFAULT_SPECULATIVECAP_TOTAL_TASKS), 1))
    # for id in straggler_ids_p2:
    #      ibm_cos.put_object(Bucket=bucket, Key="straggler_p2_{}".format(id), Body=pickle.dumps(id))
    # print(straggler_ids_p2)

    if not asynchronous:
        partitions_into_segments_csv(pw,
                                     num_workers_p1,
                                     parser_file_path,
                                     output_bucket,
                                     speculative=speculative_map)

        print("Reducing segments")
        reduce_partition_segments(pw,
                                  num_segm,
                                  parser_file_path,
                                  output_bucket,
                                  speculative=speculative_reduce)

    else:
        sort_async(pw,
                   num_workers_p1,
                   num_segm,
                   parser_file_path,
                   output_bucket,
                   speculative_map=speculative_map,
                   speculative_reduce=speculative_reduce)

    print("Sort finished, output segments at: ")
    print("bucket {}".format(output_bucket))
    print("path {}".format(output_path))

def create_sort_imzml(pw, input_data_path, format_of_data, ibm_cos, segm_n, num_workers=None, granularity=None):
    if granularity is not None:
        pywren_ibm_cloud.sort.config.DEFAULT_GRANULE_SIZE_SHUFFLE = granularity
        # ut.DEFAULT_GRANULE_SIZE_REDUCE = granularity

    print("Sort dataset in {} segments".format(segm_n))
    start_time = time.time()

    if num_workers != None:
        effective_num_workers = num_workers
    else:
        effective_num_workers = DEFAULT_NUM_WORKERS

    # Directory for temporary files
    shutil.rmtree('tmp', ignore_errors=True)
    os.mkdir("tmp")

    print("Generating parser")
    # Manage path format
    if input_data_path.startswith("cos"):
        bucket = input_data_path.split("/")

        bucket = bucket[3]
        path = '/'.join(input_data_path.split("/")[4:])
    else:
        path = input_data_path

    print("Bucket " + bucket + " and path " + path)

    prefix = "ordered_segments"
    objs = ibm_cos.list_objects_v2(Bucket=bucket, Prefix=prefix)
    while 'Contents' in objs:
        keys = [obj['Key'] for obj in objs['Contents']]
        formatted_keys = {'Objects': [{'Key': key} for key in keys]}
        ibm_cos.delete_objects(Bucket=bucket, Delete=formatted_keys)

        objs = ibm_cos.list_objects_v2(Bucket=bucket, Prefix=prefix)

    # Create local metadata object
    # imzml_metadata = ibm_cos.get_object(Bucket=bucket, Key=''.join([path, ".imzml"]))['Body'].read()
    # metadata_file_path = "tmp/" + input_data_path.split("/")[-1] + ".imzml"
    ibd_file_path = ''.join([path, ".ibd"])
    metadata_file_path = ''.join([path, ".imzml"])

    num_workers, parser_data, start_sampling_time = sample_dataset_imzml(pw, bucket,
                                                                         metadata_file_path, ibd_file_path,
                                                                         effective_num_workers,
                                                                         "random", segm_n, ibm_cos)
    print("Sampling finished")
    print("Final number of workers {}".format(num_workers))
    #
    start_time_effective = time.time()
    print("Mapping chunks into segments")

    map_chunks_into_segments_imzml_v3(pw, num_workers, bucket, ibd_file_path,
                                      bucket, "tmp/partition_file.pickle", parser_data)

    print("Reducing segments")
    # reduce_chunk_segments_imzml(pw, num_workers, bucket, bucket,
    #                             segm_n)

    end_time_effective = time.time()

    # # Sample dataset
    print("Removing temporal files")

    shutil.rmtree('tmp', ignore_errors=True)
    prefix = "tmp"
    args = {'info': {'prefix': prefix, 'bucket': bucket}}
    futures = pw.call_async(ut.clear_cos_prefix, args)
    pw.get_result(futures)

    end_time = time.time()
    print("Sort ended")
    print("--- %s seconds ---" % (end_time - start_time))
    print("--- %s seconds without loading ---" % (end_time - start_sampling_time))
    print("--- %s seconds without loading and sampling ---" % (end_time - start_time_effective))
    print("--- %s seconds without final clearing ---" % (end_time_effective - start_time))

    f = open("equations_results.txt", 'a')
    f.write("{}\n".format([(end_time - start_time), (end_time - start_sampling_time), (end_time - start_time_effective),
                           (end_time_effective - start_time)]))
    f.close()


#!/usr/bin/env python
# coding: utf-8

# ## Read data

# In[1]:

import numpy as np
import pandas as pd
import datetime
import os
import matplotlib.pyplot as plt
import matplotlib
import networkx as nx
import pickle
from collections import OrderedDict
import copy
from scipy.sparse import csr_matrix
from scipy import io
import seaborn as sns
import joblib
from base import *
from joblib import Parallel, delayed
import random
import scipy as sc
import sys

# In[2]:

def find_nearest(array, value):
    array = np.asarray(array)
    idx = (np.abs(array - value)).argmin()
    return array[idx]

# ### Read raw speed and count data

# In[3]:

start_month=int(sys.argv[1])
start_day=int(sys.argv[2])
end_month=int(sys.argv[3])
end_day=int(sys.argv[4])
year =int(sys.argv[5])
flag_weekday = int(sys.argv[6])

read_in_folder = '../../../Raw_data_hpc/Processed_data_networkL/'
read_in_pkl = '../../../Raw_data_hpc/link_dict/'
write_folder = './'

with open(read_in_pkl+'volume_dict_link_'+str(year)+"_"+str(start_month)+"_"+str(start_day)+'.pkl', 'rb') as handle:
    count_data = pickle.load(handle)
with open(read_in_pkl+'speed_dict_link_'+str(year)+"_"+str(month)+"_"+str(day)+'.pkl', 'rb') as handle:
    spd_data = pickle.load(handle)

print(spd_data)

# ### Read graph data

# In[5]:

with open(read_in_folder+'od_list.pickle', 'rb') as handle:
    (O_list, D_list) = pickle.load(handle)
# with open('graph.pickle', 'rb') as handle:
#     G = pickle.load(handle)
G = nx.read_gpickle(read_in_folder+'graph_4.pickle')

# with open(read_in_folder+'od_list_4.pickle', 'rb') as handle:
#     pickle.dump((O_list, D_list),handle, protocol=4)

# In[6]:

with open(read_in_folder+'od_list.pickle', 'rb') as handle:
    (O_list, D_list) = pickle.load(handle)
# with open('graph.pickle', 'rb') as handle:
#     G = pickle.load(handle)
G = nx.read_gpickle(read_in_folder+'graph_4.pickle')
G = nx.freeze(G)
# print(O_list)
pos=nx.get_node_attributes(G,'pos')
plt.rcParams['figure.figsize'] = [8, 6]
nx.draw_networkx(G,pos,node_size=10,node_color='blue',with_labels=False)
# print(G.number_of_edges())
# print(G.number_of_nodes())
Node = list(G.nodes)
Node.sort()
# print("Nodes: ", Node)

print(O_list)

# In[7]:

#no > flag_weekday1 & no < flag_weekday2:
if flag_weekday == 1:
    flag_weekday1 = -1
    flag_weekday2 = 7
else:
    flag_weekday1 = -1
    flag_weekday2 = 7

start_date = datetime.date(year, start_month, start_day)
end_date = datetime.date(year, end_month, end_day)
delta_date = datetime.timedelta(days = 1)
time_basis = datetime.time(0,0,0)
N_path = 1
time_interval_min = 5

# In[8]:

from itertools import islice
def k_shortest_paths(G, source, target, k, weight=None):
    def path_cost(G, path):
        return sum([G[path[i]][path[i+1]]['length'] for i in range(len(path)-1)])
    p = list(islice(nx.shortest_simple_paths(G, source, target, weight=weight), k))
    return p

# In[9]:

OD_paths = OrderedDict()
link_dict = OrderedDict()
path_list = list()
shortest_path_list = list()
count = 0;
k=1
N_path = 1
# while len(shortest_path_list) < N_path:
OD_list_output = []
print(O_list)
for O in O_list:
    for D in O_list:
        print(O, D);
        if O!=D:
            try:
                paths = k_shortest_paths(G,O, D,N_path,weight='length') #My code
                if len(paths) != 0:
                    tmp_path_list = list()
                    for path in paths:
                        path_o = Path()
                        path_o.node_list = path
                        path_o.node_to_list(G, link_dict)
                        tmp_path_list.append(path_o)
                        path_list.append(path_o)

                    OD_paths[(O, D)] = tmp_path_list
                    shortest_path_list.append(paths)
                    OD_list_output.append([O,D])
                    print("From ", O, " To ", D, "there are ", k, "paths")
                else:
                    pass
            except:
#                     print("From ", O, " To ", D, "there are no paths")
                pass
        else:
#                 print("From ", O, " To ", D, "same")
            pass
np.savetxt('path_'+start_date.strftime("%m_%d_%Y")+'_'+           str(time_interval_min)+'.txt', OD_list_output, delimiter = ',',fmt='%s')

# Find k shortest path:
# 1. for each pair of OD find shortest path
# 2. order all shortest paths
# 3. find the top k shortest path among all

# ## Generate Delta and Set Parameters

analysis_start_time = datetime.time(0, 0, 0)
time_interval = datetime.timedelta(minutes=time_interval_min)
num_OD = len(OD_paths)
link_list = list(link_dict.values())
num_link = len(link_list)
num_path_v = [len(x) for x in OD_paths.values()]
num_path = np.sum(num_path_v)
N = int(60 / time_interval_min * 24)-1

assert(len(path_list) == num_path)

# In[11]:

delta = np.zeros((num_link, num_path))
for i, link in enumerate(link_list):
    for j, path in enumerate(path_list):
        if link in path.link_list:
            delta[i,j] = 1.0

link_loc = dict()
for idx, link in enumerate(link_list):
    link_loc[link] = idx

# ## Build assignment matrix

# In[13]:

cur_date_time = datetime.datetime.combine(start_date, time_basis)
end_date_time = datetime.datetime.combine(end_date, time_basis)

date_need_to_finish = list()

while(cur_date_time <= end_date_time):
    no = cur_date_time.weekday()

    if (no > flag_weekday1) & (no < flag_weekday2):
        single_date = cur_date_time.date()
        date_need_to_finish.append(single_date)
    cur_date_time = cur_date_time + delta_date
print(date_need_to_finish)

# A parallel computing framework is used to compute the R matrix as well as P matrix. Since we have a 8 core CPU, so we use 7 process to run the program, leaving one core to ensure the desktop does not get stuck.

# In[14]:

import importlib
importlib.reload(sys.modules['base'])
Parallel(n_jobs=3, temp_folder = 'temp', max_nbytes = '10M')(delayed(save_r)(N, spd_data, analysis_start_time, time_interval,
                        tmp_date, link_dict, link_list, link_loc, path_list) for tmp_date in date_need_to_finish)

# ## Construct P matrix

# In[15]:

cur_date_time = datetime.datetime.combine(start_date, time_basis)
end_date_time = datetime.datetime.combine(end_date, time_basis)

date_need_to_finish = list()

while(cur_date_time <= end_date_time):
# #     date_need_to_finish.append(cur_date_time)
    no = cur_date_time.weekday()
    if (no > flag_weekday1) & (no < flag_weekday2):
        single_date = cur_date_time.date()
        date_need_to_finish.append(single_date)
    cur_date_time = cur_date_time + delta_date

# #### parallel computing

# In[16]:

start_time = datetime.datetime.now()
Parallel(n_jobs=3)(delayed(save_p)(N, spd_data, analysis_start_time, time_interval,
                                   tmp_date, path_list, OD_paths) for tmp_date in date_need_to_finish)
t_ = (datetime.datetime.now() - start_time)
print("solve time:", t_)

# ## Construct link flow vector

# In[17]:

o_link_list = list(filter(lambda x: x.ID in count_data.keys(), link_list))

# In[18]:

def nearest(tmp_date, items, pivot):
    return min(items, key=lambda x: abs(datetime.datetime.combine(tmp_date,x) - datetime.datetime.combine(tmp_date,pivot)))

def get_x_o(N, o_link_list, tmp_date, analysis_start_time, time_interval, count_data):
#     tmp_date = start_date
    num_o_link = len(o_link_list)
#     print("num: ", num_o_link)
    x = np.zeros(num_o_link * N)
#     x = np.zeros(N)
    #print("x: ", x)
    for h in range(N):
#         tmp_date = datetime.datetime.combine(tmp_date, analysis_start_time)
        start_time = (datetime.datetime.combine(tmp_date, analysis_start_time) + h * time_interval).time()
        end_time = (datetime.datetime.combine(tmp_date, analysis_start_time) + (h+1) * time_interval).time()
        for a, link in enumerate(o_link_list):
            possible_time = np.array(count_data[link.ID].columns)
            print('ID', link.ID);
            if count_data[link.ID].shape[0] == 0:
                if h != N-1:
                    x[h * num_o_link + a] = 0
                else:
                    x[h * num_o_link + a] = 0
            else:
                df = count_data[link.ID].loc[tmp_date,:]
                if h != N-1:
                    x[h * num_o_link + a] = df.loc[(df.index.values >= start_time) &
                                                (df.index.values <= end_time)].sum()
                else:
                    x[h * num_o_link + a] = df.loc[(df.index.values >= start_time)].sum()
#         print('x, ', x)
    date_str = tmp_date.strftime("%Y-%m-%d")
    np.save(os.path.join('X_vector', date_str), x)
    return x

# In[19]:

cur_date_time = datetime.datetime.combine(start_date, time_basis)
end_date_time = datetime.datetime.combine(end_date, time_basis)

while(cur_date_time <= end_date_time):
    no = cur_date_time.weekday()
    if (no > flag_weekday1) & (no < flag_weekday2):
        single_date = cur_date_time.date()
        date_need_to_finish.append(single_date)
        x = get_x_o(N, o_link_list, single_date, analysis_start_time, time_interval, count_data)
        date_str = cur_date_time.strftime("%Y-%m-%d")

    cur_date_time = cur_date_time + delta_date

# ## Construct sysmmetric constraints

# In[20]:

# Decide the OD region.
node_df = pd.read_csv(read_in_folder + 'node_list.csv')
regions = np.linspace(1, 26, num=26)
# print(node_df)

node_df[node_df['node_id'] == 1]['Region']

# In[21]:

regions = np.linspace(1, 26, num=26)
num_regions = len(regions)

# for tmp_date in date_need_to_finish:
row_list = []
col_list = []
data_list =[]

# num_origin = len(origin_list)
y_loc = 0
print(num_regions * num_regions/2)
Alr_sym = np.zeros((int(num_regions * num_regions), num_OD * N))

for h in range(N):
    rs = 0
    k = 0
    a = -1

    for (O,D), paths in OD_paths.items():
#         print('O, ', O,' D, ', D)
        x_loc = h * num_OD + rs # OD list
        rs+=1
        O_index = node_df[node_df['node_id'] == O].Region.iloc[0]
#         print('O_index, ',O_index)

        D_index = node_df[node_df['node_id'] == D].Region.iloc[0]
#         print('D_index, ',D_index)
        if (O_index == D_index):
            continue
        elif (O_index < D_index):
            y_loc = (O_index-1)*num_regions+D_index
            row_list.append(y_loc)
            col_list.append(x_loc)
            Alr_sym[y_loc, x_loc] = 1
            data_list.append(1)
        elif (O_index > D_index):
            y_loc = (D_index-1)*num_regions+O_index
            row_list.append(y_loc)
            col_list.append(x_loc)
            Alr_sym[y_loc, x_loc] = -1
            data_list.append(-1)

print('Alr, ', Alr_sym, 'Alr_tmp nonzero elements, ', np.count_nonzero(Alr_sym))
print('Alr shape, ', Alr_sym.shape)
b_sym = np.zeros(num_regions * num_regions)

with open('Constraints/Alr_sym.npy', 'wb') as f:
    np.save(f, Alr_sym)
with open('Constraints/b_sym.npy', 'wb') as f:
    np.save(f, b_sym)

# ## Add arterial constraint

# In[22]:

read_folder = '../Raw_data/Processed_data/arterial_2020/vol_all.pkl'
with open(read_folder, 'rb') as handle:
    vol_arterial = pickle.load(handle);

regions = np.linspace(1, 26, num=26)
num_regions = len(regions)

# for tmp_date in date_need_to_finish:
row_list = []
col_list = []
data_list = []

# num_origin = len(origin_list)
y_loc = 0
print(len(Node))
Alr_arterial = np.zeros((len(Node), num_OD * N))
b_arterial = np.zeros((len(Node),1))
for h in range(N):
    rs = 0
    k = 0
    a = -1
    for (O,D), paths in OD_paths.items():
        x_loc = h * num_OD + rs # OD list
        rs+=1

        if (O in vol_arterial.keys()):
#             print('O, ',O)
            row_list.append(O)
            col_list.append(x_loc)
            Alr_arterial[O, x_loc] = 1
            data_list.append(1)
            b_arterial[O] = vol_arterial[O]

        if (D in vol_arterial.keys()):
#             print('D, ',D)
            row_list.append(D)
            col_list.append(x_loc)
            Alr_arterial[D, x_loc] = 1
            data_list.append(1)
            b_arterial[D] = vol_arterial[D]

Alr_arterial = Alr_arterial[~np.all(Alr_arterial == 0, axis=1)]
b_arterial = b_arterial[~np.all(b_arterial == 0, axis=1)]

Alr_arterial_I = -np.identity(len(Alr_arterial))
Alr_arterial =np.concatenate((Alr_arterial, Alr_arterial_I), axis=1)

with open('Constraints/Alr_arterial.npy', 'wb') as f:
    np.save(f, Alr_arterial)
with open('Constraints/b_arterial.npy', 'wb') as f:
    np.save(f, b_arterial)

# ## Create the observed delta (time dependent)

# In[23]:

observe_index = np.fromiter(map(lambda x: x in o_link_list, link_list),dtype=np.int)
observe_index_N = np.tile(observe_index, (N,))
np.save(os.path.join("observe_index_N"), observe_index_N)

# In[24]:

print(sum(observe_index_N == 1))
print(len(observe_index_N))

# ## Load data to conduct DODE

# from pfe import nnls

# In[25]:

date_str = start_date.strftime("%Y-%m-%d")

# In[26]:

cur_date_time = datetime.datetime.combine(start_date, time_basis)
end_date_time = datetime.datetime.combine(end_date, time_basis)

date_need_to_finish = list()

while(cur_date_time <= end_date_time):
    no = cur_date_time.weekday()

    if (no > flag_weekday1) & (no < flag_weekday2):
        single_date = cur_date_time.date()
        date_need_to_finish.append(single_date)
    cur_date_time = cur_date_time + delta_date
print(date_need_to_finish)

# In[27]:

def split_select(A, mask, N):
    A_list = np.array_split(A, N)
    print(len(A_list))
    mask_list = np.array_split(mask, N)
    tmp_list = []
    for i in range(len(A_list)):
        tmp_A = A_list[i]
        print("tmp_A shape, ", tmp_A.shape)
        tmp_mask = mask_list[i]
        print("tmp_mask, ", tmp_mask.shape)
        A_mask = tmp_A[tmp_mask == 1, :]
        print("shape A_mask, ", A_mask.shape)
        tmp_list.append(A_mask)
#         break
    print("concatenate")
    A = np.concatenate(tmp_list, axis = 0)
    print("shape A, ", A.shape)
    return A

# ## Initial q vector

# In[28]:

np.random.seed(1)
regions = np.linspace(1, 26, num=26)
# num_regions = len(regions)
M = len(OD_paths)
# for tmp_date in date_need_to_finish:
row_list = []
col_list = []
data_list =[]

# num_origin = len(origin_list)
y_loc = 0
q_sym = np.zeros((M, N))
dict_q = {}

for i,((O,D), paths) in enumerate(OD_paths.items()):
    if (O,D) in dict_q.keys():
        q_sym[i,:] = dict_q[(O,D)];
    else:
        tmp = np.random.rand(1,N);
        dict_q[(O,D)] = tmp;
        dict_q[(D,O)] = tmp;
        q_sym[i,:] = tmp;

q_sym = q_sym.T.reshape(-1)
print(q_sym.shape)

# ## NNLS model

# In[37]:

import importlib
from pfe_sym import nnls
importlib.reload(sys.modules['pfe_sym'])
import math

def solve_nnls(date, sum_val = 0, penalty_beta = 0, penalty_gamma = 0):
    cur_date_time = datetime.datetime.combine(start_date, time_basis)

    all_r = []
    all_P = []
    all_x_o = np.array([])
    all_alr = []
    all_A = None

    q_dict = {}
    q_dict_2d = {}

    N = int(60 / time_interval_min * 24)
    M = len(OD_paths)

    i = 0
    observe_index_N = np.load("observe_index_N.npy")

#     while(cur_date_time <= end_date_time):
#         print(cur_date_time)
    no = cur_date_time.weekday()
    if (no > flag_weekday1) & (no < flag_weekday2):
        single_date = cur_date_time.date()
        date_str = single_date.strftime("%Y-%m-%d")

        x_o = np.load(os.path.join('X_vector', date_str + ".npy"))
#         print('X_vector', date_str," ",x)

        print('x: ',len(x_o))
        r = joblib.load(os.path.join("R_matrix", date_str + ".pickle")).tocsr()
#         r = joblib.load(os.path.join("R_matrix", date_str + ".pickle")).toarray()
        print('r: ', r.shape)
        print('r type: ', type(r))

        P = joblib.load(os.path.join("P_matrix", date_str+".pickle")).tocsr()
#         P = joblib.load(os.path.join("P_matrix", date_str+".pickle")).toarray()
        print('P: ',P.shape)

        RP = np.array(r.dot(P).toarray())[observe_index_N == 1,:]
#         RP = (r.dot(P)).toarray()
#         print('RP: ', RP.shape)
#         print('RP: ', type(RP))
#         print('observe_index_N: ', observe_index_N.shape)

        ###############################################
        ################Read in file###################
        ###############################################
        Alr_sym = np.load(os.path.join('Constraints', "Alr_sym.npy"))
        Alr_arterial = np.load(os.path.join('Constraints', "Alr_arterial.npy"))
        b_sym = np.load(os.path.join('Constraints', "b_sym.npy"))
        b_arterial = np.load(os.path.join('Constraints', "b_arterial.npy"))
        ###############################################
        ###############################################

        ###############################################
        ################Constraints####################
        ###############################################
        np.random.seed(2)

        x2,y2 = Alr_arterial.shape
        A = np.concatenate((RP, Alr_sym*penalty_beta), axis=0)
        x1,y1 = A.shape
        A = np.concatenate((A, np.zeros((x1,y2-y1))), axis=1)
        A = np.concatenate((A, Alr_arterial), axis=0)

        b = np.concatenate((x_o, b_sym), axis=0)
        b = np.concatenate((b, b_arterial.reshape(-1)), axis = 0)
        b_max = np.max(b)
        q_init = np.random.rand(A.shape[1])
#         q_init = np.concatenate((q_sym, np.random.rand(y2-y1)), axis=0)
        print('q shape, ', q_init.shape)
#         b = b/b_max
        ###############################################
        ###############################################

        ###############################################
        #########Error upper bound constraint##########
        ###############################################
        A_ones = np.ones((1,RP.shape[1]))
        A_errorUb = np.concatenate((A_ones, np.zeros((1,y2-y1))), axis=1)
        A = np.concatenate((A, -A_errorUb*penalty_gamma), axis=0)
        b = np.append(b, -(sum_val)*penalty_gamma)
        ###############################################
        ###############################################

        N = int(60 / time_interval_min * 24)-1
        M = len(OD_paths)

        print('A',np.max(A))
        print('b',np.max(b))
#         (q_est, r_norm) = sc.optimize.nnls(A, b)
        print('A shape', A.shape)
        (q_est, r_norm) = nnls(A,b,q_init,2000, 500, 5, True)
        print("solve q.")
        print('error, ', r_norm)
        print('q_est, ', q_est.sum())

        np.save(os.path.join('Q_vector', 'q_vector_2019_sym_large_beta'+
                             str(np.format_float_scientific(penalty_beta, exp_digits=2))+'_gamma'+
                             str(np.format_float_scientific(penalty_gamma, exp_digits=2))+'.npy'), q_est[:y1])

#         q_est = q_est[:y1]
        q_est_2d = q_est[:y1].reshape(N,M)

#         q_dict[cur_date_time] = q_est
#         q_dict_2d[cur_date_time] = q_est_2d[:-1,:]

#         print('q_dict_2d,', q_dict_2d[cur_date_time].shape)

        x_est =  RP.dot(q_est[:y1])

        total_flow = np.sum(A_ones.dot(q_est[:y1]))
        axis_y = i%3
        axis_x = int(i/3)

        plt.scatter(x_o, x_est, c = 'b', s=2)
        plt.plot(x_o, x_o, 'r')

        plt.title(date_str)
        plt.savefig(date_str+'.png')
        plt.close()
        date_need_to_finish.append(single_date)
        i += 1

#         cur_date_time = cur_date_time + delta_date
        return total_flow

penalty_beta = 10
penalty_gamma = 1

cur_date_time = datetime.datetime.combine(start_date, time_basis)
total_flow = solve_nnls(cur_date_time)
total_flow_constraints = solve_nnls(cur_date_time, total_flow, penalty_beta, penalty_gamma)


import numpy as np
import pandas as pd
import nltk
import tensorflow_hub as hub
import tensorflow as tf
import pickle
import logging
from tqdm import tqdm

logging.disable(logging.CRITICAL)

elmo_model = hub.Module("https://tfhub.dev/google/elmo/3", trainable=True)

DATA_PATH = './data/'
VUA_PATH = './data/vua/'
TOEFL_PATH = './data/toefl/'

def prepare_vua_data(data_file, save_file):
  '''
  :param data_file: path to file in data/vua directory from which to read the dataset (csv format)
  :param save_file: path to file in data/vua directory to which to write the formatted dataset
  '''
  data = pd.read_csv(VUA_PATH + data_file)

  sentences = []
  pos_tags = []
  labels = []
  for row in data.values:
    sen = str(row[2]).split(' ')
    sen_len = len(sen)
    cleaned_sen = []
    labels_array = []
    pos_tag = []
    for w in sen:
      if w.startswith('M_'):
        labels_array.append(1)
        w1 = w.replace('M_', '')
      else:
        labels_array.append(0)
        w1 = w
      cleaned_sen.append(w1)
      pos_tag.append(nltk.pos_tag([w1])[0][1])
    cleaned_sen = ' '.join(cleaned_sen).strip()
    sentences.append(cleaned_sen)
    labels.append(labels_array)
    pos_tags.append(pos_tag)

    try:
      assert len(pos_tags) == sen_len
    except:
      print(cleaned_sen)
      print(pos_tags)

  # prepare data according to the format of https://github.com/gao-g/metaphor-in-context
  df = pd.DataFrame({'txt_id': data['txt_id'].values, 'sen_ix': data['sentence_id'].values,
                     'sentence': sentences, 'label_seq': labels, 'pos_seq': poss,
                     'labeled_sentence': data['sentence_txt'].values, 'genre': np.empty_like(labels)})

  df.to_csv(VUA_PATH + save_file, index=False)

def prepare_toefl_data(dir_path, save_path):
  '''
  :param dir_path: path to directory in data/toefl directory which contains the essays
  :param save_path: path to file in data/toefl directory to write the formatted dataset
  '''
  essay_ids = []
  sen_ids = []
  sentences = []
  labeled_sentences = []
  label_seq = []
  pos_seq = []
  for essay_file in tqdm(sorted(os.listdir(TOEFL_PATH + dir_path))):
    fp = open(DATA_PATH + dir_path + essay_file, 'r')
    essay_id = essay_file[:-4]
    i = 1
    for line in fp:
      sen = str(line)
      labels = []
      pos_tags = []
      cleaned_sen = []
      essay_ids.append(essay_id)
      sen_ids.append(str(i))
      labeled_sentences.append(str(line))

      for w in sen.split():
        if(w.startswith('M_')):
          w1 = w.replace('M_', '')
          labels.append(1)
        else:
          w1 = w
          labels.append(0)

        pos_tags.append(nltk.pos_tag([w1])[0][1])
        cleaned_sen.append(w1)

      assert len(labels) == len(sen.split())
      assert len(pos_tags) == len(labels)
      assert len(cleaned_sen) == len(labels)
      sentences.append(' '.join(cleaned_sen))
      pos_seq.append(pos_tags)
      label_seq.append(labels)
      i += 1

  df = pd.DataFrame({'txt_id': essay_ids, 'sen_ix': sen_ids, 'sentence': sentences,
                     'label_seq': label_seq, 'pos_seq': pos_seq,
                     'labeled_sentence': labeled_sentences, 'genre': list(np.empty_like(label_seq))})

  df.to_csv(TOEFL_PATH + save_path, index=False)

def train_val_split(read_path):
  '''
  :param read_path: path to formatted train dataset file in data/ directory
  '''
  np.random.seed(101)
  df = pd.read_csv(DATA_PATH + read_path)
  df = df[df['sentence'].notna()]

  # print(len(df))
  r = int(len(df)/10)
  df = df.sample(frac=1, random_state=1001)

  val_df = df[:r]
  train_df = df[r:]

  val_path = read_path[:-4] + '_val.csv'
  train_path = read_path[:-4] + '_train.csv'

  val_df.to_csv(DATA_PATH + val_path, index=False)
  train_df.to_csv(DATA_PATH + train_path, index=False)

def compute_elmo_vectors(read_path, save_path):
  '''
  :param read_path: path to formatted dataset files in data/ directory
  :param save_path: path to pickle file in data/directory to which to write the elmo vectors
  '''
  df = pd.read_csv(DATA_PATH + read_path)

  dic = {}
  txt_ids = df['txt_id'].values
  sen_ids = df['sen_ix'].values
  sentences = df['sentence'].values
  assert len(txt_ids) == len(sentences)

  batch_sentences = [sentences[i:min(i+64, len(sentences))] for i in range(0, len(sentences), 64)]
  batch_txt_ids = [txt_ids[i:min(i+64, len(txt_ids))] for i in range(0, len(txt_ids), 64)]
  batch_sen_ids = [sen_ids[i:min(i+64, len(sen_ids))] for i in range(0, len(sen_ids), 64)]
  batch_sen_len = [[len(sen.split(' ')) for sen in batch_sen] for batch_sen in batch_sentences]

  assert len(batch_sentences) == len(batch_txt_ids)

  for i in tqdm(range(len(batch_sen_ids))):
    sen = batch_sentences[i]
    txt_id = batch_txt_ids[i]
    sen_id = batch_sen_ids[i]
    sen_len = batch_sen_len[i]
    embed = elmo_model(sen, signature='default', as_dict=True)['elmo']

    with tf.Session() as sess:
      sess.run(tf.global_variables_initializer())
      sess.run(tf.tables_initializer())
      embeddings = sess.run(embed)
      # print(embeddings.shape)

    for t, s, e, l in zip(txt_id, sen_id, embeddings, sen_len):
      if t not in dic:
        dic[t] = {}

      dic[t][str(s)] = e[:l, :]

  sum = 0
  for k in dic:
    sum += len(dic[k].keys())

  assert len(sentences) == sum

  with open(DATA_PATH + save_path, 'wb+') as f:
    pickle.dump(dic, f)

def prepare_tokens_data(label_file, save_tokens_file):
  '''
  :param label_file: path to file in data/ directory which contains the test tokens
  :param save_tokens_file: path to pickle file in data/directory to write the offsets
                           of test tokens corresponding to a sentence
  '''
  labels = pd.read_csv(DATA_PATH + label_file, header=None)

  dic = {}
  for row in labels.values:
    text_id = row[0].split('_')
    txt_id = text_id[0]
    sen_id = text_id[1]
    offset = int(text_id[2]) - 1
    if txt_id not in dic:
      dic[txt_id] = {}
    if sen_id not in dic[txt_id]:
      dic[txt_id][sen_id] = [offset]
    else:
      dic[txt_id][sen_id].append(offset)

  with open(DATA_PATH + save_tokens_file, 'wb+') as  f:
    pickle.dump(dic, f)

def main():
  if sys.argv[1] == 'vua':
    prepare_vua_data('vuamc_corpus_train.csv', 'VUA_corpus.csv')
    prepare_vua_data('vuamc_corpus_test.csv', 'VUA_corpus_test.csv')
    train_val_split('vua/VUA_corpus.csv')
    compute_elmo_vectors('vua/VUA_corpus_train.csv', 'vua/elmo_train.pkl')
    compute_elmo_vectors('vua/VUA_corpus_val.csv', 'vua/elmo_val.pkl')
    compute_elmo_vectors('vua/VUA_corpus_test.csv', 'vua/elmo_test.pkl')
    prepare_tokens_data('vua/all_pos_test_tokens.csv', 'vua/all_pos_test_tokens.pkl')
    prepare_tokens_data('vua/verb_test_tokens.csv', 'vua/verb_test_tokens.pkl')
  elif sys.argv[1] == 'toefl':
    prepare_toefl_data('train_essays/', 'TOEFL_corpus.csv')
    prepare_toefl_data('test_essays/', 'TOEFL_corpus_test.csv')
    train_val_split('toefl/TOEFL_corpus.csv')
    compute_elmo_vectors('toefl/TOEFL_corpus_train.csv', 'toefl/elmo_train.pkl')
    compute_elmo_vectors('toefl/TOEFL_corpus_val.csv', 'toefl/elmo_val.pkl')
    compute_elmo_vectors('toefl/TOEFL_corpus_test.csv', 'toefl/elmo_test.pkl')
    prepare_tokens_data('toefl/all_pos_test_tokens.csv', 'toefl/all_pos_test_tokens.pkl')
    prepare_tokens_data('toefl/verb_test_tokens.csv', 'toefl/verb_test_tokens.pkl')
  else:
    print('Unknown option')

  if __name__ == "__main__":
    main()


from classes import Coordinate

# pas testing pake partisi 1cm (0.01) makan waktu hampir semenit
# klo mau coba2 bisa pake partisi lebih gede biar waktu nunggunya manusiawi

# Editable
perimeter = 0.1
height = 0.1
partition_length = 0.02  # 0.01, 0.02, atau 0.025
center_coordinate = Coordinate(0.1, 0.1, 0.05)

# Initial variables
diameter = perimeter * 2
ver_parts = int(height // partition_length) - 1  # jumlah partisi vertikal
hor_parts = int(diameter // partition_length) - 1  # jumlah partisi horizontal

# Const
epsilon = 8.85418782 * (10 ** -12)


from user import User, Friend
from flask import Flask, Blueprint
app = Flask(__name__)

def register(Targets):
    for Target in Targets:
        register_target=Blueprint(Target.__name__.lower(), __name__, template_folder='templates')
        register_target.add_url_rule('/', view_func=Target.as_view(Target.__name__.lower()))
        app.register_blueprint(register_target, url_prefix=f'/{Target.__name__.lower()}')
        print(Target.__name__)

register([User,Friend])
# register(Friend)
# users  = Blueprint('users', __name__, template_folder='templates')
# users.add_url_rule('/', view_func=UserView.as_view('user'))
# app.register_blueprint(users, url_prefix='/user')


from typing import List
import numpy as np
import pycuda.gpuarray as gpuarray
import pycuda.driver as cuda
import pycuda.compiler as compiler

class board:
    def __init__(self, *strRepresentation:str):
        self.colors = [
            '\033[0;30;41m',
            '\033[0;30;42m',
            '\033[0;30;43m',
            '\033[0;30;44m',
            '\033[0;30;45m',
            '\033[0;30;46m',
            '\033[0;30;47m',
            '\033[0;31;41m',
            '\033[0;31;42m',
            '\033[0;31;43m',
            '\033[0;31;44m',
            '\033[0;31;45m',
            '\033[0;31;46m',
            '\033[0;31;47m',
            ]
        self.black = '\033[0;37;40m'
        self.repr = strRepresentation

        lines = strRepresentation
        self.empty = np.zeros((len(lines[0]), len(lines)), 'uint8')

        boardcode = """
        __global__ void place(%(dtype)s* board, %(dtype)s* shape, const %(dtype)s newId, const int x, const int y, const int shapew, const int shapeh, %(dtype)s* res, %(dtype)s* success)
        {
            const int idx = threadIdx.x + ncols * threadIdx.y;
            const int shapeidx = (threadIdx.x - x) + shapew * (threadIdx.y - y);
            if (x <= threadIdx.x && threadIdx.x < x + shapew && y <= threadIdx.y && threadIdx.y < y + shapeh)
                res[idx] = board[idx] + shape ? (shape[shapeidx] + newId) : 0;
            else
                res[idx] = board[idx]
            if (res[idx] >= 0b10000000) *sucess = false;
        }

        __global__ void undo(%(dtype)s* board, const %(dtype)s id, const int x, const int y)
        {
            const int idx = threadIdx.x + x + %(ncols)s * (threadIdx.y + y);
            if (board[idx] & 0b00111111 == id) board[idx] = 0;
        }
        """ % {
            'dtype':"unsigned char",
            'ncols':self.empty.shape[0],
            }

        boardmod = compiler.SourceModule(boardcode)
        self.cudaPlace = boardmod.get_function('place')
        self.cudaUndo = boardmod.get_function('undo')

        for i, line in enumerate(lines):
            for ii, c in enumerate(line):
                if c is ' ':
                    self.empty[ii,i] = 0b00000000
                elif c is 'O':
                    self.empty[ii,i] = 0b01000000
        self.empty = gpuarray.to_gpu(self.empty)
        self.res = self.empty.copy()
        self.clear()

    def clear(self):
        self.idGen = 0
        self.actual = self.empty.copy()

    def copy(self):
        return board(*self.repr)

    def place(self, shape, pos):
        newId = self.idGen + 1
        success = gpuarray.to_gpu(np.array([1], 'uint8'))

        w, h = shape.shape
        x, y = pos

        self.cudaPlace(cuda.In(self.actual), cuda.In(shape), np.ubyte(newId), np.ubyte(x), np.ubyte(y), np.ubyte(w), np.ubyte(h), success, cuda.Out(self.res),
        cuda.InOut(success))

        if not success.get()[0]:
            return False

        self.actual = self.res.copy()
        self.idGen = newId
        return True

    def undo(self):
        self.actual -= np.array(((self.actual & 0b00111111) == self.idGen) * (0b01111111 & self.idGen | 0b01000000), 'uint8')
        self.idGen -= 1

    def draw(self):
        b = np.fliplr(np.rot90(self.actual, 3))
        s = self.black
        s = '+'
        for _ in range(len(b[0] - 1)):
            s+='-'
        s+='+\n'
        for i, rows in enumerate(b):
            s+=self.black + '|'
            for ii, v in enumerate(rows):
                s+=f"{self.colors[int(b[i,ii] & 0b00111111)]}{int(b[i,ii] & 0b00111111) if b[i,ii] != 0 else self.black + ' '}"
                # s+=f"{int(b[i,ii] & 0b00111111) if b[i,ii] != 0 else ' '}|"
            s+=self.black + '|\n'
            # s+='\n+'
            # for _ in range(len(rows)):
            #     s+='-+'
            # s+='\n'
        s+= '+'
        for _ in range(len(b[0] - 1)):
            s+='-'
        s+='+\n'
        print(s)


import sys
from PyQt5 import QtWidgets,QtGui,uic
from PyQt5.QtWidgets import QMessageBox

import requests
from bs4 import BeautifulSoup

class home(QtWidgets.QDialog):
    def __init__(self,parent=None):
        super(home,self).__init__(parent)
        uic.loadUi("../venv/Lib/site-packages/QtDesigner/homepage.ui", self)
        self.show()
        self.start.clicked.connect(self.getstarted)

    def getstarted(self):
        self.hide()
        window = DashBoard(self)

class DashBoard(QtWidgets.QDialog):
    def __init__(self,parent=None):
        super(DashBoard,self).__init__(parent)
        uic.loadUi("../venv/Lib/site-packages/QtDesigner/movies.ui", self)
        self.show()
        self.cominsoon.clicked.connect(self.futureMovie)
        self.intheater.clicked.connect(self.currentMovie)

    def futureMovie(self):
        window = ComingSoon(self)

    def currentMovie(self):
        window = InTheaters(self)

class ComingSoon(QtWidgets.QDialog):
    def __init__(self, parent=None):
        super(ComingSoon, self).__init__(parent)
        uic.loadUi("../venv/Lib/site-packages/QtDesigner/displaymovie.ui", self)
        self.show()
        website = requests.get('https://www.imdb.com/movies-coming-soon/2021-04/')
        soup = BeautifulSoup(website.content, 'html.parser')
        tags = soup.find_all('h4')
        count = 0
        for soups in tags:
            count += 1
            self.numbers.addItem(str(count))
            self.displaymovie.addItem(str(soups.string))

class InTheaters(QtWidgets.QDialog):
    def __init__(self, parent=None):
        super(InTheaters, self).__init__(parent)
        uic.loadUi("../venv/Lib/site-packages/QtDesigner/theaters.ui", self)
        self.show()

        website = requests.get('https://www.imdb.com/movies-in-theaters/?ref_=cs_inth')
        soup = BeautifulSoup(website.content, 'html.parser')
        tags = soup.find_all('h4')
        count = 0
        for soups in tags:
            count += 1
            self.numbers.addItem(str(count))
            self.displaymovie.addItem(str(soups.string))

app = QtWidgets.QApplication(sys.argv)
window = home()
app.setQuitOnLastWindowClosed(True)
app.exec_()


def subset(ind,stk,n,arr):
    if ind==n:
        print(stk)
        return
    #picking an element case
    stk.append(arr[ind])
    subset(ind+1,stk,n,arr)
    l=len(stk)
    stk.pop(l-1)
    #not picking element case
    subset(ind+1,stk,n,arr)
n=int(input())
arr=list(map(int,input().split()))
stk=[]
subset(0,stk,n,arr)


# preprocessing the data
from PIL import Image
from torchvision.transforms import functional as F
from io import BytesIO
import lmdb
from torchvision.datasets import ImageFolder
from tqdm import tqdm
# from math import frexp
import argparse

# img_path = "C:/Users/Bene/PycharmProjects/StyleGAN/dataset_corgis/corgis_25k/5h8XwMyMdr.jpeg"

# function to convert pixel number inputs to their corresponding powers of 2
# def power(x):
#     assert x % 2 == 0
#     if x <= 2:
#         return 1
#     return 1 + power(x / 2)

def fast_log2(x):
    assert isinstance(x, int)
    return x.bit_length() - 1

# def power3(x):
#     return frexp(x)[1] - 1
#
# # lets see which function is fastest
# def timereps(reps, func, args):
#     from time import time
#     start = time()
#     for i in range(0, reps):
#         func(args)
#     end = time()
#     return (end - start)
#
# t1 = timereps(10000000, power, 1024)
# t2 = timereps(10000000, power2, 1024)
# t3 = timereps(10000000, power3, 1024)
# t1, t2, t3
# # 20 seconds, 1.42 seconds and 2.18 seconds respectively
# # we should use the second function
# sizes = list(map(lambda x: 2 ** x, range(power(512) + 1)))
# sizes2 = list(map(lambda x: 2 ** x, range(power2(512) + 1)))
# sizes3 = list(map(lambda x: 2 ** x, range(power3(512) + 1)))

# we need multiple sizes so we write to memory first before converting to jpgs
# all at once
# https://stackoverflow.com/questions/646286/how-to-write-png-image-to-string-with-the-pil
def resize(img_path, max_size=128, min_size=8, quality=100):
    image = Image.open(img_path)
    # range(x) starts at 0 and ends at x-1
    sizes = list(map(lambda x: 2 ** x, range(fast_log2(max_size) + 1)))
    sizes = [x for x in sizes if x >= min_size]
    ret = []

    for size in sizes:
        img = F.resize(image, size)
        with BytesIO() as buffer:
            img.save(buffer, format="jpeg", quality=quality)
            contents = buffer.getvalue()
        ret.append(contents)

    return sizes, ret

# sz, bts = resize(img_path, max_size=128)
# test = BytesIO(bts[0])
# test = Image.open(test)
# test.show()

# number of datapoints: 25000

def to_lmdb(transaction, dataset, max_size=128, min_size=8):
    files = sorted(dataset.imgs, key=lambda x: x[0])
    files = [(i, file) for i, (file, label) in enumerate(files)]
    total = 0

    for i in tqdm(range(len(files))):
        sizes, res = resize(files[i][1], max_size, min_size)
        for size, img in zip(sizes, res):
            key = f'{size}-{str(i).zfill(6)}'.encode('utf-8')
            transaction.put(key, img)

        total = total + 1

    transaction.put('length'.encode('utf-8'), str(total).encode('utf-8'))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Converting .jpg folder to lmdb'
                                                 'database.')
    parser.add_argument('--dataset_path', type=str, help='path to imagefolder')
    parser.add_argument('--out_path', type=str, help='path for output')
    parser.add_argument('--max_size', type=int, default=128, help='maximum image')
    parser.add_argument('--min_size', type=int, default=8, help='minimum image size')
    parser.add_argument('--magic_number', type=int, default=17, help='constant required for setting the data base size on windows machines')

    args = parser.parse_args()

    dataset = ImageFolder(root=args.dataset_path)
    mapsize = (args.max_size ** 4) * args.magic_number

    with lmdb.open(path=args.out_path, map_size=mapsize, readahead=False) as env:
        with env.begin(write=True) as txn:
            to_lmdb(txn, dataset, args.max_size, args.min_size)

# dataset_path = "D:/Downloads/crop1024/"
# out_path = "C:/Users/Bene/PycharmProjects/StyleGAN/lmdb_corgis/"

#
# dataset = ImageFolder(root=dataset_path)
# maxsize = 128
# minsize = 8
# how to adjust the needed size of the database dynamically?
# trial and error: starting off with 25k images and a map_size of
# 128 ** 4 * 3 leads to 24958 elements being written.
# for all 136255 images we need approx:
# 128 ** 4 * (137000 / (24950/3)) = 128 ** 4 ** 17

# needs to be adjusted relative to the original data set size.
# mapsize = (maxsize ** 4) * 17 # black magic
# the final database will require about 4.5 GB of disk space


# -*- coding: utf-8 -*-
"""
Created on Thu Aug  4 21:39:55 2022

@author: Administrator
"""

import numpy as np

import pandas as pd

import xarray as xr

import glob

from wrf import getvar,latlon_coords

from netCDF4 import Dataset

from metpy.units import units
import metpy.constants as constants
import metpy.calc as mpcalc

'''
This script is used to calculate the three terms on the right side of the equation:
horizontal advection term,
vertical advection term,
non-adiabatic heating term
'''

####################################################################
#####     read data :wind-component ,longitude,latitude,time,level,pv,omega
####################################################################
# f_wind =  r"D:\PV\wrf-read\wrf-cal-pv\wind.nc"

f_wind =  r"/Users/xpji/wind.nc"
d_wind = xr.open_dataset(f_wind)
u = d_wind.u.data*units('m/s')                                      # units('m/s')
v = d_wind.v.data*units('m/s')                                       # units('m/s')

lat = d_wind.lat
lon = d_wind.lon
lev = d_wind.level
time = d_wind.time

f_Q  =   r"/Users/xpji/Q.nc"
d_Q  =  xr.open_dataset(f_Q)

f_pv =  r"/Users/xpji/pv.nc"
pv   = xr.open_dataset(f_pv).pv

f_omega = r"/Users/xpji/omega.nc"
omega  = xr.open_dataset(f_omega).omega
###################################################################################
######  calculate dx\dy\lon\lat
###################################################################################
def  cal_dxdy(file):
    ncfile = Dataset(file)
    P = getvar(ncfile, "pressure")
    lats, lons = latlon_coords(P)
    lon = lons[0]
    lon[lon<=0]=lon[lon<=0]+360
    lat = lats[:,0]
    dx, dy = mpcalc.lat_lon_grid_deltas(lon.data, lat.data)
    return lon,lat,dx,dy

###################################################################################
######    Get the size of latitude and longitude
###################################################################################

path  = r'/Users/xpji/out0-8/'
filelist = glob.glob(path+'wrf*')
filelist.sort()
lon,lat,dx,dy = cal_dxdy(filelist[-1])
nx = lon.shape[0]
ny = lat.shape[0]
####################################################################
#####     calculate  the horizontal and vertical advection of PV
####################################################################

pv_adv = np.full((time.shape[0],lev.shape[0],lat.shape[0],lon.shape[0]),np.nan)

for i in range(time.shape[0]):

    for j in range(lev.shape[0]):

        print(i,j)
        pv_adv[i,j,:,:] = mpcalc.advection(pv[i,j,:,:],u=u[i,j,:,:],v=v[i,j,:,:],dx=dx,dy=dy,x_dim=-1,y_dim=-2)

units.define('PVU = 1e-6 m^2 s^-1 K kg^-1 = pvu')
w = omega.data*units('Pa/s')
P = pv.data*units('PVU').to('m^2 s^-1 K kg^-1')
level = d_wind.level.data*units('hPa')
wdPdp = np.zeros(shape=P.shape,dtype="float32")*(units("m^2 s^-1 K kg^-1")/units("hPa")*units('Pa/s'))
lev_dif = np.gradient(level)
p_dif   = np.gradient(P,axis=1)
for  i in range(40):

    for j in range(27):

        print(i,j)

        wdPdp[i,j,:,:] = w[i,j,:,:]*p_dif[i,j,:,:]/lev_dif[j]
pvvdv = wdPdp.to_base_units()
pvvdv_m  = np.nanmean(pvvdv,axis=0)
pvvdv_a  =( pvvdv-pvvdv_m)
da_nc=xr.Dataset(
data_vars=dict(

    pvvdv=(['time','level','lat','lon'],-np.array(pvvdv.m)),
    pvvdv_ano=(['time','level','lat','lon'],-np.array(pvvdv_a.m)),
    ),
                coords = {
                    'time':time,
                    'level':level.m,
                    'lat':lat,
                    'lon':lon,

                    },
                attrs=dict(description="PV vertical advection(PVU = 1e-6 m^2 s^-1 K kg^-1)",
                            units= 'm^2 s^-2 K kg^-1',
                            equation='-<w*(dpv/dP)>-0907',
                            ),
                )
da_nc.to_netcdf(r'./data/pvvdv_.nc')
###########################################################################################
Q    = d_Q.Q_to.data*units('K/s')
####################################################################
#####     calculate gradient in 3 direction : x,y,p
####################################################################
# calculate  gradient
dv_dif = np.gradient(v,axis=1)
du_dif = np.gradient(u,axis=1)
dq_dif = np.gradient(Q,axis=1)

dv_dp = np.zeros(Q.shape,dtype='float32')*units('m/s')/units('hPa')
du_dp = np.zeros(Q.shape,dtype='float32')*units('m/s')/units('hPa')
dq_dp = np.zeros(Q.shape,dtype='float32')*units('K/s')/units('hPa')
ny=lat.shape[0]
fcc = mpcalc.coriolis_parameter(np.deg2rad(lat.data)).reshape((ny,1))

adv = np.zeros(Q.shape,dtype='float32')*units('K/s')/units('m')*units('m/s')/units('hPa')

dx = dx*units('m')
dy = dy*units('m')
q_dx = np.gradient(Q,axis=-1)
q_dy = np.gradient(Q,axis=-2)
for i in range(40):
    for j in range(27):
        print(i,j)
        dv_dp[i,j] = dv_dif[i,j]/lev_dif[j]
        du_dp[i,j] = du_dif[i,j]/lev_dif[j]
        dq_dp[i,j] = dq_dif[i,j]/lev_dif[j]

for i in range(40):
    for j in range(27):
        adv[i,j] = -1*dv_dp[i,j]*q_dx[i,j]/dx + du_dp[i,j]*q_dy[i,j]/dy

######### cal vertical components
vdv = np.zeros(Q.shape,dtype='float32')*units('K/s')/units('hPa')*units('m/s')/units('m')
v_dx = np.gradient(v, axis=-1)
u_dy = np.gradient(u, axis=-2)
vor_z = np.zeros(Q.shape,dtype='float32')*units('m/s')/units('m')

for i in range(40):
    for j in range(27):
        vor_z[i,j] = v_dx[i,j]/dx - u_dy[i,j]/dy

absvor_z = vor_z+fcc

for i in range(40):
    for j in range(27):
        vdv[i,j] = absvor_z[i,j]*dq_dp[i,j]

heat = (adv+vdv)*constants.g
dia = heat.to_base_units()

dia_m = np.nanmean(dia,axis=0)
dia_ano = dia-dia_m

####################################################################
#####   output as Netcdf
####################################################################
heat_nc=xr.Dataset(
    data_vars=dict(

        Q_tot=(['time','level','lat','lon'],-np.array(dia.m)),
        Q_ana=(['time','level','lat','lon'],-np.array(dia_ano.m))),

        coords = {
                        'time':tim,
                        'level':lev.m,
                        'lat':lat.data,
                        'lon':lon.data,

                        },
                    attrs=dict(description=" diabatic heating ",
                                units= 'm^2 s^-1 K kg^-1'),
                    )

heat_nc.to_netcdf(r'./data/diabatic_heat_20220907.nc')


from flask import Blueprint, jsonify, make_response, request, Flask
from app.models.goal import Goal
from app.models.task import Task
from app import db
from datetime import datetime
import os, requests, sys

task_list_bp = Blueprint("task_list", __name__, url_prefix="/tasks")
goals_bp = Blueprint("goals", __name__, url_prefix="/goals")

@task_list_bp.route("", methods=["GET"])
def get_task_lists():
    """Define an endpoint for get that returns all tasks, optionally presented by a query pram of descending\
        or ascending"""
    query_sort = request.args.get("sort")
    tasks_response = []

    if query_sort == "asc":
        tasks = Task.query.order_by(Task.title)
    elif query_sort == "desc":
        tasks = Task.query.order_by(Task.title.desc())
    else:
        tasks = Task.query.all()

    for task in tasks:
        tasks_response.append(task.to_dict())

    return make_response(jsonify(tasks_response), 200)

@task_list_bp.route("/<task_id>", methods=["GET"])
def get_one_task(task_id):
    """Define an endpoing of get id where we return a dict of just that task"""
    task = Task.query.get_or_404(task_id)
    return make_response(jsonify({"task": task.to_dict()}), 200)

@task_list_bp.route("", methods=["POST"])
def create_a_valid_task():
    """Define an endpoint of post, where we can create a new task with all required attributes"""

    request_body = request.get_json()

    if "title" not in request_body or "description" not in request_body or "completed_at" not in request_body:
        return make_response(jsonify({"details": "Invalid data"}), 400)

    request_body = sanitize_date(request_body)

    new_task = Task(title = request_body["title"],
                    description = request_body["description"],
                    completed_at = request_body["completed_at"])

    add_one_database(new_task)

    return make_response(jsonify({"task": new_task.to_dict()}), 201)

@task_list_bp.route("/<task_id>", methods=["PUT"])
def update_one_task(task_id):
    """Define an endpoint that updates the title or description of the called task"""
    task = Task.query.get_or_404(task_id)
    request_body = request.get_json()

    task.title = request_body["title"]
    task.description = request_body["description"]

    db.session.commit()
    return make_response(jsonify({"task": task.to_dict()}), 200)

@task_list_bp.route("/<task_id>", methods=["DELETE"])
def delete_task(task_id):
    """Defines an endpoint for deleting a task"""
    task = Task.query.get_or_404(task_id)
    delete_one_database(task)
    return make_response(jsonify({"details": f'Task {task_id} "{task.title}" successfully deleted'}), 200)

@task_list_bp.route("/<task_id>/<complete_command>", methods=["PATCH"])
def mark_task_complete(task_id, complete_command):
    """Defines an endpoint that will mark a task as complete or incomplete"""
    task = Task.query.get_or_404(task_id)

    if complete_command == "mark_complete":
        task.completed_at = datetime.utcnow()

        # Makes a call the slackbot API
        query = {"channel": "task-notifications", "text": f"CONGRATULATIONS YOU BEAUTIFUL HUMAN!!! You completed task: {task.title}!!!"}
        slack_bot_token = os.environ.get("SLACK_BOT_TOKEN")
        headers_dict = {"Authorization": f"Bearer {slack_bot_token}"}
        requests.post("https://slack.com/api/chat.postMessage", headers=headers_dict, params=query)

    elif complete_command == "mark_incomplete":
        task.completed_at = None

    return make_response(jsonify({"task": task.to_dict()}), 200)

@goals_bp.route("", methods=["GET"])
def get_goals():
    """Defines an endpoint that gets all goals, optionally presented by a query pram of descending\
        or ascending"""
    query_sort = request.args.get("sort")
    goals_response = []

    if query_sort == "asc":
        goals = Goal.query.order_by(Goal.title)
    elif query_sort == "desc":
        goals = Goal.query.order_by(Goal.title.desc())
    else:
        goals = Goal.query.all()

    for goal in goals:
        goals_response.append(goal.to_dict())

    return make_response(jsonify(goals_response), 200)

@goals_bp.route("/<goal_id>", methods=["GET"])
def get_one_goal(goal_id):
    """Defines an endoint that returns all goals"""
    goal = Goal.query.get_or_404(goal_id)
    return make_response(jsonify({"goal": goal.to_dict()}), 200)

@goals_bp.route("", methods=["POST"])
def create_a_valid_goal():
    """Defines an endpoint that creates a valid goal with all required attributes"""
    request_body = request.get_json()

    if "title" not in request_body:
        return make_response(jsonify({"details": "Invalid data"}), 400)

    new_goal = Goal(
        title = request_body["title"]
        )
    add_one_database(new_goal)
    return make_response(jsonify({"goal": new_goal.to_dict()}), 201)

@goals_bp.route("/<goal_id>", methods=["DELETE"])
def delete_a_goal(goal_id):
    """Defines an endpoint to delete a goal"""
    goal = Goal.query.get_or_404(goal_id)
    delete_one_database(goal)
    return make_response(jsonify({"details": f'Goal {goal_id} "{goal.title}" successfully deleted'}), 200)

@goals_bp.route("/<goal_id>", methods=["PUT"])
def update_a_goal(goal_id):
    """Defines an endpoint that updates a goal with new information"""
    goal = Goal.query.get_or_404(goal_id)
    request_body = request.get_json()
    goal.title = request_body["title"]
    db.session.commit
    return make_response(jsonify({"goal": goal.to_dict()}), 200)

@goals_bp.route("/<goal_id>/tasks", methods=["POST"])
def post_tasks_for_goal(goal_id):
    """Defines an endpoint that associates a task with a goal"""
    request_body = request.get_json()
    goal = Goal.query.get_or_404(goal_id)
    for each_task in request_body["task_ids"]:
        each_task = Task.query.get(each_task)
        each_task.fk_goal_id = goal.goal_id
    return make_response(jsonify({"id": goal.goal_id, "task_ids": request_body["task_ids"]}), 200)

@goals_bp.route("/<goal_id>/tasks", methods=["GET"])
def get_tasks_for_goal(goal_id):
    """Defines an endpoint that returns all tasks associated with a goal """
    goal = Goal.query.get_or_404(goal_id)
    return make_response(goal.to_dict_with_tasks(goal_id), 200)

def sanitize_date(request_body):
    """Verifies that complete_at in correct datetime format"""
    if request_body["completed_at"]:
        try:
            val = request_body["completed_at"]
            type(val) == datetime
        except:
            sys.exit("Completed_at should be in YYYY-MM-DD hh:mm:ss format or null")
    return request_body

def add_one_database(item):
    """Defines a helper function to add an item to database"""
    db.session.add(item)
    db.session.commit()

def delete_one_database(item):
    """Defines a helper function to delete an item from database"""
    db.session.delete(item)
    db.session.commit()