import pygame
import math
from common_types import *
import random

screen_size = (800, 800)
screen = pygame.display.set_mode(screen_size)
pygame.display.set_caption("drive #likeabosch")

warnings_offset = 0

def clear_screen():
    screen.fill((255, 255, 255))

def float_to_screen(p):
    # ivert
    p = (-p[0], -p[1])
    # convert a 2d point from -80.0 to 80.0 to screen coordinates
    y = int((p[0] + 60.0) / 120.0 * screen_size[1])
    x = int((p[1] + 60.0) / 120.0 * screen_size[0])
    return (x, y)

def abs_dist_to_screen_dist(dist): # meters to pixels
    # convert a distance from -80.0 to 80.0 to screen coordinates
    return int(dist / 120.0 * screen_size[0])

def dist_between_points(p0, p1):
    return math.sqrt((p0[0] - p1[0]) ** 2 + (p0[1] - p1[1]) ** 2)

# def float_to_screen_dist(dist):
#     p1 = (dist, 0)
#     p1 = float_to_screen(p1)
#     return p1[1]

# def dist_between_points(p0, p1):
#     return ((p0[0] - p1[0])**2 + (p0[1] - p1[1])**2)**0.5

def nearest_point_on_rect_rel(p, rect_w, rect_h, rect_center):
    # get the nearest point of the rectangle's corners to the point
    # and return the relative position of that point
    # get the points of the rectangle
    p0 = (rect_center[0] - rect_w / 2, rect_center[1] - rect_h / 2)
    p1 = (rect_center[0] + rect_w / 2, rect_center[1] - rect_h / 2)
    p2 = (rect_center[0] + rect_w / 2, rect_center[1] + rect_h / 2)
    p3 = (rect_center[0] - rect_w / 2, rect_center[1] + rect_h / 2)
    # get the distances to the points
    d0 = dist_between_points(p, p0)
    d1 = dist_between_points(p, p1)
    d2 = dist_between_points(p, p2)
    d3 = dist_between_points(p, p3)
    # get the nearest point
    min_d = min(d0, d1, d2, d3)
    if min_d == d0:
        return (p0[0] - rect_center[0], p0[1] - rect_center[1])
    elif min_d == d1:
        return (p1[0] - rect_center[0], p1[1] - rect_center[1])
    elif min_d == d2:
        return (p2[0] - rect_center[0], p2[1] - rect_center[1])
    elif min_d == d3:
        return (p3[0] - rect_center[0], p3[1] - rect_center[1])

imgs_path = "assets/images/"
img_yoke = pygame.transform.scale(pygame.image.load(imgs_path + "yoke.png").convert_alpha(), (160, 100))
imgs = {
    1: pygame.transform.rotate(pygame.transform.scale(pygame.image.load(imgs_path + "truck.png").convert_alpha(), (48, 15)), 90.0),
    2: pygame.transform.rotate(pygame.transform.scale(pygame.image.load(imgs_path + "car.png").convert_alpha(), (16, 34)), 180.0),
    3: pygame.transform.scale(pygame.image.load(imgs_path + "motorbike.png").convert_alpha(), (16, 8)),
    4: pygame.transform.scale(pygame.image.load(imgs_path + "bicycle.png").convert_alpha(), (16, 8)),
    6: pygame.transform.rotate(pygame.transform.scale(pygame.image.load(imgs_path + "car.png").convert_alpha(), (40, 13)), 180.0)
}

def draw_rect(p0, p1, p2, p3, color):
    # convert 2d points to screen coordinates
    p0 = float_to_screen(p0)
    p1 = float_to_screen(p1)
    p2 = float_to_screen(p2)
    p3 = float_to_screen(p3)
    # draw the rectangle
    pygame.draw.polygon(screen, color, (p0, p1, p2, p3))

def draw_circle(p, color, radius):
    # convert 2d points to screen coordinates
    p = float_to_screen(p)
    radius = abs_dist_to_screen_dist(radius)
    # draw the circle
    pygame.draw.circle(screen, color, p, radius)

def draw_line(p0, p1, color=(255,255,255), thickness=1):
    # convert 2d points to screen coordinates
    p0 = float_to_screen(p0)
    p1 = float_to_screen(p1)
    # draw the line
    pygame.draw.line(screen, color, p0, p1, thickness)

def draw_image(img, p, center=(0, 0), angle=0, abs_pos=False):
    # convert 2d points to screen coordinates
    if not abs_pos:
        p = float_to_screen(p)
    # center = float_to_screen(center)
    # draw the image
    if angle != 0:
        img = pygame.transform.rotate(img, angle)
    screen.blit(img, (p[0] - center[0], p[1] - center[1]))

def draw_text(text, p, color=(255,255,255)):
    # draw the text
    font = pygame.font.SysFont("Roboto", 20)
    text = font.render(text, True, color)
    screen.blit(text, p)

def draw_object(obj, ref):
    # if type is ObjType.noDetection skip
    if obj.obj_type == 0:
        return
    # if the position is 0,0 skip
    if obj.dx == 0 and obj.dy == 0:
        return
    # if obj.prob < 0.5 skip
    if obj.prob != None and obj.prob < 0.5:
        return
    # if type is ObjType.undefined draw a blue circle
    if obj.obj_type == -1 or obj.obj_type == ObjType.undefined:
        draw_circle((obj.dx, obj.dy), (0, 0, 255), 1.0)
        return
    # if type is ObjType.pedestrian draw a red circle
    if obj.obj_type == 5:
        draw_circle((obj.dx, obj.dy), (255, 0, 0), 1.0)
        return
    # draw the objects as images
    center = nearest_point_on_rect_rel((0, 0), imgs[obj.obj_type].get_size()[0], imgs[obj.obj_type].get_size()[1], float_to_screen((obj.dx, obj.dy))) # !!!!!! replace truck with obj.obj_type AND replace (0,0) with rendering position !!!!!!
    draw_image(imgs[obj.obj_type], (obj.dx, obj.dy), (8, 17)) # !!!!!! replace truck with obj.obj_type !!!!!!

def draw_warning(warning, position):
    font = pygame.font.SysFont("Roboto", 35)
    text = font.render(warn_text[warning], True, (255, 0, 100)).convert_alpha()
    # replace alpha layer with white
    text.fill((255, 255, 255, 255), None, pygame.BLEND_RGBA_MULT)
    # draw the text
    screen.blit(text, position)

def draw_warnings(delta_time):
    # get the keys from warn_timeouts with values > 0
    warnings = [k for k, v in warn_timeouts.items() if v > 0]
    # if there are no warnings, return
    if len(warnings) == 0:
        return
    # draw the warnings
    for i, warning in enumerate(warnings):
        draw_warning(warning, (20, i * 40 + 480))
    # decrease the timeouts
    for warning in warnings:
        warn_timeouts[warning] -= delta_time

class Background():
    # downwards scrolling background from a jpg image with a variable speed
    def __init__(self, img_path):
        # load image and resize to screen size
        self.img = pygame.transform.scale(pygame.image.load(img_path).convert(), screen_size)
        self.speed = 0
        self.y_rel = 0
        self.y = 0

    def update(self, speed, delta_time):
        self.speed = speed
        self.y_rel += self.speed * delta_time
        self.y = abs_dist_to_screen_dist(self.y_rel)
        if self.y > screen_size[1]:
            self.y_rel = 0
            self.y = 0

    def draw(self):
        screen.blit(self.img, (0, self.y))
        screen.blit(self.img, (0, self.y - screen_size[1]))

class LaneLines():
    # scrolling lane lines which turn with the car (based on the yaw rate)
    def __init__(self, color=(255,255,255)):
        self.color = color
        self.yaw_rate = 0

    def update(self, yaw_rate):
        self.yaw_rate = yaw_rate

    def draw(self):
        # draw two lane lines curved by the yaw rate
        pass


#!/usr/bin/python3
import operator
import pandas as pd
import csv
from csv import reader

csv.field_size_limit(1000000000)

# This gets the variants, defined by GATK for each position and puts them in a dictionary
variantdict={}
with open("variantalternatives.tsv") as varalt:
        parser = csv.reader(varalt, skipinitialspace=True, delimiter='\t')
        for line in parser:
                key1 = str(line[0])
                key2 = str(line[1])
                key = "{}:{}".format(key1, key2)
                variantdict[key]=line[2]
varalt.close

# The function of the magical genotyper is defined here
def magicalgenotyper():
        "This is the magical genotyper, which defines the genotypes"
        qualified = [k for k, v in ualtvaldict.items() if v >= 3]
        soaltkey = [ele[0] for ele in sorted(ualtvaldict.items(), key=operator.itemgetter(1), reverse=True)]
        soaltval = [el[1] for el in sorted(ualtvaldict.items(), key=operator.itemgetter(1), reverse=True)]
        key = [e[0] for e in sorted(altvaldict.items(), key=operator.itemgetter(1), reverse=True)]
        val = [l[1] for l in sorted(altvaldict.items(), key=operator.itemgetter(1), reverse=True)]
        listofalleles = sorted([k for k, v in ualtvaldict.items() if v > 0 ])
        if len(qualified) == 0:
                if val[0] == val[1]:
                        genotype = "./."
                else:
                        if altvaldict.get(key[0]) > 12 and altvaldict.get(key[1]) == 0:
                                genotype = "{}/{}".format(key[0],key[0])
                        elif altvaldict.get(key[0]) > 1:
                                genotype = "{}/.".format(key[0])
                        else:
                                genotype = "./."
        elif len(qualified) == 1:
                if ualtvaldict.get(soaltkey[0]) > 9:
                        if len(listofalleles) > 1 and soaltval[1] == 2 and val[0] < val[1]*2:
                                genotype = "{}/.".format(soaltkey[0])
                        else:
                                genotype = "{}/{}".format(soaltkey[0], soaltkey[0])
                elif ualtvaldict.get(soaltkey[0]) in list(range(7,10)) and len(listofalleles) == 1:
                        genotype = "{}/{}".format(soaltkey[0], soaltkey[0])
                elif ualtvaldict.get(soaltkey[0]) in list(range(7,10)) and len(listofalleles) == 2 and soaltval[1] == 1:
                        genotype = "{}/{}".format(soaltkey[0], soaltkey[0])
                else:
                        if altvaldict.get(key[0]) > 12 and altvaldict.get(key[1]) == 0:
                                genotype = "{}/{}".format(key[0],key[0])
                        else:
                                genotype = "{}/.".format(key[0])
        elif len(qualified) == 2:
                if altvaldict.get(key[0]) > altvaldict.get(key[1])*12:
                        genotype = "{}/{}".format(key[0], key[0])
                elif altvaldict.get(key[0]) > altvaldict.get(key[1])*9:
                        genotype = "{}/.".format(soaltkey[0])
                else:
                        genotype = "{}/{}".format(qualified[0], qualified[1])
        elif len(qualified) == 3:
                if altvaldict.get(key[0]) > altvaldict.get(key[1])*12:
                        genotype = "{}/{}".format(key[0], key[0])
                elif altvaldict.get(key[0]) > altvaldict.get(key[1])*9:
                        genotype = "{}/.".format(soaltkey[0])
                elif  val[1] > val[2]*2:
                        genotype = "{}/{}".format(key[0], key[1])
                else:
                        genotype = "{}/.".format(soaltkey[0])
        else:
                genotype = "{}/.".format(soaltkey[0])
        print(chrompos, genotype, ualtvaldict)
        writer.writerow([chrompos, genotype, ualtvaldict, altvaldict])
        return

# iterating through each pileupfile, annotating variants and doing the genotyping
f = open("samplelist.txt")

for sample in f:
        sample = sample.rstrip()
        outputfile = open("thing/{}.out.genotype".format(sample), "w+", newline='')
        writer = csv.writer(outputfile, delimiter="\t")
        with open("thing/{}.out.bam.out.pileup".format(sample) ) as csvfile:
                for row in reader(csvfile, delimiter="\t", quoting=csv.QUOTE_NONE):
                        bases = list(row[4].translate({ord(c): None for c in "^]$<>"}))
                        positions = (row[6].split(","))
                        posbase = str(list(zip(positions, bases)))
                        uniqposbase = str(set(zip(positions, bases)))
                        uniqalta = uniqposbase.count("a") + uniqposbase.count("A")
                        uniqaltc = uniqposbase.count("c") + uniqposbase.count("C")
                        uniqaltg = uniqposbase.count("g") + uniqposbase.count("G")
                        uniqaltt = uniqposbase.count("t") + uniqposbase.count("T")
                        uniqaltdel = uniqposbase.count("*")
                        uniqref = uniqposbase.count(".") + uniqposbase.count("','")
                        ubasevaldict = {"A":uniqalta, "C":uniqaltc, "G":uniqaltg, "T":uniqaltt, "*":uniqaltdel}
                        alta = posbase.count("a") + posbase.count("A")
                        altc = posbase.count("c") + posbase.count("C")
                        altg = posbase.count("g") + posbase.count("G")
                        altt = posbase.count("t") + posbase.count("T")
                        altdel = posbase.count("*")
                        ref = posbase.count(".") + posbase.count("','")
                        basevaldict = {"A":alta, "C":altc, "G":altg, "T":altt, "*":altdel}
                        chrompos = "{}:{}".format(str(row[0]), str(row[1]))
                        alts = (variantdict.get(chrompos)).split(",")
                        alt1 = alts[0]
                        uvalalt1 = ubasevaldict.get(alt1)
                        valalt1 = basevaldict.get(alt1)
                        if len(alts) > 1:
                                alt2 = alts[1]
                                uvalalt2 = ubasevaldict.get(alt2)
                                valalt2 = basevaldict.get(alt2)
                                if len(alts) > 2:
                                        alt3 = alts[2]
                                        uvalalt3 = ubasevaldict.get(alt3)
                                        valalt3 = basevaldict.get(alt3)
                                        if len(alts) > 3:
                                                alt4 = alts[3]
                                                uvalalt4 = ubasevaldict.get(alt4)
                                                valalt4 = basevaldict.get(alt4)
                                                if len(alts) > 4:
                                                        alt5 = alts[4]
                                                        uvalalt5 = ubasevaldict.get(alt5)
                                                        valalt5 = ubasevaldict.get(alt5)
                                                        ualtvaldict = {"0":uniqref, "1":uvalalt1, "2":uvalalt2, "3":uvalalt3, "4":uvalalt4, "5":uvalalt5}
                                                        altvaldict = {"0":ref, "1":valalt1, "2":valalt2, "3":valalt3, "4":valalt4, "5":valalt5}
                                                        magicalgenotyper()
                                                else:
                                                        ualtvaldict = {"0":uniqref, "1":uvalalt1, "2":uvalalt2, "3":uvalalt3, "4":uvalalt4}
                                                        altvaldict = {"0":ref, "1":valalt1, "2":valalt2, "3":valalt3, "4":valalt4}
                                                        magicalgenotyper()
                                        else:
                                                ualtvaldict = {"0":uniqref, "1":uvalalt1, "2":uvalalt2, "3":uvalalt3}
                                                altvaldict = {"0":ref, "1":valalt1, "2":valalt2, "3":valalt3}
                                                magicalgenotyper()
                                else:
                                        ualtvaldict = {"0":uniqref, "1":uvalalt1, "2":uvalalt2}
                                        altvaldict = {"0":ref, "1":valalt1, "2":valalt2}
                                        magicalgenotyper()
                        else:
                                ualtvaldict = {"0":uniqref, "1":uvalalt1}
                                altvaldict = {"0":ref, "1":valalt1}
                                magicalgenotyper()
        csvfile.close()
        outputfile.close()

loci = pd.read_csv("loci.txt", sep="\t")
loci.columns = ["chrompos"]
f = open("samplelist.txt")
for sample in f:
        sample = sample.rstrip()
        df = pd.read_csv("thing/{}.out.genotype".format(sample), header=None, sep="\t")
        df.columns = ["chrompos", "{}".format(sample), "valdict1", "valdict2"]
        df = df.drop("valdict1", 1)
        df = df.drop("valdict2", 1)
        loci = loci.merge(df, how="left", on="chrompos")
loci = loci.fillna("./.")
loci.to_csv("out_output.tsv", sep="\t")
print("End of the program. :)")


#!/usr/bin/env python3

import math
from collections import defaultdict
from typing import List
from typing import Set
try:
    from _tree import *
except ImportError:
    pass

class Solution:
    def TLE_numberOfSubarrays(self, nums: List[int], k: int) -> int:
        """
        i is 0 -> size
        j is i -> size
        so O(N^2), definitely timeout...
        """
        size = len(nums)

        res = 0
        for i in range(size):
            res += self.f(i, k, size, nums)

        return res

    def f(self, i, K, size, nums: List[int]) -> int:
        res = 0
        oddcnt = 0
        for j in range(i, size):
            if nums[j] % 2 == 1:
                oddcnt += 1
            if oddcnt == K:
                res += 1

        return res

    # it's not DP...
    def numberOfSubarrays(self, nums: List[int], k: int) -> int:
        size = len(nums)

        queue = []
        dp = [0] * size

        i = 0
        while len(queue) != k and i < size:
            if nums[i] % 2 == 1:
                queue.append(i)
            i += 1

        if len(queue) != k:
            return 0

        dp[i-1] = queue[0] - (-1)

        for j in range(i, size):
            if nums[j] % 2 == 1:
                prev = queue.pop(0)
                queue.append(j)
                dp[j] = queue[0] - prev
            else:
                dp[j] = dp[j-1]

        return sum(dp)

if __name__ == '__main__':
    # assert Solution().numberOfSubarrays([1,1,2,1,1], 3) == 2
    # assert Solution().numberOfSubarrays([2,4,6], 1) == 0
    assert Solution().numberOfSubarrays([2,2,2,1,2,2,1,2,2,2], 2) == 16


from flask import Flask, request, jsonify
import math
import numpy as np
import re
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
from collections import defaultdict, Counter, ChainMap
from inverted_index_gcp import *
import time
import pickle
from pathlib import Path
from google.cloud import storage
import gc

class MyFlaskApp(Flask):
    def run(self, host=None, port=None, debug=None, **options):
        super(MyFlaskApp, self).run(host=host, port=port, debug=debug, **options)

app = MyFlaskApp(__name__)
app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False

def search_tfidf_cos_sim(query, group, norm_cos_sim, is_search=False):
    ''' Returns up to a 100 search results for the query using TFIDF AND COSINE
        SIMILARITY OF THE TITLE / BODY / ANCHOR TEXT of Wikipedia articles.

    Arguments:
    --------
        query: str
            Query String

        group: str
            "title" / "body" / "anchor"

        norm_cos_sim: bool
            Whether to divide cosine similarity by group's InvertedIndex.DL (in addition to TfIdf normalization)

        is_search: bool
            Was this function called from search function? (other option - search_body)

    Returns:
    --------
        list of up to 100 search results, ordered from best to worst where each
        element is a tuple (wiki_id, title).
        If this function was called from search function - return cos_sim_scores as well ({wiki_id: cos_sim_score})
    '''

    tok_query = tokenize(query)
    len_tok_query = len(tok_query)
    filtered_query = list(filter(lambda x: x in globals()[f"idx_{group}"].posting_locs, tok_query))  # Filter query to match words that appear in the group's posting list
    query_tf = Counter(filtered_query).items()
    query_tf_idf =  {word: tf / len(tok_query) * math.log10(N / globals()[f"idx_{group}"].df[word]) for word, tf in query_tf}

    candidates_tf_idf = {}
    for word, tf in query_tf:
        for wiki_id, freq in globals()[f"idx_{group}"].read_posting_list(word):

            candidates_tf_idf[wiki_id] = dict(candidates_tf_idf.get(wiki_id, {}), **{word:tf * freq * math.log10(N / globals()[f"idx_{group}"].df[word])})

    cos_sim_scores = get_top_n({wiki_id: sum([query_tf_idf[word] * candidates_tf_idf[wiki_id][word] / len_tok_query for word in candidates_tf_idf[wiki_id]]) for wiki_id in candidates_tf_idf}, 100)

    res = [(key, id_title_pr_pv_dict[key][0]) for key, _ in cos_sim_scores]

    return res if not is_search else (res, dict(cos_sim_scores))  # if_search - return cos_sim_scores as well for scoring calculation

def search_boolean(query, group):
    ''' Returns ALL (not just top 100) search results that contain A QUERY WORD
        IN THE TITLE / ANCHOR  TEXT (group) of Wikipedia articles, ordered in descending order of the
        NUMBER OF QUERY WORDS that appear in title / anchor text linking to the page.

    Arguments:
    --------
        query: str
            Query String
        group: str
            "title" / "anchor"

    Returns:
    --------
        list of ALL (not just top 100) search results, ordered from best to
        worst where each element is a tuple (wiki_id, title).
    '''

    tok_query = tokenize(query)
    candidates_distinct = {}
    filtered_query = list(filter(lambda x: x in globals()[f"idx_{group}"].posting_locs, tok_query))  # Filter query to match words that appear in the group's posting list
    candidates_distinct = {wiki_id: candidates_distinct.get(wiki_id, 0) + 1 for word in filtered_query for (wiki_id, _) in globals()[f"idx_{group}"].read_posting_list(word)}

    candidates_distinct = sorted({wiki_id: (cnt, *id_title_pr_pv_dict[wiki_id][1::-1]) for wiki_id, cnt in  candidates_distinct.items()}.items(), key=lambda x: (x[1][0], x[1][1]), reverse=True)  # {wiki_id: (cnt, pagerank, title)}, sorted by cnt and then pagerank for tie-breaking

    res = [(wiki_id, cnt_pr_title[2]) for wiki_id, cnt_pr_title in candidates_distinct]

    return res

def normalize_result_set_pagerank_and_pageviews(res):
    ''' filters result set's PageRank and PageViews scores and normalizes them within the filtered result set.

     Arguments:
    --------
        res: list of (wiki_id: int, title: str)
            List of up to 100 search results, where each element is a tuple (wiki_id, title).

    Returns:
    --------
        2 dictionaries - {wiki_id: normalized PageRank score} and {wiki_id: normalized PageView score}
    '''

    wiki_id_pagerank_pageviews_dict = {wiki_id: id_title_pr_pv_dict[wiki_id][1:] for wiki_id, _ in res.items()}

    pagerank_array = np.array(list({wiki_id: pagerank_pageview[0] for wiki_id, pagerank_pageview in wiki_id_pagerank_pageviews_dict.items()}.items()), dtype=[('wiki_id', 'i4'), ('pagerank', 'f4')])
    pageview_array = np.array(list({wiki_id: pagerank_pageview[1] for wiki_id, pagerank_pageview in wiki_id_pagerank_pageviews_dict.items()}.items()), dtype=[('wiki_id', 'i4'), ('pageview', 'f4')])

    # Manual MinMax Scaling as a Vetctor Operation
    pagerank_array['pagerank'] = (pagerank_array['pagerank'] - pagerank_array['pagerank'].min()) / (pagerank_array['pagerank'].max() - pagerank_array['pagerank'].min())
    pageview_array['pageview'] = (pageview_array['pageview'] - pageview_array['pageview'].min()) / (pageview_array['pageview'].max() - pageview_array['pageview'].min())

    return dict(map(tuple, pagerank_array)), dict(map(tuple, pageview_array))

def calc_score_according_to_weight(pagerank_weight, pageviews_weight, title_res=[], title_weight=0, title_cos_sim={}, body_res=[], body_weight=0, body_cos_sim={}, anchor_res=[], anchor_weight=0, anchor_cos_sim={}):
    ''' Calculates results scores according to group (title, body, anchor, pagerank, pageviews) weights and returns a sorted list of 100 (wiki_id, title) pairs, descending by score.

     Arguments:
    --------
        title_res: list of (wiki_id: int, title: str)
            List of up to 100 search results, where each element is a tuple (wiki_id, title).

        title_weight: float
            A number between 0 to 1 which indicates the weight of the title results in the scoring formula.

        title_cos_sim: dict
            {wiki_id: cos_sim_score}

        pagerank_weight: float
            A number between 0 to 1 which indicates the weight of the pagerank results in the scoring formula.

        pageviews_weight: float
            A number between 0 to 1 which indicates the weight of the pageview results in the scoring formula.

        body_res: list of (wiki_id: int, title: str)
            List of up to 100 search results, where each element is a tuple (wiki_id, title).
            Default = [] for cases when we run this function for queries of len <= 2, we only want to use title_res, pagerank and pageview.

        body_weight: float
            A number between 0 to 1 which indicates the weight of the body results in the scoring formula.

        body_cos_sim: dict
            {wiki_id: cos_sim_score}

        anchor_res: list of (wiki_id: int, title: str)
            List of up to 100 search results, where each element is a tuple (wiki_id, title).
            Default = [] for cases when we run this function for queries of len <= 2, we only want to use title_res, pagerank and pageview.

        anchor_weight: float
            A number between 0 to 1 which indicates the weight of the anchor results in the scoring formula.

        anchor_cos_sim: dict
            {wiki_id: cos_sim_score}

    Returns:
    --------
        list of up to 100 search results, ordered from best to worst where each
        element is a tuple (wiki_id, title).
    '''

    title_res_dict = dict(title_res)
    body_res_dict = dict(body_res)
    anchor_res_dict = dict(anchor_res)
    res_dict = dict(ChainMap(anchor_res_dict, title_res_dict, body_res_dict))  # merged in descending order regarding size to minimize number of merge confilcts

    pagerank_dict, pageviews_dict = normalize_result_set_pagerank_and_pageviews(res_dict)

    # Calculates a final score according to groups' weights, whether it appeared in the group's result dictionary & cosine similarity score
    return [wiki_id_title_score[0] for wiki_id_title_score in sorted({(wiki_id, title): title_weight * bool(title_res_dict.get(wiki_id, 0)) * title_cos_sim.get(wiki_id, 0) + body_weight * bool(body_res_dict.get(wiki_id, 0)) * body_cos_sim.get(wiki_id, 0) + anchor_weight * bool(anchor_res_dict.get(wiki_id, 0)) * anchor_cos_sim.get(wiki_id, 0) + pagerank_weight * pagerank_dict.get(wiki_id, 0) + pageviews_weight * pageviews_dict.get(wiki_id, 0) for wiki_id, title in res_dict.items()}.items(), key=lambda x: x[1], reverse=True)[:100]]

def weight_fine_tuning(search_query, tw, prw, pvw, bw ,aw):
    ''' Groups' weights fine-tuning for debugging purposes.

     Arguments:
    --------
        search_query: str or None
            Query String if in debug mode, None if query was sent via http

        tw: float
            A number between 0 to 1 which indicates the weight of the title results in the scoring formula.

        prw: float
            A number between 0 to 1 which indicates the weight of the pagerank results in the scoring formula.

        pvw: float
            A number between 0 to 1 which indicates the weight of the pageview results in the scoring formula.

        bw: float
            A number between 0 to 1 which indicates the weight of the body results in the scoring formula.

        aw: float
            A number between 0 to 1 which indicates the weight of the anchor results in the scoring formula.

    Returns:
    --------
        title_weight, pagerank_weight, pageviews_weight, body_weight, anchor_weight (floats)
    '''
    title_weight = float(request.args.get('title_weight', tw)) if not search_query else tw
    pagerank_weight = float(request.args.get('pagerank_weight', prw)) if not search_query else prw
    pageviews_weight = float(request.args.get('pageviews_weight', pvw)) if not search_query else pvw
    body_weight = float(request.args.get('body_weight', bw)) if not search_query else bw
    anchor_weight = float(request.args.get('anchor_weight', aw)) if not search_query else aw

    return title_weight, pagerank_weight, pageviews_weight, body_weight, anchor_weight

@app.route("/search")
def search(search_query=None):
    ''' Returns up to a 100 of your best search results for the query. This is
        the place to put forward your best search engine, and you are free to
        implement the retrieval whoever you'd like within the bound of the
        project requirements (efficiency, quality, etc.). That means it is up to
        you to decide on whether to use stemming, remove stopwords, use
        PageRank, query expansion, etc.

        To issue a query navigate to a URL like:
         http://YOUR_SERVER_DOMAIN/search?query=hello+world
        where YOUR_SERVER_DOMAIN is something like XXXX-XX-XX-XX-XX.ngrok.io
        if you're using ngrok on Colab or your external IP on GCP.

    Arguments:
    --------
        search_query: str or None
            Query String for when the query is not provided by an API call (debugging)

    Returns:
    --------
        list of up to 100 search results, ordered from best to worst where each
        element is a tuple (wiki_id, title).
    '''

    if search_query:
        query = search_query
        norm_cos_sim = False

    else:
        query = request.args.get('query', '')
        norm_cos_sim = request.args.get('norm_cos_sim', "False") in ["True", "true", "T", "t"] # Cosine Similarity Normalization switch for debugging purposes
        title_slice = int(request.args.get('title_slice', 5))
        body_slice = int(request.args.get('body_slice', 10))

    if len(query) == 0:
      return [] if search_query else jsonify([])

    # BEGIN SOLUTION

    tok_query = tokenize(query)

    # Longer queries mainly consider body scores, assisted by pagerank and pageviews (anchor had a negative influence on the M.A.P@40 score)
    # Queries that involve questions require explanations rather than retrieving an entity
    # Queries that contain a year are most likely event oriented and thus should go to a title based search and not body.
    if not bool(re.search(r'\b\d{4}\b', query)) and ((len(tok_query) >= 3) or (query[-1] == '?' and len(query.split()) >= 4) or (len(query.split()) >= 5)) :
        # weight fine-tuning for debugging purposes
        title_weight, pagerank_weight, pageviews_weight, body_weight, anchor_weight = weight_fine_tuning(search_query, 0, 0.1, 0.1, 0.8 ,0)

        #title_res, title_cos_sim = search_tfidf_cos_sim(query=query, group="title", norm_cos_sim=norm_cos_sim, is_search=True)
        body_res, body_cos_sim = search_tfidf_cos_sim(query=query, group="body", norm_cos_sim=norm_cos_sim, is_search=True)
        #anchor_res, anchor_cos_sim = search_tfidf_cos_sim(query=query, group="anchor", norm_cos_sim=norm_cos_sim, is_search=True)

        # While optimizing the weights that the anchor calculates only lowers our M.A.P@40 score.
        res = calc_score_according_to_weight(pagerank_weight=pagerank_weight, pageviews_weight=pageviews_weight, body_res=body_res, body_weight=body_weight, body_cos_sim=body_cos_sim)[:body_slice]#title_res=title_res, title_weight=title_weight, title_cos_sim=title_cos_sim, anchor_res=anchor_res, anchor_weight=anchor_weight, anchor_cos_sim=anchor_cos_sim)

    else: # Wiki scores for short queries are calculated with a larger emphasis on the title, assisted by body, pagerank and pageviews (anchor had a negative influence on the M.A.P@40 score)
        # weight fine-tuning for debugging purposes
        title_weight, pagerank_weight, pageviews_weight, body_weight, anchor_weight = weight_fine_tuning(search_query, 0.8, 0.1, 0.1, 0 ,0)

        res, title_cos_sim = search_tfidf_cos_sim(query=query, group="title", norm_cos_sim=norm_cos_sim, is_search=True)

        res = calc_score_according_to_weight(title_res=res, title_weight=title_weight, title_cos_sim=title_cos_sim, pagerank_weight=pagerank_weight, pageviews_weight=pageviews_weight)[:title_slice]

    return res if search_query else jsonify(res)
    # END SOLUTION

@app.route("/search_body")
def search_body(search_query=None):
    ''' Returns up to a 100 search results for the query using TFIDF AND COSINE
        SIMILARITY OF THE BODY OF ARTICLES ONLY. DO NOT use stemming. DO USE the
        staff-provided tokenizer from Assignment 3 (GCP part) to do the
        tokenization and remove stopwords.

        To issue a query navigate to a URL like:
         http://YOUR_SERVER_DOMAIN/search_body?query=hello+world
        where YOUR_SERVER_DOMAIN is something like XXXX-XX-XX-XX-XX.ngrok.io
        if you're using ngrok on Colab or your external IP on GCP.

     Arguments:
    --------
        search_query: str or None
            Query String for when the query is not provided by an API call (debugging / encapsulated search requested by search)

    Returns:
    --------
        list of up to 100 search results, ordered from best to worst where each
        element is a tuple (wiki_id, title).
    '''

    if search_query:
        query = search_query
        norm_cos_sim = False
    else:
        query = request.args.get('query', '')
        norm_cos_sim = request.args.get('norm_cos_sim', "False") in ["True", "true", "T", "t"] # Cosine Similarity Normalization switch for debugging purposes

    # BEGIN SOLUTION
    res = search_tfidf_cos_sim(query=query, group="body", norm_cos_sim=norm_cos_sim)
    return res if search_query else jsonify(res)
    # END SOLUTION

@app.route("/search_title")
def search_title(search_query=None):
    ''' Returns ALL (not just top 100) search results that contain A QUERY WORD
        IN THE TITLE of articles, ordered in descending order of the NUMBER OF
        DISTINCT QUERY WORDS that appear in the title. DO NOT use stemming. DO
        USE the staff-provided tokenizer from Assignment 3 (GCP part) to do the
        tokenization and remove stopwords. For example, a document
        with a title that matches two distinct query words will be ranked before a
        document with a title that matches only one distinct query word,
        regardless of the number of times the term appeared in the title (or
        query).
        http://be08-35-196-42-90.ngrok.io/search_title?query=hello+world
        Test this by navigating to the a URL like:
         http://YOUR_SERVER_DOMAIN/search_title?query=hello+world
        where YOUR_SERVER_DOMAIN is something like XXXX-XX-XX-XX-XX.ngrok.io
        if you're using ngrok on Colab or your external IP on GCP.

     Arguments:
    --------
        search_query: str or None
            Query String for when the query is not provided by an API call (debugging / encapsulated search requested by search)

    Returns:
    --------
        list of ALL (not just top 100) search results, ordered from best to
        worst where each element is a tuple (wiki_id, title).

    '''

    if search_query:
        query = search_query
    else:
        query = request.args.get('query', '')

    if len(query) == 0:
      return [] if search_query else jsonify([])
    # BEGIN SOLUTION
    res = search_boolean(query=query, group="title")
    return res if search_query else jsonify(res)
    # END SOLUTION

@app.route("/search_anchor")
def search_anchor(search_query=None):
    ''' Returns ALL (not just top 100) search results that contain A QUERY WORD
        IN THE ANCHOR TEXT of articles, ordered in descending order of the
        NUMBER OF QUERY WORDS that appear in anchor text linking to the page.
        DO NOT use stemming. DO USE the staff-provided tokenizer from Assignment
        3 (GCP part) to do the tokenization and remove stopwords. For example,
        a document with a anchor text that matches two distinct query words will
        be ranked before a document with anchor text that matches only one
        distinct query word, regardless of the number of times the term appeared
        in the anchor text (or query).

        Test this by navigating to the a URL like:
         http://YOUR_SERVER_DOMAIN/search_anchor?query=hello+world
        where YOUR_SERVER_DOMAIN is something like XXXX-XX-XX-XX-XX.ngrok.io
        if you're using ngrok on Colab or your external IP on GCP.

     Arguments:
    --------
        search_query: str or None
            Query String for when the query is not provided by an API call (debugging / encapsulated search requested by search)

    Returns:
    --------
        list of ALL (not just top 100) search results, ordered from best to
        worst where each element is a tuple (wiki_id, title).
    '''

    if search_query:
        query = search_query
    else:
        query = request.args.get('query', '')

    if len(query) == 0:
      return [] if search_query else jsonify([])
    # BEGIN SOLUTION
    res = search_boolean(query=query, group="anchor")
    return res if search_query else jsonify(res)
    # END SOLUTION

@app.route("/get_pagerank", methods=['POST'])
def get_pagerank(wiki_ids_query=None):
    ''' Returns PageRank values for a list of provided wiki article IDs.

        Test this by issuing a POST request to a URL like:
          http://YOUR_SERVER_DOMAIN/get_pagerank
        with a json payload of the list of article ids. In python do:
          import requests
          requests.post('http://YOUR_SERVER_DOMAIN/get_pagerank', json=[1,5,8])
        As before YOUR_SERVER_DOMAIN is something like XXXX-XX-XX-XX-XX.ngrok.io
        if you're using ngrok on Colab or your external IP on GCP.

     Arguments:
    --------
        wiki_ids_query: list or None
            List of wiki IDs for debugging

    Returns:
    --------
        list of floats:
          list of PageRank scores that correrspond to the provided article IDs.
    '''
    if wiki_ids_query:
        wiki_ids = wiki_ids_query
    else:
        wiki_ids = request.get_json()

    if len(wiki_ids) == 0:
      return [] if wiki_ids_query else jsonify([])

    # BEGIN SOLUTION
    filtered_ids = list(filter(lambda x: x in id_title_pr_pv_dict, wiki_ids))

    res = [(id_title_pr_pv_dict[id][0], id_title_pr_pv_dict[id][1]) for id in filtered_ids]

    return res if wiki_ids_query else jsonify(res)
    # END SOLUTION

@app.route("/get_pageview", methods=['POST'])
def get_pageview(wiki_ids_query=None):
    ''' Returns the number of page views that each of the provide wiki articles
        had in August 2021.

        Test this by issuing a POST request to a URL like:
          http://YOUR_SERVER_DOMAIN/get_pageview
        with a json payload of the list of article ids. In python do:
          import requests
          requests.post('http://YOUR_SERVER_DOMAIN/get_pageview', json=[1,5,8])
        As before YOUR_SERVER_DOMAIN is something like XXXX-XX-XX-XX-XX.ngrok.io
        if you're using ngrok on Colab or your external IP on GCP.

    Arguments:
    --------
        wiki_ids_query: list or None
            List of wiki IDs for debugging

    Returns:
    --------
        list of ints:
          list of page view numbers from August 2021 that correrspond to the
          provided list article IDs.
    '''

    if wiki_ids_query:
        wiki_ids = wiki_ids_query
    else:
        wiki_ids = request.get_json()

    if len(wiki_ids) == 0:
      return [] if wiki_ids_query else jsonify([])

    # BEGIN SOLUTION

    filtered_ids = list(filter(lambda x: x in id_title_pr_pv_dict, wiki_ids))

    res = [(id_title_pr_pv_dict[id][0], id_title_pr_pv_dict[id][2]) for id in filtered_ids]

    return res if wiki_ids_query else jsonify(res)
    # END SOLUTION

english_stopwords = frozenset(stopwords.words('english'))
# We queried ChatGPT for Wikipedia-specific StopWords and added some of our own
corpus_stopwords = ["category", "references", "also", "external", "links",
                    "first", "see", "people", "one", "two",
                    "part", "thumb", "including", "second", "following",
                    "many", "however", "would", "became", "page", "article",
                    "reference", "source", "content",
                    "fact", "year", "date", "place", "wiki",
                    "edit", "version", "user", "talk", "discussion", "template",
                    "category", "project", "author", "writer",
                    "creator", "publisher", "editor", "publication", "edition",
                    "issue", "chapter"]
all_stopwords = english_stopwords.union(corpus_stopwords)
RE_WORD = re.compile(r"""[\#\@\w](['\-]?\w){2,24}""", re.UNICODE)

def tokenize(text):
    """
    This function aims in tokenize a text into a list of tokens. Moreover, it filter stopwords.

    Parameters:
    -----------
    text: string , represting the text to tokenize.

    Returns:
    -----------
    list of tokens (e.g., list of tokens).
    """
    list_of_tokens = [token.group() for token in RE_WORD.finditer(text.lower()) if token.group() not in all_stopwords]
    return list_of_tokens

def get_top_n(sim_dict, N=3):
    """
    Sort and return the highest N documents according to the cosine similarity score.
    Generate a dictionary of cosine similarity scores

    Parameters:
    -----------
    sim_dict: a dictionary of similarity score as follows:
                                                                key: document id (e.g., wiki_id)
                                                                value: similarity score. We keep up to 5 digits after the                                                                  decimal point. (e.g., round(score,5))

    N: Integer (how many documents to retrieve). By default N = 3

    Returns:
    -----------
    a ranked list of pairs (wiki_id, score) in the length of N.
    """

    return sorted([(wiki_id, round(score, 10)) for wiki_id, score in sim_dict.items()], key=lambda x: x[1], reverse=True)[:N]

def load_pickle_from_bucket(path):
    """
    Downloads a referenced .pkl file as bytes from the bucket and loads it as a variable. used in load_data(), at engine startup

    Parameters:
    -----------
    path: str
        Path of referenced .pkl file within the bucket

    Returns:
    -----------
    Object stored as a .pkl file within the bucket
    """

    client = storage.Client()
    bucket = client.bucket('318419512_318510252')
    file = pickle.loads(bucket.get_blob(path).download_as_bytes())
    client.close()
    gc.collect()

    return file

@app.route("/load_data")
def load_data():
    """
    Loads all necessary variables from the GCP Bucket as Global Variables. Runs before the server goes up.

    Parameters:
    -----------
    None

    Returns:
    -----------
    None
    """

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'  # Prevents kernel death when downloading .pkl files from bucket
    bucket_name = '318419512_318510252'
    id_title_pr_pv_dict_path = 'gs://318419512_318510252/id_title_pr_pv_dict_path.pkl'
    # load the three indexes
    global idx_body
    global idx_title
    global idx_anchor
    idx_body = load_pickle_from_bucket("postings_gcp/body_index.pkl")
    idx_title = load_pickle_from_bucket("postings_gcp/title_index.pkl")
    idx_anchor = load_pickle_from_bucket("postings_gcp/anchor_index.pkl")

    # declare the variables we will need in other functions (search) as global
    global id_pagerank_title_pageviews_df
    global id_title_pr_pv_dict
    global N  # Number of documents in the corpus

    N = len(idx_body.DL)

    id_title_pr_pv_dict = load_pickle_from_bucket("id_title_pr_pv_dict_cast.pkl")

    # as the .pkl files are loaded with a lazy manner, we explicitly initiate their mapping to dict in load_data() by a dummy call
    idx_body.DL[309]
    idx_title.DL[309]
    idx_anchor.DL[309]
    id_title_pr_pv_dict[309]

if __name__ == '__main__':
    # run the Flask RESTful API, make the server publicly available (host='0.0.0.0') on port 8080
    load_data()
    app.run(host='0.0.0.0', port=8080, debug=True)


from django.core.management.commands.migrate import Command as RCommand
from ...db_procedures.schema import TrunkProcedures2 as TrunkProcedures
from ...db_procedures.schema import Schema
from django.db.utils import ProgrammingError
import re

ep_schema = Schema('eprectest')
tr_proc = TrunkProcedures('eprectest', 'Trunk')

class Command(RCommand):
    def handle(self, *args, **options):
        super().handle(*args, **options)

        try:
            branches = [br.model_name for br in ep_schema.trunk_branches]
        except ProgrammingError as pe:
            self.stderr.write(re.sub(r'[^\S\n]+', ' ',f"\
                Schema('app_label').branches getter returned ProgrammingError:\n\
                {str(pe)}\n\
                Probably meaning that Trunk model was not migrated.\n\
                Maybe you should run `makemigrations` first!\n"
                ))
        else:
            if not ep_schema.last_schema_filename and len(branches) > 0:
                self.stdout.write(re.sub(r'[^\S\n]+', ' ', f"\
                    It seems that Trunk contains Branches data and\
                    no schema.json is present!\n\
                    Branches from Trunk: {branches}\n\
                    If you choose refresh Trunk with no schema.json,\
                    Trunk will be rewriten with data\
                    only from models statically typed in django.\n\
                    Please choose:\n\n\
                    1  to refresh Trunk anyway\n\
                    2  to dump schema.json from Trunk and abort refreshing Trunk\n\
                    "))
                chosen = int(input())
                if chosen == 1:
                    # TrunkProcedures('eprectest', 'Trunk').refresh()
                    tr_proc.refresh()
                elif chosen == 2:
                    ep_schema.dumpschema()
                    self.stdout.write(ep_schema.dumpschema.message)
                    self.stdout.write(re.sub(r'[^\S\n]+', ' ', f"\
                        To apply last schema.json run `makemigrations`!\n\
                        "))
            else:
                # this code is run if schema.json and Trunk table exists.
                try:
                    # TrunkProcedures('eprectest', 'Trunk').refresh()
                    tr_proc.refresh()
                    self.stdout.write(tr_proc.refresh.message)
                    if ep_schema.schema_graph_changes:
                        ep_schema.update_bunches_db()
                        self.stdout.write(ep_schema.update_bunches_db.message)
                        ep_schema.dumpschema()
                        self.stdout.write(ep_schema.dumpschema.message)
                except ProgrammingError as pe:
                    self.stderr.write(re.sub(r'[^\S\n]+', ' ', f"\
                        TrunkProcedures('eprectest', 'Trunk').refresh() \
                        raised ProgrammingError:\n\
                        {str(pe)}\n\
                        Maybe you should run `makemigrations` first!\n\
                        "))


from typing import Optional

# V této ukázce budeme pracovat se «stromy». Strom je datová
# struktura, která se podobá zřetězenému seznamu, s jedním zásadním
# rozdílem: uzly nemají následníka jednoho, ale několik. Podle toho,
# kolik, dělíme stromy na binární (2 následníci), ternární (3
# následníci), atd. Lze také uvažovat stromy s proměnným počtem
# následníků (takovým se většinou říká n-ární). Počátečnímu uzlu
# (tomu, který nemá ve stromě žádné předchůdce) často říkáme kořen.

# Stromy sdílí se zřetězenými seznamy krom podobné struktury i jednu
# velmi důležitou vlastnost: jsou to «rekurzivní datové struktury».
# Co to znamená? U seznamu to, že následník uzlu seznamu tvoří opět
# seznam (navíc striktně menší seznam). A u stromu zase platí, že
# každý následník je podstrom (striktně menší strom).

# Tato struktura velmi dobře koresponduje s naší představou
# o rekurzi: problém rozdělíme na podproblémy (pro každý podstrom
# vznikne jeden) a dílčí výsledky nějak zkombinujeme na výsledek
# celkový. Elementární (bázové) podproblémy pak tvoří stromy
# o jediném uzlu (takové, které nemají žádné podstromy, známé též
# jako «listy»), případně stromy prázdné (je-li to výhodné).

# Strom budeme reprezentovat analogicky k uzlu zřetězeného seznamu.
# Prázdný strom budeme reprezentovat hodnotou ‹None›.

class Tree:
    def __init__(self, value: int, left: Optional['Tree'] = None,
                 right: Optional['Tree'] = None):
        self.value = value
        self.left = left
        self.right = right

# Jako první příklad na práci se stromy si naprogramujeme test na
# přítomnost hodnoty ve stromě. Vstupem je (potenciálně prázdný)
# strom a hledaná hodnota.

def search(tree: Optional[Tree], value: int) -> bool:

    # Aplikujeme nyní již snad dobře známý postup: nejprve vyřešíme
    # bázové (jednoduché) případy: je-li strom prázdný, hledaná
    # hodnota se v něm jistě nenachází (vracíme ‹False›).

    if tree is None:
        return False

    # Naopak, je-li hledaná hodnota uložena v aktuálním uzlu, můžeme
    # rovnou vrátit ‹True›.

    if value == tree.value:
        return True

    # Zbývají případy, které neumíme řešit přímo: víme ale, že je-li
    # hodnota ve stromě přítomna, musí to být v levém nebo v pravém
    # podstromě. Protože podstromy jsou menší (jednodušší) než celý
    # strom, jedná se o podproblémy, které můžeme řešit rekurzí.
    # Aplikujeme tedy predikát ‹search› na oba podstromy: hodnota je
    # ve stromě přítomna, je-li přítomna alespoň v jednom z jeho
    # podstromů.

    return search(tree.left, value) or search(tree.right, value)

# Nezbývá, než predikát ‹search› otestovat na několika jednoduchých
# vstupech.

def main() -> None:  # demo
    t1 = Tree(7, Tree(2), Tree(1, Tree(5), Tree(6)))
    assert search(t1, 7)
    assert search(t1, 2)
    assert search(t1, 1)
    assert search(t1, 5)
    assert search(t1, 6)
    assert not search(t1, 4)
    t2 = Tree(8, t1, Tree(10))
    assert not search(t2, 4)
    assert search(t2, 8)
    assert search(t2, 10)
    assert search(t2, 1)
    assert search(t2, 5)

if __name__ == '__main__':
    main()


import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import h5py
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import time
import pickle
import os
from imblearn import under_sampling, over_sampling
from pairing import depair
from colossus.cosmology import cosmology
from colossus.lss import peaks
##################################################################################################################
# General params
snapshot_list = [190,184] # SHOULD BE DESCENDING
p_snap = snapshot_list[0]
times_r200m = 6
curr_sparta_file = "sparta_cbol_l0063_n0256"
path_to_hdf5_file = "/home/zvladimi/MLOIS/SPARTA_data/" + curr_sparta_file + ".hdf5"
path_dict = {
    "curr_sparta_file": curr_sparta_file,
    "path_to_MLOIS": "/home/zvladimi/MLOIS/",
    "path_to_snaps": "/home/zvladimi/MLOIS/particle_data/",
    "path_to_hdf5_file": path_to_hdf5_file,
    "path_to_pickle": "/home/zvladimi/MLOIS/pickle_data/",
    "path_to_datasets": "/home/zvladimi/MLOIS/calculated_info/",
    "path_to_model_plots": "/home/zvladimi/MLOIS/xgboost_datasets_plots/"
}
snap_format = "{:04d}" # how are the snapshots formatted with 0s
##################################################################################################################
# import pygadgetreader and sparta
import sys
sys.path.insert(0, path_dict["path_to_MLOIS"] + "pygadgetreader")
sys.path.insert(0, path_dict["path_to_MLOIS"] + "sparta/analysis")
from pygadgetreader import readsnap, readheader
from sparta import sparta
from data_and_loading_functions import build_ml_dataset, check_pickle_exist_gadget, choose_halo_split, create_directory, load_or_pickle_SPARTA_data
from visualization_functions import *
#from train_xgboost_models import model_creator
##################################################################################################################
# set what the paths should be for saving and getting the data
if len(snapshot_list) > 1:
    specific_save = curr_sparta_file + "_" + str(snapshot_list[0]) + "to" + str(snapshot_list[-1]) + "_" + str(times_r200m) + "r200msearch/"
else:
    specific_save = curr_sparta_file + "_" + str(snapshot_list[0]) + "_" + str(times_r200m) + "r200msearch/"

path_to_datasets = path_dict["path_to_model_plots"] + specific_save + "datasets/"
data_location = path_dict["path_to_datasets"] + specific_save
save_location = path_dict["path_to_model_plots"] + specific_save

create_directory(save_location)

p_snapshot_path = path_dict["path_to_snaps"] + "snapdir_" + snap_format.format(p_snap) + "/snapshot_" + snap_format.format(p_snap)

p_red_shift = readheader(p_snapshot_path, 'redshift')
p_scale_factor = 1/(1+p_red_shift)
halos_pos, halos_r200m, halos_id, halos_status, halos_last_snap, ptl_mass = load_or_pickle_SPARTA_data(curr_sparta_file, p_scale_factor, p_snap, path_dict)
cosmol = cosmology.setCosmology("bolshoi")

with open(path_to_datasets + "test_dataset_all_keys.pickle", "rb") as pickle_file:
    test_all_keys = pickle.load(pickle_file)
with open(path_to_datasets + "test_dataset_" + curr_sparta_file + "_" + str(snapshot_list[0]) + "_" + str(snapshot_list[-1]) + ".pickle", "rb") as pickle_file:
    test_dataset = pickle.load(pickle_file)

num_params_per_snap = (len(test_all_keys) - 2) / len(snapshot_list)

num_bins = 30

all_file_names = []
all_models = []
for filename in os.listdir(save_location + "/models/"):
    all_file_names.append(filename)
all_file_names.sort()

for filename in all_file_names:
    with open(save_location + "/models/" + filename, "rb") as pickle_file:
        model = pickle.load(pickle_file)
    model.load_models()
    all_models.append(model)

# Determine where the scaled radii, rad vel, and tang vel are located within the dtaset
for i,key in enumerate(test_all_keys[2:]):
    if key == "Scaled_radii_" + str(p_snap):
        scaled_radii_loc = i
    elif key == "Radial_vel_" + str(p_snap):
        rad_vel_loc = i
    elif key == "Tangential_vel_" + str(p_snap):
        tang_vel_loc = i

with open(data_location + "test_indices.pickle", "rb") as pickle_file:
    test_indices = pickle.load(pickle_file)
use_halo_ids = halos_id[test_indices]
sparta_output = sparta.load(filename=path_dict["path_to_hdf5_file"], halo_ids=use_halo_ids)
dens_prf_all = sparta_output['anl_prf']['M_all'][:,p_snap,:]
dens_prf_1halo = sparta_output['anl_prf']['M_1halo'][:,p_snap,:]

# test indices are the indices of the match halo idxs used (see find_particle_properties_ML.py to see how test_indices are created)
num_test_halos = test_indices.shape[0]

# for every halo idx and pid paired in the test dataset get the halo idxs
test_halo_idxs = np.zeros(test_dataset.shape[0])
use_ptl_ids = np.zeros(test_dataset.shape[0])

for i,id in enumerate(test_dataset[:,0]):
    depaired = depair(id)
    use_ptl_ids[i] = depaired[0]
    test_halo_idxs[i] = depaired[1]

halo_masses = np.zeros(num_test_halos)

start = 0
all_accuracy = []
# get the mases for each halo
for j in range(num_test_halos):
    curr_halo_idx = test_indices[j]
    curr_test_halo = test_dataset[np.where(test_halo_idxs == curr_halo_idx)]
    halo_masses[j] = curr_test_halo.shape[0] * ptl_mass

p_red_shift = readheader(p_snapshot_path, 'redshift')
peak_heights = peaks.peakHeight(halo_masses, p_red_shift)

start_nu = np.min(peak_heights)
nu_step = np.max(peak_heights)
num_iter = 1

for i in range(num_iter):
    end_nu = start_nu + nu_step

    idx_within_nu = np.where((peak_heights >= start_nu) & (peak_heights < end_nu))[0]
    curr_test_halo_idxs = test_indices[idx_within_nu]
    print(start_nu, "to", end_nu, ":", idx_within_nu.shape, "halos")

    if curr_test_halo_idxs.shape[0] != 0:
        density_prf_all_within = np.zeros(dens_prf_all.shape[1])
        density_prf_1halo_within = np.zeros(dens_prf_1halo.shape[1])
        for j, idx in enumerate(curr_test_halo_idxs):
            density_prf_all_within = density_prf_all_within + dens_prf_all[j]
            density_prf_1halo_within = density_prf_1halo_within + dens_prf_1halo[j]
            use_ptl_idxs = np.where(test_halo_idxs == idx)
            if j == 0:
                test_halos_within = test_dataset[use_ptl_idxs]
                continue
            else:
                curr_test_halo = test_dataset[use_ptl_idxs]
                test_halos_within = np.row_stack((test_halos_within, curr_test_halo))

        test_predict = np.ones(test_halos_within.shape[0]) * -1

        for i in range(len(snapshot_list)):
            curr_dataset = test_halos_within[:,:int(2 + (num_params_per_snap * (i+1)))]

            if i == (len(snapshot_list)-1):
                use_ptls = np.where(test_halos_within[:,-1]!= 0)[0]
            else:
                use_ptls = np.where(test_halos_within[:,int(2 + (num_params_per_snap * (i+1)))] == 0)[0]

            curr_dataset = curr_dataset[use_ptls]
            all_models[i].predict(curr_dataset)
            test_predict[use_ptls] = all_models[i].get_predicts()

        actual_labels = test_halos_within[:,1]

        classification = classification_report(actual_labels, test_predict, output_dict=True)

        all_accuracy.append(classification["accuracy"])
        bins = sparta_output["config"]['anl_prf']["r_bins_lin"]
        bins = np.insert(bins, 0, 0)
        compare_density_prf(radii=test_halos_within[:,2+scaled_radii_loc], actual_prf_all=density_prf_all_within, actual_prf_1halo=density_prf_1halo_within, mass=ptl_mass, orbit_assn=test_predict, prf_bins=bins, title = str(np.floor(start_nu)) + "-" + str(np.ceil(end_nu)), show_graph = False, save_graph = True, save_location = save_location)
        plot_radius_rad_vel_tang_vel_graphs(test_predict, test_halos_within[:,2+scaled_radii_loc], test_halos_within[:,2+rad_vel_loc], test_halos_within[:,2+tang_vel_loc], actual_labels, "ML Predictions", num_bins, np.floor(start_nu), np.ceil(end_nu), show = False, save = True, save_location=save_location)
        graph_acc_by_bin(test_predict, actual_labels, test_halos_within[:,2+scaled_radii_loc], num_bins, np.floor(start_nu), np.ceil(end_nu), plot = False, save = True, save_location = save_location)

    start_nu = end_nu

print(all_accuracy)
fig, ax = plt.subplots(1,1)
ax.set_title("Number of halos at each accuracy level")
ax.set_xlabel("Accuracy")
ax.set_ylabel("Num halos")
ax.hist(all_accuracy)
fig.savefig(save_location + "all_test_halo_acc.png")


import glob
import os
import pickle
import numpy as np
import pandas as pd
from ripple_heterogeneity.assembly import assembly_reactivation, find_sig_assembly
from ripple_heterogeneity.utils import add_new_deep_sup
from ripple_heterogeneity.utils import functions, loading, add_new_deep_sup
import warnings

warnings.filterwarnings("ignore")

def get_assembly_df(results, patterns, is_member):
    """
    Returns a dataframe of the assembly properties.
    """
    assembly_df = pd.DataFrame()

    # patterns = results.get("react").patterns

    assembly_df["patterns"] = patterns.ravel()
    assembly_df["is_member"] = is_member.ravel()
    assembly_df["assembly_n"] = (
        (np.ones_like(patterns).T * np.arange(patterns.shape[0])).T.astype(int).ravel()
    )
    assembly_df["UID"] = np.tile(
        results.get("react").cell_metrics.UID.values, patterns.shape[0]
    )
    assembly_df["putativeCellType"] = np.tile(
        results.get("react").cell_metrics.putativeCellType.values, patterns.shape[0]
    )
    assembly_df["brainRegion"] = np.tile(
        results.get("react").cell_metrics.brainRegion.values, patterns.shape[0]
    )
    assembly_df["deepSuperficial"] = np.tile(
        results.get("react").cell_metrics.deepSuperficial.values, patterns.shape[0]
    )
    assembly_df["deepSuperficialDistance"] = np.tile(
        results.get("react").cell_metrics.deepSuperficialDistance.values,
        patterns.shape[0],
    )
    assembly_df = add_new_deep_sup.deep_sup_from_deepSuperficialDistance(assembly_df)
    return assembly_df

def get_strength_matrix(analog_signal):
    """
    Returns a matrix of the strength of the signal at each timepoint.
    input:
        analog_signal: a nelpy AnalogSignalArray of the assembly strengths, must have segments
    output:
        strength_matrix: a numpy array of the strength of the signal at each timepoint
    """
    strength = np.zeros([analog_signal.n_signals, analog_signal.n_intervals])
    time = strength.copy()
    for rip_i, rip in enumerate(analog_signal):
        strength[:, rip_i] = rip.max()
        time[:, rip_i] = rip.abscissa_vals[np.argmax(rip.data, axis=1)]
    return strength, time

def get_psths(
    st,
    assembly_strengths,
    assembly_df,
    labels=["Deep", "Superficial", "mixed"],
    assembly_strength_thres=5,
    bin_width=0.005,
    n_bins=200,
):
    """
    Returns a dict of the PSTH of st relative to when the assembly is active.
    input:
        st: a nelpy SpikeTrainArray of the st of the assembly
        assembly_strengths: a nelpy analogsignal array of the strength of the signal at each ripple
        assembly_df: a dataframe of the assembly properties
        labels: a list of the labels of the assemblies
        assembly_strength_thres: the threshold for the strength of the assembly
        bin_width: the width of the bins in seconds
        n_bins: the number of bins
    output:
        psths: a dict of the PSTH of st relative to when the assembly is active
    """
    psths = {}
    for label in labels:
        psth = pd.DataFrame()
        # get the max value and time of max value for each assembly over ripples
        strength, time = get_strength_matrix(assembly_strengths)

        # keep the assemblies that are specific to the label
        assembly_n = assembly_df[
            (assembly_df.assembly_label == label) & (assembly_df.assembly_keep == True)
        ].assembly_n.unique()

        strength = strength[assembly_n, :]
        time = time[assembly_n, :]

        # iterate over the assemblies and calculate the psth relative to the time of max strength
        for i in range(strength.shape[0]):
            psth = pd.concat(
                [
                    psth,
                    functions.compute_psth(
                        st.data,
                        time[i, (strength[i, :] > assembly_strength_thres)],
                        bin_width=bin_width,
                        n_bins=n_bins,
                    ),
                ],
                ignore_index=True,
                axis=1,
            )
        # add the psth to the dict
        psths[label] = psth
    return psths

def run(
    basepath,
    regions="CA1",
    target_regions=["PFC", "EC1|EC2|EC3|EC4|EC5|MEC"],
    putativeCellType="Pyr",
    weight_dt=0.02,  # dt in seconds for binning st to get weights for each assembly
    verbose=False,  # print out progress
    rip_exp_start=0.05,  # ripple expansion start, in seconds, how much to expand ripples
    rip_exp_stop=0.05,  # ripple expansion stop, in seconds, how much to expand ripples
    min_cells=5,  # minimum number of cells in analysis (n deep, n superficial, n cortex)
):
    """
    Gets the pre and post assembly strengths
    """
    # initialize session
    m1 = assembly_reactivation.AssemblyReact(
        basepath,
        brainRegion=regions,
        putativeCellType=putativeCellType,
        weight_dt=weight_dt,
    )

    # load data
    m1.load_data()

    # check if enough data is available
    m1.cell_metrics = add_new_deep_sup.deep_sup_from_deepSuperficialDistance(
        m1.cell_metrics
    )

    if not ((m1.cell_metrics.deepSuperficial == "Deep").sum() >= min_cells) | (
        (m1.cell_metrics.deepSuperficial == "Superficial").sum() >= min_cells
    ):
        print("Not enough cells")
        return None

    # if not ((m1.cell_metrics.deepSuperficial == "Deep").sum() >= min_cells) & (
    #     (m1.cell_metrics.deepSuperficial == "Superficial").sum() >= min_cells
    # ):
    #     print("Not enough cells")
    #     return None

    # extend ripples to include some extra time
    m1.ripples = m1.ripples.expand(rip_exp_start, direction="start")
    m1.ripples = m1.ripples.expand(rip_exp_stop, direction="stop")

    # check if no cells were found
    if m1.cell_metrics.shape[0] == 0:
        return None

    # restrict to pre/task/post epochs
    try:
        m1.restrict_epochs_to_pre_task_post()
    except:
        print("No pre/task/post epochs found")
        return None
    # get weights for task outside ripples
    # % (TODO: use more robust method to locate epochs than index)
    if verbose:
        print("Getting weights...")
    m1.get_weights(m1.epochs[1][~m1.ripples])

    if len(m1.patterns) == 0:
        print("No patterns found")
        return None

    # get assembly activity
    if verbose:
        print("Getting assembly activity...")
    assembly_act_pre = m1.get_assembly_act(epoch=m1.ripples[m1.epochs[0]])
    assembly_act_task = m1.get_assembly_act(epoch=m1.ripples[m1.epochs[1]])
    assembly_act_post = m1.get_assembly_act(epoch=m1.ripples[m1.epochs[2]])
    results = {
        "assembly_act_pre": assembly_act_pre,
        "assembly_act_task": assembly_act_task,
        "assembly_act_post": assembly_act_post,
        "react": m1,
    }

    # get activity for target regions
    st, cell_metrics = loading.load_spikes(
        basepath, brainRegion=target_regions, putativeCellType=putativeCellType
    )

    target_regions_1,target_regions_2 = target_regions
    if not cell_metrics.brainRegion.str.contains(target_regions_1).any() | cell_metrics.brainRegion.str.contains(target_regions_2).any():
        print("Not enough cells")
        return None

    _, _, keep_assembly, is_member = find_sig_assembly.main(
        results.get("react").patterns
    )
    assembly_df = get_assembly_df(results, results.get("react").patterns, is_member)

    # add keep assembly labels to the assembly df
    keep_assembly_df = pd.DataFrame(
        index=np.arange(results.get("react").patterns.shape[0]), data=keep_assembly
    )
    keep_assembly_dict = keep_assembly_df.to_dict()
    assembly_df["assembly_keep"] = assembly_df["assembly_n"]
    assembly_df["assembly_keep"] = assembly_df["assembly_keep"].map(
        keep_assembly_dict[0]
    )

    # iterate over unique assemblies
    for assembly_n in assembly_df.assembly_n.unique():
        assem_idx = assembly_df.assembly_n == assembly_n
        sig_mem_idx = assembly_df.is_member == True

        # check if the significant members are deep or superficial or mixed
        if (assembly_df[assem_idx & sig_mem_idx].deepSuperficial == "Deep").all():
            assembly_df.loc[assem_idx, "assembly_label"] = "Deep"
        elif (
            assembly_df[assem_idx & sig_mem_idx].deepSuperficial == "Superficial"
        ).all():
            assembly_df.loc[assem_idx, "assembly_label"] = "Superficial"
        else:
            assembly_df.loc[assem_idx, "assembly_label"] = "mixed"

    result = {}
    result["pre"] = get_psths(st, results.get("assembly_act_pre"), assembly_df)

    result["task"] = get_psths(st, results.get("assembly_act_task"), assembly_df)

    result["post"] = get_psths(st, results.get("assembly_act_post"), assembly_df)

    result["cell_metrics"] = cell_metrics

    return result

def load_results(save_path, verbose=False):
    """
    load_results: load results from a pickle file
    """
    warnings.filterwarnings("ignore")
    print("Loading results...")

    sessions = glob.glob(save_path + os.sep + "*.pkl")

    pre_deep = pd.DataFrame()
    task_deep = pd.DataFrame()
    post_deep = pd.DataFrame()
    pre_sup = pd.DataFrame()
    task_sup = pd.DataFrame()
    post_sup = pd.DataFrame()
    meta_data_deep = pd.DataFrame()
    meta_data_sup = pd.DataFrame()

    for session in sessions:
        if verbose:
            print(session)
        with open(session, "rb") as f:
            results = pickle.load(f)
        if results is None:
            continue

        if not (
            results["pre"]["Superficial"].shape[1]
            == results["task"]["Superficial"].shape[1]
            == results["post"]["Superficial"].shape[1]
        ):
            print("Error: unequal number of columns")
            continue

        if not (
            results["pre"]["Deep"].shape[1]
            == results["task"]["Deep"].shape[1]
            == results["post"]["Deep"].shape[1]
        ):
            print("Error: unequal number of columns")
            continue

        pre_deep = pd.concat(
            [pre_deep, results["pre"]["Deep"]], axis=1, ignore_index=True
        )
        task_deep = pd.concat(
            [task_deep, results["task"]["Deep"]], axis=1, ignore_index=True
        )
        post_deep = pd.concat(
            [post_deep, results["post"]["Deep"]], axis=1, ignore_index=True
        )

        pre_sup = pd.concat(
            [pre_sup, results["pre"]["Superficial"]], axis=1, ignore_index=True
        )
        task_sup = pd.concat(
            [task_sup, results["task"]["Superficial"]], axis=1, ignore_index=True
        )
        post_sup = pd.concat(
            [post_sup, results["post"]["Superficial"]], axis=1, ignore_index=True
        )

        fields_to_keep = ["UID", "brainRegion", "putativeCellType", "basepath"]

        if results["cell_metrics"].shape[0] > 0:
            n_assembly = int(
                results["pre"]["Deep"].shape[1] / results["cell_metrics"].shape[0]
            )
            for n_assem in range(n_assembly):
                temp_df = results["cell_metrics"][fields_to_keep].copy()
                temp_df["assembly_n"] = n_assem
                meta_data_deep = pd.concat([meta_data_deep, temp_df], ignore_index=True)

            if not (meta_data_deep.shape[0] == post_deep.shape[1]):
                print("Error: unequal number of columns")

            n_assembly = int(
                results["pre"]["Superficial"].shape[1]
                / results["cell_metrics"].shape[0]
            )
            for n_assem in range(n_assembly):
                temp_df = results["cell_metrics"][fields_to_keep].copy()
                temp_df["assembly_n"] = n_assem
                meta_data_sup = pd.concat([meta_data_sup, temp_df], ignore_index=True)
        else:
            print("No cell metrics found")
    return (
        pre_deep,
        task_deep,
        post_deep,
        pre_sup,
        task_sup,
        post_sup,
        meta_data_deep,
        meta_data_sup,
    )


import json
import numpy as np
import en_core_web_sm

def get_ids(idfile):
    with open(idfile, 'r') as infile:
        lines = infile.read().split('\n')[:-1]
        ilist = [''] * len(lines)
        for pair in lines:
            p = pair.split('\t')
            ilist[int(p[1])] = p[0]
    return ilist

def to_json(eidfile, ridfile, triplefile, combined_rels, jsonfile):
    jsondict = {"nodes": [], "links": []}
    entities = get_ids(eidfile)
    relations = get_ids(ridfile)

    nlp = en_core_web_sm.load()
    f = lambda x: '#69b3a2' if [token for token in nlp(x) if token.ent_type_ == 'PERSON'] == [] else '#589bdd'
    for i in range(len(entities)):
        jsondict["nodes"].append({"id": i, "name": entities[i].replace('_', ' '), "color": f(entities[i].replace('_', ' '))})

    with open(triplefile, 'r') as infile:
        for line in infile.read().split('\n'):
            if line == '':
                continue
            triple = line.split('\t')
            jsondict["links"].append({"source": entities.index(triple[0]),"target": entities.index(triple[1]),"name":combined_rels[relations.index(triple[2])].replace('_', ' ')})

    with open(jsonfile, 'w') as outfile:
        outfile.write(json.dumps(jsondict))

def main():
    to_json('./data/entity2id.txt', './data/relation_tuples.txt', './templates/relations.json')

if __name__ == '__main__':
    main()


import sys, pickle, time

sys.path.append("/Users/samuelarnesen/Desktop/projects/rl-papers/gym/")
sys.path.append("/usr/local/lib/python3.8/site-packages")

import gym
import torch
import torch.nn as nn
import torch.optim as optim

from models import *

INCLUDE_NOISE = False
ENV_NAME = "MountainCarContinuous-v0"
#ENV_NAME = "Pendulum-v1"

env = gym.make(ENV_NAME)

FILE_NAME = "actor.p"
CRITIC_FILE = "critic.p"

model = load_model(FILE_NAME)
critic = load_model(CRITIC_FILE)
model.eval()
critic.eval()

min_action = -2
max_action = 2

step_size = 0.1

observation = env.reset()
done = False
step_count = 0

while not done:

	action = model(observation=observation, include_noise=INCLUDE_NOISE).item()

	observation, reward, done, info = env.step(np.asarray([action]))
	score = critic(observation, torch.tensor(action).unsqueeze(0).unsqueeze(0)).item()

	possible_action = min_action
	while possible_action < max_action:
		possible_score = critic(observation, torch.tensor(float(possible_action)).unsqueeze(0).unsqueeze(0)).item()
		print(step_count, "\t", round(possible_action, 2), round(possible_score, 2), "\t", round(action, 2), round(score, 2))
		possible_action += step_size

	print()

	step_count += 1

print(step_count)


from collections import defaultdict
import itertools
from typing import List

class Solution:
    def pyramidTransition(self, bottom: str, allowed: List[str]) -> bool:
        f = defaultdict(set)
        for s in allowed:
            f[s[:2]].add(s[2:])

        def dfs(row, next, map, i):
            if len(row) == 1:
                return True
            if len(next) + 1 == len(row):
                return dfs(next, "", map, 1)

            for c in map.get(row[i-1:i+1], set()):
                if dfs(row, next + c, map, i + 1):
                    return True
            return False
        return dfs(bottom, "", f, 1)

bottom = "AABA"
allowed = ["AAA", "AAB", "ABA", "ABB", "BAC"]

s = Solution()
print(s.pyramidTransition(bottom, allowed))


def label_customer(customer_id,logins, prediction_date, churn_days,   return_trans = False):
    """
    Make label times for a single customer. Returns a dataframe of labels with times, the binary label,
    and the number of days until the next churn.

    Params
    --------
        customer_id (str): unique id for the customer
        customer_logs (dataframe): logs dataframe for the customer
        prediction_date (str): time at which predictions are made. Either "MS" for the first of the month
                               or "SMS" for the first and fifteenth of each month
        churn_days (int): integer number of days without an active membership required for a churn. A churn is
                          defined by exceeding this number of 5 weeks  without being an active player.
        lead_time (int): number of periods in advance to make predictions for. 3 weeks
        prediction_window(int): number of periods over which to consider churn. 5 weeks
        return_trans (boolean): whether or not to return the transactions for analysis. Defaults to False.

    Return
    --------
        label_times (dataframe): a table of customer id, the cutoff times at the specified frequency, the
                                 label for each cutoff time, the number of days until the next churn for each
                                 cutoff time, and the date on which the churn itself occurred.
        transactions (dataframe): [optional] dataframe of customer transactions if return_trans = True. Useful
                                  for making sure that the function performed as expected

       """

    assert(prediction_date in ['MS', 'SMS']), "Prediction day must be either 'MS' or 'SMS'"
    assert(logins['actor_account_id'].unique() == [customer_id]), "Transactions must be for only customer"

    # Don't modify original
    logs = logins.copy()

    #Range for cutoff times is from first to last  log
    first_log = logs['min_time']
    last_log = logs['max_time']
    start_date = first_log
   # pd.datetime(first_log.month, first_log.day)

    # Find number of days between last log and cutoff

    logs['difference_days'] = logs['cutof']- logs['max_time']
    # Determine which actor are associated with a churn
    logs['churn'] = logs['difference_days'].astype('timedelta64[D]') > churn_days
    logs['last_day_active']=last_log

    # Find date of each churn
    logs.loc[logs['churn'] == True,
                     'churn_date'] = logs.loc[logs['churn'] == True,
                                                      'last_day_active'] + pd.Timedelta(churn_days, 'd')

    #time labels
    label_times=pd.DataFrame({'actor_account_id':[customer_id]})
    label_times.insert(1, "cutoff_time", "05-11-2016")
    label_times['cutoff_time'] = label_times['cutoff_time'].astype('datetime64[ns]')

    lead_time=21
    prediction_window=35
    # Use the lead time and prediction window parameters to establish the prediction window
    # Prediction window is for each cutoff time
    label_times['prediction_window_start'] = label_times['cutoff_time'] + pd.Timedelta(lead_time, 'd')
    label_times['prediction_window_end'] = label_times['cutoff_time'] + pd.Timedelta(lead_time, 'd') + pd.Timedelta(prediction_window, 'd')

    previous_churn_date = None

    #when no churn
    if (logs['churn']==False).all():
        label_times['label']=0
        label_times['days_to_churn']=np.nan
        label_times['churn_date']=np.nan
        if return_trans:
            return label_times[['actor_account_id', 'cutoff_time', 'label', 'days_to_churn', 'churn_date']], logs
        return label_times[['actor_account_id', 'cutoff_time', 'label', 'days_to_churn', 'churn_date']]

    # Iterate through every cutoff time
    for i, row in label_times.iterrows():

        # Default values if unknown
        churn_date = pd.NaT
        label = np.nan
        # Find the window start and end
        window_start = row['prediction_window_start']
        window_end = row['prediction_window_end']
        # Determine if there were any churns during the prediction window
        churns = logs.loc[(logs['churn_date'] >= window_start) &
                                  (logs['churn_date'] < window_end), 'churn_date']

        # Positive label if there was a churn during window
        if not churns.empty:
            label = 1
            churn_date = churns.values[0]

            # Find number of days until next churn by
            # subsetting to cutoff times before current churn and after previous churns
            if not previous_churn_date:
                before_idx = label_times.loc[(label_times['cutoff_time'] <= churn_date)].index
            else:
                before_idx = label_times.loc[(label_times['cutoff_time'] <= churn_date) &
                                             (label_times['cutoff_time'] > previous_churn_date)].index

            # Calculate days to next churn for cutoff times before current churn
            label_times.loc[before_idx, 'days_to_churn'] = (churn_date - label_times.loc[before_idx,
                                                                                'cutoff_time']).\
                                                            dt.total_seconds() / (3600 * 24)
            previous_churn_date = churn_date
        # No churns, but need to determine if an active member
        else:
            # Find transactions before the end of the window that were not cancelled ### here the carracter deleted
            logs_before = logs.loc[(logs['time'] < window_end)].copy()
            # If the membership expiration date for this membership is after the window start, the custom has not churned
            if np.any(logs_before['time'] >= window_start):
                label = 0

        # Assign values
        label_times.loc[i, 'label'] = label
        label_times.loc[i, 'churn_date'] = churn_date

        # Handle case with no churns
        if not np.any(label_times['label'] == 1):
            label_times['days_to_churn'] = np.nan
            label_times['churn_date'] = pd.NaT

    if return_trans:
        return label_times.drop(columns = ['actor_account_id']), logs

    return label_times[['actor_account_id', 'cutoff_time', 'label', 'days_to_churn', 'churn_date']].copy()


import numpy as np
import statistics

def reorder_cut_img(images, n_img_per_col):
    """
    :param images: list of 3d images
    :param n_img_per_col: number of images per col
    :return: a matrix of images [n_img_per_col][len(images) - n_img_per_col]
    """
    print("rebuilding preparation")
    m_img = []
    test = len(images)
    n_img_per_row = len(images) // n_img_per_col
    for r in range(n_img_per_col):
        # new row
        m_img.append([])
        for c in range(n_img_per_row):
            # we are at the image: r * n_img_per_row + c
            m_img[r].append(images[r * n_img_per_row + c])
    # now we should have each image in the right place (as in sol of the tests for cut)
    return m_img

def step1_to_rebuild(m_images, n_s_row_max, n_s_col_max, overlap_w, overlap_h, W=4172, H=1202):
    """
    :param m_images: matrix of images
    :param n_s_row_max: number of rows max for each img
    :param n_s_col_max: number of cols max for each img
    :param overlap_w:
    :param overlap_h:
    :param W: width of the original img
    :param H: height of the original img
    :return: a matrix of list of pixels as big as the original image
    """
    print("start step 1 rebuilding")
    img_step1 = [[[] for i in range(W)] for j in range(H)]
    for r in range(len(m_images)):
        for c in range(len(m_images[r])):
            c_in_step_2, r_in_step_2 = get_start_index(r, c, overlap_w, overlap_h, n_s_row_max, n_s_col_max)

            # To handle easily edge cases
            if c == len(m_images[r]) - 1:
                c_in_step_2 = W - len(m_images[r][c][0])
            if r == len(m_images) - 1:
                r_in_step_2 = H - len(m_images[r][c])

            # print("c: ", c_in_step_2, " w: ", r_in_step_2)

            for r1 in m_images[r][c]:
                tmp = c_in_step_2
                for c1 in r1:
                    img_step1[r_in_step_2][tmp].append(c1)
                    tmp += 1
                r_in_step_2 += 1
    return img_step1

def get_start_index(r, c, overlap_w, overlap_h, n_s_row_max, n_s_col_max):
    # It does not handle edge cases
    pad_h = n_s_row_max - overlap_h
    pad_w = n_s_col_max - overlap_w
    return c * pad_w, r * pad_h

def final_step_rebuild(img_step1, interpolator=lambda x: mean(x)):
    """
    :param img_step1: return image of step1_to_rebuild
    :param interpolator: function which decide how to handle the overlapped value of the pixels
    :return: starting image
    """
    print("start rebuilding")
    img_final = []
    for r in range(len(img_step1)):
        # add row
        img_final.append([])
        for c in range(len(img_step1[r])):
            param = [[] for i in range(len(img_step1[r][c][0]))]
            for p in range(len(img_step1[r][c])):
                for v in range(len(img_step1[r][c][p])):
                    param[v].append(img_step1[r][c][p][v])
            pix_to_add = []
            for l_to_int in param:
                pix_to_add.append(interpolator(l_to_int))
            # add pix
            img_final[r].append(pix_to_add)
    print("rebuilt")
    return img_final

def mean(l):
    return statistics.mean(l)

def median(l):
    return statistics.median(l)

def rebuild(images, n_img_per_col, n_s_row_max, n_s_col_max, overlap_w, overlap_h, W, H, interpolator=lambda x: mean(x)):
    """
    :param images: list of images
    :param n_img_per_col: number of images per col (regarding original img)
    :param n_s_row_max: max number of rows for single image
    :param n_s_col_max: max number of cols for single image
    :param overlap_w: overlap value on w
    :param overlap_h: overlap value on h
    :param W: W of original image in pix
    :param H: H of original image in pix
    :param interpolator: function which decide how to handle the overlapped value of the pixels
    :return: image rebuilt
    """
    m_images = reorder_cut_img(images, n_img_per_col)
    step1 = step1_to_rebuild(m_images, n_s_row_max, n_s_col_max, overlap_w, overlap_h, W, H)
    return final_step_rebuild(step1, interpolator)


# In this challenge, you are tasked with creating a Python script for analyzing the financial records of your company. You will give a set of financial data called [budget_data.csv](PyBank/Resources/budget_data.csv). The dataset is composed of two columns: `Date` and `Profit/Losses`. (Thankfully, your company has rather lax standards for accounting so the records are simple.)
# Your task is to create a Python script that analyzes the records to calculate each of the following:

import os
import csv
import statistics

bd_csv = os.path.join('../Resources', 'budget_data.csv')

#holders for data
months=[]
profitsLoss=[]

# Define function and have it accept 'budget_data'

#variables
count = 0
totalProfitLosses = 0
total_months = 0
greatest_increase = ["",0]
greatest_decrease = ["",9999999999]

#Open and read csv
with open(bd_csv) as csvfile:
    csvreader = csv.reader(csvfile, delimiter = ",")
    print(csvreader)

    csv_header = next(csvreader)

    first_row = next(csvreader)
    total_months = total_months + 1
    total_net = int(first_row[1])
    previous_net = total_net
# loop through data to count total months and calculate net change for profits loss
    for row in csvreader:
        total_months += 1
        total_net += int(row[1])
        net_change = int(row[1]) - previous_net
        previous_net = int(row[1])

# append profits loss

        profitsLoss.append(total_net)

        if net_change > greatest_increase[1]:
            greatest_increase[0] = row[0]
            greatest_increase[1] = net_change

        if net_change < greatest_decrease[1]:
            greatest_decrease[0] = row[0]
            greatest_decrease[1] = net_change

net_monthly_avg = sum(profitsLoss) / len(profitsLoss)

#   In addition, your final script should both print the analysis to the terminal and export a text file with the results.
# print("```text")
print("Financial Analysis")
print("----------------------------------------------------------")
print(f"Total Months: {total_months}")
print(f"Total Profits: ${total_net}")
print(f"Average  Change: ${net_monthly_avg}")
print(f"Greatest Increase in Profits: {greatest_increase[0]} (${greatest_increase[1]})")
print(f"Greatest Decrease in Profits: {greatest_decrease[0]} (${greatest_decrease[1]})")


from __future__ import print_function
try:
    import argparse
except ImportError:
    from itertools import chain
    import optparse
    USING_ARGPARSE = False
else:
    USING_ARGPARSE = True
import os
import sys
try:
    reload(sys)
except NameError:
    pass
else:
    sys.setdefaultencoding('utf-8')

from builtins import input

from ivre import db, graphroute, nmapout
from ivre.activecli import display_short, display_distinct, \
    displayfunction_json, displayfunction_honeyd, displayfunction_nmapxml, \
    displayfunction_gnmap, displayfunction_graphroute, \
    displayfunction_explain, displayfunction_remove, displayfunction_csv
from ivre.utils import display_top, CLI_ARGPARSER

def main():
    if USING_ARGPARSE:
        parser = argparse.ArgumentParser(
            description='Access and query the active scans database.',
            parents=[db.db.nmap.argparser, CLI_ARGPARSER],
        )
    else:
        parser = optparse.OptionParser(
            description='Access and query the active scans database.',
        )
        for args, kargs in chain(db.db.nmap.argparser.args,
                                 CLI_ARGPARSER.args):
            parser.add_option(*args, **kargs)
        parser.parse_args_orig = parser.parse_args
        parser.parse_args = lambda: parser.parse_args_orig()[0]
        parser.add_argument = parser.add_option
    parser.add_argument('--no-screenshots', action='store_true',
                        help='When used with --json, do not output '
                        'screenshots data.')
    parser.add_argument('--honeyd', action='store_true',
                        help='Output results as a honeyd config file.')
    parser.add_argument('--nmap-xml', action='store_true',
                        help='Output results as a nmap XML output file.')
    parser.add_argument('--gnmap', action='store_true',
                        help='Output results as a nmap grepable output file.')
    parser.add_argument(
        '--graphroute',
        choices=["dot", "rtgraph3d"] if graphroute.HAVE_DBUS else ["dot"],
        help='Create a graph from traceroute results. '
        'dot: output result as Graphviz "dot" format to stdout.'
        '%s' % (" rtgraph3d: send results to rtgraph3d."
                if graphroute.HAVE_DBUS else "")
    )
    parser.add_argument('--graphroute-cluster', choices=['AS', 'Country'],
                        help='Cluster IP according to the specified criteria'
                        '(only for --graphroute dot)')
    if graphroute.HAVE_DBUS:
        parser.add_argument('--graphroute-dont-reset', action='store_true',
                            help='Do NOT reset graph (only for '
                            '--graphroute rtgraph3d)')
    parser.add_argument('--graphroute-include', choices=['last-hop', 'target'],
                        help='How far should graphroute go? Default if to '
                        'exclude the last hop and the target for each result.')
    parser.add_argument('--top', metavar='FIELD / ~FIELD',
                        help='Output most common (least common: ~) values for '
                        'FIELD, by default 10, use --limit to change that, '
                        '--limit 0 means unlimited.')
    parser.add_argument('--csv', metavar='TYPE',
                        help='Output result as a CSV file',
                        choices=['ports', 'hops'])
    parser.add_argument('--csv-separator', metavar='SEPARATOR',
                        default=",",
                        help='Select separator for --csv output')
    parser.add_argument('--csv-add-infos', action='store_true',
                        help="Include country_code and as_number"
                        "fields to CSV file")
    parser.add_argument('--csv-na-str', default="NA",
                        help='String to use for "Not Applicable" value '
                        '(defaults to "NA")')
    args = parser.parse_args()

    out = sys.stdout

    hostfilter = db.db.nmap.parse_args(args)
    sortkeys = []
    if args.init:
        if os.isatty(sys.stdin.fileno()):
            sys.stdout.write(
                'This will remove any scan result in your database. '
                'Process ? [y/N] ')
            ans = input()
            if ans.lower() != 'y':
                sys.exit(-1)
        db.db.nmap.init()
        sys.exit(0)
    if args.ensure_indexes:
        if os.isatty(sys.stdin.fileno()):
            sys.stdout.write(
                'This will lock your database. '
                'Process ? [y/N] ')
            ans = input()
            if ans.lower() != 'y':
                sys.exit(-1)
        db.db.nmap.ensure_indexes()
        sys.exit(0)
    if args.top is not None:
        display_top(db.db.nmap, args.top, hostfilter, args.limit)
        sys.exit(0)
    if args.sort is not None:
        sortkeys = [(field[1:], -1) if field.startswith('~') else (field, 1)
                    for field in args.sort]
    if args.short:
        display_short(db.db.nmap, hostfilter, sortkeys, args.limit, args.skip)
        sys.exit(0)
    if args.distinct is not None:
        display_distinct(db.db.nmap, args.distinct, hostfilter, sortkeys,
                         args.limit, args.skip)
        sys.exit(0)
    if args.explain:
        displayfunction_explain(hostfilter, db.db.nmap)
        sys.exit(0)
    if args.json:
        def displayfunction(x):
            return displayfunction_json(
                x, db.db.nmap, args.no_screenshots
            )
    elif args.honeyd:
        displayfunction = displayfunction_honeyd
    elif args.nmap_xml:
        displayfunction = displayfunction_nmapxml
    elif args.gnmap:
        displayfunction = displayfunction_gnmap
    elif args.graphroute is not None:
        def displayfunction(x):
            return displayfunction_graphroute(
                x, args.graphroute, args.graphroute_include,
                args.graphroute_dont_reset
            )
    elif args.delete:
        def displayfunction(x):
            return displayfunction_remove(x, db.db.nmap)
    elif args.csv is not None:
        def displayfunction(x):
            return displayfunction_csv(
                x, args.csv, args.csv_separator, args.csv_na_str,
                args.csv_add_infos
            )
    else:

        def displayfunction(cursor):
            nmapout.displayhosts(cursor, out=out)

    if args.update_schema:
        db.db.nmap.migrate_schema(args.version)
    elif args.count:
        out.write(
            str(db.db.nmap.count(hostfilter)) + '\n'
        )
    else:
        kargs = {}
        if args.limit is not None:
            kargs["limit"] = args.limit
        if args.skip is not None:
            kargs["skip"] = args.skip
        if sortkeys:
            kargs["sort"] = sortkeys
        cursor = db.db.nmap.get(hostfilter, **kargs)
        displayfunction(cursor)
        sys.exit(0)


import pika
import process
import json
import time
import datetime
import os
import sys
import math
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import backend as K
from tensorflow.keras import layers
from tensorflow.python.keras import initializers
import re
import string
import transformers
from transformers import TFAutoModel, AutoTokenizer, AutoModelForMaskedLM, TFAutoModelForSequenceClassification, PhobertTokenizer
sys.path.insert(0, '/home/thanh/DATN/FakeNewDetection')

roberta = '/home/thanh/DATN/FakeNewDetection/Model/phobert-base'
MAX_LEN = 256

from Data.clean_text import clean_post, extract_num
from Data.regular_encode import regular_encode
from Data.tokenizer import VnCoreTokenizer
from Model.build_model import build_model

output_dir = '/home/thanh/DATN/FakeNewDetection/Model/17122020'

connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.queue_declare(queue='simulations')
channel.queue_declare(queue='results')
model = build_model(max_len=MAX_LEN)

def callback(ch, method, properties, body):
    requestParams = json.loads(body.decode('utf-8'))
    print(requestParams)
    timestamp_string = requestParams[1]
    timestamp = time.mktime(datetime.datetime.strptime(timestamp_string,"%Y-%m-%dT%H:%M").timetuple())
    data_test = [{'post_message': str(requestParams[0]) ,
              'timestamp_post': float(timestamp),
              'num_like_post': float(requestParams[2]) ,
              'num_comment_post': float(requestParams[3]),
              'num_share_post': float(requestParams[4])}]
    df_test_1 = pd.DataFrame(data = data_test)
    pred_result = str(predict(df_test_1))
    message = str(requestParams[0])
    timestamp = str(requestParams[1])
    results = {
        'message': message,
        'timestamp': timestamp,
        'result' : pred_result
    }
    channel.basic_publish(exchange='', routing_key='results', body=json.dumps(results, ensure_ascii=False))

def predict(test_dataframe):

  test_dataframe['post_message'] = test_dataframe['post_message'].apply(clean_post)
  test_dataframe["post_message"] = test_dataframe["post_message"].apply(VnCoreTokenizer().tokenize)

  test_dataframe["post_message"] = test_dataframe["post_message"].astype(str)
  test_dataframe["timestamp_post"] = test_dataframe["timestamp_post"].astype(float)

  for col in ["timestamp_post", "num_like_post", "num_comment_post", "num_share_post"]:
    test_dataframe[col] = test_dataframe[col].apply(extract_num)
    test_dataframe[col] = test_dataframe[col].astype(float)
    test_dataframe[col] = test_dataframe[col].fillna(0)

  X_test = regular_encode(test_dataframe, max_len=MAX_LEN)
  y_test = np.zeros((len(test_dataframe),1))
  preds = []
  for i, file_name in enumerate(os.listdir(output_dir)):
      print('_'*80)
      K.clear_session()
      model_path = os.path.join(output_dir, file_name)
      print(f'Inferencing with model from: {model_path}')
      model.load_weights(model_path)
      pred = model.predict(X_test,y_test)
      print(pred)
      preds.append(pred)
  preds = np.mean(preds, axis=0)
  print("predict weight is: ", preds)
  return preds[0][0]

# data_test = [{'post_message': '3 công nhân trung quốc làm cty hòa phát bị nhiễm visrut corona là sự thật mọi người đề phòng mặc dù hòa phát ngăn chặn yếm mọi thông tin nhưng công nhân làm ở hòa phát đã nói lại sự thật trên... mình đăng tin này có thể bị lực lượng chức năng ngăn chặn nguồn tin và xử phạt và bắt giữ mình nhưng mình cũng muốn nói vì đó là sự thật. sau 1 giờ mình sẽ xóa tin này vì k muốn bị lực lượng nhà nước cưỡng chế áp giải bắt bớ mình dù đó là sự thật.' ,
#               'timestamp_post': 1596255660,
#               'num_like_post': 0 ,
#               'num_comment_post': 0,
#               'num_share_post': 0}]
# df_test_1 = pd.DataFrame(data = data_test)
# start_time = time.time()
# predict(df_test_1)
# print("--- Time predict is %s seconds ---" % (time.time() - start_time))

channel.basic_consume(queue='simulations', on_message_callback=callback, auto_ack=True)
channel.start_consuming()


from graph import Graph

def has_cycle(graph):
    unvisited = set()
    visiting = set()
    visited = set()

    for vertex in graph.getAllVertices():
        unvisited.add(vertex)

    while unvisited:
        cur_node = unvisited.pop()
        if dfs(graph, cur_node, unvisited, visiting, visited):
            return True

    return False

def dfs(graph: Graph, vertex: int, unvisited: set, visiting:set, visited:set):
    visiting.add(vertex)

    for nbr in graph.getNeighbors(vertex):
        if nbr in visited:
            continue

        if nbr in visiting:
            return True

        if dfs(graph, nbr, unvisited, visiting, visited):
            return True

    visiting.remove(vertex)
    visited.add(vertex)
    return False

if __name__ == "__main__":
    graph = Graph(6)
    graph.addEdge(0,1)
    graph.addEdge(0,2)
    graph.addEdge(1,2)
    graph.addEdge(3,0)
    graph.addEdge(3,4)
    graph.addEdge(4,5)
    graph.addEdge(5,3)

    print(has_cycle(graph))


#write a program in Python to print the GCD of all the numbers in the list

import math

def gcdList(list):
    gcd=math.gcd(list[0],list[1])

    for index in range(2,len(list)):
        gcd=math.gcd(gcd,list[index])

    return gcd

list=[12,18,30,96,84,72]
print(gcdList(list))