from random import randrange, sample
from converter import *
from functools import reduce

def calculate_B(k, ID, n, e, inv, size = 2**20):
    B, quadruples = [[0 for i in range(2*k)] for i in range(2)]
    f_x_y = []
    for i in range(2 * k):
        a_i, c_i, d_i, r_i = [randrange(1, size) for x in range(4)]
        B[i] = calculate_B_i(a_i, c_i, d_i, r_i, ID, e, n, f_x_y)
        quadruples[i] = a_i, c_i, d_i, r_i
    return B, quadruples, f_x_y

def calculate_B_i(a, c, d, r, ID, e, n, f = None):
    x_i = sha1_hash(a + c)
    y_i = sha1_hash(a ^ ID + d)
    f_x_y = bytes_to_int(x_i) * bytes_to_int(y_i)

    if f is not None:
        f.append(f_x_y)
    B_i = r**e * f_x_y % n
    return B_i

def receive_signature(B, ID, priv_key, e, n, R, quadruples):
    for i in R:
        a, c, d, r = quadruples[i]
        B_i = calculate_B_i(a, c, d, r, ID, e, n)
        if(B_i != B[i]):
            raise Exception("Values are not equal")
    R_sign = [i for i in range(len(B)) if i not in R]
    B_to_sign = [B[i] for i in R_sign]
    signed_B = pow(reduce(lambda x, y : x * y %n, B_to_sign), priv_key, n)
    return signed_B, R_sign

def calc_S(signed_B, quadruples, n, R_sign):
    r = [quadruples[i][3] for i in R_sign]
    S = signed_B * mulinv(reduce(lambda x, y : x * y , r), n) % n
    return S

# Taken from
#https://en.wikibooks.org/wiki/Algorithm_Implementation/Mathematics/Extended_Euclidean_algorithm
def extendex_euc_alg(x, n):
    x0, x1, y0, y1 = 1, 0, 0, 1
    while n != 0:
        q, x, n = x // n, n, x % n
        x0, x1 = x1, x0 - q * x1
        y0, y1 = y1, y0 - q * y1
    return  x, x0, y0

# x = mulinv(b) mod n, (x * b) % n == 1
def mulinv(b, n):
    g, x, _ = extendex_euc_alg(b, n)
    if g == 1:
        return x % n

def totient(p, q):
    return (p - 1) * (q - 1)

p, q, e = 1300097, 1299721, 17
n = p * q # 127*89= 11303
totient_n = totient(p, q)
ID =  1251261711
k = 100
priv_key  = mulinv(e, totient_n)
# print("INV", priv_key)
for i in range(1000):
    B, quadruples, f_x_y = calculate_B(k, ID, n, e, priv_key) # STEP 1
    R = sample(range(len(B)), len(B) // 2) # STEP 2
    quadruples_R = {i: quadruples[i] for i in R}
    signed_B, R_sign = receive_signature(B, ID, priv_key, e, n, R, quadruples_R) #STEP 3.2

    f_sign = [f_x_y[i] for i in R_sign]
    prod_sum_f = reduce(lambda x, y: x * y, f_sign)
    x = pow(prod_sum_f, priv_key, n)

    S = calc_S(signed_B, quadruples, n, R_sign)
    print("SIGNED VALUE OK", S == x)
    print("SIGNATURE OK", prod_sum_f % n == pow(S, e, n))
    # print(S)


from flask import Flask, redirect, render_template, request, session, url_for
import os

app = Flask(__name__)

@app.route("/")
def index():
    return render_template("index.html")

@app.route("/result", methods=["POST"])
def result():
    # Imports the Google Cloud client library
    from google.cloud import language
    import numpy, re

    # Instantiates a client
    language_client = language.Client()

    # The text to analyze
    para = request.form["text"]
    thresholdFactor = int(request.form["thresholdFactor"])
    text = re.split(r'[.!?]+', para)

    pair_message = []

    for data in text:
        data = data.strip()
        document = language_client.document_from_text(data)
        # Detects the sentiment of the text
        sentiment = document.analyze_sentiment().sentiment
        pair_message.append([data, sentiment.magnitude])

    mean_mag = numpy.mean([x[1] for x in pair_message])
    median_mag = numpy.median([x[1] for x in pair_message])

    print
    processed_text = ""
    count = 0
    threshold = min(mean_mag, median_mag)
    for sentence in pair_message:
        if sentence[1] >= threshold * ((thresholdFactor - 0.5)/ 2):
            count += 1
            location = para.index(sentence[0])
            processed_text += para[location:location + len(sentence[0]) + 1] + " "
    return render_template("success.html", display = processed_text)

if __name__ == '__main__':
     app.debug = True
     port = int(os.environ.get("PORT", 5000))
     app.run(host='0.0.0.0', port=port)


for aa in range(int(input())):
    n=int(input())
    li=list(map(int,input().split()))
    s=1
    x=[]
    for i in li:
      s*=i
      x.append(s)
    k=x[n-1]
    r=-1
    for i in range(n-1):
       if(k/x[i]==x[i]):
          r=i+1
          break
    # print(r,x)
    if(r==-1):
       print(-1)
       continue
    else:
       print(r)


import math
import random
import gym
from gym import spaces
import numpy as np

DRAW = 3

class TicTacToe(gym.Env):
    def __init__(self):
        self.board = np.zeros(shape=(3, 3), dtype=np.int)
        self.observation_space = spaces.Box(
            low=0, high=2, shape=(3, 3), dtype=np.int)
        self.action_space = spaces.Discrete(9)
        #self.play_against_human = play_against_human

        self.done = False
        self.symbol = 1
        self.opp_symbol = 2

    def get_observation(self):
        return np.copy(self.board)

    def board_is_full(self):
        if np.isin(0, self.board):
            return False
        else:
            return True

    def pos_is_available(self, pos):
        x = pos % 3
        y = int(pos/3)

        #print("Coords:", x, y)

        if self.board[x, y] == 0:
            return True
        else:
            return False

    def put_random(self):
        x = random.randint(0, 8)

        while not self.pos_is_available(x):
            x = random.randint(0, 8)

        return self.mark(x, self.opp_symbol)

    def reset(self):
        self.done = False
        self.board = np.zeros(shape=(3, 3), dtype=np.int)

        x = random.randint(0, 1)

        if x == 1:
            self.put_random()

        return self.get_observation()

    def mark(self, pos, sym):
        x = pos % 3
        y = int(pos/3)

        #print("Coords:", x, y)
        reward = -1

        if self.board[x, y] == 0:
            self.board[x, y] = sym
            valid = True
        else:
            valid = False

        if not valid:
            reward = -40
            self.done = True
        else:
            win = self.check_end()

            if win == self.symbol:
                reward = 20
                self.done = True
            elif win == self.opp_symbol:
                reward = -20
                self.done = True
            elif win == DRAW:
                reward = 10
                self.done = True

        return reward

    def check_end(self):
        for i in range(3):
            if self.board[i, 0] == self.board[i, 1] == self.board[i, 2]:
                sym = self.board[i, 0]
                if sym != 0:
                    return sym
            if self.board[0, i] == self.board[1, i] == self.board[2, i]:
                sym = self.board[0, i]
                if sym != 0:
                    return sym

        if self.board[0, 0] == self.board[1, 1] == self.board[2, 2]:
            sym = self.board[0, 0]
            if sym != 0:
                return sym
        elif self.board[0, 2] == self.board[1, 1] == self.board[2, 0]:
            sym = self.board[0, 2]
            if sym != 0:
                return sym

        if not self.board_is_full():
            return 0
        else:
            return DRAW

    def step(self, act):
        reward = self.mark(act, self.symbol)

        if not self.done:
            reward = self.put_random()

        return self.get_observation(), reward, self.done, {}

    def render(self, mode='human'):
        print("---------------")
        for x in range(3):
            for y in range(3):
                print(self.board[y, x], end=" ")
            print("")
        print("---------------")

Reinforcement Learning/V2 - Tyro/tictactoe/tictactoe.py


from flask import Flask,render_template,request,redirect
from cs50 import SQL

app=Flask(__name__)

db=SQL("sqlite:///froshims.db")

@app.route("/",methods=["POST","GET"])
def index():
    if request.method=="GET":
        return render_template("main.html")
    if request.method=="POST":

        x = request.form.get("name")
        if x=="":
            return "Name required"
        y=request.form.get("sport")
        db.execute("INSERT INTO registrants (name,sport) VALUES(?,?)",x,y)

        return redirect("/done")

@app.route("/done")
def done():
    l = db.execute("SELECT * FROM registrants ")
    return render_template("added.html",l=l)

if __name__== "__main__":
    app.run()


import sys
from enum import Enum
from heapq import *
from math import cos, sin, pi, sqrt, tan, nan
import random

import pygame
import copy

from client.game.src.core.bot.a_star import A_Star
from client.game.src.core.bot.maze import Maze, path_from
from client.game.src.core.stat_bar.stat_bar import StatBar
from client.game.src.utils.config import Config

class Drive(Enum):
    FORWARD = 0
    BACKWARD = 1

class Rotate(Enum):
    LEFT = 0
    RIGHT = 1

class ExpectRotate(float, Enum):
    UP = 270
    DOWN = 90
    LEFT = 180
    RIGHT = 0

class DirectionAngle(float, Enum):
    UP = 90
    DOWN = 270
    LEFT = 180
    RIGHT = 0

class BotController:
    SHOT_DISTANCE = 30
    ANGLE_OFFSET = 10
    POS_OFFSET = 20
    COLISION_OFFSET = 10
    DETECTION_OFFSET = 30
    FLEE_ANGLE = 70
    DISTANCE_OFFSET = 15
    ANGLE_SENSITIVITY = 30

    def __init__(self, screen, game, player, id):
        self.id = id
        self.screen = screen
        self.game = game
        self.player = player
        self.map = self.read_map(game.map.data)
        self.node = None
        self.x = 0
        self.y = 0
        self.glitch_time = 0
        self.flee_timer = 0
        self.flee_dir = 0
        self.last_bullet_angle = 0
        self.enemy = None

    def distance(self, player):
        return abs(self.player.position[0] - player.position[0]) + abs(self.player.position[1] - player.position[1])

    def set_enemy(self):
        self.enemy = self.find_closest_player()

    def find_closest_player(self):
        minDistance = sys.maxsize
        minPlayer = None

        for player in self.game.players:
            if player != self.player:
                if minDistance > self.distance(player):
                    minDistance = self.distance(player)
                    minPlayer = player
        return minPlayer

    def read_map(self, map):
        # for i in range(len(map)):
        #     print(map[i])
        for i in range(len(map)):
            map[i] = list(map[i])
            for j in range(len(map[i])):
                if map[i][j] == 'S':
                    map[i][j] = '.'

        return map

    def astar(self, maze):
        start_node = maze.find_node('S')
        self.node = (start_node.x, start_node.y)
        start_node.visited = True
        end_node = maze.find_node('E')
        start_node.cost = abs(end_node.x - start_node.x) + abs(end_node.y - start_node.y)
        q = []
        id = 0
        heappush(q, (start_node.cost, id, start_node))
        while q:
            node = heappop(q)[2]  # LIFO
            node.visited = True
            if node.type == 'E':
                return path_from(node)

            children = maze.get_possible_movements(node)
            for child in children:
                if not child.visited:
                    child.parent = node
                    child.moves_cost = node.moves_cost + maze.move_cost(child)
                    child.cost = abs(end_node.x - child.x) + abs(end_node.y - child.y) + child.moves_cost
                    id += 1
                    heappush(q, (child.cost, id, child))

        return None

    def find_path(self):
        # player = self.find_closest_player()  # gdyby bylo wiecej graczy trzeba uzywac tej metody
        player = self.enemy
        new_map = copy.deepcopy(self.map)
        width = self.game.assets.width
        height = self.game.assets.height

        self.x = int(self.player.position[0] / width)
        self.y = int(self.player.position[1] / height)

        new_map[self.y][self.x] = 'S'
        x2 = int(player.position[0] / width)
        y2 = int(player.position[1] / height)
        new_map[y2][x2] = 'E'

        if (self.x, self.y) == (x2, y2):
            return None, None, 0
        maze = Maze(new_map)
        maze.path = self.astar(maze)
        # maze.draw()

        act = maze.path[len(maze.path) - 1]
        new = maze.path[len(maze.path) - 2]
        return (act.x, act.y), (new.x, new.y), len(maze.path)

    def move(self):
        if self.enemy.is_alive():
            actual_point, new_point, length = self.find_path()
            if actual_point:
                if actual_point[1] < new_point[1]:
                    return length, ExpectRotate.UP
                if actual_point[1] > new_point[1]:
                    return length, ExpectRotate.DOWN
                if actual_point[0] < new_point[0]:
                    return length, ExpectRotate.RIGHT
                if actual_point[0] > new_point[0]:
                    return length, ExpectRotate.LEFT
        return 0, self.player.angle  # stay

    def diagonal_distance(self):
        """
        :return: diagonal distance between bot and an enemy.
        :rtype: float
        """
        e_x, e_y = self.enemy.position
        b_x, b_y = self.player.position
        return sqrt(pow(b_x - e_x, 2) + pow(b_y - e_y, 2))

    def checkMove(self, directory, pos):
        # for i in range(len(self.map)):
        #   print(self.map[i])

        if directory == DirectionAngle.DOWN and self.map[pos[1] + 1][pos[0]] != "#":
            return True
        elif directory == DirectionAngle.UP and self.map[pos[1] - 1][pos[0]] != "#":
            return True
        elif directory == DirectionAngle.RIGHT and self.map[pos[1]][pos[0] + 1] != "#":
            return True
        elif directory == DirectionAngle.LEFT and self.map[pos[1]][pos[0] - 1] != "#":
            return True

        return False

    @property
    def move2(self):
        moves = []
        self.actual_pos = ""
        if self.enemy.is_alive():
            self.actual_pos, new_point, length = self.find_path()
            if self.actual_pos:
                moves = [DirectionAngle.DOWN, DirectionAngle.UP, DirectionAngle.RIGHT, DirectionAngle.LEFT]
                opponent_possition = None
                if (self.actual_pos[1] < new_point[1]):
                    opponent_possition = DirectionAngle.DOWN
                elif (self.actual_pos[1] > new_point[1]):
                    opponent_possition = DirectionAngle.UP
                elif (self.actual_pos[0] < new_point[0]):
                    opponent_possition = DirectionAngle.RIGHT
                elif (self.actual_pos[0] > new_point[0]):
                    opponent_possition = DirectionAngle.LEFT

                # print("oppo:", opponent_possition)
                moves.remove(opponent_possition)
                for move in moves:
                    if not self.checkMove(move, self.actual_pos):
                        moves.remove(move)

                if len(moves) == 0 and self.checkMove(opponent_possition, self.actual_pos):
                    moves.append(opponent_possition)

        if len(moves):
            # print(self.actual_pos, new_point, moves, self.player.angle, self.player.lastMove)
            for angle in moves:
                # if self.lastMove and ((angle - 1 < self.lastMove < angle + 1) or (angle == DirectionAngle.RIGHT and 359 < self.lastMove < 361)):  # stay
                if self.player.lastMove != None and angle == self.player.lastMove and self.checkMove(self.player.lastMove, self.actual_pos):
                    return length, self.player.lastMove

            r = random.randrange(0, len(moves), 1)
            self.player.lastMove = moves[r]
            # print("a? ", moves[r])
            return length, moves[r]
        # print("C")
        self.player.lastMove = self.player.angle
        return 0, self.player.angle  # stay

    def shot_condition(self, length):
        """
        Checks if enemy is in a straight line with a bot and if bot faces the enemy.
        :param int length: Length of a shortest path between bot and an enemy in a number of tiles.
        :return: If bot is supposed to shot.
        :rtype: bool
        """
        if self.enemy.is_alive():
            e_x, e_y = self.enemy.position
            b_x, b_y = self.player.position
            b_angle = self.player.angle
            width = self.game.assets.width
            height = self.game.assets.height
            tile_size = (width + height) / 2
            if length - 1 <= self.diagonal_distance() / tile_size:
                if b_x - self.POS_OFFSET <= e_x <= b_x + self.POS_OFFSET:
                    # enemy above
                    if b_y > e_y:
                        if DirectionAngle.UP - self.ANGLE_OFFSET <= b_angle <= DirectionAngle.UP + self.ANGLE_OFFSET:
                            return True
                    # enemy below
                    if b_y < e_y:
                        if DirectionAngle.DOWN - self.ANGLE_OFFSET <= b_angle <= DirectionAngle.DOWN + self.ANGLE_OFFSET:
                            return True
                elif b_y - self.POS_OFFSET <= e_y <= b_y + self.POS_OFFSET:
                    # enemy on a left
                    if b_x > e_x:
                        if DirectionAngle.LEFT - self.ANGLE_OFFSET <= b_angle <= DirectionAngle.LEFT + self.ANGLE_OFFSET:
                            return True
                    # enemy on a right
                    if b_x < e_x:
                        if DirectionAngle.RIGHT - self.ANGLE_OFFSET <= b_angle <= DirectionAngle.RIGHT + self.ANGLE_OFFSET:
                            return True
        return False

    @staticmethod
    def whole_angle(angle):
        """
        Check which angle from {0, 90, 180, 270} is closest to the given one.
        :param float angle:
        :return: angle from a range of {0, 90, 180, 270}.
        :rtype: float
        """
        if 45 < angle <= 135:
            return 90
        elif 135 < angle <= 225:
            return 180
        elif 225 < angle <= 315:
            return 270
        else:
            return 0

    def on_trajectory(self, bullet):
        """
        Checks if bot is on the bullet whole trajectory, including the one behind it.
        :param bullet: Bullet Object
        :return: If bot is on the bullet trajectory. If yes also bullet angle.
        :rtype: bool, float
        """
        bullet_x, bullet_y = bullet.position
        bot_x, bot_y = self.player.position
        offset = 2
        if 90 - offset <= bullet.angle <= 90 + offset:
            hit_x = bullet_x
            hit_y = bot_y
        elif 0 <= bullet.angle <= offset or 360 - offset <= bullet.angle <= 360:
            hit_x = bot_x
            hit_y = bullet_y
        elif 270 - offset <= bullet.angle <= 270 + offset:
            hit_x = bullet_x
            hit_y = bot_y
        elif 180 - offset <= bullet.angle <= 180 + offset:
            hit_x = bot_x
            hit_y = bullet_y
        else:
            bullet_a = tan(-bullet.angle * pi / 180)
            bullet_b = bullet_y - (bullet_a * bullet_x)

            bot_a = - 1 / bullet_a if bullet_a else - 1 / pow(10, -20)
            bot_b = bot_y - (bot_a * bot_x)

            hit_x = (bot_b - bullet_b) / (bullet_a - bot_a) if bullet_a - bot_a else (bot_b - bullet_b) / pow(10, -20)
            hit_y = (bullet_a * hit_x) + bullet_b

        dist = sqrt(pow(bot_x - hit_x, 2) + pow(bot_y - hit_y, 2))

        if dist < self.DETECTION_OFFSET:
            return True, bullet.angle
        return False, None
        # return True, object.angle

    def at_gunpoint(self):
        """
        Check if bot might get hit by any bullet on a map except his.
        :return: If bot might get hit if not moved. If yes also bullet angle.
        :rtype: bool, float
        """
        for bullet in self.game.bullet_controller.bullets:
            if bullet.player.id != self.player.id:

                bullet_x, bullet_y = bullet.position
                bot_x, bot_y = self.player.position
                if bullet_x < bot_x - self.DETECTION_OFFSET and 90 <= bullet.angle <= 270:
                    # print("x< ", self.id)
                    continue
                elif bullet_x > bot_x + self.DETECTION_OFFSET and (
                        0 <= bullet.angle <= 90 and 270 <= bullet.angle <= 360):
                    # print("x> ", self.id)
                    continue
                elif bullet_y < bot_y - self.DETECTION_OFFSET and 0 <= bullet.angle <= 180:
                    # print("y< ", self.id)
                    continue
                elif bullet_y > bot_y + self.DETECTION_OFFSET and 180 <= bullet.angle <= 360:
                    # print("y> ", self.id)
                    continue
                else:
                    # print("check")
                    endangered, angle = self.on_trajectory(bullet)
                    if endangered:
                        return endangered, angle
        return False, None

    def paul(self, time):
        if self.player.is_alive():
            flee, bullet_angle = self.at_gunpoint()
            if self.flee_timer <= 0 or flee:
                self._reload(time)
                length, rotate = self.move()
                shot = self.shot_condition(length)
                angle = self.player.angle - rotate
                width = self.game.assets.width
                height = self.game.assets.height

                if self.player.angle > 180 and rotate == DirectionAngle.RIGHT:
                    angle = angle - 360
                elif self.player.angle <= 0 and rotate == DirectionAngle.DOWN:
                    angle = (360 + angle)

                if angle != 0 and (-1 < self.player.angle < 1 and self.x * width > self.player.position[0] - width / 2
                                   or 179 < self.player.angle < 181 and self.player.position[0] > (
                                           self.x + 1) * width - width / 2
                                   or 269 < self.player.angle < 271 and self.y * height >= self.player.position[
                                       1] - height / 2
                                   or 89 < self.player.angle < 91 and self.player.position[1] > (
                                           self.y + 1) * height - height / 2)\
                                   or 359 < self.player.angle < 361 and self.player.position[0] > (
                                           self.x + 1) * width - width / 2:
                    angle = 0

                if shot:
                    self.shot()

                move_value = 0
                if flee:
                    self.last_bullet_angle = bullet_angle
                    self.flee_timer = 20
                    rev_b_angle = (bullet_angle + 180) % 360
                    if rev_b_angle - self.FLEE_ANGLE <= self.player.angle <= rev_b_angle + self.FLEE_ANGLE:
                        if self.flee_dir:
                            self.rotate(Rotate.LEFT, 3 * time)
                        else:
                            self.rotate(Rotate.RIGHT, 3 * time)
                        self.drive(Drive.BACKWARD, time)
                    else:
                        self.drive(Drive.BACKWARD, time)
                elif self.glitch_time < 0:
                    self.drive(Drive.BACKWARD, time)
                    if self.flee_dir:
                        self.rotate(Rotate.LEFT, time / 2)
                    else:
                        self.rotate(Rotate.LEFT, time / 2)
                    move_value = 1
                elif angle > 1:
                    self.rotate(Rotate.RIGHT, time)
                elif angle < -1:
                    self.rotate(Rotate.LEFT, time)
                else:
                    self.flee_dir = 0 if self.flee_dir else 1
                    self.player.rotate(self.whole_angle(self.player.angle) - self.player.angle, time)
                    b_x, b_y = self.player.position
                    e_x, e_y = self.enemy.position
                    re_angle = (self.enemy.angle + 180) % 360
                    dist_offset = self.DISTANCE_OFFSET
                    ang_offset = self.ANGLE_SENSITIVITY
                    # bot and enemy facing each other and on the +/- same x position
                    if re_angle - ang_offset <= self.player.angle <= re_angle + ang_offset and \
                            e_x - dist_offset <= b_x <= e_x + dist_offset and \
                            e_y - self.COLISION_OFFSET <= b_y <= e_y + self.COLISION_OFFSET:
                        if length <= 3 and \
                                (90 - ang_offset <= re_angle <= 90 + ang_offset or \
                                270 - ang_offset <= re_angle <= 270 + ang_offset):
                            move_value = -10
                        else:
                            move_value = self.drive(Drive.FORWARD, time)
                    # bot and enemy facing each other and on the +/- same y position
                    elif re_angle - ang_offset <= self.player.angle <= re_angle + ang_offset and \
                            e_y - dist_offset <= b_y <= e_y + dist_offset and \
                            e_x - self.COLISION_OFFSET <= b_x <= e_x + self.COLISION_OFFSET:
                        if length <= 3 and \
                                (0 <= re_angle <= ang_offset or 360 - ang_offset <= re_angle <= 360 or \
                                180 - ang_offset <= re_angle <= 180 + ang_offset):
                            move_value = -10
                        else:
                            move_value = self.drive(Drive.FORWARD, time)
                    # bot or enemy is turned to the other one in a 90 degree angle
                    elif re_angle + 90 - ang_offset <= self.player.angle <= re_angle + 90 + ang_offset or \
                          re_angle - 90 - ang_offset <= self.player.angle <= re_angle - 90 + ang_offset: \
                        move_value = self.drive(Drive.FORWARD, time)
                    elif length > 2:
                        move_value = self.drive(Drive.FORWARD, time)
                    else:
                        move_value = -10 - random.randrange(10)
                self.glitch_time += move_value
                if self.glitch_time >= 100:
                    self.glitch_time = -40 + random.randrange(10)
                elif move_value == 0:
                    self.glitch_time = 0
            else:
                shot = self.shot_condition(1)
                self.flee_timer -= 1

    def piotrek(self, time):
        if self.player.is_alive():
            self._reload(time)
            length, rotate = self.move()
            shot = self.shot_condition(length)
            angle = self.player.angle - rotate
            width = self.game.assets.width
            height = self.game.assets.height

            if self.player.angle > 180 and rotate == DirectionAngle.RIGHT:
                angle = angle - 360
            elif self.player.angle <= 0 and rotate == DirectionAngle.DOWN:
                angle = (360 + angle)

            if angle != 0 and (-1 < self.player.angle < 1 and self.x * width > self.player.position[0] - width / 2
                               or 179 < self.player.angle < 181 and self.player.position[0] > (
                                       self.x + 1) * width - width / 2
                               or 269 < self.player.angle < 271 and self.y * height >= self.player.position[
                                   1] - height / 2
                               or 89 < self.player.angle < 91 and self.player.position[1] > (
                                       self.y + 1) * height - height / 2)\
                               or 359 < self.player.angle < 361 and self.player.position[0] > (
                                       self.x + 1) * width - width / 2:
                angle = 0

            if shot:
                self.shot()

            move_value = 0
            if self.glitch_time < 0:
                self.drive(Drive.BACKWARD, time)
                if self.flee_dir:
                    self.rotate(Rotate.LEFT, time / 2)
                else:
                    self.rotate(Rotate.LEFT, time / 2)
                move_value = 1
            elif angle > 1:
                self.rotate(Rotate.RIGHT, time)
            elif angle < -1:
                self.rotate(Rotate.LEFT, time)
            else:
                self.flee_dir = 0 if self.flee_dir else 1
                self.player.rotate(self.whole_angle(self.player.angle) - self.player.angle, time)
                b_x, b_y = self.player.position
                e_x, e_y = self.enemy.position
                re_angle = (self.enemy.angle + 180) % 360
                dist_offset = self.DISTANCE_OFFSET
                ang_offset = self.ANGLE_SENSITIVITY
                # bot and enemy facing each other and on the +/- same x position
                if re_angle - ang_offset <= self.player.angle <= re_angle + ang_offset and \
                        e_x - dist_offset <= b_x <= e_x + dist_offset and \
                        e_y - self.COLISION_OFFSET <= b_y <= e_y + self.COLISION_OFFSET:
                    if length <= 3 and \
                            (90 - ang_offset <= re_angle <= 90 + ang_offset or \
                            270 - ang_offset <= re_angle <= 270 + ang_offset):
                        move_value = -10
                    else:
                        move_value = self.drive(Drive.FORWARD, time)
                # bot and enemy facing each other and on the +/- same y position
                elif re_angle - ang_offset <= self.player.angle <= re_angle + ang_offset and \
                        e_y - dist_offset <= b_y <= e_y + dist_offset and \
                        e_x - self.COLISION_OFFSET <= b_x <= e_x + self.COLISION_OFFSET:
                    if length <= 3 and \
                            (0 <= re_angle <= ang_offset or 360 - ang_offset <= re_angle <= 360 or \
                            180 - ang_offset <= re_angle <= 180 + ang_offset):
                        move_value = -10
                    else:
                        move_value = self.drive(Drive.FORWARD, time)
                # bot or enemy is turned to the other one in a 90 degree angle
                elif re_angle + 90 - ang_offset <= self.player.angle <= re_angle + 90 + ang_offset or \
                      re_angle - 90 - ang_offset <= self.player.angle <= re_angle - 90 + ang_offset: \
                    move_value = self.drive(Drive.FORWARD, time)
                elif length > 2:
                    move_value = self.drive(Drive.FORWARD, time)
                else:
                    move_value = -10 - random.randrange(10)
            self.glitch_time += move_value
            if self.glitch_time >= 100:
                self.glitch_time = -40 + random.randrange(10)
            elif move_value == 0:
                self.glitch_time = 0

    def john(self, time):
        if self.player.is_alive():
            self._reload(time)
            length, rotate = self.move2
            shot = self.shot_condition(length)
            angle = self.player.angle - rotate
            width = self.game.assets.width
            height = self.game.assets.height

            if self.player.angle > 180 and rotate == DirectionAngle.RIGHT:
                angle = angle - 360
            elif self.player.angle <= 0 and rotate == DirectionAngle.DOWN:
                angle = (360 + angle)

            if angle != 0 and (-1 < self.player.angle < 1 and self.x * width > self.player.position[0] - width / 2
                               or 179 < self.player.angle < 181 and self.player.position[0] > (
                                       self.x + 1) * width - width / 2 + 6
                               or 269 < self.player.angle < 271 and self.y * height >= self.player.position[
                                   1] - height / 2 + 7
                               or 89 < self.player.angle < 91 and self.player.position[1] > (
                                       self.y + 1) * height - height / 2
                    or 359 < self.player.angle < 361 and self.player.position[0] > (
                    self.x + 1) * width - width / 2 + 6):

                # print("b")
                angle = 0

            if shot:
                self.shot()
            if angle > 1:
                self.rotate(Rotate.RIGHT, 2*time)
            elif angle < -1:
                self.rotate(Rotate.LEFT, 2*time)
            else:
                if angle > 0.05:
                    self.rotate(Rotate.RIGHT, time/5)
                elif angle < -0.05:
                    self.rotate(Rotate.LEFT, time/5)

                if self.glitch_time < 0:
                    self.drive(Drive.BACKWARD, time)
                    move_value = 1
                    self.player.lastMove = None
                else:
                    move_value = self.drive(Drive.FORWARD, time)

                self.glitch_time += move_value
                if self.glitch_time == 100:
                    self.glitch_time = -50
                elif move_value == 0:
                    self.glitch_time = 0

    def _reload(self, time):
        self.player.reload_time -= time
        StatBar.show_reload(self.screen, self.player)
        if self.player.reload_time <= 0:
            if self.player.is_current:
                StatBar.show_magazine(self.screen, self.player)

    def _reload_magazine(self):
        if self.player.bullets != Config.player['tank']['magazine']:
            self.player.reload_magazine()
            self.player.reload_time = Config.player['tank']['reload_magazine']

    def drive(self, drive: Drive, time):
        x, y = self.player.position
        radians = -self.player.angle * pi / 180
        if drive == Drive.FORWARD:
            speed = Config.player['speed']['drive']['forward'] * time
            new_x = x + (speed * cos(radians))
            new_y = y + (speed * sin(radians))
        else:
            speed = Config.player['speed']['drive']['backward'] * time
            new_x = x - (speed * cos(radians))
            new_y = y - (speed * sin(radians))

        new_position = (new_x, new_y)

        self.player.move(new_position)
        if (x, y) == self.player.position:
            return 1
        else:
            return 0
        # TODO: Send new position to the server

    def rotate(self, angle: Rotate, time):
        rotate_speed = Config.player['speed']['rotate'] * time
        if angle == Rotate.LEFT:
            new_angle = rotate_speed
        else:
            new_angle = -rotate_speed

        if new_angle > 360:
            new_angle -= 360
        elif new_angle < -360:
            new_angle += 360

        self.player.rotate(new_angle, 1)
        # TODO: Send new angle to the server

    def shot(self):
        if self.player.reload_time <= 0:
            self.player.reload_time = Config.player['tank']['reload_bullet']
            self.player.shot()
            if self.player.is_current:
                StatBar.show_magazine(self.screen, self.player)
            if self.player.bullets == 0:
                self._reload_magazine()
            # TODO: Send bullet position to the server


# By Marco Anaya

# python3 plot.py (module)

import sys
import os
import math
import string

import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator
from matplotlib.backends.backend_pdf import PdfPages

# supports coloring 12 categories, repeating once until 24
colors = dict(zip(string.ascii_uppercase,
            ['#E58606', '#5D69B1', '#52BCA3', '#99C945', '#CC61B0', '#24796C',
            '#DAA51B', '#2F8AC4', '#764E9F', '#ED645A', '#CC3A8E', '#A5AA99',
            '#E58606', '#5D69B1', '#52BCA3', '#99C945', '#CC61B0', '#24796C',
            '#DAA51B', '#2F8AC4', '#764E9F', '#ED645A', '#CC3A8E', '#A5AA99',
            '#E58606', '#5D69B1', '#52BCA3']))

min_size = 5

def plot(project, module, class_distributions=None, class_performance_per_req=None):
    class_perf, class_distr = class_performance_per_req, class_distributions

    csv_path = "./" + project + "/" + module + '/csv/'
    graph_path = "./" + project + "/" + module + '/graphs/'
    if class_perf is None:
        class_perf = pd.read_csv(csv_path + module + '_class_performance_per_req.csv')
    if class_distr is None:
        class_distr = pd.read_csv(csv_path + module + '_class_distributions.csv')

    class_perf = class_perf[class_perf['Class Size'] > min_size]
    class_distr = class_distr[class_distr['Class Size'] > min_size]

    # Runs plot_grade() for each grade level, generating three graphs
    grades = list(filter(lambda entry: os.path.isdir(csv_path + entry), os.listdir(csv_path)))
    print('Graphing ' + module + ':')
    for grade in grades:

        plot_grade(graph_path, module, grade, class_perf, class_distr)
        print('  performance and distribution graphs for grade ' + str(grade) + '.')

    # Generates teacher analysis for the whole module
    plot_module(graph_path, module, grades, class_perf)
    print('  performance by classroom and grade.')
    plot_module_by_req(graph_path, module, grades, class_perf)
    print('  performance for a single requirement by classroom and grade.')
    print('Done.')

def plot_grade(path, module, grade, class_perf, class_distr):
    df = class_perf
    df2 = class_distr

    pdf = PdfPages(path + module + '-' + grade + '.pdf')

    df = df.loc[df['Grade'] == grade]

    by_req_data = df.iloc[:, 4:len(df.columns) - 1]

    tIDs = list(df['Teacher ID'])
    cIDs = list(df['Studio ID'])
    class_sizes = list(df.iloc[:, 3])

    # construct per-requirement graph
    plt.subplots(figsize=(10, 6))
    plt.title((module + ' grade ' + grade + ' Classroom Performance Per Requirement').title())

    req_count = len(by_req_data.columns)

    class_count = len(by_req_data.index)
    pos = list(range(req_count))
    width = .7 / (class_count)

    labels = [tID + '-' + str(cID) + ' (n=' + str(class_size) + ')' for (tID, cID, class_size) in zip(tIDs, cIDs, class_sizes)]

    # plotting a set of bars for each grade level
    for i, (index, row) in enumerate(by_req_data.iterrows()):
        plt.bar(pos if i == 0 else [p + width * i for p in pos],
            row,
            width * .9,
            alpha=0.7,
            color=colors[tIDs[i]],
            label=labels[i],
            zorder=3)
    plt.xlabel('Requirements')
    plt.xticks([p + (width * .9 * class_count / 2) for p in pos], labels=[p + 1 for p in pos])
    plt.ylabel('Percent Complete')
    plt.ylim([0, 100])
    plt.grid(axis='y', zorder=0, which='both')
    plt.legend(bbox_to_anchor=(1.04, 1), loc='upper right', ncol=1)

    plt.savefig(path + 'pngs/' + module + '-' + grade + '-per-req.png', bbox_inches='tight')
    pdf.savefig(bbox_inches='tight')
    plt.close()

    # construct totals graph
    if class_count > 1:
        total_data = df['Total']
        plt.subplots(figsize=(10, 6))
        plt.title((module + ' grade ' + grade + ' Classroom Performance Totals').title())
        plt.bar(
            list(range(class_count)),
            total_data,
            alpha=0.7,
            color=[colors[tID] for tID in tIDs],
            zorder=3
        )
        plt.xlabel('Classroom')
        plt.xticks(list(range(class_count)), labels=labels, rotation=20, horizontalalignment='right')
        plt.ylabel('Percent Complete')
        plt.ylim([0, 100])
        plt.grid(axis='y', zorder=0, which='both')

        plt.savefig(path + 'pngs/' + module + '-' + grade + '-totals.png', bbox_inches='tight')
        pdf.savefig(bbox_inches='tight')
        plt.close()

    df2 = df2.loc[df2['Grade'] == grade]
    distr_data = df2.iloc[:, 4:]

    # construct distribution graph
    fig, axs = plt.subplots(math.ceil(class_count / 2), 2, figsize=(10, 5 * math.ceil(class_count / 2)))

    for i, (ax, (index, row)) in enumerate(zip(axs.reshape(-1), distr_data.iterrows())):
        pos = list(distr_data.columns)
        ax.bar(pos, row, .95,
            alpha=0.7, zorder=3, color=colors[tIDs[i]])

        if class_count > 1:
            ax.set_title(('Classroom ' + str(cIDs[i])).title())
        else:
            ax.set_title(('Classroom ' + str(cIDs[i]) + ' Distribution').title())

        ax.set_xlabel('Score')
        ax.set_xticks(pos)
        ax.set_xticklabels = row.index

        ax.set_ylabel('Students')
        ax.yaxis.set_major_locator(MaxNLocator(integer=True))

        ax.grid(axis='y', zorder=0, which='major', alpha=0.5)

    if class_count % 2 != 0:
        axs.reshape(-1)[-1].remove()
    if class_count > 1:
        fig.suptitle('Grade Distributions')

    plt.savefig(path + 'pngs/' + module + '-' + grade + '-distributions.png', bbox_inches='tight')
    pdf.savefig(bbox_inches='tight')

    pdf.close()
    plt.close()

# plots teacher analysis using TOTALS column from CLASS PERFORMANCE PER REQUIREMENT
def plot_module(path, module, grades, class_perf):
    df = class_perf
    totals_data = [df.loc[df['Grade'] == grade] for grade in grades]

    # construct graph measuring performance by classroom and grade
    fig, axs = plt.subplots(1, len(grades), sharey=True, sharex=True, figsize=(3 * len(grades), 6))
    fig.suptitle((module + ' Requirement Completion by Classroom and Grade').title())
    fig.text(0.5, 0.05, 'Grades', ha='center', va='center')
    fig.text(0.08, 0.5, 'Classroom Completion (%)', ha='center', va='center', rotation='vertical')

    for i, [d, ax] in enumerate(zip(totals_data, axs)):
        bars = len(d.index)
        labels = [list(d['Teacher ID'])[i] + '-' + str(list(d['Studio ID'])[i]) + ' (n=' + str(list(d.iloc[:, 3])[i]) + ')' for i in range(bars)]
        bar = ax.bar(
            list(range(bars)),
            d['Total'],
            .9,
            color=[colors[tID] for tID in list(d['Teacher ID'])],
            alpha=0.7,
            zorder=3
        )
        ax.set_ylim([0, 100])
        ax.set_xlabel(grades[i])
        ax.tick_params(labelbottom=False)
        ax.legend(bar, labels, bbox_to_anchor=(1, -0.1), loc='upper right', ncol=1)

    plt.savefig(path + 'pngs/' + module + '-by-classroom.png', bbox_inches='tight')
    plt.savefig(path + module + '-teacher-analysis.pdf', bbox_inches='tight')
    plt.close()

# plots teacher analysis by requirement
def plot_module_by_req(path, module, grades, class_perf):
    df = class_perf
    totals_data = [df.loc[df['Grade'] == grade] for grade in grades]

    for req_index in range(4,len(df.columns)-1):
        # construct graph measuring performance by classroom and grade
        fig, axs = plt.subplots(1, len(grades), sharey=True, sharex=True, figsize=(3 * len(grades), 6))
        fig.suptitle((module + ' Single Requirement Completion by Classroom and Grade').title())
        fig.text(0.5, 0.93, 'Requirement: ' + df.columns[req_index] + ' [' + str(req_index-3) + ']', ha='center', va='center')
        fig.text(0.5, 0.05, 'Grades', ha='center', va='center')
        fig.text(0.08, 0.5, 'Classroom Completion for Requirement (%)', ha='center', va='center', rotation='vertical')

        for i, [d, ax] in enumerate(zip(totals_data, axs)):
            bars = len(d.index)
            labels = [list(d['Teacher ID'])[i] + '-' + str(list(d['Studio ID'])[i]) + ' (n=' + str(list(d.iloc[:, 3])[i]) + ')' for i in range(bars)]
            bar = ax.bar(
                list(range(bars)),
                d.iloc[:,req_index],
                .9,
                color=[colors[tID] for tID in list(d['Teacher ID'])],
                alpha=0.7,
                zorder=3
            )
            ax.set_ylim([0, 100])
            ax.set_xlabel(grades[i])
            ax.tick_params(labelbottom=False)
            ax.legend(bar, labels, bbox_to_anchor=(1, -0.1), loc='upper right', ncol=1)

        plt.savefig(path + 'pngs/' + module + "-req" + str(req_index-3) + '-analysis.png', bbox_inches='tight')
        plt.savefig(path + module + "-req" + str(req_index-3) + '-analysis.pdf', bbox_inches='tight')
        plt.close()

if __name__ == '__main__':
    plot(sys.argv[1], sys.argv[2])


import json
import os
import datetime

from analysis.analysis_constants import (
    REPLACE_LIST, USER_DATA_PATH, EDIT_TYPES,
    MSG_TYPES, HIGH_LEVEL_PROCESS, EDITING_PROCESS,
    PARTICIPANT_DATABASE, EDIT_TYPES,
)

class Participant:
    def __init__(self, email, id, order, study, videos, baseline):
        self.email = email
        self.id = id
        self.order = order
        self.study_id = study
        self.videos = videos
        self.baseline = baseline
        self.json_files = []
        self.found_file = False
        for filename in os.listdir(USER_DATA_PATH):
            if filename.endswith(".json"):
                if filename.startswith(email):
                    self.json_files.append(os.path.join(USER_DATA_PATH, filename))

        self.data = {}
        if len(self.json_files) == 0:
            self.found_file = False
        else:
            self.found_file = True
            for json_file in self.json_files:
                with open(json_file, "r") as f:
                    cur_data = json.load(f)
                    log_length = len(cur_data.get("logs", []))
                    cur_log_length = len(self.data.get("logs", []))
                    if log_length > cur_log_length:
                        # print("!!!", json_file, log_length)
                        self.data = cur_data

            self.adjust_data()

    def exists(self):
        return self.found_file

    def video_id(self):
        return self.videos[0]

    def video_name(self):
        if self.video_id() == "video-1":
            return "cooking"
        elif self.video_id() == "video-2":
            return "talking-head"
        return "unknown"

    ### user logs
    def logs(self):
        return self.data.get("logs", [])

    def get_specific_logs(self, msg):
        logs = self.logs()
        processing_logs = []
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            if log["msg"] == msg:
                processing_logs.append(log)
        return processing_logs

    def get_unique_msgs(self):
        logs = self.logs()
        unique_msgs = []
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            if log["msg"] not in unique_msgs:
                unique_msgs.append(log["msg"])
        return unique_msgs

    def get_unqiue_msg_types(self):
        unique_msgs = self.get_unique_msgs()
        unique_msg_types = {}
        for msg in unique_msgs:
            msg_type = msg[0:4]
            if msg_type not in unique_msg_types:
                unique_msg_types[msg_type] = []
            unique_msg_types[msg_type].append(msg)
        return unique_msg_types

    def adjust_data(self):
        logs = self.logs()
        new_logs = []
        for log in logs:
            if self.email == "kh.mukashev@gmail.com":
                ## skip logs after 12:00:00
                date = datetime.datetime.fromtimestamp(log["time"] / 1000)
                if date.hour >= 12:
                    continue
            if self.email == "ysamargarita2002@gmail.com":
                date = datetime.datetime.fromtimestamp(log["time"] / 1000)
                if date.hour >= 18 and date.minute >= 30:
                    continue
            if self.email == "kamila240373@gmail.com":
                date = datetime.datetime.fromtimestamp(log["time"] / 1000)
                if date.month < 10 or date.day < 1 or date.day > 3 or date.hour < 15 or date.hour > 17:
                    continue
            if log["msg"] in REPLACE_LIST:
                if REPLACE_LIST[log["msg"]] == None:
                    continue
                else:
                    log["msg"] = REPLACE_LIST[log["msg"]]
            new_logs.append(log)
        self.data["logs"] = new_logs

    def count_msg(self, msg):
        logs = self.logs()
        count = 0
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            if log["msg"] == msg:
                count += 1
        return count

    def count_msg_type(self, msg_type):
        logs = self.logs()
        count = 0
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            if log["msg"].startswith(msg_type):
                count += 1
        return count

    def get_processing_logs(self):
        processing_logs = self.get_specific_logs("processingComplete")
        formatted_logs = []
        for log in processing_logs:
            date = datetime.datetime.fromtimestamp(log["time"] / 1000)
            formatted_date = date.strftime("%H:%M:%S")
            formatted_logs.append({
                "intent_id": log["data"]["intentId"],
                "text": log["data"]["text"],
                "sketch": log["data"]["sketch"],
                "mode": log["data"]["mode"],
                "explanation": json.dumps(log["data"]["explanation"], separators=(',', ': ')),
                "formatted_time": formatted_date,
                "time": log["time"],
            })
        formatted_logs.sort(key=lambda x: x["time"])
        return formatted_logs

    def count_processing_logs(self):
        PROCESSING_COMPLETE_MSG = "processingComplete"
        logs = self.logs()
        count = 0
        count_iterations = 0
        count_processings = {}

        combined_data = []
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            if log["msg"] == PROCESSING_COMPLETE_MSG:
                data = log["data"]
                if data["intentId"] not in count_processings.keys():
                    count_processings[data["intentId"]] = 0
                count_processings[data["intentId"]] += 1
                combined_data.append(data)

        iterations_list = [num - 1 for num in count_processings.values()]
        count_iterations = sum(iterations_list)
        count = sum(count_processings.values())

        processed_data = []
        total_count_edits = 0
        count_from_scratch = 0
        count_add_more = 0
        count_has_text = 0
        count_has_sketch = 0
        count_has_text_and_sketch = 0
        for data in combined_data:
            point = {
                "text": data["text"],
                "hasSketch": len(data["sketch"]) > 0,
                "count_edits": len(data["edits"]),
                "start": data["start"],
                "finish": data["finish"],
                "mode": data["mode"],
            }
            processed_data.append(point)
            total_count_edits += len(data["edits"])
            if data["mode"] == "from-scratch":
                count_from_scratch += 1
            elif data["mode"] == "add-more":
                count_add_more += 1

            if len(data["text"]) > 0:
                count_has_text += 1
            if len(data["sketch"]) > 0:
                count_has_sketch += 1
            if len(data["text"]) > 0 and len(data["sketch"]) > 0:
                count_has_text_and_sketch += 1
        return {
            "total": count,
            "total_count_suggested": total_count_edits,
            "total_count_iterations": count_iterations,
            "iterations_list": iterations_list,
            "count_from_scratch": count_from_scratch,
            "count_add_more": count_add_more,
            "count_has_text": count_has_text,
            "count_has_sketch": count_has_sketch,
            "count_has_text_and_sketch": count_has_text_and_sketch,
        }

    def count_decision(self):
        REJECT_MSG = "timelineDecisionReject"
        ACCEPT_MSG = "timelineDecisionAccept"
        logs = self.logs()
        reject_count = 0
        accept_count = 0
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            if log["msg"] == REJECT_MSG:
                reject_count += 1
            elif log["msg"] == ACCEPT_MSG:
                accept_count += 1
        return {
            "reject": reject_count,
            "accept": accept_count,
        }

    ### user tasks
    def tasks(self):
        return self.data.get("tasks", [])

    def get_task(self, task_id):
        tasks = self.tasks()
        for task in tasks:
            if task["projectMetadata"]["projectId"] == task_id:
                return task
        return None

    def get_all_intents(self):
        return self.data.get("intents", [])

    def get_intent(self, intent_id):
        all_intents = self.get_all_intents()
        for intent in all_intents:
            if intent["id"] == intent_id:
                return intent

    def get_task_summary(self, task_id):
        accepted_suggestion_ids = []
        logs = self.logs()
        for log in logs:
            if log["msg"] == "timelineDecisionAccept" and log["videoId"] == self.video_id():
                data = log["data"]
                if "addedEdits" in data:
                    accepted_suggestion_ids.extend(data["addedEdits"])

        task = self.get_task(task_id)
        intents = task["intents"]

        count_intents = len(intents)
        count_edits = 0
        count_history_points = 0
        count_edits_from_suggestion = 0
        edit_types_count = {}

        intents_data = []
        for intent_id in intents:
            intent = self.get_intent(intent_id)
            intents_data.append({
                "intent_id": intent_id,
                "edit_type": intent["editOperationKey"],
                "count_edits": len(intent["activeEdits"]),
                "history_points": len(intent["history"]),
                "count_suggestions": len(intent["suggestedEdits"]),
            })
            count_edits += len(intent["activeEdits"])
            count_history_points += len(intent["history"]) - 1

            edits_from_suggested = []
            for edit_id in intent["activeEdits"]:
                if edit_id in accepted_suggestion_ids:
                    edits_from_suggested.append(edit_id)

            count_edits_from_suggestion += len(edits_from_suggested)

            edit_type = intent["editOperationKey"]
            if edit_type not in edit_types_count:
                edit_types_count[edit_type] = {
                    "count_intents": 0,
                    "count_edits": 0,
                    "count_history_points": 0,
                    "count_edits_from_suggestion": 0,
                    "edit_type": edit_type,
                }
            edit_types_count[edit_type]["count_intents"] += 1
            edit_types_count[edit_type]["count_edits"] += len(intent["activeEdits"])
            edit_types_count[edit_type]["count_history_points"] += len(intent["history"]) - 1
            edit_types_count[edit_type]["count_edits_from_suggestion"] += len(edits_from_suggested)

        return {
            "count_intents": count_intents,
            "count_edits": count_edits,
            "count_history_points": count_history_points,
            "count_edits_from_suggestion": count_edits_from_suggestion,
            "edit_types_count": edit_types_count,
            #"intents": intents_data,
        }

    def get_edit_process(self, high_level = False):
        logs = self.logs()
        process = []
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            process.append({
                "msg": log["msg"].strip(),
                "data": log["data"],
                "time": log["time"],
            })
        process.sort(key=lambda x: x["time"])

        # combine the same logs near each other
        for log in process:
            found_edit_types = []
            for edit_type in EDITING_PROCESS.keys():
                if log["msg"] in EDITING_PROCESS[edit_type]:
                    found_edit_types.append(edit_type)
            if len(found_edit_types) == 0:
                log["msg"] = "unknown"
            else:
                found_edit_type = ""
                if len(found_edit_types) > 1:
                    if "suggested" in log["data"] and log["data"]["suggested"] == True:
                        found_edit_type = "examine"
                    else:
                        found_edit_type = "manual"
                    #print("!!!", log["msg"], found_edit_types)
                else:
                    found_edit_type = found_edit_types[0]
                if high_level:
                    log["msg"] = HIGH_LEVEL_PROCESS[found_edit_type]
                else:
                    log["msg"] = found_edit_type

        # filter out anything or unknown
        new_process = []
        for log in process:
            if log["msg"] == "anything" or log["msg"] == "unknown":
               continue
            new_process.append(log)
        process = new_process

        combined_process = []
        for log in process:
            if len(combined_process) == 0:
                combined_process.append(log)
                continue
            last_log = combined_process[-1]
            if last_log["msg"] == log["msg"]:
                last_log["time"] = log["time"]
            else:
                combined_process.append(log)

        # add duration
        for log in combined_process:
            log["duration"] = 0
        for i in range(1, len(combined_process)):
            log = combined_process[i]
            prev_log = combined_process[i - 1]
            log["duration"] = log["time"] - prev_log["time"]

        formatted_process = []
        for log in combined_process:
            date = datetime.datetime.fromtimestamp(log["time"] / 1000)
            duration = datetime.datetime.fromtimestamp(log["duration"] / 1000)
            formatted_date = date.strftime("%H:%M:%S")
            formatted_duration = duration.strftime("%M:%S")

            formatted_process.append({
                "msg": log["msg"],
                "date": formatted_date,
                "duration": formatted_duration,
                "time": log["time"],
            })
        return formatted_process


from typing import List

class ParsingError(Exception):
    pass

# 'IF' isn't reserved... treat it as a normal internal function.
RESERVED_WORDS = (['ALL', 'AND', 'ANY', 'ARRAY', 'AS', 'ASC',
                   'ASSERT_ROWS_MODIFIED', 'AT', 'BETWEEN', 'BY', 'CASE',
                   'CAST', 'COLLATE', 'CONTAINS', 'CREATE', 'CROSS',
                   'CUBE', 'CURRENT', 'DEFAULT', 'DEFINE', 'DESC',
                   'DISTINCT', 'ELSE', 'END', 'ENUM', 'ESCAPE', 'EXCEPT',
                   'EXCLUDE', 'EXISTS', 'EXTRACT', 'FALSE', 'FETCH',
                   'FOLLOWING', 'FOR', 'FROM', 'FULL', 'GROUP', 'GROUPING',
                   'GROUPS', 'HASH', 'HAVING', 'IGNORE', 'IN', 'INNER',
                   'INTERSECT', 'INTERVAL', 'INTO', 'IS', 'JOIN',
                   'LATERAL', 'LEFT', 'LIKE', 'LIMIT', 'LOOKUP', 'MERGE',
                   'NATURAL', 'NEW', 'NO', 'NOT', 'NULL', 'NULLS', 'OF',
                   'ON', 'OR', 'ORDER', 'OUTER', 'OVER', 'PARTITION',
                   'PRECEDING', 'PROTO', 'RANGE', 'RECURSIVE', 'RESPECT',
                   'RIGHT', 'ROLLUP', 'ROWS', 'SELECT', 'SET', 'SOME',
                   'STRUCT', 'TABLESAMPLE', 'THEN', 'TO', 'TREAT', 'TRUE',
                   'UNBOUNDED', 'UNION', 'UNNEST', 'USING', 'WHEN', 'MINUS',
                   'WHERE', 'WINDOW', 'WITH', 'WITHIN', 'QUALIFY', 'RECORDS'])

class SQLLexer:

    def __init__(self, sql_str):
        self._str = sql_str
        self._pos = 0
        self._comments = []

        self.consume_all_space()

    CONTEXT = 25

    def error(self, msg):

        # Find beginning and end of snippet.
        pos = self._pos
        linec = self._str[0:pos].count('\n') + 1
        line_start = 0
        if linec > 1:
            line_start = self._str[0:pos + 1].rindex('\n') + 1

        snippet_beg = int(max(line_start, pos - SQLLexer.CONTEXT))
        try:
            snippet_end = int(
                min(self._str[line_start:].index('\n') + line_start + 1,
                    pos + SQLLexer.CONTEXT))
        except ValueError:
            snippet_end = int(min(len(self._str), pos + SQLLexer.CONTEXT))

        # Raise exception
        raise ParsingError('\n'.join([
            f'{msg} at [{linec}:{pos - line_start + 1}]',
            f'Snippet: {self._str[snippet_beg:snippet_end]}',
            f'         {"-" * (pos - snippet_beg)}^'
        ]))

    def get_comments(self) -> List[str]:
        rcomments = self._comments
        self._comments = []
        return rcomments

    def peek(self, elem):
        """Peek for an element.

        Args:
          elem: Element. If a string, an identifier is searched for. Otherwise
              an operator.

        Returns:
          Returns True/False if the element is available to consume.
        """
        if isinstance(elem, list):
            rpos = self._pos
            for el in elem:
                if not self.peek(el):
                    self._pos = rpos
                    return False
                self._pos += len(el)
                self.consume_all_space()
            self._pos = rpos
            return True

        # If elem does not start with an alpha character,
        # just check if it matches the next available text
        if not elem[0].isalpha():
            return self._str[self._pos:self._pos + len(elem)] == elem

        # If it *doesn't* match the upper case of next available next,
        # then it doesn't match.
        if self._str[self._pos:self._pos + len(elem)].upper() != elem:
            return False

        # If this is the end of the string, it's a match
        if self._pos + len(elem) >= len(self._str):
            return True

        # Check the next character -- if it's not an alpha character,
        # and not a digit or underscore then we have a match.
        ch = self._str[self._pos + len(elem)]
        if not ch.isalpha() and not ch.isdigit() and ch != '_':
            return True

        return False

    def consume_any(self, elems):
        """Consume any of a list of elements.

        Args:
            elems: List of elements
                See peek() for what is an element
                Tested by consume(elem)

        Returns:
            Returns elem from elems that was consumed or None
        """
        for e in elems:
            if self.consume(e):
                return e
        return None

    def consume(self, elem):
        """Consume for an element or list of elements.

        Args:
          elem(s): Element or list of elements.
              See peek() for what is an element.
              If a list, then all elements must be consumed to be satisfied.

        Returns:
          Returns True/False if the element(s) is/are available to consume.
        """
        if isinstance(elem, list):
            rpos = self._pos
            for e in elem:
                if not self.consume(e):
                    self._pos = rpos
                    return None
            return elem

        if self.peek(elem):
            self._pos += len(elem)
            self.consume_all_space()
            return elem

        return None

    def expect(self, elem):
        return (self.consume(elem) or
                self.error('Expected "' + elem + '"'))

    def expect_end(self):
        if self._pos < len(self._str):
            self.error('Expected end')

    def consume_all_space(self):
        """Consume all space and comments.

        Returns:
          This moves the position forward to the next non-space non-comment.
        """
        while self._pos < len(self._str):

            if self._str[self._pos].isspace():
                self._pos += 1
                continue

            # It's a SQL comment
            if self._str[self._pos:self._pos + 2] == '--':
                start = self._pos + 2
                try:
                    move = self._str[self._pos:].index('\n')
                except ValueError:
                    move = len(self._str) - self._pos
                self._comments.append(self._str[start:start + move - 2].strip())
                self._pos += move
                continue

            # It's a hash comment
            if self._str[self._pos] == '#':
                try:
                    move = self._str[self._pos:].index('\n')
                except ValueError:
                    move = len(self._str) - self._pos
                self._pos += move
                continue

            # It's a /* */ comment
            if self._str[self._pos:self._pos + 2] == '/*':
                start = self._pos + 2
                while self._pos < len(self._str):
                    if self._str[self._pos:self._pos + 2] == '*/':
                        self._comments.append(
                            self._str[start:self._pos].strip())
                        self._pos += 2
                        break
                    self._pos += 1
                continue

            # Something else - stop now!
            break

    def consume_identifier(self):
        """Consume identifier.

        Returns:
          This returns an identifier (that's not reserved) or None if
          one isn't available.
        """
        if self._pos >= len(self._str):
            return None

        is_escaped = None
        ch = self._str[self._pos]
        if ch in ('`', '"'):
            is_escaped = ch

        if not (is_escaped or
                ch.isalpha() or
                ch == ':' or
                ch == '_'):
            return None

        pos = self._pos

        if is_escaped:
            pos += 1

        v = []

        if ch == ':':
            v.append(ch)
            pos += 1

        while pos < len(self._str):
            ch = self._str[pos]
            if is_escaped:
                if ch == is_escaped:
                    break
            else:
                if not (ch.isalpha() or ch == '_' or ch.isdigit()):
                    break
            v.append(ch)
            pos += 1

        v = ''.join(v)
        if v.upper() in RESERVED_WORDS:
            return None

        if is_escaped:
            pos += 1

        self._pos = pos

        self.consume_all_space()

        return v

    def consume_number(self):
        """Consume number.

        Returns:
          This returns a number (float or integer) or None
          if one isn't available.
        """
        if self._pos == len(self._str):
            return None

        if not self._str[self._pos].isdigit():
            return None

        # Do the parsing itself.
        pos = self._pos
        is_frac = False
        divisor = 1
        num = 0
        while pos < len(self._str):
            ch = self._str[pos]
            if ch.isdigit():
                num *= 10
                num += int(ch)
                if is_frac:
                    divisor *= 10
            elif not is_frac and ch == '.':
                is_frac = True
            else:
                break
            pos += 1

        # TODO(scannell) need 'e'

        # TODO(scannell) This is awkward. Fix.
        self._pos = pos
        self.consume_all_space()

        if is_frac:
            return float(num) / divisor

        return num

    def consume_string(self):
        """Consume string.

        Returns:
          This returns a string or None if one isn't available.
        """
        spos = self._pos
        flag = None

        if spos == len(self._str):
            return None

        if self._str[spos] == 'b' or self._str[spos] == 'r':
            flag = self._str[spos]
            spos += 1

        if spos == len(self._str) or self._str[spos] not in ('"', '\''):
            return None

        if self._str[spos:spos + 3] == '"""':
            quotechar = self._str[spos:spos + 3]
        else:
            quotechar = self._str[spos]

        spos += len(quotechar)
        epos = spos
        while (epos < len(self._str) and
               self._str[epos:epos + len(quotechar)] != quotechar):
            epos += 1

        if epos == len(self._str):
            self.error('Expected string')

        self._pos = epos + len(quotechar)

        self.consume_all_space()

        return (self._str[spos:epos], flag)


import sys
import os
import codecs
import re
import numpy as np
import pandas as pd
from keras_bert import load_trained_model_from_checkpoint, Tokenizer
from find_redactions import clean_dataframe, get_line_pairs, find_repeated_substring

DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'data')

REDACTION_MARKERS = [
    '[Harm to Ongoing Matter - Grand Jury]',
    '[Harm to Ongoing Matter - Personal Privacy]',
    '[Harm to Ongoing Matter]',
    '[Personal Privacy - Grand Jury]',
    '[Personal Privacy]',
    '[HOM]',
]

REDACTION_MARKER = REDACTION_MARKERS[2]
REDACTION_BERT_TOKEN = '[MASK]'

UNZIPPED_MODEL_PATH = os.path.join(DATA_DIR, 'uncased_L-12_H-768_A-12')

if len(sys.argv) == 2:
    UNZIPPED_MODEL_PATH = sys.argv[1]

UNZIPPED_MODEL_PATH = (
    os.path.abspath(os.path.expanduser(os.environ.get('UNZIPPED_MODEL_PATH') or UNZIPPED_MODEL_PATH)))

print(f'UNZIPPED_MODEL_PATH: {UNZIPPED_MODEL_PATH}')

BERT_MODEL_CASED = 'uncased' not in UNZIPPED_MODEL_PATH.lower()

global P
P = None

def get_unredacted_sentences(df='mueller-report-with-redactions-marked.csv',
                             min_line_length=60, max_line_length=150):
    df = clean_dataframe(df) if isinstance(df, str) else df
    line_pairs = get_line_pairs(df, min_line_length=min_line_length, max_line_length=max_line_length)
    print(pd.DataFrame(line_pairs, columns='text redacted_text'.split()).head())

    for i, (text, redacted) in enumerate(line_pairs):
        if len(re.findall(r'^[-:.0-9 \t]{1,2}', text.strip())) > 0:
            print(f'Skipping: {text[:30]}')
            continue
        print(f"Redacting: {text[:30]}")
        print()

# (TEXT, REDACTION_MARKER, PAGE_NUM, NUM_WORDS_IN_REDACTION)
TEXTS = [
    ('''
    The presidential campaign of Donald J. Trump ("Trump Campaign" or "Campaign") showed
    interest in WikiLeaks\'s releases of documents and welcomed their potential to damage
    candidate Clinton. Beginning in June 2016, [Harm to Ongoing Matter] forecast to senior
    Campaign officials that WikiLeaks would release information damaging to candidate Clinton.
    WikiLeaks\'s first release came in July 2016. Around the same time, candidate Trump announced
    that he hoped Russia would recover emails described as missing from a private server used by
    linton when she was Secretary of State (he later said that he was speaking sarcastically).
    ''', '[Harm to Ongoing Matter]', 5, 4),

    ('''
    On October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special
    Counsel's investigative authority as to several individuals and entities. First, as part of
    a full and thorough investigation of the Russian government's efforts to interfere in the
    2016 presidential election," the Special Counsel was authorized to investigate the pertinent
    ctivities of Michael Cohen, Richard Gates, [Personal Privacy] , Roger Stone, and
    ''', '[Personal Privacy]', 12, 2),

    ('''By February 2016, internal IRA documents referred to support for the Trump Campaign
    and opposition to candidate Clinton.49 For example, [Harm to Ongoing Matter] directions to
    IRA operators
    ''', '[Harm to Ongoing Matter]', 23, 5),

    ('''The focus on the U.S. presidential campaign continued throughout 2016.
    In [Harm to Ongoing Matter] 2016 internal [Harm to Ongoing Matter] reviewing the
    IRA-controlled Facebook group "Secured Borders,"
    ''', '[Harm to Ongoing Matter]', 23, 3),

    ('''IRA employees frequently used Investigative Technique Twitter, Facebook, and Instagram
    to contact and recruit U.S. persons who followed the group. The IRA recruited U.S. persons
    from across the political spectrum. For example, the IRA targeted the family
    of [Personal Privacy] and a number of black social justice activists
    ''', '[Personal Privacy]', 31, 3),

    ('''
    A. GRU Hacking Directed at the Clinton Campaign 1. GRU Units Target the Clinton Campaign Two military units of the GRU
    carried out the computer intrusions into the Clinton Campaign, DNC, and DCCC: Military Units 26165 and 74455.110
    Military Unit 26165 is a GRU cyber unit dedicated to targeting military, political, governmental, and non-governmental
    organizations outside of Russia, including in the United States.111 The unit was sub-divided into departments with
    different specialties. One department, for example, developed specialized malicious software ("malware"), while another
    department conducted large-scale spearphishing campaigns. 112 Investigative Technique a bitcoin mining operation
    to 109 As discussed in Section V below, our Office charged 12 GRU officers for crimes arising from the hacking of these
    computers, principally with conspiring to commit computer intrusions, in violation of 18 U.S.C. $$1030 and 371.
    See Volume 1, Section V.B, infra; Indictment, United States v. Netyksho, No., [Investigative Technique] a bitcoin mining
    operation to secure bitcoins used to purchase computer infrastructure used in hacking operations.
    ''', "[Investigative Technique]", 36, 3),

    ('''
    Footnote: 113. Bitcoin mining consists of unlocking new bitcoins by solving computational problems. [IT] kept its
    newly mined coins in an account on the bitcoin exchange platform CEX.io. To make purchases, the GRU routed funds
    into other accounts through transactions designed to obscure the source of funds. Netyksho Indictment 62.
    ''', '[IT]', 37, 2),

    ('''
    The first set of GRIU-controlled computers, known by the GRU as "middle servers," sent and received messages to and
    from malware on the DNC/DCCC networks. The middle servers, in turn, relayed messages to a second set of
    GRU-controlled computers, labeled internally by the GRU as an "AMS Panel." The AMS Panel [Investigative Technique]
    served as a nerve center through which GRU officers monitored and directed the malware's operations on the
    DNC/DCCC networks. 127 The AMS Panel used to control X-Agent during the DCCC and DNC intrusions was housed on a
    leased computer located near IT Arizona.
    ''', '[Investigative Technique]', 39, 3),

    ('''Footnote: 140. See, e.g., Internet Archive, "https://dcleaks.com/" (archive date Nov. 10, 2016). Additionally,
    DCLeaks released documents relating to [Personal Privacy] , emails belonging to [PP] , and emails from  2015 relating
    to Republican Party employees (under the portfolio name "The United States Republican Party\').
    "The United States Republican Party" portfolio contained approximately 300 emails from a variety of GOP members, PACs,
    campaigns, state parties, and businesses dated between May and October 2015. According to open-source reporting,
    these victims shared the same Tennessee-based web-hosting company, called Smartech Corporation.'
    ''', '[Personal Privacy]', 41, 5),

    ('''Footnote: 140. See, e.g., Internet Archive, "https://dcleaks.com/" (archive date Nov. 10, 2016). Additionally,
    DCLeaks released documents relating to [PP], emails belonging to [Personal Privacy] , and emails from  2015 relating
    to Republican Party employees (under the portfolio name "The United States Republican Party\').
    "The United States Republican Party" portfolio contained approximately 300 emails from a variety of GOP members, PACs,
    campaigns, state parties, and businesses dated between May and October 2015. According to open-source reporting,
    these victims shared the same Tennessee-based web-hosting company, called Smartech Corporation.'
    ''', '[Personal Privacy]', 41, 1),
    ]

# to manually generate plausible redaction unredactions:
# df = clean_dataframe()

# if len(sys.argv) != 4:
#     print('USAGE: python load_model.py CONFIG_PATH CHECKPOINT_PATH DICT_PATH')
#     print()
#     sys.argv = [
#         sys.argv[0],
#         os.environ.get('CONFIG_PATH') or os.path.join(UNZIPPED_MODEL_PATH, 'bert_config.json'),
#         os.environ.get('CHECKPOINT_PATH') or os.path.join(UNZIPPED_MODEL_PATH, 'bert_model.ckpt'),
#         os.environ.get('DICT_PATH') or os.path.join(UNZIPPED_MODEL_PATH, 'vocab.txt')]
#     SCRIPTNAME, CONFIG_PATH, CHECKPOINT_PATH, DICT_PATH = sys.argv
#     print(f'CONFIG_PATH:     {CONFIG_PATH}')
#     print(f'CHECKPOINT_PATH: {CHECKPOINT_PATH}')
#     print(f'DICT_PATH:       {DICT_PATH}')

# config_path, checkpoint_path, dict_path = tuple(sys.argv[1:])

# model = load_trained_model_from_checkpoint(config_path, checkpoint_path, training=True)
# model.summary(line_length=120)

# token_dict = {}
# with codecs.open(dict_path, 'r', 'utf8') as reader:
#     for line in reader:
#         token = line.strip()
#         token_dict[token] = len(token_dict)
# token_dict_rev = {v: k for k, v in token_dict.items()}

# tokenizer = Tokenizer(token_dict)

class NLPPipeline(dict):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        for k, v in self.items():
            setattr(self, k, v)

def load_pipeline(UNZIPPED_MODEL_PATH=UNZIPPED_MODEL_PATH, cased=BERT_MODEL_CASED):
    if len(sys.argv) != 4:
        print('python load_model.py CONFIG_PATH CHECKPOINT_PATH DICT_PATH')
        print('CONFIG_PATH:     $UNZIPPED_MODEL_PATH/bert_config.json')
        print('CHECKPOINT_PATH: $UNZIPPED_MODEL_PATH/bert_model.ckpt')
        print('DICT_PATH:       $UNZIPPED_MODEL_PATH/vocab.txt')
        sys.argv = [
            sys.argv[0],
            os.path.abspath(os.environ.get('CONFIG_PATH') or os.path.join(UNZIPPED_MODEL_PATH, 'bert_config.json')),
            os.path.abspath(os.environ.get('CHECKPOINT_PATH') or os.path.join(UNZIPPED_MODEL_PATH, 'bert_model.ckpt')),
            os.path.abspath(os.environ.get('DICT_PATH') or os.path.join(UNZIPPED_MODEL_PATH, 'vocab.txt')),
        ]

    print(sys.argv)
    if not all([os.path.exists(p) for p in sys.argv[1:4]]):
        print("You must specify the path where you've downloaded the pretrained BERT model in $UNZIPPED_MODEL_PATH or on the commandline.")

    config_path, checkpoint_path, dict_path = tuple(sys.argv[1:])

    model = load_trained_model_from_checkpoint(config_path, checkpoint_path, training=True)
    model.summary(line_length=120)

    token_dict = {}
    with codecs.open(dict_path, 'r', 'utf8') as reader:
        for line in reader:
            token = line.strip()
            token_dict[token] = len(token_dict)
    token_dict_rev = {v: k for k, v in token_dict.items()}
    if cased:
        print('***************CASED TOKENIZER*******************')
    else:
        print('***************uncased tokenizer*******************')
    tokenizer = Tokenizer(token_dict, cased=cased)

    return NLPPipeline(model=model, token_dict=token_dict, token_dict_rev=token_dict_rev, tokenizer=tokenizer)

def find_first_hom_tokens(df, text=None, substring='of documents and', marker='[Personal Privacy]'):
    global P
    P = P or load_pipeline()
    if not text:
        df = clean_dataframe(df) if isinstance(df, str) else df
        for t in df.text:
            if substring in t:
                text = t
                break
    # print(f'TEXT: {text}')
    tokens = P.tokenizer.tokenize(text)
    joined_tokens = ' '.join(tokens)
    # print(f'joined_tokens: {joined_tokens}')
    hom = ' '.join(P.tokenizer.tokenize(marker)[1:-1])
    # print(f'joined_hom: {hom}')
    hom_start = joined_tokens.find(hom)
    hom_stop = hom_start + len(hom)
    # print(f'hom_start: {hom_start}, hom_stop: {hom_stop}')
    prefix_tokens = joined_tokens[:hom_start].split()
    suffix_tokens = joined_tokens[hom_stop:].split()
    # print(f'HOM prefix_tokens: {prefix_tokens}\nHOM suffix_tokens: {suffix_tokens}')

    return prefix_tokens, suffix_tokens

sentences = [
    'The IRA later used social media accounts and interest groups to sow discord in the U.S. political system through what it termed "information warfare."',
    ('The campaign evolved from a generalized program designed in 2014 and 2015 to undermine the U.S. electoral system,' +
        'to a targeted operation that by early 2016 favored candidate Trump and disparaged candidate Clinton.'),
    ('The IRA\'s operation also included the purchase of political advertisements on social media in the names of U.S. persons and entities,' +
        'as well as the staging of political rallies inside the United States.'),
    ('To organize those rallies, IRA employees posed as U.S. grassroots entities and persons and made contact' +
        ' with Trump supporters and Trump Campaign officials in the United States.'),
    ]

MASK_TOKEN = '[MASK]'

def unredact_tokens(prefix_tokens=[], suffix_tokens=[], num_redactions=5):
    global P
    if not P:
        P = load_pipeline()
    tokens = list(prefix_tokens) + [MASK_TOKEN] * num_redactions + list(suffix_tokens)
    tokens = tokens[:512]
    tokens_original = tokens.copy()
    # text = ' '.join(tokens)
    # print(f"Predicting {num_redactions} MASK tokens in: {' '.join(tokens)}")

    indices = np.asarray([[P.token_dict[token] for token in tokens] + [0] * (512 - len(tokens))])
    segments = np.asarray([[0] * len(tokens) + [0] * (512 - len(tokens))])
    masks = np.asarray([[0] * 512])
    redactions = []
    for i, t in enumerate(tokens):
        if t == MASK_TOKEN:
            redactions.append(i - 1)
            masks[0][i] = 1

    # masks = np.asarray([[0, 1, 1] + [0] * (512 - 3)])

    predicts = P.model.predict([indices, segments, masks])[0]
    predicts = np.argmax(predicts, axis=-1)
    predictions_parameterized = list(
        map(lambda x: P.token_dict_rev[x],
            [x for (j, x) in enumerate(predicts[0]) if j - 1 in redactions])
        )
    # predictions_hardcoded = list(map(lambda x: token_dict_rev[x], predicts[0][3:5]))
    print(f'Predictions: {predictions_parameterized}')

    # print(f'Hardcoded fill with: {predictions_hardcoded}')
    # list(map(lambda x: token_dict_rev[x], predicts[0][1:3]))
    print(f'.    Actual: {[t for (i, t) in enumerate(tokens_original) if i - 1 in redactions]}')
    print()
    print()
    # if len(predictions) > 10:
    #     break
    return (predictions_parameterized, tokens)

def unredact_text(text, redactions=[2, 3]):
    print(f"Redacting tokens {redactions} in: {text}")
    global P
    P = P or load_pipeline()

    tokens = P.tokenizer.tokenize(text)
    tokens_original = tokens.copy()
    for r in redactions:
        tokens[r + 1] = MASK_TOKEN

    # print(f'Tokens: {tokens}')

    indices = np.asarray([[P.token_dict[token] for token in tokens] + [0] * (512 - len(tokens))])
    segments = np.asarray([[0] * len(tokens) + [0] * (512 - len(tokens))])
    masks = np.asarray([[0] * 512])
    for r in redactions:
        masks[0][r + 1] = 1
    # masks = np.asarray([[0, 1, 1] + [0] * (512 - 3)])

    predicts = P.model.predict([indices, segments, masks])[0]
    predicts = np.argmax(predicts, axis=-1)
    predictions_parameterized = list(
        map(lambda x: P.token_dict_rev[x],
            [x for (j, x) in enumerate(predicts[0]) if j - 1 in redactions])
        )
    # predictions_hardcoded = list(map(lambda x: token_dict_rev[x], predicts[0][3:5]))
    print(f'Predictions: {" ".join(predictions_parameterized)}')

    # print(f'Hardcoded fill with: {predictions_hardcoded}')
    # list(map(lambda x: token_dict_rev[x], predicts[0][1:3]))
    print(f'.    Actual: {[t for (i, t) in enumerate(tokens_original) if i - 1 in redactions]}')
    print()
    print()
    # if len(predictions) > 10:
    #     break
    return (predictions_parameterized, text)

def unredact_examples(examples=TEXTS):
    for text, marker, page, num_redactions in examples:
        print('\n\n****************************')
        prefix_tokens, suffix_tokens = find_first_hom_tokens(df=None, text=text, marker=marker)
        print(f'page: {page}\nnum_words: {num_redactions}\nprefix_tokens: {prefix_tokens}\nsuffix_tokens: {suffix_tokens}')
        print(unredact_tokens(prefix_tokens=prefix_tokens, suffix_tokens=suffix_tokens, num_redactions=num_redactions))
        print('\n\n****************************')

    predictions = []
    for sentnum, text in enumerate(sentences):
        print(sentnum)
        predictions.append(unredact_text(text))

def unredact_interactively():
    global P
    if not P:
        P = load_pipeline()
    unredacted = ' '
    while unredacted:
        text = input('Text: ')
        marker = input('Redaction marker: ')
        marker = marker or 'unk'
        redactions = find_repeated_substring(text, substring=marker)
        if not redactions:
            print('No redactions found')
            unredacted = text
            continue
        # print(redactions)
        start, stop = redactions[0], redactions[-1] + len(marker)
        prefix, suffix = text[:start], text[stop:]
        # print(start, stop)
        # print(f'prefix: {prefix}')
        # print(f'suffix: {suffix}')
        prefix_tokens = P.tokenizer.tokenize(prefix)[:-1]
        suffix_tokens = P.tokenizer.tokenize(suffix)[1:]
        # print(f'prefix_tokens: {prefix_tokens}')
        # print(f'suffix_tokens: {suffix_tokens}')
        unredacted_tokens, all_tokens = unredact_tokens(prefix_tokens=prefix_tokens, suffix_tokens=suffix_tokens, num_redactions=len(redactions))
        print(f'all_tokens: {all_tokens}')
        print(f'unredacted_tokens: {unredacted_tokens}')
        j = 0
        for (i, tok) in enumerate(all_tokens):
            if tok == '[MASK]' and j < len(unredacted_tokens):
                all_tokens[i] = unredacted_tokens[j]
                j += 1

        unredacted = ' '.join(all_tokens)
        # unredacted = ' '.join([t[2:] if t.startswith('##') else t for t in unredacted_tokens])
        print(f'Unredacted text: {unredacted}')

if __name__ == '__main__':
    unredact_interactively()

# sentence_1 = text
# sentence_2 = 'Joseph Conrad said "We live as we dream, alone." '
# print('Tokens:', tokenizer.tokenize(first=sentence_1, second=sentence_2))
# indices, segments = tokenizer.encode(first=sentence_1, second=sentence_2, max_len=512)
# masks = np.array([[0] * 512])

# predicts = model.predict([np.array([indices]), np.array([segments]), masks])[1]
# print('%s is random next: ' % sentence_2, bool(np.argmax(predicts, axis=-1)[0]))

# sentence_2 = 'Mathematicians use patterns to formulate new conjectures; they resolve the truth or falsity of conjectures with proof. '
# print('Tokens:', tokenizer.tokenize(first=sentence_1, second=sentence_2))
# indices, segments = tokenizer.encode(first=sentence_1, second=sentence_2, max_len=512)

# predicts = model.predict([np.array([indices]), np.array([segments]), masks])[1]
# print('%s is random next: ' % sentence_2, bool(np.argmax(predicts, axis=-1)[0]))


from .codenames import (
    account_manager,
    administration,
    ae,
    ae_review,
    auditor,
    celery_manager,
    clinic,
    data_manager,
    data_query,
    everyone,
    export,
    lab,
    lab_view,
    pharmacy,
    pii,
    pii_view,
    rando,
    site_data_manager,
    tmg,
)
from .group_names import (
    AE,
    AE_REVIEW,
    ACCOUNT_MANAGER,
    ADMINISTRATION,
    AUDITOR,
    CELERY_MANAGER,
    CLINIC,
    DATA_MANAGER,
    DATA_QUERY,
    EVERYONE,
    EXPORT,
    LAB,
    LAB_VIEW,
    PHARMACY,
    PII,
    PII_VIEW,
    RANDO,
    SITE_DATA_MANAGER,
    TMG,
)

default_codenames_by_group = {
    AE: ae,
    AE_REVIEW: ae_review,
    ACCOUNT_MANAGER: account_manager,
    ADMINISTRATION: administration,
    AUDITOR: auditor,
    CELERY_MANAGER: celery_manager,
    CLINIC: clinic,
    DATA_MANAGER: data_manager,
    DATA_QUERY: data_query,
    EVERYONE: everyone,
    EXPORT: export,
    LAB: lab,
    LAB_VIEW: lab_view,
    PHARMACY: pharmacy,
    PII: pii,
    PII_VIEW: pii_view,
    RANDO: rando,
    SITE_DATA_MANAGER: site_data_manager,
    TMG: tmg,
}


import numpy as np
import DipoleLatticeField
import pickle
import gzip

def tup_to_array(tup):
    mat = np.array([[tup[0], tup[1], tup[2]],[tup[3], tup[4], tup[5]],[tup[6], tup[7], tup[8]]])
    return mat

#Define SI units
pi = np.pi
mB=9.274*10**(-24)
k=1.380*10**(-23)
NA=6.022*10**23
mu0=4*pi*10**(-7)

ErLiF4 = DipoleLatticeField.Tetragonal()
ErLiF4.axes(5.162, 10.70)
ErLiF4.g_tensor(8.105, 3.147)
#Define the ions to be in the bilayered antiferromagnetic ground state, aligned along the x-axis
#Ion orientation doesn't matter when calculating the dipole tensors
ErLiF4.add_position(0.5, 0.0, 0.00, (0,0,1))
ErLiF4.add_position(0.5, 0.5, 0.25, (0,0,1))
ErLiF4.add_position(0.0, 0.5, 0.50, (0,0,1))
ErLiF4.add_position(0.0, 0.0, 0.75, (0,0,1))

try:
    f = gzip.open('ErLiF4_Jmatricies.dat.gz', 'rb')
    print("Loading matricies")
    J00mat, J01mat, J02mat, J03mat, J10mat, J11mat, J12mat, J13mat, J20mat, J21mat, J22mat, J23mat, \
    J30mat, J31mat, J32mat, J33mat = pickle.load(f)

except FileNotFoundError:
    print("Making matricies")

    J00 = ErLiF4.J_terms(200,0,0)
    J01 = ErLiF4.J_terms(200,0,1)
    J02 = ErLiF4.J_terms(200,0,2)
    J03 = ErLiF4.J_terms(200,0,3)
    J10 = ErLiF4.J_terms(200,1,0)
    J11 = ErLiF4.J_terms(200,1,1)
    J12 = ErLiF4.J_terms(200,1,2)
    J13 = ErLiF4.J_terms(200,1,3)
    J20 = ErLiF4.J_terms(200,2,0)
    J21 = ErLiF4.J_terms(200,2,1)
    J22 = ErLiF4.J_terms(200,2,2)
    J23 = ErLiF4.J_terms(200,2,3)
    J30 = ErLiF4.J_terms(200,3,0)
    J31 = ErLiF4.J_terms(200,3,1)
    J32 = ErLiF4.J_terms(200,3,2)
    J33 = ErLiF4.J_terms(200,3,3)

    J00mat = tup_to_array(J00)
    J01mat = tup_to_array(J01)
    J02mat = tup_to_array(J02)
    J03mat = tup_to_array(J03)
    J10mat = tup_to_array(J10)
    J11mat = tup_to_array(J11)
    J12mat = tup_to_array(J12)
    J13mat = tup_to_array(J13)
    J20mat = tup_to_array(J20)
    J21mat = tup_to_array(J21)
    J22mat = tup_to_array(J22)
    J23mat = tup_to_array(J23)
    J30mat = tup_to_array(J30)
    J31mat = tup_to_array(J31)
    J32mat = tup_to_array(J32)
    J33mat = tup_to_array(J33)

    ELF_Jmats = [ J00mat, J01mat, J02mat, J03mat, J10mat, J11mat, J12mat, J13mat, J20mat, J21mat, J22mat, J23mat, J30mat, J31mat, J32mat, J33mat]
    pickle.dump(ELF_Jmats, gzip.open('ErLiF4_Jmatricies.dat.gz', 'wb'))

## Jonos test stuff

# Calculate E1

xhat = np.array([1, 0, 0])
yhat = np.array([0, 1, 0])
zhat = np.array([0, 0, 1])

S1 = 0.5*xhat
S2 = -0.5*xhat
S3 = -0.5*xhat
S4 = 0.5*xhat

V = ErLiF4.a[0]*ErLiF4.b[1]*ErLiF4.c[2]*(10**(-10))**3
E1 = (-mu0*mB**2)/(8*pi*V)*S1.dot(J00mat.dot(S1) + J01mat.dot(S2) + J02mat.dot(S3) + J03mat.dot(S4))
print('E1 = ' + str(E1/k) + ' K')


import os
from splinter import Browser
from selenium import webdriver

# open headless browser
# laptop
# driver=webdriver.Firefox(executable_path='/Users/jmetzger/anaconda3/bin/geckodriver')
# des machines
driver=webdriver.Firefox(executable_path='/data/des30.a/data/annis/dae-haven/py-lib/lib/python2.7/site-packages/geckodriver/geckodriver')
browser=Browser(headless=True)
# the geckodriver executable needs to be in the environmental variable $PATH
# export PATH=$PATH:/data/des30.a/data/annis/dae-haven/py-lib/lib/python2.7/site-packages/geckodriver/

# to decimal degrees
def RA_convert(RA):
    RA=RA.split(':')
    RA=[float(x) for x in RA]
    return (RA[0] + RA[1]/60. + RA[2]/3600.)*(360/24.)

# to decimal degrees
def DEC_convert(DEC):
    DEC=DEC.split(':')
    DEC=[float(x) for x in DEC]
    if DEC[0]!=0: return (abs(DEC[0]) + DEC[1]/60. + DEC[2]/3600.)*abs(DEC[0])/DEC[0]
    else: return DEC[1]/60. + DEC[2]/3600.

#
#   Get Panstarrs template image
#       save template images to new folder
#
def get_template_image(RA,DEC,size,browser,path):
    #open PS1 query
    url='http://ps1images.stsci.edu/cgi-bin/ps1cutouts?pos='+str(RA)+'%2C'+str(DEC)+\
        '&filter=i&filetypes=stack&auxiliary=data&size='+str(size)+'&output_size=0&verbose=0&autoscale=99.500000&catlist='
    browser.visit(url)
    fitsfile=browser.find_link_by_partial_text('FITS-cutout')[0]['href']

    newfile=fits.open(fitsfile)

    # write file
    new_filename=path+'RA'+str(RA)+'_DEC'+str(DEC)+'.fits'
    try: newfile.writeto(new_filename)
    except OSError:
        try:
            os.remove(new_filename)
            newfile.writeto(new_filename)
        except OSError: print('file saving error')
    return newfile[0].data


entrada = int(input())
lista = []
dentro = fora = 0
for i in range(0, entrada):
    lista.append(int(input()))
    if 10 <= lista[i] <= 20:
        dentro += 1
    else:
        fora += 1
print(f"{dentro} in")
print(f"{fora} out")