from datetime import datetime
import json
import logging
import os

class HarFileHandler(logging.FileHandler):
    def __init__(self, filename, mode: str, encoding: str=None, delay: bool=False, errors=None,
        version="1.2", creator_version="537.36", pages_title="HAR Generator") -> None:
        super().__init__(filename, mode=mode, encoding=encoding, delay=delay)
        self._starter(version, creator_version, pages_title)

    def _starter(self, version, creator_version, pages_title):
        starter = f'''{{
  "log": {{
    "version": "{version}",
    "creator": {{
      "name": "WebInspector",
      "version": "{creator_version}"
    }},
    "pages": [
      {{
        "startedDateTime": "{datetime.now().strftime("%Y-%m-%dT%H:%M:%S.123Z")}",
        "id": "page_1",
        "title": "{pages_title}",
        "pageTimings": {{
          "onContentLoad": 908.7060000747442,
          "onLoad": 2029.8569998703897
        }}
      }}
    ],
    "entries": [
'''
        self.emit(logging.makeLogRecord({'msg': starter}))

    def _closer(self):
        closer = '''    ]
  }
}
'''
        self.emit(logging.makeLogRecord({'msg': closer}))

    def close(self) -> None:
        self._closer()
        return super().close()

class HarLogger:
    def __init__(self):
        self.logger = logging.getLogger()
        self.logger.setLevel(logging.DEBUG)
        file_handler = HarFileHandler(os.path.join('logs', 'test.har'), mode='w')
        file_handler.setLevel(logging.DEBUG)
        file_handler.setFormatter(logging.Formatter(fmt=f'%(message)s'))
        self.logger.addHandler(file_handler)
        self.entry_delimiter = ""

    def info(self, request, response):
        self.debug(request, response)

    def debug(self, request, response):
        entry = self._make_entry(request, response)
        self.logger.info(entry)

    def warning(self, request, response):
        self.debug(request, response)

    def critical(self, request, response):
        self.debug(request, response)

    def error(self, request, response):
        self.debug(request, response)

    def _make_entry(self, request, response):
        entry = self.entry_delimiter + f'''{{
        "_fromCache": "disk",
        "_initiator": {{
          "type": "script",
          "stack": {{}}
        }},
        "_priority": "High",
        "_resourceType": "fetch",
        "cache": {{}},
        "pageref": "page_1",
        "request": {{
          "method": "{response._method}",
          "url": "{request.full_url}",
          "httpVersion": "http/2.0",
          "headers": {self._make_headers(request.header_items())},
          "queryString": [],
          "cookies": [],
          "headersSize": -1,
          "bodySize": 0
        }},
        "response": {{
          "status": {response.status},
          "statusText": "{response.reason}",
          "httpVersion": "http/2.0",
          "headers": {self._make_headers(response.headers._headers)},
          "cookies": [],
          "content": {{
            "size": 0,
            "mimeType": "{response.headers._default_type}",
            "text": {json.dumps(response.read().decode(response.headers.get_content_charset()))}
          }},
          "redirectURL": "",
          "headersSize": -1,
          "bodySize": 0,
          "_transferSize": 0,
          "_error": null
        }},
        "serverIPAddress": "140.82.113.4",
        "startedDateTime": "{datetime.now().strftime("%Y-%m-%dT%H:%M:%S.123Z")}",
        "time": 4.252000013366342,
        "timings": {{
          "blocked": 1.195999818906188,
          "dns": -1,
          "ssl": -1,
          "connect": -1,
          "send": 0,
          "wait": 1.8329999623298645,
          "receive": 1.223000232130289,
          "_blocked_queueing": 1.076999818906188
        }}
      }}'''
        self.entry_delimiter = ",\n"
        return entry

    def _make_headers(self, r_headers, indent=2*6):
      list_dict = [{"name": header[0], "value": header[1]} for header in r_headers]
      return json.dumps(list_dict, indent=indent)

if __name__ == '__main__':
    a = HarLogger()
    from urllib import request
    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0", "Accept":"application/json,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Content-Type": "application/json", "DNT":"1","Connection":"close", "Upgrade-Insecure-Requests":"1"}
    req = request.Request('https://google.com', headers=headers)
    with request.urlopen(req, timeout=3) as response:
        print(dir(req))
        print(dir(response))
        a.info(req, response)
        b =1

    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0", "Accept":"application/json,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Content-Type": "application/json", "DNT":"1","Connection":"close", "Upgrade-Insecure-Requests":"1"}
    req = request.Request('https://reqres.in/api/users?page=2', headers=headers)
    with request.urlopen(req, timeout=6) as response:
        print(req.full_url)
        a.info(req, response)
        b =1


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
# recusion
# class Solution:
#     def inorderTraversal(self, root: Optional[TreeNode]) -> List[int]:
#         self.result = []
#         self.inorder(root)
#         return self.result
#     def inorder(self, parent):
#         if parent == None:
#             return
#         self.inorder(parent.left)
#         self.result.append(parent.val)
#         self.inorder(parent.right)
# iteration
class Solution:
    def inorderTraversal(self, root: Optional[TreeNode]) -> List[int]:
        stack = []
        result = []
        while root:
            stack.append(root)
            root = root.left
        while len(stack) != 0:
            current = stack.pop()
            result.append(current.val)
            current = current.right
            while current:
                stack.append(current)
                current = current.left
        return result


#Libs
import matplotlib.pyplot as plt
import cv2
import copy

#Variables
#Graphics
data = [-3,-2,-1,0,1,2,3]
print(data)

plt.stem(data)
plt.show()

data2 = [-20,-19,-18,-17,-16,-15,-14,-13,-12,-11,-10,-9,-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]
plt.stem(data2)
plt.show()

#Cv2
image_example = cv2.imread('images/bmw.jpg')
cv2.imshow("BMW",image_example)
cv2.waitKey(0)

print('Dados da imagem colorida:')
print(image_example.shape)

image_gray_example = cv2.imread("images/bmw.jpg", 0)
cv2.imshow("Gray BMW",image_gray_example)
cv2.waitKey(0)

print("Dados da imagem em tons de cinza: ")
print(image_gray_example.shape)

image_2_example = cv2.imread("images/renault_megane_st.jpeg")
image_show = cv2.imshow("Renault Megane", image_2_example)
cv2.waitKey(0)

image_2_example_blue = copy.copy(image_2_example)

image_2_example_blue[:,:,1] = 0
image_2_example_blue[:,:,2] = 0
cv2.imshow("Blue Megane",image_2_example_blue)
cv2.waitKey(0)

image_2_example_green = copy.copy(image_2_example)

image_2_example_green[:,:,0] = 0
image_2_example_green[:,:,2] = 0
cv2.imshow("Green Megane",image_2_example_green)
cv2.waitKey(0)

image_2_example_red = copy.copy(image_2_example)

image_2_example_red[:,:,0] = 0
image_2_example_red[:,:,1] = 0
cv2.imshow("Red Megane",image_2_example_red)
cv2.waitKey(0)

image_3_example_jimny = cv2.imread("images/suzuki_jimny.jpg")
cv2.imshow("Suzuki Jimny", image_3_example_jimny)
cv2.waitKey(0)

image_3_example_convertida_jimny = cv2.cvtColor(image_3_example_jimny, cv2.COLOR_BGR2GRAY)
cv2.imshow("Gray Suzuki Jimny",image_3_example_convertida_jimny)
cv2.waitKey(0)

image_example_jimny_convertida_HSV = cv2.cvtColor(image_3_example_jimny, cv2.COLOR_BGR2HSV)
cv2.imshow("HSV Suzuki Jimny", image_example_jimny_convertida_HSV)
cv2.waitKey(0)

image_example_jimny_convertida_RGB = cv2.cvtColor(image_3_example_jimny, cv2.COLOR_BGR2RGB)
cv2.imshow("RGB Suzuki Jimny",image_example_jimny_convertida_RGB)
cv2.waitKey(0)

image_example_megane_convertida_RGB = cv2.cvtColor(image_2_example, cv2.COLOR_BGR2RGB)
cv2.imshow("RGB Renault Megane",image_example_megane_convertida_RGB)
cv2.waitKey(0)

image_Sobel_example_Gray = cv2.cvtColor(image_2_example, cv2.COLOR_BGR2GRAY)
print('Imagem Original')
cv2.imshow("Renault Megane",image_Sobel_example_Gray)
print('\n\n')
cv2.waitKey(0)

gX = cv2.Sobel(image_Sobel_example_Gray, ddepth=cv2.CV_64F, dx=1, dy=0, ksize=3) #ksize: tamanho da matriz de vizinhaça dos bits
gY = cv2.Sobel(image_Sobel_example_Gray, ddepth=cv2.CV_64F, dx=0, dy=1, ksize=3) #o ddepth converte a imagem para outro formato

#Converter a imagem novamente para o formato 8 bits
gX = cv2.convertScaleAbs(gX)
gY = cv2.convertScaleAbs(gY)

#Juntando os gradientes em X e Y com o mesmo grau de importância
Sobel_grads_combined = cv2.addWeighted(gX, 0.5, gY, -.5, 0)

cv2.imshow("Gradiente de Sobel em X", gX)
cv2.waitKey(0)
cv2.imshow("Gradiente de Sobel em Y", gY)
cv2.waitKey(0)
cv2.imshow("Gradientes de Sobel em X e Y combinados", Sobel_grads_combined)
cv2.waitKey(0)

image_Canny_example_Gray = cv2.cvtColor(image_2_example, cv2.COLOR_BGR2GRAY)

Canny_1 = cv2.Canny(image_Canny_example_Gray, 10,50)
Canny_2 = cv2.Canny(image_Canny_example_Gray, 150,400)

cv2.imshow("Imagem Original", image_Canny_example_Gray)
cv2.waitKey(0)
cv2.imshow("Imagem 1 por Canny", Canny_1)
cv2.waitKey(0)
cv2.imshow("Imagem 2 por Canny", Canny_2)
cv2.waitKey(0)

image_Laplaciano_example_Gray = cv2.cvtColor(image_2_example, cv2.COLOR_BGR2GRAY)
cv2.imshow("Imagem original", image_Laplaciano_example_Gray)
cv2.waitKey(0)

laplaciano_image = cv2.Laplacian(image_Laplaciano_example_Gray, cv2.CV_64F, ksize=3)

laplaciano_image_abs = cv2.convertScaleAbs(laplaciano_image)

cv2.imshow("Imagem após a aplicação do laplaciano", laplaciano_image_abs)
cv2.waitKey(0)


__author__ = "Suyash Soni"
__email__ = "suyash.soni248@gmail.com"

from flask import request
from util.response import intercept, Response
from controller.base import BaseController
from util.injector import inject
from service.plag_dao import PlagiarismDAO
from util.constants.error_codes import HttpErrorCode
from util.error_handlers.exceptions import ExceptionBuilder, BadRequest

class Document(BaseController):
    plag_dao: PlagiarismDAO = inject(PlagiarismDAO)

    @intercept()
    def post(self, *args, **kwargs):
        """Adds a new document to repo"""

        data = request.get_json(force=True)

        content = data.get('content', '')
        title = data.get('title', '')
        description = data.get('description', '')
        author = data.get('author', '')

        if content and title:
            doc = self.plag_dao.create_doc(content, title, description=description, author=author)
        else:
            ExceptionBuilder(BadRequest).error(HttpErrorCode.REQUIRED_FIELD, 'content', 'title').throw()

        return Response(status_code=201, message='Document added successfully!')

    @intercept()
    def get(self):
        """
        Fetches all the documents(paginated).
        :return:
        """
        res = self.plag_dao.get_docs(page=int(request.args.get("page", 1)),
                                    per_page=int(request.args.get("per_page", 10)), all='all' in request.args)
        docs_info = dict(data=[d.to_dict() for d in res['data']], count=res['count'])
        print(docs_info)
        return Response(data=docs_info)


## RENAME this file YourLastName_OOP_FinalProjectSpr2018.py

##Assignment: Add to the constructor and methods of a base class and superclasses
##            which inherit the base class properties.

## Begin with the base Seq class and the DNA class we created in lecture below.

### Seq Class
#
#  Constructor:
#  (1) Use the string functions upper and string to clean up self.sequence.
#  (2) Add a variable self.kmers to the constructor and make it equal to an empty list.

#  Methods:
#  (1) Add a method called make_kmers that makes kmers of a given length from self.sequence
#      appends these to self.kmers. Default kmer paramter=3.
#  (2) Add a method called fasta that returns a fasta formatted string like this:
#      >species gene
#      AGATTGATAGATAGATAT

class Seq:

    def __init__(self,sequence,gene,species):
        self.sequence=sequence.strip(' ').upper()
        self.gene=gene
        self.species=species
        self.kmers=[]

    def __str__(self):
        return self.sequence

    def print_record(self):
        print(self.species + " " + self.gene + ": " + self.sequence)

    def make_kmers(self, k=3):
        n= len(self.sequence)
        for i in range(0, n-k+1):
           self.kmers.append(self.sequence[i:i+k])
        return self.kmers

    def fasta(self):
        return '>' + self.species + ' ' + self.gene + '\n' + self.sequence

import re

## DNA Class: INHERITS Seq class

 # Constructor:
 # Use re.sub to change any non nucleotide characters in self.sequence into an 'N'.
 #     re.sub('[^ATGCU]','N',sequence) will change any character that is not a
 #     capital A, T, G, C or U into an N. (Seq already uppercases and strips.)
 #
 # Methods:
 # Add a method called print_info that is like print_record, but adds geneid and an
 #     empty space to the beginning of the string.

class DNA(Seq):

    def __init__(self,sequence,gene,species,geneid,**kwargs):
        super().__init__(sequence,gene,species)
        self.sequence=re.sub('[^ATGCU]','N',self.sequence)
        self.geneid=geneid
        # self.sequence=re.sub('[^ATGCU]','N',sequence)

    def analysis(self):
        gc=len(re.findall('G',self.sequence) + re.findall('C',self.sequence))
        return gc

    def print_info(self):
       print(self.geneid + ' ' + self.species + " " + self.gene + ": " + self.sequence)

## RNA Class:  INHERITS DNA class

 # Construtor:
 # Use the super() function (see DNA Class example).
 # (1) Automatically change all Ts to Us in self.sequence.
 # (2) Add self.codons equals to an empty list
 #
 # Methods:
 # (1) Add make_codons which breaks the self.sequence into 3 letter codons
 #     and appends these codons to self.codons unless they are less than 3 letters long.
 # (2) Add translate which uses the Global Variable standard_code below to
 #     translate the codons in self.codons and returns a protein sequence.

standard_code = {
     "UUU": "F", "UUC": "F", "UUA": "L", "UUG": "L", "UCU": "S",
     "UCC": "S", "UCA": "S", "UCG": "S", "UAU": "Y", "UAC": "Y",
     "UAA": "*", "UAG": "*", "UGA": "*", "UGU": "C", "UGC": "C",
     "UGG": "W", "CUU": "L", "CUC": "L", "CUA": "L", "CUG": "L",
     "CCU": "P", "CCC": "P", "CCA": "P", "CCG": "P", "CAU": "H",
     "CAC": "H", "CAA": "Q", "CAG": "Q", "CGU": "R", "CGC": "R",
     "CGA": "R", "CGG": "R", "AUU": "I", "AUC": "I", "AUA": "I",
     "AUG": "M", "ACU": "T", "ACC": "T", "ACA": "T", "ACG": "T",
     "AAU": "N", "AAC": "N", "AAA": "K", "AAG": "K", "AGU": "S",
     "AGC": "S", "AGA": "R", "AGG": "R", "GUU": "V", "GUC": "V",
     "GUA": "V", "GUG": "V", "GCU": "A", "GCC": "A", "GCA": "A",
     "GCG": "A", "GAU": "D", "GAC": "D", "GAA": "E", "GAG": "E",
     "GGU": "G", "GGC": "G", "GGA": "G", "GGG": "G"}

class RNA(DNA):

    def __init__(self,sequence,gene,species,geneid,**kwargs):
        super().__init__(sequence,gene,species,geneid)
        self.sequence = re.sub('T','U',self.sequence)
        self.codons = []

    def make_codons(self):
        x = re.findall('.{1,3}',self.sequence)
        for i in x:
            if len(i) == 3:
                self.codons.append(i)
            else:
                pass

    def translate(self):
        seq = ''
        for i in self.codons:
            try:
                AA = standard_code[i]
                seq += AA
            except KeyError:
                AA = 'X'
                seq += AA
        print(seq)

# ## Protein Class: INHERITS Seq class

 # Construtor:
 # Use the super() function (see DNA Class example).
 # (1) Use re.sub to change any non LETTER characters in self.sequence into an 'X'.
 # (2) self.aa_counts has already been added - it is a dictionary with all 21
 #     amino acids as keys and values set to 0
 #
 # Methods:
 # Add tabulate_amino_acids, which counts the amino acids in self.sequence
 #     and puts these in the dictionary self.counts

class Protein(Seq):
    def __init__(self,sequence,gene,species,geneid,**kwargs):
        super().__init__(sequence,gene,species)
        self.sequence =  re.sub('[^A-Z]','X',self.sequence.rstrip())
        self.aa_counts={'A':0,'C':0,'D':0,'E':0,'F':0,'G':0,'H':0,'I':0,'K':0,'L':0,'M':0,'P':0,'Q':0,'R':0,'S':0,'T':0,'U':0,'V':0,'W':0,'X':0,'Y':0}

    def tabulate_amino_acids(self):
        for aa in self.sequence:
            if aa in self.aa_counts:
                self.aa_counts[aa] +=1
            else:
                self.aa_counts[aa] = 1
        return(self.aa_counts)


''' STICKY PERCEPTION: SUPPORT FOR PREDICTIVE CODING IN THE UNIFORMITY ILLUSION?

    This code runs an experiment which investigates the effect of perceptual
    'stickiness' following exposure to the uniformity illusion

    If you want to read my thesis to get more information about the experiment contact me :D

    Code & study design by Nina Fitzmaurice
    Contact: ninafitzmaurice@gmail.com

    Citation APA
    N.B. Fitzmaurice, Sticky Perception in the Uniformity Illusion, (2022),
    GitHub repository, https://github.com/ninafitzmaurice/Sticky-Perception-Uniformity-Illusion
 '''

import psychopy.visual
from psychopy.visual import Aperture
import psychopy.event
import psychopy.tools
from psychopy.tools import monitorunittools
from psychopy import monitors
import numpy as np
import random
from  psychopy.iohub import launchHubServer
from psychopy import gui
from psychopy import data
import pandas as pd
import os

# This code runs the uniformity illusion with delayed central stimuli updating
#### Nina Fitzmaurice thesis project 2022

## Conditions:
# Experimental = UI (also acts as control as the illusion does not consistently occur)
# control = short latency of stimuli (optional, can be removed)
# control = catch trial illusion forced (optional, can be removed)

# saves to new folder 'data', new patch to child dir
current_dir = os.path.dirname(os.path.abspath(__file__))
new_dir = current_dir + os.sep + u'Nina_UI_Data'
# create directory
if os.path.isdir(new_dir) == False:
    os.mkdir(new_dir)

# change to new dir
os.chdir(new_dir)

print('new dir to save data to:     ', os.getcwd())

experiment_name = 'Uniformity_illusion'

info = {'Participant_nr': 'xxx'}
dlg = gui.DlgFromDict(dictionary=info, title=experiment_name)

if dlg.OK == False:
    psychopy.core.quit()

sub_ID = info['Participant_nr']
info['date'] = data.getDateStr()

filename = '%s_%s_%s.csv' % (info['Participant_nr'],
            experiment_name, info['date'])

test_filename = '%s_%s_%s_%s.csv' % (info['Participant_nr'],
            experiment_name,'TEST_', info['date'])

# live updating into txt file as back up:
# Open a file with access mode 'a'
# so it can be appended
backUp = open(filename+'_backUp.txt', 'a+')

###############################################################

#### EXPERIMENT PARAMS
# ignore this
#10.6 = short latency trial duration estimate
#15.6 = longer dur trials
# 156 % 3 == 0
# 156 trials will last around 32 mins
# 12 trials per stimuli (13 unique stimuli in total)

## SET AS DESIRED
# duration of stimuli (secs)
exp_stim_duration = 8
# short latency catch stimuli duration
catch_duration = 2
# duration of mask (secs)
mask_duration = 0.1
# inter stim noise duration (secs) = for noise btw centre change and REPRODUCTION
inter_stim_dur = 1
# inter TRIAL duration (secs) = for noise btwn TRIALS
inter_trial_dur = 1.5

# number of trials
# ok this is a mess, I will probably change it later right now I cant be bothered
# will result in uneven number of control and experimental conditions
# I dont think its a huge deal??
# IMPORTANT!!! every trial % 3 == 0 will be a SHORT LATENCY CATCH TRIAL
# ----------------- ONLY FOR NON UI CATCH TRIALS
# N trials must % 3 == 0!! So 1/3 of trials are short latenct catch trials
n_trials = 9
if n_trials % 3 != 0:
    raise Exception("n_trials % 3 MUST = 0!")
# 16 stimuli to test, so 13/156 = trials per stimuli
# n_trials /= 16

## SCALE - work out for different aspect ratios
# I tried to do this automatically but psychopy is annoying so this is how it is
# preserves scaling of the stimuli
# my aspect ratio = 16:9
# so, scaling = 9/16
# you might also need to adjust the number of dots per row and col in the stimuli to
# make it look better on different ratios but that code is later
scale = 0.563

#### QUIT KEY
# quit key to abort study at any time
quit_key = 'q'

def quit_key_pressed():
    win.close()

psychopy.event.globalKeys.add(quit_key, quit_key_pressed)

## WINDOW
win = psychopy.visual.Window(
    #size=[1368, 912],
    size=[1368/2, 912/2],
    allowStencil=True,
    units='pix',
    fullscr=True,
    color=[-1, -1, -1],
    monitor = monitors.Monitor('samplingExperiment'))

####################################################

## Setting up the eyetracker, uncomment if needed
#iohub_config = {
#    'eyetracker.hw.sr_research.eyelink.EyeTracker':{
#        'name': 'tracker',
#        'model_name': 'EYELINK 1000 DESKTOP',
#        'calibration': {
#            'auto_pace': False,
#            'screen_background_color': [75,75,75]
#            },
#        'runtime_settings': {
#            'sampling_rate': 500,
#            'track_eyes': 'RIGHT'
#            }
#
#        }
#    }
#
#iohub_config['eyetracker.hw.sr_research.eyelink.EyeTracker']['default_native_data_file_name']= ('NINA' + '%s' % (info['Participant_nr']))
#iohub_config['eyetracker.hw.sr_research.eyelink.EyeTracker']['simulation_mode'] = False
#io = launchHubServer(window = win, **iohub_config)
#tracker = io.devices.tracker

#################################################################

# not ideal, couldnt get the others to work smoothly
# feel free to change it but then you must remove scaling in the whole script,
# I think
win.units = 'norm'

## MOUSE
myMouse = psychopy.event.Mouse(visible=True)
# [0] = left, [1] = wheel in the middle is pressed, [2], right
# leftClick, wheelClick, rightClick = myMouse.getPressed()

## APERTURE:
# experimental only
aperture = psychopy.visual.Aperture(win, size= (0.97,1), pos=(0,0), shape='square',
    inverted=False, units=None)

# for demo only
demo_aperture = psychopy.visual.Aperture(win, size= (0.3,0.3), pos=(0,-0.475), shape='square',
    inverted=False, units=None)

# aperture should be off initially
aperture.enabled = False
demo_aperture.enabled = False

####### STIMULI COORDINATES

def make_stim_coords(STIM_TYPE, x_circles, y_circles, x_start, x_end, y_start, y_end):
    # normalisation = length of x, y = 2
    # set step size from point x/y start to x/y end
    x_len =  abs(x_start) + abs(x_end)
    #print('x len: ',x_len)
    # if negative y start coord, then convert both to pos values and
    # subract smaller from bigger (to get distance between points)
    if y_start < 0:
        y_len =  round(abs(y_end) - abs(y_start), 1)

    elif y_start > 0 and y_end < 0:
        y_len = round(y_start + abs(y_end), 1)

    #print('y len ', y_len)

    x_step = x_len/x_circles
    y_step = y_len/y_circles

    # coords for circles, upper left to lower right
    if STIM_TYPE == 'colour':
        x_coords = np.array(np.arange(x_start, x_end + x_step, x_step))
        y_coords = np.array(np.arange(y_start, y_end, -1*y_step))
    else:
        x_coords = np.array(np.arange(x_start, x_end, x_step))
        y_coords = np.array(np.arange(y_start, y_end, -1*y_step))
    #print(x_coords)
    #print(y_coords)
    # makes coordinate grid
    coords = np.array(np.meshgrid(x_coords, y_coords)).T.reshape(-1, 2)

    if STIM_TYPE == 'size':
        # for the shift of rows.......
        x = np.empty((x_circles)) # number of rows (y coords in meshgrid)
        x[::2] = 0 # for every odd row = every x coord is 0
        x[1::2] = 0.7* x_step # for every odd row = every x coord is half x step
        y = np.zeros((y_circles)) # for every instance of x, the y coord is 0

        # formatt in same way as coords
        shift = np.array(np.meshgrid(y, x)).T.reshape(-1, 2)
        shift = np.fliplr(shift)

        # add n to every x coord on the yth row
        coords_shifted = shift + coords

        # dots will be shifted over!!

        return coords_shifted

    elif STIM_TYPE == 'colour':
        # dots will NOT be shifted over!!
        return coords

## TO CUSTOMISE
# args: STIM_TYPE, x_circles, y_circles, x_start, x_end, y_start, y_end

# STIM_TYPE = for size or colour stim
#   size stim = coords will be shifted
#   colour stim = coords will not be shifted

# x_circles MUST BE EVEN
# y_circles MUST BE EVEN

# values are normalised!!

# coordinates for the mini demo in instructions
coords_demo_periph = make_stim_coords('colour', 10,8,-0.3,0.3,-0.2,-0.75)
coords_demo_cent = make_stim_coords('colour', 10,8,-0.3,0.3,-0.2,-0.75)

# coordinates for experimental stimuli grids/element arrays
coords_colour = make_stim_coords('colour', 18,12,-0.95,0.975,0.95,-1.05)
coords_size = make_stim_coords('size', 18,12,-0.95,0.975,0.95,-1.05)

## DEMO ELEMENT ARRAY
# makes a grid of dots like the experimental stimuli but with fewer params
def demo_stimuli_grid(coords, size, R,G,B):
    grid = psychopy.visual.ElementArrayStim(win, units = None,
    nElements=len(coords), sizes=size,
    xys=coords, elementTex=None, elementMask = 'circle')

    grid.colors = [R,G,B]
    grid.sizes *= [scale,1]

    return grid

## DEMO STIMULI can customise
demo_cent = demo_stimuli_grid(coords_demo_periph, 0.045,0,1,0)
demo_periph = demo_stimuli_grid(coords_demo_cent, 0.045,1,0,0)

## STIMULI ELEMENT ARRAY
# makes grid of stimuli
# opacity
# sizes = size of elements
# mask = gaussian blur, circle, other feature if you want
# RGB = colour
# orientation = orientation of each element
def stimuli_grid(coordinates, opacity, sizes, mask, orientation, R,G,B):

    grid = psychopy.visual.ElementArrayStim(win, units = None,
        nElements=len(coordinates), sizes=sizes,
        xys=coordinates, elementTex=None, elementMask = mask)

    grid.colors = [R,G,B]
    grid.colorSpace='rgb'
    grid.sizes *= [scale,1]

    grid.opacities = opacity

    if orientation == 'rand':
        # possible orientations from 0 deg to 80 deg
        rotation = np.arange(0,80,1)
        # assigns random orientation in range above for each element
        grid.oris = np.random.choice(rotation, len(coords_shifted), replace=True)
    else:
        grid.oris = orientation

    # I removed fringe width because it wasn't blury enough, but code is here incase
    # change of mind or someone wants to use it
#    if mask == 'raisedCos':
#        grid.maskParams = {'fringeWidth': fw}

    return grid

## NOISE
noiseTexture = np.random.rand(300, 300) * 2.0 - 1
noise = psychopy.visual.GratingStim(win, tex=noiseTexture,
        size=(2,2), units='norm',
        interpolate=False, autoLog=False)

## STIMULI VALUES
# Size
small = 0.06
big = 0.09
repro_size = small+((big-small)/2)
colour_stim_size = 0.1

## BLOCKS list of block types
# easy to customise
block_list = ['centFill_RT','centFill_Repro','blackOut']

## STIMULI TRIALS, for participants to practice before experiment starts
# customise values for the stimuli grid in the stimuli dicts
# stimuli_grid(coordinates, opacity, sizes, mask, orientation, R,G,B)
CF_test_stim_list = [
            # centre small periph big
            {'Trial': {'Cent': stimuli_grid(coords_size,1,small,'circle',0,1,1,0),
                        'Periph': stimuli_grid(coords_size,1,big,'circle',0,1,1,0),
                        'Cent_new': stimuli_grid(coords_size,1,big,'circle',0,1,1,0),
                        'Repro': stimuli_grid(coords_size,1,repro_size,'circle',0,1,1,0),
                        'Condition': ['Exp', 'Cent_big'],
                        'Name': 'Size'}},

            # exp 001
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0,1),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0.75,0.7),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0.75,0.7),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,0,0,1),
                        'Condition': ['Exp', '001'],
                        'Name': 'Colour'}}

                        ]

BO_test_stim_list = [
            # exp 010 blackOut
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,1,0),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,1,0,0),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,-1,-1,-1),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,0,1,0),
                        'Condition': ['Exp', '010', 'blackOut'],
                        'Name': 'Colour'}},

            # exp 001 blackOut
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0,1),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0.75,0.7),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,-1,-1,-1),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,0,0,1),
                        'Condition': ['Exp', '001', 'blackOut'],
                        'Name': 'Colour'}}
                        ]

## EXPERIMENT for data collection
## STIMULI centFill
centFill_stim_list = [

    # CREATING STIMULI:
    # opacity = x, size = x, mask = x, orientation = x, R = x, G = x, B = x

            ## SIZE
            # centre big periph small
            {'Trial': {'Cent': stimuli_grid(coords_size,1, big,'circle',0,1,1,0),
                        'Periph': stimuli_grid(coords_size,1,small,'circle',0,1,1,0),
                        'Cent_new': stimuli_grid(coords_size,1,small,'circle',0,1,1,0),
                        # for reproduction task = size between small and big!
                        'Repro': stimuli_grid(coords_size,1,repro_size,'circle',0,1,1,0),
                        'Condition': ['Exp', 'Cent_small'],
                        'Name': 'Size'}},

            # centre small periph big
            {'Trial': {'Cent': stimuli_grid(coords_size,1,small,'circle',0,1,1,0),
                        'Periph': stimuli_grid(coords_size,1,big,'circle',0,1,1,0),
                        'Cent_new': stimuli_grid(coords_size,1,big,'circle',0,1,1,0),
                        'Repro': stimuli_grid(coords_size,1,repro_size,'circle',0,1,1,0),
                        'Condition': ['Exp', 'Cent_big'],
                        'Name': 'Size'}},

            # diff too big for illusion
            # eh I didnt use this in the end, kinda messy but hey ho do what you will with it
            {'Trial': {'Cent': stimuli_grid(coords_size,1,0.03,'circle',0,1,1,0),
                        'Periph': stimuli_grid(coords_size,1,0.09,'circle',0,1,1,0),
                        'Cent_new': stimuli_grid(coords_size,1,0.09,'circle',0,1,1,0),
                        'Repro': stimuli_grid(coords_size,1,repro_size,'circle',0,1,1,0),
                        'Condition': ['Exp', 'no_UI'],
                        'Name': 'Size'}},

            # Catch trial
            # stimuli gets BIGGER to match big centre during catch trial
            {'Trial': {'Cent': stimuli_grid(coords_size,1,big,'circle',0,1,1,0),
                        'Periph': stimuli_grid(coords_size,1,small,'circle',0,1,1,0),
                        'Cent_new': stimuli_grid(coords_size,1,big,'circle',0,1,1,0),
                        # for reproduction task = size between small and big!
                        'Repro': stimuli_grid(coords_size,1,repro_size,'circle',0,1,1,0),
                        'Condition': ['Catch'],
                        'Name': 'Size'}},

            ## COLOUR
            # Condition value is the colour of the centre
            # exp 010
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,1,0),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,1,0,0),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,1,0,0),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,0,1,0),
                        'Condition': ['Exp', '010'],
                        'Name': 'Colour'}},

            # exp 001
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0,1),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0.75,0.7),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0.75,0.7),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,0,0,1),
                        'Condition': ['Exp', '001'],
                        'Name': 'Colour'}},

            # exp 101
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,1,0,1),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0.91,0.78,0),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0.91,0.78,0),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,1,0,1),
                        'Condition': ['Exp', '101'],
                        'Name': 'Colour'}},

            # Catch trials
            # Catch 001
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0,1),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0.75,0.7),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0.75,0.7),
                        'Catch': stimuli_grid(coords_colour,0,colour_stim_size,'circle',0,0,0,1),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,0,0,1),
                        'Condition': ['Catch', '001'],
                        'Name': 'Colour'}},

            # Catch 010
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,1,0),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,1,0,0),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,1,0,0),
                        'Catch': stimuli_grid(coords_colour,0,colour_stim_size,'circle',0,0,1,0),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,0,1,0),
                        'Condition': ['Catch', '010'],
                        'Name': 'Colour'}},

            # Catch 101
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,1,0,1),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0.91,0.78,0),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0.91,0.78,0),
                        'Catch': stimuli_grid(coords_colour,0,colour_stim_size,'circle',0,1,0,1),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,1,0,0),
                        'Condition': ['Catch', '101'],
                        'Name': 'Colour'}}#,

## made these stimuli didnt use them, the code to run them might be broken if you chose to
## add them in be aware it may not be so smooth I dont remember
#            # Orientation
#
#            {'Trial': {'Cent': stimuli_grid(0.07,'circle',0,0,0,1),
#                        'Periph': stimuli_grid(0.07,'circle',0,1,0,0),
#                        'Cent_new': stimuli_grid(0.07,'circle',0,1,0,0),
#                        'Condition': ['Exp', '001'],
#                        'Name': 'Ori'}},
#
#                        {'Trial': {'Cent': stimuli_grid(0.07,'circle',0,0,0,1),
#                        'Periph': stimuli_grid(0.07,'circle',0,1,0,0),
#                        'Cent_new': stimuli_grid(0.07,'circle',0,1,0,0),
#                        'Condition': ['Exp', '001'],
#                        'Name': 'Ori'}},
#
#                        {'Trial': {'Cent': stimuli_grid(0.07,'circle',0,0,0,1),
#                        'Periph': stimuli_grid(0.07,'circle',0,1,0,0),
#                        'Cent_new': stimuli_grid(0.07,'circle',0,1,0,0),
#                        'Condition': ['Exp', '001'],
#                        'Name': 'Ori'}},

#            # Donuts
#            # centre donut periph circle
#            {'Trial': {'Cent': stimuli_grid(1,0.07,'circle',0,1,1,1),
#                        'Cent_2': stimuli_grid(1,0.03,'circle',0,0,0,0),
#                        'Periph': stimuli_grid(1,0.07,'circle',0,1,1,1),
#                        'Cent_2_new': stimuli_grid(1,0.07,'circle',0,1,1,1),
#                        'Condition': ['Exp', 'Cent_donut'],
#                        'Name': 'Donut'}},
#
#            # centre circle periph donut
#            {'Trial': {'Cent': stimuli_grid(1,0.07,'circle',0,1,1,1),
#                        'Periph': stimuli_grid(1,0.07,'circle',0,1,1,1),
#                        'Periph_2': stimuli_grid(1,0.03,'circle',0,0, 0, 0),
#                        'Cent_2_new': stimuli_grid(1,0.03,'circle',0,0, 0, 0),
#                        'Condition': ['Exp', 'Periph_donut'],
#                        'Name': 'Donut'}},
#
#            # Catch trail
#            {'Trial': {'Cent': stimuli_grid(1,0.07,'circle',0,1,1,1),
#                        'Cent_2': stimuli_grid(1,0.03,'circle',0,0, 0, 0),
#                        'Periph': stimuli_grid(1,0.07,'circle',0,1,1,1),
#                        'Periph_2': stimuli_grid(1,0,'circle',0,0, 0, 0),
#                        'Cent_2_new': stimuli_grid(1,0.03,'circle',0,0, 0, 0),
#                        'Condition': ['Catch'],
#                        'Name': 'Donut'}}

             ]

## STIMULI blackOut
blackOut_stim_list = [
    # CREATING STIMULI:
    # opacity = x, size = x, mask = x, orientation = x, R = x, G = x, B = x

            # Centre black out trials
            # exp 010 blackOut
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,1,0),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,1,0,0),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,-1,-1,-1),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,0,1,0),
                        'Condition': ['Exp', '010', 'blackOut'],
                        'Name': 'Colour'}},

            # exp 001 blackOut
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0,1),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0.75,0.7),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,-1,-1,-1),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,0,0,1),
                        'Condition': ['Exp', '001', 'blackOut'],
                        'Name': 'Colour'}},

            # exp 101 blackOut
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,1,0,1),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0.91,0.78,0),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,-1,-1,-1),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,1,0,1),
                        'Condition': ['Exp', '101', 'blackOut'],
                        'Name': 'Colour'}},

            # Catch trials
            # Catch 001
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0,1),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,0.75,0.7),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,-1,-1,-1),
                        'Catch': stimuli_grid(coords_colour,0,colour_stim_size,'circle',0,0,0,1),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,0,0,1),
                        'Condition': ['Catch', '001','blackOut'],
                        'Name': 'Colour'}},

            # Catch 010
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0,1,0),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,1,0,0),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,-1,-1,-1),
                        'Catch': stimuli_grid(coords_colour,0,colour_stim_size,'circle',0,0,1,0),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,0,1,0),
                        'Condition': ['Catch', '010','blackOut'],
                        'Name': 'Colour'}},

            # Catch 101
            {'Trial': {'Cent': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,1,0,1),
                        'Periph': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,0.91,0.78,0),
                        'Cent_new': stimuli_grid(coords_colour,1,colour_stim_size,'circle',0,-1,-1,-1),
                        'Catch': stimuli_grid(coords_colour,0,colour_stim_size,'circle',0,1,0,1),
                        'Repro': stimuli_grid(coords_colour,0.5,colour_stim_size,'circle',0,1,0,0),
                        'Condition': ['Catch', '101','blackOut'],
                        'Name': 'Colour'}}

                        ]

## TEXT STIM & FIXATION & BUTTONS

text_screen = psychopy.visual.TextBox(win,
                         text='q',
                         font_size=32,
                         font_color=[1,1,1],
                         size=(1,1),
                         pos=(0,0.2),
                         grid_horz_justification='centre',
                         units='norm'
                         )

fix = psychopy.visual.TextStim(win, text='+', pos=0, color=[1,1,1],
                                colorSpace ='rgb', opacity=0.5, units='norm',
                                height=0.09)

welcome = u'''Thank you for participating in this study!

Please read the instructions carefully.

Press SPACE to see the instructions.'''

#-------------------------

CF_instructions_RT = u'''During each trial you will be shown different visual illusions. Maintain fixation on the central fixation cross while viewing the illusions.

After a few seconds of viewing the illusion, the centre of the screen is going to change. Maintain fixation during this change.

LEFT CLICK the mouse when all the dots on the screen look the same:'''

CF_instructions_REPRO_a = u'''During each trial you will be shown different visual illusions. Maintain fixation on the central fixation cross while viewing the illusions.

After a few seconds of viewing the illusion, the centre of the screen is going to change. Maintain fixation on the cross during this change.

Example:'''

BO_instructions_a = u'''During each trial you will be shown different visual illusions. Maintain fixation on the central fixation cross while viewing the illusions.

After a few seconds of viewing the illusion, the centre of the screen is going to black out. Maintain fixation on the cross during this change.

Example:'''

#------------------------

CF_instructions_REPRO_b = u'''A few seconds after the centre changes, a new screen will appear.

Try to re-create the stimuli you saw in the periphery AFTER the centre changed.

SCROLL the mouse to change the features of the stimuli. Try this out now!'''

BO_instructions_b = u'''A few seconds after the centre blacks out, a new screen will appear.

Try to re-create the stimuli you saw in the periphery AFTER the centre went blank.

SCROLL the mouse to change the features of the stimuli. Try this out now!'''

start_eyetracking = u'''The experimenter will set up the eyetracker now. '''

#------------------------

start_test_trials = u'''Press SPACE to start practice trials'''

test_trials_complete = u'''Test trials completed!

Press SPACE to start the experiment.'''

start_of_block = u'''starting block...'''

end_of_block = u'''This block has ended.

Take a break, youre doing great!'''

end_of_experiment = '''Thank you for your time, the experiment is complete!'''

## STRUCTURE PER TRIAL
''' kinda a mess T_T
this function will run each trial in the stim list through the trial format

1. centre is different to periphey, and UI occurs or not (includes catch trials)
1.a. if participant left click = reports seeing the UI, no click = reports
     didnt see the UI

2. the centre changes to match the periphery, waits for click from
particpant (indicated when they see the illusory periphery destablise).
2.a. RT at time of click is stored
2.b. there is NOT timeout! wait until participant reports uniformity

3. a noise grating to mask any retinal after-effects (at least a bit)

4. REPRODUCTION TASK: only the periphery is visible, scroll wheel to change
attributes of the periphery to match the illusory perception. left click
to submit choice '''

def RUN_TRIALS(trials, block, blockNumber, trialNumber):
    # SEND MESSAGE TO EYE TRACKER
    # tracker.sendMessage(f'''block {block} {blockNumber} starts''')

    ### LATENCY & UI CATCH TRIALS
    '''MESS MESS MESS - I will change this at some point
    for now its ok

    making sure that a random 1/3 of all trials are latency catch
     and that 1/3 of the UI catch trials are run rather than all of them
     NOTE = it is set up this way because the UI catch trials are in the stimuli
           dicts (stim_lists) but the latency catches are not.
           UI catch must also be 1/3 of trials or the trial split would
           not be even.
           this code will make sure only a random 1/3 of UI catch trials run!

     1. make dict with all trial indexes as the keys
     2. randomly select 1/3 of trials from each index and
        add this random 1/3rd of trials to the dict.
        i.e.     dict = {'1':[2,4,7],'2':[5,11,3]...}'''
    catch_reps = {}
    # if not trial blocks...
    if blockNumber != 'NaN':
        # for every trial type in the stim list
        for i in range(1,len(trials.trialList)+1):
            # add trial index as a key in the dict
            if i not in catch_reps:
                # for each index that is not in the list, make a new key
                # and append random 1/3 of trials to that list
                catch_reps[i] = np.random.choice(np.arange(1,n_trials+1),
                                                        int(n_trials/3), replace=False)

    skipped_counter = 0

    for trial in trials:
        #tracker.setRecordingState(True)

        # 1. setting up trial....

        # 1.a.
        # organising number of trials that are catch trials
        ## LATENCY & UI CATCH TRIALS NEW VERSION
        # 2/3rds of trials are a latency or UI catch
        # excluding trial blocks
        # part of the aforementioned mess
        if blockNumber != 'NaN':
            if 'Catch' in trial['Trial']['Condition']:
                # if thisRepN in dict[trial index]
                #        this rep N is saved to dict as a catch trial
                if trials.thisRepN+1 in catch_reps.get(trials.thisIndex+1):
                    stim_duration = exp_stim_duration
                    data['Catch_latency'].append(0)

                elif trials.thisRepN+1 not in catch_reps.get(trials.thisIndex+1):
                    skipped_counter += 1
                    #print('SKIPPED ',trials.thisN, trial['Trial']['Condition'])
                    #trials.next()
                    #print('trial n before continue: ', trials.thisN)
                    continue

            elif 'Catch' not in trial['Trial']['Condition']:
                # LATENCY CATCH
                if trials.thisRepN+1 in catch_reps.get(trials.thisIndex+1):
                    stim_duration = catch_duration
                    data['Catch_latency'].append(1)

                elif trials.thisRepN+1 not in catch_reps.get(trials.thisIndex+1):
                    stim_duration = exp_stim_duration
                    data['Catch_latency'].append(0)

        else:
            stim_duration = exp_stim_duration
            data['Catch_latency'].append(0)

#-----------------------------------------------------------------------
#          ignore
#        ## LATENCY CATCH TRIALS: OLD VERSION === NOT RANDOMISED
#        # each trial INDEX (the trial identifier) is appended to a list
#        # ---- to keep track of how many times each trial was completed
#        # every third trial will be a short latency catch trial
#        # IF n trials % 3 == 0!!!
#        # THIS EXCLUDES FORCED UI CATCH TRIALS
#        complete_trials.append(trials.thisIndex+1)
#        #print('count   ', complete_trials.count(trials.thisIndex))
#        if complete_trials.count(trials.thisIndex+1) % 3 == 0:
#            if not 'Catch' in trial['Trial']['Condition']:
#                #print('MODULOOOO 3')
#                stim_duration = catch_duration
#                data['Catch_latency'].append(1)
#                data['Catch_UI'].append(0)
#            else:
#                stim_duration = exp_stim_duration
#                data['Catch_latency'].append(0)
#        else:
#            stim_duration = exp_stim_duration
#            data['Catch_latency'].append(0)
#--------------------------------------------------------------------

        # ONLY THE TRIALS RAN!!
        trialNumber += 1

        ## saving trial related data
        # sub ID
        data['Sub_ID'].append(sub_ID)
        # +1 bc python indexing starts at 0
        # rep number
        data['Rep_n'].append(trials.thisRepN+1)
        # block number
        data['Block_n'].append(blockNumber)
        # block type (centFill or blackOut or test)
        data['Block_type'].append(block)
        # trial number out of total trials
        data['Trial_n'].append(trialNumber)
        # trial number out of current repeat
        data['Trial_Rep_n'].append(trials.thisTrialN+1)
        # index of trial
        data['Trial_index'].append(trials.thisIndex+1)
        # the name of the stimuli group
        data['Stimuli'].append(trial['Trial']['Name'])
        # more details about the stimuli condition in string
        data['Condition'].append(trial['Trial']['Condition'])

        # these values are saved to compare the reproduction task values to
        # colour = compare repro opacity to actual opacity (100% opace)
        if trial['Trial']['Name'] == 'Colour':
            data['Cent_opacity'].append(1)
        else:
            # not import for size trials
            data['Cent_opacity'].append('NaN')

        # compare the repro values to these values
        # these values match the centre values for each stimuli
        if trial['Trial']['Name'] == 'Size':
            if 'Cent_small' in trial['Trial']['Condition']:
                data['Cent_size'].append(big)

            elif  'Cent_big' in trial['Trial']['Condition']:
                data['Cent_size'].append(small)

            elif 'no_UI' in trial['Trial']['Condition']:
                data['Cent_size'].append(0.03)

            elif 'Catch' in trial['Trial']['Condition']:
                data['Cent_size'].append(big)
        else:
            data['Cent_size'].append('NaN')

        if 'Exp' in trial['Trial']['Condition']:
            data['Exp'].append(1)
        else:
            data['Exp'].append(0)

        if 'Catch' in trial['Trial']['Condition']:
            data['Catch_UI'].append(1)
        else:
            data['Catch_UI'].append(0)

        ### INIT PARAMS FOR CATCH TRIALS
        radius_catch = small
        # the size of the inner circle is set to 0 in catch trial
        radius_catch_donut = 0
        # set opacity catch to 0 == fully transparent, will increase to 1!
        opacities_catch = 0

        ### INIT REPRODUCTION TASK VALUES
        if trial['Trial']['Name'] == 'Size':
            trial['Trial']['Repro'].setSizes(repro_size)
            trial['Trial']['Repro'].sizes *= [scale,1]

        if trial['Trial']['Name'] == 'Colour':
            if '010' in trial['Trial']['Condition']:
                trial['Trial']['Repro'].colors = [0,1,0]
            if '001' in trial['Trial']['Condition']:
                trial['Trial']['Repro'].colors = [0,0,1]
            if '101' in trial['Trial']['Condition']:
                trial['Trial']['Repro'].colors = [1,0,1]

        timer = psychopy.core.Clock() # sets a clock for each trial

        # NOISE BEFORE EACH TRIAL TO CLEAR EFFECTS OF LAST TRIAL
        while timer.getTime()<inter_trial_dur:
                # remove apature
                aperture.enabled = False
                noise.phase += (0.01 / 2, 0.005 / 2)
                noise.draw()
                win.flip()
                if timer.getTime()>inter_trial_dur:
                    break

        # SEND MESSAGE TO EYE TRACKER
        #tracker.sendMessage(f'''UI: trial index: {trials.thisIndex}. trial n: {trialNumber}, rep: {trials.thisRepN+1}, block: {block}''')
        # 2. running trials...
        myMouse.clickReset()
        UI_was_seen = False

        while timer.getTime()<stim_duration:

            # to report seeing UI
            mouseClicks = myMouse.getPressed()
            if mouseClicks[0]==1:
                UI_was_seen = True

            # draw peripheral stimuli
            aperture.enabled = True
            aperture.inverted= True

            # this will select UI CATCH TRIALS (UI forced to occur)
            if 'Catch' in trial['Trial']['Condition']:

                if trial['Trial']['Name'] == 'Size':
                    # speed(steps) = distance/time
                    # distance = big-small
                    # time = stim duration
                    radius_catch += ((big-small)/(stim_duration*60))
                    trial['Trial']['Periph'].sizes = radius_catch
                    trial['Trial']['Periph'].sizes *= [scale,1]
                    trial['Trial']['Periph'].draw()

                elif trial['Trial']['Name'] == 'Donut':
                    # speed(steps) = distance/time
                    # distance = from 0 to 0.03
                    # time = stim duration
                    trial['Trial']['Periph'].draw()
                    radius_catch_donut += (0.03/(stim_duration*60))
                    trial['Trial']['Periph_2'].sizes = radius_catch_donut
                    trial['Trial']['Periph_2'].sizes *= [scale,1]
                    trial['Trial']['Periph_2'].draw()

                elif trial['Trial']['Name'] == 'Colour':
                    trial['Trial']['Periph'].draw()
                    # this will slowly increase the opacity of the second periphery:
                    # same values as centre and repro
                    opacities_catch += (1/(stim_duration*60)) # from one to 0 over the duration of the stimuli
                    trial['Trial']['Catch'].opacities = opacities_catch
                    trial['Trial']['Catch'].draw()

            # draw the double periph stimuli for donut condition
            elif 'Exp' and 'Periph_donut' in trial['Trial']['Condition']:
                trial['Trial']['Periph'].draw()
                trial['Trial']['Periph_2'].draw()

            # periphery for all other conditions
            elif 'Exp'and not 'Periph_donut' in trial['Trial']['Condition']:
                trial['Trial']['Periph'].draw()

            # draw central stimuli for all trials OTHER than donut_cent
            aperture.inverted= False
            trial['Trial']['Cent'].draw()

            # for the donut conditions = because needs to draw second central stimuli
            # donut catch trial AND donut_cent need dobould stimuli drawn in centre
            if trial['Trial']['Name'] == 'Donut':
                if 'Catch' in trial['Trial']['Condition'] or 'Cent_donut' in trial['Trial']['Condition']:
                    trial['Trial']['Cent_2'].draw()

            # flip to window
            win.flip()

            # after set time, win cleared
            if timer.getTime()>stim_duration:
                win.flip()
                run = False
                break

        # append in data file if UI was reported or not
        if UI_was_seen == True:
            data['Uniformity'].append(1)
        else:
            data['Uniformity'].append(0)

        # MSG TO EYETRACKER
        #tracker.sendMessage(f'''MASK: trial index: {trials.thisIndex}. trial n: {trialNumber}, rep: {trials.thisRepN+1}, block: {block}''')
        # MASK = short blank before centre change to prevent
        # effects of movement/change on the retina in the centre only
        timer.reset()
        while timer.getTime()<(mask_duration):
            win.flip()

            if timer.getTime()>=(mask_duration):
                break

        # SEND MESSAGE TO EYE TRACKER
        #tracker.sendMessage(f'''CENTRE CHANGE: trial index: {trials.thisIndex}. trial n: {trialNumber}, rep: {trials.thisRepN+1}, block: {block}''')
        # 3. centre change
        # CENTRE CHANGE = the central stimuli drawn match the periphery
        # this while loop includes the reproduction task because I made a mess
        # I am too lazt to fix it so just dont touch anything
        myMouse.clickReset()
        run_centre_change = True
        # so the reproduction task doesnt start to run before mouse click
        run_repro = False
        # TIMER FOR REACTIONS TIMES
        RT_timer = psychopy.core.Clock()
        while run_centre_change and timer.getTime()>=mask_duration:
            # re-draw peripheral stimuli
            aperture.enabled = True
            aperture.inverted= True

            # draws the same periphery as before
            trial['Trial']['Periph'].draw()
            # only for Donut conditions:
            # incl catch and periph_donut because they both need a double centre
            if trial['Trial']['Name'] == 'Donut':
                if 'Catch' in trial['Trial']['Condition'] or 'Periph_donut' in trial['Trial']['Condition']:
                    trial['Trial']['Periph_2'].draw()

            # draw central stimuli
            aperture.inverted= False
            # for all other stimuli types
            if trial['Trial']['Name'] != 'Donut':
                trial['Trial']['Cent_new'].draw()

            if trial['Trial']['Name'] == 'Donut':
                trial['Trial']['Cent'].draw()
                trial['Trial']['Cent_2_new'].draw()

            win.flip()

            mouseClicks = [0]
            if block == 'centFill_RT':
                mouseClicks = myMouse.getPressed()
            # SET DURATION FOR BLACKOUT AND CENTFILL_REPRO TRIALS
            elif block == 'blackOut' or block == 'centFill_Repro':
                psychopy.core.wait(1.7)
                mouseClicks[0] = 1
                # SEND MESSAGE TO EYE TRACKER
                #tracker.sendMessage(f'''CENT CHANGE TIMEOUT: trial index: {trials.thisIndex}. trial n: {trialNumber}, rep: {trials.thisRepN+1}, block: {block}''')

            if mouseClicks[0]==1:
                # NO RT for black out trials, no uniformity reoccurs
                # NP RT for repro cent fill, set duration
                if block == 'blackOut' or block =='centFill_Repro':
                    data['RT'].append('NaN')
                elif block == 'centFill_RT':
                    # SEND MESSAGE TO EYE TRACKER
                    #tracker.sendMessage(f'''RT CENT CHANGE:  trial index: {trials.thisIndex}. trial n: {trialNumber}, rep: {trials.thisRepN+1}, block: {block}''')
                    data['RT'].append(round(RT_timer.getTime(),4))

                # so RT timer doesnt run in back
                RT_timer = None
                # reset timer for inter stim noise grating
                timer.reset()

                # after click, noise grating shown
                while timer.getTime()<inter_stim_dur:
                    if block == 'centFill_Repro' or block == 'blackOut':
                        # remove apature
                        aperture.enabled = False
                        noise.phase += (0.01 / 2, 0.005 / 2)
                        noise.draw()
                        win.flip()
                    elif block == 'centFill_RT':
                    # SKIPS THE NOISE BETWEEN REPRO CENT CHANGE
                        data['Reproduction'].append('NaN')
                        break

                # when the timer exceeds the noise stim duration (specified at
                # start of script) then reset mouse clicks and init reproduction
                # task by setting run_repro to TRUE and run_centre_change to FALSE
                # This way the loops do not over-ride eachother, and kinda resolves my mess
                if timer.getTime()>=inter_stim_dur:
                    if block == 'centFill_Repro' or block =='blackOut':
                        myMouse.clickReset()
                        run_repro = True
                        run_centre_change = False
                    elif block == 'centFill_RT':
                        run_repro = False

            # REPRODUCTION TASK
            # ONLY the periph is drawn up
            # scroll wheel changes the stimuli attributes of the periphery and
            # draws to screen, participant clicks right mouse to submit
            aperture.enabled = True
            aperture.inverted= True
            myMouse.clickReset()
            psychopy.event.clearEvents('mouse')

            while run_repro and timer.getTime()>inter_stim_dur:
                # to exit reproduction task left click
                leftClick_repro = myMouse.getPressed()[0]
                ## SCROLL UP = Negative
                ## SCROLL DOWN = Positive
                scroll = myMouse.getWheelRel()[1]
                # uses reproduction task element of stimulus dict
                # can scroll up and down to change size uptil reaching limits
                # limits = big and small stimuli +/-0.005 (bit of overshoot)

                if scroll < 0:
                    ## SIZE INCREASE
                    if trial['Trial']['Name'] == 'Size':
                        trial['Trial']['Repro'].sizes += [(0.002*scale),0.002]
                        if trial['Trial']['Repro'].sizes[1,1] > (big + 0.005):
                            trial['Trial']['Repro'].sizes -= [(0.002*scale),0.002]

                    ## COLOUR opacity INCREASE
                    elif trial['Trial']['Name'] == 'Colour':
                            trial['Trial']['Repro'].opacities += 0.02
                            if trial['Trial']['Repro'].opacities[1] > 1:
                                trial['Trial']['Repro'].opacities -= 0.02

                elif scroll > 0:
                    ## SIZE DECREASE
                    if trial['Trial']['Name'] == 'Size':
                        trial['Trial']['Repro'].sizes -= [(0.002*scale),0.002]
                        if trial['Trial']['Repro'].sizes[1,1] < (small - 0.005):
                            trial['Trial']['Repro'].sizes += [(0.002*scale),0.002]

                    ## COLOUR opacity DECREASE
                    elif trial['Trial']['Name'] == 'Colour':
                            trial['Trial']['Repro'].opacities -= 0.02
                            if trial['Trial']['Repro'].opacities[1] < 0:
                                trial['Trial']['Repro'].opacities += 0.02

                #trial['Trial']['Repro'].sizes *= [scale,1]

                if trial['Trial']['Name'] == 'Colour':
                    trial['Trial']['Periph'].draw()

                trial['Trial']['Repro'].draw()
                win.flip()

                if (leftClick_repro):
                    # append data of final reproduction to dataframe
                    if trial['Trial']['Name'] == 'Size':
                        data['Reproduction'].append(trial['Trial']['Repro'].sizes[1,1])
                    elif trial['Trial']['Name'] == 'Colour':
                        data['Reproduction'].append(trial['Trial']['Repro'].opacities[1])

                    run_repro = False

        ## EYE TRACKER RECORDING FOR THIS TRIAL DONE
        #tracker.setRecordingState(False)

        ### BACK UP DATA - live updating to txt file
        # make list of current trial data
        new_data_row = []
        # for every key in the data dict, append a new value
        for key in data:
        #for key in wtf_trial_numbers:
            # for the other trials: for key in keys, get the number
            # in each list appended to they keys that
            # corresponds to the current trial number
            # INDEXING STARTS AT 0 AND TRIALS START AT 1!!!
            # so TRIAL NUMBER - 1
            #new_data_row.append(wtf_trial_numbers[key][trialNumber-1])
            new_data_row.append(data[key][trialNumber-1])

        # Append new line to txt file
        new_data_row = ",".join(map(str, new_data_row))
        backUp.write(str(new_data_row)+"\n")

        # clear window
        win.flip()
        myMouse.clickReset()

## INIT DATA FRAME

data = {'Sub_ID':[],'Block_n':[],'Block_type': [],'Trial_n':[],'Trial_Rep_n':[],
        'Rep_n':[], 'Trial_index':[],'Stimuli':[],'Condition':[],'Exp':[],
        'Catch_UI':[], 'Catch_latency':[],'Cent_size':[],'Cent_opacity':[],
        'RT':[],'Reproduction':[],'Uniformity':[]}

# headers to txt file
backUp.write (str(list(data.keys()))+'\n')

# making a df
df = pd.DataFrame(data)

#### EXPERIMENT STRUCTURE
# Button for demos
instructions_button = psychopy.visual.Rect(win, width=0.2, height=0.1,
                            units='norm', fillColor=[1,1,1],
                            fillColorSpace='rgb', pos=(0.7,-0.7), size=None)

instructions_button_text = psychopy.visual.TextBox(win,
                         text='Ok, got it!',
                         font_size=30,
                         font_color=[0,0,0],
                         background_color=None,
                         size=(0.2,0.1),
                         pos=(0.7,-0.7),
                         grid_horz_justification='centre',
                         units='norm'
                         )

myMouse.clickReset()
experiment_run = True
START = True

while experiment_run:
    # instructions
    # welcome screen, press space tp start experiment
    if START == True:
        text_screen.setText(welcome)
        text_screen.draw()
        win.flip()
        psychopy.event.waitKeys()
        START == False

    # block counter
    block_n = 0

    # for every clock in the list of blocks (3 blocks)
    for block in block_list:
        win.setMouseVisible(True)

        block_n += 1

        instructions_1 = True
        instructions_2 = False
        demo_run = True
        demo_timer = psychopy.core.Clock()

        while demo_run == True:
            while instructions_1 == True:
                # instructions 1 + centre change demo
                if block == 'centFill_Repro' or block == 'blackOut':
                    keys = psychopy.event.getKeys(keyList=['space'])
                    demo_aperture.enabled = True
                    demo_aperture.inverted= True
                elif block == 'centFill_RT':
                    demo_aperture.enabled = False
                    keys = None

                # TEXT
                # for centre fill blocks, set the instructions
                if block == 'centFill_RT':
                    text_screen.setText(CF_instructions_RT)
                # for black out blocks, set the instructions
                elif block == 'centFill_Repro':
                    text_screen.setText(CF_instructions_REPRO_a)
                # for black out blocks, set the instructions
                elif block == 'blackOut':
                    text_screen.setText(BO_instructions_a)

                text_screen.draw()
                instructions_button.draw()
                instructions_button_text.draw()

                # draw periph
                if demo_timer.getTime() < 7:
                    demo_periph.draw()

                # CENTRE STIMULI CHANGE + PERIPH CHANGE
                # first centre drawn
                # ONLY REPRO AND BLACKOUT cent change
                if block == 'centFill_Repro' or block == 'blackOut':
                    demo_aperture.inverted= False
                    if demo_timer.getTime() < 4:
                        demo_cent.draw()
                    # draw new central stimuli
                    elif demo_timer.getTime() > 4 and demo_timer.getTime() < 7:
                        # for filled centre
                        if block == 'centFill_Repro':
                            demo_periph.draw()
                        # for black out centre
                        elif block == 'blackOut':
                            demo_cent.opacities = 0
                            demo_cent.draw()
                    # reset timer and opaicites
                    elif demo_timer.getTime() > 7.5:
                        demo_periph.opacities = 1
                        demo_cent.opacities = 1
                        demo_timer.reset()

                win.flip()

                if (keys):
                    instructions_1 = False
                    instructions_2 = True
                    # setting opacities to 1
                    demo_periph.opacities = 1
                    # set opacity of overlay color for repro task 0.5
                    demo_cent.opacities = 0.5
                elif myMouse.isPressedIn(instructions_button, buttons=[0]):
                    demo_timer = None
                    instructions_2 = False
                    instructions_1 = False
                    demo_run = False

            while instructions_2 == True:
                keys = psychopy.event.getKeys(keyList=['space'])
                scroll = myMouse.getWheelRel()[1]
                demo_aperture.inverted= True

                # TEXT
                # for centre fill blocks, set the instructions
                if block == 'centFill_Repro':
                    text_screen.setText(CF_instructions_REPRO_b)
                # for black out blocks, set the instructions
                if block == 'blackOut':
                    text_screen.setText(BO_instructions_b)
                text_screen.draw()
                instructions_button.draw()
                instructions_button_text.draw()

                # draw the background colour
                demo_periph.draw()
                demo_cent.draw()

                if scroll < 0:
                    demo_cent.opacities += 0.05
                    #demo_cent.draw()
                    if demo_cent.opacities[1] > 1:
                        demo_cent.opacities -= 0.05
                        #demo_cent.draw()

                elif scroll > 0:
                    demo_cent.opacities -= 0.05
                    #demo_cent.draw()
                    if demo_cent.opacities[1] < 0:
                        demo_cent.opacities += 0.05
                        #demo_cent.draw()

                demo_aperture.inverted= False

                win.flip()

                if (keys):
                    demo_timer.reset()
                    instructions_2 = False
                    instructions_1 = True
                    demo_periph.opacities = 1
                    demo_cent.opacities = 1
                elif myMouse.isPressedIn(instructions_button, buttons=[0]):
                    demo_timer = None
                    instructions_2 = False
                    instructions_1 = False
                    demo_run = False

        if demo_run == False:
            aperture.enabled = False
            text_screen.setText(start_eyetracking)
            text_screen.draw()
            win.flip()
        psychopy.event.waitKeys()

        # EYE TRACKER CALIBRATION
        #tracker.runSetupProcedure()

        text_screen.setText(start_of_block)
        text_screen.draw()
        win.flip()
        psychopy.event.waitKeys()

        fix.autoDraw = True
        win.setMouseVisible(False)

        if block == 'centFill_RT' or block == 'centFill_Repro':
            # RUNS TEST TRIALS FOR BLOCK
            test_trials_CF = psychopy.data.TrialHandler(trialList=CF_test_stim_list, nReps=2, method = 'sequential')
            # trial number starts at 0, will end at 4 after this practice block
            if block == 'centFill_RT':
                RUN_TRIALS(test_trials_CF, block, 'NaN', 0)
            elif block == 'centFill_Repro':
                RUN_TRIALS(test_trials_CF, block, 'NaN', len(data['Reproduction']))

            aperture.enabled = False
            # after test trials
            text_screen.setText(test_trials_complete)
            text_screen.draw()
            win.flip()
            psychopy.event.waitKeys()

            trials = psychopy.data.TrialHandler(trialList= centFill_stim_list, nReps=n_trials, method = 'random')
            # next trial number always starts at the current length of the dataframe - 1!
            RUN_TRIALS(trials, block, block_n, len(data['Reproduction']))

            # save dict as df
            df =  pd.DataFrame.from_dict(data)
            # save dataframe as csv
            df.to_csv(filename, index = False, header=True)

            fix.autoDraw = False
            aperture.enabled = False
            # END OF BLOCK BREAK
            text_screen.setText(end_of_block)
            text_screen.draw()
            win.flip()
            psychopy.event.waitKeys()

        elif block == 'blackOut':
            # RUNS TEST TRIALS FOR BLOCK
            test_trials_BO = psychopy.data.TrialHandler(trialList=BO_test_stim_list, nReps=2, method = 'sequential')
            RUN_TRIALS(test_trials_BO, block, 'NaN', len(data['Reproduction']))

            # after test trials
            aperture.enabled = False
            text_screen.setText(test_trials_complete)
            text_screen.draw()
            win.flip()
            psychopy.event.waitKeys()

            trials = psychopy.data.TrialHandler(trialList= blackOut_stim_list, nReps=n_trials, method = 'random')
            RUN_TRIALS(trials, block, block_n, len(data['Reproduction']))

            # save dict as df
            df =  pd.DataFrame.from_dict(data)
            # save dataframe as csv
            df.to_csv(filename, index = False, header=True)

        aperture.enabled = False
        fix.autoDraw = False

    # save dict as df
    df =  pd.DataFrame.from_dict(data)
    # save dataframe as csv
    df.to_csv(filename, index = False, header=True)
    # Close the file with live updating
    backUp.close()

    # end experiment
    text_screen.setText(end_of_experiment)
    text_screen.draw()
    win.flip()
    psychopy.event.waitKeys()

    # is set to true automatically at start
    # tracker.setConnectionState(False)

    experiment_run = False


import sys, heapq

def get_freq(msg):
	# create a char-freq mapping
	chars = dict()
	for char in msg:
		if chars.get(char):
			chars[char] +=1
		else:
			chars[char] = 1
	return chars.items()

def make_tree(freqs):

	# freq: list(tuple(int, str))

	heap = []
	for freq in freqs:
		heapq.heappush(heap, [freq])

	while len(heap)>1:
		left = heapq.heappop(heap)
		right = heapq.heappop(heap)
		freq_0, label_0 = left[0]
		freq_1, label_1 = right[0]
		node = [(freq_0 + freq_1, label_0+label_1), left, right]
		heapq.heappush(heap, node)
	return heap.pop()

def make_map(tree, map = dict(), prefix=''):
	# could also initialize global dict
	if len(tree) == 1:
		label, freq = tree[0]
		map[label] = prefix
	else:
		value, left, right = tree
		make_map(left, map, prefix+'0')
		make_map(right, map, prefix+'1')
	return map

def huffman_encoding(msg):
	tree = make_tree(get_freq(msg))
	map = make_map(tree)
	res = ''.join([map[letter] for letter in msg])
	return res, tree

def huffman_decoding(data, tree):
	tmp_tree = tree
	res = []

	for d in data:
		if d == '0':
			tmp_tree = tmp_tree[1]
		else:
			tmp_tree = tmp_tree[2]

		if len(tmp_tree)==1:
			label, freq = tmp_tree[0]
			res.append(label)
			tmp_tree = tree

	return ''.join(res)

if __name__ == "__main__":
    codes = {}

    a_great_sentence = "The bird is the word"

    print ("The size of the data is: {}\n".format(sys.getsizeof(a_great_sentence)))
    print ("The content of the data is: {}\n".format(a_great_sentence))

    encoded_data, tree = huffman_encoding(a_great_sentence)

    print ("The size of the encoded data is: {}\n".format(sys.getsizeof(int(encoded_data, base=2))))
    print ("The content of the encoded data is: {}\n".format(encoded_data))

    decoded_data = huffman_decoding(encoded_data, tree)

    print ("The size of the decoded data is: {}\n".format(sys.getsizeof(decoded_data)))
    print ("The content of the encoded data is: {}\n".format(decoded_data))

    print("----------------------------------------------")
    a_long_sentence = "kono dio da"
    print ("The content of the data is: {}\n".format(a_long_sentence))
    print ("The size of the data is: {}\n".format(sys.getsizeof(a_long_sentence)))
    encoded_data, tree = huffman_encoding(a_long_sentence)
    print ("The size of the encoded data is: {}\n".format(sys.getsizeof(int(encoded_data, base=2))))
    print ("The content of the encoded data is: {}\n".format(encoded_data))
    decoded_data = huffman_decoding(encoded_data, tree)
    print ("The content of the encoded data is: {}\n".format(decoded_data))
    print ("The size of the decoded data is: {}\n".format(sys.getsizeof(decoded_data)))

    print("----------------------------------------------")
    a_long_sentence = "yakamashii"
    print ("The content of the data is: {}\n".format(a_long_sentence))
    print ("The size of the data is: {}\n".format(sys.getsizeof(a_long_sentence)))
    encoded_data, tree = huffman_encoding(a_long_sentence)
    print ("The size of the encoded data is: {}\n".format(sys.getsizeof(int(encoded_data, base=2))))
    print ("The content of the encoded data is: {}\n".format(encoded_data))
    decoded_data = huffman_decoding(encoded_data, tree)
    print ("The content of the encoded data is: {}\n".format(decoded_data))
    print ("The size of the decoded data is: {}\n".format(sys.getsizeof(decoded_data)))


# https://www.acmicpc.net/problem/1929
# 1929번: 소수 구하기

m, n = map(int, input().split())
primeNum = []
if n>2:
    for i in range(m,n+1):
        if i==1:
            continue
        if i==2:
            primeNum.append(2)
            continue
        for j in range(2,n-1):
            if i%j==0:
                break
            if j >= i-1 and i%j != 0:
                primeNum.append(i)
elif n==2:
    primeNum.append(2)
# for item in primeNum:
#     print(item)
print(len(primeNum))


import re, os
import pandas as pd
import numpy as np
from nltk.tokenize import word_tokenize
from nltk import pos_tag
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from sklearn.preprocessing import LabelEncoder
from collections import defaultdict
from nltk.corpus import wordnet as wn
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.metrics import accuracy_score
from textclassify import Prediction
from nltk.stem import PorterStemmer
from visual import Visual

class Process:
    def __init__(self, pth, sf, sw, stem):
        self.path = pth
        self.singleFile = sf
        self.sw = sw
        self.stem = stem

    def cleanSW(self, words):
        sw = []
        swords = open("sw.txt", 'r')
        swords = swords.read()
        for word in swords.split():
            word =  word.strip()
            sw.append(word)
        out = [word for word in words.split() if not word.lower() in sw]
        output = " ".join(out)
        return output

    def cleanPunct(self, words):
        words = re.sub(r'[^\x00-\x7f]',r'', words) # removes garbage chars
        return re.sub(r'[^\w\s]', '', words) # removes punctuation

    def cleanNums(self, words):
        return re.sub(r'\d+', 'num', words)

    def stemWords(self, words):
        porter = PorterStemmer()
        out = ""
        for word in words.split():
            out = out + porter.stem(word) + ' '
        return out

    def cleanContents(self, content):
        if self.sw:
            content = self.cleanSW(content)
        if self.stem:
            content = self.stemWords(content)

        content = self.cleanNums(content)
        content = self.cleanPunct(content.lower())

        return content

    def preProcess(self):
        outputStr = "text,date\n"
        if self.singleFile:
            content = open(self.path, "r", encoding = "latin-1")
            link = content.readline()
            date = content.readline()
            article = content.read().replace('\n', '')
            article = self.cleanContents(article, self.sw)

            outputStr += article + "," + date
        else:
            for f in os.listdir(self.path):
                content = open(self.path + f, "r", encoding = "latin-1")
                link= content.readline()
                date=content.readline()
                article = content.read().replace('\n', '')

                article = self.cleanContents(article)

                outputStr += article + "," + date

        tempFile = open("temp-data.csv", 'w')
        tempFile.write(outputStr)
        tempFile.close()

    def process(self):
        predictor = Prediction("temp-data.csv")
        predictor.predict()

    def visualize(self, o):
        if o == 1:
            os.system("bokeh serve visual.py")
        elif o == 2:
            os.system("bokeh serve visual2.py")

path = os.path.dirname(os.path.abspath(__file__)) + '/separated'
test = Process(r'/Users/ryanhebert/Desktop/Research/Data/CNN/TweetsHK/', False, True, True)
test.preProcess()
test.process()
test.visualize(1)


import concurrent
from concurrent.futures import Future, ThreadPoolExecutor, ALL_COMPLETED
from dataclasses import dataclass
from typing import Callable, List, Tuple, Deque

from dragoneye.utils.app_logger import logger

@dataclass
class ThreadedFunctionData:
    callable: Callable
    args: Tuple
    error_msg: str
    timeout_msg: str = None

def execute_parallel_functions_in_threads(tasks_groups: Deque[List[ThreadedFunctionData]], max_workers, timeout=None)\
        -> List[Tuple[Future, str, str]]:
    """
    This function takes a Queue (tasks_groups: Deque[List[ThreadedFunctionData]])
    of lists, when each queued item represents a list of functions that should be run in parallel.
    """
    tasks_responses: List[Tuple[Future, str, str]] = []
    with ThreadPoolExecutor(max_workers=max_workers) as executor:

        for tasks_group in tasks_groups:
            for task in tasks_group:
                tasks_responses.append((
                    executor.submit(task.callable, *task.args),
                    task.error_msg, task.timeout_msg))

            concurrent.futures.wait([response[0] for response in tasks_responses], timeout=timeout, return_when=ALL_COMPLETED)

        timeout_tasks = [response for response in tasks_responses if response[0].running()]
        for timeout_task in timeout_tasks:
            logger.exception(timeout_task[2])
        failed_tasks = [response for response in tasks_responses if response[0].exception()]
        for failed_task in failed_tasks:
            logger.exception(failed_task[1], exc_info=failed_task[0].exception())

    return tasks_responses


# https://www.acmicpc.net/problem/2751
# 2751번: 수 정렬하기2
# 시간 초과

def mergeSort(arr):
    if len(arr) < 2:
        return arr

    mid = len(arr)//2
    frontArr = mergeSort(arr[mid:])
    backArr = mergeSort(arr[:mid])

    mergedSort = []
    l, h = 0, 0
    while l < len(frontArr) and h < len(backArr):
        if frontArr[l] < backArr[h]:
            mergedSort.append(frontArr[l])
            l += 1
        else:
            mergedSort.append(backArr[h])
            h += 1
    mergedSort += frontArr[l:]
    mergedSort += backArr[h:]
    return mergedSort

n = int(input())

number = []
for i in range(n):
    tmp = int(input())
    number.append(tmp)

sorted = mergeSort(number)
for item in sorted:
    print(item)


#!/usr/bin/env python

# Exercise is suppose we have a below list
# [1,2,3], [4,5,6], [7,8,9]
# Output should be average = (1+4+7)/3, (2+5+8)/3, (3,6,9)/3

# First we will do it for 2 lists using zip function
def average_finder(list1,list2):
	average = []
	for pair in zip(list1,list2): # pair value would be (1,4), (2,5), (3,6)
		average.append(sum(pair)/len(pair))
	return average

print(average_finder([1,2,3], [4,5,6]))
print("\n")

# Now we will make this anonymous function to pass for any number of list
def average_finder(*args):
	average = []
	for pair in zip(*args): # Imp here: As *args takes input in tuple: ([], []) and here using zip function we are unpacking those lists from tuple
		average.append(sum(pair)/len(pair))
	return average

print(average_finder([1,2,3], [4,5,6], [7,8,9])) # output is in floating point: [4.0, 5.0, 6.0]
print("\n")

# If we want output in integer just use '//' instead '/' in line #21

# Now we will do the same in just 2 lines using lambda and LC together
average_finder_1 = lambda *args:[sum(pair)//len(pair) for pair in zip(*args)]
print(average_finder_1([1,2,3], [4,5,6], [7,8,9]))

# Lets break Line #31
# 1. Meaning of lambda *args:[sum(pair)/len(pair) for pair in zip(*args)] (lambda syntax: lambda input:ouput)
# 2. here output is in Lc - [sum(pair)/len(pair) for pair in zip(*args)], first what we want is average of pairs from where in *args


"""boolean_default

Revision ID: 978129bf15fe
Revises: dfefdac3d90c
Create Date: 2021-03-04 15:29:30.427535

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '978129bf15fe'
down_revision = 'dfefdac3d90c'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column('samplers', 'private',
               existing_type=sa.BOOLEAN(),
               nullable=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column('samplers', 'private',
               existing_type=sa.BOOLEAN(),
               nullable=True)
    # ### end Alembic commands ###


#!/usr/bin/python3
import yaml
import json
import fileinput
import os

# ET CORE DOCKER COMPOSE FILES (ETM, EDM, EIM, ESM, EMP and EPM)
emp = '../emp/deploy/docker-compose.yml'
edm = '../edm/deploy/docker-compose.yml'
edm_lite = '../edm/deploy/docker-compose-lite.yml'
esm = '../esm/deploy/docker-compose.yml'
eim = '../eim/deploy/docker-compose.yml'
epm = '../epm/deploy/docker-compose.yml'

etm_complementary_singlenode = '../etm/deploy/docker-compose-complementary.yml'
etm_main_singlenode = '../etm/deploy/docker-compose-main.yml'
platform_services = '../platform-services/docker-compose.yml'

etm_complementary_lite = '../etm/docker/docker-compose-complementary.yml'
mysql_elasticsearch_lite = '../docker-compose-mysql-elasticsearch-lite.yml'
etm_main_lite = '../etm/docker/docker-compose-main.yml'
eim_in_etm = '../etm/elastest-torm/src/main/resources/test_engines/eim.yml'

etm_proxy = '../etm/docker/docker-compose-proxy.yml'

# ET TSS JSON FILES (EUS, EBS, ESS, EDS, EMS)
eus = '../eus/elastestservice.json'
ebs = '../ebs/elastestservice.json'
ess = '../ess/elastestservice.json'
eds = '../eds/elastestservice.json'
ems = '../ems/elastestservice.json'

# ET Test Engines DOCKER COMPOSE FILES (ECE and ERE)
ece = '../etm/elastest-torm/src/main/resources/test_engines/ece.yml'
ere = '../etm/elastest-torm/src/main/resources/test_engines/ere.yml'

# EUS Browsers file
eus_browsers_properties_file = '../eus/browsers/docker-browser.properties'
eus_properties_file = '../eus/application.properties'
etm_properties_file = '../etm/application.properties'

# Images to pull at the start
imagesFilesToPrePull = {'eusBrowsers': eus_browsers_properties_file,
                        'etmSocat': etm_properties_file,
                        'eus': eus}

tss_images_files = {'eus': eus, 'ebs': ebs, 'ess': ess, 'eds': eds, 'ems': ems}

def getCoreList():
    core_list = [emp, edm_lite, esm, eim, eim_in_etm, epm, etm_complementary_singlenode, platform_services,
                 etm_main_singlenode, etm_complementary_lite, etm_main_lite, etm_proxy]
    return core_list

def getCoreListByExecMode(mode):
    # Mini
    if (mode == 'mini'):
        core_list = [etm_complementary_lite,
                     etm_main_lite, etm_proxy, platform_services]
    # singlenode
    else:
        core_list = [emp, edm_lite, esm, eim, epm, etm_complementary_singlenode,
                     etm_main_singlenode, etm_proxy, platform_services]
    return core_list

def getTSSFile(tss):
    return tss_images_files[tss]

def getTSSList():
    tss_list = [eus, ebs, ess, eds, ems]
    return tss_list

def getEnginesList():
    engines_list = [ece, ere]
    return engines_list

def getFilePathByImage(imageToPull):
    return imagesFilesToPrePull[imageToPull]

# Yaml

def getYml(path):
    with open(path, 'r') as stream:
        try:
            return yaml.load(stream)
        except yaml.YAMLError as exc:
            return yaml.load('')

def saveYml(path, yml):
    with open(path, 'w') as outfile:
        yaml.dump(yml, outfile, default_flow_style=False)

# Json

def getJson(path):
    with open(path, 'r') as stream:
        data = json.load(stream)
        return data
    return json.load('')

def saveJson(path, json_file):
    with open(path, 'w') as outfile:
        json.dump(json_file, outfile, indent=4, separators=(',', ': '))

def getLineByContent(content, path):
    result = ''
    for line in fileinput.input(path):
        if(content in line):
            result = line
    return result

def checkIfDirExists(path):
    return os.path.exists(path)

def createDir(path):
    os.makedirs(path)

def checkIfFileExists(path):
    return os.path.isfile(path)

def readFileByLines(path, linesToRead):
    myfile = open(path)
    with myfile:
        head = [next(myfile) for x in xrange(linesToRead)]
    myfile.close()
    return head

def writeFile(path, content):
    file = open(path, 'a')
    file.write(content + '\n')
    file.close()


import math
import gym
from gym.utils import seeding
from SuperSonic.policy_definition.action import action_functions
from SuperSonic.policy_definition.observation import observation_function
from SuperSonic.policy_definition.reward import reward_function
import numpy as np
import sqlite3
import time

class StokeEnv(gym.Env):
    """A :class:
        STOKE is a stochastic optimizer and program synthesizer for the x86-64 instruction set.
        This classical compiler optimization task finds a valid code sequence to maximize the performance
        of a loop-free sequence of instructions. Superoptimizaiton is an expensive
        optimization technique as the number of possible configurations grows exponentially as
        the instruction count to be optimized increases.

    Source:
        This environment corresponds to the version of the STOKE
        described by stanfordPL. (https://github.com/StanfordPL/stoke)
        paper link: https://raw.githubusercontent.com/StanfordPL/stoke/develop/docs/papers/asplos13.pdf

    Observation:
        Type: Box(100)
        Optimized code will be convert to vectors by different embedding approaches,
        e.g. Word2vec, Doc2vec, CodeBert ...

    Actions:
        Type: Discrete(9)
        NUm      Action      Description
        0        add_nops	 Adds one extra nop instruction into the rewrite.
        1        delete	     Deletes one instruction at random.
        2        instruction Replaces an instruction with another one chosen at random.
        3        opcode	     Replaces an instruction's opcode with a new one that takes operands of the same type.
        4        operand	 Replaces an operand of one instruction with another.
        5        rotate	     Formerly "resize". Moves an instruction from one basic block to another, and shifts all the instructions in between.
        6        local_swap	 Takes two instructions in the same basic block and swaps them.
        7        global_swap Takes two instructions in the entire program and swaps them.
        8        weighted	 Selects from among several other transforms at random.

    Reward:
        In all cases, lower cost is better. We combine the value of correctness with other values we want to optimize for.
        Name	    Description
        binsize	    The size (in bytes) of the assembled rewrite using the x64asm library.
        correctness	How "correct" the rewrite's output appears. Very configurable.
        size	    The number of instructions in the assembled rewrite.
        latency	    A poor-man's estimate of the rewrite latency, in clock cycles, based on the per-opcode latency table in src/cost/tables.
        measured	An estimate of running time by counting the number of instructions actually executed on the testcases. Good for loops and algorithmic improvements.
        sseavx	    Returns '1' if both avx and sse instructions are used (this is usually bad!), and '0' otherwise. Often used with a multiplier like correctness + 1000*sseavx
        nongoal	    Returns '1' if the code (after minimization) is found to be equivalent to one in --non_goal. Can also be used with a multiplier.

    Starting State:
        All observations are assigned a uniform random value in [-1..1]

    """

    def __init__(
        self, state_function, action_function, reward_function
    ):
        """ Defines the reinforcement leaning environment. Initialise with an environment.

        :param state_function:  a state function that can summarize the program after each action as a
                                finite feature vector.
        :param action_function: an action function that can discrete set of actions or transformations that can be applied
                                to a program, such as passes in a compiler
        :param reward_function: a reward function that reports the quality of the actions taken so far.
        """

        self.state_function = state_function
        self.action_function = action_function
        self.reward_function = reward_function

        self.interleave_action_length, self.obsv_size = 9, 100
        self.obsv_low = 0
        self.obsv_high = 1
        self.action_space, self.observation_space = action_functions().init_actions(
            self.interleave_action_length,
            self.obsv_low,
            self.obsv_high,
            self.obsv_size,
            self.action_function,
        )

        self.state = None
        self.tstart = time.time()

    def get_reward(self,reward,reward_func):
        """ Calculate reward with method "reward_function".

            :param action: What will agent do in next step.
            :param reward: Describe how the agent "ought" to behave.
            :return: return a reward score after calculating.
        """
        reward = reward_function().get_rew(reward,reward_function
        )
        return reward

    def get_obs(self, action_history, state_code):
        """ feedback the observation with method "observation_function".

            :param action_history: This is the action history in past few steps.
            :param state_code: The X86 code after executing specific action.
            :return: return an embedding vector.
        """
        self.state_code = state_code
        self.input = [self.state_code]
        observation = observation_function().get_observation(
            self.input, self.obsv_size, self.state_function
        )
        return observation

    def step(self, action, state_code, reward):
        """Take a step.

            :param action: An action, or a sequence of actions. When multiple
                    actions are provided the observation and reward are returned after
                    running all of the actions.
            :param reward: A reward, to describe how the agent "ought" to behave.
            :param state_code: The X86 code after executing specific action.
            :return: A tuple of observation, reward, done, and info. Observation and
                    reward are None if default observation/reward is not set. If done is
                    True, observation and reward may also be None (e.g. because the
                    service failed).
        """
        assert self.action_space.contains(action), "%r (%s) invalid" % (
            action,
            type(action),
        )
        obs = self.get_obs(action, state_code)  # update observation
        self.state = obs
        done = True
        rew = self.get_reward(reward,self.reward_function)  # update reward
        info = {}
        return self.state, rew, done, info

    def reset(self):
        """ reset the RL environment.
        """
        self.state = np.random.random(100)
        return self.state

    def render(self, mode="human"):
        return None

    def close(self):
        return None

# if __name__ == '__main__':
#     env = StokeEnv("Doc2vec","transform","hamming")
#     env.reset()
#     env.step(env.action_space.sample())
#     print(env.state)
#     env.step(env.action_space.sample())
#     print(env.state)