"""
BASIC USAGE OF THE timeData Class AND SOME PLOTTINGS
In this file you

"""
# Change main directory to the main folder and import folders with own libraries
import os
os.chdir("../../")
import import_folders

####### Public Libraries ######
import datetime as dt
import matplotlib.pyplot as plt
import numpy as np

####### Own Libraries ######
from graph_lib import gl  # Own graphical library
import CTimeData as CTD  # Data Structures for basic timeseries
# Import functions independent of DataStructure
import utilities_MQL5 as ul5 # General utilities related to MQL5
import DDBB_lib as DBl      # Database library
plt.close("all") # Close all previous Windows

############ print options ############
storage_f = 0           # Print
show_data_shape = 0
guessing_markerhours = 0
basic_timeSeries_functions = 0

################# Plotting  Options! ################
basic_plotting = 0  # Basic chart with signal, volume, return
intrabydays_f = 0   # We divide the intraday data into its dayly components and plot all days on top of each other

################ SELECT SOURCE AND FOLDER NAMES ################
dataSource =  "MQL5"  # Hanseatic  FxPro GCI Yahoo
[storage_folder,  updates_folder] = ul5.get_foldersData(source = dataSource)
folder_images = "../pics/gl/"

######## SELECT SYMBOLS AND PERIODS ########
symbols = ["EURUSD", "Alcoa_Inc"]
periods = [15]

######## SELECT DATE LIMITS ###########
sdate = dt.datetime.strptime("15-8-2018", "%d-%m-%Y")
edate = dt.datetime.strptime("25-8-2018", "%d-%m-%Y")

######## CREATE THE OBJECT AND LOAD THE DATA ##########
# Create the information object for the timeSeries
timeData = CTD.CTimeData(symbols[0],periods[0])

# Create the pandas dataframe with the OHCLV timeSeries and add it to the timeData
# Both approaches do the same
if(1):
    TD = DBl.load_TD_from_csv(storage_folder, symbols[0],periods[0])
    timeData.set_TD(TD)
else:
    timeData.set_csv(storage_folder)  # Load the data into the model

# Set the interval period to be analysed
timeData.set_interval(sdate,edate, trim = True)

# In order to plot intraday data without discontnuities we need to know
# its market hours.
opentime, closetime = timeData.guess_openMarketTime()
period = timeData.guess_period() # Not needed but implemented
# Information needed for proper plotting.
dataTransform = ["intraday", opentime, closetime]

print ("Loaded: %s %i minutes"%(timeData.symbolID, timeData.period))
print ("Market Hours " + str(opentime) +" - " + str(closetime))
print ("Loaded dates: ", sdate," to ", edate)

################# STORAGE FUNCTIONS ##############
## Add more candlesticks from the MQL trading
if (storage_f == 1):
    timeData.add_csv(updates_folder) # Add more data to the tables
    timeData.save_to_csv(storage_folder)

############## Data filling  #################################
# TODO: call data filling functions

############## OBTAIN TIME SERIES #################################
# We can obtain some basic preprocessed time series
price = timeData.get_timeSeries(["Open","Close","Average"]);
dates = timeData.get_dates()  ## The dates of the selected data

if (show_data_shape):
    """
    Show the shape of different data structures
    """
    print (price.shape) # np.array(Nsamples, Nseries)
    print (dates.shape) # np.array(Nsamples, ) of selected Dates
    print (timeData.time_mask.shape) # # np.array(Nsamples, ) of indices of dates

############## OWN BASIC PLOTING FUNC #########################################
if (basic_timeSeries_functions):
    """
    Functions to obtain the basic timeSeries data from the object.
    """
    # If we do not specify new time series, we use the last ones, default = close.
    price = timeData.get_timeSeries(["RangeHL","RangeCO"]);
    price = timeData.get_timeSeries(["magicDelta"]);

    returns = timeData.get_timeSeriesReturn(["Close","Average"])
    cumReturns = timeData.get_timeSeriesCumReturn()
    SortinoRatio = timeData.get_SortinoR()
    get_SharpR = timeData.get_SharpR()

################# BASIC PLOTTING MIXTURES ############################
if (basic_plotting):
    """
    We aim to plot the price, volume, return and cummulative return for the selected security
    in the selected time frame.
    """
    gl.set_subplots(4,1)
    ############# 1: Basic Time Series and Volume
    seriesNames = ["Average", "High"]
    dataHLOC = timeData.get_timeSeries(["High","Low","Open","Close"])
    prices = timeData.get_timeSeries(seriesNames);
    dates = timeData.get_dates()
    volume = timeData.get_timeSeries(["Volume"]);
    Returns = timeData.get_timeSeriesReturn(seriesNames = ["Close"]);
    CumReturns = timeData.get_timeSeriesCumReturn(seriesNames = ["Close"]);
    nSMA = 10
    SMA = timeData.SMA(n = nSMA)

    ax1 = gl.plot(dates, SMA, labels = [timeData.symbolID + str(timeData.period), "Time", "Price"],
            legend = ["SMA(%i)" % nSMA], nf = 1, dataTransform = dataTransform,
            AxesStyle = "Normal - No xaxis", color = "cobalt blue")

    gl.barchart(dates, dataHLOC, lw = 2, dataTransform = dataTransform, color = "k",
                AxesStyle = "Normal - No xaxis", legend = ["OHCL"])
    # Notice the Weekends and the displacement between bars and step

    ############# 2: Volume
    gl.stem(dates, volume, sharex = ax1, labels = ["", "Time", "Volume"],
            legend = ["Volume"], nf = 1, alpha = 0.8, dataTransform = dataTransform,
            AxesStyle = "Normal - No xaxis")

    ############# 3: Returns
    gl.stem(dates, Returns, sharex = ax1, labels = ["", "Time", "Return"],
           legend = ["Return"], nf = 1, dataTransform = dataTransform,
           AxesStyle = "Normal - No xaxis")

    ############# 4: Commulative Returns
    seriesNames = ["Close"]

    gl.plot(dates, CumReturns, sharex = ax1,labels = ["", "Time", "Cum Return"],
           legend =  ["Cum Return"], nf = 1, dataTransform = dataTransform,
            AxesStyle = "Normal")
            ############## Dayly things obtaining  ####################################
# We separate the data into a list of days in order to be able to analyze it easier

if (dayly_data_f == 1):
    prices_day, dates_day = timeData.get_intra_by_days()

    plot_flag = 1
    for day in range (len(prices_day)):
        # type(dates_day[1]) <class 'pandas.tseries.index.DatetimeIndex'>
        # I could not find a fucking way to modify it inline TODO
        # This is done this way becasue we cannot asume that we have all the data
        # for all the days
        list_time = []
        for i in range(len(dates_day[day])):
            # datetime.replace([year[, month[, day[, hour[, minute[, second[, microsecond[, tzinfo]]]]]]]])
            dateless = dates_day[day][i].replace(year = 2000, month = 1, day = 1)
            list_time.append(dateless)
        gl.plot(list_time, prices_day[day],
                nf = plot_flag)
        plot_flag = 0

    gl.subplots_adjust(left=.09, bottom=.10, right=.90, top=.95, wspace=.20, hspace=0)
    image_name = "timeDataExample.png"
    gl.savefig(folder_images + image_name,
               dpi = 100, sizeInches = [30, 12])
    # TODO: Plot reconstruction to show that it is not the same.

if(intrabydays_f):
    """
    We aim to plot the intraday price and volume for different days on top of each other.
    For this purpose, we first divide the entire timeseries into its corresponding days and
    then we plot day by day
    """

    TD = timeData.get_TD()
    #caca = TD.groupby(TD.index.map(lambda x: x.date))
    caca = TD.groupby(TD.index.date)
    groups_of_index_dict = caca.groups # This is a dictionary with the dates as keys and the indexes of the TD as values
    days_dict = caca.indices # This is a dictionary with the dates as keys and the indexes of the TD as valu
    keys = days_dict.keys()# list of datetime.date objects
    keys = sorted(keys)
    set_indexes = days_dict[keys[0]]

    gl.set_subplots(2,1)
    ax1 = gl.plot([],[], nf = 1)
    ax2 = gl.plot([],[], nf = 1, sharex = ax1)

    for key in keys:
        set_indexes = days_dict[key]
        set_indexes.sort()
        set_indexes = timeData.time_mask[set_indexes] # Since it changed.

        times = timeData.get_dates(set_indexes).time
        values = timeData.get_timeSeriesCumReturn(["Close"],set_indexes)

        gl.plot(times, values, ax = ax1)
        volume = timeData.get_timeSeries(["Volume"],set_indexes)
        gl.scatter(times, volume, ax = ax2)

Examples/1.1 Trading Information Classes/1.1 main_CtimeData.py


from unicodedata import name
from django.urls import path
from . import views

urlpatterns=[
    path('',views.home,name='home'),
    path('about/',views.about,name='about'),
    path('dogs/',views.index,name='index'),
    path('dogs/<int:dog_id>/',views.detail,name='detail'),
    path('dogs/create',views.DogCreate.as_view(),name='dogs_create'),
    path('dogs/<int:pk>/update/',views.DogUpdate.as_view(),name='dogs_update'),
    path('dogs/<int:pk>/delete/',views.DogDelete.as_view(),name='dogs_delete'),
    path('dogs/<int:dog_id>/add_price',views.add_price,name='add_price'),
    path('sports/',views.SportList.as_view(),name='sports_index'),
    path('sports/<int:pk>/',views.SportDetail.as_view(),name='sports_detail'),
    path('sports/create/', views.SportCreate.as_view(), name='sports_create'),
    path('sports/<int:pk>/update/', views.SportUpdate.as_view(), name='sports_update'),
    path('sports/<int:pk>/delete/', views.SportDelete.as_view(), name='sports_delete'),
    path('dogs/<int:dog_id>/assoc_sport/<int:sport_id>/', views.assoc_sport, name='assoc_sport'),
]


### Code written for STDP tutorial @ CAMP 2016
### Adapted from Song et al 2000 (Competitive Hebbian Learning through spike-timing-dependent synaptic plasticity)
### With independent Poisson inputs (relevant figures : Fig 2)

### Author: Harsha Gurnani
### Date: June 15, 2016
#######################################

from brian2 import *
from time import time
set_device('cpp_standalone')

#### Simulation parameters ####
simtime = 150 *second
delta = 0.1*ms

defaultclock.dt = delta

########################
#### Model Parameters
########################

### Neuron and synapses:

taum = 20*ms		# Membrane time constant
Vrest = -70*mV		# Resting membrane potential
Erev = 0*mV         	# Excitatory synapse - reversal potential
Irev = -70*mV       	# Inhibitory synapse - reversal potential
taue = 5*ms		# Excitatory time constants
taui = 5*ms		# Inhibitory time constants

gmax = 0.015        	#Max excitatory conductance //in unnits of leak conductance
ginh = 0.05          	#Inhibitory conductance

Vt = -54*mV         	# Spike threshold
Vr = -60*mV         	# Reset potential

#### How does no. of synapses/ ratio influence firing rates?
Ne = 1000           	#No. of excitatory synapses
Ni = 200             	#No. of ihibitory synapses

# How does final distribution of weights depend on presynaptic firing rates? Why???
FRe = 15*Hz         	#Firing rate (FR) for excitatory input
FRi = 10*Hz         	#FR for inhibitory input

### Neuron model
eqs = Equations('''

    dV/dt = ( (Vrest - V) + ge*(Erev - V) + gi*(Irev - V) )/taum :volt
    dge/dt = -ge/taue :1        #Conductance of exc synapse
    dgi/dt = -gi/taui :1        #Conductance of inh synapse
''')

NRN = NeuronGroup(1, model=eqs, threshold = 'V>Vt', reset = 'V=Vr', method='euler')

########################
#### Synapses and STDP
########################

## STDP parameters for excitatory synapses
taupre = 20*ms		# Time constant of potentiation for pre-post pairing
taupost = 20*ms		# Time constant of depression for post-pre pairing
Weakratio = 1.05        # Apost*taupost/(Apre*taupre)
                        # Depression:Potentiation ratio (slightly greater than 1 for stabilising network)
Apre = 0.005            # %Strengthening with pre-post pair
Apost = Apre*(taupre/taupost)*Weakratio

### Excitatory Synapse Model:
syneqs = '''
    gsyn    :1
    dx/dt = -x/taupre :1	(event-driven)
    dy/dt = -y/taupost :1	(event-driven)
    '''

preeqs = '''
    ge_post += gsyn
    x += Apre
    gsyn += -y*gmax
    gsyn = clip(gsyn, 0, gmax)
    '''

posteqs = '''
    y += Apost
    gsyn += x*gmax
    gsyn = clip(gsyn, 0, gmax)
    '''

### Poisson Input at Excitatory synapses, at firing rate FRe
InpE = PoissonGroup(Ne, rates = FRe)

S_exc = Synapses(InpE, NRN, model=syneqs, on_pre=preeqs, on_post=posteqs )
S_exc.connect()				# Connect all
S_exc.gsyn[:] = gmax*rand(Ne)    	# Initialise uniformly between 0 and gmax

### Inhibitory synapses:
InpI = PoissonGroup(Ni, rates = FRi)

S_inh = Synapses(InpI, NRN, model = ' ', on_pre = '''gi_post += ginh''')	# Constant inhibitory conductance = ginh
S_inh.connect()

################
### Monitors
################

VMon = StateMonitor( NRN, 'V', record = 0 )			## Membrane potential
FR = PopulationRateMonitor(NRN)					## Firing rate of NRN
Weights = StateMonitor( S_exc, 'gsyn', record = [0,Ne-1])	## Two example synaptic weights

run( simtime, report='text')

figure()
### Histogram of sinal synaptic weights
subplot(211)
hist(S_exc.gsyn[:] /gmax, 20)
ylabel('No. of synapses')
xlabel('Normalised synaptic weight')

### Initial membrane potential trace
subplot(223)
plot(VMon.t[0:3000] /ms, VMon.V[0,0:3000] /mV)
ylim([-80,-40])
ylabel('Membrane potential (mV)')
legend('Initial V')

### Final membrane potential trace
subplot(224)
plot(VMon.t[-3000:-1] /ms, VMon.V[0,-3000:-1] /mV)
ylim([-80,-40])
legend('Final V')

figure()
### Evolution of Firing rate in time
subplot(211)
plot(FR.t /second, FR.smooth_rate(window='gaussian', width=100*ms)/Hz)
ylabel('Firing rate (Hz)')

### Evolution of two example synaptic weights
subplot(212)
plot(Weights.t /second, Weights.gsyn[0,:], 'b', label='Synapse 1')
plot(Weights.t /second, Weights.gsyn[1,:], 'g', label='Synapse N')
xlabel('Time (second)')
ylabel('Synaptic weight')

tight_layout()
show()


from network import *
import torch
import torch.nn as nn
from collections import OrderedDict
from torch import optim
import os
import torchvision
from PIL import Image
import numpy as np
from torchvision.utils import save_image
from network import *
from attention import *

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class Model(nn.Module):
    def __init__(self,hp):
        super(Model, self).__init__()
        self.hp = hp
        self.net = network()

        self.net.apply(weights_init_normal).to(device)

        self.sample_train_params = self.parameters()
        self.loss = nn.L1Loss()
        self.optimizer = optim.Adam(self.sample_train_params, hp.learning_rate)
        self.hp = hp
        self.step = 0

    def forward(self, img, img_gt):
        self.optimizer.zero_grad()

        output = self.net(img)

        loss = self.loss(output, img_gt)

        loss.backward()
        self.optimizer.step()

        return loss

    def append_dir(self, name):
        if not os.path.exists(self.hp.saved_models):
            os.mkdir(self.hp.saved_models)
        pathappend = self.hp.saved_models + "/" + name
        if not os.path.exists(pathappend):
            os.makedirs(pathappend)
        return pathappend

    def test(self, dataloader_Test, epoch):
        pathvrst = self.append_dir("val_output")
        self.optimizer.zero_grad()
        self.eval()
        img_stack = []
        for i, data in enumerate(dataloader_Test):
            img, img_gt= data
            img = img.to(device)
            with torch.no_grad():
                msk_big = self.net(img)
            img_stack.append(img[0])
            img_stack.append(img_gt[0])
            img_stack.append(msk_big[0])

            img_stack = torch.stack(img_stack, dim=0)
            save_image(img_stack, f"{pathvrst}/{i}+{str(epoch)+'epochs'}.png")

            img_stack = []

            # save_image(img_gt, f"{pathvrst}/{str(i) + 'gt'}+{str(epoch)+'epochs'}.png")
            # save_image(img, f"{pathvrst}/{str(i) + 'org'}+{str(epoch)+'epochs'}.png")


from django.contrib import admin
from django.urls import path,include

urlpatterns = [
    path('admin/', admin.site.urls),
    path('', include('Compte.urls')),
    path('dash', include('Dashbord.urls')),
    path('entreprise', include('Entreprise.urls')),
    path('employé', include('Employé.urls'))
]


# 9.1 2차 정렬
# 9.1.2 선택 정렬
# O(n^2), 안정적이지 않음
def selection_sort(seq):
    length = len(seq)
    for i in range(length-1):
        min_j = i
        for j in range(i+1, length):                # 최솟값을 찾는 과정
            if seq[min_j] > seq[j]:                 # 최솟값을 찾으면
                min_j = j
        seq[i], seq[min_j] = seq[min_j], seq[i]     # i번째 위치와 최솟값 위치를 바꿈

    return seq

def test_selection_sort():
    seq = [11, 3, 28, 43, 9, 4]
    assert(selection_sort(seq) == sorted(seq))
    print("테스트 통과!")

if __name__ == '__main__':
    test_selection_sort()


#!/usr/bin/python

parsed = {}
with open('/proc/meminfo') as meminfo:
        for line in meminfo:
                id, data = line.strip().split()[:2]
                parsed[id[:-1]] = int(data)

gb_total = parsed['MemTotal']

gb_used = (parsed['MemTotal'] + parsed['Shmem'] - parsed['MemFree'] -
           parsed['Buffers'] - parsed['Cached'] - parsed['SReclaimable'])

print( '%.0f MB' % (gb_used / 1024.0))


#!/usr/bin/python
# ex:set fileencoding=utf-8:

from __future__ import unicode_literals

from django.forms import ModelForm

from django.forms.models import inlineformset_factory

from djangobmf.forms import BMFForm
# rom djangobmf.layouts import Helper, Layout

from .models import Quotation, QuotationProduct

class QuotationUpdateForm(ModelForm):
    class Meta:
        model = Quotation
        exclude = ['quotation_number', ]

class QuotationCreateForm(ModelForm):
    class Meta:
        model = Quotation
        exclude = ['quotation_number', 'state', 'shipping_address', 'invoice_address', 'valid_until']

QuotationInlineFormset = inlineformset_factory(Quotation, QuotationProduct, extra=2, exclude=[])

class BMFQuotationUpdateForm(BMFForm):
    class Meta:
        form_class = QuotationUpdateForm
        inlines = {'products': QuotationInlineFormset}

class BMFQuotationCreateForm(BMFForm):
    class Meta:
        form_class = QuotationCreateForm
        inlines = {'products': QuotationInlineFormset}


import discord
from discord.ext import commands
import os
import random
from discord import TextChannel

class FunCog(commands.Cog):

    def __init__(self, bot):
        self.bot = bot
        self.whip_images = ["https://cdn.discordapp.com/attachments/460111213649461248/461643645985095690/whip2.gif", "https://cdn.discordapp.com/attachments/366335793658331137/461641836856082432/whip1.gif", "https://cdn.discordapp.com/attachments/460111213649461248/461643739677458454/whip6.gif", "https://cdn.discordapp.com/attachments/460111213649461248/461643930987921409/whip4.gif", "https://cdn.discordapp.com/attachments/460111213649461248/461643973937856522/whip3.gif", "https://cdn.discordapp.com/attachments/460111213649461248/461644021454864384/whip5.gif", "https://cdn.discordapp.com/attachments/460111213649461248/461644224396525569/whip10.gif", "https://cdn.discordapp.com/attachments/460111213649461248/461644305245667339/whip8.gif", "https://cdn.discordapp.com/attachments/460111213649461248/461644476641837067/whip9.gif", "https://cdn.discordapp.com/attachments/460111213649461248/461644603066548225/whip7.gif"]
        self.bam_images = ["https://cdn.discordapp.com/attachments/480798534409650186/509420445980819467/giphy_2.gif", "https://cdn.discordapp.com/attachments/480798534409650186/509420392558100501/giphy_3.gif", "https://cdn.discordapp.com/attachments/480798534409650186/509420341202780196/giphy_4.gif", "https://cdn.discordapp.com/attachments/480798534409650186/509420304708272159/giphy.gif", "https://cdn.discordapp.com/attachments/480798534409650186/509420229823037460/O3DHIA5.gif", "https://cdn.discordapp.com/attachments/480798534409650186/509420143839936522/azCR8D1.gif"]

    @commands.command()
    async def whip(self, ctx, target: discord.Member):
        whip_image = random.choice(self.whip_images)
        await ctx.send("You whipped the shit out of "+target.name, set_image=whip_image, image=False)

    @commands.command()
    async def bam(self, ctx, target: discord.Member):
        bam_image = random.choice(self.bam_images)
        await ctx.send(f"You bammed the shit out of {target.name}", set_image=bam_image, image=False)

    @commands.command()
    async def set_channel(self, ctx, channel_id):
        channel = self.bot.get_channel(int(channel_id))
        if channel:
            self.channel = channel
            await ctx.send("Setting the channel succeeded")
        else:
            await ctx.send("Setting the channel failed")

    @commands.command()
    async def speak(self, ctx):
        message = ctx.message.content.split(".speak ")
        message = message[1]
        if self.channel:
            await self.channel.send(message)
            await ctx.message.delete()
        else:
            await ctx.send("No channel set")

    async def get_user_reaction(self, ctx, message, reactions):

        reaction, user = await self.bot.wait_for('reaction_add',)

    @commands.Cog.listener("on_raw_reaction_add")
    async def on_reaction_add(self, payload):
        reactions = ["⚪", "⚫"]
        # YIN                                  #YANG
        role_ids = [611866314738368521, 611866869778874369]

        # The annoucement message id
        message_id = 611933810837422091
        # This means the reaction was done to the selected message
        guild = self.bot.get_guild(payload.guild_id)
        member = guild.get_member(payload.user_id)

        # Check if the member has one of the roles already
        for role in member.roles:
            if role.id in role_ids:
                return

        if payload.message_id == message_id:
            # If the reaction is one of the selected:
            print(payload.emoji)
            emoji = str(payload.emoji)
            if emoji in reactions:
                #White / Yang
                if emoji == "⚪":
                    role = guild.get_role(611866869778874369)
                    await self.assign_role(member, role)
                #black / Yin
                elif emoji == "⚫":
                    role = guild.get_role(611866314738368521)
                    await self.assign_role(member, role)

    async def assign_role(self, member, role):
        if member.bot:
            return
        print("Assign:", role, "to:", member)
        await member.add_roles(role)

def setup(bot):
    bot.add_cog(FunCog(bot))


import math

import numpy as np
import pandas as pd
from turfpy.measurement import boolean_point_in_polygon
from geojson import Point, Polygon, Feature

def tendancy(all_data, court_points, fps, video_width):
    '''
    This function predict
    0:other
    1:center
    2:left
    3:right
    '''
    region_points = calc_region_points(court_points, video_width)
    polygons = make_polygons(region_points)

    forehand_frames, backhand_frames = detect_fore_back_frame(all_data)

    forehand_regions = get_region_classes(all_data, forehand_frames, fps, polygons)
    backhand_regions = get_region_classes(all_data, backhand_frames, fps, polygons)

    region_ratio_forehand = get_region_ratio(forehand_regions, player_position=None, all_data=all_data)
    region_ratio_backhand = get_region_ratio(backhand_regions, player_position=None, all_data=all_data)
    region_ratio_left_forehand = get_region_ratio(forehand_regions, player_position='left', all_data=all_data)
    region_ratio_right_forehand = get_region_ratio(forehand_regions, player_position='right', all_data=all_data)
    region_ratio_left_backhand = get_region_ratio(backhand_regions, player_position='left', all_data=all_data)
    region_ratio_right_backhand = get_region_ratio(backhand_regions, player_position='right', all_data=all_data)

    return [region_ratio_forehand, region_ratio_backhand, region_ratio_left_forehand, region_ratio_right_forehand, region_ratio_left_backhand, region_ratio_right_backhand]

def calc_region_points(court_points, video_width):
    bottom_right = np.array(court_points['court_point_1'])
    top_right = np.array(court_points['court_point_2'])
    top_left = np.array(court_points['court_point_3'])
    bottom_left = np.array(court_points['court_point_4'])

    top_center_left = (1.1*top_right + 2*top_left)/3.1
    top_center_right = (2*top_right + 1.1*top_left)/3.1
    bottom_center_left = (3*bottom_right + 4*bottom_left)/7
    bottom_center_right = (4*bottom_right + 3*bottom_left)/7
    middle_center_left = (1*bottom_left + 2*top_left)/3
    middle_center_right = (1*bottom_right + 2*top_right)/3

    #
    intersection_1 = calc_intersection(top_center_left, bottom_center_left, top_center_right, bottom_center_right)
    intersection_2 = calc_intersection(top_center_left, bottom_center_left, middle_center_left, middle_center_right)
    intersection_3 = calc_intersection(top_center_right, bottom_center_right, middle_center_left, middle_center_right)
    intersection_4 = calc_intersection(intersection_2, intersection_3, [0, 0], [0, 1])
    intersection_5 = calc_intersection(intersection_2, intersection_3, [video_width-1, 0], [video_width-1, 1])
    delta = bottom_center_left - bottom_center_right
    temporary_point = intersection_1 + delta
    intersection_6 = calc_intersection(intersection_1, temporary_point, [0, 0], [0, 1])
    intersection_7 = calc_intersection(intersection_1, temporary_point, [video_width-1, 0], [video_width-1, 1])

    region_points = [intersection_1, intersection_2, intersection_3, intersection_4, intersection_5, intersection_6, intersection_7]

    return region_points

def calc_intersection(point1, point2, point3, point4):
    x1, y1, x2, y2, x3, y3, x4, y4 = point1[0], point1[1], point2[0], point2[1], point3[0], point3[1], point4[0], point4[1]
    if x1 == x2 and x3 == x4:
        x = y = np.nan
    elif x1 == x2:
        x = x1
        y = (y4 - y3) / (x4 - x3) * (x1 - x3) + y3
    elif x3 == x4:
        x = x3
        y = (y2 - y1) / (x2 - x1) * (x3 - x1) + y1
    else:
        a1 = (y2-y1)/(x2-x1)
        a3 = (y4-y3)/(x4-x3)
        if a1 == a3:
            x = y = np.nan
        else:
            x = (a1*x1-y1-a3*x3+y3)/(a1-a3)
            y = (y2-y1)/(x2-x1)*(x-x1)+y1
    return (x, y)

def make_polygons(intersections):
    center_polygon = Polygon([[
                intersections[0],
                intersections[1],
                intersections[2]
                ]])
    left_polygon = Polygon([[
                intersections[0],
                intersections[1],
                intersections[3],
                intersections[5]
                ]])
    right_polygon = Polygon([[
                intersections[0],
                intersections[2],
                intersections[4],
                intersections[6]
                ]])
    polygons = [center_polygon, left_polygon, right_polygon]

    return polygons

def judge_region(points, polygons):
    '''
    0:other
    1:center
    2:left
    3:right
    '''
    region_classes = []
    geopoints = [Feature(geometry=Point(point)) for point in points]

    for index, geopoint in enumerate(geopoints):
        for i, polygon in enumerate(polygons):
            if boolean_point_in_polygon(geopoint, polygon):
                resion_class = i + 1
                region_classes.append(resion_class)

        if len(region_classes) != (index + 1):
            region_classes.append(0)
    return region_classes

def detect_fore_back_frame(all_data):
    forehand_frames = []
    backhand_frames = []
    for frame in range(len(all_data)-3):
        if (all_data['modified_swing_classes'][frame] == 1) and (all_data['modified_swing_classes'][frame+1] != 1) and (all_data['modified_swing_classes'][frame+2] == 0) and (all_data['modified_swing_classes'][frame+3] != 1):
            # forehand
            forehand_frames.append(frame)
        elif (all_data['modified_swing_classes'][frame] == 2) and (all_data['modified_swing_classes'][frame+1] != 2) and (all_data['modified_swing_classes'][frame+2] == 0) and (all_data['modified_swing_classes'][frame+3] != 2):
            #backhand
            backhand_frames.append(frame)
    return forehand_frames, backhand_frames

def get_region_classes(all_data, frames, fps, polygons):
    regions = []
    for frame in frames:
        try:
            region_class = predcit_shot_region(all_data, frame, polygons, fps)
            regions.append([frame, region_class])
        except IndexError:
            pass
    regions = np.array(regions, dtype=int)

    return regions

def predcit_shot_region(all_data, frame, polygons, fps):
    frame_range = [frame + int(fps*0), frame + int(fps*1)]
    x_points = all_data['x_ball'][frame_range[0]:frame_range[1]]
    y_points = all_data['y_ball'][frame_range[0]:frame_range[1]]
    points = np.array([x_points, y_points], dtype=int).transpose().tolist()
    region_classes = judge_region(points, polygons)
    region_classes = np.delete(region_classes, np.where(region_classes == 0))
    unique, freq = np.unique(region_classes, return_counts=True)
    try:
        region_class = unique[np.argmax(freq)]
    except ValueError:
        region_class = 0

    # print(frame, frame_range[0], region_classes, frame_range[1])
    # print(region_class)

    return region_class

def get_region_count(regions, region_class, player_position=None, all_data=None, tennis_court_width=823):
    try:
        if player_position is None:
            count = np.count_nonzero(regions[:, 1] == region_class)
        elif player_position == 'left':
            count = np.count_nonzero(regions[all_data['x_player_converted'][regions[:, 0]].values < tennis_court_width/2, 1] == region_class)
        elif player_position == 'right':
            count = np.count_nonzero(regions[all_data['x_player_converted'][regions[:, 0]].values >= tennis_court_width/2, 1] == region_class)
        else:
            raise('input valid player_position')
    except IndexError:
        count = 0

    return count

def get_region_ratio(regions, player_position, all_data):
    num_center = get_region_count(regions, 1, player_position, all_data)
    num_left = get_region_count(regions, 2, player_position, all_data)
    num_right = get_region_count(regions, 3, player_position, all_data)

    num_total = num_center + num_left + num_right
    region_ratio = np.array([num_left, num_center, num_right])*100/num_total
    region_ratio = region_ratio.tolist()

    for i in range(3):
        if math.isnan(region_ratio[i]):
          region_ratio[i] = 'undetected'
        else: region_ratio[i] = int(region_ratio[i])
    return region_ratio


import cv2

# loading the frame
frame = cv2.cvtColor(cv2.imread('/home/zack/v1_robotws/src/trying_again/map_19.jpg'),cv2.COLOR_BGR2GRAY)
# thresholding the frame so if the value in a cell is 30 or above it is true showing its okat to go in
ret,thresh = cv2.threshold(frame,30,255,cv2.THRESH_BINARY)
# kernel for the mask
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
# closeing the image to remove anything that might be nothing
mask = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel)
# showing the masked image and the original image
cv2.imshow('regular image',frame)
cv2.imshow('masked image',mask)

while(1):
    if cv2.waitKey(1) & 0xFF == 27:
        break
cv2.destroyAllWindows()
# finding the contours in the mask
contours, hierarchy = cv2.findContours(mask,cv2.RETR_LIST,cv2.CHAIN_APPROX_NONE)


import math
import re
import numpy as np
from stemming.porter2 import stem
import random
import time

from scipy.sparse import dok_matrix
from scipy.stats import ttest_ind as t

from sklearn.decomposition import LatentDirichletAllocation
from sklearn.datasets import make_multilabel_classification

from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, precision_recall_fscore_support
from sklearn.pipeline import make_pipeline

from gensim.test.utils import common_texts, common_corpus
from gensim.corpora.dictionary import Dictionary
from gensim.models import LdaModel
from gensim.models import TfidfModel

def tt_test(a, b):
    print(t(a, b))

def format_number(n):
    # formats output numbers
    # all integers are cast to a string
    # all floats have three decimal places and are cast to a string

    # if sys/qrn num or already two decimal float
    # note query number goes to 10, so need to check for length 1 and 2
    if len(str(n)) == 1 or len(str(n)) == 2 or len(str(n)) == 4:
        return str(n)
    # if less than three decimal places add zeroes
    if len(str(n)) < 5:
        diff = 5-len(str(n))
        return str(n) + '0'*diff
    # if more than two decimal places then round
    if len(str(n)) > 4:
        # get the last digits and cast to integer
        to_round = int(str(n)[2:6])
        # get the final digit which will determine if rounding up or down
        final_digit = int(str(to_round)[-1])

        # if the last digit is less than five use floor function
        if final_digit < 5:
            post_digit = str(math.floor(to_round/10)/1000)
        # else use ceil function
        else:
            post_digit = str(math.ceil(to_round/10)/1000)

        # check if n is greater than 1, if so get the digits before the decimal point
        if n > 1:
            return str(n).split('.')[0] + post_digit[1:]
        else:
            return post_digit

class System:
    def __init__(self, num):
        # system number
        self.num = num
        # associated queries
        self.queries = []

    def add_query_result(self, query_result):
        # add a result to queries
        self.queries.append(query_result)

    def get_query(self, qry_num):
        # get list of QueryResults for qry_num
        queries = []
        for query in self.queries:
            if query.qry_num == qry_num:
                queries.append(query)
        return queries

class QueryResult:
    def __init__(self, sys_num, qry_num, doc_num, doc_rank, score):
        # has form system_number, query_number, doc_number, rank_of_doc, score
        self.sys_num  = sys_num
        self.qry_num  = qry_num
        self.doc_num  = doc_num
        self.doc_rank = doc_rank
        self.score    = score

    def pretty_print(self):
        # pretty printer for testing and debugging
        print("system:", self.sys_num, " query:", self.qry_num, " doc:", self.doc_num, " doc rank:",
              self.doc_rank, " score: ", self.score)

class SystemResults:
    def __init__(self):
        # list of system results
        self.systems = []

    def add_systems(self, systems):
        # add systems
        [add_system(sys) for sys in systems]

    def add_system(self, sys):
        # add system
        self.systems.append(sys)

    def get_system(self, num):
        # return system with corresponding system number
        for sys in self.systems:
            if sys.num == num:
                return sys

class Qrel:
    # list of Results for query x in 1-6
    # there will be 6 objects
    def __init__(self, qry_num):
        self.qry_num = qry_num
        self.res_list = []

    def add_result(self, res):
        self.res_list.append(res)

    def get_doc_rel(self, doc):
        # get relevance of document
        for res in self.res_list:
            if res.doc_num == doc:
                return res.rel
        # if document not present return 0
        return 0

class Result:
    # individual result
    def __init__(self, qry_num, doc_num, rel):
        self.qry_num = qry_num
        self.doc_num = doc_num
        self.rel = rel

    def pretty_print(self):
        print("query: ", self.qry_num, " doc: ", self.doc_num, " rel: ", self.rel)

class Qrels:
    def __init__(self):
        self.qrels = []

    def add_qrel(self, qrel):
        self.qrels.append(qrel)

    def get_qrel(self, qry_num):
        # get qrel from query number
        for qrel in self.qrels:
            if qrel.qry_num == qry_num:
                return qrel

class EVAL:
    def __init__(self):
        pass

    def read_in_sys_res(self, filename):

        curr_sys = 1
        system_results = SystemResults()
        sys = System(curr_sys)
        # don't want the first line as it is just format
        for line in open(filename, 'r').readlines()[1:]:
            # split input string on commas
            split = line.split(',')
            # cast values to appropriate type and add to list
            vals = [int(val) for val in split[:-1]] + [float(split[-1])]
            # create
            qry = QueryResult(vals[0], vals[1], vals[2], vals[3], vals[4])
            if qry.sys_num == curr_sys:
                # add query result to system
                sys.add_query_result(qry)
            else:
                # add system to system results
                system_results.add_system(sys)
                # increment sys number
                curr_sys += 1
                # create new system
                sys = System(curr_sys)
                # add qrt
                sys.add_query_result(qry)
        # catch the last sys
        system_results.add_system(sys)
        # for query in system_results.get_system(1).queries:
        #     query.pretty_print()
        return system_results

    def read_in_qrels(self, filename):

        # final object - list of qrels. one for each query
        qrels = Qrels()

        curr_query = 1
        # queries 1 to 6
        # list of Result objects for query
        qrel = Qrel(curr_query)

        for line in open(filename, 'r').readlines()[1:]:
            split = line.split(',')
            vals = [int(val) for val in split]

            res = Result(vals[0], vals[1], vals[2])
            if res.qry_num == curr_query:
                qrel.add_result(res)
            else:
                # add qrel to qrels
                qrels.add_qrel(qrel)
                # increment query number
                curr_query += 1
                # create new qrel
                qrel = Qrel(curr_query)
                # add res
                qrel.add_result(res)
        # catch the last qrel
        qrels.add_qrel(qrel)
        # for res in qrels.get_qrel(4).res_list:
        #     res.pretty_print()
        return qrels

    def get_system_results(self):
        # reads in system_results.csv
        # has form system_number, query_number, doc_number, rank_of_doc, score

        # get system_results
        return self.read_in("system_results.csv")

    def get_qrels(self):
        # reads in qrels.csv
        # has form query_id, doc_id, relevance

        # get qrels
        return self.read_in("qrels.csv")

    def evaluate(self):
        # gets top n results
        n = 10
        # uses precision or recall metric

        # fetch the system results and qrels
        # only want the top 10 results as cutoff at 10 and ranked linearly

        file = open("ir_eval.csv", 'w')

        # TODO: change this, they don't need to pass the filename
        qrels = self.read_in_qrels("qrels.csv")
        sys_res = self.read_in_sys_res("system_results.csv")

        # TODO: first beginning with system1, need top 10 results for each query

        # output to be written, updated every iteration
        # of form [system_number,query_number,P@10,R@50,r-precision,AP,nDCG@10,nDCG@20]
        output = ""
        # IR systems are numbered 1-6

        # write the header
        file.write("system_number,query_number,P@10,R@50,r-precision,AP,nDCG@10,nDCG@20\n")
        for sys_num in range(1, 7):
            # print("=-=-=-=-=-=-=-=-=-=-=-=-=- IR SYSTEM ", sys_num, " =-=-=-=-=-=-=-=-=-=-=-=-=-")
            # queries are numbered 1-10
            mean_vals = {}
            mean_counts = {}
            for qry_num in range(1, 11):
                output = ""
                print("-----")
                # if sys_num == 1 and qry_num == 10:
                #     for a in sys_res.get_system(1).get_query(10):
                #         a.pretty_print()
                print("-----")
                print(sys_num, " ", qry_num)
                # print("~~~~~~~~~~~~~ system:1 , query:", qry_num, " ~~~~~~~~~~~~~")
                a = self.precision(sys_res.get_system(sys_num).get_query(qry_num)[:10], qrels)
                b = self.recall(sys_res.get_system(sys_num).get_query(qry_num)[:50], qrels)
                c = self.ap(sys_res.get_system(sys_num).get_query(qry_num), qrels)
                d = self.r_precison(sys_res.get_system(sys_num).get_query(qry_num), qrels)
                e = self.nDCG(sys_res.get_system(sys_num).get_query(qry_num)[:10], qrels)
                f = self.nDCG(sys_res.get_system(sys_num).get_query(qry_num)[:20], qrels)

                metrics = [['a', a], ['b', b], ['c', c], ['d', d], ['e', e], ['f', f]]

                # add metrics to dictionary to calculate mean for each IR system
                for m in metrics:
                    if m[0] in mean_vals.keys():
                        mean_vals[m[0]] += m[1]
                        mean_counts[m[0]] += 1
                    else:
                        mean_vals[m[0]] = m[1]
                        mean_counts[m[0]] = 1

                # do formatting
                for count, i in enumerate([sys_num, qry_num] + [a, b, c, d, e, f]):
                    # if sys_num == 1 and qry_num == 1:
                    #     print("\n\n\n\n\nYESYESYESYESYESYESYESYESYESYESYESYES\n\n\n\n")
                    #     print(format_number(1))
                    # calculating output string for line
                    # for the sys_num and qry_num (count < 1) just append the number cast to a string
                    formatted = format_number(i)
                    # print("formatted: ", formatted)
                    output += formatted
                    output += ","
                print(output)
                # print(output)
                if output[-1] == ",":
                    output = output[:-1]
                file.write(output + '\n')

            output = ""
            # do formatting for mean
            mean_metrics = [mean_vals[k]/mean_counts[k] for k in mean_vals.keys()]
            for count, i in enumerate([sys_num, "mean"] + mean_metrics):
                # calculating output string for line
                # for the sys_num and qry_num (count < 1) just append the number cast to a string
                formatted = format_number(i)
                # print("formatted: ", formatted)
                output += formatted
                output += ","
                # print(output)
            if output[-1] == ",":
                output = output[:-1]

            # don't want blank line at the end of the file
            if sys_num == 6:
                file.write(output)
            else:
                file.write(output + '\n')
        file.close()

    def get_relevant_for_query(self, qrels, qry_num):
        relevant = []
        for rel in qrels.get_qrel(qry_num).res_list:
            relevant.append(rel.doc_num)
        return relevant

    def get_retrieved_for_query(self, sys):
        retrieved = []
        for result in sys:
            retrieved.append(result.doc_num)
        return retrieved

    def precision(self, sys, qrels):
        # precision at cutoff 10
        # using precision = relevant and retrieved / retrieved
        # that is P = TP / (TP + FP)

        # get the query number
        qry_num = sys[0].qry_num

        # get all retreived documents for the query
        retrieved = self.get_retrieved_for_query(sys)

        # get all relevant documents for the query
        relevant = self.get_relevant_for_query(qrels, qry_num)

        # true positives are in retrieved and relevant
        TP = len([r for r in retrieved if r in relevant])
        # false positives are in retrived but not relevant
        # FP = len([r for r in retrieved if r not in relevant])

        return TP/len(retrieved)

    def recall(self, sys, qrels):
        # precision at cutoff 10
        # using precision = relevant and retrieved / retrieved
        # that is P = TP / (TP + FP)
        # # exit()
        # for i in range(1,11):
        #     print(i, "==========================================================================")
        #     for rel in qrels.get_qrel(i).res_list:
        #         print(rel.doc_num)
        # exit()

        # get the query number
        qry_num = sys[0].qry_num

        # get all retreived documents for the query
        retrieved = self.get_retrieved_for_query(sys)

        # get all relevant documents for the query
        relevant = self.get_relevant_for_query(qrels, qry_num)
        # print(qry_num)
        # print(relevant)

        # true positives are in retrieved and relevant
        TP = len([r for r in retrieved if r in relevant])
        # false positives are in retrived but not relevant
        # FP = len([r for r in retrieved if r not in relevant])

        return TP / len(relevant)

    def ap(self, sys, qrels):
        # r: number of relevant docs for q
        # n: number of docs retrieved
        # P(k): p@k
        # Rel(k): 1 if retrieved doc@k is relevant otherwise 0

        # ap = 1/r * sum from k=1 to n: p@k * rel(k)
        qry_num = sys[0].qry_num

        # get all relevant documents for the query to calculate r

        relevant = self.get_relevant_for_query(qrels, qry_num)
        r = len(relevant)

        # get all retreived documents for the query to calculate n
        retrieved = self.get_retrieved_for_query(sys)
        n = len(retrieved)

        # summation loop
        sum = 0
        for k in range(1, n):
            sum += self.precision(sys[:k], qrels) * self.relevant(retrieved[k], relevant)
        return sum / r

    def r_precison(self, sys, qrels):
        # get the query number
        # r-precision is defined as r/R

        # where R is the total number of relevant documents
        # and r is the number of relevant documents out of the R fetched
        qry_num = sys[0].qry_num
        relevant = self.get_relevant_for_query(qrels, qry_num)
        R = len(relevant)

        retrieved = self.get_retrieved_for_query(sys[:R])

        r = len([r for r in retrieved if r in relevant])

        return r/R

    def relevant(self, doc, rels):
        # returns binary value; 1 if document is relevant to query, 0 otherwise
        return 1 if doc in rels else 0

    def nDCG(self, sys, qrels):
        # TODO: make sure it's top n
        # nDCG = rel_1 + sum_{i=2 to rank n} rel_i / log_2(i)

        # get the query number
        qry_num = sys[0].qry_num

        # get qrel for qry_num
        qrel = qrels.get_qrel(qry_num)

        # get all retreived documents for the query
        retrieved = self.get_retrieved_for_query(sys)

        # get all relevant documents for the query
        relevant = self.get_relevant_for_query(qrels, qry_num)

        # instansiate (actual) DCG to 0
        DCG = 0
        # instansiate (ideal) iDCG to 0
        iDCG = 0
        # loop through ranks 1 to n
        # TODO: check index
        for i in range(1, len(retrieved)):

            # get document relevance
            relevance = qrel.get_doc_rel(retrieved[i])
            if i == 1:
                iDCG += i

            # discount the gain if rank is not 1
            if i != 1:
                relevance /= math.log(i, 2)
                iDCG += math.log(i,2)

            # increment sum
            DCG += relevance
        nDCG = DCG / iDCG
        return nDCG

def N_t_c(term, t, c, docs, c_no):
    # t: boolean value
    # c: boolean value
    # there are four cases, N_0_0, N_0_1, N_1_0, N_1_1
    N_t_c = 0

    if c == 0:
        for doc_no, doc in enumerate(docs):
            if doc_no != c_no:
                if t == 0:
                    if term not in doc:
                        N_t_c += 1
                if t == 1:
                    if term in doc:
                        N_t_c += 1
    if c == 1:
        for doc_no, doc in enumerate(docs):
            if doc_no == c_no:
                if t == 0:
                    if term not in doc:
                        N_t_c += 1
                if t == 1:
                    if term in doc:
                        N_t_c += 1
    return N_t_c

def N_t(term, t, docs):
    # documents containing term
    # t: boolean value
    N_t = 0
    if t == 0:
        for doc in docs:
            if term not in doc:
                N_t += 1
    if t == 1:
        for doc in docs:
            if term in doc:
                N_t += 1
    return N_t

def N_c(c, N):
    # documents not in class c
    # c: boolean value
    if c == 0:
        return N - 1
    if c == 1:
        return 1

class PreProcessor:
    # this is an altered version of my code.py module from CW1

    def __init__(self):
        pass
    def read_in(self, filename):

        # if file to be read is stop words, return string
        if filename == "englishST.txt":
            return open(filename, 'r').read()
        # otherwise process different documents

        # holds different documents
        OT = []
        NT = []
        QR = []

        # iterate over input and split into documents
        lines = open(filename, 'r').readlines()
        for line in lines:
            # check the first letter to determine OT, NT, or QR
            if line[0] == 'O':
                OT.append(line)
            elif line[0] == 'N':
                NT.append(line)
            else:
                QR.append(line)
        # space = " "
        # print(len(OT), space, len(NT), space, len(QR))
        return OT, NT, QR

    def tokeniser(self, doc):
        tokens = []
        # iterate over lines in the document
        for line in doc:
            # split the line using regex, and iterate over the list
            # note the first item is discarded as we do not want "OT", "NT", or "Quran"
            for word in re.split(r'[^\w+\']+', line)[1:]:
                if word != "":
                    # append case folded word to token list
                    tokens.append(word.lower())
        # TODO: it looks like there are words like "'" in the token list
        # print(tokens)
        return tokens

    def sentence_tokeniser_and_stemmer(self, doc):
        # tokensises, stems, and stops
        sentences = []
        # read stop words from file
        stop_words = self.read_in("englishST.txt").split()
        # iterate over lines in the document
        # get list of lists
        for line in doc:
            # split the line using regex, and iterate over the list
            # note the first item is discarded as we do not want "OT", "NT", or "Quran"
            sentence = []
            # for word in re.split(r'[^\w+\']+', line)[1:]:
            for word in re.split(r'[^\w+\']+', line):
                # if word != "" and word not in ["ot", "nt", "quran", "OT",  "NT", "Quran"]: TODO: TODO TODO TODO TODO TODO TODO
                if word != "" and word != "'":
                    # append case folded word to sentence
                    sentence.append(word.lower())
            # use porter stemmer to stem tokens if not present in stop words list
            sentence = [stem(t) for t in sentence if t not in stop_words]
            # print(sentence)
            if sentence: sentences.append(sentence)
        return sentences

    def stopping_and_stemming(self, tokens):
        # performs stopping
        # makes a list of stop words and removes any such words in token stream

        # read stop words from file
        stop_words = self.read_in("englishST.txt").split()
        # use porter stemmer to stem tokens if not present in stop words list
        tokens = [stem(t) for t in tokens if t not in stop_words]
        return tokens

    def pre_process(self):
        print("preprocessing...")
        # get documents
        OT, NT, QR = self.read_in("train_and_dev.tsv")
        # pre-processed documents
        docs = []
        doc_verses = []

        # for each document
        for doc in [OT, NT, QR]:
            # tokenise
            # tokens = self.tokeniser(doc)
            # stop and stem
            # docs.append(self.stopping_and_stemming(tokens))
            doc_verses.append(self.sentence_tokeniser_and_stemmer(doc))
            # exit()
            # docs.append(self.sentence_tokeniser_and_stemmer(doc))
        # print(self.mutual_information(docs))

        # corpus = self.sentence_tokeniser_and_stemmer([OT, NT, QR])
        # self.LDA(docs, doc_verses)

        # print(self.mutual_information(docs))
        docs = doc_verses[0] + doc_verses[1] + doc_verses[2]
        # print(len(docs), "         +        ")
        # print("len of each doc: ", len(OT), " ", len(NT), " ", len(QR))
        # for doc in doc_verses:
        #     x,y = self.chi_mi(doc)
        #     print("CHI")
        #     print(x)
        #     print("MI")
        #     print(y)
        # exit()

        self.LDA(docs, doc_verses)
        print("Complete.")

    def chi_mi(self, docs):
        # x^2(D,t,c) = (A * B) / (C * D * E * F)

        chi_sqr = {}
        m_i = {}
        print("Beginning metric calculations...")
        for c_no, doc in enumerate(docs):
            N = len(docs)
            print(c_no, " / ", N, "...")
            used_terms = []
            for term in doc:
                if term not in used_terms:
                    used_terms.append(term)

                    # chi squared =-=-=-=-=-=-=-=-=-=-=-=-=-
                    # x^2(D,t,c) = (A * B) / (C * D * E * F)
                    N00 = N_t_c(term, 0, 0, docs, c_no)
                    N01 = N_t_c(term, 0, 1, docs, c_no)
                    N10 = N_t_c(term, 1, 0, docs, c_no)
                    N11 = N_t_c(term, 1, 1, docs, c_no)

                    # A = N00 + N01 + N10 + N11
                    # B = ((N11 * N00) - (N10 * N01)) ** 2
                    #
                    # C = N11 + N01
                    # D = N11 + N10
                    # E = N10 + N00
                    # F = N01 + N00
                    #
                    # if C*D*E*F <= 0:
                    #     cs = 0
                    # else:
                    #     cs = (A * B) / (C * D * E * F)
                    # # if not zero
                    # if cs != 0:
                    #     chi_sqr = self.form_top_ten(chi_sqr, cs, term)

                    # mutual information =-=-=-=-=-=-=-=-=-=-=-=-=-
                    # I = A + B + C + D

                    # Additional parameters required for mutual information
                    N1_ = N_t(term, 1, docs)
                    N_1 = N_c(1, N)
                    N0_ = N_t(term, 0, docs)
                    N_0 = N_c(0, N)

                    # values are calculated from another function as a divison by zero error may occur, which must be handled properly
                    # if the calculation fails, a value of zero is returned
                    A = self.try_calc(N, N11, N1_, N_1)
                    B = self.try_calc(N, N10, N1_, N_0)
                    C = self.try_calc(N, N01, N0_, N_1)
                    D = self.try_calc(N, N00, N0_, N_0)

                    I = A + B + C + D
                    # print(I)
                    # if not zero
                    if I != 0:
                        m_i = self.form_top_ten(m_i, I, term)

        return chi_sqr, m_i

    def try_calc(self, N, Ntc, Nt, Nc):
        # try except in order to make sure th log doesn't fail
        try:
            # print(N, Ntc, Nt, Nc)
            s1 = Ntc / N
            s2 = (N * Ntc) / (Nt * Nc)
            x = s1 * math.log(s2, 2)
            return x
        except:
            return 0

    def form_top_ten(self, dict, val, term):
        # need top ten terms
        # add to dictionary if in top ten
        to_remove = None

        if not dict:
            return {term:val}
        # print(len(dict))
        if len(dict) < 10:
            dict[term] = val
            return dict
        else:
            for k in dict.keys():
                if val > dict[k]:
                    to_remove = k
                    break
            if to_remove:
                del dict[to_remove]
                dict[term] = val
                return dict
            else:
                return dict

    def mutual_information(self, docs):
        # N - total number of documents
        # N_t_c - documents that do/don't contain e_t, e_c depending on the t, c values
        # N_t - documents that contain/don't contain e_t
        # N_c - documents that contain/don't contain e_c

        # list to hold the mutual information
        m_i = {}
        N = len(docs)
        print("Beginning mutual information calculations...")
        for c_no, doc in enumerate(docs):
            print(c_no)
            used_terms = []
            for term in doc:
                # print(m_i)
                formula = ""
                I = 0
                # print(term)
                if term not in used_terms:
                    used_terms.append(term)
                    for t in range(0, 2):
                        for c in range(0, 2):
                            # print("==============")
                            # print("term: ", term)
                            # print("N_", t,"_", c, ": ", self.N_t_c(term, t, c, docs, c_no))
                            # print("N_", t, ": ", self.N_t(term, t, docs))
                            # print("N_", c, ": ", self.N_c(c, N))
                            # print("==============")
                            formula += "N" + str(t) + str(c) + "/N log " + "(N N" + str(t) + str(c) + \
                                       ") /(N" + str(t) + "N" + str(c) + ")        +        "

                            Ntc = N_t_c(term, t, c, docs, c_no)
                            Nt = N_t(term, t, docs)
                            Nc = N_c(c, N)
                            # print(Ntc, Nt, Nc)
                            # need to check if the log will give an error
                            # if an exception is thrown, skip this
                            try:
                                s1 = Ntc / N
                                # print(s1)
                                s2 = (N*Ntc) / (Nt * Nc)
                                # print(s2)
                                I += s1 * math.log(s2, 2)
                                # print(I)
                            except:
                                pass
                    # if not zero
                    if I != 0:
                        # print("RETURNED SHOULD NOT BE NONE ", self.form_top_ten(m_i, I, term))
                        m_i = self.form_top_ten(m_i, I, term)
        return m_i

    def LDA(self, docs, doc_verses):
        # do I need OT, etc at the start of the line
        docs = [d[1:] for d in docs]
        # Create a corpus from a list of texts

        common_dictionary = Dictionary(docs)

        common_corpus = [common_dictionary.doc2bow(verse) for verse in docs]
        print(len(common_corpus), "    <- corpus length")
        # print(common_corpus)

        # Train the model on the corpus.
        print("model trained")
        lda = LdaModel(common_corpus, num_topics=20, id2word=common_dictionary)

        # apply lda to each doc corpus
        # ot_lda = lda[[common_dictionary.doc2bow(verse) for verse in doc_verses[0]]]
        # nt_lda = lda[[common_dictionary.doc2bow(verse) for verse in doc_verses[1]]]
        # qr_lda = lda[[common_dictionary.doc2bow(verse) for verse in doc_verses[2]]]
        #
        #
        #
        #
        #
        #
        #
        # print("~~~~~~~~~~~~~~~~~~~~~~~")
        # print("document topic probabilities for the three corpora...")
        # print("~~nt~~")
        # nt = [lda.get_document_topics(common_corpus[0])]
        # print(nt)
        # print("~~ot~~")
        # ot = [lda.get_document_topics(common_corpus[1])]
        # print(ot)
        # print("~~qe~~")
        # qr = [lda.get_document_topics(common_corpus[2])]
        # print(qr)
        # we need to go deeper...
        print("we need to go deeper...")

        l = lda.show_topics(num_topics=10, num_words=10, formatted=True)
        for i in l:
            print(i)

        # nt_docs = [doc for doc in common_corpus[0]]
        # print(docs[0])
        # loop through the list of verses for each holy book
        print(len(doc_verses))
        topics = []
        for doc_verse in doc_verses:
            # instansiate dictionaries
            topic_scores = {}
            topic_counts = {}
            # loop through the verses
            for verse in doc_verse:
                # get the document topic probabilities for the verse
                # topic_probs = lda.get_document_topics(common_dictionary.doc2bow(verse))
                topic_probs = lda[common_dictionary.doc2bow(verse)]

                # loop through these probabilities
                for tp in topic_probs:
                    # tp form is (topic, prob)

                    topic = tp[0]
                    prob = tp[1]

                    # add probability
                    if topic in topic_scores.keys():
                        topic_scores[topic] += prob
                    else:
                        topic_scores[topic] = prob
                    # update topic count in order to calculate mean
                    if topic in topic_counts.keys():
                        topic_counts[topic] += 1
                    else:
                        topic_counts[topic] = 1
            # calculate the mean probability for each topic
            for topic in topic_scores.keys():
                mean = topic_scores[topic] / topic_counts[topic]
                topic_scores[topic] = mean
            # get the topic with the highest mean probabiity
            most_freq_topic = max(topic_scores, key=topic_scores.get)
            # get the associated probability
            highest_topic_prob = topic_scores[most_freq_topic]
            print("=====////====")
            print("the predicted topic is topic ", most_freq_topic, " with probability ", highest_topic_prob)
            print("=====////====")
            topics.append(most_freq_topic)
            # get the top 10 most likely words for that topic

        for i in topics:
            print(i)
            print(lda.get_topic_terms(topicid=i, topn=10))
            for idx, prob in lda.get_topic_terms(topicid=i, topn=10):
                # print(idx, "     ", prob)
                print(common_dictionary[idx], "    ", prob)

        # for i in range(len(docs)):
            # print(lda.get_document_topics(common_dictionary.doc2bow(docs[i])))
        print("")
        print("~~~~~~~~~~~~~~~~~~~~~~~")
        #     lda.show_topic(i)
        #     print(lda.print_topic(i, 10))

        # for a in lda.get_document_topics(common_corpus):
        #     print(a)

        print("hello")

        pass

def unpack_array(arr):
    # unpacks numpy arrays to return first three items
    # used for unpacking precision_recall_fscore_support returned np arrays
    return [arr.item(idx) for idx in range(3)]

class TextClassifier:
    def __init__(self, pp):
        self.pre_processor = pp

    def pre_process_dev(self, dictionary, common_corpus):
        OT, NT, QR = self.pre_processor.read_in("test.tsv")

        # OT, NT, QR = self.preprocess_for_svm(d, stop_and_stem=True)

        docs = []
        # lines = open("")

        # iterate over documents
        for doc in [OT, NT, QR]:
            docs += self.preprocess_for_svm(doc, stop_and_stem=True)

        # train tfidf model
        tfidf = TfidfModel(common_corpus)
        # get X and y for each set
        pre_X, y = self.form_data_and_labels(docs)

        # get BOW representation and counts in order to form spare matrices
        X_corp = [dictionary.doc2bow(verse) for verse in pre_X]

        # get tf-idf weighting
        X_tfidf = [tfidf[verse] for verse in X_corp]

        # instansiate our sparse matrices
        X = dok_matrix((len(pre_X), len(dictionary.token2id.items())))

        # (i,j) is the count of the number of times the word j appears in document i
        # for i, loop over documents for Xtrn
        print("forming dev...")
        # number documents linearly from 0 - len(docs), i is the identifier and doc is the document
        for i, bow_doc in enumerate(X_tfidf):
            for id, count in bow_doc:
                X[i, id] = count
        print("finished dev...")
        return [X, y]

    def preprocess_for_svm(self, doc, stop_and_stem):
        # tokensises, does NOT stem or stop
        sentences = []
        # iterate over lines in the document
        for line in doc:
            # split the line using regex, and iterate over the list
            sentence = []
            for word in re.split(r'[^\w+\']+', line):
                if word != "" and word != "'":
                    # append case folded word to sentence
                    sentence.append(word.lower())
            if stop_and_stem:
                sentence = self.pre_processor.stopping_and_stemming(sentence)
            if sentence: sentences.append(sentence)
        return sentences

    def get_baseline_model(self, docs, doc_verses, dictionary, common_corpus, unique_tokens):
        n_terms = len(unique_tokens)

        # shuffle and split data into train and test
        train, test = self.shuffle_and_split(docs, "9:1")
        # train, test = self.shuffle_and_split(sentences, "7:3")
        print("split train length", len(train))

        # get X and y for each set
        pre_Xtrn, ytrn = self.form_data_and_labels(train)
        pre_Xtst, ytst = self.form_data_and_labels(test)

        # get lens for matrices
        pre_Xtrn_len = len(pre_Xtrn)
        pre_Xtst_len = len(pre_Xtst)
        print("pre-train length", pre_Xtrn_len)

        # instansiate our sparse matrices
        Xtrn = dok_matrix((pre_Xtrn_len, n_terms))
        Xtst = dok_matrix((pre_Xtst_len, n_terms))

        print("forming train set...")
        # fill matrix
        # TODO: can make this a loop
        # (i, j) is number of times word j appears in document i
        for i in range(pre_Xtrn_len):
            for j in range(n_terms):
                cnt = self.word_count(pre_Xtrn[i], unique_tokens[j])
                if cnt != 0:
                    Xtrn[i, j] = cnt
        print("finished train...")

        print("forming test set...")
        for i in range(pre_Xtst_len):
            for j in range(n_terms):
                cnt = self.word_count(pre_Xtst[i], unique_tokens[j])
                if cnt != 0:
                    Xtst[i, j] = cnt
        print("finished test...")

        print("training baseline model...")
        # now training the baseline SVC on train data
        baseline = SVC(C=1000)
        baseline.fit(Xtrn, ytrn)
        print("baseline model trained...")

        return baseline, [Xtrn, ytrn], [Xtst, ytst]

    def get_improved_model(self, docs, doc_verses, dictionary, common_corpus):
        # train tfidf model
        tfidf = TfidfModel(common_corpus)

        # shuffle and split data into train and test
        train, test = self.shuffle_and_split(docs, "9:1")

        # get X and y for each set
        pre_Xtrn, ytrn = self.form_data_and_labels(train)
        # pre_Xtst, ytst = self.form_data_and_labels(test)
        pre_Xtst, ytst = self.form_data_and_labels(test)

        # get BOW representation and counts in order to form spare matrices
        Xtrn_corp = [dictionary.doc2bow(verse) for verse in pre_Xtrn]
        Xtst_corp = [dictionary.doc2bow(verse) for verse in pre_Xtst]

        # get tf-idf weighting
        trn_tfidf = [tfidf[verse] for verse in Xtrn_corp]
        tst_tfidf = [tfidf[verse] for verse in Xtst_corp]

        # instansiate our sparse matrices
        Xtrn = dok_matrix((len(pre_Xtrn), len(dictionary.token2id.items())))
        Xtst = dok_matrix((len(pre_Xtst), len(dictionary.token2id.items())))

        # (i,j) is the count of the number of times the word j appears in document i
        # for i, loop over documents for Xtrn
        print("forming train set...")
        # number documents linearly from 0 - len(docs), i is the identifier and doc is the document
        for i, bow_doc in enumerate(trn_tfidf):
            for id, count in bow_doc:
                Xtrn[i, id] = count
        print("finished train...")

        print("forming test set...")
        for i, bow_doc in enumerate(tst_tfidf):
            for id, count in bow_doc:
                Xtst[i, id] = count
        print("finished test...")

        print("training improved model...")
        model = SVC(C=600)
        model.fit(Xtrn, ytrn)
        print("improved model trained...")
        return model, [Xtrn, ytrn], [Xtst, ytst]

    def classify(self):
        # extracts bag of words from corpus
        a = time.perf_counter()

        # read in the data
        OT, NT, QR = self.pre_processor.read_in("train_and_dev.tsv")

        # preprocess

        # all verses
        docs = []
        # verses for each document
        doc_verses = []
        # unique tokens
        unique_tokens = []

        # iterate over documents
        for doc in [OT, NT, QR]:
            verses = self.preprocess_for_svm(doc, stop_and_stem=True)

            # add unique terms
            for verse in verses:
                for token in verse:
                    if token not in unique_tokens:
                        unique_tokens.append(token)

            # add to nested document verse list
            doc_verses.append(verses)

            # add to verse list
            docs += verses

        # unique tokens
        dictionary = Dictionary(docs)
        # corpus of all verses
        common_corpus = [dictionary.doc2bow(verse) for verse in docs]

        # read in the dev set
        dev = self.pre_process_dev(dictionary, common_corpus)
        # print(dev)
        # exit()
        baseline_model, b_train, b_test = self.get_baseline_model(docs, doc_verses, dictionary, common_corpus, unique_tokens)
        improved_model, i_train, i_test = self.get_improved_model(docs, doc_verses, dictionary, common_corpus)

        f = open("classification.csv", 'w')
        acc_results = open("accuracy_results", 'w')

        f.write("system,split,p-quran,r-quran,f-quran,p-ot,r-ot,f-ot,p-nt,r-nt,f-nt,p-macro,r-macro,f-macro\n")

        c = ','
        for system, model in zip(["baseline", "improved"], [baseline_model, improved_model]):
            for split, data_set in zip(["train", "dev", "test"], [i_train, dev, i_test]):
                ypred = model.predict(data_set[0])
                print(split)
                print(data_set[1])
                print(ypred)
                precision, recall, fscore, _ = precision_recall_fscore_support(y_true=data_set[1], y_pred=ypred)

                acc_results.write(str(accuracy_score(y_true=data_set[1], y_pred=ypred)) + '\n')

                p_ot, p_nt, p_qr = unpack_array(precision)
                p_macro = np.sum(precision)
                r_ot, r_nt, r_qr = unpack_array(recall)
                r_macro = np.sum(recall)
                f_ot, f_nt, f_qr = unpack_array(fscore)
                f_macro = np.sum(fscore)

                output = ""
                for item in [system, split, p_qr, r_qr, f_qr, p_ot, r_ot, f_ot, p_nt, r_nt, f_nt, p_macro, r_macro, f_macro]:
                    print(item)
                    if isinstance(item, str):
                        output += item + c
                    # format not string values for output
                    else:
                        output += format_number(item) + c

                # replace end of line comma with newline
                output = output[:-1] + "\n"
                print("--------------------")

                f.write(output)
        f.close()
        acc_results.close()

        b = time.perf_counter()

        print("\n\ntime: ", b-a, "\n\n")
        print("Goodbye Cruel World")

    def shuffle_and_split(self, lst, ratio):
        # shuffles list of docs and splits into test and train
        # ratio of form n:1-n, meaning train=data[:n] test=data[n+1:] where n + (1-n) = 10

        # get multipliers from ratio
        trn_mult = int(ratio[0]) / 10 * len(lst)
        trn_mult = math.floor(trn_mult)
        # tst_mult = int(ratio[2]) / 10 * len(lst)

        print(len(lst))
        print(math.floor(trn_mult))
        # print(math.floor(tst_mult))

        # shuffle data
        random.shuffle(lst)

        # split
        # TODO: check that the split indices are correct
        return lst[:trn_mult], lst[trn_mult+1:]

    def form_data_and_labels(self, sentences):
        X = []
        y = []
        for sentence in sentences:
            # print(sentence)
            # the first word in the sentence defines the corpus
            # get the relevant id
            corpus = sentence[0]
            # print("CORPUS: ", corpus)
            y.append(self.get_corpus_id(corpus))
            # the remainder of the sentence is to be added to X
            X.append(sentence[1:])
        # ignore warning for making a ragged nested sequence TODO for now
        np.warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning)
        return np.array(X), np.array(y)

    def get_token_ids(self, unique_tokens):
        # enumerate tokens and use that as ID
        token_ids = {}
        for id, tok in enumerate(unique_tokens):
            token_ids[tok] = id
        return token_ids

    def get_corpus_id(self, corpus):
        if corpus == "ot":
            return 0
        if corpus == "nt":
            return 1
        if corpus == "quran":
            return 2

    def word_count(self, doc, word):
        # returns occurnces of word in doc
        count = 0
        # to speed up search, first ensure that word is present
        # trying to optimise this
        # if word in doc:
        #     print(doc.count(word))
        #     return doc.count(word)
        # else:
        #     return 0
        #
        if word in doc:
            for w in doc:
                if w == word:
                    count += 1
            return count
        else:
            return count

if __name__ == "__main__":
    print("Hello World")

    # IR EVALUATION =-=-=-=-=-=-=-=-=-=-=-=-=-#
    # ev = EVAL()                               #
    # ev.evaluate()                             #
    # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-#

    # TEXT ANALYSIS =-=-=-=-=-=-=-=-=-=-=-=-=-#
    pp = PreProcessor()
    # pp.pre_process()
    # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-#

    # TEXT CLASSIFICATION =-=-=-=-=-=-=-=-=-=-#
    tc = TextClassifier(pp)                   #
    tc.classify()                             #
    # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-#


import gym
import torch.optim as optim

from dqn_model import DQN
from dqn_learn import OptimizerSpec, dqn_learing
from utils.gym import get_env, get_wrapper_by_name
from utils.schedule import LinearSchedule
from fractalenv import *
from learn import *

BATCH_SIZE = 32
GAMMA = 0.99
REPLAY_BUFFER_SIZE = 1000000
LEARNING_STARTS = 50000
LEARNING_FREQ = 4
FRAME_HISTORY_LEN = 4
TARGER_UPDATE_FREQ = 10000
LEARNING_RATE = 0.00025
ALPHA = 0.95
EPS = 0.01

def main(env, num_timesteps):

    def stopping_criterion(env):
        return get_wrapper_by_name(env, "Monitor").get_total_steps() >= num_timesteps

    optimizer_spec = OptimizerSpec(
        constructor=optim.RMSprop,
        kwargs=dict(lr=LEARNING_RATE, alpha=ALPHA, eps=EPS),
    )

    exploration_schedule = LinearSchedule(1000000, 0.1)

    dqn_learing(
        env=env,
        q_func=DQN,
        optimizer_spec=optimizer_spec,
        exploration=exploration_schedule,
        stopping_criterion=stopping_criterion,
        replay_buffer_size=REPLAY_BUFFER_SIZE,
        batch_size=BATCH_SIZE,
        gamma=GAMMA,
        learning_starts=LEARNING_STARTS,
        learning_freq=LEARNING_FREQ,
        frame_history_len=FRAME_HISTORY_LEN,
        target_update_freq=TARGER_UPDATE_FREQ,
    )

if __name__ == '__main__':

    env = FractalEnv()

    main(env, 1000)


def find_wire_intersections(wires):
    fewest_combined_steps = -1
    wire_cell_dicts = []
    for wire_steps in wires:
        x, y = 0, 0
        total_distance = 0
        current_wire_cell_dict = {(x, y): total_distance}
        for vector in wire_steps:
            direction = vector[0]
            distance = int(vector[1:])
            while distance > 0:
                distance -= 1
                total_distance += 1
                if direction == 'R':
                    x += 1
                elif direction == 'L':
                    x -= 1
                elif direction == 'U':
                    y += 1
                elif direction == 'D':
                    y -= 1
                if (x, y) not in current_wire_cell_dict:
                    for wire_cell_dict in wire_cell_dicts:
                        if (x, y) in wire_cell_dict:
                            combined_steps = wire_cell_dict[x, y] + total_distance
                            if fewest_combined_steps == -1 or combined_steps < fewest_combined_steps:
                                fewest_combined_steps = combined_steps
                    current_wire_cell_dict[x, y] = total_distance
        wire_cell_dicts.append(current_wire_cell_dict)
    return fewest_combined_steps

with open('input/day03.txt') as input_file:
    first_wire_steps = input_file.readline().split(',')
    second_wire_steps = input_file.readline().split(',')
print(find_wire_intersections([first_wire_steps, second_wire_steps]))


import os
from builtins import bool
os.environ["CUDA_VISIBLE_DEVICES"]="6"

import torch
from torch import multiprocessing, cuda
from torch.utils.data import DataLoader
import torch.nn.functional as F
from torch.backends import cudnn
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np
import importlib

import imageio
import argparse
from data import data_voc, data_coco
from tool import torchutils, pyutils

cudnn.enabled = True

def overlap(img, seed_map):
    seed_map = seed_map.squeeze()
    seed_map = seed_map.astype(int)

    color = np.array([[0, 0, 0], [255, 0, 0]])
    seed_map = color[seed_map] * 2

    if seed_map.shape == np.array(img).astype(np.float).shape:
        out = (seed_map + np.array(img).astype(np.float)) / 3
        out = (out / np.max(out) * 255).astype(np.uint8)
    else:
        print(seed_map.shape)
        print(np.array(img).shape)
    return out

def draw_seedmap(seed_map, gt_label, orig_img, save_path, img_name):

    seed_map = F.interpolate(seed_map, orig_img.shape[:-1], mode='bilinear', align_corners=False)
    seed_map = seed_map.squeeze()
    seed_map = seed_map.numpy()

    gt_cat = np.where(gt_label==1)[0]

    for _, gt in enumerate(gt_cat):
        heatmap = overlap(orig_img, seed_map[gt])
        cam_viz_path = os.path.join(save_path, img_name + '_{}.png'.format(gt))
        imageio.imsave(cam_viz_path, heatmap)

def denormalization(image) :
    mean=(0.485, 0.456, 0.406)
    std=(0.229, 0.224, 0.225)

    if image.dim() == 3:
        assert image.dim() == 3, "Expected image [CxHxW]"
        assert image.size(0) == 3, "Expected RGB image [3xHxW]"

        for t, m, s in zip(image, mean, std):
            t.mul_(s).add_(m)
    elif image.dim() == 4:
        # batch mode
        assert image.size(1) == 3, "Expected RGB image [3xHxW]"

        for t, m, s in zip((0,1,2), mean, std):
            image[:, t, :, :].mul_(s).add_(m)

    return image

def _work(process_id, model, dataset, args):
    databin = dataset[process_id]
    n_gpus = torch.cuda.device_count()
    data_loader = DataLoader(databin, shuffle=False, num_workers=args.num_workers // n_gpus, pin_memory=False)
    os.makedirs(os.path.join(args.session_name, "visual_per_size"), exist_ok=True)

    with torch.no_grad(), cuda.device(process_id):

        model.cuda()
        model.training = False

        for iter, pack in enumerate(data_loader):
            img_name = pack['name'][0]
            label = pack['label'][0]
            size = pack['size']
            label = F.pad(label, (1, 0), 'constant', 1.0)

            # original
            #outputs = [model(img[0].cuda(non_blocking=True), label.cuda(non_blocking=True).unsqueeze(-1).unsqueeze(-1)) for img in pack['img']]

            # print(pack['img'].shape) # torch.Size([1, 2, 3, 281, 500])

            # with PAMR
            label_cuda = label.cuda(non_blocking=True).unsqueeze(-1).unsqueeze(-1)
            img = pack['img'][0]
            outputs = model(img.cuda(non_blocking=True), label_cuda, denormalization(img).cuda(non_blocking=True))

            # multi-scale fusion
            seeds_x1 = outputs[0].cpu()
            seeds_x2 = outputs[1].cpu()
            seeds_x3 = outputs[2].cpu()
            seeds_x4 = outputs[3].cpu()
            seeds_hie = outputs[4].cpu()

            # visualize seed map
            if args.visualize:
                orig_img = np.array(Image.open(pack['img_path'][0]).convert('RGB'))
                draw_seedmap(seeds_x1, label, orig_img, os.path.join(args.session_name, 'visual'), f"{img_name}_x1")
                draw_seedmap(seeds_x2, label, orig_img, os.path.join(args.session_name, 'visual'), f"{img_name}_x2")
                draw_seedmap(seeds_x3, label, orig_img, os.path.join(args.session_name, 'visual'), f"{img_name}_x3")
                draw_seedmap(seeds_x4, label, orig_img, os.path.join(args.session_name, 'visual'), f"{img_name}_x4")
                draw_seedmap(seeds_hie, label, orig_img, os.path.join(args.session_name, 'visual'), f"{img_name}_hie")

            if process_id == n_gpus - 1 and iter % (len(databin) // 100) == 0:
                print("%d " % ((5*iter+1)//(len(databin) // 100)), end='')

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument("--network", default="network.resnet50_AE_SIPE", type=str)
    # parser.add_argument("--num_workers", default=os.cpu_count()//2, type=int)
    parser.add_argument("--num_workers", default=1, type=int)
    parser.add_argument("--session_name", default="exp", type=str)
    parser.add_argument("--ckpt", default="final.pth", type=str)
    parser.add_argument("--visualize", default=True, type=bool)
    parser.add_argument("--dataset", default="voc", type=str)
    parser.add_argument("--make_dataset", default='train', type=str) # trainaug

    args = parser.parse_args()

    os.makedirs(os.path.join(args.session_name, 'npy'), exist_ok=True)
    os.makedirs(os.path.join(args.session_name, 'visual'), exist_ok=True)
    pyutils.Logger(os.path.join(args.session_name, 'infer.log'))
    print(vars(args))

    assert args.dataset in ['voc', 'coco'], 'Dataset must be voc or coco in this project.'

    if args.dataset == 'voc':
        dataset_root = '../PascalVOC2012/VOCdevkit/VOC2012'
        model = getattr(importlib.import_module(args.network), 'CAM')(num_cls=21)
        # dataset = data_voc.VOC12ClsDatasetMSF(f'data/{args.make_dataset}_' + args.dataset + '.txt', voc12_root=dataset_root, scales=(1.0, 0.5, 1.5, 2.0))
        dataset = data_voc.VOC12ClsDatasetMSF(f'data/{args.make_dataset}_' + args.dataset + '.txt', voc12_root=dataset_root, scales=(1.0, ))

    elif args.dataset == 'coco':
        dataset_root = "../ms_coco_14&15/images"
        model = getattr(importlib.import_module(args.network), 'CAM')(num_cls=81)
        dataset = data_coco.COCOClsDatasetMSF('data/train_' + args.dataset + '.txt', coco_root=dataset_root, scales=(1.0, 0.5, 1.5, 2.0))

    checkpoint = torch.load(args.session_name + '/ckpt/' + args.ckpt)
    model.load_state_dict(checkpoint['net'], strict=True)
    model.eval()

    n_gpus = torch.cuda.device_count()

    dataset = torchutils.split_dataset(dataset, n_gpus)

    print('[ ', end='')
    multiprocessing.spawn(_work, nprocs=n_gpus, args=(model, dataset, args), join=True)
    print(']')

    torch.cuda.empty_cache()


import os
import torch
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import torch.nn.functional as F
import cv2

def tensor2numpy(tensor, batch_idx=0):
    if tensor is not None and torch.is_tensor(tensor):
        if len(tensor.shape) == 4:
            tensor = tensor.permute(0, 2, 3, 1) # BxHxWxC
        tensor = tensor.clone().detach().cpu().numpy()[batch_idx, ...]
    return tensor

def restore_from_norm(img, normalize_mean = [0.485, 0.456, 0.406], normalize_std = [0.229, 0.224, 0.225]):
    img = img * normalize_std + normalize_mean
    img = np.clip(img, 0, 1)
    return img

def imshow(ax, im, title=''):
    ax.cla()
    ax.axis('off')
    if im is not None:
        ax.imshow(im)
        ax.set_title(title)

def add_rect(ax, bbox, color='r'):
    if bbox is not None:
        rect = patches.Rectangle((bbox[0], bbox[1]), bbox[2], bbox[3],
                                  linewidth=1, edgecolor=color, facecolor='none')
        ax.add_patch(rect)

def visualize_results(fig, axes, frame_id,
                      color, depth, gt_bbox, search_color, search_depth,
                      pred_mask, pred_bbox, pred_prob, pred_bbox_in_search,
                      use_norm=True):

    if fig is None and len(axes) == 0:
        fig, axes = plt.subplots(2, 4, figsize=(6, 6))

    color = tensor2numpy(color)
    depth = tensor2numpy(depth)
    gt_bbox = tensor2numpy(gt_bbox) # (4,)

    search_color = tensor2numpy(search_color)
    search_depth = tensor2numpy(search_depth)
    pred_prob = tensor2numpy(pred_prob) # HxWx1
    pred_mask = tensor2numpy(pred_mask) # HxWx1
    pred_bbox = tensor2numpy(pred_bbox) # (4,)
    pred_bbox_in_search = tensor2numpy(pred_bbox_in_search) # (4,)

    if use_norm:
        # color = restore_from_norm(color)
        # depth = restore_from_norm(depth)
        search_color = restore_from_norm(search_color)
        search_color = np.clip(search_color, 0.0, 1.0)
        search_depth = restore_from_norm(search_depth)
        search_depth = np.clip(search_depth, 0.0, 1.0)

    plt.cla()
    imshow(axes[0, 0], color, title='Frame %d, Color'%frame_id)
    imshow(axes[0, 1], depth, title='Depth')

    imshow(axes[1, 0], search_color, title='search color')
    imshow(axes[1, 1], search_depth, title='search depth')

    imshow(axes[2, 0], pred_mask, title='pred mask')
    imshow(axes[2, 1], pred_prob, title='pred prob : %f'%np.max(pred_prob))

    add_rect(axes[0, 0], gt_bbox, color='g')
    add_rect(axes[0, 0], pred_bbox, color='r')
    add_rect(axes[1, 0], pred_bbox_in_search, color='r')

    plt.pause(0.0001)
    plt.show(block=False)

def save_debug(temp_path, epoch, batch,
               train_images, train_depths, train_masks, train_boxes,
               test_images, test_depths, test_masks, test_boxes, test_gauss,
               prediction_mask, scores):

    train_rgb = tensor2numpy(train_images)
    train_rgb = restore_from_norm(train_rgb)
    train_d = tensor2numpy(train_depths)
    train_d = restore_from_norm(train_d)

    train_m = tensor2numpy(train_masks)
    train_b = tensor2numpy(train_boxes)

    test_rgb = tensor2numpy(test_images)
    test_rgb = restore_from_norm(test_rgb)

    test_d = tensor2numpy(test_depths)
    test_d = restore_from_norm(test_d)

    test_m = tensor2numpy(test_masks)
    test_b = tensor2numpy(test_boxes)
    test_g = tensor2numpy(test_gauss) # Bx7x7

    scores = tensor2numpy(scores) # Bx7x7

    prediction_mask = tensor2numpy(F.softmax(prediction_mask, dim=1))
    pred_fg = prediction_mask[:, :, 0]
    # pred_bg = prediction_mask[:, :, 1]

    fig, axes = plt.subplots(3, 3, figsize=(8, 8))

    imshow(axes[0, 0], train_rgb, title='train rgb')
    add_rect(axes[0, 0], train_b, color='r')
    imshow(axes[0, 1], train_d, title='depth')
    imshow(axes[0, 2], train_m, title='mask')

    imshow(axes[1, 0], test_rgb, title='test rgb')
    add_rect(axes[1, 0], test_b, color='r')
    imshow(axes[1, 1], test_d, title='depth')
    imshow(axes[1, 2], test_m, title='mask')

    imshow(axes[2, 0], test_g, title='test score')
    imshow(axes[2, 1], scores, title='pred score : %f'%np.max(scores))
    imshow(axes[2, 2], pred_fg, title='pred mask')

    plt.savefig(os.path.join(temp_path,'debug-%d-%d.png'%(epoch, batch)))
    plt.close(fig)


import sys
import os
import json
import glob
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestClassifier

def main():
    # Checks
    checkUsage()

    dataSetPath = sys.argv[1]
    testMethodsPath = sys.argv[2]

    dataSet = pd.read_json(dataSetPath)
    testMethods = pd.read_json(testMethodsPath)

    # Build X_train
    x_train = dataSet.iloc[:, [1,2,3,6,7,8]].values
    print("X train: ")
    print(x_train)
    print("\n")

    # Build Y_train
    y_train = dataSet.iloc[:, 4].values
    print("Y train: ")
    print(y_train)
    print("\n")

    # Build X_test
    x_test = testMethods.iloc[:, [1,2,3,6,7,10]].values

    # Create classifier object
    classifier = RandomForestClassifier(n_estimators = 100, random_state = 0)

    # Fit the regressor with x and y data
    classifier.fit(x_train, y_train)

    print("\nTest Methods predictions:")
    prediction = classifier.predict_proba(x_test)
    #print(prediction)

    # Global Analysis
    flakyTestsFound = []
    nbTests = len(prediction)
    for i in range(0, nbTests):
        el = prediction[i]

        # Count number of Flaky Tests
        if el[0] < 0.5:
            flakyTestsFound.append(i)

    print("Number of Tests: ", str(nbTests))
    print("Number of Flaky Tests: ", str(len(flakyTestsFound)))
    print("Percentage: ", str(int(100 * len(flakyTestsFound) / nbTests)),"%")

    # Flaky Tests Found Analysis
    # for flakyTestFoundIndex in flakyTestsFound:
    #     print("\n")
    #     print(testMethods.iloc[flakyTestFoundIndex])

    print(classifier.feature_importances_)

def checkUsage():
    #Check the programs' arguments
    if len(sys.argv) != 3 or not os.path.isfile(sys.argv[1]) or not os.path.isfile(sys.argv[2]):
        print("Usage: python3 testModel.py [path/to/dataset.json] [path/to/testMethods.json]")
        sys.exit(1)

if __name__ == "__main__":
    main()


""" Urls for ecommerce app. """

from django.conf.urls.defaults import patterns, url

from ecommerce.views.add_flyers_views import (show_add_flyer_by_list,
    show_add_flyer_by_map, show_add_flyer_dates, show_buy_market_flyer)
from ecommerce.views.ecommerce_views import show_receipt
from ecommerce.views.add_slot_views import show_add_a_new_display

urlpatterns = patterns('',
    url(r'^receipt/(?P<order_id>\d+)/$', show_receipt, name="receipt"),
    url(r'^add-flyer-by-map/(?P<slot_id>\d+)/$', show_add_flyer_by_map,
        name='add-flyer-by-map'),
    url(r'^add-flyer-by-list/(?P<slot_id>\d+)/$', show_add_flyer_by_list,
        name='add-flyer-by-list'),
    url(r'^buy-market-flyer/(?P<slot_id>\d+)/$', show_buy_market_flyer,
        name='buy-market-flyer'),
    url(r'^add-flyer-dates/(?P<slot_id>\d+)/$', show_add_flyer_dates,
        name='add-flyer-dates'),
    url(r'^add-a-display/(?P<coupon_id>\d+)/$', show_add_a_new_display,
        name="add-a-new-display"),
)


# -*- coding: utf-8 -*-
"""
Created on Fri Apr 22 22:04:42 2022

@author: kalya_kl8c3da
"""

import os
import cv2
import timeit
import matplotlib.pyplot as plt

'''
dataset_path = 'C:/Users/kalya_kl8c3da/Downloads/facedata/dataset/celebrity_face_dataset/'
path = 'C:/Users/kalya_kl8c3da/Downloads/facedata/dataset/celebrity_face_dataset/deepika padukone face/'
destination = 'C:/Users/kalya_kl8c3da/Downloads/facedata/shah rukh khan face_copy_test/'
folder = os.listdir(path)

names = []
for file in folder:
    names.append(file)
face_detector = cv2.CascadeClassifier('C:/ProgramData/Anaconda3/Lib/site-packages/cv2/data/haarcascade_frontalface_default.xml')
for image_names in range(len(names)):
    img_path = path + names[image_names]
    img = cv2.imread(img_path)
    faces = face_detector.detectMultiScale(img, 1.3, 5)
    for (x,y,w,h) in faces:
        cv2.rectangle(img, (x,y), (x+w,y+h), (0, 0, 0), 0)
        cv2.imwrite(destination + str(names[image_names]) , img[y:y+h,x:x+w])
'''

# dataset_path = 'C:/Users/kalya_kl8c3da/Downloads/facedata/dataset/celebrity_face_dataset/'
dataset_path = 'C:/Users/kalya_kl8c3da/Downloads/facedata/cropped_kalyan_banaja_saurav_color_gray/'
destination = 'C:/Users/kalya_kl8c3da/Downloads/facedata/shah rukh khan face_copy_test/'

def face_extraction(face_dataset_path):
    start = timeit.default_timer()
    folder = os.listdir(face_dataset_path)
    face_detector = cv2.CascadeClassifier('C:/ProgramData/Anaconda3/Lib/site-packages/cv2/data/haarcascade_frontalface_default.xml')

    names = []
    sub_folder_path = []
    # image_pth = []
    sf_images = []

    for folder_names in folder:
        names.append(folder_names)
    for file in range(len(names)):
        sub_folder = face_dataset_path + names[file]  + '/'
        sub_folder_path.append(sub_folder)
        for i in range(len(sub_folder_path)):
            sf_img1 = os.listdir(sub_folder_path[i])
            for j in range(len(sf_img1)):
                sf_img = str(sub_folder_path[i])  + str(sf_img1[j])
                sf_images.append(sf_img)
    print(sf_images)

    for image_names in range(len(sf_images)):
        img_path =  sf_images[image_names]
        img = cv2.imread(img_path)
        faces = face_detector.detectMultiScale(img, 1.3, 5)
        print(faces)
        for (x,y,w,h) in faces:
            cv2.rectangle(img, (x,y), (x+w,y+h), (0, 0, 0), 0)
            cv2.imwrite(str(sf_images[image_names]) , img[y:y+h,x:x+w])

    stop = timeit.default_timer()
    print('Tptal Time Taken to crop dataset: ', stop - start)
face_extraction(dataset_path)

#Your statements here


import pickle
from rdkit.Chem import AllChem
from tqdm import tqdm

if __name__ == '__main__':
    dataPathSave = './chembl_raw/raw/'

    f = open(dataPathSave + 'labelsHard.pckl', 'rb')
    targetMat = pickle.load(f)
    sampleAnnInd = pickle.load(f)
    targetAnnInd = pickle.load(f)
    f.close()

    index_list = targetAnnInd.index.tolist()

    fin = open('chembl_full/assay_id.tsv', 'w')
    for idx, chembl_idx in enumerate(index_list):
        print('{}\t{}'.format(chembl_idx, idx), file=fin)

    ########## Double-check smiles and rdkit ##########
    root_path = './chembl_full'
    fout = open('{}/smiles.csv'.format(root_path), 'r')
    smiles_list = []
    for line in fout:
        smiles_list.append(line.strip())

    f = open('{}/rdkit_molecule.pkl'.format(root_path), 'rb')
    rdkit_mol_list = pickle.load(f)

    for i, (smiles, rdkit_mol) in enumerate(zip(smiles_list, rdkit_mol_list)):
        s = AllChem.MolToSmiles(rdkit_mol)
        print(i, '\t', smiles, '\t', rdkit_mol)
        assert smiles == s