#!/usr/bin/python3
import sys
import os

import peewee

from slugify import slugify

sys.path.insert(1, os.path.join(sys.path[0], '../src'))

from faker import Faker
from main import database, db, Proposal, Community, User, Tag

import random

from peewee import fn

fake = Faker('sv_SE')

def get_random_user():
    return User.select(User.id).order_by(fn.Random()).get()

def get_random_text():
  return fake.text() + '\n' + fake.text()

def get_random_sentence():
  return fake.sentence()

items = [
  ("Natur, Miljö och Resurser", "politik"),
  ("Konst och Kultur", "politik"),
  ("Socialpolitik och Mönsterbrytning", "politik"),
  ("Integration och Arbetsmarknaden", "politik"),
  ("Freds- och Försvarspolitik", "politik"),
  ("Bostad, Städer och Landsbygd", "politik"),
  ("IT och Digitala Rättigheter", "politik"),
  ("Skola, Utbildning och Forskning", "politik"),
  ("Finans-, Ekonomi- och Skattepolitik", "politik"),
  ("Asylpolitik", "politik"),
  ("Ny Politisk Kultur", "politik"),
  ("Transport", "politik"),
  ("Entreprenör, Näringsliv och Handel", "politik"),
  ("Religion och Kyrka", "politik"),
  ("Sundhet, Livskvalité och Familjeliv", "politik"),
  ("Rättspolitik", "politik"),
  ("EU- och Utrikespolitik", "politik"),
  ("Mat, Jordbruk och Fiske", "politik"),
  ("Inrikespolitik och Offentliga Sektorn", "politik"),
  ("Principer och Värderingar", "politik"),
  ("Mångfald och Jämställdhet", "politik"),
  ("Förslag under utveckling", "politiska-förslag"),
  ("Förslag i samråd", "politiska-förslag"),
  ("Förslag som söker medlemmars stöd", "politiska-förslag"),
  ("Förslag under utveckling", "politiska-förslag"),
  ("Politisk katalog (snälla betygsätt)", "politiska-förslag"),
  ("Annullerade förslag", "politiska-förslag"),
  ("Söker medarbetare", "politiska-förslag"),
  ("Visioner i samråd", "politisk-vision"),
  ("Visioner som väntar på behandling i Politiska Forum", "politisk-vision"),
  ("Visioner under utveckling", "politisk-vision"),
  ("Antagna visioner", "politisk-vision"),
  ("Hemsidan", "hemsidan"),
  ("Funktioner och önskemål", "meta"),
  ("Support", "meta"),
  ("Felanmälan", "meta"),
  ("Lokal Karlshamn", "lokalförening")
]

me = User.get(User.username == 'ganehag')

for title, tag in items:
  c = Community()
  c.name = slugify(title)
  c.description = title
  c.maintainer = me

  with database.atomic():
    try:
      c.save()
    except peewee.IntegrityError as e:
      pass

  c = Community.get(Community.name == slugify(title))

  t = Tag()
  t.title = tag
  t.community = c

  with database.atomic():
    try:
      t.save()
    except peewee.IntegrityError as e:
      pass

  continue

  for z in range(0, 5):
    p = Proposal()
    p.title = fake.sentence(nb_words=5, variable_nb_words=False)
    p.author = get_random_user()
    p.community = c
    p.content = get_random_text()
    p.update_search_index()
    p.published = True
    p.upvotes = random.randint(0, 10000)
    p.downvotes = random.randint(0, 10000)
    try:
      with database.atomic():
        p.save()
    except peewee.IntegrityError as e:
      print(e)
      print("Duplicate title", p.title)


import os, config, time, utils
import requests, sys, json, os
import numpy as np
#from PIL import Image
#import pandas as pd
import argparse
#from utils import LoadDataset
from requests.exceptions import HTTPError, ConnectTimeout
#from glob import glob
#from load_dataset import load_test_caltech_256
#from torchvision.utils import save_image
import logging, torch
from PIL import Image

def sendImage(url, filePath, target, nr_branch_edge, p_tar, distortion_type, distortion_lvl):

	data_dict = {"p_tar": str(p_tar), "target": str(target), "nr_branch_edge": str(nr_branch_edge), "distortion_type": distortion_type,
	"distortion_lvl": str(distortion_lvl), "mode": ""}

	files = [
	('img', (filePath, open(filePath, 'rb'), 'application/octet')),
	('data', ('data', json.dumps(data_dict), 'application/json')),]

	try:
		r = requests.post(url, files=files, timeout=config.timeout)
		r.raise_for_status()

	except HTTPError as http_err:
		raise SystemExit(http_err)

	except requests.Timeout:
		logging.warning("Timeout")
		pass

def sendDistortedImage(imgPath, target, nr_branch_edge, p_tar, distortion_lvl, distortion_type):
	sendImage(config.url_ee, imgPath, target, nr_branch_edge, p_tar, distortion_type, distortion_lvl)
	sendImage(config.url_ensemble, imgPath, target, nr_branch_edge, p_tar, distortion_type, distortion_lvl)
	sendImage(config.url_naive_ensemble, imgPath, target, nr_branch_edge, p_tar, distortion_type, distortion_lvl)
	sendImage(config.url_cloud_backbone, imgPath, target, nr_branch_edge, p_tar, distortion_type, distortion_lvl)

def sendDistortedImageSet(dataset_path_list, target_list, distortion_lvl_list, p_tar, nr_branch_edge):

	for imgPath, target, distortion_lvl in zip(dataset_path_list, target_list, distortion_lvl_list):
		sendDistortedImage(imgPath, target, nr_branch_edge, p_tar, distortion_lvl, distortion_type="gaussian_blur")

def inferenceTimeExp(distorted_datasetPath, p_tar_list, nr_branch_edge_list):

	file_path_list, distortion_lvl_list, target_list = utils.getImageFilePath(distorted_datasetPath)

	for p_tar in p_tar_list:

		for nr_branch_edge in nr_branch_edge_list:

			logging.debug("p_tar: %s, Number Branches at Edge: %s"%(p_tar, nr_branch_edge))

			sendDistortedImageSet(file_path_list, target_list, distortion_lvl_list, p_tar, nr_branch_edge)

def main(args):
	#Number of side branches that exists in the early-exit DNNs
	#nr_branches_model_list = np.arange(config.nr_min_branches, config.nr_max_branches+1)

	distorted_model_path =  os.path.join(config.DIR_NAME, "models", args.dataset_name, args.model_name,
		"pristine_ee_model_mobilenet_%s_branches_id_%s.pth"%(args.n_branches, args.model_id) )

	p_tar_list = [0.8, 0.82, 0.83, 0.85, 0.9]

	indices_path = os.path.join(config.DIR_NAME, "indices")

	dataset_path = config.dataset_path_dict[args.dataset_name]

	logPath = "./log_inference_time_exp_%s_%s.log"%(args.model_name, args.dataset_name)

	logging.basicConfig(level=logging.DEBUG, filename=logPath, filemode="a+", format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')

	#This line defines the number of side branches processed at the edge
	nr_branch_edge_list = np.arange(2, args.n_branches+1)

	#device = 'cuda' if (torch.cuda.is_available() and args.cuda) else 'cpu'
	inferenceTimeExp(config.distorted_dataset_path, p_tar_list, nr_branch_edge_list)

if (__name__ == "__main__"):
	# Input Arguments. Hyperparameters configuration
	parser = argparse.ArgumentParser(description="Evaluating early-exits DNNs perfomance")

	parser.add_argument('--dataset_name', type=str, default=config.dataset_name,
		choices=["caltech256"],
		help='Dataset name (default: Caltech-256)')

	parser.add_argument('--model_name', type=str, default=config.model_name,
		choices=["mobilenet"], help='DNN model name (default: MobileNet)')

	parser.add_argument('--n_branches', type=int, default=3, help='Number of exit exits.')

	#parser.add_argument('--location', type=str, choices=["ohio", "sp"], help='Location of Cloud Server')

	parser.add_argument('--seed', type=int, default=config.seed, help='Seed.')

	parser.add_argument('--distortion_prob', type=float, default=1)

	parser.add_argument('--model_id', type=int, default=config.model_id, help='Model Id.')

	parser.add_argument('--batch_size_train', type=int, default=config.batch_size_train, help='Size of train batch.')

	parser.add_argument('--exit_type', type=str, default=config.exit_type, help='Exit type.')
	parser.add_argument('--distribution', type=str, default=config.distribution, help='Distribution of early exits.')
	parser.add_argument('--pretrained', type=bool, default=config.pretrained, help='Pretrained ?')

	parser.add_argument('--cuda', type=bool, default=True, help='Cuda ?')

	args = parser.parse_args()

	main(args)


import pandas as pd
import matplotlib.pyplot as plt
from scipy.stats import linregress

def draw_plot():
    # Read data from file
    df = pd.read_csv('epa-sea-level.csv')

    # Create scatter plot
    plt.scatter(df['Year'], df['CSIRO Adjusted Sea Level'])

    # Create first line of best fit
    y = range(2014, 2051)
    x = {'Year': list(y)}
    new_year = pd.DataFrame(x)
    df_new = df.append(new_year, ignore_index = True)
    plt.scatter(df['Year'], df['CSIRO Adjusted Sea Level'])
    slope, intercept, r, p, std_err = linregress(df['Year'], df['CSIRO Adjusted Sea Level'])
    y_pred = intercept + slope * df_new['Year']
    plt.plot(df_new['Year'],y_pred, color="red")

    # Create second line of best fit
    i = df_new[(df_new['Year'] < 2000)].index
    df_new.drop(i, inplace=True)
    plt.scatter(df_new['Year'], df_new['CSIRO Adjusted Sea Level'])
    slope, intercept, r, p, std_err = linregress(df['Year'], df['CSIRO Adjusted Sea Level'])
    y_pred = intercept + slope * df_new['Year']
    plt.plot(df_new['Year'],y_pred, color="red")

    # Add labels and title
    plt.xlabel('Year')
    plt.ylabel('Sea Level (inches)')
    plt.title('Rise in Sea Level')

    # Save plot and return data for testing (DO NOT MODIFY)
    plt.savefig('sea_level_plot.png')
    return plt.gca()


import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
from scipy.io import wavfile
import mpmath

#-- Create the list of musical notes

scale=[]
for k in range(35, 65):
    note=440*2**((k-49)/12)
    if k%12 != 0 and k%12 != 2 and k%12 != 5 and k%12 != 7 and k%12 != 10:
        scale.append(note) # add musical note (skip half tones)
n_notes = len(scale) # number of musical notes

#-- Generate the data

n = 300
sigma = 0.5
min_t = 400000
max_t = 400020

def create_data(f, nobs, min_t, max_t, sigma):
    z_real = []
    z_imag = []
    z_modulus = []
    incr_t = (max_t - min_t) / nobs
    for t in np.arange(min_t, max_t, incr_t):
        if f == 'Zeta':
            z = mpmath.zeta(complex(sigma, t))
        elif f == 'Eta':
            z = mpmath.altzeta(complex(sigma, t))
        z_real.append(float(z.real))
        z_imag.append(float(z.imag))
        modulus = np.sqrt(z.real*z.real + z.imag*z.imag)
        z_modulus.append(float(modulus))
    return(z_real, z_imag, z_modulus)

(z_real, z_imag, z_modulus) = create_data('Eta', n, min_t, max_t, sigma)

size = len(z_real) # should be identical to nobs
x = np.arange(size)

# frequency of each note
y = z_real
min = np.min(y)
max = np.max(y)
yf = 0.999*n_notes*(y-min)/(max-min)

# duration of each note
z = z_imag
min = np.min(z)
max = np.max(z)
zf = 0.1 + 0.4*(z-min)/(max-min)

# volume of each note
v = z_modulus
min = np.min(v)
max = np.max(v)
vf = 500 + 2000*(1 - (v-min)/(max-min))

#-- plot data

mpl.rcParams['axes.linewidth'] = 0.3
fig, ax = plt.subplots()
ax.tick_params(axis='x', labelsize=7)
ax.tick_params(axis='y', labelsize=7)
plt.rcParams['axes.linewidth'] = 0.1
plt.plot(x, y, color='red', linewidth = 0.3)
plt.plot(x, z, color='blue', linewidth = 0.3)
plt.plot(x, v, color='green', linewidth = 0.3)
plt.legend(['frequency','duration','volume'], fontsize="7",
    loc ="upper center", ncol=3)
plt.show()

#-- Turn the data into music

def get_sine_wave(frequency, duration, sample_rate=44100, amplitude=4096):
    t = np.linspace(0, duration, int(sample_rate*duration))
    wave = amplitude*np.sin(2*np.pi*frequency*t)
    return wave

wave=[]
for t in x: # loop over dataset observations, create one note per observation
    note = int(yf[t])
    duration = zf[t]
    frequency = scale[note]
    volume = vf[t]  ## 2048
    new_wave = get_sine_wave(frequency, duration = zf[t], amplitude = vf[t])
    wave = np.concatenate((wave,new_wave))
wavfile.write('sound.wav', rate=44100, data=wave.astype(np.int16))


import pytest
from utils import utils as utils
def pytest_addoption(parser):
    parser.addoption("--browser",action="store",default="chrome",help="type in browser_name")

@pytest.fixture(scope="class")
def test_setup(request):
    from selenium import webdriver
    browser=request.config.getoption("--browser")
    if browser=='chrome':
        driver = webdriver.Chrome(executable_path="C:/Users/kruthi.p/PycharmProjects/Test_New/drivers/chromedriver.exe")
        driver.get(utils.EOTS_URL)
        driver.find_element_by_id("details-button").click()
        driver.find_element_by_id("proceed-link").click()
    elif browser=='firefox':
        driver = webdriver.Firefox(executable_path="C:/Users/kruthi.p/PycharmProjects/Test_New/drivers/geckodriver.exe")
        driver.get(utils.EOTS_URL)

    driver.implicitly_wait(5)
    driver.maximize_window()
    request.cls.driver=driver
    yield
    driver.close()
    driver.quit()


import numpy as np
import matplotlib.pyplot as plt

w1=np.load('weights_1.npy')
w2=np.load('weights_3.npy')

print(w1.shape)
#print(w1)
print(w2.shape)
#print(w2)

wc=np.dot(w1,w2)
wc=np.reshape(wc,[60,4])
print(wc.shape)
#print(wc)

# Plot the grid
plt.imshow(np.transpose(np.abs(wc)))
plt.gray()
plt.show()

# Normalize by the input mean
x_train=np.load('x_train.npy')
print(x_train.shape)
print(np.mean(np.mean(x_train,1),0))

plt.imshow(np.transpose(np.abs(wc)*np.mean(np.mean(x_train,1),0)))
plt.gray()
plt.show()


# Run this Dash app with `python app.py`
# visit http://127.0.0.1:8050/ in web browser.

####################################### Import dependencies and other files #####################################################
import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import plotly.express as px
import plotly.graph_objects as go
import plotly.figure_factory as ff
import pandas as pd
import os, mediacloud.api
from io import BytesIO
from dotenv import load_dotenv
import sys
import csv
from datetime import date
from SubDirectory import dataprocessing

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

#Variable to switch to offline testing without requesting new
OFFLINE = False

####################################### Set variables & import data #####################################################
app = dash.Dash(__name__, external_stylesheets=external_stylesheets)

#define app colors
colors = {
    'background': '#f7f7f5',
    'text': '#2a2a2a'
}

#define colors for different media sources
media_colors = {
    "South China Morning Post": "#EFB938",
    "New York Times": "#647CAA",
    "BBC": "#7BB48D",
    "Peoples Daily": "#C0003A",
    "english.people.com.cn": "#C0003A"
}

# gets data from csv document which is updated by the dataprocessing file continously
# this makes it possible to both use online + offline, as well as update the data, eg. change sources /timespan

if OFFLINE:
    print("OFFLINE MODE")
    story_df= pd.read_csv('story-list.csv',parse_dates=True, squeeze=True)
    word_df= pd.read_csv('word_df.csv',parse_dates=True, squeeze=True)
    orgs_df= pd.read_csv('orgs_df.csv',parse_dates=True, squeeze=True)

    print(story_df.groupby('media_id')['subjectivity'].mean())

else:
    print("ONLINE MODE")
    #story_df= pd.read_csv('story-list.csv',parse_dates=True, squeeze=True)

    story_df= dataprocessing.getData(dataprocessing.media_dict)
    print(story_df.groupby('media_id')['subjectivity'].mean())

    #calculate most used words (for some reason does not want to work in dataprocessing file)
    word_df= pd.DataFrame()
    for i in dataprocessing.media_dict:
        query = f'"Hong Kong" AND (protest* OR unrest OR "Anti-Extradition" OR "democracy movement" OR assembly OR demonstration* OR “human chain” OR rally) and media_id:{i}'
        df= dataprocessing.get_wordcount(query,i)
        df['media_name'] = dataprocessing.media_dict[i]
        word_df = word_df.append(df)
    word_df.to_csv('word_df.csv')

    orgs_df= pd.DataFrame()
    for j in dataprocessing.media_dict:
        query = f'"Hong Kong" AND (protest* OR unrest OR "Anti-Extradition" OR "democracy movement" OR assembly OR demonstration* OR “human chain” OR rally) and media_id:{j}'
        df= dataprocessing.get_orgscount(query,j)
        df['media_name'] = dataprocessing.media_dict[j]
        orgs_df = orgs_df.append(df)
    orgs_df.to_csv('orgs_df.csv')

####################################### Create Visualisations & other dashboard elements ####################################

# create a scatterplot
fig = px.scatter(story_df, x="publish_date",y="polarity",
                 #size="subjectivity",
                 color="media_name", color_discrete_map= media_colors, hover_name="title",
                )
fig.update_layout(hovermode="x unified")

# Chart to compare coverage amount of different news sources

fig2 = px.histogram(story_df, x="publish_date", marginal="rug",color="media_name", hover_name= 'title', hover_data={'publish_date':False}, color_discrete_map= media_colors, barmode='group')
fig2.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        hovermode="x unified",
        yaxis=dict(
        title='Articles published',
        ),
        xaxis=dict(
        title='Date',
        )
    )

fig3= px.scatter(story_df, x="publish_date", color="polarity", color_continuous_scale= px.colors.diverging.balance, y="media_name", hover_name="title")
fig3.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        yaxis=dict(
        title='Media Outlet',
        ),
        xaxis=dict(
        title='Date',
        )
)
fig3.update_traces(marker_opacity=1, marker_symbol= 42, marker_size= 50)

fig4 =px.bar(word_df, x="count", y= "term", barmode="group", color= "media_name", color_discrete_map= media_colors,)
fig4.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        height=600,
        yaxis=dict(
        title='Term',
        categoryorder= 'total descending'
        ),
        xaxis=dict(
        title='Count',
        )
    )

fig5 =px.bar(orgs_df, x="count", y= "description", color= "media_name", color_discrete_map= media_colors, width=30)

# Text to go with the visualisations
# also make this responsive in app callback?
markdown_text = f'''
### Development of the Hong Kong protests in the News

How much news content does each media source dedicate to the topic of the Hong Kong protests? \n
The following graph compares the number of news articles published about the topic by different news sources. \n
While the number of stories alone does not say much, it is interesting to compare what time each news media picked up \n
a news event or to explore when the number of articles spiked and what the headlines were around that time. \n
The ratio to overall stories of each source reveals the attention a media source gave to a topic: &nbsp
\n
**The percentage of a sources' news articles that mentioned the Hong Kong protests:**   \n
{dataprocessing.media_dict.get(1, 0)} : **{dataprocessing.ratio_dict.get(1,0):.2%}** \n
{dataprocessing.media_dict.get(39590,0)} : **{dataprocessing.ratio_dict.get(39590,0):.2%}** \n
{dataprocessing.media_dict.get(1094, 0)} : **{dataprocessing.ratio_dict.get(1094,0):.2%}** \n
{dataprocessing.media_dict.get(65173,0)} : **{dataprocessing.ratio_dict.get(65173,0):.2%}** \n
'''

# create small tables from text? or visualise the numbers in a better way?

markdown_text2 = f'''
### Sentiment and Polarity of Article Headlines

Since news headlines set the tone for an article and reach an even larger audience than the article text itself, it is vital that they are neutral and objective. \n
Looking at headlines that are subjective or overly positive/negative in tone can reveal biases of news sources. \n
The calculated polarity values range from from -1 (negative tone) to +1 (positive tone), the subjectivity values from 0 (objective) to 1 (subjective). \n
**Average polarity values for the headlines of each news source:**   \n
{dataprocessing.media_dict.get(1, 0)} : **{story_df[story_df['media_name']=='New York Times']['polarity'].mean():.2}** \n

{dataprocessing.media_dict.get(39590,0)} : **{story_df[story_df['media_name']=='BBC']['polarity'].mean():.2}** \n

{dataprocessing.media_dict.get(1094, 0)} : **{story_df[story_df['media_name']=='South China Morning Post']['polarity'].mean():.2}** \n

{dataprocessing.media_dict.get(65173,0)} : **{story_df[story_df['media_name']=='english.people.com.cn']['polarity'].mean():.2}** \n
'''

markdown_text3 = f'''
### Taking a closer look at the most used words

Analysing the most used words can show each news source's perspective, as well as which aspects of the news events they focus on. \n
Instances of news bias can be revealed for example through different labels chosen for the same entity or event.
'''

#display data table
def generate_table(dataframe, max_rows=10):
    return html.Table([
        html.Thead(
            html.Tr([html.Th(col) for col in dataframe.columns])
        ),
        html.Tbody([
            html.Tr([
                html.Td(dataframe.iloc[i][col]) for col in dataframe.columns
            ]) for i in range(min(len(dataframe), max_rows))
        ])
    ])

####################################### Set app layout + HTML #####################################################

app.layout = html.Div(style={'backgroundColor': colors['background'], 'font-family': "Verdana"}, children=[   html.H1(
        children='News Explorer',
        style={
            'textAlign': 'center',
            'color': colors['text'],
            'letter-spacing': '5px',
            'font-weight': 'lighter',
            'padding-top': '1em',
        }
    ),

    #display subtitle
    html.Div(children='visualising digital news coverage', style={
        'textAlign': 'center',
        'color': '#5a5a5a',
        'margin-bottom': '3em'
    }),

    #Dropdown menu to select multiple news sources
    html.Label('Select a News Sources to explore'),
    dcc.Dropdown(
        id='dropdown',
        options=[
            {'label': 'New York Times (US)', 'value': 1},
            {'label': 'South China Morning Post (Hong Kong)', 'value': 39590},
            {'label': 'BBC (UK)', 'value': 1094},
            {'label': 'Peoples Daily (China)', 'value': 65173},
        ],
        value=[1, 39590, 1747,65173],
        multi=True,
        style= {
            'color': colors['text'],
        }
    ),

    #Date Picker to change time range
    html.Label('Pick a date range '),
    dcc.DatePickerRange(
        id='my-date-picker-range',
        min_date_allowed=min(story_df['publish_date']),
        max_date_allowed=max(story_df['publish_date']),
        initial_visible_month=date(2019, 1, 1),
        end_date=date(2019, 12, 31),
        start_date=min(story_df['publish_date']),
        clearable= False,
    ),

    #display markdown text
    dcc.Markdown(
        id="text",
        children=markdown_text,
        style={'backgroundColor': '#f0f0f0', 'margin' : '42px', 'padding': '8px'}
    ),

    # display graph
    dcc.Graph(
        id='bar-chart',
        figure=fig2
    ),

    #display markdown text
    dcc.Markdown(
        id="text2",
        children=markdown_text2,
        style={'backgroundColor': '#f0f0f0', 'margin' : '42px', 'padding': '8px'}
    ),

    # display graph
    dcc.Graph(
        id='rug-chart',
        figure=fig3
    ),

     # display graph
    dcc.Graph(
        id='scatter-plot',
        figure=fig
    ),

    #display markdown text
    dcc.Markdown(
        id="text3",
        children=markdown_text3,
        style={'backgroundColor': '#f0f0f0', 'margin' : '42px', 'padding': '8px'}
    ),

    # display graph
    dcc.Graph(
        id='word-bars',
        figure=fig4
    ),

    # # display graph
    # dcc.Graph(
    #     id='fifth-graph',
    #     figure=fig5
    # ),

    # # display table
    # html.Div(children=[
    # html.H4(children='All Stories'),
    # generate_table(story_df)
    # ],
    # style={
    #         'textAlign': 'center',
    #         'color': colors['text']
    #     })
])
####################################### Callback functions #####################################################

#Callback for dropdown menu (media sources) and date range picker -> Updates graphs
@app.callback(
    Output('bar-chart', 'figure'),
    Output('scatter-plot', 'figure'),
    Output('rug-chart', 'figure'),
    Output('word-bars', 'figure'),
    Input('dropdown', 'value'),
    [Input('my-date-picker-range', 'start_date'),
    Input('my-date-picker-range', 'end_date')])

def update_figure(selected_media, start_date, end_date):
    time_df = story_df.loc[(story_df['publish_date'] >= start_date) & (story_df['publish_date'] < end_date)]
    filtered_df= time_df[time_df.media_id.isin(selected_media)]
    filtered_word_df= word_df[word_df.media_id.isin(selected_media)]

    fig = px.scatter(filtered_df, x="publish_date",y="polarity",
                 marginal_y="rug", opacity= 0.6,
                 color="media_name", color_discrete_map= media_colors, hover_name="title",
                )

    fig.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        yaxis=dict(
        title='Polarity',
        )
    )

    fig.update_xaxes(
        dtick="M1",
        title= "Date",
        tickformat="%b\n%Y",
        ticklabelmode="period"
        )

    fig.update_traces(marker_opacity=0.8)

    fig2 = px.histogram(filtered_df, x="publish_date", marginal="rug",color="media_name", hover_name= 'title', hover_data={'publish_date':False}, color_discrete_map= media_colors, barmode= 'group')

    fig2.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        hovermode="x unified",
        yaxis=dict(
        title='Articles published',
        ),
        xaxis=dict(
        title='Date',
        )
    )

    fig3= px.scatter(filtered_df, x="publish_date", color="subjectivity", #color_continuous_scale= px.colors.diverging.balance,
    color_continuous_scale= px.colors.sequential.matter, y="media_name", hover_name="title")

    fig3.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        yaxis=dict(
        title='Media Outlet',
        ),
        xaxis=dict(
        title='Date',
        )
    )
    fig3.update_traces(marker_opacity=1, marker_symbol= 42, marker_size= 50)

    #new word counts

    # calculating new word counts -> works, but takes too long, slows overall performance
    # would only work if the word_df is differently structured, so that it can be filtered by time

    # word_df= pd.DataFrame()
    # for i in selected_media:
    #     df= pd.DataFrame()
    #     query = f'"Hong Kong" AND (protest* OR unrest OR "Anti-Extradition" OR "democracy movement*" OR "extradition") and media_id:{i}'
    #     df= dataprocessing.get_wordcount(query,i)
    #     #df['media_name'] = dataprocessing.media_dict[i]
    #     word_df = word_df.append(df)

    fig4 =px.bar(filtered_word_df, x="count", y= "term", barmode="group", color= "media_name", color_discrete_map= media_colors,)
    fig4.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        height=800,
        yaxis=dict(
        title='Term',
        tickmode='linear', # displays every label instead of only every other,
        categoryorder= 'total ascending'
        ),
        xaxis=dict(
        title='Count',
        )
    )

    return fig2, fig, fig3, fig4

if __name__ == '__main__':
    app.run_server(debug=True)


print("You have 0 coins.")
count = 0
while True and count != 6:
    answer = input("Do you want another?  ")
    if count == 5:
        print("Sorry, I am out of coins to give.  Goodbye.")
        break
    elif answer == "yes":
        count += 1
        print(f"You have {count} coins.")

    else:
        if answer != "yes":
            print("Bye")
            break


import math
import matplotlib.pyplot as plt

def powInt(x, n):
    if n == 0:
        return 1
    elif n == 1:
        return x
    else:
        q = math.floor(n / 2)
        r = n % 2
        if r == 0:
            return powInt(x * x, q)
        else:
            return x * powInt(x * x, q)

def expEuler(x, n):
    if n == 0:
        return 1
    else:
        return powInt(1.0 + (1.0 * x / n), n)

def expTaylor(x, n):
    acc = 0
    for k in range(0, n + 1):
        acc += (1.0 * powInt(x, k)) / math.factorial(k)
    return acc

def expHybrid(x, n, k):
    if n == 0:
        return 1
    else:
        acc = 0
        t = (1.0 * x) / n
        for l in range(0, k + 1):
            acc += (1.0 * powInt(t, l)) / math.factorial(l)
        return powInt(acc, n)

def exp_experiment(t, n):
    for i in range(0, n):
        print(str(i) + "\t" + str(expEuler(t, i)) + "\t" + str(expHybrid(t, i, 2)) + "\t" + str(
            expHybrid(t, i, 3)) + "\t" + str(
            expHybrid(t, i, 10)) + "\t" + str(expTaylor(t, i)) + "\t" + str(
            math.exp(t)))

def exp_error_experiment(t, n):
    exp = math.exp(t)
    acc = [[], [], [], [], []]
    for i in range(0, n):
        acc[0].append(math.fabs(expEuler(t, i) - exp))
        acc[1].append(math.fabs(expTaylor(t, i) - exp))
        acc[2].append(math.fabs(expHybrid(t, i, 2) - exp))
        acc[3].append(math.fabs(expHybrid(t, i, 3) - exp))
        acc[4].append(math.fabs(expHybrid(t, i, 10) - exp))
        print(str(i) + "\t" + str(acc[0][i]) + "\t" + str(acc[1][i]) + "\t" + str(acc[2][i]) + "\t" + str(
            acc[3][i]) + "\t" + str(acc[4][i]))
    plt.plot(acc[0], 'r', label='euler')
    plt.plot(acc[1], 'g', label='taylor')
    plt.plot(acc[2], 'b', label='hybrid2')
    plt.plot(acc[3], 'c', label='hybrid3')
    plt.plot(acc[4], 'y', label='hybrid10')
    plt.xlabel('iterations')
    plt.ylabel('error')
    plt.legend()
    plt.show()

# exp_experiment(100, 150)
exp_error_experiment(10, 150)


import itertools
import multiprocessing
import numpy as np
from catboost import CatBoostClassifier, datasets
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss, accuracy_score, roc_auc_score
from tqdm import tqdm, trange
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
import pickle
import sys
import telegram_send
import os

def test_keras_parameters(all_params):
    self, dataset, params = all_params
    loss = 0.0
    acc = 0.0
    roc_auc = 0.0
    for iteration in range(self.KERAS_HYPERPARAMETER_ITERATIONS):
        model = self.get_compiled_keras_model_from_parameters(*params)
        X, y, _ = self.get_balanced_dataset(dataset, 9999999999, 9999999999)
        X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.25)
        callbacks = [
            tf.keras.callbacks.EarlyStopping(
                monitor="val_loss",
                min_delta=0,
                patience=0,
                verbose=0,
                mode="auto",
                baseline=None,
                restore_best_weights=False,
            )
        ]
        model.fit(
            X_train, y_train,
            epochs=self.KERAS_EPOCHS, validation_data=(X_valid, y_valid),
            callbacks=callbacks, verbose=False,
        )
        test_loss, test_acc, test_auc = model.evaluate(X_valid,  y_valid, verbose=0)
        loss += test_loss
        acc += test_acc
        roc_auc += test_auc
    metrics = (
        loss / self.KERAS_HYPERPARAMETER_ITERATIONS,
        acc / self.KERAS_HYPERPARAMETER_ITERATIONS,
        roc_auc / self.KERAS_HYPERPARAMETER_ITERATIONS,
    )
    return params, metrics

class BaseExperiment:
    KERAS_MAX_ONEHOT_VALUES = 50  # if unique(feature) < MAX_ONEHOT_VALUES, one-hot encoding, otherwise integer encoding
    KERAS_EPOCHS = 32
    POSITIVE_STEPS = [9999999999]
    NEGATIVE_STEPS = [9999999999]
    ITERATIONS = 5
    PLOT_FIG_SIZE = (9, 12)
    KERAS_HYPERPARAMETER_WORKERS = 8
    KERAS_HYPERPARAMETER_ITERATIONS = 4
    KERAS_LAYERS_RANGE = [4, 5, 6]
    KERAS_LAYER_SIZE_RANGE = [64, 128, 192, 256]
    KERAS_DROPOUT_RANGE = [0.2, 0.35, 0.5]
    KERAS_ACTIVATIONS = ['relu', 'tanh']
    KERAS_OPTIMIZERS_LIST = ['adam']  # ['adam', 'adadelta', 'rmsprop']  # in most cases adam was better
    USE_CTR = False
    LOG_TO_STDOUT = True
    LOG_TO_TELEGRAM = False

    def get_dataset(self):
        raise NotImplementedError()

    def get_balanced_dataset(self, dataset, positives, negatives):
        X, y, cat_features = dataset
        data = X.copy()
        assert 'label' not in data, '"label" can not be a name of a feature'  # the easiest way
        data['label'] = y
        assert (data[data['label'] == 0].shape[0] + data[data['label'] == 1].shape[0]) == data.shape[0], 'labels should be only 0 or 1'
        data = pd.concat([
            data[data['label'] == 1][:positives],
            data[data['label'] == 0][:negatives],
        ]).sample(frac=1).reset_index(drop=True)
        return data.drop(columns=['label']), data['label'], cat_features

    def run(self):
        keras_params = self.tune_keras_hyperparameters()
        catboost_metrics = self.run_catboost()
        keras_metrics = self.run_keras(keras_params)
        self.plot_metrics(catboost_metrics, keras_metrics)
        self.plot_metrics_diff(catboost_metrics, keras_metrics)
        self.print_summary(catboost_metrics, keras_metrics, keras_params)
        self.write_metrics(catboost_metrics, keras_metrics)
        self.log_progress('Finished')

    def run_catboost(self):
        dataset = self.get_dataset()
        metrics = {
            'loss': np.zeros(shape=(len(self.NEGATIVE_STEPS), len(self.POSITIVE_STEPS))),
            'accuracy': np.zeros(shape=(len(self.NEGATIVE_STEPS), len(self.POSITIVE_STEPS))),
            'roc_auc': np.zeros(shape=(len(self.NEGATIVE_STEPS), len(self.POSITIVE_STEPS))),
            'mean_prediction': np.zeros(shape=(len(self.NEGATIVE_STEPS), len(self.POSITIVE_STEPS))),
            'max_epochs': np.ones(shape=(len(self.NEGATIVE_STEPS), len(self.POSITIVE_STEPS)), dtype='int32'),
        }
        for i, positive in enumerate(tqdm(self.POSITIVE_STEPS, desc='Catboost')):
            self.log_progress(f'{i}/{len(self.POSITIVE_STEPS)}')
            for j, negative in enumerate(tqdm(self.NEGATIVE_STEPS, leave=False)):
                loss = 0.0
                acc = 0.0
                roc_auc = 0.0
                mean_pred = 0.0
                for iteration in range(self.ITERATIONS):
                    X, y, cat_features = self.get_balanced_dataset(dataset,  positive, negative)
                    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.25)

                    params = {
                        'loss_function':'Logloss',
                        'eval_metric':'AUC',
                        'cat_features': cat_features,
                        'early_stopping_rounds': 200,
                        'verbose': False,
                    }
                    cbc_1 = CatBoostClassifier(**params)
                    cbc_1.fit(
                        X_train, y_train,
                        eval_set=(X_valid, y_valid),
                        use_best_model=True,
                        plot=False,
                    )
                    loss += log_loss(y_valid, cbc_1.predict_proba(X_valid)[:, 1])
                    acc += accuracy_score(y_valid, cbc_1.predict(X_valid))
                    roc_auc += roc_auc_score(y_valid, cbc_1.predict_proba(X_valid)[:, 1])
                    mean_pred += cbc_1.predict(X_valid).mean()

                metrics['loss'][j, i] = loss / self.ITERATIONS
                metrics['accuracy'][j, i] = acc / self.ITERATIONS
                metrics['roc_auc'][j, i] = roc_auc / self.ITERATIONS
                metrics['mean_prediction'][j, i] = mean_pred / self.ITERATIONS
        return metrics

    def run_keras(self, model_hyperparameters):
        dataset = self.transform_dataset_for_keras(self.get_dataset())
        metrics = {
            'loss': np.zeros(shape=(len(self.NEGATIVE_STEPS), len(self.POSITIVE_STEPS))),
            'accuracy': np.zeros(shape=(len(self.NEGATIVE_STEPS), len(self.POSITIVE_STEPS))),
            'roc_auc': np.zeros(shape=(len(self.NEGATIVE_STEPS), len(self.POSITIVE_STEPS))),
            'mean_prediction': np.zeros(shape=(len(self.NEGATIVE_STEPS), len(self.POSITIVE_STEPS))),
            'max_epochs': np.ones(shape=(len(self.NEGATIVE_STEPS), len(self.POSITIVE_STEPS)), dtype='int32'),
        }
        for i, positive in enumerate(tqdm(self.POSITIVE_STEPS, desc='Keras')):
            self.log_progress(f'{i}/{len(self.POSITIVE_STEPS)}')
            for j, negative in enumerate(tqdm(self.NEGATIVE_STEPS, leave=False)):
                loss = 0.0
                acc = 0.0
                roc_auc = 0.0
                mean_pred = 0.0
                max_epochs = 0
                for iteration in range(self.ITERATIONS):
                    X, y, _ = self.get_balanced_dataset(dataset, positive, negative)
                    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.25)

                    model = self.get_compiled_keras_model_from_parameters(*model_hyperparameters)
                    callbacks = [
                        tf.keras.callbacks.EarlyStopping(
                            monitor="val_loss",
                            min_delta=0,
                            patience=0,
                            verbose=0,
                            mode="auto",
                            baseline=None,
                            restore_best_weights=False,
                        )
                    ]
                    history = model.fit(
                        X_train, y_train,
                        epochs=self.KERAS_EPOCHS, validation_data=(X_valid, y_valid),
                        callbacks=callbacks, verbose=False,
                    )
                    test_loss, test_acc, test_auc = model.evaluate(X_valid,  y_valid, verbose=0)
                    loss += test_loss
                    acc += test_acc
                    roc_auc += test_auc
                    mean_pred += model.predict_classes(X_valid).mean()
                    max_epochs = max(max_epochs, len(history.history))

                metrics['loss'][j, i] = loss / self.ITERATIONS
                metrics['accuracy'][j, i] = acc / self.ITERATIONS
                metrics['roc_auc'][j, i] = roc_auc / self.ITERATIONS
                metrics['mean_prediction'][j, i] = mean_pred / self.ITERATIONS
                metrics['max_epochs'][j, i] = max_epochs
        return metrics

    def transform_dataset_for_keras(self, dataset):
        X, y, cat_features = dataset
        numeric_features = [x for x in X.columns if x not in cat_features]

        columns_to_onehot_encode = [x for x in cat_features if X[x].nunique() <= self.KERAS_MAX_ONEHOT_VALUES]
        if self.USE_CTR:
            columns_to_ctr_encode = [x for x in cat_features if X[x].nunique() > self.KERAS_MAX_ONEHOT_VALUES]
            columns_to_integer_encode = []
        else:
            columns_to_ctr_encode = []
            columns_to_integer_encode = [x for x in cat_features if X[x].nunique() > self.KERAS_MAX_ONEHOT_VALUES]

        new_ctr_features = []
        ctr_combinations = [(column,) for column in columns_to_ctr_encode] + list(itertools.combinations(columns_to_ctr_encode, 2))
        for column_set in ctr_combinations:
            values = X[column_set[0]].astype('str')
            if len(column_set) > 1:
                for column in column_set[1:]:
                    values += ';' + X[column].astype('str')
            unique_counts = values.value_counts()
            value_to_count = {value: count for (value, count) in zip(unique_counts.axes[0], unique_counts)}
            new_column_name = '__'.join(column_set) + '__CTR'
            new_ctr_features.append(new_column_name)
            X[new_column_name] = 0.0
            total_count = X.shape[0]
            PRIOR = 0.5
            for value, count in value_to_count.items():
                X.loc[values == value, (new_column_name,)] = (count + PRIOR) / (total_count + 1)
        X = X.drop(columns=columns_to_ctr_encode)

        for col in X.columns:
            if col in columns_to_integer_encode:
                X[col] = X[col].astype('category').cat.codes.astype('float32')
            if col in numeric_features or col in columns_to_integer_encode or col in new_ctr_features:
                if X[col].max() == X[col].min():
                    col_range = 1
                else:
                    col_range = X[col].max() - X[col].min()
                X[col] = (X[col] - X[col].min()) / col_range

        X = pd.get_dummies(X, columns=columns_to_onehot_encode)

        X = X.astype('float32')
        y = y.astype('float32')
        return X,  y, cat_features

    def get_keras_model(self):
        raise NotImplementedError()

    def tune_keras_hyperparameters(self):
        dataset = self.transform_dataset_for_keras(self.get_dataset())
        loss_by_params = {}
        accuracy_by_params = {}
        roc_auc_by_params = {}
        parameter_tuples = []
        for layers_num in self.KERAS_LAYERS_RANGE:
            for layer_sizes in itertools.product(self.KERAS_LAYER_SIZE_RANGE, repeat=layers_num):
                # In most cases the larger first two layers were – there better, so we cut small layers for faster tuning
                if layer_sizes[0] <= 64:
                    continue
                if layer_sizes[1] <= 32:
                    continue
                if len(layer_sizes) >= 2 and layer_sizes[-1] >= layer_sizes[-2]:
                    continue  # small optimization: we believe that the network should become more "narrow" at the end
                if len(layer_sizes) >= 3 and layer_sizes[-2] > layer_sizes[-3]:
                    continue  # small optimization: we believe that the network should become more "narrow" at the end
                for activation in self.KERAS_ACTIVATIONS:
                    for dropout in self.KERAS_DROPOUT_RANGE:
                        for optimizer in self.KERAS_OPTIMIZERS_LIST:
                            params_key = (layer_sizes, activation, dropout, optimizer)
                            parameter_tuples.append(params_key)

        with multiprocessing.Pool(processes=self.KERAS_HYPERPARAMETER_WORKERS) as pool:
            all_param_tuples = [(self, dataset, param_tuple) for param_tuple in parameter_tuples]
            # results = pool.imap_unordered(test_keras_parameters, all_param_tuples)
            results = pool.map(test_keras_parameters, all_param_tuples)
            for _i, result in enumerate(tqdm(results, total=len(parameter_tuples), desc='Tuning Keras')):
                if _i % 4 == 0:
                    self.log_progress(f'{_i}/{len(parameter_tuples)}')
                params_key, metrics = result
                loss, acc, auc = metrics
                loss_by_params[params_key] = loss
                accuracy_by_params[params_key] = acc
                roc_auc_by_params[params_key] = auc

        ordered_params = sorted(roc_auc_by_params.items(), key=lambda item: item[1], reverse=True)
        print('BEST NETWORK PARAMETERS:')
        for item in ordered_params[:5]:
            params_key, roc_auc = item
            print(f'{params_key}: auc={roc_auc:.3f}, acc={accuracy_by_params[params_key]:.3f}')
        return ordered_params[0][0]

    def get_compiled_keras_model_from_parameters(self, layer_sizes, activation, dropout, optimizer):
        layers = []
        for layer_size in layer_sizes:
            layers.append(keras.layers.Dense(layer_size, activation=activation))
            if dropout > 0:
                layers.append(keras.layers.Dropout(dropout))
        layers.append(keras.layers.Dense(1, activation='sigmoid'))
        model = keras.Sequential(layers)
        model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy', keras.metrics.AUC()])
        return model

    def plot_metrics(self, catboost_metrics, keras_metrics):
        if not os.path.exists('experiments'):
            os.mkdir('experiments')
        if not os.path.exists(f'experiments/plots'):
            os.mkdir(f'experiments/plots')
        cmap = np.concatenate([
            catboost_metrics['roc_auc'],
            np.full((1, len(self.POSITIVE_STEPS)), min(catboost_metrics['roc_auc'].min(), keras_metrics['roc_auc'].min())),
            keras_metrics['roc_auc']
        ])
        plt.figure(figsize=self.PLOT_FIG_SIZE)
        plt.title(f'AUC', fontsize=18)
        plt.imshow(cmap, cmap=plt.get_cmap("PiYG", 7))
        plt.xlabel('Positives', fontsize=14)
        plt.ylabel('Negatives', fontsize=14)
        plt.xticks(range(len(self.POSITIVE_STEPS)), self.POSITIVE_STEPS)
        plt.yticks(
            range(len(self.NEGATIVE_STEPS)*2+1),
            [f'{x} (Catboost)' for x in self.NEGATIVE_STEPS] + [''] + [f'{x} (Keras)' for x in self.NEGATIVE_STEPS]
        )
        for i in range(len(self.NEGATIVE_STEPS)):
            for j in range(len(self.POSITIVE_STEPS)):
                plt.text(j-0.45, i-0.15, f'true+={round(self.POSITIVE_STEPS[j] / (self.POSITIVE_STEPS[j] + self.NEGATIVE_STEPS[i]), 2)}')
                # plt.text(j-0.45, i, f"pred={round(catboost_metrics['mean_prediction'][i, j], 2)}")
                plt.text(j-0.45, i+0.15, f"auc={round(catboost_metrics['roc_auc'][i, j], 3)}")
                plt.text(j-0.45, i-0.15 + len(self.NEGATIVE_STEPS) + 1, f"true+={round(self.POSITIVE_STEPS[j] / (self.POSITIVE_STEPS[j] + self.NEGATIVE_STEPS[i]), 2)}")
                # plt.text(j-0.45, i + len(self.NEGATIVE_STEPS) + 1, f"pred={round(keras_metrics['mean_prediction'][i, j], 2)}")
                plt.text(j-0.45, i+0.15 + len(self.NEGATIVE_STEPS) + 1, f"auc={round(keras_metrics['roc_auc'][i, j], 3)}")
        plt.savefig(f'experiments/plots/{self.__class__.__name__}_stacked.png')

    def write_metrics(self, catboost_metrics, keras_metrics):
        if not os.path.exists('experiments'):
            os.mkdir('experiments')
        if not os.path.exists('experiments/metrics'):
            os.mkdir('experiments/metrics')
        filename = f'experiments/metrics/{self.__class__.__name__}.pickle'
        result = {
            'catboost': catboost_metrics,
            'keras': keras_metrics,
        }
        with open(filename, 'wb') as f:
            pickle.dump(result, f)

    def plot_metrics_diff(self, catboost_metrics, keras_metrics):
        if not os.path.exists('experiments'):
            os.mkdir('experiments')
        if not os.path.exists('experiments/plots'):
            os.mkdir('experiments/plots')
        diff = (catboost_metrics['roc_auc'] - keras_metrics['roc_auc']) / keras_metrics['roc_auc']
        min_intensity = 0.5
        clip_threshold = 0.7
        diff_rgb = np.full((diff.shape[0], diff.shape[1], 3), 1.0)
        for i in range(diff_rgb.shape[0]):
            for j in range(diff_rgb.shape[1]):
                clipped = max(min(diff[i, j], clip_threshold), -clip_threshold)
                coef = (clip_threshold - abs(clipped)) / clip_threshold
                blue = 1 * coef + min_intensity * (1 - coef)
                if clipped > 0:
                    green = 1
                    red = 1 * coef + min_intensity * (1 - coef)
                else:
                    green = 1 * coef + min_intensity * (1 - coef)
                    red = 1
                diff_rgb[i, j, 0] = red
                diff_rgb[i, j, 1] = green
                diff_rgb[i, j, 2] = blue
        plt.figure(figsize=(self.PLOT_FIG_SIZE[0], self.PLOT_FIG_SIZE[1]/2))
        plt.title('Catboost over Keras', fontsize=18)
        plt.imshow(diff_rgb)
        plt.xlabel('Positives', fontsize=14)
        plt.ylabel('Negatives', fontsize=14)
        plt.xticks(range(len(self.POSITIVE_STEPS)), self.POSITIVE_STEPS)
        plt.yticks(range(len(self.NEGATIVE_STEPS)), self.NEGATIVE_STEPS)
        for i in range(len(self.NEGATIVE_STEPS)):
            for j in range(len(self.POSITIVE_STEPS)):
                plt.text(j-0.45, i-0.15, f'true+={round(self.POSITIVE_STEPS[j] / (self.POSITIVE_STEPS[j] + self.NEGATIVE_STEPS[i]), 2)}')
                plt.text(j-0.45, i+0.15, f"auc={'+' if diff[i, j] > 0 else ''}{round(diff[i, j]*100, 1)}%")
        plt.savefig(f'experiments/plots/{self.__class__.__name__}_diff.png')

    def print_summary(self, catboost_metrics, keras_metrics, keras_params):
        if not os.path.exists('experiments'):
            os.mkdir('experiments')
        if not os.path.exists('experiments/summaries'):
            os.mkdir('experiments/summaries')
        with open(f'experiments/summaries/{self.__class__.__name__}.txt', 'w') as f:
            print('==== METRICS SUMMARY ====\n')
            f.write('==== METRICS SUMMARY ====\n\n')
            for key in sorted(catboost_metrics):
                if key == 'mean_prediction':
                    continue  # not a very intereting metric
                catboost_value = catboost_metrics[key][-1, -1]
                keras_value = keras_metrics[key][-1, -1]
                catboost_diff = (catboost_value - keras_value) / max(keras_value, 1e-6) * 100
                catboost_diff_sign = '+' if catboost_diff >= 0.0 else ''
                keras_diff = (keras_value - catboost_value) / max(catboost_value, 1e-6) * 100
                keras_diff_sign = '+' if keras_diff >= 0.0 else ''
                print(f'{key:>10}: catboost={catboost_value:.3f} ({catboost_diff_sign}{catboost_diff:.2f}%), keras={keras_value:.3f} ({keras_diff_sign}{keras_diff:.2f}%)\n')
                f.write(f'{key:>10}: catboost={catboost_value:.3f} ({catboost_diff_sign}{catboost_diff:.2f}%), keras={keras_value:.3f} ({keras_diff_sign}{keras_diff:.2f}%)\n\n')
            layer_sizes, activation, dropout, optimizer = keras_params
            print(f'keras hyperparameters: layers={str(layer_sizes)}, activation={activation}, dropout={dropout}, optimizer={optimizer}')
            f.write(f'keras hyperparameters: layers={str(layer_sizes)}, activation={activation}, dropout={dropout}, optimizer={optimizer}\n')
            print('=========================')
            f.write('=========================\n')

    def log_progress(self, message):
        caller_class = self.__class__.__name__
        caller_fn = sys._getframe().f_back.f_code.co_name
        full_message = f'{caller_class}.{caller_fn}: {message}'
        if self.LOG_TO_STDOUT:
            print(full_message)
        if self.LOG_TO_TELEGRAM:
            try:
                telegram_send.send(messages=[full_message])
            except Exception as exc:
                print(exc)


from abstraits.obase import BaseObj

from .direction import Direction
from .vecteur import Vecteur
from .force import Propulsion, Frottement

class Vehicule(BaseObj):

    """Classe représentant un véhicule

    Un véhicule est un objet potentiellement enregistrable (hérité d'BaseObj)
    possédant plusieurs attributs et méthodes caractérisant le déplacement
    d'un véhicule d'un type standard en trois dimensions.

    Le système de déplacement est géré par des vecteurs  modélisant :
    *   La vitesse
    *   La direction
    *   Les valeurs des forces agissant sur le véhicule

    Un véhicule standard possède par défaut deux forces agissant sur lui :
    *   Sa propulsion (force propulsive, absente du modèle abstrait [1])
    *   Son frottement (sans quoi sa vitesse tend vers l'infini).

    Ce comportement peut être surchargé dans des classes filles.

    Attributs :
        masse -- la masse du véhicule
        position -- la position, représentée par un vecteur (X, Y, Z)
        vitesse -- la vitesse actuelle, là encore sous la forme d'un vecteur [1]
        direction -- la direction du véhicule sous la forme d'un vecteur
        frottement -- sa force de frottement
        forces -- une liste comportant les forces agissant sur le véhicule
        salles -- un dictionnaire des salles constituant le véhicule [2]

    [1] La force propulsive n'est pas présente directement dans cette classe.
        En effet, elle est souvent propre au type de véhicule que l'on
        souhaite créer (les navires, par exemple, n'ont pas de force
        propulsive à proprement parlé).

    [2] Le vecteur permet de représenter autant une direction ((1, 0, 0)
        représente un vecteur vers l'est par exemple) qu'une force (le
        vecteur (2, 0, 0) est plus long que (1, 0, 0)). Ces vecteurs
        sont donc utilisés pour représenter la direction, la vitesse
        du véhicule, mais aussi la valeur des forces s'appliquant sur lui.

    [3] Les salles déclarées dans ce dictionnaire sont celles du véhicule
        (son intérieur, par exemple). Toutes ces salles se déplaceront
        avec le véhicule. Le dictionnaire est sous la forme :
        {coordonnes_relatives: salle_correspondante}
        Les coordonnées relatives sont celles par rapport au centre du véhicule
        et restent donc fixes pour le véhicule. Par contre, les coordonnées
        des salles (salle_correspondante.coords) sont modifiées à chaque
        déplacement du véhicule.

    """

    def __init__(self):
        """Constructeur du véhicule.

        Initialise les valeurs et crée deux forces toujours présentes,
        la propulsion qui sert à faire avancer le véhicule ainsi
        que les frottements qui servent à garantir que la vitesse
        ne pourra pas augmenter indéfiniment.

        """
        BaseObj.__init__(self)
        self.masse = 1
        self.position = Vecteur(0, 0, 0, self)
        self.vitesse = Vecteur(0, 0, 0, self)
        self.acceleration = Vecteur(0, 0, 0, self)
        self.direction = Direction(self, 1, 0, 0)

        self.frottement = Frottement(self, 0.7)

        self.forces = [self.frottement]

        self.salles = {}
        self.en_collision = False

        self._construire()

    def __getnewargs__(self):
        return ()

    def energie_cinetique(self):
        return self.m * self.vitesse.norme ** 2

    def maj_salles(self):
        d = self.direction.direction
        i = self.direction.inclinaison
        operation = lambda v: self.position + v.tourner_autour_z(d).incliner(i)
        for vec, salle in self.salles.items():
            vec = Vecteur(*vec)
            vec = operation(vec)
            salle.coords.x = vec.x
            salle.coords.y = vec.y
            salle.coords.z = vec.z

    def avancer(self, temps):
        """Fait avancer le véhicule.

        Cette fonction place le véhicule à sa nouvelle position, celle après
        qu'il se soit écoulé temps seconde virtuelle.

        """
        # Calcul de la nouvelle position
        if not self.en_collision:
            self.position += temps * self.vitesse

        # Si on a une masse nulle, on ne peut pas continuer
        if self.masse == 0:
            raise ValueError("ce véhicule a une masse nulle")

        # On calcule l'accélération à partir des forces
        self.acceleration = Vecteur(0, 0, 0, self)
        for force in self.forces:
            self.acceleration += (1 / self.masse) * force.valeur

        self.forces = [force for force in self.forces if not force.desuette]

        # On calcule la nouvelle vitesse à partir de l'accélération
        self.vitesse += temps * self.acceleration

        # On modifie les coordonnées des salles
        self.maj_salles()

        # On renvoie la nouvelle position
        return self.position

    def collision(self, impacts):
        """Fonction appelé lorsqu'une collision se produit avec ce véhicule.

        La surcharger pour faire réagir un type de véhicule spécifiquement.

        """
        self.en_collision = True

    def get_prochaine_coordonnees(self, temps):
        """Retourne les prochaines coordonnées.

        Cette fonction trouve les prochaine coordonnées du véhicule après
        temps secondes virtuelles en supposant la vitesse constante.

        """
        position = self.position + temps * self.vitesse
        resultat = []
        d = self.direction.direction
        i = self.direction.inclinaison
        operation = lambda v: position + v.tourner_autour_z(d).incliner(i)
        for vec, salle in self.salles.items():
            vec = Vecteur(*vec)
            resultat += operation(vec).coordonnees.entier()

        return resultat


import os
import pandas as pd
import numpy as np
from src.utils import constant
from decouple import config
from src.utils.get_data import get_tweets_by_id
from src.utils.get_data import concatenate_datasets
from src.utils.get_data import get_datasets
from src.utils.preprocessing import preprocessing_restricted
from src.utils.cross_validator import validate_parameters_via_cross_validation
from src.utils import usa_hate_speech_calculator
from src.utils import dataset_balancer
from src.utils.manage_classifiers import choose_and_create_classifier

def load_labeled_dataset():
    """
    Concatenate the data sets from csv-files (labeled_data.csv, hatespeech_text_label_vote_RESTRICTED_100K.csv,
    tweets.csv) together and return it as a pandas dataframe.

    Returns
    -------
    df_concatenated:        Pandas dataframe
                            The dataframe containing all data from the mentioned csv-files.
    """
    # if tweets not already loaded from TwitterAPI
    if not os.path.isfile(os.path.join('data', 'tweets.csv')):
        # load dataset from https://github.com/zeerakw/hatespeech, loads tweets via tweet id
        df = get_tweets_by_id(config, os.path.join('data', 'NAACL_SRW_2016.csv'))

    # load datasets from
    #  https://github.com/t-davidson/hate-speech-and-offensive-language/tree/master/data (df2)
    #  and https://github.com/jaeyk/intersectional-bias-in-ml (df3)
    df2, df3 = get_datasets(os.path.join('data', 'labeled_data.csv'),
                            os.path.join('data', 'hatespeech_text_label_vote_RESTRICTED_100K.csv'))

    df_concatenated = concatenate_datasets(os.path.join('data', 'tweets.csv'), df2, df3)

    return df_concatenated

def run_experiment(df_dataset, preprocessing='preprocessing_restricted'):
    """
    Execute preprocessing on given dataset and validates best classifier and parameters by f1-score.
    Parameters
    ----------

    df_dataset          Pandas dataframe
                        Dataframe including features and labels.
                        On Dataframe Experiments will be executed.

    preprocessing       String
                        Preprocessing methode which should be used.
                        preprocessing_restricted or otherwise preprocessing methode will
                        be used.

    Returns
    -------
    Parameters:         Array
                        including 0: classifer, 1: vectorizer, 2: grams
                        classifier: svm, decison_tree,random_forest, log_reg or ensemble
                        vectorizer: count or tfidf
                        ngram represented as Tuple e.g (1, 2) for uni- and bigrams

    x_balanced          Array
                        Balanced and preprocessed training features

    y_balanced          Array
                        balanced training labels
    """

    # runs chosen preprocess-method on the text-column of the dataframe
    if preprocessing is constant.PREPROCESSING_RESTRICTED:
        df_dataset['preprocessed'] = df_dataset['text'].apply(preprocessing_restricted)
    else:
        df_dataset['preprocessed'] = df_dataset['text'].apply(preprocessing)

    # balance data -> ~9k normal vs ~9k hate speech tweets
    x_balanced, y_balanced = dataset_balancer.balance_data(df_dataset[['preprocessed']],
                                                           df_dataset[['hate_speech']])

    # cross validation
    parameters = validate_parameters_via_cross_validation(x_data=x_balanced,
                                                          y_data=y_balanced)

    return parameters, x_balanced, y_balanced

def make_prediction(dataset_filepath, classifier, vectorizer, ngrams, x_train, y_train,
                    preprocessing='preprocessing_restricted'):
    """
    Make predictions to the input csv-file with the given classifier and parameters.

    Parameters
    ----------
    dataset_filepath:  	String.
                        Filepath to the csv-file which includes the data which should be investigate.
                        The feature column has to be named 'full_text'.

    classifier:         String
                        Names of classifier which should be used.
                        svm, decison_tree,random_forest, log_reg or ensemble

    vectorizer: 		String
                        Name of Vectorizer which should be used.
                        count or tfidf

    ngrams:             Tuple
                        Unigram: (1,1) ,Uni-/Bigrams: (1,2)

    x_train             Array.
                        Features of the training data

    y_train             Array.
                        Labels of the training data

    preprocessing       String
                        Preprocessing methode which should be used.
                        preprocessing_restricted or otherwise preprocessing methode will be used.

    Returns
    -------

    y_pred              Array
                        Predicted labels for the input dataset (csv-file)

    df                  Pandas Dataframe
                        Dataframe loaded from csv-file
    """
    # load dataset from the given filepath
    df = pd.read_csv(dataset_filepath, sep=',')

    # runs chosen preprocess-method on the full_text-column of the dataframe
    if preprocessing is constant.PREPROCESSING_RESTRICTED:
        df['preprocessed'] = df['full_text'].apply(preprocessing_restricted)
    else:
        df['preprocessed'] = df['full_text'].apply(preprocessing)

    model, vec, x_test = choose_and_create_classifier(classifier, x_train, y_train, df[['preprocessed']], vectorizer,
                                                      ngrams)
    x_test = vec.transform(df['preprocessed'].values)
    y_pred = model.predict(x_test)

    return y_pred, df

def calculate_hatespeech_per_us_state(df_usa, y_pred, print_out=True):
    """
    Make predictions to the input csv-file with the given classifier and parameters.

    Parameters
    ----------
    df_usa:  	Pandas dataframe
                Dataset of the collected tweets from different US-States

    y_pred:     Array.
                Predicted labels for the dataset features

    print_out:  Bool.
                True if each average should be print to console

    Returns
    -------

    y_pred      Array
                Predicted labels for the input dataset (csv-file)
    """
    # Calculate avg hate speech per US state
    avg_hate_speech_per_state = usa_hate_speech_calculator.calculate_hate_speech_ratio(df_usa, y_pred)
    unique_city_names = df_usa.city_name.unique()

    # Print avg hate speech rate per US state
    if print_out:
        for state in range(len(unique_city_names)):
            print('US state: ' + unique_city_names[state] + ' | hate speech rate: ' + str(
                np.round(avg_hate_speech_per_state[state], 2)) + '%')

    return avg_hate_speech_per_state


def loot_command(treasure, items):
    for item in items:
        if item not in treasure:
            treasure.insert(0, item)
    return treasure

def drop_command(treasure, index):
    if 0 <= index < len(treasure):
        a = treasure.pop(index)
        treasure.append(a)
    return treasure

def steal_command(treasure, count):
    count *= -1
    steal_treasure = treasure[count::]
    print(', '.join(steal_treasure))
    return treasure[:count]

treasure_chest = input().split('|')
commands = input()

while not commands == "Yohoho!":
    commands = commands.split()
    command = commands[0]
    if command == 'Loot':
        commands.pop(0)
        treasure_chest = loot_command(treasure_chest, commands)
    elif command == 'Drop':
        indx = int(commands[1])
        treasure_chest = drop_command(treasure_chest, indx)
    elif command == 'Steal':
        counter = int(commands[1])
        treasure_chest = steal_command(treasure_chest, counter)
    commands = input()

if len(treasure_chest) > 0:
    divide = len(treasure_chest)
    all_treasure = 0
    for treasure in treasure_chest:
        all_treasure += len(treasure)
    average = all_treasure / divide
    print(f"Average treasure gain: {average:.2f} pirate credits.")
else:
    print("Failed treasure hunt.")


import logging
from fn_microsoft_sentinel.lib.function_common import PACKAGE_NAME, SentinelProfiles
from fn_microsoft_sentinel.lib.sentinel_common import SentinelAPI
from fn_microsoft_sentinel.lib.resilient_common import ResilientCommon
from resilient_circuits import ResilientComponent, function, handler, StatusMessage, FunctionResult, FunctionError
from resilient_lib import ResultPayload, validate_fields

class FunctionComponent(ResilientComponent):
    """Component that implements Resilient function 'sentinel_get_incident_comments''"""

    def __init__(self, opts):
        """constructor provides access to the configuration options"""
        super(FunctionComponent, self).__init__(opts)
        self.options = opts.get(PACKAGE_NAME, {})
        self.sentinel_profiles = SentinelProfiles(opts, self.options)

    @handler("reload")
    def _reload(self, event, opts):
        """Configuration options have changed, save new values"""
        self.options = opts.get(PACKAGE_NAME, {})
        self.sentinel_profiles = SentinelProfiles(opts, self.options)

    @function("sentinel_get_incident_comments")
    def _sentinel_get_incident_comments_function(self, event, *args, **kwargs):
        """Function: Get Comments from a Sentinel Incident"""
        try:
            validate_fields(
                ["sentinel_profile", "sentinel_incident_id"],
                kwargs
            )

            yield StatusMessage("Starting 'sentinel_get_incident_comments'")

            rc = ResultPayload(PACKAGE_NAME, **kwargs)

            # Get the function parameters:
            incident_id = kwargs.get("incident_id")  # int
            sentinel_incident_id = kwargs.get("sentinel_incident_id")  # text
            sentinel_profile = kwargs.get("sentinel_profile")  # text

            log = logging.getLogger(__name__)
            log.info("incident_id: %s", incident_id)
            log.info("sentinel_incident_id: %s", sentinel_incident_id)
            log.info("sentinel_profile: %s", sentinel_profile)

            sentinel_api = SentinelAPI(self.options['tenant_id'],
                                       self.options['client_id'],
                                       self.options['app_secret'],
                                       self.opts, self.options)

            resilient_api = ResilientCommon(self.rest_client())

            profile_data = self.sentinel_profiles.get_profile(sentinel_profile)
            result, status, reason = sentinel_api.get_comments(profile_data, sentinel_incident_id)

            new_comments = []
            if status:
                new_comments = resilient_api.filter_resilient_comments(incident_id, result['value'])

            yield StatusMessage("Finished 'sentinel_get_incident_comments'")

            results = rc.done(status, { "value": new_comments }, reason=reason)

            # Produce a FunctionResult with the results
            yield FunctionResult(results)
        except Exception:
            yield FunctionError()


# -*- coding: utf-8 -*-
"""
Created on Wed Feb 23 21:39:31 2022

@author: stife
"""

class LRUCache:
    def __init__(self, capacity: int=10) -> None:
        """
        Parameters
        ----------
        capacity : int, optional
            DESCRIPTION. Максимальный размер кэша в количестве добавленных значений.
            capacity >= 1. Значение по умолчанию 10.
        """
        self.capacity = capacity
        self.data = {}  # словарь: key = запрос, value - значение

    def get(self, key: str) -> str:
        if key in self.data:
            val = self.data[key][:]
            del self.data[key]
            self.data[key] = val
            return val
        else:
            return ""

    def set(self, key: str, value: str) -> None:
        if key in self.data: # уже есть
            del self.data[key]
            self.data[key] = value
        else:
            if len(self.data) + 1 > self.capacity: # проверка на переполнение
                # удаляем первое значение (в порядке добавление)
                del self.data[next(iter(self.data.keys()))]
            self.data[key] = value

    def rem(self, key: str) -> None:
        if key in self.data:
            del self.data[key]

    def __str__(self):
        return "capacity = %d, data = %s" % (self.capacity, self.data)


# -*- coding: utf-8 -*-
'''

@Author: Isabella
@Create time: 2018/6/13 14:57

'''
import requests
from bs4 import BeautifulSoup
import pymongo
import time

def save_to_mongo(result):
    try:
        db[MONGO_Collection].insert(result)
        print("存储成功")
    except Exception:
        print("存储失败")

def get_html(i):
    headers = {
        'x-requested-with': 'XMLHttpRequest',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
        'cookie':'_ga=GA1.2.1332905533.1528701127; route=6098c46d36cc5825a33759f7c165d408; _gid=GA1.2.1710498787.1528795610; Hm_lvt_17521045a35cb741c321d016e40c7f95=1528701129,1528795610,1528874985; JOSESSIONID=2D1A9F3D1488671C95B8BAD861CAD404-n1; Hm_lpvt_17521045a35cb741c321d016e40c7f95=1528884226'
    }
    rurl = 'https://search.jobmd.cn/s?wd='+ keyword +'&locations=420000&t='+str(i)
    r = requests.get(rurl,headers=headers)
    r.encoding = 'utf-8'
    soup = BeautifulSoup(r.text,'lxml')
    return soup

def get_detail(url):
    rd = requests.get(url)
    rd.encoding = 'utf-8'
    rdsoup = BeautifulSoup(rd.text,'lxml')
    return rdsoup

def get_page_num():
    soup = get_html(i=1)
    sum = soup.find('span', class_='num').text
    return sum

def get_jobs():
    soup = get_html(i)
    form = soup.find('div',class_ = 'list-box mt10 list-box-B')
    try:
        divs = form.select('.list-dl')
        for div in divs:
            # 基础信息
            jobname = div.find('a',class_='item-jobs-name').text
            hosname = div.find('a',class_='item-ent-name').text
            jobloc = div.find('span',class_='job-loc').text
            jobcate = div.find('span', class_='job-cate').text
            jobedu = div.find('span',class_='job-edu').text
            workyear = div.find('span',class_='job-work').text
            jobcontents = div.find('div',class_='job-class-content')
            posttime = jobcontents.find('span', class_='job-class-meta').text

            # 职位详情
            jobhref = div.find('a',class_='show-more')['href']
            rdsoup = get_detail(jobhref)
            items = rdsoup.find('div', class_='box-info_base')
            salary = items.find('span').text.strip().replace(',','')
            try:
                salarys = salary.split('-')
                salary_low = int(salarys[0])
                salary_high = int(salarys[1])
            except:
                salary_low = salary
                salary_high = None
                print('薪资为面议')
            jd = rdsoup.find('div', class_='work-content_box-detail c-box').text.strip()

            # 公司详情
            hosinfo = rdsoup.find('div',class_='work-hospital_info')
            hosinfos = hosinfo.find_all('span')
            industry = hosinfos[0].text
            hoslevel = hosinfos[1].text
            stuffcount = hosinfos[2].text

            # 创建dict。因为mongodb基于js
            dic = {
                '职位':jobname.strip(),
                '薪资最低':salary_low,
                '薪资最高':salary_high,
                '地址':jobloc.strip()[6:],
                '专业要求':jobcate.strip()[6:],
                '学历':jobedu.strip()[5:],
                '工作年限':workyear.strip()[5:-1],
                '招聘开始时间':posttime.replace("\n","")[0:10],
                '招聘截止时间':posttime.replace("\n","")[11:21],
                '职位链接': jobhref.strip(),
                '公司': hosname.strip(),
                '具体行业':industry.strip(),
                '医院类型':hoslevel.strip()[0:4],
                '医院级别':hoslevel.strip()[5:7],
                '公司人数':stuffcount.strip(),
                '职位详情': jd.replace('\n', ' '),
            }
            print(dic)
            save_to_mongo(dic)
    except Exception as err:
        print(err)
        print('可能是因为该岗位无数据')

def txt2list(file):
    with open(file,encoding='utf-8') as f:
        content = [line.strip() for line in f]
        return content

if __name__ == "__main__":
    worktime = time.strftime("%Y%m%d", time.localtime())

    SERVICE_ARGS = ['--load-images=false', '--disk-cache=true']
    client = pymongo.MongoClient('localhost', 27017)
    db = client['dingxiang']

    for keyword in txt2list('list.txt'):
        MONGO_Collection = worktime + '-' + keyword + '-' + '湖北'
        pagenum = int(get_page_num())//20+1
        for i in range(1,pagenum+1):
            get_jobs()


class ServerCommands(dict):
    def __init__(self):
        super().__init__()
        self.update({
            '!connect': 'on_client_connect',
            '!disconnect': 'on_client_disconnect'
        })

    def register_command(self, command, callback_name):
        self[command] = callback_name

    def remove_command(self, command):
        try:
            del self[command]
            return True
        except KeyError:
            return False

    def get_command(self, get_command):
        try:
            self[get_command]
        except KeyError:
            return None


import logging
import binascii
import Crypto.Random
from Crypto.Hash import SHA256
from Crypto.PublicKey import RSA
from Crypto.Signature import PKCS1_v1_5

logger = logging.getLogger(__name__)

class Wallet:
    def __init__(self):
        self.private_key = None
        self.public_key = None

    def create_keys(self):
        """
        Function to create private and public keys to validate transactions
        """
        private_key, public_key = self.generate_keys()
        self.private_key = private_key
        self.public_key = public_key

    def save_keys(self):
        """
        Function to save keys to the disk
        """
        if not (self.public_key is None and self.private_key is None):
            try:
                with open('wallet.txt', 'w') as f:
                    f.write(self.public_key)
                    f.write("\n")
                    f.write(self.private_key)
                return True
            except (IOError,
                    IndexError):  # IndexError - File exists but is empty
                logger.error('Saving wallet failed')
                print('Saving wallet failed')
                return False

    def load_keys(self):
        """
        Function to load keys from the disk
        """
        try:
            with open('wallet.txt', 'r') as f:
                keys = f.readlines()
                # All characters except the line break
                self.public_key = keys[0][:-1]
                self.private_key = keys[1]
            return True
        except (IOError, IndexError):
            logger.error('Loading wallet failed')
            print('Loading wallet failed')
            return False

    def generate_keys(self):
        """
        Function to generate private and public keys
        """
        private_key = RSA.generate(1024, Crypto.Random.new().read)
        public_key = private_key.publickey()
        # Returning string representation of the key
        return (binascii.hexlify(
            private_key.exportKey(format='DER')).decode('ascii'),
                binascii.hexlify(
                    public_key.exportKey(format='DER')).decode('ascii'))

    def sign_transaction(self, sender, recipient, amount):
        signer = PKCS1_v1_5.new(
            RSA.importKey(binascii.unhexlify(self.private_key)))
        hhash = SHA256.new(
            (str(sender) + str(recipient) + str(amount)).encode('utf8'))

        signature = signer.sign(hhash)
        return binascii.hexlify(signature).decode('ascii')

    @staticmethod
    def verify_transaction_signature(transaction):
        public_key = RSA.importKey(binascii.unhexlify(transaction.sender))
        verifier = PKCS1_v1_5.new(public_key)
        hhash = SHA256.new(
            (str(transaction.sender) + str(transaction.recipient) +
             str(transaction.amount)).encode('utf8'))

        return verifier.verify(hhash, binascii.unhexlify(
            transaction.signature))  # Returns Boolean value


# multiAgents.py
# --------------
# Licensing Information:  You are free to use or extend these projects for
# educational purposes provided that (1) you do not distribute or publish
# solutions, (2) you retain this notice, and (3) you provide clear
# attribution to UC Berkeley, including a link to http://ai.berkeley.edu.
#
# Attribution Information: The Pacman AI projects were developed at UC Berkeley.
# The core projects and autograders were primarily created by John DeNero
# (denero@cs.berkeley.edu) and Dan Klein (klein@cs.berkeley.edu).
# Student side autograding was added by Brad Miller, Nick Hay, and
# Pieter Abbeel (pabbeel@cs.berkeley.edu).

from curses import newpad
import dis
from re import A
from shutil import move
from util import manhattanDistance
from game import Directions
import random, util
import statistics

from game import Agent

class ReflexAgent(Agent):
    """
    A reflex agent chooses an action at each choice point by examining
    its alternatives via a state evaluation function.

    The code below is provided as a guide.  You are welcome to change
    it in any way you see fit, so long as you don't touch our method
    headers.
    """

    def getAction(self, gameState):
        """
        You do not need to change this method, but you're welcome to.

        getAction chooses among the best options according to the evaluation function.

        Just like in the previous project, getAction takes a GameState and returns
        some Directions.X for some X in the set {NORTH, SOUTH, WEST, EAST, STOP}
        """
        # Collect legal moves and successor states
        legalMoves = gameState.getLegalActions(self.index)
        legalMoves.remove("Stop")

        # Choose one of the best actions
        scores = [self.evaluationFunction(gameState, action) for action in legalMoves]
        bestScore = max(scores)
        bestIndices = [index for index in range(len(scores)) if scores[index] == bestScore]
        chosenIndex = random.choice(bestIndices) # Pick randomly among the best

        "Add more of your code here if you want to"

        return legalMoves[chosenIndex]

    def evaluationFunction(self, currentGameState, action):
        """
        Design a better evaluation function here.

        The evaluation function takes in the current and proposed successor
        GameStates (pacman.py) and returns a number, where higher numbers are better.

        The code below extracts some useful information from the state, like the
        remaining food (newFood) and Pacman position after moving (newPos).

        Print out these variables to see what you're getting, then combine them
        to create a masterful evaluation function.
        """
        # Useful information you can extract from a GameState (pacman.py)
        score=0
        successorGameState = currentGameState.generatePacmanSuccessor(self.index, action)
        newPos = successorGameState.getPacmanPosition(self.index)
        newFood = successorGameState.getFood()
        newGhostStates = successorGameState.getGhostStates()
        ghostStates = currentGameState.getGhostStates()
        newGhostPositions = successorGameState.getGhostPositions()
        currentPos = currentGameState.getPacmanPosition(self.index)
        distances = []
        for position in newGhostPositions:
            distances.append(manhattanDistance(newPos,position))

        nearestGhostDis = min(distances)
        if(nearestGhostDis<2):#very dangerous
           score-=100
        elif(nearestGhostDis<3):#quite dangerous
           score-=50
        elif(nearestGhostDis<4):#less dangerous
           score-=25
        elif(nearestGhostDis<5):#less dangerous
            score-=10
        food = successorGameState.getFood()
        foodCoordinates = food.asList()#generates a list of coordinates with remaining food
        distancesToFood = []
        #print(pacPos)
        foodRemaining = successorGameState.getNumFood()
        for foodpos in foodCoordinates:
            distancesToFood.append(manhattanDistance(foodpos,newPos))
        if(foodRemaining>=1):
            nearestFoodDis = min(distancesToFood)
            score+=10/nearestFoodDis
        if(foodRemaining<3):
            score+=25/nearestFoodDis
            score=score

        currentFoodCount = currentGameState.getNumFood()
        newFoodCount = successorGameState.getNumFood()
        if(newFoodCount<currentFoodCount):
            score+=5

        remainingCapPos = successorGameState.getCapsules() #positions of remaining capsules
        numRemainingCaps = len(remainingCapPos) #number of capsules remaining
        #score-=25*numRemainingCaps #the less capsules remaining, the better
        distancesToCaps = []
        #score += 0.1*intgamestate.getScore()
        for cappos in remainingCapPos:
            distancesToCaps.append(manhattanDistance(cappos,newPos))
            #print(manhattanDistance(cappos,pacPos))
        if(numRemainingCaps>=1):
            nearestCapDistance = min(distancesToCaps)
            score += 20/nearestCapDistance

        capsules = currentGameState.getCapsules()
        newcapsules = successorGameState.getCapsules()
        closestCapsuleDis = 999999
        closestCapsulePos = (0,0)
        for cap in capsules:
            olddistocap = manhattanDistance(currentPos,cap)
            if(olddistocap<closestCapsuleDis):
                closestCapsuleDis=olddistocap
                closestCapsulePos=cap
        olddistocap=manhattanDistance(closestCapsulePos,currentPos)
        newdistocap = manhattanDistance(closestCapsulePos,newPos)
        print(olddistocap)
        print(newdistocap)
        foods = currentGameState.getFood()
        if(len(newcapsules)>=1):

            if(newdistocap<olddistocap):
                #score+=1000
                print("YESSS")
        elif(newFoodCount==currentFoodCount):
            closestFoodPos = (0,0)
            closesfoodDis = 99999
            #for x in range(0,len(foods)):
            #    for y in range(0,len(foods[x])):
            #        if(currentGameState.hasFood(x,y)):
            #            if(manhattanDistance((x,y),currentPos)<closesfoodDis):
            #                closesfoodDis=manhattanDistance((x,y),currentPos)
            #                closestFoodPos=(x,y)
            #if(manhattanDistance(closestFoodPos,newPos)<closestFoodPos):
            #    score+=5
            #get closer to next available food
        if((len(capsules)-len(newcapsules))>=1):
            score+=10*(newFoodCount)

        #if len(newFood.asList()):
        #    fooddist = util.manhattanDistance(newPos, newFood.asList()[0])
        #else:
        #    fooddist = 0
        print(action)
        print(score)
        return score

def scoreEvaluationFunction(currentGameState, index):
    """
    This default evaluation function just returns the score of the state.
    The score is the same one displayed in the Pacman GUI.

    This evaluation function is meant for use with adversarial search agents
    (not reflex agents).
    """
    return currentGameState.getScore()[index]

class MultiAgentSearchAgent(Agent):
    """
    This class provides some common elements to all of your
    multi-agent searchers.  Any methods defined here will be available
    to the MinimaxPacmanAgent & AlphaBetaPacmanAgent.

    You *do not* need to make any changes here, but you can if you want to
    add functionality to all your adversarial search agents.  Please do not
    remove anything, however.

    Note: this is an abstract class: one that should not be instantiated.  It's
    only partially specified, and designed to be extended.  Agent (game.py)
    is another abstract class.
    """

    def __init__(self, index = 0, evalFn = 'scoreEvaluationFunction', depth = '2'):
        self.index = index # Pacman is always agent index 0
        self.evaluationFunction = lambda state:util.lookup(evalFn, globals())(state, self.index)
        self.depth = int(depth)

class MultiPacmanAgent(MultiAgentSearchAgent):
    """
    You implementation here
    """
    def heuristic(self,gamestate):
        if(gamestate.isWin()):
            return 99999999
        elif(gamestate.isLose()):
            return -9999999
        score = 0
        index = self.index #index of pacman
        pacPos = gamestate.getPacmanPosition(index) #position of pacman
        foodRemaining = gamestate.getNumFood() #amount of food remaining in the game
        #score -= 2*foodRemaining #the less food remaining, the better
        food = gamestate.getFood()
        foodCoordinates = food.asList()#generates a list of coordinates with remaining food
        distancesToFood = []
        #print(pacPos)
        for foodpos in foodCoordinates:
            distancesToFood.append(manhattanDistance(foodpos,pacPos))
        if(foodRemaining>=1):
            nearestFoodDis = min(distancesToFood)
            score+=10/nearestFoodDis
        if(foodRemaining<3):
            score+=25/nearestFoodDis
            score=score
        ghostPositions = gamestate.getGhostPositions()
        distancesToGhost = []
        for ghostpos in ghostPositions:
            distancesToGhost.append(manhattanDistance(ghostpos,pacPos))

        nearestGhostDis = min(distancesToGhost)

        if(nearestGhostDis<2):#very dangerous
           score-=50
        elif(nearestGhostDis<3):#quite dangerous
           score-=25
        elif(nearestGhostDis<4):#less dangerous
           score-=10
        #elif(nearestGhostDis<5):#less dangerous
        #    score-=10
        averageGhostdis = statistics.mean(distancesToGhost)
        #score +=2*averageGhostdis
        remainingCapPos = gamestate.getCapsules() #positions of remaining capsules
        numRemainingCaps = len(remainingCapPos) #number of capsules remaining
        #score-=25*numRemainingCaps #the less capsules remaining, the better
        distancesToCaps = []
        #score += 0.1*intgamestate.getScore()
        for cappos in remainingCapPos:
            distancesToCaps.append(manhattanDistance(cappos,pacPos))
            #print(manhattanDistance(cappos,pacPos))
        if(numRemainingCaps>=1):
            nearestCapDistance = min(distancesToCaps)
            score += 30/nearestCapDistance #incentivize capsules before food raised to 30 from 20
        score+=100/(foodRemaining+1)
        score+=250/(numRemainingCaps+1) #raised from 200 to 250
        score+=scoreEvaluationFunction(gamestate,index) #incentivize eating food faster and avoiding idle states

        return score

    #My implementation extends the minmax algo for multiple minimizing agents. By splitting the search into three functions, I'm
    #able to compute the min and max scores for each action and then return the action with the highest score.

    def minimumSearchAgent(self,gamestate,index,depth):
        if((gamestate.isWin())): #depth reached or game is over
            return 9999999
        if(gamestate.isLose()):
            return -999999
        if(depth==self.depth):
            return self.heuristic(gamestate) #must cut search slightly early to avoid timeout on gradescope
        lowestScore = float('inf') # set the initial lowest score to infinity so that all scores are worse
        nextagent = ""
        for i in range(0,len(gamestate.getLegalActions(index))):
            nextState = gamestate.generateSuccessor(index,gamestate.getLegalActions(index)[i])
            if(index<gamestate.getNumAgents()-1): #iterate for next ghost
                lowestScore = min(lowestScore,self.minimumSearchAgent(nextState,index+1,depth))
            elif(index>=gamestate.getNumAgents()-1): #iterated for all ghosts, return to maxsearch
                lowestScore = min(lowestScore,self.maximumSearchAgent(nextState,0,depth)) #pass control back to maxsearch
        return lowestScore

    def maximumSearchAgent(self,gamestate,index,depth):
        if((gamestate.isWin())): #depth reached or game is over
            return 9999999
        if(gamestate.isLose()):
            return -999999
        if(depth==self.depth):
            return self.heuristic(gamestate)
        greatestScore = float('-inf') #set the initial best score to negative infinity so all scores are better

        for i in range(0,len(gamestate.getLegalActions(self.index))):
            nextState = gamestate.generateSuccessor(index,gamestate.getLegalActions(self.index)[i])
            greatestScore = max(self.minimumSearchAgent(nextState,index+1,depth+1),greatestScore)
        return greatestScore

    def minmaxSearch(self,gamestate,index,depth):
        bestScore = float('-inf') #initialize the initial best score to be better than all possible scores
        for i in range(0,len(gamestate.getLegalActions(index))):
            scores = []
            scoresandmoves = []
            nextstate = gamestate.generateSuccessor(index,gamestate.getLegalActions(index)[i]) #evaluate for all possible pacman moves
            if(nextstate.isWin()):
                return gamestate.getLegalActions(index)[i]
            currentGameScore = self.minimumSearchAgent(nextstate,index+1,depth) #pacman just moved so evaluate ghost actions next
            scores.append(currentGameScore)
            scoresandmoves.append([currentGameScore,move])
            if(currentGameScore>bestScore):     #choose the best state at the end of the search
                bestScore, bestmove=currentGameScore, gamestate.getLegalActions(index)[i]

        return bestmove,bestScore #return the move with the highest score and corresponging score

    def getAction(self, gameState):
        index = self.index # pacman index
        """
        Returns the expectimax action using self.depth and self.evaluationFunction

        All ghosts should be modeled as choosing uniformly at random from their
        legal moves.

        Some functions you may need:
        if gameState.isWin() or gameState.isLose():
            return self.evaluationFunction(gameState)
        legalMoves = gameState.getLegalActions(agent)
        legalNextState = [gameState.generateSuccessor(agent, action)
                          for action in legalMoves]
        """
        "*** YOUR CODE HERE ***"
        #print("Number of Pacmans:", gameState.getNumPacman(), ", Number of ghosts:", gameState.getNumGhosts())
        pos = gameState.getPacmanPosition(self.index)
        legalmoves = gameState.getLegalActions(self.index)
        for move in legalmoves:
            if(gameState.generateSuccessor(self.index,move).isWin()):
                return move

        return self.minmaxSearch(gameState,index,1)[0]

        util.raiseNotDefined()

class RandomAgent(MultiAgentSearchAgent):
    def getAction(self, gameState):
        legalMoves = gameState.getLegalActions(self.index)
        return random.choice(legalMoves)


import concurrent.futures
import functools
import pathlib
import re
import sys
import tarfile

import pandas as pd
import tqdm

sys.path.insert(0, '../src/')
import utils  # noqa:E402

def get_subfiles(archive_file_path, n_drugs):
    """
    Find all score, interaction, and log files within a `.tar.gz` archive.
    Each subfile is represented as a list.

    For OFFSIDES, files are [drug, bootstrap_iteration, file_type,
    subfile_name, archive_path].

    For TWOSIDES, files are [drug_1, drug_2, file_type, subfile_name,
    archive_path]
    """
    file_locations = list()
    try:
        tar = tarfile.open(archive_file_path, mode='r:gz')
        subfiles = tar.getnames()
    except tarfile.ReadError:
        return None
    except EOFError:
        return None

    if n_drugs == 1:
        for subfile in subfiles:
            if 'interaction' in subfile:
                drug = re.match(r'(?:interactions__)([0-9]+)(?:\.npy)', subfile)
                if not drug:
                    raise ValueError(f'{archive_file_path.name} contained {subfile} not matched')
                drug = drug.group(1)
                bootstrap = None
                file_type = 'interaction'
            else:
                drug, bootstrap = utils.extract_indices(subfile)
                file_type = re.match('^[a-z]+(?=_.+)', subfile).group()
            file_locations.append([drug, bootstrap, file_type, subfile,
                                   archive_file_path.name])
    elif n_drugs == 2:
        for subfile in subfiles:
            file_name_match = re.match(r'([a-z]+)(?:.*?__)([0-9]+)(?:_)([0-9]+)(?=\.npy)', subfile)
            if not file_name_match:
                raise ValueError(f'{archive_file_path.name} contained {subfile} not matched')
            file_type, drug_1, drug_2 = file_name_match.groups()
            drug_1, drug_2 = int(drug_1), int(drug_2)
            file_locations.append([drug_1, drug_2, file_type, subfile,
                                   archive_file_path.name])
    return file_locations

def compute_file_map(n_drugs, archives_path):
    """Compute a file map for an entire directory of archives"""
    get_subfiles_partial = functools.partial(get_subfiles, n_drugs=n_drugs)

    archive_files = list(archives_path.glob('scores_*.tgz'))
    with concurrent.futures.ProcessPoolExecutor() as executor:
        file_locations = list(tqdm.tqdm(
            executor.map(get_subfiles_partial, archive_files),
            total=len(archive_files)
        ))

    # Flatten list of lists of tuples to a list of tuples
    flattened_file_locations = [l for l in file_locations if l is not None]
    flattened_file_locations = [i for l in flattened_file_locations for i in l]

    # First two column names depend on the number of drugs
    n_drug_to_column_names = {
        1: ('drug', 'bootstrap'),
        2: ('drug_index_1', 'drug_index_2'),
    }
    column_names = [*n_drug_to_column_names[n_drugs], 'file_type', 'file_name',
                    'archive_file']

    # Instantiate DataFrame and save to disk immediately
    return pd.DataFrame(flattened_file_locations, columns=column_names)

def compute_all_filemaps():
    # Path to where the `.tgz` archives are stored
    archives_path = pathlib.Path('/data/archives/')

    # Path where the file maps will be saved
    meta_path = pathlib.Path('/data/meta')

    # Compute and save OFFSIDES file map
    offsides_file_map = compute_file_map(1, archives_path.joinpath('1/'))
    offsides_file_map.to_csv(meta_path.joinpath('file_map_offsides.csv'),
                             index=False)

    # Compute and save TWOSIDES file map
    twosides_file_map = compute_file_map(2, archives_path.joinpath('2/'))
    twosides_file_map.to_csv(meta_path.joinpath('file_map_twosides.csv'),
                             index=False)

if __name__ == "__main__":
    compute_all_filemaps()


# Kieran Fitzgerald 101276562

"""
- must NOT import anything other than the pygame, random, and sys libraries
• must get the name of the source image as a command-line argument
• must use nested loops to visit each pixel of the source image and get its colour value
• must use one or more loops to draw an appropriate1 number of 'points' in your 'new' image
• must randomly alter the position of the drawn 'points' in your 'new' image2
• must NOT use colours for drawing 'points' other than red, green, blue and (if you wish) black3
• can assume that the user will always provide the name of a valid image when prompted
"""

import pygame
import sys
import random

#initialize pygame
pygame.init()

#initialize image source as the command line argument user provides
img_src = pygame.image.load(sys.argv[1])

#set width and height variables by calling get_size function
(wid,hgt) = img_src.get_size()

#set new width and new height variables
new_wid = wid * 5

new_hgt = hgt * 5

#create a blank window surface that is 4 times larger than original
win_sfc = pygame.display.set_mode((new_wid,new_hgt))

#start nested loop (first loop is for rows, second is for columns)
for y in range(hgt):
    for x in range(wid):

        #retrieving red green and blue values from each pixel that we iterate over
        (r,g,b, _) = img_src.get_at((x,y))

        #determining how many of each color pixel will be used in this block of pixels
        #if all three amount to 0, nothing will be printed so it will just be black in that region
        num_red = (r/50)
        num_green = (g/50)
        num_blue = (b/50)

        #precondition while loop that ends when we have printed all of the red pixels that we need to print
        while num_red > 0:
            #draw a circle at a random position in a range of 25 possible values
            pygame.draw.circle(win_sfc,(255,0,0),(random.randint((x*5)-5,(x*5)),random.randint((y*5)-5,(y*5))),1)
            #subtract one from number of pixels for the precondition
            num_red = num_red - 1
        while num_green > 0:
            pygame.draw.circle(win_sfc,(0,255,0),(random.randint((x*5)-5,(x*5)),random.randint((y*5)-5,(y*5))),1)
            num_green = num_green - 1
        while num_blue > 0:
            pygame.draw.circle(win_sfc,(0,0,255), (random.randint((x*5)-5,(x*5)),random.randint((y*5)-5,(y*5))),1)
            num_blue = num_blue - 1

#update the display
pygame.display.update()

#run while loop and check for user hitting quit button
while True:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            exit()


# Solve the ODE for a conducting ring spinning in a magnetic field.
import numpy as np
import scipy.integrate
import matplotlib.pyplot as plt

# This function evaluates the derivatives for the equation
# d^2 theta/dt^2 = - * sin(theta) -q* d theta/dt + Fsin(k*t)
# We work in the transformed variables y[0] = theta, y[1] = d(theta)/dt
k=0
F=0
q=0

def derivatives(y,t,q,f,k):
    return [y[1], -np.sin(y[0]) - q*y[1] + F*np.sin(k*t)]
# Main code starts here
t=np.linspace(0.0, 100.0, 10000)
y0=[0.01, 0.0]

y=scipy.integrate.odeint(derivatives,y0,t,args=(q,k,F))
for i in range(len(y)):
    print("{:8.4g} {:8.4g} {:8.4g}".format(t[i],y[i,0],y[i,1]))

#plotting
plt.plot(t, 0.5*y[:, 1]**2 + (1-np.cos(y[:, 0])), 'b', label='energy(t)')
plt.plot(t, 0.5*(0.01*np.sin(t))**2  + (1-np.cos(0.01*np.cos(t))), 'g', label='theoretical solution(t)')
plt.legend(loc='best')
plt.xlabel('t')
plt.xlim(0,100)
plt.grid()
plt.show()
plt.savefig('pendulum1.pdf')