import numpy as np

from pylatex import Document, Section, Subsection, Tabular, Math, TikZ, Axis, \
    Plot, Figure, Package, Matrix
from pylatex.utils import italic
import os

if __name__ == '__main__':
    item="XT-500"
    platforms=["NSC","WMEH","FRC"]
    device = {
    'XTL-5000': ['NSC'],
    'RT-300': ['WMEH','NSC'],
    'AN/AP-213':['WHEC','FRC','NSC']
    }
    image_filename = os.path.join(os.path.dirname(__file__), 'kitten.jpg')

    doc = Document()
    doc.packages.append(Package('geometry', options=['tmargin=1cm',
                                                     'lmargin=10cm']))

    with doc.create(Section('Hessam has created this!')):
        doc.append('Some regular text and some')
        doc.append(italic('italic text. '))
        doc.append('\nAlso some crazy characters: $&#{}')
        with doc.create(Subsection('Math that is incorrect')):
            doc.append(Math(data=['2*3', '=', 9]))

        with doc.create(Subsection('Table of something')):
            with doc.create(Tabular('rc|cl')) as table:
                table.add_hline()
                table.add_row((1, 2, 3, device['XTL-5000']))
                table.add_hline()
                table.add_row((-1, 'ss', 3, 4))
                table.add_hline(1, 2)
                table.add_empty_row()
                table.add_row((4, 5, 6, 7))

    a = np.array([[100, 10, 20]]).T
    M = np.matrix([[2, 3, 4],
                   [0, 0, 1],
                   [0, 0, 2]])

    with doc.create(Section('The fancy stuff')):
        with doc.create(Subsection('Correct matrix equations')):
            doc.append(Math(data=[Matrix(M), Matrix(a), '=', Matrix(M * a)]))

        with doc.create(Subsection('Beautiful graphs')):
            with doc.create(TikZ()):
                plot_options = 'height=6cm, width=6cm, grid=major'
                with doc.create(Axis(options=plot_options)) as plot:
                    plot.append(Plot(name='model', func='-x^5 - 242'))

                    coordinates = [
                        (-4.77778, 2027.60977),
                        (-3.55556, 347.84069),
                        (-2.33333, 22.58953),
                        (-1.11111, -493.50066),
                        (0.11111, 46.66082),
                        (1.33333, -205.56286),
                        (2.55556, -341.40638),
                        (3.77778, -1169.24780),
                        (5.00000, -3269.56775),
                    ]

                    plot.append(Plot(name='estimate', coordinates=coordinates))

        with doc.create(Subsection('Cute kitten pictures')):
            with doc.create(Figure(position='h!')) as kitten_pic:
                kitten_pic.add_image(image_filename, width='120px')
                kitten_pic.add_caption('Look it\'s on its back')

    doc.generate_pdf('full')


from rocksample import Rocksample
import time
import numpy as np

MAX_STEPS = 100000

if __name__ == "__main__":
    env = Rocksample(7, 8, True)
    start = time.time()
    i = 0
    r = 0
    while i < MAX_STEPS:
        r = env.step(np.random.choice(env.get_legalactions()))
        s = env.get_state()
        done = s[-1]
        i += 1
        if done:
            break
    end = time.time()

    time_per_step = (end - start) / MAX_STEPS
    reward_per_step = r / MAX_STEPS
    print(f"total iterations: {i}")
    print(f"Time per step: {time_per_step :.2} s")
    print(f"Reward per step: {reward_per_step :.2}")


"""
Rozważmy ciąg (a0, . . . , an-1) liczb naturalnych. Załóżmy, że został podzielony na k spójnych podciągów: (a0, . . . , a`1), (a`1+1, . . . , a`2), . . . , (a`k-1+1, . . . , an-1). Przez wartość i-go podciągu rozumiemy sumę jego elementów a przez najgorszy podciąg rozumiemy podciąg o najmniejszej wartości (rozstrzygając remisy w dowolny sposób). Wartością podziału jest wartość jego najgorszego podciągu. Zadanie polega na znalezienie podziału ciągu (a0, . . . , an-1) o maksymalnej wartości.
"""

def maxmin(T, K):
    n = len(T)
    F = [[-float("inf")]*(K+1) for I in range(n)]

    # base
    F[0][1] = T[0]
    for i in range(1, n):
        F[i][1] = F[i-1][1] + T[i]

    # relation
    for k in range(2, K+1):
        for i in range(k-1, n):
            for x in range(k-1, i+1):
                F[i][k] = max(F[i][k], min(F[i][1]-F[x-1][1], F[x-1][k-1]))

    return F[n-1][K]

tab = [1,3,7,2,13]

print(maxmin(tab, 3))


import math
import numpy
import collections
from PIL import Image

from ImageDecoder import ImageDecoder
from Unification import Unification
from ImageHelper import ImageHelper
from Commons import Commons

class HistogramColor(object):
    def __init__(self, firstPath, imageType):
        self.firstDecoder = ImageDecoder(firstPath, imageType)
        self.imageType = imageType

    # Ex6.1
    def calculateHistogram(self):
        print('Compute histogram from color image {}'.format(self.firstDecoder.name))
        height, width = self.firstDecoder.height, self.firstDecoder.width
        image = self.firstDecoder.getPixels()
        bins, histogram = Commons.CalculateColorHistogram(image, height, width)
        ImageHelper.SaveColorHistogram(bins, histogram, 'calculate', self.firstDecoder)

    # Ex6.2
    def moveHistogram(self, constValue=(0,0,0)):
        print('Move histogram in color image {} about {}'.format(self.firstDecoder.name, constValue))
        height, width = self.firstDecoder.height, self.firstDecoder.width
        image = self.firstDecoder.getPixels()
        maxValue = float(numpy.iinfo(image.dtype).max)
        minValue = float(numpy.iinfo(image.dtype).min)
        result = numpy.ones((height, width, 3), numpy.uint8)
        for h in range(height):
            for w in range(width):
                computedR = int(image[h, w, 0]) + constValue[0]
                computedR = maxValue if computedR > maxValue else computedR
                computedR = minValue if computedR < minValue else computedR
                result[h, w, 0] = computedR
                computedG = int(image[h, w, 1]) + constValue[1]
                computedG = maxValue if computedG > maxValue else computedG
                computedG = minValue if computedG < minValue else computedG
                result[h, w, 1] = computedG
                computedB = int(image[h, w, 2]) + constValue[2]
                computedB = maxValue if computedB > maxValue else computedB
                computedB = minValue if computedB < minValue else computedB
                result[h, w, 2] = computedB

        ImageHelper.Save(result, self.imageType, 'move-histogram-image', False, self.firstDecoder, None, constValue)
        bins, histogram = Commons.CalculateColorHistogram(result, height, width)
        ImageHelper.SaveColorHistogram(bins, histogram, 'move-histogram', self.firstDecoder, constValue)

    # Ex6.3
    def extendHistogram(self):
        print('Extend histogram in color image {}'.format(self.firstDecoder.name))
        height, width = self.firstDecoder.height, self.firstDecoder.width
        image = self.firstDecoder.getPixels()
        result = Commons.Normalization(image, image)

        ImageHelper.Save(result, self.imageType, 'extend-histogram-image', False, self.firstDecoder)
        bins, histogram = Commons.CalculateColorHistogram(result, height, width)
        ImageHelper.SaveColorHistogram(bins, histogram, 'extend-histogram', self.firstDecoder)

    # Ex6.4
    def localSingleThresholding(self, contrastThreshold=15, windowSize=3):
        print('Local single thresholding color image {} with contrast threshold of {}'.format(self.firstDecoder.name, contrastThreshold))
        if windowSize % 2 == 0:
            raise ValueError("Window size can't be even")

        height, width = self.firstDecoder.height, self.firstDecoder.width
        image = self.firstDecoder.getPixels()
        maxValue = int(numpy.iinfo(image.dtype).max)
        minValue = int(numpy.iinfo(image.dtype).min)
        if contrastThreshold <= minValue or contrastThreshold >= maxValue:
            raise ValueError("Contrast threshold has to be in range of ({},{})".format(minValue, maxValue))

        result = numpy.zeros((height, width, 3), numpy.uint8)
        overlap = int(math.ceil(windowSize/2))
        for h in range(height):
            for w in range(width):
                minH = 0 if h-overlap < 0 else h-overlap
                maxH = height if h+overlap+1 > height else h+overlap+1
                minW = 0 if w-overlap < 0 else w-overlap
                maxW = width if w+overlap+1 > width else w+overlap+1
                if minH >= maxH or minW >= maxW:
                    continue

                window = image[minH:maxH, minW:maxW]
                for color in range(3):
                    localMin = numpy.amin(window[:, :, color])
                    localMax = numpy.amax(window[:, :, color])
                    localContrast = localMax-localMin
                    midColor = (int(localMax)+int(localMin))/2
                    if localContrast < contrastThreshold:
                        if midColor >= maxValue/2:
                            result[minH:maxH, minW:maxW, color] = numpy.full((maxH-minH, maxW-minW), maxValue)
                        else:
                            result[minH:maxH, minW:maxW, color] = numpy.full((maxH-minH, maxW-minW), minValue)
                    else:
                        if image[h, w, color] >= midColor:
                            result[h, w, color] = maxValue
                        else:
                            result[h, w, color] = minValue

        ImageHelper.Save(result, self.imageType, 'single-local-threshold-image', False, self.firstDecoder, None, contrastThreshold)
        bins, histogram = Commons.CalculateColorHistogram(result, height, width)
        ImageHelper.SaveColorHistogram(bins, histogram, 'single-local-threshold', self.firstDecoder)

    # Ex6.5
    def localMultiThresholding(self, amountOfThresholds=4, windowSize=3):
        print('Local multi thresholding color image {} with amount of thresholds equals to {}'.format(self.firstDecoder.name, amountOfThresholds))
        if windowSize % 2 == 0:
            raise ValueError("Window size can't be even")

        height, width = self.firstDecoder.height, self.firstDecoder.width
        image = self.firstDecoder.getPixels()

        result = numpy.zeros((height, width, 3), numpy.uint8)
        overlap = int(math.ceil(windowSize/2))
        for h in range(height):
            for w in range(width):
                minH = 0 if h-overlap < 0 else h-overlap
                maxH = height if h+overlap+1 > height else h+overlap+1
                minW = 0 if w-overlap < 0 else w-overlap
                maxW = width if w+overlap+1 > width else w+overlap+1
                if minH >= maxH or minW >= maxW:
                    continue

                window = image[minH:maxH, minW:maxW]
                for color in range(3):
                    localMax = float(numpy.amax(window[:, :, color]))
                    scale = localMax / (amountOfThresholds-1) if localMax != 0 else 1
                    result[h, w, color] = int(round(image[h, w, color] / scale)) * int(scale)

        ImageHelper.Save(result, self.imageType, 'multi-local-threshold-image', False, self.firstDecoder, None, amountOfThresholds)
        bins, histogram = Commons.CalculateColorHistogram(result, height, width)
        ImageHelper.SaveColorHistogram(bins, histogram, 'multi-local-threshold', self.firstDecoder, amountOfThresholds)

    # Ex6.6
    def globalSingleThresholding(self):
        print('Global single thresholding color image {}'.format(self.firstDecoder.name))
        height, width = self.firstDecoder.height, self.firstDecoder.width
        image = self.firstDecoder.getPixels()
        maxValue = int(numpy.iinfo(image.dtype).max)
        minValue = int(numpy.iinfo(image.dtype).min)

        thresholdR = numpy.mean(image[:, :, 0])
        thresholdG = numpy.mean(image[:, :, 1])
        thresholdB = numpy.mean(image[:, :, 2])

        result = numpy.zeros((height, width, 3), numpy.uint8)
        for h in range(height):
            for w in range(width):
                result[h, w, 0] = maxValue if image[h, w, 0] >= thresholdR else minValue
                result[h, w, 1] = maxValue if image[h, w, 1] >= thresholdG else minValue
                result[h, w, 2] = maxValue if image[h, w, 2] >= thresholdB else minValue

        ImageHelper.Save(result, self.imageType, 'single-global-threshold-image', False, self.firstDecoder)
        bins, histogram = Commons.CalculateColorHistogram(result, height, width)
        ImageHelper.SaveColorHistogram(bins, histogram, 'single-global-threshold', self.firstDecoder)

    # Ex6.7
    def globalMultiThresholding(self, amountOfThresholds=4):
        print('Global multi thresholding color image {} with amount of thresholds equals to {}'.format(self.firstDecoder.name, amountOfThresholds))
        height, width = self.firstDecoder.height, self.firstDecoder.width
        image = self.firstDecoder.getPixels()

        maxR = numpy.amax(image[:, :, 0])
        maxG = numpy.amax(image[:, :, 1])
        maxB = numpy.amax(image[:, :, 2])
        scaleR = maxR / (amountOfThresholds-1)
        scaleG = maxG / (amountOfThresholds-1)
        scaleB = maxB / (amountOfThresholds-1)

        result = numpy.zeros((height, width, 3), numpy.uint8)
        for h in range(height):
            for w in range(width):
                result[h, w, 0] = int(round(image[h, w, 0] / scaleR)) * int(scaleR)
                result[h, w, 1] = int(round(image[h, w, 1] / scaleG)) * int(scaleG)
                result[h, w, 2] = int(round(image[h, w, 2] / scaleB)) * int(scaleB)

        ImageHelper.Save(result, self.imageType, 'multi-global-threshold-image', False, self.firstDecoder, None, amountOfThresholds)
        bins, histogram = Commons.CalculateColorHistogram(result, height, width)
        ImageHelper.SaveColorHistogram(bins, histogram, 'multi-global-threshold', self.firstDecoder, amountOfThresholds)


## @file Statistics.py
#  @title Statistics
#  @author Polly Yao
#  @date 1/28/2017

## @brief This module has three methods(average, stdDev, rank) for CircleADT.py
#  @details This module takes in a list of instances of circles

import CircleADT
import numpy as np

## @brief This method calculate the average of radius of a list circle instances
#  @param thelist is a list of instance of circles
#  @return the average of all the radius
def average(thelist):
    ans = []

    for element in thelist:
        ans.append(element[2])

    avg = np.average(ans)
    return (avg)

## @brief This method calculate the standard deviation of radius of a list circle instances
#  @param thelist is a list of instance of circles
#  @return the standard deviation of all the radius
def stdDev(thelist):
    ans = []
    for element in thelist:
        i = ans.append(element[2])

    stdD = np.std(ans)
    return (stdD)

## @brief This method rank a list of instances CircleT by radius
#  @param thelist is a list of instance of circles
#  @assumption if there are element duplicates in the list, the function will remove all the duplicates and leave only one copy of that element
#  @return the list ranked by radius
def rank(thelist):
    array = []
    arrai = []
    arraa = []
    for element in thelist:
        array.append(element[2])
        arrai.append(element[2])

    arrayR = []
    sawA = set()
    for element in array:
        if element not in sawA:
            arrayR.append(element)
            sawA.add(element)

    arraiR = []
    sawB = set()
    for element in arrai:
        if element not in sawB:
            arraiR.append(element)
            sawB.add(element)

    for i in range(len(arrayR)):
        for j in range(i+1, len(arrayR)):
            if arrayR[j] > arrayR[i]:
                arrayR[j], arrayR[i] = arrayR[i], arrayR[j]

    for element in arrayR:
        if (element in arraiR):
            c = arraiR.index(element)
            arraa.append(c+1)

    return arraa


"""
This class is used primarily for the extraction and processing of stock data. It can extract price
and volumes data as well as investment ratings and news headlines. This data can then be
processed to produce technical indicators and daily sentiment values from both news headlines
and investment ratings.
"""

from constants import *
from storage import *
# General
from datetime import datetime
import numpy as np
import pandas as pd
import math
import time
# Scraping
import requests
from bs4 import BeautifulSoup
import threading
import re
import json
import selenium
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# Sentiment
from transformers import pipeline
from textblob import TextBlob
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer

class Stock:
    def __init__(self, name, code, ic_name="", start_date="2014-01-01", end_date="2021-01-01", df=pd.DataFrame(), search_term=None, driver=None):
        self.name = name
        self.code = code
        self.ic_name = ic_name
        self.start_date = self._iso_to_datetime(start_date)
        self.end_date = self._iso_to_datetime(end_date)
        self.df = self._initialize_df() if df.empty else df
        self.search_term = name if search_term == None else search_term
        self.driver = driver

        # Parameters
        self.rs_decay = 0.9
        self.ss_decay = 0.9

        # Links
        self.investing_link =f"https://uk.investing.com/equities/{self.ic_name}"
        self.yahoo_link = f"https://uk.finance.yahoo.com/quote/{self.code}"

    def __str__(self):
        rep = f"{self.code} - {self.name}\n{self.df.describe()}"
        return rep

    # Data Extraction Methods
    def _initialize_df(self):
        """Extract price data and use this to create a new dataframe for the stock"""
        def _convert_type(value):
            data_types = [int, float, str]
            for d in data_types:
                try:
                    return d(value)
                except ValueError:
                    pass
        data_points = []
        period1 = str(int(self.start_date.timestamp()))
        period2 = str(int(self.end_date.timestamp()))
        interval = "1d"
        file_link = (f"https://query1.finance.yahoo.com/v7/finance/download/"
            f"{self.code}?period1={period1}&period2={period2}&interval={interval}")
        request = requests.get(file_link, headers=STANDARD_HEADERS)
        if request.status_code == 401:
            raise Exception("Yahoo finance rejected request")
        content = str(request.content).replace("'", "").split("\\n")
        # cols = content[0].split(",")
        for i in range(1, len(content)):
            new_point = [_convert_type(d) for d in [self.code] + content[i].split(",")]
            new_point[1] = self._iso_to_datetime(new_point[1])
            data_points.append(new_point)
        if len(data_points) == 0:
            raise Exception("No data was retrieved by the extraction function")
        return pd.DataFrame(data=data_points, columns=['tic', 'date', 'open', 'high', 'low', 'close', 'adj_close', 'volume'])

    def extract_investment_ranking_data(self):
        url = f"https://uk.finance.yahoo.com/quote/{self.code}/analysis"
        page = requests.get(url, headers=STANDARD_HEADERS)
        if page.status_code == 401:
            raise Exception("Yahoo finance rejected request")

        # 5 lines below adapted from M Button, 2019
        # https://www.mattbutton.com/
        #   how-to-scrape-stock-upgrades-and-downgrades-from-yahoo-finance/
        soup = BeautifulSoup(page.content,'html.parser')
        script = soup.find('script', text=re.compile(r'root\.App\.main'))
        json_text = re.search(r'^\s*root\.App\.main\s*=\s*({.*?})\s*;\s*$', script.string, flags=re.MULTILINE).group(1)
        data = json.loads(json_text)
        rankings_scraped = data['context']['dispatcher']['stores']\
            ['QuoteSummaryStore']['upgradeDowngradeHistory']['history']

        rankings_dict = dict()
        for r in rankings_scraped:
            investment_ranking = {'action':r['action'], 'from':r['fromGrade'], 'to': r['toGrade']}
            iso = datetime.fromtimestamp(r['epochGradeDate']).strftime('%Y-%m-%d')
            date = self._iso_to_datetime(iso)
            if date in rankings_dict:
                rankings_dict[date].append(investment_ranking)
            else:
                rankings_dict[date] = [investment_ranking]
        # Rankings by date list
        rankings_by_date_list = []
        ranking_scores = []
        previous_rank_score = 0
        change_scores = []
        previous_change_score = 0
        for i in range(len(self.df)):
            if self.df.iloc[i]['date'] in rankings_dict:
                rankings = rankings_dict[self.df.iloc[i]['date']]
                ranking_values = [RANKING_VALUES[r['to']] if r['to'] in RANKING_VALUES else 0 for r in rankings]
                for r in rankings:
                    if r['to'] not in RANKING_VALUES:
                        print(r)
                ranking_score = (self.rs_decay * previous_rank_score) + sum(ranking_values)
                change_values = [CHANGE_VALUES[r['action']] for r in rankings]
                change_score = (self.rs_decay * previous_change_score) + sum(change_values)
            else:
                rankings = []
                ranking_score = self.rs_decay * previous_rank_score
                change_score = self.rs_decay * previous_change_score
            previous_rank_score = ranking_score
            previous_change_score = change_score
            rankings_by_date_list.append(rankings)
            ranking_scores.append(ranking_score)
            change_scores.append(change_score)

        self.df['rankings'] = rankings_by_date_list
        self.df['ranking_score'] = ranking_scores
        self.df['ranking_change_score'] = change_scores

    def extract_news_data(self, google=True, investing=False, threads=5, verbose=False):

        def _extract_from_investing_page(articles_dict, n, i, valid):

            def _convert_investing_date(news_date):
                # Converts format of date e.g. Apr 14, 2021 -> 2021-04-14
                if "ago" in news_date:
                    return datetime.today().strftime('%Y-%m-%d')

                date_components = news_date.replace(",", "").split(" ")
                return (f"{date_components[2]}-"
                    f"{MONTHS[date_components[0].lower()]}-{date_components[1]}")

            url = f"https://uk.investing.com/equities/{self.ic_name}-news/{str(n)}"
            page = requests.get(url, headers=STANDARD_HEADERS)
            soup = BeautifulSoup(page.content,'html.parser')

            # Check you haven't looped
            p_number = soup.find('div', id='paginationWrap').find('a', class_='pagination selected').text

            if n != int(p_number):
                valid[i] = False
            else:
                articles_section = soup.find('section', id='leftColumn')
                articles = articles_section.find_all('article')
                for article in articles:
                    d = article.find('div')
                    a = d.find('a')
                    details_sec = d.find(class_= 'articleDetails')
                    details = details_sec.find_all('span')

                    title = a.text
                    link = a['href']
                    author = details[0].text[3:]
                    date = self._iso_to_datetime(_convert_investing_date(details[1].text[3:]))

                    new_article = {'title': title, 'link': link, 'author': author}

                    if date in articles_dict:
                        articles_dict[date].append(new_article)
                    else:
                        articles_dict[date] = [new_article]

                valid[i] = True

        if investing:
            if verbose: print("Extracting Investing News")
            # Populate a dictionary with scraped articles
            investing_articles_dict = dict()
            # Use threading to request pages and extract articles
            start_page = 1
            searching = True
            while searching:
                if verbose: print(f"{start_page} - {start_page + threads}")
                threads_list = []
                valid = [None] * threads
                for n in range(start_page, start_page + threads):
                    t = threading.Thread(target=_extract_from_investing_page, args=(investing_articles_dict, n, n - 1 - start_page, valid))
                    t.start()
                    threads_list.append(t)
                for t in threads_list:
                    t.join()
                if not all(valid):
                    searching = False
                else:
                    start_page += threads

            investing_articles = []
            for i in range(len(self.df)):
                if self.df.loc[i,'date'] in investing_articles_dict:
                    investing_articles.append(investing_articles_dict[self.df.loc[i,'date']])
                else:
                    investing_articles.append([])

            # Add articles to DataFrame
            self.df['investing_articles'] = investing_articles

        def _extract_from_google_news_searchs(articles_dict, driver, pages_per_range=10): #, from_date, to_date, max_pages=1000

            def _convert_google_date(date_string):
                # Converts date in format e.g. 22 Sept 2020 -> 2020-09-22
                date_components = date_string.split(" ")
                return (f"{date_components[2]}-"
                    f"{MONTHS[date_components[1].lower()[:3]]}-{date_components[0]}")

            def _check_for_captcha(driver):
                try:
                    driver.find_element_by_xpath('//*[@id="captcha-form"]')
                    time.sleep(2)
                    print("\nThere appears to be Captcha form. Complete this and then press enter.")
                    input()
                except selenium.common.exceptions.NoSuchElementException as e:
                    pass
            def _return_element(driver, xpath):
                WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, xpath)))
                return driver.find_element_by_xpath(xpath)

            def _change_date_range(driver, from_date, to_date):
                _return_element(driver, '//*[@id="hdtbMenus"]/span[2]/g-popup/div[1]').click()
                _return_element(driver, '//*[@id="lb"]/div/g-menu/g-menu-item[8]').click()
                from_date_box = _return_element(driver, '//*[@id="OouJcb"]')
                from_date_box.clear()
                from_date_box.send_keys(from_date)
                to_date_box = _return_element(driver, '//*[@id="rzG2be"]')
                to_date_box.clear()
                to_date_box.send_keys(to_date)
                _return_element(driver, '//*[@id="T3kYXe"]/g-button').click()

            def _change_search_term(driver, search):
                search_box = _return_element(driver, '//*[@id="lst-ib"]')
                search_box.clear()
                search_box.send_keys(search)
                # search_box.submit()
                _return_element(driver, '//*[@id="mKlEF"]').click()

            # Change search term
            _change_search_term(driver, self.search_term)
            # Check to see if there is a Capcha form that needs to be completed
            _check_for_captcha(driver)

            earliest_year = self.df.loc[0,'date'].year
            latest_year = self.df.loc[len(self.df) - 1,'date'].year
            for y in range(earliest_year, latest_year + 1):
                if str(y) == "2018": time.sleep(60 * 7) # Mid extraction throttle
                from_date = f"01/01/{y}"
                to_date = f"12/31/{y}"
                _change_date_range(driver, from_date, to_date)
                # Iterate through pages
                for i in range(pages_per_range):
                    _check_for_captcha(driver)
                    # Extract articles from page
                    page = driver.page_source
                    soup = BeautifulSoup(page,'html.parser')
                    articles_div = soup.find('div', id='rso')
                    if articles_div == None:
                        pass
                    else:
                        a_divs = articles_div.find_all('div', class_="dbsr")
                        for a_div in a_divs:
                            article = dict()
                            article['title'] = a_div.find('div', role="heading").text
                            article['link'] = a_div.find('a')['href']
                            try:
                                iso = _convert_google_date(a_div.find('span', class_="WG9SHc").text)
                                date = self._iso_to_datetime(iso)
                                if date in articles_dict:
                                    articles_dict[date].append(article)
                                else:
                                    articles_dict[date] = [article]
                            except KeyError:
                                pass

                    # Move to next page
                    WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '//*[@id="pnnext"]')))
                    next_button = driver.find_element_by_xpath('//*[@id="pnnext"]')
                    next_button.click()

        if google:
            if verbose: print("Extracting Google News")
            if self.driver == None:
                driver_local = True
                driver = self.get_google_news_driver()
            else:
                driver_local = False
                driver = self.driver

            # Extract news headlines from google news
            google_articles_dict = dict()
            _extract_from_google_news_searchs(google_articles_dict, driver)

            # Quit driver when finished
            if driver_local: driver.quit()
            self.driver = None

            # Create dated list from dictionary
            google_articles = []
            for i in range(len(self.df)):
                if self.df.loc[i,'date'] in google_articles_dict:
                    google_articles.append(google_articles_dict[self.df.loc[i,'date']])
                else:
                    google_articles.append([])

            # Add articles to DataFrame
            self.df['google_articles'] = google_articles

    # Data Processing Methods
    def calculate_technical_indicators(self):
        """Calculates technical indicators and adds them to the DataFrame."""
        # MACD
        smoothing = 2
        ema12 = [None for _ in range(11)]
        ema26 = [None for _ in range(25)]
        macd = [None for _ in range(25)]
        ema12.append(sum(self.df.iloc[:12]['close'])/12)
        ema26.append(sum(self.df.iloc[:26]['close'])/26)
        for i in range(12, len(self.df)):
            if i > 11: # EMA 12
                ema12.append((self.df.iloc[i]['close'] * (smoothing / 13)) + (ema12[i-1] * (1 - (smoothing / 13))))
            if i > 25: # EMA 26
                ema26.append((self.df.iloc[i]['close'] * (smoothing / 27)) + (ema26[i-1] * (1 - (smoothing / 27))))
            if i > 24: # MACD
                macd.append(ema12[i] - ema26[i])
        signal_line = [None for _ in range(33)]
        signal_line.append(sum(macd[25:34]) / 9)
        for i in range(34, len(self.df)):
            signal_line.append((macd[i] * (smoothing / 10)) + (signal_line[i-1] * (1 - (smoothing / 10))))

        # RSI
        # Calculating close changes
        close_change = [None]
        for i in range(1, len(self.df)):
            close_change.append(self.df.iloc[i]['close'] - self.df.iloc[i-1]['close'])
        up_sma14 = [None for _ in range(14)]
        down_sma14 = [None for _ in range(14)]
        rsi = [None for _ in range(14)]
        normalized_rsi = [None for _ in range(14)]
        # Up and Down SMA 14
        for i in range(14, len(self.df)):
            # ups = list(filter(lambda x: x > 0, close_change[i-13:i+1]))
            ups = [x for x in close_change[i-13:i+1] if x > 0]
            up_sma14.append(1e-8 if len(ups) == 0 else sum(ups) / 14)
            # downs = list(map(lambda x:abs(x), filter(lambda x: x < 0, close_change[i-13:i+1])))
            downs = [abs(x) for x in close_change[i-13:i+1] if x < 0]
            down_sma14.append(1e-8 if len(downs) == 0 else sum(downs) / 14)
            rsi.append(100 - (100 / (1 + (up_sma14[i] / down_sma14[i]))))
            normalized_rsi.append(rsi[i] / 100)

        # Bollinger Bands
        sma20 = [None for _ in range(19)]
        std_dev20 = [None for _ in range(19)]
        bb_upper = [None for _ in range(19)]
        bb_lower = [None for _ in range(19)]
        std_devs_out = [None for _ in range(19)]
        for i in range(19, len(self.df)):
            sma20.append(sum(self.df.iloc[i-19:i+1]['close']) / 20)
            std_dev20.append(math.sqrt(sum([(d - sma20[i])**2 for d in self.df.loc[:,'close'].iloc[i-19:i+1]]) / 20))
            bb_upper.append(sma20[i] + (2 * std_dev20[i]))
            bb_lower.append(sma20[i] - (2 * std_dev20[i]))
            std_devs_out.append((self.df.iloc[i]['close'] - sma20[i]) / std_dev20[i])

        # Relative Volume
        vol_sma60 = [None for _ in range(59)]
        relative_vol = [None for _ in range(59)]
        for i in range(59, len(self.df)):
            vol_sma60.append(sum(self.df.loc[i-59:i,'volume']) / 60)
            relative_vol.append(self.df.iloc[i]['volume'] / vol_sma60[i])

        generated_lists = {
            'ema12': ema12, 'ema26': ema26, 'macd': macd, 'signal_line': signal_line, 'close_change': close_change,
            'up_sma14': up_sma14, 'down_sma14': down_sma14, 'rsi': rsi, 'normalized_rsi': normalized_rsi, 'sma20': sma20,
            'std_dev20': std_dev20, 'bb_upper': bb_upper, 'bb_lower': bb_lower, 'std_devs_out': std_devs_out,
            'vol_sma60': vol_sma60, 'relative_vol': relative_vol}

        # Check that the list lengths are correct
        if not all([len(self.df) == len(l) for l in generated_lists.values()]):
            raise Exception("One of the lists in technical indicators calculation was wrong length")

        # Add technical indicator lists to the dataframe
        for key, value in generated_lists.items():
            self.df[key] = value

    def calculate_news_sentiment(self, hugging_face=True, text_blob=False, vader=False, verbose=False):
        """Use various libraries to extract sentiment from all available news articles"""
        # Check which sources we have extracted from
        possible_sources = ["google_articles", "investing_articles"]
        available_sources = []
        for source in possible_sources:
            if source in self.df:
                available_sources.append(source)
        # Extract sentiment from available sources
        for source in available_sources:
            if hugging_face:
                if verbose: print("Hugging face")
                classifier = pipeline("sentiment-analysis")
                for i in range(len(self.df)):
                    for a in self.df.loc[i, source]:
                        a['hugging_face'] = classifier(a['title'])[0]
                        if verbose: print(a['hugging_face'])

                if verbose: print("Calculating Hugging face scores")
                previous = 0
                scores = []
                for i in range(len(self.df)):
                    if len(self.df.loc[i, source]) > 0:
                        values = [HF_LABEL_VALUES[a['hugging_face']['label']] for a in self.df.loc[i, source]]
                        score = (self.ss_decay * previous) + sum(values)
                    else:
                        score = (self.ss_decay * previous)
                    scores.append(score)
                    previous = score
                self.df[f'hf_{source}_score'] = scores

            if text_blob:
                if verbose: print("Text blob")
                for i in range(len(self.df)):
                    for a in self.df.loc[i, source]:
                        title_sentiment = TextBlob(a['title']).sentiment
                        a['text_blob'] = {"polarity": title_sentiment.polarity, "subjectivity": title_sentiment.subjectivity}

                if verbose: print("Calculating Text blob scores")
                previous = 0
                scores = []
                for i in range(len(self.df)):
                    if len(self.df.loc[i, source]) > 0:
                        values = [a['text_blob']['polarity'] for a in self.df.loc[i, source]]
                        score = (self.ss_decay * previous) + sum(values)
                    else:
                        score = (self.ss_decay * previous)
                    scores.append(score)
                    previous = score
                self.df[f'tb_{source}_score'] = scores

            if vader:
                if verbose: print("Vader")
                analyzer = SentimentIntensityAnalyzer()
                for i in range(len(self.df)):
                    for a in self.df.loc[i, source]:
                        a['vader'] = analyzer.polarity_scores(a['title'])

                if verbose: print("Calculating Vader scores")
                previous = 0
                scores = []
                for i in range(len(self.df)):
                    if len(self.df.loc[i, source]) > 0:
                        values = [a['vader']['compound'] for a in self.df.loc[i, source]]
                        score = (self.ss_decay * previous) + sum(values)
                    else:
                        score = (self.ss_decay * previous)
                    scores.append(score)
                    previous = score
                self.df[f'vader_{source}_score'] = scores

    def calculate_cheat_values(self):
        cheats = []
        for i in range(0, len(self.df) - 1):
            cheats.append((self.df.loc[i+1,'close'] / self.df.loc[i,'close']) - 1)
        cheats.append(0)
        self.df['cheats'] = cheats

    # Compound Methods
    def extract_and_calculate_basic(self, verbose=True):
        if verbose: print("Extracting investment ranking data")
        self.extract_investment_ranking_data()
        if verbose: print("Calculating technical indicators")
        self.calculate_technical_indicators()
        if verbose: print("Calculating cheat values")
        self.calculate_cheat_values()

    def extract_and_calculate_all(self, verbose=True):
        self.extract_and_calculate_basic(verbose=verbose)
        if verbose: print("Extracting news data")
        self.extract_news_data(investing=False)
        if verbose: print("Calculating news sentiment")
        self.calculate_news_sentiment()

    # Other Methods
    def save_as_excel(self):
        self.df.to_excel(f"{self.code}.xlsx")

    @staticmethod
    def _iso_to_datetime(string_date):
        try:
            return datetime(*[int(d) for d in string_date.split("-")])
        except ValueError as e:
            print(f"Failed to convert '{string_date}'")

    @staticmethod
    def get_google_news_driver(headless=False):
        if WEBDRIVER_PATH == "other/chromedriver/":
            raise Exception("No webdriver found, add webdriver file to other/chromedriver folder")
        # Setup webdriver
        options = webdriver.ChromeOptions()
        if headless: options.add_argument('headless')
        options.add_argument('window-size=1200x600')
        driver = webdriver.Chrome(WEBDRIVER_PATH, chrome_options=options)
        # Accept conditions
        driver.get("https://www.google.co.uk/search?")
        button_xpath = '//*[@id="L2AGLb"]'
        WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, button_xpath)))
        button = driver.find_element_by_xpath(button_xpath)
        button.click()
        url = ("https://www.google.co.uk/search?q=Apple&"
            "tbs=cdr:1,cd_min:01/01/2014,cd_max:12/31/2014&tbm=nws")
        driver.get(url)
        try:
            driver.find_element_by_xpath('//*[@id="captcha-form"]')
            time.sleep(2)
            print("\nThere appears to be Captcha form. Complete this and then press enter.")
            input()
        except selenium.common.exceptions.NoSuchElementException as e:
            pass
        return driver

if __name__ == "__main__":
    stocks = retrieve_stocks_from_folder("data/snp_50_stocks_full")
    print(stocks[0].df.columns)
    # for s in stocks:
    #     s.calculate_news_sentiment(hugging_face=False, text_blob=True, vader=True)
    #     save_stock(s, "data/snp_50_stocks_full")

    stocks = retrieve_stocks_from_folder("data/snp_50_stocks_full")
    print(stocks[0].df.columns)
    # stocks = [Stock(*sa) for sa in SNP_500_TOP_100]

    # driver = Stock.get_google_news_driver()

    # for s in stocks:
    #     s.extract_investment_ranking_data()
    # s1 = Stock("Apple", "AAPL", "apple-computer-inc", driver=driver)
    # s2 = Stock('Cisco', 'CSCO', 'cisco-sys-inc', driver=driver)
    # s = Stock('General Electric', 'GE', 'general-electric')
    # s1 = Stock(*SNP_500_TOP_100[0])
    # print(s1.name)
    # s1.extract_and_calculate_basic()
    # s1.calculate_cheat_values()
    # print(s1.df)
    # s1.extract_investment_ranking_data()
    # s2 = Stock(code="XOM", name="XOM", ic_name=None)
    # print(s2.name)
    # s2.extract_investment_ranking_data()
    # print(s.name)
    # save_stock(s, "data")
    # s.extract_investment_ranking_data()
    # for i in range(len(s.df)):
    #     print("RS:", s.df.loc[i, 'ranking_score'], "CS", s.df.loc[i, 'ranking_change_score'])
    # s1.extract_news_data(investing=False, google=True)
    # s2.extract_news_data(investing=False, google=True)
    # # save_stock(s1, "data")

    # # s2.extract_news_data(investing=False, google=True)
    # s1.calculate_news_sentiment(verbose=True)
    # s2.calculate_news_sentiment(verbose=True)
    # save_stock(s2, "data")

    # s = retrieve_dill_object("data\GE_2021-08-05.dill")
    # s1.save_as_excel()
    # s2.save_as_excel()

    # print(pd.DataFrame())

    # df = pd.read_excel("AAPL.xlsx")
    # ns = Stock("Apple", "AAPL", "apple-computer-inc", df=df)

    # print(ns.df)

    # s.ss_decay = 0.8
    # s.calculate_news_sentiment(verbose=True, hugging_face=False, vader=False, text_blob=False)
    # save_stock(s, "data")

    # for v in s.df.loc[:, 'hf_score']:
    #     print(v)

    # s.save_as_excel()
    # for day_articles in s.df['articles']:
    #     for a in day_articles:
    #         print(a['title'])
    #         print(a['text_blob'])
    #         print(a['vader'])

    # df = pd.read_excel("APPL_df")
    # s = Stock("Apple", "AAPL", "apple-computer-inc", df=df)
    # s.calculate_news_sentiment(verbose=True)
    # save_stock(s, "data")

    # # s.extract_price_data()
    # s.calculate_technical_indicators()
    # print(s.df)
    # s.extract_news_data()
    # print(s.loc[:,'inv_articles'])
    # for a in s.df['inv_articles']:

# # CODE TO CHECK DF AND DAILY DATA MATCHES
# for i in range(len(self.data)):
#     # print(f"{getattr(self.data[i], 'rsi')} ... {self.df.loc[i,'rsi']}")
#     try:
#         for k in generated_lists:
#             if (getattr(self.data[i], k) != self.df.loc[i,k]): # and k not in ['rsi', 'normalized_rsi', 'down_sma14', 'up_sma14']:
#                 if not (math.isnan(self.df.loc[i,k]) and getattr(self.data[i], k) == None):
#                     if abs(getattr(self.data[i], k) - self.df.loc[i,k]) > 1e-4:
#                         print(k, "does not match.")
#                         print(f"{getattr(self.data[i], k)} != {self.df.loc[i,k]}")

#                         raise ValueError("Values don't match")

#         for k in ['close', 'volume', 'adj_close', 'date']:
#             if (getattr(self.data[i], k) != self.df.loc[i,k]) and k not in ['rsi', 'normalized_rsi', 'down_sma14', 'up_sma14']:
#                 if not (math.isnan(self.df.loc[i,k]) and getattr(self.data[i], k) == None):
#                     print(k, "does not match.")
#                     print(f"{getattr(self.data[i], k)} != {self.df.loc[i,k]}")

#     except ValueError as e:
#         print(k)
#         print(f"{getattr(self.data[i], k)} != {self.df.loc[i,k]}")
#         raise e


import json

from django.http import JsonResponse
from django.shortcuts import render, redirect
from django.views.generic import TemplateView
from django.contrib.auth.mixins import LoginRequiredMixin
from django.contrib import messages
from django.contrib.auth.decorators import login_required

#Para conectarse a mongoDB desde python
from utils import *
from bson import ObjectId
import pymongo
#Cast from str to array
from ast import literal_eval

# Create your views here.

def delete_course(request, id):
    db = connect("proesCol")

    query = {"_id": ObjectId(id)}
    print(query)
    db.delete_one(query)
    return redirect('courses')

@login_required
def Courses(request):
    # Para saber las asignaturas de un profesor
    db = connect("proesCol")
    if request.method == 'POST':
        #"""""
        my_id = request.POST["id_materia"]
        nuevo_nombre = request.POST["nuevo_nombre"]
        query = {"_id": ObjectId(my_id)}

        new_values = {"$set":{"asignatura.nombre":nuevo_nombre}}
        db.update_one(query, new_values)
        #"""
        return redirect('courses')
    profesor = request.user.username
    asignaturas = []
    salida = db.find({"profesor": profesor}, {"asignatura.nombre": 1})
    for x in salida:
        x["id"] = str(x["_id"])
        asignaturas.append(x)
    return render(request, "courses/courses.html", {"contexto": asignaturas})  # "about.html"

def TopWorst(request, id):
    #se debe recibir un id en el argumento, cuando esté listo se debe borrar el id de abajo
    #Solo funciona por ahora con una sola materia
    # id = '6201dd30cf634ef49138dec8'
    db = connect("proesCol")
    salida = db.find_one({"_id":ObjectId(id)},{"estudiantes":1,"_id":0})["estudiantes"]

    ordered = sorted(salida, key=lambda d: d['definitiva'])
    salidaArr=[]

    if len(ordered)>= 3:
        salidaArr={"est1":ordered[0],"est2":ordered[1],"est3":ordered[2]}
    else:
        for i in range(len(ordered)):
            salidaArr.append(ordered[i])
    return render(request,"courses/topworst.html",{"contexto":salidaArr})  # "about.html"

def TopBest(request, id):
    #se debe recibir un id en el argumento, cuando esté listo se debe borrar el id de abajo
    #Solo funciona por ahora con una sola materia
    # id = '6201dd30cf634ef49138dec8'
    db = connect("proesCol")
    salida = db.find_one({"_id":ObjectId(id)},{"estudiantes":1,"_id":0})["estudiantes"]

    ordered = sorted(salida, key=lambda d: d['definitiva'], reverse=True)
    salidaArr=[]

    if len(ordered)>= 3:
        salidaArr={"est1":ordered[0],"est2":ordered[1],"est3":ordered[2]}
    else:
        for i in range(len(ordered)):
            salidaArr.append(ordered[i])
    return render(request,"courses/topbest.html",{"contexto":salidaArr})  # "about.html"

@login_required
def NewCourse(request):
    #id = str(db.find_one({"asignatura.nombre":"ESTADÍSTICA"},{"_id":1})["_id"])
    #out = db.find_one({"_id":ObjectId(id)})
    if request.method == 'POST':
        nombre = request.POST["nombre"]
        min = float(request.POST["min"])
        max = float(request.POST["max"])
        profesor = request.user.username
        query = {"asignatura":{"nombre":nombre,"min":min,"max":max},"profesor":profesor,"tipo_notas":[],"estudiantes":{}}
        db = connect("proesCol")
        db.insert(query)
        return redirect('courses')
    return render(request, "courses/newcourse.html")  # "about.html"

@login_required
def Spreadsheet(request, id):
    #tipo_notas y estudiantes es lo que se va a recibir del spreedsheet

    db = connect("proesCol")
    query = {"_id":ObjectId(id)}
    salida= db.find(query)[0]

    if request.method == 'POST':
        estudiantes = literal_eval(request.POST['estudiantes'])
        tipo_notas = literal_eval(request.POST['tipo_notas'])
        if tipo_notas != salida["tipo_notas"]:
            db.update_one(query,{"$set":{"tipo_notas":tipo_notas}})
        if estudiantes != salida["estudiantes"]:
            db.update_one(query,{"$set":{"estudiantes":estudiantes}})
        salida= db.find(query)[0]

    salida["id"] = str(salida["_id"])
    return render(request, "courses/spreadsheet.html",{"contexto":salida})   # "about.html"


import sqlite3
import random
import pandas as pd
import numpy as np
import datetime as dt
import gcalcheck as goog
import gcalevent as ct
from dateutil.parser import parse

print('Please enter a monday in the future, following this format: yyyy-m-d')
startDate = raw_input('> ')

con = sqlite3.connect("./coffeechats.sqllite.s3db")
df = pd.read_sql(sql="select * from Person", con=con)

def pairPeople(df):
    # empty
    pairColumns = ['PersonID1', 'Name 1', 'Email 1', 'PersonID2', 'Name 2', 'Email 2']
    dfPairs = pd.DataFrame(columns=pairColumns)

    # Loop until we have paired up everyone based on this number
    unpairedPeople = len(df.index)

    while unpairedPeople > 1:
        print('{0} people left to pair up ...'.format(unpairedPeople))
        isNewPair = False

        while isNewPair == False:

            # Get a random index from 0-num people left
            first = random.randint(0, unpairedPeople - 1)
            second = first

            # Find another person but make sure its not the same as the first person
            while second == first:
                second = random.randint(0, unpairedPeople - 1)

            print('Checking if these people have been paired up before')
            # Lookup Chats table in SQLite db for Person1/Person2

            id1 = df.iloc[[first]]['Id'].values[0]
            dep1 = df.iloc[[first]]['DepartmentID'].values[0]
            name1 = df.iloc[[first]]['Name'].values[0]
            email1 = df.iloc[[first]]['Email'].values[0]

            id2 = df.iloc[[second]]['Id'].values[0]
            dep2 = df.iloc[[second]]['DepartmentID'].values[0]
            name2 = df.iloc[[second]]['Name'].values[0]
            email2 = df.iloc[[second]]['Email'].values[0]

            # different departments
            if dep1 == dep2:
                print('People are in same Department, searching again')
            else:
                # check to see if pair exists in DB
                chat = pd.read_sql_query(sql='SELECT * FROM Chat WHERE PersonId1 = %d AND PersonId2 = %d' % (id1, id2),
                                         con=con)

                if len(chat.index) == 0:
                    print('Pair is new')
                    isNewPair = True

                    print('Paired up Person ID {0} with Person ID {1}'.format(id1, id2))

                    # Create a row with Name1/Email/Name2/Email and append this to our "paired up" list
                    pair = [(id1, name1, email1, id2, name2, email2)]

                    # Convert record ndarray to DataFrame
                    pairDf = pd.DataFrame.from_records(pair, columns=pairColumns)
                    dfPairs = dfPairs.append(pairDf, ignore_index=True)

        # Remove the people from the list of unpaired people
        df = df[df['Name'] != name1]
        df = df[df['Name'] != name2]

        # Remove 2 more people from our list
        unpairedPeople -= 2
        # Eventually df will end up empty

    # Here is our paired up list of people
    dfPairs.to_csv('./paired_people.csv', index=False)
    print(dfPairs)

    # Adding pairs to db. SQLite requires insertions to have an exact header match, so we manipulate df accordingly.
    dfPairs.loc[:, 'ChatTime'] = np.NaN
    dfPairs = dfPairs[['PersonID1', 'PersonID2', 'ChatTime']]
    dfPairs.to_sql("Chat", con, if_exists="append", index=False)
    return dfPairs

def find_appointment(df, date):
    for i in range(0, len(df)):
        row = df.iloc[i]

        name1 = row['Name 1']
        email1 = row['Email 1']
        name2 = row['Name 2']
        email2 = row['Email 2']

        # Get both peoples calendars
        cal1 = goog.get_calendar(email1, date)
        cal2 = goog.get_calendar(email2, date)

        print('email1: %s' % email1)
        print('email2: %s' % email2)
        # This outputs the start and end times for each person's appointments during the week period
        person_1_busy = [{'start': parse(slot['start']), 'end': parse(slot['end'])} for slot in cal1['calendars'][email1]['busy']]
        person_2_busy = [{'start': parse(slot['start']), 'end': parse(slot['end'])} for slot in cal2['calendars'][email2]['busy']]

        #adding a timedelta as defined in nointerns. This defaults to friday 5PM EST of the week entered.
        after, before = goog.compute_end_datetime(date)

        first_available_slot = first_both_available(person_1_busy, person_2_busy, parse(after), parse(before))
        print('First Available Slot: {}'.format(first_available_slot))

        if first_available_slot is not None:
            ct.insert_event(email1, email2, first_available_slot['start'].strftime('%Y-%m-%dT%H:%M:%S%z'), first_available_slot['end'].strftime('%Y-%m-%dT%H:%M:%S%z'))

# this always returns the first slot in the time window, defined by line 99 in no interns
def first_both_available(person_1_busy, person_2_busy, after, before):
    for slot in slots(before, after, end_time_delta=5, step=30):
        if is_available(slot, person_1_busy) \
                and is_available(slot, person_2_busy):
            return slot
        else:
            continue

# Returns whether a given calendar slot is available for a meeting
def is_available(slot, busy):
    for busy_slot in busy:
        if overlap(slot, busy_slot):
            return False
    return True

def overlap(slot1, slot2):
    # assuming slot are datetime, dictionaries
    return ((slot2['start'] <= slot1['start']) and (slot1['start'] <= slot2['end'])) or \
        ((slot1['start'] <= slot2['start']) and (slot2['start'] <= slot1['end']))

# generates slots to check appointment times against
def slots(start, end, end_time_delta, step):
    now = start
    time_slots = []
    while now <= end:
            original = now
            end_hour = now + dt.timedelta(hours=end_time_delta)
            while now < end_hour:
                time_slots.append(now)
                now += dt.timedelta(minutes=step)
            now = original + dt.timedelta(days=1)
    slot_times = []

    for i in range(0, len(time_slots) - 1):
        slot = {'start': time_slots[i], 'end': time_slots[i + 1]}
        slot_times.append(slot)
    return slot_times

pairs = pairPeople(df)

find_appointment(pairs, startDate)


import os
import re
import syslog
import tempfile
import time
import fcntl
from configparser import ConfigParser
import requests

def uri_reader(uri):
    req_opts = {
        'url': uri,
        'timeout': 120,
        'stream': True
    }
    try:
        req = requests.get(**req_opts)
    except Exception as e:
        syslog.syslog(syslog.LOG_ERR,'blacklist download : unable to download file from %s (error : %s)' % (uri, e))
        return

    if req.status_code >= 200 and req.status_code <= 299:
        req.raw.decode_content = True
        prev_chop  = ''
        while True:
            chop = req.raw.read(1024).decode()
            if not chop:
                if prev_chop:
                    yield prev_chop
                break
            else:
                parts = (prev_chop + chop).split('\n')
                if parts[-1] != "\n":
                    prev_chop = parts.pop()
                else:
                    prev_chop = ''
                for part in parts:
                    yield part
    else:
        syslog.syslog(syslog.LOG_ERR,
            'blacklist download : unable to download file from %s (status_code: %d)' % (uri, req.status_code)
        )

if __name__ == '__main__':
    # check for a running download process, this may take a while so it's better to check...
    try:
        lck = open('/tmp/unbound-download_blacklists.tmp', 'w+')
        fcntl.flock(lck, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except IOError:
        # already running, exit status 99
        sys.exit(99)

    startup_time = time.time()
    syslog.openlog('unbound', logoption=syslog.LOG_DAEMON, facility=syslog.LOG_LOCAL4)
    blacklist_items = set()
    if os.path.exists('/var/unbound/etc/blacklists.ini'):
        cnf = ConfigParser()
        cnf.read('/var/unbound/etc/blacklists.ini')
        # exclude (white) lists, compile to regex to be used to filter blacklist entries
        if cnf.has_section('exclude'):
            exclude_list = set()
            for exclude_item in cnf['exclude']:
                try:
                    re.compile(cnf['exclude'][exclude_item], re.IGNORECASE)
                    exclude_list.add(cnf['exclude'][exclude_item])
                except re.error:
                    syslog.syslog(syslog.LOG_ERR,
                        'blacklist download : skip invalid whitelist exclude pattern "%s" (%s)' % (
                            exclude_item, cnf['exclude'][exclude_item]
                        )
                    )
            if not exclude_list:
                exclude_list.add('$^')

            wp = '|'.join(exclude_list)
            whitelist_pattern = re.compile(wp, re.IGNORECASE)
            syslog.syslog(syslog.LOG_NOTICE, 'blacklist download : exclude domains matching %s' % wp)

        # fetch all blacklists
        if cnf.has_section('blacklists'):
            for blacklist in cnf['blacklists']:
                file_stats = {'uri': cnf['blacklists'][blacklist], 'skip' : 0, 'blacklist': 0, 'lines' :0}
                for line in uri_reader(cnf['blacklists'][blacklist]):
                    file_stats['lines'] += 1
                    # cut line into parts before comment marker (if any)
                    tmp = line.split('#')[0].split()
                    entry = None
                    while tmp:
                        entry = tmp.pop(-1)
                        if entry not in ['127.0.0.1', '0.0.0.0']:
                            break
                    if entry:
                        if whitelist_pattern.match(entry):
                            file_stats['skip'] += 1
                        else:
                            file_stats['blacklist'] += 1
                            blacklist_items.add(entry)

                syslog.syslog(
                    syslog.LOG_NOTICE,
                    'blacklist download %(uri)s (lines: %(lines)d exclude: %(skip)d black: %(blacklist)d' % file_stats
                )

    # write out results
    with open("/var/unbound/etc/dnsbl.conf", 'w') as unbound_outf:
        if blacklist_items:
            unbound_outf.write('server:\n')
            for entry in blacklist_items:
                unbound_outf.write("local-data: \"%s A 0.0.0.0\"\n" % entry)

    syslog.syslog(syslog.LOG_NOTICE, "blacklist download done in %0.2f seconds (%d records)" % (
        time.time() - startup_time, len(blacklist_items)
    ))


"""
https://www.acmicpc.net/problem/17472

7 8
0 0 0 0 0 0 1 1
1 1 0 0 0 0 1 1
1 1 0 0 0 0 0 0
1 1 0 0 0 1 1 0
0 0 0 0 0 1 1 0
0 0 0 0 0 0 0 0
1 1 1 1 1 1 1 1
"""

import collections

N, M = map(int, input().split())
matrix = [list(map(int, input().split())) for _ in range(N)]

dr = (0, 1, 0, -1)
dc = (1, 0, -1, 0)

def bfs(r: int, c: int, visited: set):
    deque = collections.deque([(r, c)])
    island = set([(r, c)])

    while deque:
        _r, _c = deque.popleft()

        for i in range(4):
            nr = _r + dr[i]
            nc = _c + dc[i]
            if (
                (0 <= nr < N)
                and (0 <= nc < M)
                and (nr, nc) not in visited
                and matrix[nr][nc] == 1
            ):
                visited.add((nr, nc))
                deque.append((nr, nc))
                island.add((nr, nc))

    return list(island)

def find_islands():
    """
    Find all islands using BFS
    """
    visited = set()
    islands = []

    for r in range(N):
        for c in range(M):
            if matrix[r][c] == 1 and (r, c) not in visited:
                islands.append(bfs(r, c, visited))

    return islands

def build_bridges(id: int, island: list):
    def can_go(r, c):
        return (0 <= r < N) and (0 <= c < M) and matrix[r][c] != 1

    def find_reached_island(r, c):
        for i, island in enumerate(islands):
            if (r, c) in island:
                return i

        return -1

    bridges = set()
    for r, c in island:
        for i in range(4):
            nr = r + dr[i]
            nc = c + dc[i]

            len = 0
            while can_go(nr, nc):
                len += 1

                nr = nr + dr[i]
                nc = nc + dc[i]

            reached_island = find_reached_island(nr, nc)

            if len < 2 or reached_island == -1:
                continue

            bridges.add((id, reached_island, len))

    return bridges

def find_parent(x):
    if parent[x] != x:
        parent[x] = find_parent(parent[x])
    return parent[x]

def union_parent(a, b):
    a = find_parent(a)
    b = find_parent(b)

    if a != b:
        parent[a] = b

def kruskal():
    answer, count = 0, 0

    for a, b, length in bridges:
        if find_parent(a) != find_parent(b):
            union_parent(a, b)

            answer += length
            count += 1

    return answer, count

"""
Solve Problem
"""
islands = find_islands()
bridges_dict = collections.defaultdict(list)
bridges = []
for id, island in enumerate(islands):
    _bridges = build_bridges(id, island)
    for _bridge in _bridges:
        a, b, length = _bridge
        bridges_dict[a].append((b, length))
        bridges.append(_bridge)

parent = [i for i in range(len(islands) + 1)]
bridges.sort(key=lambda x: x[2])

answer, count = kruskal()

print(answer if count == len(islands) - 1 else -1)


import time
import boto3

ROOT_BUCKET_NAME = "virtual-waiting-room-on-aws"

regions = ['ap-northeast-1', 'ap-south-1', 'ap-southeast-1', 'ca-central-1', \
    'eu-central-1', 'eu-north-1', 'eu-west-1', 'sa-east-1', \
    'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2']
print(regions)

for region in regions:
    try:
        s3 = boto3.client('s3', region_name=region)
        name = f"{ROOT_BUCKET_NAME}-{region}"
        response = s3.create_bucket(
            Bucket=name,
            CreateBucketConfiguration={'LocationConstraint': region})
        print(response)
    except Exception as exception:
        print(exception)
    time.sleep(2)


import streamlit as st
from pandas import DataFrame
from common import df_covid_data, df_covid_data_our_countries

def display_datasets_introduction():
    """Display the Importation des données page."""
    st.write(
        "The dataset used for this project is the COVID-19 dataset from \
        Our World in Data: \
        https://github.com/owid/covid-19-data/tree/master/public/data "
    )
    st.write(
        "It is updated daily and includes data on confirmed cases, deaths,\
        hospitalizations, testing, and vaccinations, as well as other \
        variables of potential interest."
    )

def display_dataframe(dataframe: DataFrame):
    """Display a dataframe.

    Parameters
    ----------
    dataframe : DataFrame
        The dataframe to display.
    """
    st.dataframe(dataframe)

def display_datasets_description():
    """Display dataset description."""
    st.write(
        "The dataset with only countries selected for this project,\
        wich are France, Germany, Spain, Italy, Sweden and England:"
    )

def display_datasets_page():
    """Display the page dedicated to the overview of the datasets."""
    st.header("Overview of the datasets")
    display_datasets_introduction()
    display_dataframe(df_covid_data)
    display_datasets_description()
    display_dataframe(df_covid_data_our_countries)

# if __name__ == "__main__":
#     display_datasets_page()


import smtplib
import sys
from email.mime.multipart import MIMEMultipart as text

########### Crredentials #####################
gmail_user = 'teliscos@gmail.com'
gmail_password = '***************' # App Password
##############################################
sent_from = gmail_user
_help = "commmand <email to> <subject> <file/text> <file path/content>"

try:
    to = str(sys.argv[1])
    to = to.split(",")
    subject = str(sys.argv[2])
    opt = str(sys.argv[3])

except:
    print(_help)
    sys.exit()

if opt == "file":
    try:
        body = str(sys.argv[4])
        file_con = open(body,"r")
        body = file_con.read()
    except:
        print("Path Error")
else:
    body=opt

email_text = """ From: From Person <{}>
To: To Person <{}>
MIME-Version: 1.0
Content-type: text/html
Subject: {}

{}
""".format(sent_from, ", ".join(to), subject, body)

try:
    server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
    server.ehlo()
    server.login(gmail_user, gmail_password)
    server.sendmail(sent_from, to, email_text)
    server.close()

    print('Email sent!')
except:
    print('Something went wrong...')


import csv
# НЕ УДАЛЯЙ ЭТО КОД КРАТК ДОРОЖ ПУТИ
with open('plantis.csv', mode='r', encoding='utf8') as f:
    reader = csv.reader(f, delimiter=';', quotechar='"')
    lst = list(reader)
cities_start = []
cities_continue = []
res = []
x_min = 0
point = []
for i in range(len(lst)-1):
    if lst[-1][0] == lst[i][0]:
        cities_start.append(lst[i])

for i in cities_start:

    if lst[-1][1] == i[1]:
        x_min = int(i[-1])
    else:
        point = i[1]
        for z in range(len(lst) - 1):
            for j in point:
                if j == lst[z][0]:
                    if lst[-1][1] == lst[z][1]:
                        print(lst[z], cities_continue)
                        if int(lst[z][-1]) + int(i[-1]) < x_min:
                            print(1)
                            x_min = int(lst[z][-1]) + int(i[-1])
                            res.append(i)
print(res)


import json
import os
from datetime import datetime
from collections import defaultdict
from flask import Flask, render_template, jsonify, request
from flask_socketio import SocketIO, emit

total_data=defaultdict(list)
prefetch_channellist = list()
i=0
last_used_id=''

app = Flask(__name__)
app.config["SECRET_KEY"] = os.getenv("SECRET_KEY")
socketio = SocketIO(app)

@app.route("/lastused", methods=["post", "get"])
def lastused():
	global last_used_id
	if request.method == "POST":
		lst= request.form.get("lst");
		last_used_id= lst
	# print(last_used_id)
	return jsonify({"lastused":last_used_id})

@app.route("/channeldata", methods=["post"])
def channeldata():
	title = request.form.get("title")
	if title in total_data:
		return jsonify({"success":True,
			"chatdetail":total_data[title]
			})
	else:
		return jsonify({"success":False
			})

@app.route("/channelname",methods=["GET","POST"])
def channelname():
	print("entered channelname")
	if request.method == "POST":
		title = request.form.get("title")
		prefetch_channellist.append(title)
		print(prefetch_channellist)
	return jsonify({"success":True,
		"prefetch_channellist":prefetch_channellist
		})

@app.route("/")
def index():
    return render_template('index.html')

@app.route("/chats")
def chats():
	# print(total_data)
	return render_template('chat.html')

@socketio.on("send message")
def sendmessage(data):
	global i
	now = datetime.now()
	instance = tuple( (data["username"], data["mes"],
		 now.strftime('%d-%m-%Y'), now.strftime( '%H:%M')) )
	if data["chatroom_name"] not in total_data:
		i=0

	total_data[data["chatroom_name"]].append(instance)
	username = total_data[data["chatroom_name"]][i][0];
	mes = total_data[data["chatroom_name"]][i][1];
	date = total_data[data["chatroom_name"]][i][2];
	time = total_data[data["chatroom_name"]][i][3];
	i += 1
	emit("announce message", {"date":date,
			"mes": mes,
		 	"username":username,
			"time":time},
			 broadcast = True)


#coding: utf8

import sys
sys.path.append('..')

import os
import src.libs.platform_version
import json
from flask import Flask, request, Response, current_app, g, render_template, redirect, make_response
from flask_cors import CORS
from src.config import Config as C
# from src.common.key import KEY
from src.libs.log import L
from src.common.response import NormalResponseJson, NormalResponse, ErrorResponse, ErrorResponseJson, ErrorResponseData

import src.data_china as DC
import src.data_pos as DP
import json

app = Flask(__name__, static_url_path='')
CORS(app, supports_credentials=True)
app.config['SECRET_KEY'] = 'AreUOK'
app.config['TOKEN_EXPIRATION'] = 86400
BASE_DIR = os.path.abspath(os.path.dirname(__file__)) + "/static/upload/"
FILE_PATH = os.path.dirname(__file__) + "/"

count_file = "./logs/count.txt"
rumor_user_file = "./logs/rumor_user.json"

def read_count():
    with open(count_file, 'r') as file:
        return int(file.read().strip())

def add_count():
    old = read_count()
    with open(count_file, 'w') as file:
        file.write(str(old+1))

@app.route('/getCount')
def get_count():
    data = dict()
    data['count'] = read_count()
    return str(data)

@app.route('/')
def index():
    add_count()
    return "Welcome..."

@app.route('/test')
def test(r):
    R = request.form if request.method=='POST' else request.args
    status = R.get('status', '')
    if status=='': return ErrorResponseJson("请求的参数有误！")
    return NormalResponseJson(request, {'status': status})

@app.route('/getDataSummary')
def get_data_summary():
    add_count()
    R = request.form if request.method=='POST' else request.args
    level = int(R.get('level', 1))
    code = R.get('name', '86')
    if code == 'china' or code == '': code = '86'

    data = DC.get_data_summary(level, code)
    return NormalResponseJson(request, data)

@app.route('/getDataDetails')
def get_data_details():
    add_count()
    R = request.form if request.method=='POST' else request.args
    level = int(R.get('level', 1))
    code = R.get('name', '')

    data = DC.get_data_latest(level, code)
    return NormalResponseJson(request, data)

@app.route('/getTimeData')
def get_time_data():
    add_count()
    R = request.form if request.method=='POST' else request.args
    level = int(R.get('level', 1))
    code = R.get('name', '86')
    if code == 'china' or code == '': code = '86'

    data = DC.get_time_data(level, int(code))
    return NormalResponseJson(request, data)

@app.route('/getNewsData')
def get_news_data():
    add_count()
    R = request.args
    province = R.get('province', None)
    keyword = R.get('keyword', None)
    date = R.get('pubDate', None)
    page = R.get('page', None)
    num = R.get('num', None)
    data, length = DC.get_news_data(province, keyword, date, page, num)
    return NormalResponseJson(request, data, length)

@app.route('/getRumorData')
def get_rumor_data():
    add_count()
    R = request.args
    keyword = R.get('keyword', None)
    type = R.get('rumorType', None)
    page = R.get('page', None)
    num = R.get('num', None)
    data, length = DC.get_rumor_data(keyword, type, page, num)
    return NormalResponseJson(request, data, length)

@app.route('/testRumor')
def test_rumor_data():
    add_count()
    R = request.args
    sentence = R.get('sentence', None)
    if not sentence:
        return ErrorResponseJson("参数不正确")
    data = DC.test_rumor(sentence)

    rumor_user_list = read_rumor_user()
    had = False
    for i in rumor_user_list:
        if sentence == i["info"]:
            had = True
            break
    if not had:
        item = dict()
        item["id"] = len(rumor_user_list)
        item["info"] = sentence
        item["url"] = ""
        rumor_user_list.append(item)
        write_rumor_user(rumor_user_list)

    return NormalResponseJson(request, data)

@app.route('/getTopic')
def get_topic():
    add_count()
    R = request.args
    date = R.get('date', None)
    data = DC.get_topic(date)
    return NormalResponseJson(request, data, len(data))

# @app.route('/testNewsData')
# def test_news_data():
#     pass
#     data = DC.get_news_data_example()
#     return NormalResponseJson(request, data)
#
# @app.route('/testRumorData')
# def test_rumor_data():
#     data = DC.get_rumor_data_example()
#     return NormalResponseJson(request, data)

@app.route('/getInfoUser', methods=['GET'])
def get_rumor_user():
    add_count()
    if request.method == "GET":
        rumor_user_list = read_rumor_user()
        return NormalResponseJson(request, rumor_user_list)
    return ErrorResponseJson()

@app.route('/setInfoUser', methods=['GET'])
def set_rumor_user():
    add_count()
    if request.method == 'GET':
        R = request.args
        id = R.get("id", None)
        url = R.get("url", None)
        if id and url:
            rumor_user_list = read_rumor_user()
            if int(id) < len(rumor_user_list):
                rumor_user_list[int(id)]["url"] = url
                write_rumor_user(rumor_user_list)
                return NormalResponseJson(request, rumor_user_list[int(id)])
    return ErrorResponseJson()

def read_rumor_user():
    with open(rumor_user_file, 'r') as file:
        rumor_user_list = json.load(file)
        return rumor_user_list

def write_rumor_user(rumor_user_list):
    with open(rumor_user_file, 'w') as file:
        json.dump(rumor_user_list, file)

@app.route('/getDataPos')
def get_data_pos():
    add_count()
    R = request.form if request.method=='POST' else request.args
    code = R.get('code', '420000')

    data = DP.get_region_data(int(code))
    return NormalResponseJson(request, data)

@app.route('/getMap')
def get_map():
    R = request.form if request.method=='POST' else request.args
    name = R.get('id', '')
    path = FILE_PATH + "/data/geojson/{}.json".format(name)
    if not os.path.exists(path):
        return ErrorResponseJson("地图文件不存在")

    with open(path, encoding='utf8') as fp:
        data = ''.join(fp.readlines())
        return NormalResponseJson(request, data)

if __name__ == '__main__':
    L.info("Server Start...")
    app.run(host="0.0.0.0",port=C.web.PORT, debug=True)