import os
from flask import Blueprint, render_template, request, current_app, session, redirect, url_for
from db_context_manager import DBConnection
from db_work import select_dict, insert
from sql_provider import SQLProvider
from cache.wrapper import fetch_from_cache
from datetime import datetime

blueprint_order = Blueprint('bp_order', __name__, template_folder='templates', static_folder='static')
provider = SQLProvider(os.path.join(os.path.dirname(__file__), 'sql'))

@blueprint_order.route('/', methods=['GET','POST'])
def order_index():
    db_config = current_app.config['db_config']
    cache_config = current_app.config['cache_config']
    cached_select = fetch_from_cache('all_items_cached', cache_config)(select_dict) #явное задание декоратора
    if request.method == 'GET':
        sql = provider.get('all_items.sql')
        items = cached_select(db_config, sql)
        basket_items = session.get('basket', {})
        return render_template('basket_order_list.html', items=items, basket=basket_items)
    else:
        prod_id = request.form['id_med']
        sql = provider.get('all_items.sql')
        items = select_dict(db_config, sql)

        add_to_basket(prod_id, items)

        return redirect(url_for('bp_order.order_index'))

def add_to_basket(prod_id: str, items: dict):
    item_description = [item for item in items if str(item['id_med']) == str(prod_id)]
    print('item_description before=', item_description)
    item_description = item_description[0]
    curr_basket = session.get('basket', {})

    if prod_id in curr_basket:
        curr_basket[prod_id]['amount'] = curr_basket[prod_id]['amount'] + 1
    else:
        curr_basket[prod_id] = {
            'med_name': item_description['med_name'],
            'med_price': item_description['med_price'],
            'med_amount': item_description['med_amount'],
            'amount': 1
        }
        session['basket'] = curr_basket
        session.permanent = True
    return True

@blueprint_order.route('/clear-basket')
def clear_basket():
    if 'basket' in session:
        session.pop('basket')
    return redirect(url_for('bp_order.order_index'))

@blueprint_order.route('/save_order', methods=['GET', 'POST'])
def save_order():
    user_id = session.get('user_id')
    current_basket = session.get('basket', {})
    order_id = save_order_with_list(current_app.config['db_config'], user_id, current_basket)
    if order_id:

        session.pop('basket')
        return render_template('order_created.html', order_id=order_id)
    else:
        return 'Что-то пошло не так :('

def save_order_with_list(dbconfig:dict, user_id:int, current_basket:dict):
    with DBConnection(dbconfig) as cursor:
        if cursor is None:
            raise ValueError('Курсор не создан')
        _sql1 = provider.get('insert_order.sql', user_id=user_id, order_date=datetime.now())
        result1 = cursor.execute(_sql1)
        if result1 == 1:
            _sql2 = provider.get('select_order_id.sql', user_id=user_id)
            cursor.execute(_sql2)
            order_id = cursor.fetchall()[0][0]
            print('order_id=', order_id)
            if order_id:
                for key in current_basket:
                    print(key, current_basket[key]['amount'])
                    prod_amount = current_basket[key]['amount']
                    _sql3 = provider.get('insert_order_list.sql', order_id=order_id, prod_id=key, prod_amount=prod_amount)
                    cursor.execute(_sql3)
                return order_id


from to_do_app import db
from datetime import datetime

# Post
class Reminder(db.Model):

    __tablename__ = 'reminders'

    id = db.Column(db.Integer, primary_key=True)
    title = db.Column(db.String(64), nullable=False)
    text = db.Column(db.Text, nullable=False)
    reminder_date = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)

    # Relationships
    tag_id = db.Column(db.Integer, db.ForeignKey('tags.id'), nullable=False)
    tag = db.relationship('Tag', backref=db.backref('reminders', lazy=True))

    def __init__(self, title, text, tag_id):
        self.title = title
        self.text = text
        self.tag_id = tag_id

    def __repr__(self):
        return f' ID: {self.id} --- Title: {self.title} --- Text.. --- Date:{self.reminder_date} --- TAG_ID: {self.tag_id} TAG:{self.tag} ---'

# Tag
class Tag(db.Model):

    __tablename__ = 'tags'

    id = db.Column(db.Integer, primary_key=True)
    tag = db.Column(db.String(32), nullable=False, unique=True)

    def __init__(self, tag):
        self.tag = tag

    def __repr__(self):
        return f'{self.tag}'

Programming Languages/Python/External Libraries/Flask/Flask_website_examples/00 To-Do WebApp/to_do_app/models.py


import tkinter as tk
import tkinter.ttk as ttk
from DAO import *
from Service import *

class CustomersApp(tk.Toplevel):

    def __init__(self, parent, service, table_to_update):
        super().__init__()
        self.service = service
        self.parent = parent
        self.table_to_update = table_to_update
        self.parent.title("Редактирование таблицы \"Покупатель\"")

        self.info_frame = tk.Frame(self)
        self.btn_frame = tk.Frame(self)
        self.table_frame = tk.Frame(self)

        tk.Label(self.info_frame, text="Покупатель", anchor=tk.W, width=23).grid(row=0, column=0)
        tk.Label(self.info_frame, text="Адрес покупателя", anchor=tk.W, width=23).grid(row=0, column=2)

        self.name = tk.Entry(self.info_frame, width=30)
        self.address = tk.Entry(self.info_frame, width=40)
        self.name.grid(row=0, column=1, pady=5, padx=(0, 5))
        self.address.grid(row=0, column=3, pady=5)

        self.table = ttk.Treeview(self.table_frame)

        self.table["columns"] = ("Код покупателя",
                                 "Покупатель",
                                 "Адрес покупателя"
                                 )

        self.table.column("#0", width=0)
        self.table.column("Код покупателя", width=80, minwidth=80, anchor=tk.CENTER)
        self.table.column("Покупатель", width=200, minwidth=200)
        self.table.column("Адрес покупателя", width=300, minwidth=300)

        self.table.heading("#0", text="")
        self.table.heading("Код покупателя", text="Код покупателя", anchor=tk.CENTER)
        self.table.heading("Покупатель", text="Покупатель", anchor=tk.CENTER)
        self.table.heading("Адрес покупателя", text="Адрес покупателя", anchor=tk.CENTER)

        self.table.bind("<<TreeviewSelect>>", self.on_select, "+")

        self.table.pack()

        self.edit = tk.Button(self.btn_frame, text="Изменить", width=10, command=self.change)
        self.add = tk.Button(self.btn_frame, text="Добавить", width=10, command=self.add)
        self.delete = tk.Button(self.btn_frame, text="Удалить", width=10, command=self.remove)
        self.exit = tk.Button(self.btn_frame, text="Выход", width=10, command=self.on_closing)
        self.edit.grid(row=0, column=0, padx=5, pady=5)
        self.add.grid(row=1, column=0, padx=5, pady=5)
        self.delete.grid(row=2, column=0, padx=5, pady=5)
        self.exit.grid(row=3, column=0, padx=5, pady=5)

        self.info_frame.grid(row=0, column=0, columnspan=4)
        self.table_frame.grid(row=1, column=0, columnspan=3)
        self.btn_frame.grid(row=1, column=3)

        self.update_table()

    def get_table(self):
        return self.table

    def update_table(self):
        for child in self.table.get_children():
            self.table.delete(child)
        customers = self.service.get_customers()
        for i, val in zip(customers.keys(), customers.values()):
            self.table.insert("", i, values=(val.get_iid(),
                                             val.get_name(),
                                             val.get_address()
                                             ))

    def on_closing(self):
        self.destroy()

    def add(self):
        self.service.add("customers",
                         name=self.name.get(),
                         address=self.address.get()
                         )
        self.update_table()

    def change(self):
        self.service.change("customers",
                            self.table.item(self.table.selection())["values"][0],
                            name=self.name.get(),
                            address=self.address.get()
                            )
        self.update_table()
        self.table_to_update.update_table()

    def remove(self):
        iid = self.table.item(self.table.selection())["values"][0]
        self.service.delete("customers", iid)
        self.update_table()
        self.table_to_update.update_table()

    def on_select(self, event):
        self.service.get_orders()
        self.current = self.table.selection()

        self.clear_fields()
        values = self.table.item(self.table.selection())["values"]
        self.name.insert(0, values[1])
        self.address.insert(0, values[2])

    def clear_fields(self):
        self.name.delete(0, tk.END)
        self.address.delete(0, tk.END)

    def change_fields(self):
        self.table.delete(self.current)
        self.current = None
        self.table.insert("", 0, text="0", values=("3", "Дивядюк", "Хитачи", "FGH", "Good", "100", "200.3",))

    # def get_values(self):

if __name__ == '__main__':
    root = tk.Tk()
    root.title("asdd")
    serv = Service()
    CustomersApp(root, serv, )
    root.mainloop()


# 전기버스2

def dfs(cnt, k, bat):
    global result
    bat -= 1
    if bat < 0:
        return
    if k >= N-1:
        result = min(result, cnt)
        return
    if cnt >= result:
        return
    dfs(cnt+1, k+1, M[k])
    dfs(cnt, k+1, bat)

T = int(input())
for tc in range(T):
    M = list(map(int, input().split()))
    N = M.pop(0)
    result = 1000
    dfs(0, 1, M[0])
    print('#{} {}' .format(tc+1, result))


import json
from typing import List

none = "d3043820717d74d9a17694c176d39733"

# region Prometheus
class Prometheus:
    """
    # Arguments
    address : str
    """

    def __init__(
            self,
            address: str = none):
        self.address = address
# endregion

# region Datadog
class Datadog:
    """
    # Arguments
    address : str
    api_key : str
    app_key : str
    """

    def __init__(
            self,
            address: str = none,
            api_key: str = none,
            app_key: str = none):
        self.address = address
        self.api_key = api_key
        self.app_key = app_key
# endregion

# region New Relic
class NewRelic:
    """
    # Arguments
    account_id : str
    base_url_nerd_graph : str
    base_url_rest : str
    personal_api_key : str
    region : str
    """

    def __init__(
            self,
            account_id: str = none,
            base_url_nerd_graph: str = none,
            base_url_rest: str = none,
            personal_api_key: str = none,
            region: str = none):
        self.account_id = account_id
        self.base_url_nerd_graph = base_url_nerd_graph
        self.base_url_rest = base_url_rest
        self.personal_api_key = personal_api_key
        self.region = region
# endregion

# region CloudWatch
class CloudWatch:
    """
    # Arguments
    iam_arn : str
    """

    def __init__(
            self,
            iam_arn: str = none):
        self.iam_arn = iam_arn
# endregion

# region Verification Provider
class VerificationProvider:
    """
    # Arguments
    name : str
    cluster_ids : List[str]
    prometheus : Prometheus
    datadog : Datadog
    newRelic : NewRelic
    cloudWatch : CloudWatch
    """

    def __init__(
            self,
            name: str = none,
            cluster_ids: List[str] = none,
            prometheus: Prometheus = none,
            datadog: Datadog = none,
            new_relic: NewRelic = none,
            cloud_watch: CloudWatch = none):
        self.name = name
        self.cluster_ids = cluster_ids
        self.prometheus = prometheus
        self.datadog = datadog
        self.new_relic = new_relic
        self.cloud_watch = cloud_watch
# endregion

# region Client Requests
class CreateVerificationProviderRequest:
    """
    # Arguments
    verification_provider : VerificationProvider
    """

    def __init__(self, verification_provider: VerificationProvider):
        self.verification_provider = verification_provider

    def toJSON(self):
        return json.dumps(self, default=lambda o: o.__dict__,
                          sort_keys=True, indent=4)
# endregion


'''
Created on May 28, 2016

@author: Marcus
'''
from urllib import request
import random

def download_web_image(url):
    number=random.randrange(1,1000)
    full_name=str(number)+".jpg"
    request.urlretrieve(url, full_name)

download_web_image("http://i.imgur.com/dbg7fdP.jpg")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import vilperg_amazonreview as amaz
import sys
import os

def get_all_products(start_url):
    """Get all products from an amazon overview site.

    Args:
        start_url: The url of the overview page.

    Returns:
        products (list): The Amazon_product objects for the products on the
            overview page.
    """
    overview = amaz.Amazon_overview(start_url)
    products = overview.get_products()
    return products

def write_products(products, out_dir):
    """Write products and their reviews to a directory structure under out_dir.

    For each product in the list, a directory is created under out_dir. In this
    directory, product_info file and a reviews directory are created. The
    product_info file contains information about the product. The reviews
    directory contains files describing one review each.

    Args:
        products (list): A list of Amazon_product objects.
        out_dir (str): The top level directory under which the products and
            their reviews are written.
    """
    ensure_dir(out_dir)
    for p in products:
        p_dir = '{0}/{1}'.format(out_dir, p.id)
        ensure_dir(p_dir)
        p_info = '{0}/product_info'.format(p_dir)
        with open(p_info, 'w') as p_f:
            p_f.write('{0}\n'.format(p))
        r_dir = '{0}/reviews'.format(p_dir)
        write_reviews(p, r_dir)

def write_reviews(product, top_dir):
    """Write the reviews of a product in a directory.

    Args:
        product (Amazon_product): An Amazon_product object.
        top_dir (str): The directory the reviews will be written in.
    """
    reviews = product.get_reviews()
    ensure_dir(top_dir)
    for r in reviews:
        r_dir = '{0}/{1}'.format(top_dir, r.id)
        ensure_dir(r_dir)
        with open('{0}/content'.format(r_dir), 'w') as f:
            f.write(r.text)

        with open('{0}/info'.format(r_dir), 'w') as f:
            info1 = '{0.id}\n{0.title}\n{0.stars}\n'
            info2 = '{1}\n{0.product_id}\n{0.date}\n'
            info = (info1 + info2).format(
                    r, amaz.formatHelpfulness(r.helpfulness))
            f.write(info)

def ensure_dir(d):
    """Create a directory if it does not already exist.

    Args:
        d (str): A directory.
    """
    if not os.path.exists(d):
        os.makedirs(d)

def write_all_overviews(start_url, out_dir):
    """Write all reviews of products of this and following overview pages.

    Get the amazon products and their reviews from an overview page and
    write all the products and their reviews to a directory structure, the top
    of which is out_dir. Then get the next overview page and do it again and
    so on.

    Args:
        start_url: The url of the overview page.
        out_dir: The top of the directory structure for the products and
            reviews.
    """
    overview = amaz.Amazon_overview(start_url)
    write_products(overview.get_products(), out_dir)

    while True:
        # Print all following overviews.
        try:
            overview = overview.get_next_overview()
        except amaz.No_next_overview:
            # All overviews have been processed.
            break
        write_products(overview.get_products(), out_dir)

def main():
    """Write all reviews of products of this and following overview pages.

    Get the amazon products and their reviews from an overview page and
    write all the products and their reviews to a directory structure, the top
    of which is out_dir. Then get the next overview page and do it again and
    so on.

    Command line arguments:
        start_url: The url of the overview page.
        out_dir: The top of the directory structure for the products and
            reviews.
    """
    start_url = sys.argv[1]
    out_dir = sys.argv[2]
    write_all_overviews(start_url, out_dir)

if __name__ == '__main__':
    main()


a, b = list(map(int, input().split()))

if a <= 0 and b >= 0:
    print('Zero')
elif a > 0 and b > 0:
    print('Positive')
else:
    r = abs(a - b)
    if r % 2 == 0:
        print('Negative')
    else:
        print('Positive')


# Based on
# https://github.com/roholazandie/boids/tree/master

import py5
import numpy as np

def setup():
    global flock
    py5.size(500, 500)
    flock = [
        Boid(py5.random(py5.width),
             py5.random(py5.height))
        for _ in range(20)
        ]
    py5.stroke_weight(2)
    py5.stroke(255)

def draw():
    py5.background(30, 30, 47)
    for boid in flock:
        boid.update()

    py5.window_title(f'{py5.get_frame_rate():.2f}')

class Boid():

    max_force = 0.3
    max_force_sq = max_force ** 2
    max_speed = 5
    max_speed_sq = max_speed ** 2
    perception = 200
    perception_sq = perception ** 2

    def __init__(self, x, y):
        self.position = py5.Py5Vector(x, y)
 #       vec = py5.Py5Vector.random(2) - 0.5 * 10
        vec = (np.random.rand(2) - 0.5) * 10
        self.velocity = py5.Py5Vector(*vec)
        vec = (np.random.rand(2) - 0.5) / 2
        self.acceleration = py5.Py5Vector(*vec)

    def update(self):
        self.edges()
        self.apply_behaviour(flock)
        self.position += self.velocity
        self.velocity += self.acceleration
        #limit
        if self.velocity.mag_sq > self.max_speed_sq:
            self.velocity.mag = self.max_speed

        py5.line(self.position.x,
                 self.position.y,
                 self.position.x + self.velocity.x * 5,
                 self.position.y + self.velocity.y * 5,
                 )

    def apply_behaviour(self, boids):
        alignment = self.align(boids)
        cohesion = self.cohesion(boids)
        separation = self.separation(boids)

        self.acceleration = alignment
        self.acceleration += cohesion
        self.acceleration += separation

    def edges(self):
        if self.position.x > py5.width:
            self.position.x = 0
        elif self.position.x < 0:
            self.position.x = py5.width

        if self.position.y > py5.height:
            self.position.y = 0
        elif self.position.y < 0:
            self.position.y = py5.height

    def align(self, boids):
        steering = py5.Py5Vector2D()
        total = 0
        avg_vector = py5.Py5Vector2D()
        for boid in boids:
            if (boid.position - self.position).mag_sq < self.perception_sq:
                avg_vector += boid.velocity
                total += 1
        if total > 0:
            avg_vector /= total
            avg_vector = avg_vector.norm * self.max_speed
            steering = avg_vector - self.velocity

        return steering

    def cohesion(self, boids):
        steering = py5.Py5Vector2D()
        total = 0
        center_of_mass = py5.Py5Vector2D()
        for boid in boids:
            if (boid.position - self.position).mag_sq < self.perception_sq:
                center_of_mass += boid.position
                total += 1
        if total > 0:
            center_of_mass /= total
            vec_to_com = center_of_mass - self.position
            if vec_to_com.mag_sq > 0:
                vec_to_com.mag = self.max_speed
            steering = vec_to_com - self.velocity
            if steering.mag_sq > self.max_force_sq:
                steering.mag = self.max_force

        return steering

    def separation(self, boids):
        steering = py5.Py5Vector2D()
        total = 0
        near_vector = py5.Py5Vector2D()
        for boid in boids:
            if self != boid:
                diff = boid.position - self.position
                if diff.mag_sq < self.perception_sq:
                    near_vector += diff.norm
                    total += 1
        if total > 0:
            avg_vector = near_vector / total
            if steering.mag_sq > 0:
                avg_vector.mag = self.max_speed
            steering = avg_vector - self.velocity
            if steering.mag_sq > self.max_force_sq:
                steering.mag = self.max_force

        return steering

py5.run_sketch()


import tkinter as tk
from tkinter.messagebox import showerror

from constants.frames import MAIN_FRAME_NAME
from util import add_new_quantity

class AddQuantityFrame(tk.Frame):

    def __init__(self, root, controller):
        tk.Frame.__init__(self, root)
        self.controller = controller

        self.main_label = tk.Label(self, text="Добавление новой величины", font="Helvetica 30 bold")
        self.main_label.pack(pady=50)

        self.info_label = tk.Label(self, text="Введите название величины", font="Helvetica 20")
        self.info_label.pack(pady=40)

        self.quantity_name_entry = tk.Entry(self, width=24)
        self.quantity_name_entry.pack()

        self.add_button = tk.Button(self, text="Добавить величину", width=20, height=3, command=self.__add_quantity)
        self.add_button.pack(pady=40)

        self.back_button = tk.Button(self, text="Назад", width=20, height=3,
                                     command=lambda: self.controller.show_frame(MAIN_FRAME_NAME))
        self.back_button.pack()

    def __add_quantity(self):
        quantity_name = self.quantity_name_entry.get()
        if quantity_name == "":
            showerror("Название величины", "Введите название величины")
            return

        if len(quantity_name) > 30:
            showerror("Длинное название", "Название величины может содержать не более 30 символов")
            return

        add_new_quantity(quantity_name)
        self.controller.show_frame(MAIN_FRAME_NAME)

    def render(self):
        self.clear()

    def clear(self):
        self.quantity_name_entry.delete(0, tk.END)


from tkinter import *
from tkinter import ttk, messagebox
from consume import OficialVenta

class Aplicacion():
    __ventana = None
    __pesos = None
    __dolares = None

    def __init__(self):
        self.__ventana = Tk()
        self.__ventana.geometry("290x115")
        self.__ventana.title("PESO A DOLAR")
        mainframe = ttk.Frame(self.__ventana, padding = "3 3 12 12")
        mainframe.grid(column = 0, row = 0, sticky = (N, W, E, S))
        mainframe.columnconfigure(0, weight = 1)
        mainframe.rowconfigure(0, weight = 1)
        mainframe["borderwidth"] = 2
        mainframe["relief"] = "sunken"
        self.__pesos = StringVar()
        self.__dolares = StringVar()
        self.__pesos.trace('w', self.calcular)
        self.pesosEntry = ttk.Entry(mainframe, width=7, textvariable=self.__pesos)
        self.pesosEntry.grid(column=2, row=1, sticky=(W, E))
        ttk.Label(mainframe, textvariable=self.__dolares).grid(column=2, row=2, sticky=(W, E))
        ttk.Button(mainframe, text='Salir', command=self.__ventana.destroy).grid(column=3, row=3,sticky=W)
        ttk.Label(mainframe, text="dolares").grid(column=3, row=1, sticky=W)
        ttk.Label(mainframe, text="es equivalente a").grid(column=1, row=2, sticky=E)
        ttk.Label(mainframe, text="pesos").grid(column=3, row=2, sticky=W)
        for child in mainframe.winfo_children():
            child.grid_configure(padx=5, pady=5)
        self.pesosEntry.focus()
        self.__ventana.mainloop()

    def calcular(self, *args):
        if self.pesosEntry.get() != "":
            try:
                pesos = float(self.pesosEntry.get())
                self.__dolares.set(pesos * OficialVenta)
            except ValueError:
                messagebox.showerror(title='Error de tipo',
                                     message='Debe ingresar un valor numérico')
            self.pesosEntry.focus()
        else:
            self.__dolares.set("")


#!/usr/bin/python
# -*- coding: utf-8 -*-
#
from __future__ import unicode_literals

import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import time
import traceback

from logs.log import logger
from eosapi.httpapi.client import Client
from eosapi.httpapi.exceptions import HttpAPIError

TMP_PATH = './tmp'

############################################
#            Start run testcase            #
############################################

def check_exist_acct(conf_dict):
    '''check the common_params.exist_account availability'''

    # NOTE: get account that you need open http plugin.

    assert conf_dict['common_params']['exist_account']
    client = Client([conf_dict['common_params']['nodeos_url']])
    try:
        client.get_account(conf_dict['common_params']['exist_account'])
    except HttpAPIError as err:
        return False
    return True

def setup_env():
    if os.path.exists(TMP_PATH):
        shutil.rmtree(TMP_PATH)
    os.mkdir(TMP_PATH)

def run_testcase(case_dict, common_params):
    cmdline = ''
    try:
        if 'pre_call' in case_dict and case_dict['pre_call']:
            cmdline = case_dict['pre_call']
            logger.info('Going to execute PRE call: {} {}'.format(case_dict['casename'], cmdline))
            pmsg = subprocess.check_output(cmdline, stderr=subprocess.STDOUT, shell=True)
            logger.info(pmsg)
        if 'cmdline' not in case_dict or not case_dict['cmdline']:
            logger.error('Error: cmdline can NOT be empty {}'.format(case_dict['casename']))
            return False
        params = {}
        params.update(common_params)
        params.update(case_dict['params'])
        params_file = os.path.join(TMP_PATH, str(re.sub(r"\s+", "", case_dict['casename'])).lower() + ".json")
        with open(params_file, "w") as fp:
            fp.write(json.dumps(params, indent=True, sort_keys=True, ensure_ascii=False))

        cmdline = case_dict['cmdline'] + " " + params_file
        logger.info('Going to execute cmdline: {} {}'.format(case_dict['casename'], cmdline))
        pmsg = subprocess.check_output(cmdline, stderr=subprocess.STDOUT, shell=True)
        logger.info(pmsg)

        if 'post_call' in case_dict and case_dict['post_call']:
            cmdline = case_dict['post_call']
            logger.info('Going to execute POST call: {} {}'.format(case_dict['casename'], cmdline))
            pmsg = subprocess.check_output(cmdline, stderr=subprocess.STDOUT, shell=True)
            logger.info(pmsg)
        return True
    except Exception as e:
        logger.error('get exception: {} {}'.format(case_dict['casename'], cmdline))
        logger.error(traceback.print_exc())
        return False

def run_monster(conf_dict):
    for case_dict in conf_dict['testcases']:
        result = run_testcase(case_dict, conf_dict['common_params'])
        if case_dict['stoponfail'] and not result:
            return False
    return True

####################################################################

def main():
    parser = argparse.ArgumentParser(description='EOSIO testcase collections running tool.')
    parser.add_argument('--config', default="./config.json", type=str, help='config.json config file path')
    args = parser.parse_args()
    conf_file = os.path.abspath(os.path.expanduser(args.config))
    # Check the parameters
    if not os.path.exists('testcases'):
        logger.error('call startmonster.py in the eostestmonster directory')
        sys.exit(1)

    conf_dict = None
    with open(conf_file, 'r') as fp:
        conf_dict = json.loads(fp.read())
    if not conf_dict:
        logger.error('validator config can not be empty: {}'.format(conf_file))
        sys.exit(1)
    if not check_exist_acct(conf_dict):
        logger.error('ERROR: the exist_account is NOT available')
        sys.exit(1)

    setup_env()

    # Start the testcase
    try:
        time_start = time.time()
        result = run_monster(conf_dict)
        if not result:
            logger.error('!!! Call testcases FAILED !!!')
        else:
            logger.info('Call testcases SUCCESS !!!')
        time_usage = time.time() - time_start
        logger.info('TIME USAGE:%ss' % (time_usage,))
        return result
    except Exception as e:
        logger.error(traceback.print_exc())

if __name__ == '__main__':
    sys.exit(0 if main() else 1)


from django.urls import path,include
from sources import views

urlpatterns = [
    # path('',include(router.urls)),
    path(r'sms', views.sms,name='sms'),
    path(r'email', views.email,name='email'),
    path(r'facebook', views.facebook,name='facebook'),
    path(r'twitter', views.twitter,name='twitter'),
    path(r'whatsapp', views.whatsapp,name='whatsapp'),
    path(r'telegram', views.telegram,name='telegram'),
]


#!/usr/bin/env python3
"""module 1-minor.py
Calculates the minor of a matrix
"""

def determinant(matrix):
    """Calculates the determinant of a matrix"""

    # Check if matrix is list of lists
    if not isinstance(matrix, list) or (len(matrix) == 0 and not all
                                        (isinstance(row, list)
                                         for row in matrix)):
        raise TypeError("matrix must be a list of lists")
    if len(matrix) == 0 or (len(matrix) == 1 and len(matrix[0]) == 0):
        return 1
    for i in range(len(matrix)):
        if type(matrix[i]) is not list:
            raise TypeError("matrix must be a list of lists")
        if len(matrix) != len(matrix[i]):
            raise ValueError("matrix must be a square matrix")

    matrix_size = len(matrix)

    # Base case for 0x0 matrix
    if matrix_size == 0:
        return 1

    # Base case for 1x1 matrix
    if matrix_size == 1:
        return matrix[0][0]

    # Base case for 2x2 matrix
    if matrix_size == 2:
        return matrix[0][0] * matrix[1][1] - matrix[0][1] * matrix[1][0]

    # Recursive calculation of the determinant for matrices larger than 2x2
    det = 0
    for index, value in enumerate(matrix[0]):
        # Create a submatrix by removing the first row and the current column
        submatrix = [row[:index] + row[index + 1:] for row in matrix[1:]]

        # Add or subtract current value times determinant of the submatrix
        det += value * determinant(submatrix) * (-1) ** index

    return det

def minor(matrix):
    """Calculates the minor matrix of a matrix"""
    # Check if matrix is a list of lists
    if type(matrix) is not list or not all(isinstance(row, list)
                                           for row in matrix):
        raise TypeError("matrix must be a list of lists")

    # Check if matrix is square
    size = len(matrix)
    if not all(len(row) == size for row in matrix) or size == 0:
        raise ValueError("matrix must be a non-empty square matrix")

    # Calculate the minor matrix
    minor_matrix = []
    for i in range(size):
        minor_row = []
        for j in range(size):
            sub_matrix = [row[:j] + row[j + 1:] for
                          row in (matrix[:i] + matrix[i + 1:])]
            minor_row.append(determinant(sub_matrix))
        minor_matrix.append(minor_row)

    return minor_matrix


import os
import sys
import numpy as np
import torch

sys.path.append("/home/lyf/Desktop/FWA/benchmarks/cec2013")
import cec13
sys.path.append("/home/lyf/Desktop/FWA/benchmarks/cec2017")
import cec17

def func_wrapper(func, func_id):

    def wrapped(x):

        origin_type = type(x)
        if origin_type is not list:
            origin_shape = x.shape
            dim = origin_shape[-1]

            if origin_type is torch.Tensor:
                x = x.cpu().numpy()
            x = x.reshape((-1, dim)).tolist()

        if func == "cec13":
            tmp = cec13.eval(x, func_id+1)
        elif func == "cec17":
            tmp = cec17.eval(x, func_id+1)
        else:
            raise Exception("No such benchmark!")

        if origin_type is np.ndarray:
            return np.array(tmp).reshape(origin_shape[:-1])
        elif origin_type is torch.Tensor:
            return torch.tensor(tmp).reshape(origin_shape[:-1])
        elif type(x) is list and type(x[0]) is list:
            return tmp
        else:
            return tmp[0]

    return wrapped

class Benchmark(object):

    def __init__(self):
        self.cec13 = [func_wrapper("cec13", func_id) for func_id in range(28)]
        self.cec17 = [func_wrapper("cec17", func_id) for func_id in range(30)]


class CommandNode():
    def __init__(self, name, command, level, required):
        self.name = name
        self.command = command
        self.childs = {}
        self.level = level
        self.category = False
        self.required = required
        self.parent = None

    def addChild(self, key, node):
        self.childs[key] = node

    def printNode(self):
        print ('\t' * self.level) + self.name
        for name, node in self.childs.iteritems():
            node.printNode()


# python Extractor_v2.py corpusBody user
# Coded by Chen Hu and Yinzhi Yu
import sys
import re
import json
from cStringIO import StringIO
# Class State is a finite state machine for the thread and head
class State:
	# Meta state
	state_set = set(['Body','Head'])
	# Sub state in head
	headType_set = set(['Original','Forward','To','From','Email'])

	def __init__(self):
		self.state = None
		self.headType = None
		self.headEnd = None

	# transition between states, given the current text line and context
	def transition(self,line,text,lineno):
		if self.state == None or self.state == "Body":
			return self.transFromBody(line,text,lineno)
		else:
			if self.headType == "Original":
				return self.transInOrg(line,text,lineno)
			elif self.headType == "Forward":
				return self.transInFwd(line,text,lineno)
			elif self.headType == "To":
				return self.transInTo(line,text,lineno)
			elif self.headType == "From":
				return self.transInFm(line,text,lineno)
			elif self.headType == "Email":
				return self.transInEml(line,text,lineno)

	# Check if the current line is the begining of a head
	def check_head(self,line,text,lineno):
		if "--Original Message--" in line:
			self.headEnd = self.findOrgEnd(line,text,lineno)
			return 'Original'
		if "-- Forwarded by" in line:
			self.headEnd = self.findFwdEnd(line,text,lineno)
			return "Forward"
		if "To: " in line[:5]:
			self.headEnd = self.findToEnd(line,text,lineno)
			return "To"
		if "From: " in line[:8]:
			tmp = self.findFromEnd(line,text,lineno)
			if tmp == -1:
				return None
			else:
				self.headEnd = tmp
				return "From"
		if re.search('<?[^@]+@[^@]+\.[^@]+>? on [\d | /]+ [\d | :]+ [(AM) | (PM)]',line) != None :
			self.headEnd = self.findEmlEnd(line,text,lineno)
			return "Email"
		return None

	# Transition in state Body
	def transFromBody(self,line,text,lineno):
		tag = self.check_head(line,text,lineno)
		if tag != None:
			self.state = "Head"
			self.headType = tag
			return True
		else:
			self.state = "Body"
			return False

	# Transition in state Head and sub_state in Orginal
	def transInOrg(self,line,text,lineno):
		if lineno == self.headEnd:
			self.headEnd = None
			self.state = "Body"
			return True

		return False

	# Transition in state Head and sub_state in Forward
	def transInFwd(self,line,text,lineno):
		if lineno == self.headEnd:
			self.headEnd = None
			self.state = "Body"
			return True

		return False

	# Transition in state Head and sub_state in To
	def transInTo(self,line,text,lineno):
		if lineno == self.headEnd:
			self.headEnd = None
			self.state = "Body"
			return True

		return False

	# Transition in state Head and sub_state in From
	def transInFm(self,line,text,lineno):
		if lineno == self.headEnd:
			self.headEnd = None
			self.state = "Body"
			return True

		return False

	# Transition in state Head and sub_state in Email
	def transInEml(self,line,text,lineno):
		if lineno == self.headEnd:
			self.headEnd = None
			self.state = "Body"
			return True

		return False

	# Given the context, find the end line of this Forward head
	def findFwdEnd(self,line,text,lineno):
		i = lineno + 1
		from_index = 0
		while i < len(text) and (not "-- Forwarded by" in text[i]):
			if "Subject:" in text[i]:
				return i
			if "From:" in text[i]:
				from_index = i
			i += 1

		if from_index!=0:
			return from_index
		else:
			return lineno + 1

	# Given the context, find the end line of this To head
	def findToEnd(self,line,text,lineno):
		i = lineno + 1
		re_index = 0
		while i < len(text):
			if "Subject:" in text[i]:
				return i
			if "X-Mailer" in text[i]:
				return i
			if (re_index == 0) and ("Re:" in text[i]):
				re_index = i
			i += 1
		if re_index != 0:
			return re_index
		return lineno + 1

	# Given the context, find the end line of this From head
	def findFromEnd(self,line,text,lineno):
		i = lineno + 1
		sent_index = 0
		to_index = 0
		while i < len(text):
			if "Subject:" in text[i]:
				return i
			if "Re:" in text[i]:
				return i
			if "Date:" in text[i]:
				return i
			if (sent_index == 0) and ("Sent:" in text[i]):
				sent_index = i
			if (to_index ==0 ) and ("To:" in text[i]):
				to_index = i
			i += 1
		if (sent_index !=0) and (to_index != 0):
			return max(sent_index,to_index)
		else:
			return -1

	# Given the context, find the end line of this Original head
	def findOrgEnd(self,line,text,lineno):
		i = lineno + 1
		from_index = 0
		while i < len(text):
			if "Subject:" in text[i]:
				return i
			if (from_index == 0) and "From:" in text[i]:
				from_index = i
			i += 1
		if from_index != 0:
			return from_index
		else:
			return lineno + 1

	# Given the context, find the end line of this Email head
	def findEmlEnd(self,line,text,lineno):
		i = lineno + 1
		to_index = 0
		while i < len(text):
			if "Subject:" in text[i]:
				return i
			if (to_index == 0) and ("To:" in text[i]):
				to_index = i
				while i < len(text):
					if text[i] == '\n':
						to_index = i - 1
						break
					i += 1
			i += 1
		if to_index != 0:
			return to_index
		else:
			return lineno + 1

class Extractor:
	# Find some missing header lines before current line
	@staticmethod
	def popLine(i,text):
		pre = i
		while pre>=1:
			if not re.match('>*$',text[pre-1]):
				pre = pre-1
			else:
				break
		return pre

	# Extarct the header from the emial in filesName
	@staticmethod
	def extract(email_doc):
		reader = StringIO(email_doc)
		text= []
		start = []
		end = []
		length = []
		for line in reader:
			text.append(line)
			length.append(len(line))
		doc_state = State()

		i = 0
		while i<len(text):
			if doc_state.transition(text[i],text,i):
				if doc_state.state == "Head":
					if(doc_state.headType == "To" or doc_state.headType == "From"):
						startIndex = Extractor.popLine(i,text)
					else:
						startIndex = i
					start.append(startIndex)
					if i == len(text) - 1:
						end.append(i)
				else:
					end.append(i)
			i += 1

		doc = []
		thread = []
		i=0
		threadCount = 0
		charCount = 0
		for threadCount in range(len(start)):
			if start[threadCount]!=0:
				tmp = ""
				for k in range(i,start[threadCount]):
					tmp+=text[k]
				tmpCount = sum(length[0:i])
				doc.append((tmpCount, tmp))
			tmp=""
			for k in range(start[threadCount],end[threadCount]+1):
				tmp += text[k]
			tmpCount = sum(length[0:start[threadCount]])
			thread.append((tmpCount, tmp))
			i=end[threadCount]+1

		tmp=""
		for k in range(i,len(text)):
			tmp+=text[k]
		tmpCount = sum(length[0:i])
		doc.append((tmpCount, tmp))

		return (doc,thread)


#sol 2 https://www.youtube.com/watch?v=r-F1sCTmZxA
# 1 word - pad spaces to the right
# n words - uniformly distribute spaces, round robin:reinitialization
#words, padded to the right
class Solution:
    def fullJustify(self, words: List[str], maxWidth: int) -> List[str]:
        if not words:
            return
        result = []
        cur = []
        num_letters=0
        for w in words:
            if num_letters + len(cur)+len(w) > maxWidth:
                if len(cur)==1:
                    result.append(cur[0]+' '*(maxWidth-num_letters))
                else:
                    num_spaces = maxWidth - num_letters
                    space_between_words, extra = divmod(num_spaces, len(cur)-1)
                    for i in range(extra): #add EXTRA spaces after each word e.g. 10%3=1 extra
                        cur[i]+=' '
                    result.append((' '*space_between_words).join(cur))  #add spaces btw words e.g. 10/3=3 spaces btw words
                cur = []
                num_letters = 0
            cur.append(w)
            num_letters +=len(w)
            print(w,cur,num_letters, result)
        result.append(' '.join(cur)+ ' '*(maxWidth - num_letters-len(cur)+1)) #left justify the last line. len(cur)+1==spaces btw words
        return result
####################################################################################################################
#sol 3 https://www.youtube.com/watch?v=RORuwHiblPc
#dynamic programming approach

####################################################################################################################

#simple sol https://leetcode.com/problems/text-justification/discuss/528789/Python-20ms-short-solution-explained
#sliding window
def fullJustify(self, words: List[str], maxWidth: int) -> List[str]:
    def justify(s, e, l):
        d = e-s
        if e == len(words)-1 or not d:
            lines.append(' '.join(words[s:e+1]) + ' ' * (maxWidth-l-d))
        else:
            line, (q, r) = '', divmod(maxWidth-l, d)
            for i in range(d):
                line += words[s+i] + ' ' * (q+1 if i < r else q)
            lines.append(line + words[e])

    lines, s, l = [], 0, 0
    for e in range(len(words)):
        l += len(words[e])
        if e == len(words)-1:
            justify(s, e, l)
        elif l + len(words[e+1]) + (e-s+1) > maxWidth:
            justify(s, e, l)
            s, l = e+1, 0


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Oct 12 15:50:56 2018

@author: Miguel Monteiro

Se x² + y² <= 1 então caiu dentro do circulo
"""

import sys
import random

randomPoints = int(sys.argv[1]) # n assume valor dado pelo argumento na linha de comandos
insidePoints = 0  # Pontos que cairam dentro do circulo

for i in range(randomPoints):
    #random.randon() da um valor "aleatorio" entre 0 e 1
    x2 = random.random()**2 # x²
    y2 = random.random()**2 # y²
    if x2 + y2 <= 1:
        insidePoints += 1

pi = 4 * insidePoints / randomPoints

print("\nIteracoes: ",randomPoints,"\nPontos interiores: ",insidePoints)
print("\nPi = ",pi)


"""
Game Lobby class to keep information about each player in a game,
and the overall game progress

Written by Trevor Zapiecki and Gabriel Brown
"""

from .player import Player
from . import events
from .task_generator import TaskGenerator

class GameLobby:

    INITIAL_NUM_TASKS_TO_COMPLETE = 5
    INITIAL_TASK_COMPLETION_TIME = 15000    # In milliseconds

    def __init__(self, lobby_id, numPlayers):
        """Constructor for Lobby"""

        self.lobby_id = lobby_id
        self.numPlayers = numPlayers
        self.connectedPlayers = 0
        self.players = {}                   # KEY: user_id/cookies, VALUE: Player object

        # KEY: user/id cookie, VALUE: first task that user has to complete
        self.initial_task_assignments = {}

        # KEY: userid/cookie, VALUE: a list of tasks that this particular
        # user can complete (should appear with buttons on their screen)
        self.user_tasks = {}

        self.ship_health = 100              # The overall ship health

        self.task_fail_damage = 10          # The amount of damage failing a task deals to the ship
        self.bad_input_damage = 5           # The amount of damage pressing a the wrong button deals to the ship

        # The number of tasks players must complete before moving on to a new sector
        self.num_tasks_to_complete = self.INITIAL_NUM_TASKS_TO_COMPLETE

        # The number of tasks players have actually completed
        self.num_tasks_completed = 0

        self.has_won = False
        self.section_complete = False

        self.section_number = 0
        self.task_completion_time = self.INITIAL_TASK_COMPLETION_TIME  # The time alotted to complete each assigned task in milliseconds.

    def add_player(self, player):
        self.players[player.user_id] = player

    def start_game(self):
        """This should be called whenever all the players connect to a game for the first time"""

        self.task_generator = TaskGenerator(len(self.players))
        self.adjust_challenge()
        self.assign_tasks()

    def assign_tasks(self):
        """
        Assign tasks to each player, both the task that they must complete
        as well as the list of tasks that they are able to complete
        (which buttons should appear on their screen)
        """

        player_cookies = list(self.players.keys())
        initial_task_ids = list(self.task_generator.current_tasks.keys())
        usable_task_ids = list(self.task_generator.usable_tasks.keys())

        # Assign tasks to be completed. Tasks are assigned in no particular order,
        # just however the dictionaries organize their keys.
        for i in range(len(player_cookies)):

            player_cookie = player_cookies[i]
            task_id = initial_task_ids[i]

            # Assign task to do
            self.initial_task_assignments[player_cookie] = self.task_generator.current_tasks[task_id]

            # Assign a list of tasks that each user can complete on their own

            index_of_first_task = i * TaskGenerator.NUM_TASKS_PER_PLAYER

            # initialize with empty list, so we can add tasks to it in loop
            self.user_tasks[player_cookie] = []
            for j in range(TaskGenerator.NUM_TASKS_PER_PLAYER):

                # figure out which task we can add from usable_tasks
                interactable_task_id = usable_task_ids[ (i * TaskGenerator.NUM_TASKS_PER_PLAYER) + j ]

                # add that task
                self.user_tasks[player_cookie].append(self.task_generator.usable_tasks[interactable_task_id])

    def task_failed(self):
        """Update the ship health and loss status when a task fails"""

        self.ship_health = self.ship_health - self.task_fail_damage
        self.has_lost = self.ship_health <= 0

    def bad_input(self):
        """
        Update the ship health and loss status when a user
        presses a button that doesn't complete an active task
        """

        self.ship_health = self.ship_health - self.bad_input_damage
        self.has_lost = self.ship_health <= 0

    def task_completed(self):
        """
        Update the num_tasks_completed and win status
        when a user successfully completes a task
        """

        self.num_tasks_completed += 1
        self.section_complete = self.num_tasks_completed >= self.num_tasks_to_complete

        if self.section_complete:
            self.section_number += 1
            self.num_tasks_completed = 0

    def adjust_challenge(self):
        """
        Update the task completion time and and the number of
        tasks to complete based on the section number
        and number of players
        """

        # Base time should be 10 seconds plus 2 seconds per player, then subtract
        # 1 second each time we go up a section
        self.task_completion_time = self.INITIAL_TASK_COMPLETION_TIME + (self.numPlayers - 2)*2000 - self.section_number*1000

        # Add an additional task per player when the section increases
        self.num_tasks_to_complete = self.INITIAL_NUM_TASKS_TO_COMPLETE + (self.section_number * self.numPlayers)

    def reset(self):
        """
        Reset win conditions and ship health, ask the task generator
        for a new set of usable tasks, and assign them to the players
        """

        self.section_number = 0
        self.num_tasks_completed = 0
        self.ship_health = 100
        self.has_won = False
        self.has_lost = False
        self.task_generator.new_section()
        self.adjust_challenge()
        self.assign_tasks()

    def new_section(self):
        """
        Essentially a lighter version of reset(). The win conditions
        and section number should not be reset with every new section.
        """

        self.num_tasks_completed = 0
        self.ship_health = 100
        self.task_generator.new_section()
        self.adjust_challenge()
        self.assign_tasks()


import argparse
import os
import torch
import tqdm
import cv2
import numpy as np
import joblib
import torch.nn.functional as F
import shutil
from tools.BBAM.BBAM_utils import numpy_to_torch, save_mask, tv_norm, idx_to_class, get_max_iou, get_single_iou, selected_positives, save_pred
from maskrcnn_benchmark.config import cfg
from maskrcnn_benchmark.data import make_data_loader
from maskrcnn_benchmark.engine.inference import inference
from maskrcnn_benchmark.modeling.detector import build_detection_model
from maskrcnn_benchmark.utils.checkpoint import DetectronCheckpointer
from maskrcnn_benchmark.utils.collect_env import collect_env_info
from maskrcnn_benchmark.utils.comm import synchronize, get_rank
from maskrcnn_benchmark.utils.logger import setup_logger
from maskrcnn_benchmark.utils.miscellaneous import mkdir
from tools.BBAM.data_utils import BBAM_data_loader
import random
import copy
from maskrcnn_benchmark.structures.bounding_box import BoxList
import time
from tools.BBAM.BBAM_FRCNN import process
# os.environ["CUDA_VISIBLE_DEVICES"] = "3"
# os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
idx_to_class = {0 : 'aeroplane', 1 : 'bicycle', 2 : 'bird', 3 : 'boat', 4 : 'bottle', 5 : 'bus', 6 : 'car', 7 : 'cat',
                8 : 'chair', 9 : 'cow', 10 : 'table', 11 : 'dog', 12 : 'horse', 13 : 'motorbike', 14 : 'person',
                15 : 'plant', 16 : 'sheep', 17 : 'sofa', 18 : 'train', 19 : 'monitor'}

# time.sleep(60*60*2)

parser = argparse.ArgumentParser(description="PyTorch Object Detection Training")
parser.add_argument(
    "--config-file",
    default="configs/pascal_voc/BBAM_VOC_aug_FPN_R101_FasterRCNN_test.yaml",
    help="path to config file",
    type=str,
)
parser.add_argument(
    "--ckpt",
    default="logs/ckpoints/Faster_RCNN_final_voc.pth",
    type=str,
)
parser.add_argument(
    "--lr_mask",
    help="Modify config options using the command-line",
    default=0.02,
)
parser.add_argument(
    "--max_iter_mask",
    help="Modify config options using the command-line",
    default=300,
)
parser.add_argument(
    "--tv_beta",
    help="Modify config options using the command-line",
    default=3,
)
parser.add_argument(
    "--tv_coeff",
    help="Modify config options using the command-line",
    default=1e-3,
)
parser.add_argument(
    "--noise_method",
    help="constant | blur | blurrandom",
    default="constant",
)
parser.add_argument(
    "opts",
    help="Modify config options using the command-line",
    default=None,
    nargs=argparse.REMAINDER,
)
args = parser.parse_args()

cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()

output_dir = cfg.OUTPUT_DIR

mode = 'val'  # val | train_aug_cocostyle
data_dir = 'Dataset/VOC2012_SEG_AUG/'

image_root = os.path.join(data_dir, 'JPEGImages')

save_path_root = 'Faster_VOC_val'
if not os.path.exists(save_path_root):
    os.makedirs(save_path_root)
#
l1_coeff = 0.07

image_id = '2007_000727'
queries = [
        [3, 1200, 3]]
print(queries)
results = joblib.Parallel(n_jobs=-1)(
        [joblib.delayed(process)(i) for i in queries]
    )


"""
Code for initial processing of ctd and bottle data from nanoos cruises.

Hack notes:

2021
- has three cruises (April: RC0051, July: RC0058, September: RC0063)
- none have DIC or TA, but samples were taken. They just were slow to be
  processed by PMEL. I may get draft values from Anna Boyar.
- the original September bottle file had a column alignment type to it
  appeared to have near-zero nitrate. Anna fixed this and the copy I have
  in LO_data is corrected. Not sure about the website.
- decimal longitudes are positive (fixed with hack)
- April has bad rows at the end of the xlsx with only lat, lon (fixed)
- April bottle file has a typo for the latitude of station 5 (repeats that of 28)
  (fixed with a hack)
"""

from datetime import datetime, timedelta
import numpy as np
import gsw
import sys
import pandas as pd
from time import time

from lo_tools import Lfun
Ldir = Lfun.Lstart()

testing = False

# CTD
# Not written yet

# BOTTLE
source = 'nanoos'
otype = 'bottle'

in_dir0 = Ldir['data'] / 'obs' / source

# process bottles
out_dir = Ldir['LOo'] / 'obs' / source / otype
if testing:
    Lfun.make_dir(out_dir, clean=True)
else:
    Lfun.make_dir(out_dir)

year_list = [2017]

for year in year_list:
    ys = str(year)

    # name output files
    out_fn = out_dir / (str(year) + '.p')
    info_out_fn = out_dir / ('info_' + str(year) + '.p')

    in_dirs = list(in_dir0.glob('*'+ys+'*'))

    fn_list = []
    for in_dir in in_dirs:
        fn_list.append(list(in_dir.glob('*labupcast*'))[0])

    if testing:
        fn_list = [fn_list[1]]

    cid0 = 0
    DF = pd.DataFrame()
    for fn in fn_list:
        print(fn.name)
        # selected columns to ingest
        if year in [2021]:
            """
            Available columns:
            ['record no', 'CRUISE_ID', 'DATE_UTC', 'TIME_UTC', 'DATE_LOCAL',
            'TIME_LOCAL', 'LATITUDE_DEG', 'LONGITUDE_DEG', 'LATITUDE_DEC',
            'LONGITUDE_DEC', 'STATION_NO', 'NISKIN_NO', 'NISKIN_NO_FLAG_W',
            'CTDPRS_DBAR', 'DEPTH (M)', 'CTDTMP_DEG_C_ITS90', 'CTDTMP2_DEC_C_ITS90',
            'CTDTMP_FLAG_W', 'CTD/TEMP_COMMENTS', 'CTDSAL_PSS78', 'CTDSAL2_PSS78',
            'CTDSAL_FLAG', 'CTD/SAL_COMMENTS', 'SIGMATHETA_KG_M3',
            'SIGMATHETA2_KG_M3', 'CTDOXY_MG_L_1', 'CTDOXY_MG_L_2',
            'CTDOXY_MG_L_AVG', 'CTDOXY_FLAG', 'CTD/O2_COMMENTS', 'OXYGEN_MG_L_1',
            'OXYGEN_MG_L_2', 'OXYGEN_MG_L_3', 'OXYGEN_avg_mg_L', 'OXYGEN_UMOL_KG',
            'OXYGEN_FLAG_W', 'OXYGEN COMMENTS', 'CTD_PH', 'SALINITY_PSS78',
            'SALINITY_FLAG', 'Nutrient lab temperature', 'NITRATE_UMOL_L',
            'NITRITE_UMOL_L', 'AMMONIUM_UMOL_L', 'PHOSPHATE_UMOL_L',
            'SILICATE_UMOL_L', 'NUTRIENTS_FLAG_W', 'CTD FLU (mg/m3)', 'CHLA (ug/l)',
            'CHLA 2 (ug/l)', 'CHLA avg (ug/l)', 'CHLA_FLAG', 'PHAEOPIGMENT (ug/l)',
            'PHAEOPIGMENT 2 (ug/l)', 'PHAEOPIGMENT avg (ug/l)', 'PHAEOPIGMENT_FLAG',
            'TA_UMOL_KG', 'TA_FLAG_W', 'DIC_UMOL_KG', 'DIC_FLAG_W',
            'SECCHI DEPTH (m)']
            """
            cols = ['CRUISE_ID', 'DATE_UTC', 'TIME_UTC',
            'LATITUDE_DEC', 'LONGITUDE_DEC', 'STATION_NO',
            'CTDPRS_DBAR', 'CTDTMP_DEG_C_ITS90', 'CTDSAL_PSS78',
            'OXYGEN_avg_mg_L',
            'NITRATE_UMOL_L', 'NITRITE_UMOL_L', 'AMMONIUM_UMOL_L', 'PHOSPHATE_UMOL_L',
            'SILICATE_UMOL_L', 'CTD FLU (mg/m3)',
            'CHLA avg (ug/l)','PHAEOPIGMENT avg (ug/l)',
            'TA_UMOL_KG', 'DIC_UMOL_KG', 'SECCHI DEPTH (m)']
            # load the data to a DataFrame
            df = pd.read_excel(fn, usecols=cols, dtype={'DATE_UTC':str, 'TIME_UTC':str})
            # NOTE: this would have been a lot easier using
            # parse_dates={'time':['DATE_UTC', 'TIME_UTC']} in the pd.read_excel() call,
            # but there are some bad rows with missing data, so we slog through doing it
            # from scratch.
            dstr = df.DATE_UTC
            tstr = df.TIME_UTC
            tlist = []
            for ii in range(len(dstr)):
                try:
                    tlist.append(pd.to_datetime(dstr[ii][:10] + ' ' + tstr[ii][:8]))
                except:
                    tlist.append(np.nan)
            df['time'] = tlist
            df = df.dropna(axis=0, how='all') # drop rows with no good data
            df = df[df.time.notna()] # drop rows with bad time

        elif year in [2017]:
            # this only works for the first two cruises!
            cols = ['Station number',
               'Date_UTC', 'Time_UTC',
               ' CruiseID',
               ' Potemp090C', ' Sal00', ' PrDM',
               ' Latitude', ' Longitude',
               'CTDOXY_UMOL_KG_ADJ',
               'NITRATE_UMOL_L', 'NITRITE_UMOL_L',
               'AMMONIUM_UMOL_L', 'PHOSPHATE_UMOL_L', 'SILICATE_UMOL_L',
               'CHLA avg (ug/l)',
               'PHAEOPIGMENT avg (ug/l)']
            df = pd.read_excel(fn, usecols=cols, parse_dates={'time':['Date_UTC', 'Time_UTC']})
            df = df.dropna(axis=0, how='all') # drop rows with no good data

        # if we got any good rows, proceed with processing
        if len(df) > 0:
            # rename variables for convenience (some will be final names)
            if year in [2021]:
                df = df.rename({'STATION_NO':'name', 'CRUISE_ID':'cruise',
                    'LONGITUDE_DEC':'lon', 'LATITUDE_DEC':'lat',
                    'CTDPRS_DBAR':'P (dbar)', 'CTDTMP_DEG_C_ITS90':'PT', 'CTDSAL_PSS78':'SP',
                    'OXYGEN_avg_mg_L':'DO (mg/L)',
                    'NITRATE_UMOL_L':'NO3 (uM)', 'NITRITE_UMOL_L':'NO2 (uM)',
                    'AMMONIUM_UMOL_L':'NH4 (uM)',
                    'PHOSPHATE_UMOL_L':'PO4 (uM)', 'SILICATE_UMOL_L':'SiO4 (uM)',
                    'CTD FLU (mg/m3)':'Fluor (ug/L)','CHLA avg (ug/l)':'ChlA (ug/L)',
                    'PHAEOPIGMENT avg (ug/l)':'Phaeo (ug/L)',
                    'TA_UMOL_KG':'TA (umol/kg)', 'DIC_UMOL_KG':'DIC (umol/kg)',
                    'SECCHI DEPTH (m)':'Secchi (m)',
                }, axis=1)
            elif year in [2017]:
                df = df.rename({'Station number':'name',
                   ' CruiseID':'cruise',
                   ' Potemp090C':'PT', ' Sal00':'SP', ' PrDM':'P (dbar)',
                   ' Latitude':'lat', ' Longitude':'lon',
                   'CTDOXY_UMOL_KG_ADJ':'DO (umol/kg)',
                   'NITRATE_UMOL_L':'NO3 (uM)', 'NITRITE_UMOL_L':'NO2 (uM)',
                   'AMMONIUM_UMOL_L':'NH4 (uM)', 'PHOSPHATE_UMOL_L':'PO4 (uM)', 'SILICATE_UMOL_L':'SiO4 (uM)',
                   'CHLA avg (ug/l)':'ChlA (ug/L)',
                   'PHAEOPIGMENT avg (ug/l)':'Phaeo (ug/L)'
                }, axis=1)

            # add the "cid" (cast ID) column
            #
            # Note that we will save the field "name" for station number, since this dataset has
            # repeat stations which is helpful for plotting sections. Then we will generate our own
            # cid, a unique one for each cast, being careful to keep them unique for the collection
            # of cruises in this year, even though a station may be repeated on all cruises.
            #
            # We will also save the field "cruise" as a convenient way to select a collection of
            # casts from a given month.
            df['cid'] = np.nan
            cid = cid0
            for name in df.name.unique():
                df.loc[df.name==name,'cid'] = cid
                cid += 1
            for cid in df.cid.unique():
                # Check that there are not two different casts associated with the same station
                # by looking for large time differences. Pretty ad hoc.
                time_diff = df[df.cid==cid].time.values[-1] - df[df.cid==cid].time.values[0]
                time_diff = pd.to_timedelta(time_diff)
                if time_diff.days > 1 or time_diff.days < -1:
                    print('Station %d has time diff of %d days' % (cid, time_diff.days))
                # RESULT: the time_diffs are all zero, so it appears that in this database
                # the Station field is a unique cast identifier.

                # Force certain fields to be the same throughout the cast.
                df.loc[df.cid==cid,'lon'] = - df[df.cid==cid].lon.values[0] # also fix sign
                df.loc[df.cid==cid,'lat'] = df[df.cid==cid].lat.values[0]
                df.loc[df.cid==cid,'time'] = df[df.cid==cid].time.values[0]

            # Next make derived quantities and do unit conversions

            # (1) Create CT, SA, and z
            # - pull out variables
            SP = df.SP.to_numpy()
            pt = df.PT.to_numpy()
            p = df['P (dbar)'].to_numpy()
            lon = -(np.abs(df.lon.to_numpy())) # force sign to be negative
            df['lon'] = lon
            lat = df.lat.to_numpy()
            # - do the conversions
            SA = gsw.SA_from_SP(SP, p, lon, lat)
            CT = gsw.CT_from_pt(SA, pt)
            z = gsw.z_from_p(p, lat)
            # - add the results to the DataFrame
            df['SA'] = SA
            df['CT'] = CT
            df['z'] = z

            # (2) units
            if 'DO (gg/L)' in df.columns:
                df['DO (uM)'] = (1000/32) * df['DO (mg/L)']
            elif 'DO (umol/kg)' in df.columns:
                rho = gsw.rho(SA,CT,p)
                df['DO (uM)'] = (rho/1000) * df['DO (umol/kg)']

            # (3) retain only selected variables
            cols = ['cid', 'cruise', 'time', 'lat', 'lon', 'name', 'z',
                'CT', 'SA', 'DO (uM)',
                'NO3 (uM)', 'NO2 (uM)', 'NH4 (uM)', 'PO4 (uM)', 'SiO4 (uM)',
                'Fluor (ug/L)', 'ChlA (ug/L)', 'Phaeo (ug/L)', 'TA (umol/kg)',
                'DIC (umol/kg)', 'Secchi (m)']
            this_cols = [item for item in cols if item in df.columns]
            df = df[this_cols]
            DF = pd.concat([DF,df])

            print(' - processed %d casts' % ( len(df.cid.unique()) ))
            cid0 = df.cid.max() + 1

    # Sort the result by time, and sort each cast to be bottom to top
    DF = DF.sort_values(['time','z'], ignore_index=True)

    # Rework cid to also be increasing in time
    a = DF[['time','cid']].copy()
    a['cid_alt'] = np.nan
    ii = 0
    for t in a.time.unique():
        a.loc[a.time==t,'cid_alt'] = ii
        ii += 1
    DF['cid'] = a['cid_alt'].copy()

    # clean up spaces in cruise names
    a = DF['cruise'].to_list()
    aa = [item.strip() for item in a]
    DF['cruise'] = aa

    # Hack: fix a location typo
    DF.loc[(DF.cruise=='RC0051') & (DF.name==5), 'lat'] = 47.883

    # Save the data
    DF.to_pickle(out_fn)

    # Also pull out a dateframe with station info to use for model cast extractions.
    ind = DF.cid.unique()
    col_list = ['lon','lat','time','name','cruise']
    info_df = pd.DataFrame(index=ind, columns=col_list)
    for cid in DF.cid.unique():
        info_df.loc[cid,col_list] = DF.loc[DF.cid==cid,col_list].iloc[0,:]
    info_df.index.name = 'cid'
    info_df['time'] = pd.to_datetime(info_df['time'])
    info_df.to_pickle(info_out_fn)


#!/usr/bin/python3.10

import common

HELP_INFO = [
    "Script is solving task 2 for day 3 of advent of code 2021",
    "Arguments:",
    common.TAB + "input file"
]
arguments_keywords = ["inputFile"]

script_arguments = common.parse_arguments(arguments_keywords, HELP_INFO)
if script_arguments is None:
    exit(1)

inputFileName = script_arguments["inputFile"]
print("solving file: " + inputFileName)
inputLines = common.read_lines_from_file(inputFileName)

def get_one_item(lines, bit_for_gt, bit_for_other):
    i = 0
    lines_left = lines
    while not common.is_empty(lines_left):
        zeros = 0
        ones = 0
        lines = lines_left
        lines_left = []
        for line in lines:
            bit = line[i]
            if bit == "0":
                zeros += 1
            else:
                ones += 1

        if zeros > ones:
            chosen_bit = bit_for_gt
        else:
            chosen_bit = bit_for_other

        for line in lines:
            bit = line[i]
            if bit == chosen_bit:
                lines_left.append(line)
        if len(lines_left) == 1:
            return lines_left[0]
        i += 1

print("solving file: " + inputFileName)
inputLines = common.read_lines_from_file(inputFileName)

oxygenLeft = get_one_item(inputLines, "0", "1")
co2Left = get_one_item(inputLines, "1", "0")

oxygenStr = str(oxygenLeft)
co2Str = str(co2Left)

oxygen = int(oxygenStr, 2)
co2 = int(co2Str, 2)

print("oxygen: %s -> %d" % (oxygenStr, oxygen))
print("co2 %s -> %d" % (co2Str, co2))

print("%d * %d = %d" % (oxygen, co2, oxygen * co2))


numbers = [1, 10, 5 , 16, 4 ,9 ,10]
letters = ['a', 'g', 's','b','y','a','s']

value = min(numbers)
value = max(numbers)
value = max(letters)
value = min(letters)

value = numbers[3:6]
value = numbers[:3]
value = numbers[4:]

numbers[4] = 40

numbers.append(49)      #adds item we want
numbers.append(59)

numbers.insert(3,78)    #adds item in index we want
numbers.insert(-1 ,20)

numbers.pop()           #delete the last one
numbers.pop(0)
numbers.pop(-1)

numbers.remove(49)      #delete we want
numbers.remove(40)

numbers.sort()
numbers.reverse()

letters.sort()
letters.reverse()

print(letters)
print(numbers)

print(len(numbers))
print(len(letters))

numbers.clear()

print(numbers.count(10))
print(letters.count('a'))

message = 'Hello There . My name is Gülcihan'.split()
print(message)
print(message[0])

my_list = [1,2,3]
my_list= ['one' , 2, True]
print(my_list)

list1= ['one', 'two', 'there']
list2 = ['four', 'five', 'six']

number = list1+ list2
print(number)
print(len(number))

usersA = ['Derya', 20]
usersB = ['Pelin', 19]

users = [usersA, usersB]  #

print(usersA)
print(usersB)
print(users)

print(users[0][0])

list1 = ['Bmw', ' Mercedes', 'Opel', 'Mazda']

result = (len(list1))

result = list1[0]
result = list1[3]
result = list1[-1]

list1[-1] = 'Toyota'
result = list1

result  = 'Mercedes' in list1

result = list1[-2]

result = list1[0:3]
result = list1[:3]
result = list1[-2:]

result[-2] = ['Toyota', 'Renault']
result = list1

result = list1 + ['Audi', 'Nissan']

del list1[-1]
result = list1

result = list1[::-1]

studentA = ['Yiğit', 'Bilgi', 2010,[70,60,70]]
studentB = ['Ahmet', 'Sevim',1999,[80,80,70]]
studentC = ['Ahmet', 'Turan',1998,[80,70,90]]

result = studentA[0]
result = studentB[1]
result = studentC[3][1]

result = f"{studentA[0]} {studentA[1]} {2019-studentA[2]} age and grade point average {(studentA[3][0]) +(studentA[3][1]) + (studentA[3][2])} "

print(result)


import os

from topaz.objects.stringobject import W_StringObject
from topaz.utils.ll_file import O_BINARY

def map_filemode(space, w_mode):
    encoding = ""
    if w_mode is space.w_nil:
        mode = os.O_RDONLY
    elif isinstance(w_mode, W_StringObject):
        mode_str = space.str_w(w_mode)
        mode = 0
        invalid_error = space.error(space.w_ArgumentError,
            "invalid access mode %s" % mode_str
        )
        major_mode_seen = False
        readable = writeable = append = False

        pos = 0
        for ch in mode_str:
            pos += 1
            if ch == "b":
                mode |= O_BINARY
            elif ch == "+":
                readable = writeable = True
            elif ch == "r":
                if major_mode_seen:
                    raise invalid_error
                major_mode_seen = True
                readable = True
            elif ch == "a":
                if major_mode_seen:
                    raise invalid_error
                major_mode_seen = True
                mode |= os.O_CREAT
                append = writeable = True
            elif ch == "w":
                if major_mode_seen:
                    raise invalid_error
                major_mode_seen = True
                mode |= os.O_TRUNC | os.O_CREAT
                writeable = True
            elif ch == ":":
                encoding = mode_str[pos + 1:]
                break
            else:
                raise invalid_error
        if readable and writeable:
            mode |= os.O_RDWR
        elif readable:
            mode |= os.O_RDONLY
        elif writeable:
            mode |= os.O_WRONLY
        if append:
            mode |= os.O_APPEND
    else:
        mode = space.int_w(w_mode)
    return (mode, encoding)


import socket
import json

s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
bank_host = socket.gethostname()
bank_port = 1024

market_host = socket.gethostname()
market_port = 5055

inp = input("Do you want to create Bank Account")

if (inp == 'y'):
    s.sendto(inp.encode(), (bank_host, bank_port))
    name = input("What is your name?")
    s.sendto(name.encode(), (bank_host, bank_port))

    while True:
        print("Select from options")
        print("1. Deposit Money")
        print("2. Withdraw Money")
        print("3. Check Balance")
        print("4. Buy Stock")
        print("5. Sell Stock")

        option = input("Choose Option: ")
        option = str(option)

        if option == '1':
            s.sendto(option.encode(), (bank_host, bank_port))
            amt = input("How much money do you want to deposit?")
            s.sendto(name.encode(), (bank_host, bank_port))
            s.sendto(amt.encode(), (bank_host, bank_port))

        elif option == '2':
            s.sendto(option.encode(), (bank_host, bank_port))
            amt = input("How much money do you want to withdraw?")
            s.sendto(name.encode(), (bank_host, bank_port))
            s.sendto(amt.encode(), (bank_host, bank_port))

        elif option == '3':
            s.sendto(option.encode(), (bank_host, bank_port))
            s.sendto(name.encode(), (bank_host, bank_port))
            amt, addr = s.recvfrom(1024)
            amt = amt.decode('utf')
            print("You have " + amt + " in your account")

        elif option == '4':
            s2.sendto(option.encode(), (market_host, market_port))
            stock = input("What stock do you want to buy?")
            quantity = input("What is the quantity?")
            s2.sendto(stock.encode(), (market_host, market_port))
            price, addr = s2.recvfrom(1024)
            avaliable_quantity, addr = s2.recvfrom(1024)
            avaliable_quantity = avaliable_quantity.decode('utf-8')
            print(avaliable_quantity)
            if int(quantity) > int(avaliable_quantity):
                print("Not enough quantity")
                continue
            price = int(price.decode('utf-8'))
            buy_amt = price * int(quantity)
            inp = '3'
            inp = inp.encode()
            s.sendto(inp, (bank_host, bank_port))
            s.sendto(name.encode(), (bank_host, bank_port))
            bank_balance, addr = s.recvfrom(1024)
            if int(bank_balance) < buy_amt:
                print("Not enough money")
                continue

            data = "OK"
            data = data.encode()

            s2.sendto(data, (market_host, market_port))
            s2.sendto(name.encode(), (market_host, market_port))
            s2.sendto(quantity.encode(), (market_host, market_port))

            option = '4'
            option = option.encode()
            s.sendto(option, (bank_host, bank_port))
            s.sendto(str(buy_amt).encode(), (bank_host, bank_port))
            s.sendto(stock.encode(), (bank_host, bank_port))
            s.sendto(name.encode(), (bank_host, bank_port))

        elif option == '5':
            s2.sendto(option.encode(), (market_host, market_port))
            stock = input("What stock do you want to sell?")
            quantity = input("What is the quantity?")
            s2.sendto(stock.encode(), (market_host, market_port))
            price, addr = s2.recvfrom(1024)
            s2.sendto(name.encode(), (market_host, market_port))
            avaliable_quantity, addr = s2.recvfrom(1024)

            if int(quantity) > int(avaliable_quantity):
                data = "NO"
                data = data.encode()
                s2.sendto(data, (market_host, market_port))
                print("No enough quantity")
                continue

            data = "OK"
            data = data.encode()

            s2.sendto(data, (market_host, market_port))
            quantity = str(quantity)
            qunat = int(quantity)
            quantity = quantity.encode()
            s2.sendto(quantity, (market_host, market_port))

            option = '5'
            s.sendto(option.encode(), (bank_host, bank_port))
            price = price.decode()
            price = int(price)
            sell_amt = qunat * price
            print(sell_amt)
            sell_amt = str(sell_amt)
            sell_amt = sell_amt.encode()
            s.sendto(sell_amt, (bank_host, bank_port))
            s.sendto(name.encode(), (bank_host, bank_port))
            s.sendto(stock.encode(), (bank_host, bank_port))


import time
import inspect
import random, os
import numpy as np
import pandas as pd
os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(str(x) for x in [5, 6, 7])

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

from torch.nn import init
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader

from diffusers import DDPMScheduler, DDIMScheduler
from diffusers.optimization import get_cosine_schedule_with_warmup
from diffusers import DDPMPipeline, UNet2DModel, UNet2DConditionModel

from tqdm.auto import tqdm
from dataclasses import dataclass
from accelerate import Accelerator
from sklearn.model_selection import train_test_split
from typing import List, Optional, Tuple, Union

seed = 42
n_classes = 5
data_path = './harbox_sensys21_dataset/'
data_save_folder = './backup/harbox/'

@dataclass
class TrainingConfig:
    data_size = (3, 100)
    train_batch_size = 128
    test_batch_size = 128
    ae_num_epochs = 100
    dm_num_epochs = 200
    num_train_timesteps = 1000
    num_inference_steps = 100
    gradient_accumulation_steps = 1
    learning_rate = 5e-4
    lr_warmup_steps = 10
    eta = 0
    train_size = 0.99
    channel_list = [64, 32, 16]
    kernel_size = 3
    window_length = 100
    mixed_precision = 'fp16'  # `no` for float32, `fp16` for automatic mixed precision
    output_dir = data_save_folder+'hippo_harbox'  # the model namy locally and on the HF Hub
    IMUD_norm = False # IMUD normalized--not good
    splitD_norm = True

    autoencoder_train = True # whether to re-train the model *****************************************
    layer_num = 3 # the i-th layer of extracted feature embedding *************************************
    corss_attention_dim = 3  # here needs to change according to the layer number :1-7, 2-5, 3-3 ******
    layer_embedding_name = data_save_folder+'layer_embedding_'+str(layer_num)+'.pt' # the saved embedding feature file *******************
    save_model_name = data_save_folder+'scheduler_model_l'+str(layer_num)+'.pt' # the saved diffusion model file ************************

config = TrainingConfig()

# Initialize accelerator and tensorboard logging
accelerator = Accelerator(
    mixed_precision=config.mixed_precision,
    gradient_accumulation_steps=config.gradient_accumulation_steps,
    log_with="tensorboard",
    logging_dir=os.path.join(config.output_dir, "logs"),
)

# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = accelerator.device
generator_fix = torch.Generator(device=device)
generator_fix.manual_seed(seed)

class Encoder(nn.Module):
    def __init__(self, in_channel, out_channel, kernel_size, stride=1, padding=0):
        super(Encoder,self).__init__()
        self.in_channel = in_channel
        self.out_channel = out_channel
        self.kernel_size = kernel_size
        self.stride = stride
        self.padding = padding
        self.encoder = nn.Sequential(nn.Conv2d(in_channel, out_channel, kernel_size, stride, padding))
        self.pooling = nn.MaxPool2d(kernel_size, stride, padding, return_indices=True)
        self.decoder = nn.Sequential(nn.ConvTranspose2d(out_channel, in_channel, kernel_size, stride, padding))
        self.unpooling = nn.MaxUnpool2d(kernel_size, stride, padding)

    def forward(self,x):
        x = self.encoder(x)
        x, indices = self.pooling(x)
        x = self.unpooling(x, indices) # filter out part of information since non-maximal values are lost
        x = self.decoder(x)
        return x

    def extract_feature_raw(self, x_data, batch_size, window_length, device):
        result = []
        batched_x_data = []
        batched_y_data = [] # we don't need the y_info, return []
        for idx in range(0, len(x_data)-batch_size, batch_size):
            sample = torch.tensor(x_data[idx:idx+batch_size, :, :]).reshape(batch_size, 1, window_length, -1).float()
            batched_x_data.append(sample)
            sample = sample.to(device)
            sample_embedding = self.encoder(sample)
            result.append(sample_embedding)
        return result, batched_x_data, batched_y_data

    def extract_feature_embed(self, data):
        result = []
        for idx in range(len(data)):
            sample_embedding = self.encoder(data[idx])
            result.append(sample_embedding)
        return result

class Decoder(nn.Module):
    # minmize the error between raw data and reconstructed data
    def __init__(self, in_channel, out_channel, kernel_size_1, kernel_size_2, stride=1, padding=0):
        super(Decoder, self).__init__()
        self.conv_trans1 = nn.ConvTranspose2d(in_channel, out_channel, kernel_size_1, strFalseide, padding)
        self.conv_trans2 = nn.ConvTranspose2d(out_channel, 1, kernel_size_2, stride, padding)

    def forward(self, x):
        x = self.conv_trans1(x)
        x = self.conv_trans2(x)
        return x

class ExternalCrossAttention(nn.Module):
    def __init__(self, d_model,S=64):
        super().__init__()
        self.mq = nn.Linear(d_model, S, bias=False)
        self.mk = nn.Linear(d_model,S,bias=False)
        self.mv = nn.Linear(S,d_model,bias=False)
        self.softmax = nn.Softmax(dim=1)
        self.init_weights()

    def init_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                init.kaiming_normal_(m.weight, mode='fan_out')
                if m.bias is not None:
                    init.constant_(m.bias, 0)
            elif isinstance(m, nn.BatchNorm2d):
                init.constant_(m.weight, 1)
                init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                init.normal_(m.weight, std=0.001)
                if m.bias is not None:
                    init.constant_(m.bias, 0)

    def forward(self, latent_unet, latent_condition):
        Q = self.mq(latent_unet)
        K = self.mk(latent_condition)
        V = self.mv(latent_condition)
        attn=torch.bmm(Q, K)
        attn=self.softmax(attn)
        attn=attn/torch.sum(attn,dim=2,keepdim=True) #bs,n,S
        out=torch.bmm(attn, V)
        return out

class UNet_Block(nn.Module):
    def __init__(self, in_ch, out_ch):
        super().__init__()
        self.conv1 = nn.Conv2d(in_ch, out_ch, 3)
        self.relu = nn.ReLU()
        self.conv2 = nn.Conv2d(out_ch, out_ch, 3)
    def forward():
        return self.relu(self.conv2(self.relu(self.conv1(x))))

class UNet_Encoder(nn.Module):
    def __init__(self, chs=(1, 8, 16, 32)):
        super().__init__()
        self.enc_blocks = nn.ModuleList([UNet_Block(chs[i], chs[i+1]) for i in range(len(chs)-1)])
        self.pool = nn.MaxPool2d(2)
    def forward():
        ftrs = []
        for block in self.enc_blocks:
            x = block(x)
            ftrs.append(x)
            x = self.pool(x)
        return ftrs

class UNet_Decoder(nn.Module):
    def __init__(self, chs = (32, 16, 8), cond_embed = None):
        super().__init__()
        self.chs = chs
        self.upconvs = nn.ModuleList([nn.ConvTranspose2d(chs[i], chs[i+1], 2, 2) for i in range(len(chs)-1)])
        self.dec_blocks = nn.ModuleList([UNet_Block(chs[i], chs[i+1]) for i in range(len(chs)-1)])
        self.crossAttention = ExternalCrossAttention(d_model = 32, S = 128)
    def forward(self, x, encoder_features, embed_condition):
        x = self.crossAttention(x, embed_condition)
        for i in range(len(self.chs)-1):
            x = self.upconvs[i](x)
            enc_ftrs = self.crop(encoder_features[i], x)
            x = torch.cat([x, enc_ftrs], dim = 1) # note here is cat., so channel doubled
            x = self.dec_blocks[i](x)
        return x
    def crop(self, enc_ftrs, x):
        _, _, H, W = x.shape
        enc_ftrs = torchvision.transforms.CenterCrop([H, W])(enc_ftrs)
        return enc_ftrs

class UNet(nn.Module):
    def __init__(self, enc_chs=(1, 8, 16, 32), dec_chs=(32, 16, 8), num_class = 5, retrain_dim = False, out_sz = (1, 100)):
        super().__init__()
        self.encoder = UNet_Encoder(enc_chs)
        self.decoder = UNet_Decoder(dec_chs)
        self.head = nn.Conv2d(dec_chs[-1], num_class, 1)
        self.retain_dim = retain_dim

    def forward(self, x, embed_condition):
        enc_ftrs = self.encoder(x)
        out = self.decoder(enc_ftrs[::-1][0], enc_ftrs[::-1][1:], embed_condition)
        # out = self.head(out)
        if self.retain_dim:
            out = F.interpolate(out, out_sz)
        return out

class IMUDataset(Dataset):
    def __init__(self, rootpath = data_path, saved = False, normalize = True):
        super().__init__()
        self.X = []
        self.y = []
        if saved == False:
            dataset_dict = load_data()
        for key in dataset_dict.keys():
            self.X.extend(dataset_dict[key])
            self.y.extend([int(key)]*np.ones(len(dataset_dict[key])).astype(int))
        if normalize == True:
            v = np.array(self.X)
            v_min = v.min(axis=(0, 1), keepdims=True)
            v_max = v.max(axis=(0, 1), keepdims=True)
            self.X = 2*(v - v_min)/(v_max - v_min)-1
    def __len__(self):
        return len(self.y)
    def __getitem__(self, idx):
        sample = torch.tensor(self.X[idx])
        sample = sample[None,:]
        return sample, self.y[idx]

def read_data(file_path):
    column_names = ['timestamp', 'acc_x', 'acc_y', 'acc_z', 'gyro_x', 'gyro_y', 'gyro_z', 'mag_x', 'mag_y', 'mag_z']
    data = pd.read_csv(file_path, sep=' ', header = None, names = column_names)
    return data

def load_data(window_length=100, axis_num=9, data_path=data_path):
    dataset = {}
    class_set = ['Call','Hop','typing','Walk','Wave']
    class_num = len(class_set)
    for user_id in range(0, 121): #!(0, 121)
        for class_id in range(class_num):
            read_path = data_path +  str(user_id) + '/' + str(class_set[class_id]) + '_train' + '.txt'
            if os.path.exists(read_path):
                tmp_original_data = read_data(read_path)
                tmp_coll = tmp_original_data.iloc[:, 1:axis_num+1].to_numpy().reshape(-1, window_length, axis_num)
                if class_id not in dataset.keys():
                    dataset[class_id]=[]
                dataset[class_id].extend(tmp_coll)
    return dataset

def set_loader(config):
    imu_data = IMUDataset(rootpath = data_path, saved = False, normalize = config.IMUD_norm)
    train_len = int(len(imu_data)*config.train_size)
    test_len = len(imu_data)-train_len
    train_set, test_set = torch.utils.data.random_split(imu_data, [train_len, test_len])
    train_loader = torch.utils.data.DataLoader(train_set, batch_size=config.train_batch_size, num_workers=16, pin_memory=True, drop_last=True)
    test_loader = torch.utils.data.DataLoader(test_set, batch_size=config.test_batch_size, num_workers=16, pin_memory=True, drop_last=True)
    return train_loader, test_loader

def split_dataset(train_size=0.8, window_length=100, normalize = True):
    dataset = load_data(window_length=window_length)
    test_size = 1 - train_size
    x_dataset = []
    y_dataset = []

    for key in dataset.keys():
        x_dataset.extend(dataset[key])
        y_dataset.extend(key*np.ones(len(dataset[key])).astype(int))

    if normalize == True:
        v = np.array(x_dataset)
        v_min = v.min(axis=(0, 1), keepdims=True)
        v_max = v.max(axis=(0, 1), keepdims=True)
        x_dataset = 2*(v - v_min)/(v_max - v_min)-1

    x_train, x_test, y_train, y_test = train_test_split(x_dataset, y_dataset, test_size = test_size, random_state=seed)
    x_train, x_test, y_train, y_test = np.array(x_train), np.array(x_test), np.array(y_train), np.array(y_test)

    return x_train, x_test, y_train, y_test

def autoencoder_train_raw(model, loss_function, optimizer, x_train, window_length=100, batch_size=32, epochs=1, device='cpu'):
    model.train()
    print("******* Autoencoder train raw *******")
    for epoch in range(epochs+1):
        for idx in range(0, len(x_train)-batch_size, batch_size):
            sample = torch.tensor(x_train[idx:idx+batch_size, :, :]).reshape(batch_size, 1, window_length, -1).float()
            sample = sample.to(device)
            model.zero_grad()
            recon_sample = model(sample)
            loss = loss_function(recon_sample, sample)
            loss.backward()
            optimizer.step()
        if epoch % 10  ==0:
            print('Train Epoch: {} \tLoss: {:.6f}'.format(epoch, loss.item()/batch_size))
    return model

def autoencoder_train_embed(model, loss_function, optimizer, data, window_length=100, batch_size=32, epochs=1, device='cpu'):
    model.train()
    print("******* Autoencoder train embed *******")
    for epoch in range(epochs+1):
        for idx in range(len(data)):
            sample = data[idx]
            sample = sample.to(device)
            model.zero_grad()
            recon_sample = model(sample)
            loss = loss_function(recon_sample, sample)
            loss.backward(retain_graph=True)
            optimizer.step()
        if epoch % 10 == 0:
            print('Train Epoch: {} \tLoss: {:.6f}'.format(epoch, loss.item()/batch_size))
    return model

def create_latent(x_train, layer_num=3, channel_list=[32, 64, 128], kernel_size=3, window_length=100, batch_size=32, epochs=100, device='cpu', mode_train = False):
    if mode_train == True:
        # stacked autoencoder training--first layer encoder
        model = Encoder(1, channel_list[0], kernel_size).to(device)
        optimizer = optim.SGD(model.parameters(), lr=1*1e-4)
        loss_function = nn.MSELoss()
        model = autoencoder_train_raw(model, loss_function, optimizer, x_train, window_length, batch_size, epochs, device)
        torch.save(model, data_save_folder+'autoencoder_raw.pt')
    else:
        model = torch.load(data_save_folder+'autoencoder_raw.pt', map_location=torch.device(device))

    layer_1, batched_x_data, batched_y_data = model.extract_feature_raw(x_train, batch_size, window_length, device)

    if layer_num == 1:
        return layer_1, batched_x_data, batched_y_data

    if mode_train == True:
        # stacked autoencoder training--second layer encoder
        model_2 = Encoder(channel_list[0], channel_list[1], kernel_size).to(device)
        optimizer_2 = optim.SGD(model_2.parameters(), lr=5*1e-4)
        loss_function = nn.MSELoss()
        model_2 = autoencoder_train_embed(model_2, loss_function, optimizer_2, layer_1, window_length, batch_size, epochs, device)
        torch.save(model, data_save_folder+'autoencoder_raw.pt')
        torch.save(model_2, data_save_folder+'autoencoder_sec.pt')
    else:
        model_2 = torch.load(data_save_folder+'autoencoder_sec.pt', map_location=torch.device(device))

    layer_2 = model_2.extract_feature_embed(layer_1)

    if layer_num == 2:
        return layer_2, batched_x_data, batched_y_data

    if mode_train == True:
        # stacked autoencoder training--third layer encoder
        model_3 = Encoder(channel_list[1], channel_list[2], kernel_size).to(device)
        optimizer_3 = optim.SGD(model_3.parameters(), lr=5*1e-4)
        loss_function = nn.MSELoss()
        model_3 = autoencoder_train_embed(model_3, loss_function, optimizer_3, layer_2, window_length, batch_size, epochs, device)
        torch.save(model, data_save_folder+'autoencoder_raw.pt')
        torch.save(model_2, data_save_folder+'autoencoder_sec.pt')
        torch.save(model_3, data_save_folder+'autoencoder_third.pt')
    else:
        model_3 = torch.load(data_save_folder+'autoencoder_third.pt', map_location=torch.device(device))

    layer_3 = model_3.extract_feature_embed(layer_2)

    if layer_num == 3:
        return layer_3, batched_x_data, batched_y_data

def train_loop(config, model, encoder_hidden_states, noise_scheduler, optimizer, train_dataloader, lr_scheduler, do_classifier_free_guidance = True, save_model_name= 'scheduler_model.pt'):
    # global varabile: accelerator
    if accelerator.is_main_process:
        accelerator.init_trackers("train_example")

    # Prepare everything
    # There is no specific order to remember, you just need to unpack the
    # objects in the same order you gave them to the prepare method.
    model, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
        model, optimizer, train_dataloader, lr_scheduler
    )
    global_step = 0
    accepts_eta = "eta" in set(inspect.signature(noise_scheduler.step).parameters.keys())
    extra_step_kwargs = {}
    if accepts_eta:
        extra_step_kwargs["eta"] = config.eta

    for epoch in range(config.dm_num_epochs):
        progress_bar = tqdm(total=len(train_dataloader), disable=not accelerator.is_local_main_process)
        progress_bar.set_description(f"Epoch {epoch}")

        for idx, (clean_data, y) in enumerate(train_dataloader):
            noise = torch.randn(clean_data.shape, generator=generator_fix, device = device, dtype = torch.float32).to(device)
            noise = noise * noise_scheduler.init_noise_sigma
            bs = clean_data.shape[0]

            # Sample a random timestep for each image ---todo test if fix the timestep works better---
            timesteps = torch.randint(int(noise_scheduler.num_train_timesteps/10), noise_scheduler.num_train_timesteps, (bs,), device = device).long()
            noisy_images = noise_scheduler.add_noise(clean_data, noise, timesteps)

            with accelerator.accumulate(model):
                # print(np.shape(noisy_images), np.shape(timesteps))
                noisy_images = noisy_images.to(torch.float32)
                # print(np.shape(encoder_hidden_states[idx]))
                hidden_view = encoder_hidden_states[idx].view(*encoder_hidden_states[idx].shape[:1], -1, encoder_hidden_states[idx].shape[-1])
                # print(np.shape(noisy_images), np.shape(hidden_view))
                if do_classifier_free_guidance == True:
                    if random.uniform(0, 1) > 0.95:
                        hidden_view = 1e-5*torch.ones(hidden_view.size()).to(device)
                noise_pred = model(noisy_images, timesteps, hidden_view, return_dict = False)[0]
                loss = F.mse_loss(noise_pred, noise)
                accelerator.backward(loss, retain_graph=True)

                # accelerator.clip_grad_norm_(model.parameters(), 1.0)
                optimizer.step()
                lr_scheduler.step()
                optimizer.zero_grad()

            progress_bar.update(1)
            logs = {"loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0], "step": global_step}
            progress_bar.set_postfix(**logs)
            accelerator.log(logs, step = global_step)
            global_step+=1

    save_object_dict = {"noise_scheduler": noise_scheduler, "model": accelerator.unwrap_model(model = model)}
    torch.save(save_object_dict, save_model_name)

    return noise_scheduler, model

@torch.no_grad()
def evaluate(latents_shape, scheduler, model, encoder_hidden_states, device, num_inference_steps = 1,  guidance_scale = 2, do_classifier_free_guidance = True):
    extra_step_kwargs = {}

    noise_latents = torch.randn(latents_shape, device=device)
    random_data = noise_latents
    scheduler.set_timesteps(num_inference_steps)
    timesteps = scheduler.timesteps.to(device)
    noise_latents = noise_latents*scheduler.init_noise_sigma

    hidden_view = encoder_hidden_states[0].view(*encoder_hidden_states[0].shape[:1], -1, encoder_hidden_states[0].shape[-1])
    uncond_view = 1e-5*torch.ones(hidden_view.size()).to(device)
    hidden_view = torch.cat([uncond_view, hidden_view])
    # print(np.shape(latent_model_input), np.shape(hidden_view))

    for i, t in enumerate(timesteps):
        latent_model_input = torch.cat([noise_latents]*2) if do_classifier_free_guidance else noise_latents
        latent_model_input = scheduler.scale_model_input(latent_model_input, t)

        noise_pred = model(latent_model_input, t, hidden_view).sample
        if do_classifier_free_guidance:
            noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
            noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)

        noise_latents = scheduler.step(noise_pred, t, noise_latents, **extra_step_kwargs).prev_sample
    return random_data, noise_latents

if __name__ == "__main__":

    x_train, x_test, y_train, y_test = split_dataset(train_size = config.train_size, window_length = config.window_length, normalize = config.splitD_norm)
    latent_shape = (config.train_batch_size, 1, *np.shape(x_train[0]))
    # latent_shape = (1, 1, *np.shape(x_train[0]))

    layer_embedding, batched_data, true_scores = create_latent(x_train, config.layer_num, config.channel_list, config.kernel_size, config.window_length, config.train_batch_size, config.ae_num_epochs, device=device, mode_train=config.autoencoder_train)
    print(len(layer_embedding), layer_embedding[0].shape)
    torch.save(layer_embedding, config.layer_embedding_name)
    # layer_embedding = torch.load(config.layer_embedding_name)

    train_loader, test_loader = set_loader(config)

    # check the source file here https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/unet_2d_condition.py
    model = UNet2DConditionModel(
        sample_size=config.data_size,  # the target image resolution
        in_channels=1,  # the number of input channels, 3 for RGB images
        out_channels=1,  # the number of output channels
        layers_per_block=1,  # how many ResNet layers to use per UNet block
        # down_block_types = ("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D"),
        # up_block_types = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"),
        block_out_channels=(32, 64, 128, 256),  # the number of output channes for each UNet block.
        cross_attention_dim=config.corss_attention_dim,
    )

    save_model_name = config.save_model_name
    noise_scheduler = DDIMScheduler(num_train_timesteps = config.num_train_timesteps)
    optimizer = torch.optim.AdamW(model.parameters(), lr = config.learning_rate)
    lr_scheduler = get_cosine_schedule_with_warmup(optimizer=optimizer, num_warmup_steps=config.lr_warmup_steps, num_training_steps=(len(train_loader)*config.dm_num_epochs))

    noise_scheduler, model = train_loop(config, model, layer_embedding, noise_scheduler, optimizer, train_loader, lr_scheduler, save_model_name=save_model_name)

    saved_object_dict = torch.load(save_model_name, map_location=torch.device(device))
    noise_scheduler, model = saved_object_dict["noise_scheduler"], saved_object_dict["model"]
    begin=time.time()
    random_data, generated_data = evaluate(latent_shape, noise_scheduler, model, encoder_hidden_states = layer_embedding, device = device, num_inference_steps = config.num_inference_steps)
    end=time.time()
    print("Time spent: ", end-begin)
    print(np.shape(generated_data))


from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.shortcuts import render, redirect, get_object_or_404
from django.urls import reverse

from home.forms import UploadFileForm
from home.models import LearningFileStorage
from personal_space.models import UserBookmarkArticle
from .forms import ArticlesForm
from .models import Article

def article(request, article_id):
    """ Display the user's profile.
    :param request:
    :param article_id:
    """

    article = get_object_or_404(Article, id=article_id)
    user_bookmark = None
    if request.user.is_authenticated:
        user_bookmark = UserBookmarkArticle.objects.filter(user=request.user, article=article)

    context = {
        'article': article,
        'user_bookmark': user_bookmark
    }

    return render(request, 'article.html', context)

@login_required(redirect_field_name='home')
def write_article(request):
    """
        Allows content Manager to write his own articles
    """
    if not request.user.is_staff:
        messages.error(request, 'Sorry, only content Managers can create new content Articles.')
        return redirect(reverse('learning_area'))

    if request.method == 'POST':
        form = ArticlesForm(request.POST)
        if form.is_valid():
            form.save()
            messages.success(request, 'Article Stored')
        return redirect('learning_area')
    else:
        form = ArticlesForm()
        file_form = UploadFileForm()
        files_on_page = LearningFileStorage.objects.all()

        context = {'form': form,
                   'file_form': file_form,
                   'files': files_on_page
                   }
    return render(request, 'write_article.html', context)

@login_required(redirect_field_name='home')
def edit_article(request, article_id):
    """
        Allows content Managers to edit articles
        :param request:
        :param article_id:
        :return:
    """

    if not request.user.is_staff:
        messages.error(request, 'Sorry, You are not authorized to do this action.')
        return redirect(reverse('learning_area'))

    article = get_object_or_404(Article, pk=article_id)
    if request.method == 'POST':
        form = ArticlesForm(request.POST, instance=article)
        if form.is_valid():
            form.save()
            messages.success(request, 'Article changes are saved.')
            return redirect(reverse('article', args=[article.id]))
        else:
            messages.error(request,
                           'Failed to update Article. Please try again!')
    else:
        form = ArticlesForm(instance=article)
        file_form = UploadFileForm()
        files_on_page = LearningFileStorage.objects.all()
        context = {
            'form': form,
            'article': article,
            'file_form': file_form,
            'files': files_on_page
        }
        return render(request, 'write_article.html', context)

@login_required(redirect_field_name='home')
def delete_article(request, article_id):
    """ Check if is is_staff user or author before deleting article"""
    if not request.user.is_staff:
        messages.error(request, 'Sorry, you are not allowed to remove an article')
        return redirect(reverse('home'))

    article = get_object_or_404(Article, pk=article_id)
    article.delete()
    messages.success(request, 'Article removed Successfully!')
    return redirect(reverse('learning_area'))