#! /usr/bin/env python3
# coding: utf-8

"""
Ubuntu:
1. pip3 install pdfkit markdown
2. apt-get install wkhtmltopdf
   more platform: https://github.com/JazzCore/python-pdfkit/wiki/Installing-wkhtmltopdf

Windows：
1. scoop install python wkhtmltopdf
2. pip3 install pdfkit markdown
"""

import os
import re
import sys
import string

import pdfkit

def get_title(prefix, current):
    if prefix == "":
        title = "{}".format(current)
    else:
        title = "{}.{}".format(prefix, current)

    return title

def generate(prefix, line_list, current_sharp, start_index):
    size = len(line_list)
    if start_index >= size:
        return size + 1

    current = 1

    head_pattern = re.compile(r"^(#+)\s*(.*)$")
    title_pattern = re.compile(r"^[\s\.\d]+(.*)$")
    code_pattern = '```'
    code_cnt = 0

    i = start_index
    while True:
        if i >= size:
            break

        line = line_list[i]

        line_trip = line.strip()
        if line_trip.startswith(code_pattern):
            code_cnt += 1

        if code_cnt % 2 == 1:
            i += 1
            continue;

        match = head_pattern.match(line)
        if match:
            sharp = match.group(1)
            current_size = len(sharp)
            if current_size > current_sharp:
                i = generate(get_title(prefix, current - 1), line_list, current_size, i)

            elif current_size == current_sharp:
                title = get_title(prefix, current)
                right = match.group(2)
                r = title_pattern.match(right)
                if r:
                    right = r.group(1)

                if right != "":
                    line_list[i] = "{} {} {}".format(sharp, title, right)
                else:
                    line_list[i] = "{} {}".format(sharp, title)

                current += 1
                i += 1

            else:
                break

        else:
            i += 1

    return i

def reformat_md(filename):
    if not os.path.exists(filename):
        print ("%s file is not exist." % (filename))
        return

    with open(filename, mode="r", encoding="utf-8") as f:
        contents = f.read()

    line_list = contents.split("\n")
    generate("", line_list, 1, 0)

    with open(filename, "w", encoding="utf-8") as f:
        size = len(line_list)
        i = 0
        while i < size - 1:
            f.write(line_list[i])
            f.write("\n")
            i += 1

        f.write(line_list[size-1])

# 设置环境变量
def to_pdf(file_name):
    import markdown

    out_file = '.'.join([file_name.rsplit('.', 1)[0], 'pdf'])
    out_html = '.'.join([file_name.rsplit('.', 1)[0], 'html'])

    input_file = open(file_name, mode="r", encoding="utf-8")
    text = input_file.read()

    exts = ['markdown.extensions.extra', 'markdown.extensions.codehilite','markdown.extensions.tables','markdown.extensions.toc', 'markdown.extensions.nl2br']

    html = markdown.markdown(text, extensions=exts)

    output_file = open(out_html, "w",encoding="utf-8",errors="xmlcharrefreplace")

    # 可以根据选择修改css样式， 可以参考github.css和default.css
    css = '''
    <html">
    <head>
    <meta content="text/html; charset=utf-8" http-equiv="content-type" />
        <style>
            font-family: Helvetica, arial, sans-serif;
            font-size: 14px;
            line-height: 1.6;
            padding-top: 10px;
            padding-bottom: 10px;
            background-color: white;
            padding: 30px; }

            h1, h2, h3, h4, h5, h6 {
              margin: 20px 0 10px;
              padding: 0;
              font-weight: bold;
              -webkit-font-smoothing: antialiased;
              cursor: text;
              position: relative; }

            h1:hover a.anchor, h2:hover a.anchor, h3:hover a.anchor, h4:hover a.anchor, h5:hover a.anchor, h6:hover a.anchor {
              background: url("../../images/modules/styleguide/para.png") no-repeat 10px center;
              text-decoration: none; }

            h1 tt, h1 code {
              font-size: inherit; }

            h2 tt, h2 code {
              font-size: inherit; }

            h3 tt, h3 code {
              font-size: inherit; }

            h4 tt, h4 code {
              font-size: inherit; }

            h5 tt, h5 code {
              font-size: inherit; }

            h6 tt, h6 code {
              font-size: inherit; }

            h1 {
              font-size: 28px;
              color: black; }

            h2 {
              font-size: 24px;
              border-bottom: 1px solid #cccccc;
              color: black; }

            h3 {
              font-size: 18px; }

            h4 {
              font-size: 16px; }

            h5 {
              font-size: 14px; }

            h6 {
              color: #777777;
              font-size: 14px; }

            table {
              padding: 0; }
              table tr {
                border-top: 1px solid #cccccc;
                background-color: white;
                margin: 0;
                padding: 0; }
                table tr:nth-child(2n) {
                  background-color: #f8f8f8; }
                table tr th {
                  font-weight: bold;
                  border: 1px solid #cccccc;
                  text-align: left;
                  margin: 0;
                  padding: 6px 13px; }
                table tr td {
                  border: 1px solid #cccccc;
                  text-align: left;
                  margin: 0;
                  padding: 6px 13px; }
                table tr th :first-child, table tr td :first-child {
                  margin-top: 0; }
                table tr th :last-child, table tr td :last-child {
                  margin-bottom: 0;
        </style>
    </head>
    <body>
    %s
    </body>
    </html>
    '''

    output_file.write(css % html)
    output_file.close()

    pdfkit.from_file(out_html, out_file)

    os.remove(out_html)

if __name__ == "__main__":
    if len(sys.argv) <= 1:
        print ("Help >\n" +
			"> generate sequece number: markdown-tools.py name.md>\n" +
			"> genereate pdf: markdown-tools.py pdf name.md>")
        os._exit(0)

    if sys.argv[1] == 'pdf':
        to_pdf(sys.argv[2])
        os._exit(0)

    reformat_md(sys.argv[1])


from datetime import datetime
from functools import partial

from bouncebox.util import Object
from  bouncebox.core.series import create_series

CLASS_SERIES_CACHE = {}

class Event(Object):
    repr_attrs = ['timestamp']

    # check __setattr__ immutable == True means immutable
    immutable = False

    def __init__(self, timestamp=None, source_event=None, series=None):
        self.generated = datetime.now()
        self.source_event = source_event
        if timestamp is None and source_event:
            timestamp = source_event.timestamp
        self.timestamp = timestamp

        if series is None:
            series = self.__class__.class_series()
        self.series = series
        self.immutable = True

    @classmethod
    def class_series(self):
        # quick way to get default series
        if not self in CLASS_SERIES_CACHE:
            CLASS_SERIES_CACHE[self] = create_series(self)
        return CLASS_SERIES_CACHE[self]

    def ___setattr__(self, name, value):
        # events are immutable
        if self.immutable:
            assert False, 'boo'
        super(Event, self).__setattr__(name, value)

# Would like this switchable? Eventually remove non cython version?
try:
    from bouncebox.core.event_cython import Event
except:
    print('No cython Event')

class SourceEvent(Event):
    """
        Would like to disintuish events that are generated externally
    """

class EndEvent(Event):
    graphable = False

class StartEvent(Event):
    """
        Used to signal to bouncebox that it is beginning.
    """
    graphable = False

class EventFactory(object):
    """
        Events should come from some sort of factory.
        Not sure if factory is the right word.
        Just a place to keep grouped events metadata together
    """

    def __init__(self, event_cls):
        self.event_cls = event_cls

    def build_event(self, *args, **kwargs):
        event = self.event_cls(*args, **kwargs)
        event.factory = self
        return event

class SeriesEventFactory(EventFactory):
    """
        Takes in a Series object and generates events based of those
    """
    def __init__(self, series):
        self.series = series
        self.event_cls = series.event_cls
        self.event_args = series.event_args
        self.create_event_func()
        super(SeriesEventFactory, self).__init__(series.event_cls)

    def create_event_func(self):
        kw = {}
        kw['series'] = self.series
        for arg in self.event_args:
            kw[arg] = getattr(self.series, arg)
        func = partial(self._build_event, **kw)
        self.build_event = func

    def _build_event(self, *args, **kwargs):
        event = self.event_cls(*args, **kwargs)
        return event


T = 10
for tc in range(1, T + 1):
    v, e = map(int, input().split())
    graph = [[] for _ in range(v + 1)]
    visited = [True for _ in range(v + 1)]
    # 노드와 간선의 리스트
    temp_list = list(map(int, input().split()))
    count = v
    # 선행 순서이므로 반대로 넣어줌
    # a = 1, b = 2 일 경우
    # 2는 1이 수행완료 되어야 가능하다
    for i in range(0, len(temp_list), 2):
        v, e = temp_list[i], temp_list[i + 1]
        graph[e].append(v)
    # 결과누적 리스트
    result = []
    # 노드가 모두 수행될때까지 반복
    while count:
        # 1번부터 마지막 노드까지 반복
        for i in range(1, len(graph)):
            # 현재 노드가 수행되지 않았다면 (True가 수행되지 않은것)
            if visited[i]:
                # 선행 노드를 확인
                for j in graph[i]:
                    # 선행노드가 수행되지 않았다면 멈추고 빠져나와서 다음 노드 확인
                    if visited[j]:
                        break
                    # 선행노드가 수행되었다면
                    # 현재노드 수행했다고 바꾸고 결과에 추가
                    # 노드 수행되었으므로 count 감소
                else: # graph가 빈 요소이면 (선행될 것이 없으면) (for else 문)
                    visited[i] = False
                    result.append(i)
                    count -= 1

    print(f'#{tc}', *result)

SWEA/D6/1267. ［S／W 문제해결 응용］ 10일차 － 작업순서/［S／W 문제해결 응용］ 10일차 － 작업순서.py

［S／W 문제해결 응용］ 10일차 － 작업순서.py


import tensorflow as tf
# 设置日志等级（取消warning）
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

def tensorflow_demo():
    a = 2
    b = 3
    c = a + b
    print('Python:\n', c)
    tf.compat.v1.disable_v2_behavior()
    a_t = tf.constant(2)
    b_t = tf.constant(3)
    c_t = a_t + b_t
    print('c_t:\n', c_t)
    # 开启会话
    # sess = tf.compat.v1.Session()
    # print(sess.run(c_t))
    with tf.compat.v1.Session() as sess:
        print(sess.run(c_t))

    return None

def graph_demo():
    tf.compat.v1.disable_v2_behavior()
    a_t = tf.constant(2, name='a_t')
    b_t = tf.constant(3, name='b_t')
    c_t = tf.add(a_t, b_t, name='c_t')

    # 法一：调用方法
    default_g = tf.compat.v1.get_default_graph()
    print("default graph:\n", default_g)
    # 法二：查看属性
    print("a_t.graph:\n", a_t.graph)
    print("c_t.graph:\n", c_t.graph)
    sess = tf.compat.v1.Session()
    print("sess.graph:\n", sess.graph)

    # 自定义图：
    new_g = tf.Graph()
    with new_g.as_default():
        a_new = tf.constant(20)
        b_new = tf.constant(30)
        c_new = a_new + b_new
        print('c_new:\n', c_new)
    new_sess = tf.compat.v1.Session(graph=new_g)
    print('c_new_value:\n', new_sess.run(c_new))
    print("a_new.graph:\n", a_new.graph)
    print("c_new.graph:\n", c_new.graph)
    print("new_sess.graph:\n", new_sess.graph)

    # 图的可视化
    # step1 将图写入本地events文件
    # tf.compat.v1.summary.FileWriter("./tmp/summary", graph=sess.graph)
    # step2 调用tensorboard

    return None

def session_demo():
    tf.compat.v1.disable_v2_behavior()
    a_t = tf.constant(2, name='a_t')
    b_t = tf.constant(3, name='b_t')
    c_t = tf.add(a_t, b_t, name='c_t')
    print('a_t:\n', a_t)
    print('b_t:\n', b_t)
    print('c_t:\n', c_t)

    a_ph = tf.compat.v1.placeholder(tf.float32)
    b_ph = tf.compat.v1.placeholder(tf.float32)
    c_ph = tf.add(a_ph, b_ph)

    with tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(allow_soft_placement=True, log_device_placement=True)) as \
            sess:
        print("sess.run(c_t):\n", sess.run(c_t))
        print("c_t.eval(session=sess):\n", c_t.eval(session=sess))
        abc = sess.run([a_t, b_t, c_t])
        print("abc:\n", abc)
        c_ph_value = sess.run(c_ph, feed_dict={a_ph: 3.9, b_ph: 4.8})
        print("c_ph_value:\n", c_ph_value)
    return None

def tensor_demo():
    # 张量的类型修改
    tensor1 = tf.constant(4.0)
    tensor2 = tf.constant([1, 2, 3, 4])
    linear_squares = tf.constant([[4], [9], [16], [25]], dtype=tf.int32)
    print("tensor1:\n", tensor1)
    print("tensor2:\n", tensor2)
    print("linear_squares:\n", linear_squares)
    print("linear_squares_cast:\n", tf.cast(linear_squares, dtype=tf.float32))
    with tf.compat.v1.Session() as sess:
        print("tf.zeros.eval():\n", tf.zeros(shape=[3, 4], dtype=tf.float32).eval())
    # 张量静态形状的修改
    tf.compat.v1.disable_v2_behavior()
    a_ph = tf.compat.v1.placeholder(dtype=tf.float32, shape=[None, None])
    b_ph = tf.compat.v1.placeholder(dtype=tf.float32, shape=[None, 10])
    c_ph = tf.compat.v1.placeholder(dtype=tf.float32, shape=[3, 2])
    a_ph.set_shape([2, 3])
    b_ph.set_shape([2, 10])
    b_ph_reshape = tf.reshape(b_ph, shape=[2, 2, 5])
    print('a_ph:\n', a_ph)
    print("b_ph:\n", b_ph)
    print("b_ph_reshape:\n", b_ph_reshape)
    return None

def variable_demo():
    tf.compat.v1.disable_v2_behavior()
    # 创建变量
    with tf.compat.v1.variable_scope("my_scope"):
        a = tf.Variable(initial_value=50)
        b = tf.Variable(initial_value=40)
    with tf.compat.v1.variable_scope("your_scope"):
        c = tf.add(a, b)
    print(a)
    print(c)
    # 创建初始化变量
    init = tf.compat.v1.global_variables_initializer()
    # 开启会话
    with tf.compat.v1.Session() as sess:
        sess.run(init)
        a_value, b_value, c_value = sess.run([a, b, c])
        print("{}   {}   {}".format(a_value, b_value, c_value))
    return None

# tensorflow_demo()
# graph_demo()
# session_demo()
# tensor_demo()
variable_demo()


import requests

API_URL = "http://localhost:3000/api/v1/prediction/85ca04f1-94c1-4366-8183-a270e07a61dc"

def query(payload):
    response = requests.post(API_URL, json=payload)
    return response.json()

output = query({
    "question": "Hey, how are you?",
})

print(output)


from django.contrib import admin
from django.contrib import admin, messages
from django.utils.translation import gettext, gettext_lazy as _, ngettext
from jalali_date.admin import ModelAdminJalaliMixin

from .models import Products, Comments

class CommentsInline(admin.TabularInline):
    model = Comments
    fields = ['user', 'text', 'stars', 'is_active', 'recommend']
    extra = 1

@admin.register(Products)
class ProductsAdmin(ModelAdminJalaliMixin, admin.ModelAdmin):
    list_display = ['title', 'cover_img', 'slug', 'price', 'status', 'active', 'datetime_created']
    search_field = ('title',)
    prepopulated_fields = {'slug': ('title',)}
    actions = ['make_not_available', 'make_available', 'make_active', 'make_de_active']
    inlines = [
        CommentsInline,
    ]

    @admin.action(description=_('Selected products are not available'))
    def make_not_available(self, request, queryset):
        updated = queryset.update(status='not')
        self.message_user(request, ngettext(
            _('%d story was successfully marked as not available.'),
            _('%d stories were successfully marked as not available.'),
            updated,
        ) % updated, messages.SUCCESS)

    @admin.action(description=_('Selected products are active'))
    def make_active(self, request, queryset):
        updated = queryset.update(active=True)
        self.message_user(request, ngettext(
            _('%d story was successfully marked as activated.'),
            _('%d stories were successfully marked as activated.'),
            updated,
        ) % updated, messages.SUCCESS)

    @admin.action(description=_('Selected products are de_active'))
    def make_de_active(self, request, queryset):
        updated = queryset.update(active=False)
        self.message_user(request, ngettext(
            _('%d story was successfully marked as deactivated.'),
            _('%d stories were successfully marked as deactivated.'),
            updated,
        ) % updated, messages.SUCCESS)

    @admin.action(description=_('Selected products are available'))
    def make_available(self, request, queryset):
        updated = queryset.update(status='ava')
        self.message_user(request, ngettext(
            _('%d story was successfully marked as available.'),
            _('%d stories were successfully marked as available.'),
            updated,
        ) % updated, messages.SUCCESS)

@admin.register(Comments)
class CommentsAdmin(ModelAdminJalaliMixin, admin.ModelAdmin):
    list_display = ['user', 'product', 'is_active', 'recommend', 'stars', 'datetime_created']
    search_field = ('product',)


import os
import numpy as np
from sklearn.model_selection import cross_val_score
from sklearn.feature_selection import VarianceThreshold
from collections import defaultdict
import time
from scipy import sparse
import pandas as pd
import glob

"""
the folowing function definition are based on:

Single-Resolution Multi-quantity Multi-variate BOPF (SMM-BOPF)
Multi-Resolution Multi-quantity  Multi-variate BOPF (MMM-BOPF)

"""

_BANDS = ["lsstg", "lssti", "lsstr", "lsstu", "lssty", "lsstz"]

def cv_score(x, labels, classes, _pipeline, message="", cv=5, n_jobs=8):
    ini = time.time()
    dropped, max_dropped = _pipeline.check_dropped(x, labels)
    if dropped > max_dropped:
        print("[{}] dropped because {} > {}".format(message, dropped, max_dropped))
        return [-1], None, dropped, [-1, -1], 0

    n_variables = len(_BANDS)
    n_observations, m_temp = x.shape
    n_features = m_temp // n_variables
    # print(":::::::: initial matrix shape:", x.shape,
    #       "(n_observations: %d, n_features: %d, n_variables: %d)" % (n_observations,
    #                                                                  n_features,
    #                                                                  n_variables))
    sklearn_pipeline = _pipeline.get_compact_pipeline(n_variables, n_features, classes)

    print("[%s]: DOING CROSS VALIDATION..." % message, end="\r")
    ini = time.time()
    scores = cross_val_score(sklearn_pipeline, x, labels,
                             scoring="balanced_accuracy", cv=cv, n_jobs=n_jobs, verbose=0)
    end = time.time()
    print("[%s]: %.3f += %.3f (time: %.3f sec)" % (message, float(np.mean(scores)),
                                                   float(np.std(scores)), end - ini))

    return scores, sklearn_pipeline, dropped, [-1, -1], end - ini

def cv_ssm_bopf(data, labels, wins, wls, _pipeline, cv=5, n_jobs=8,
                drop_zero_variance=True, outfile=None, C="LSA"):
    classes = np.unique(labels)
    q_code = _pipeline.Q[0]
    data_mr_repr = defaultdict(lambda: defaultdict(object))
    cv_results = defaultdict(lambda: defaultdict(object))
    result_lists = defaultdict(list)
    best_data = None
    best_acc = -1
    print("LOADING DATA FROM QUANTITY SEARCH... ", end="")
    iniq = time.time()
    for wl in wls:
        for win in wins:
            message = "[win: %.3f, wl: %d, q: %s]" % (win, wl, q_code)
            # try:
            if True:
                data_repr_i_full, elapse = _pipeline.ssm_bopf(data, win, wl, q_code)
                message += "{BOPF time: %.3f Secs}" % elapse
                if drop_zero_variance:
                    try:
                        drop_zero_variance = VarianceThreshold()
                        data_repr_i = drop_zero_variance.fit_transform(data_repr_i_full)
                    except:
                        data_repr_i = data_repr_i_full
                else:
                    data_repr_i = data_repr_i_full
                # print(":::::::: zero variance reduces features: ",
                #       data_repr_i_full.shape, " -> ", data_repr_i.shape)
                data_mr_repr[win][wl] = data_repr_i
                cv_results_i = cv_score(data_repr_i, labels, classes, _pipeline, message=message, cv=cv, n_jobs=n_jobs)
                cv_results[win][wl] = cv_results_i
                result_lists["win"].append(win)
                result_lists["wl"].append(wl)

                if cv_results_i[1] is not None:
                    result_lists["score"].append(np.mean(cv_results_i[0]))
                else:
                    result_lists["score"].append(-1)
                acc_i = float(np.mean(cv_results_i[0]))
                if acc_i > best_acc:
                    best_acc = acc_i
                    best_data = (cv_results_i, win, wl)
                if outfile is not None:
                    cv_mean = float(np.mean(cv_results_i[0]))
                    cv_std = float(np.std(cv_results_i[0]))
                    line = "%s,%s,%.3f,%d,%d,%d,%s,%.3f,%.3f,%.3f,%.3f" % (
                        C, q_code, win, wl, _pipeline.doc_kw["alphabet_size"],
                        cv_results_i[2], str(cv_mean > -1), cv_mean, cv_std,
                        elapse, cv_results_i[4])
                    f = open(outfile, "a+")
                    f.write(line + "\n")
                    f.close()
            # except Exception as e:
            #     print("failed iteration wl=%d, win=%f, error: %s" % (wl, win, e))
    endq = time.time()
    print("DONE (%.3f secs)" % (endq - iniq))
    return data_mr_repr, cv_results, result_lists, best_acc, best_data

def cv_smm_bopf(data, labels, wins, wls, _pipeline, cv=5, n_jobs=8,
                drop_zero_variance=True, outfile=None, C="-", data_folder=None):
    """
    compute cross validation score for

    Single-Resolution Multi-quantity Multi-variate BOPF (SMM-BOPF)

    using different combinations of (window width, word length)
    """
    classes = np.unique(labels)
    q_code = _pipeline.quantities_code()
    data_mr_repr = defaultdict(lambda: defaultdict(object))
    cv_results = defaultdict(lambda: defaultdict(object))
    result_lists = defaultdict(list)
    best_data = None
    best_acc = -1

    for wl in wls:
        for win in wins:
            try:
                if data_folder is None:
                    data_repr_i_full, elapse = _pipeline.smm_bopf(data, win, wl)
                    if drop_zero_variance:
                        drop_zero_variance = VarianceThreshold()
                        data_repr_i = drop_zero_variance.fit_transform(data_repr_i_full)
                    else:
                        data_repr_i = data_repr_i_full
                else:
                    _pipeline.doc_kw["alphabet_size"] = np.array([_pipeline.alpha])
                    quantities_keys = _pipeline.quantities_code()[1:-1].split("-")
                    data_keys = []
                    for q_key in quantities_keys:
                        data_key = sparse.load_npz(os.path.join(data_folder, q_key + "_%d_%.3f.npz" % (wl, win)))
                        data_keys.append(data_key)
                    data_repr_i = sparse.hstack(data_keys, format="csr")
                    elapse = -1.0
                # print(":::::::: zero variance reduces features: ",
                #       data_repr_i_full.shape, " -> ", data_repr_i.shape)
                data_mr_repr[win][wl] = data_repr_i
                message = "[win: %.3f, wl: %d, q: %s, n_fea: %d, bopf_time: %.3f]" % (
                    win, wl, q_code, data_repr_i.shape[1], elapse)
                cv_results_i = cv_score(data_repr_i, labels, classes, _pipeline, message=message, cv=cv, n_jobs=n_jobs)
                cv_results[win][wl] = cv_results_i
                result_lists["win"].append(win)
                result_lists["wl"].append(wl)

                if cv_results_i[1] is not None:
                    result_lists["score"].append(np.mean(cv_results_i[0]))
                else:
                    result_lists["score"].append(-1)
                acc_i = float(np.mean(cv_results_i[0]))
                if acc_i > best_acc:
                    best_acc = acc_i
                    best_data = (cv_results_i, win, wl)
                if outfile is not None:
                    cv_mean = float(np.mean(cv_results_i[0]))
                    cv_std = float(np.std(cv_results_i[0]))
                    line = "%s,%s,%.3f,%d,%d,%d,%s,%s,%.3f,%.3f,%.3f,%.3f" % (
                        C, q_code, win, wl, _pipeline.doc_kw["alphabet_size"][0],
                        cv_results_i[2], str(data_repr_i.shape[1]), str(cv_mean > -1),
                        cv_mean, cv_std, elapse, cv_results_i[4])
                    f = open(outfile, "a+")
                    f.write(line + "\n")
                    f.close()

            except Exception as e:
                print("failed iteration wl=%d, win=%f, error: %s" % (wl, win, e))

    return data_mr_repr, cv_results, result_lists, best_acc, best_data

def cv_mmm_bopf(data, labels, _pipeline, cv=5, wins=None, wls=None,
                resolution_max=4, top_k=4, out_path=None,
                out_base_bopf_path=None, cv_smm_bopf_results=None,
                n_jobs=8, drop_zero_variance=True, timestamp=None):
    classes = np.unique(labels)
    q_code = _pipeline.quantities_code()
    if timestamp is None:
        timestamp = time.strftime("%Y%m%d-%H%M%S")

    base_file_name = "multi_ress_%s_%s.csv" % (q_code, timestamp)

    if cv_smm_bopf_results is None and wins is not None and wls is not None:
        cv_smm_bopf_results = cv_smm_bopf(data, labels, wins,
                                          wls, _pipeline,
                                          cv=cv, n_jobs=n_jobs,
                                          drop_zero_variance=drop_zero_variance)
    elif cv_smm_bopf_results is None and (wins is None or wls is None):
        raise ValueError("need to specify the CV_SMM_BOPF results or the values ofr wins and wls")

    data_mr_repr, cv_results, result_list, smm_best_acc, smm_best_data = cv_smm_bopf_results
    win_list = result_list["win"]
    wl_list = result_list["wl"]
    score_list = result_list["score"]

    if out_base_bopf_path is not None:
        try:
            save_base_bopf(data_mr_repr, win_list, wl_list, out_base_bopf_path)
            print("basic BOPF repr saved to %s" % out_base_bopf_path)
        except Exception as e:
            print("failed to save basic BOPF repr, error: ", e)

    # if out_path is not None:
    #     # save results of single_resolution
    #     save_first_stack(cv_results, win_list, wl_list, _pipeline, base_file_name, out_path)

    # rank results by acc
    rank_by_idx = np.argsort(score_list)[::-1]
    optimal_pairs = []
    optimal_acc = -1

    out_file = os.path.join(out_path, base_file_name)
    f = open(out_file, "a+")
    header = "top-k,stack,base,wl,win,dropped,shape_before,shape_after,mean_cv,std_cv,exp_var,n_comp,scheme\n"
    f.write(header)
    f.close()

    for ii, top_idx in enumerate(rank_by_idx[:top_k]):
        # incremental config
        x = data_mr_repr[win_list[top_idx]][wl_list[top_idx]]
        record_idxs = [top_idx]
        record_acc = score_list[top_idx]
        print("---> starting stack of configurations for top ", ii + 1)
        msg = "[(win:wl)"
        for used_idx in record_idxs:
            msg += "-(%.3f:%d)" % (win_list[used_idx], wl_list[used_idx])
        msg += "]"
        print("---> best config stack 1: %s ::::: acc: %.3f" % (msg, float(record_acc)))
        # stack up until resolution_max configs
        for jj in range(2, resolution_max + 1):
            try:
                print("---> starting search on stack level ", jj)
                # define an output file

                # define iter parameters to find best config
                best_config = None
                best_acc = record_acc

                # start iter to find next stack config
                for next_idx in rank_by_idx:
                    # if next_idx was not added to incremental config
                    if next_idx not in record_idxs:
                        # try to stack this idx and check acc
                        try:
                            # generate a compy of the repr matrix
                            x_i = sparse.hstack([x, data_mr_repr[win_list[next_idx]][wl_list[next_idx]]], format="csr")

                            # generate a custom message
                            message = "[(win:wl)"
                            for used_idx in record_idxs:
                                message += "-(%.3f:%d)" % (win_list[used_idx], wl_list[used_idx])
                            message += "-(%.3f:%d)]" % (win_list[next_idx], wl_list[next_idx])

                            # get a cr score from this current stack of configs
                            score, pipeline, dropped, shapes, cv_time = cv_score(x_i, labels, classes, _pipeline,
                                                                                message=message, cv=cv, n_jobs=n_jobs)
                            # line to write to file
                            line = "%d,%d,%s,%d,%f,%d,%d,%d," % (
                                ii+1, jj, message, wl_list[next_idx], win_list[next_idx], dropped, shapes[0], shapes[1])

                            # if the iteration  was invalid (should always be valid here)
                            if pipeline is None:
                                # raise an error
                                raise ValueError("the cv failed for some reason")
                            # else
                            else:
                                # generate the rest of the line to write
                                if "lsa" in pipeline:
                                    exp_var = np.sum(pipeline["lsa"].explained_variance_ratio_)
                                else:
                                    exp_var = None
                                n_comps = _pipeline.K
                                scheme_name = pipeline["vsm"].get_scheme_notation()
                                mean_cv = float(np.mean(score))
                                std_cv = float(np.std(score))
                                line += "%f,%f,%f,%d,%s\n" % (mean_cv, std_cv,
                                                              exp_var if exp_var is not None else -1,
                                                              n_comps,
                                                              pipeline["vsm"].get_scheme_notation())

                                # and if this stack is the best so far, update variables
                                if mean_cv > best_acc:
                                    best_config = record_idxs + [next_idx]
                                    best_acc = mean_cv
                        except Exception as e:
                            print("failed on stack step k=%d, for wl=%d, win=%f, error: %s" % (
                                jj, wl_list[next_idx], win_list[next_idx], e))
                        else:

                            # write line to file
                            f = open(out_file, "a+")
                            try:
                                f.write(line)
                            except:
                                f.close()
                            else:
                                f.close()

                # after the iteration, if no improvement was achieve, stop the process
                if best_config is None or best_acc - record_acc < 0.01:  # at least improvement of 1%
                    print("ACCURACY IS NOT IMPROVING, STOPPING CODE")
                    break
                else:
                    # update the incremental configuration and advance to next stack level
                    x_arr = []
                    for used_idx in best_config:
                        x_arr.append(data_mr_repr[win_list[used_idx]][wl_list[used_idx]])
                    x = sparse.hstack(x_arr, format="csr")
                    record_idxs = best_config
                    record_acc = best_acc
                    msg = "[(win:wl)"
                    for used_idx in record_idxs:
                        msg += "-(%.3f:%d)" % (win_list[used_idx], wl_list[used_idx])
                    msg += "]"
                    print("---> best config stack %d: %s ::::: acc: %.3f" % (_pipeline.n, msg, float(record_acc)))
            except Exception as e:
                print("failed stack step k=%d, error: %s" % (_pipeline.n, e))

        if record_acc > optimal_acc:
            optimal_acc = record_acc
            optimal_pairs = record_idxs
    R = [(win_list[i], wl_list[i]) for i in optimal_pairs]
    print("FINAL BEST CONFIG: ", R, ", acc: ", optimal_acc)
    return R, timestamp, optimal_acc

def save_base_bopf(data_mr_repr, win_list, wl_list, out_base_bopf_path):
    metadata_dict = defaultdict(list)
    time_stamp = time.strftime("%Y%m%d-%H%M%S")
    for win, wl in zip(win_list, wl_list):
        sparse_repr_matrix = data_mr_repr[win][wl]
        out_file = os.path.join(
            out_base_bopf_path,
            "data_" + str(win) + "#" + str(round(wl, 3)) + "_" + time_stamp + ".npz")
        sparse.save_npz(out_file, sparse_repr_matrix)
        metadata_dict["win"].append(win)
        metadata_dict["wl"].append(wl)
        metadata_dict["file"].append(out_file)

    out_file = os.path.join(out_base_bopf_path, "metadata_" + time_stamp + ".csv")
    df = pd.DataFrame(metadata_dict, index=False)
    df.to_csv(out_file)

def save_first_stack(x_score, win_list, wl_list, _pipeline, base_file_name, out_path):
    out_file = os.path.join(out_path, base_file_name + "1-top_0-" + time.strftime(
        "%Y%m%d-%H%M%S") + ".csv")
    f = open(out_file, "a+")
    header = "base,wl,win,dropped,shape_before,shape_after,mean_cv,std_cv,exp_var,n_comp,scheme\n"
    f.write(header)
    f.close()

    msg = "[(win:wl)]"
    for i in range(len(win_list)):
        win = win_list[i]
        wl = wl_list[i]
        scores, cv_pipeline, dropped, shapes = x_score[win][wl]
        # line to write to file
        line = "%s,%d,%f,%d,%d,%d," % (msg, wl, win, dropped, shapes[0], shapes[1])
        exp_var = -1
        n_comps = _pipeline.K
        scheme = _pipeline.get_scheme_notation()
        mean_cv = -1
        std_cv = -1

        if cv_pipeline is not None:
            mean_cv = float(np.mean(scores))
            std_cv = float(np.std(scores))
            if "lsa" in cv_pipeline:
                exp_var = np.sum(cv_pipeline["lsa"].explained_variance_ratio_)

        line += "%f,%f,%f,%d,%s\n" % (mean_cv, std_cv, exp_var, n_comps, scheme)

        f = open(out_file, "a+")
        try:
            f.write(line)
        except:
            f.close()
        else:
            f.close()

def load_base_bopf(path, timestamp, _pipeline, labels, cv=5, n_jobs=8):
    data_mr_repr = defaultdict(lambda: defaultdict(object))
    classes = np.unique(labels)
    q_code = _pipeline.quantities_code()
    cv_results = defaultdict(lambda: defaultdict(object))
    result_lists = defaultdict(list)
    for f in glob.glob(os.path.join(path, "*%s*" % timestamp)):
        win, wl = os.path.split(f)[1].split("_")[1].split("#")
        win = float(win)
        wl = int(wl)
        message = "[win: %.3f, wl: %d, q: %s]" % (win, wl, q_code)
        try:
            data_repr_i_full = sparse.load_npz(f)
            drop_zero_variance = VarianceThreshold()
            data_repr_i = drop_zero_variance.fit_transform(data_repr_i_full)
            print(":::::::: zero variance reduces features: ",
                  data_repr_i_full.shape, " -> ", data_repr_i.shape)
            data_mr_repr[win][wl] = data_repr_i
            cv_results_i = cv_score(data_repr_i, labels, classes, _pipeline, message=message, cv=cv, n_jobs=n_jobs)
            cv_results[win][wl] = cv_results_i
            result_lists["win"].append(win)
            result_lists["wl"].append(wl)

            if cv_results_i[1] is not None:
                result_lists["score"].append(np.mean(cv_results_i[0]))
            else:
                result_lists["score"].append(-1)
        except Exception as e:
            print("failed iteration wl=%d, win=%f, error: %s" % (wl, win, e))

    return data_mr_repr, cv_results, result_lists

def load_bopf_from_quantity_search(path_data, path_cv_results, Q):
    data_mr_repr = defaultdict(lambda: defaultdict(object))
    cv_results = defaultdict(lambda: defaultdict(object))
    result_lists = defaultdict(list)

    df_cv_results = pd.read_csv(path_cv_results)
    df = df_cv_results[df_cv_results["quantity"] == Q]
    # set new wins and awls
    wins2 = np.unique(df["win"])
    wls2 = np.unique(df["wl"])
    Q_splitted = Q[1:-1].split("-")
    best_acc = -1
    best_data = None
    for wl in wls2:
        for win in wins2:
            try:
                bopf_data = []
                for q in Q_splitted:
                    f = os.path.join(path_data, "%s_%d_%.3f.npz" % (q, wl, win))
                    bopf_data.append(sparse.load_npz(f))
                data_i = sparse.hstack(bopf_data, format="csr")
                data_mr_repr[win][wl] = data_i
                # line = df.loc[(df["win2"] == int(win)) & (df["wl"] == wl)]
                line = df.loc[(df["win"] == win) & (df["wl"] == wl)]
                cv_results_i = ([line.cv_mean.values[0], line.cv_std.values[0]],
                                None, line.dropped.values[0],
                                [-1, -1], line.cv_time.values[0])
                cv_results[win][wl] = cv_results_i
                result_lists["win"].append(win)
                result_lists["wl"].append(wl)
                result_lists["score"].append(line.cv_mean.values[0])
                acc_i = line.cv_mean.values[0]
                if acc_i > best_acc:
                    best_acc = acc_i
                    best_data = (cv_results_i, win, wl)
            except Exception as e:
                print("failed load for [%d, %.3f] with error: " % (wl, win), win, e)
                raise e

    return data_mr_repr, cv_results, result_lists, best_acc, best_data

# scores, sklearn_pipeline, dropped, [-1, -1], end - ini


#!/usr/bin/env python3.5
from MyPDB import XYZ

class RigidBody():
    def __init__(self, p1: XYZ, p2: XYZ, p3: XYZ, p4: XYZ):
        self.p1 = p1
        self.p2 = p2
        self.p3 = p3
        self.p4 = p4

    def __str__(self):
        return '%r\n%r\n%r\n%r' % (self.p1, self.p2, self.p3, self.p4)

    def __repr__(self):
        return self.__str__()

def read_RBs(file_name):
    results = []
    with open(file_name, 'r') as fin:
        for l in fin:
            floats = [float(a) for a in l.split()[1:]]
            rb = RigidBody(XYZ(floats[0], floats[1], floats[2]), XYZ(floats[3], floats[4], floats[5]),
                           XYZ(floats[6], floats[7], floats[8]), XYZ(floats[9], floats[10], floats[11]))
            results.append(rb)
            break
    print(results)

if __name__ == '__main__':
    read_RBs('all_dir_0_RB.db')


#!/usr/bin/python
# -*- coding: utf-8 -*-

"""ring info user system local code.

__author__ = 'kylinfish@126.com'
__date__ = '2014/09/22'
"""

# import base64

from .base import get, post, delete, build_query_url
from .. import config

def create_users(auth, play_load):
    u"""注册IM用户[批量] 建议批量不要过多, 在20-60之间.

        :param auth: 身份认证
        :param play_load: [{'username':'${用户名1}','password':'${密码}'},…]

        Path : /{org_name}/{app_name}/users
        HTTP Method : POST
        URL Params ： 无
        Request Headers : {“Content-Type”:”application/json”,”Authorization”:”Bearer ${token}”}
        Request Body ： [{“username”:”${用户名1}”,”password”:”${密码}”},…,{“username”:”${用户名2}”,”password”:”${密码}”}]
        Response Body ： 详情参见示例返回值, 返回的json数据中会包含除上述属性之外的一些其他信息，均可以忽略。
    """

    url = "%s/%s/%s/users" % (config.HOST_SERVER, config.APP_ORG, config.APP_NAME)
    return post(url, play_load, auth)

def pickup_users(auth, limit=None, cursor=None):
    u"""获取IM用户[批量查询].

    :param auth: 身份认证
    :param limit: 数量值
    :param cursor: 游标

    该接口默认返回最近创建的10个用户, 如果需要指定获取数量，需加上参数limit=N，N为数量值.
    关于分页：如果DB中的数量大于N, 返回json会携带一个字段“cursor”, 我们把它叫做”游标”, 该游标可理解为结果集的指针, 值是变化的.

    往下取数据的时候带着游标, 就可以获取到下一页的值.
    如果还有下一页, 返回值里依然还有这个字段, 直到没有这个字段, 说明已经到最后一页.
    cursor的意义在于数据(真)分页.

    1) 不分页

        Path : /{org_name}/{app_name}/users
        HTTP Method : GET
        URL Params ： limit=20
        Request Headers : {“Authorization”:”Bearer ${token}”}
        Request Body ： 无
        Response Body ： 详情参见示例返回值, 返回的json数据中会包含除上述属性之外的一些其他信息，均可以忽略。

    2) 分页

        Path : /{org_name}/{app_name}/users
        HTTP Method : GET
        URL Params ： limit=20&cursor=LTU2ODc0MzQzOnNmdTlxdF9LRWVPaVFvMWlBZmc4S3c
        Request Headers : {“Authorization”:”Bearer ${token}”}
        Request Body ： 无
        Response Body ： 详情参见示例返回值, 返回的json数据中会包含除上述属性之外的一些其他信息，均可以忽略。
    """

    url = "%s/%s/%s/users" % (config.HOST_SERVER, config.APP_ORG, config.APP_NAME)

    payload = []
    if limit:
        payload.append("limit=%d" % limit)
    if cursor:
        payload.append("cursor=%s" % cursor)

    if payload:
        url = ''.join((url, "?", "&".join(payload)))

    # 处理数据的另外方式
    # query = {"limit": limit, "cursor": cursor}
    # url = build_query_url(url, query)

    return get(url, auth)

def delete_users(auth, ql=None, limit=None):
    u""""删除IM用户[批量].

    :param auth: 身份认证
    :param ql: sql
    :param limit: 数量值

    删除某个app下指定数量的环信账号。可一次删除N个用户, 数值可以修改.
    建议这个数值在100-500之间, 不要过大.
    需要注意的是, 这里只是批量的一次性删除掉N个用户, 具体删除哪些并没有指定, 可以在返回值中查看到哪些用户被删除掉了.

    可以通过增加查询条件来做到精确的删除, 例如:

        按照创建时间来排序(降序)
            DELETE /{org_name}/{app_name}/users?ql=order+by+created+desc&limit=300

        按照创建时间来排序(升序)
            DELETE /{org_name}/{app_name}/users?ql=order+by+created+asc&limit=300

        按时间段来删除 使用ql=created> {起始时间戳} and created < {结束时间戳} 的查询语句, 时间戳是timestamp类型的, 并且需要对ql进行http url encode
            DELETE /{org_name}/{app_name}/users?ql=created > 1409506121910 and created < 1409576121910

        Path : /{org_name}/{app_name}/users
        HTTP Method : DELETE
        URL Params : limit=30
        Request Headers : {“Authorization”:”Bearer ${token}”}
        Request Body ： 无
        Response Body ： 详情参见示例返回值, 返回的json数据中会包含除上述属性之外的一些其他信息，均可以忽略。
    """

    url = "%s/%s/%s/users" % (config.HOST_SERVER, config.APP_ORG, config.APP_NAME)

    """
    payload = []
    if ql:
        payload.append("ql=%s" % base64.urlsafe_b64encode(ql)) # urlsafe_b64encode 接不上REST API
    if limit:
        payload.append("limit=%d" % limit)

    if payload:
        url = ''.join((url, "?", "&".join(payload)))
    """

    # 处理数据的另外方式
    query = {"limit": limit, "ql": ql}
    url = build_query_url(url, query)

    return delete(url, auth)

def add_group_users(auth, group_id, usernames):
    u"""群组批量添加成员.

        :param auth: 身份认证
        :param group_id: 群组ID, 注意: 非群组名称
        :param usernames: 用户列表

        Path : /{org_name}/{app_name}/chatgroups/{chatgroupid}/users
        HTTP Method : POST
        URL Params ： 无
        Request Headers : {“Authorization”:”Bearer ${token}”}
        Request Body ：{“usernames”:[“username1”,”username2”]}’
                        — usernames固定属性，作为json的KEY；username1/username2 要添加到群中的成员用户名，可变

        Response Body ：详情参见示例返回值, 返回的json数据中会包含除上述属性之外的一些其他信息，均可以忽略
    """

    payload = {"usernames": usernames}
    url = "%s/%s/%s/chatgroups/%s/users" % (config.HOST_SERVER, config.APP_ORG, config.APP_NAME, group_id)
    return post(url, payload, auth)


from odoo import models

class DasCalculPlanningReport(models.AbstractModel):
    _name = "report.das.calcul_planning_xlsx"
    _description = "Das Calcul Planning Report"
    _inherit = "report.report_xlsx.abstract"

    def generate_xlsx_report(self, workbook, data, lines):
        first_format = workbook.add_format({'font_size': 12, 'align': 'vcenter', 'bold': True})
        sheet = workbook.add_worksheet('Calcul planning')
        sheet_col = ['Num', 'Appelation', 'Nom', 'Login Jira', 'Post', 'CAT', 'Dep', 'Ref Name', 'Account', 'Project',
                     'Ref BC',
                     'Project category', 'Client', 'Locality', 'Date',  'Planned H', 'Planned J', 'Project type',
                     'FAC', 'PNF',
                     'INT', 'RH', 'AVV', 'MGT', 'PRV', 'REF DATE']

        for i in range(len(sheet_col)):
            sheet.write(0, i, sheet_col[i], first_format)
            i += 1

        i = 1
        for res_id in lines:
            # num
            sheet.write(i, 0, i)
            # appelation
            sheet.write(i, 1, res_id.resource_id.appelation)
            # Nom
            sheet.write(i, 2, res_id.resource_name)
            # Login_jira
            sheet.write(i, 3, res_id.resource_id.login_jira)
            # Post
            sheet.write(i, 4, res_id.resource_id.job_id.display_name)
            # cat
            sheet.write(i, 5, res_id.account_id.category_id.display_name)
            # centre
            sheet.write(i, 6, res_id.resource_id.department_id.name)
            # ref name
            sheet.write(i, 7, res_id.resource_id.ref_name)
            # compte
            sheet.write(i, 8, res_id.account_id.display_name)
            # projet
            sheet.write(i, 9, res_id.project_id.display_name)
            # ref_bc
            sheet.write(i, 10, res_id.account_id.reference_id.display_name)
            # #project_category
            sheet.write(i, 11, res_id.account_id.category_id.display_name)
            # #client
            sheet.write(i, 12, res_id.project_id.partner_id.display_name)
            # #localite
            sheet.write(i, 13, res_id.account_id.locality)
            # #date
            sheet.write(i, 14, str(res_id.date))
            # Semaine

            # ID : num-date-poste
            # Ferié
            # Planned days (daily hours)
            sheet.write(i, 15, res_id.daily_hours)
            # Planned days J
            sheet.write(i, 16, res_id.daily_hours / 8)
            # sequence : yearweek_num
            # week_num
            # month
            # ref date project
            # Project type : ref-date-project
            sheet.write(i, 17, res_id.type)
            # FAC
            sheet.write(i, 18, res_id.fac_hours)
            # PNF
            sheet.write(i, 19, res_id.pnf_hours)
            # int
            sheet.write(i, 20, res_id.int_hours)
            # RH
            sheet.write(i, 21, res_id.rh_hours)
            # AVV
            sheet.write(i, 22, res_id.avv_hours)
            # MGT
            sheet.write(i, 23, res_id.mgt_hours)
            # PRV
            sheet.write(i, 24, res_id.prv_hours)
            # ref date
            ref_date = str(res_id.resource_id.reference) + '-' + str(res_id.date)
            sheet.write(i, 25, ref_date)

            # IC J
            # IC %

            i += 1

class DasResourceReport(models.AbstractModel):
    _name = "report.das.resource_xlsx"
    _description = "Das Resource Report"
    _inherit = "report.report_xlsx.abstract"

    def generate_xlsx_report(self, workbook, data, lines):
        first_format = workbook.add_format({'font_size': 12, 'align': 'vcenter', 'bold': True})
        sheet = workbook.add_worksheet('Resource ')
        sheet_col = ['Reference', 'Department', 'Jira login', 'Reg Number', 'Name', 'First name', 'Appelation', 'Job',
                     'Gender', 'Previous Dep', 'Jira name', 'CC key', 'REF_NAME']

        for i in range(len(sheet_col)):
            sheet.write(0, i, sheet_col[i], first_format)
            i += 1

        i = 1
        for res_id in lines:
            # reference
            sheet.write(i, 0, res_id.resource_id.reference)
            # Department
            sheet.write(i, 1, res_id.department_id.name)
            # Login_jira
            sheet.write(i, 2, res_id.resource_id.login_jira)
            # Matricule
            sheet.write(i, 3, res_id.resource_id.registration_number)
            # name
            sheet.write(i, 4, res_id.resource_id.name)
            # first name
            sheet.write(i, 5, res_id.resource_id.first_name)
            # appelation
            sheet.write(i, 6, res_id.resource_id.appelation)
            # Job
            sheet.write(i, 7, res_id.resource_id.job_id.display_name)
            # sexe
            sheet.write(i, 8, res_id.resource_id.gender)
            # Previous department
            sheet.write(i, 9, res_id.resource_id.previous_department_id.name)
            # Jira name
            sheet.write(i, 10, res_id.resource_id.name_jira)
            # Code cc
            sheet.write(i, 11, res_id.resource_id.code_cc)
            # ref name
            sheet.write(i, 12, res_id.resource_id.ref_name)
            i += 1

class DasConsolidationPlanningReport(models.AbstractModel):
    _name = "report.das.consolidation_planning_xlsx"
    _description = "Das Consolidation Planning Report"
    _inherit = "report.report_xlsx.abstract"

    def generate_xlsx_report(self, workbook, data, lines):
        first_format = workbook.add_format({'font_size': 12, 'align': 'vcenter', 'bold': True})
        sheet = workbook.add_worksheet('Consolidation planning')
        sheet_col = ['Num', 'Department', 'Name', 'Account']

        # Add Column name
        for i in range(len(sheet_col)):
            sheet.write(0, i, sheet_col[i], first_format)
            i += 1

        i = 1
        for res_id in lines:
            # num
            sheet.write(i, 0, i)
            # Department
            sheet.write(i, 1, res_id.department_id.name)
            # name
            sheet.write(i, 2, res_id.resource_id.name)
            # Account
            sheet.write(i, 3, res_id.account_id.display_name)

            i += 1
        #
        # list_date = []
        # for res_id in lines:
        #     list_date.append(res_id.date)
        i = 1
        j = 4
        temp_date = lines[0].date
        for planning in lines:
            # Write same date in same column
            if planning.date != temp_date:
                j += 1
                temp_date = planning.date

            sheet.write(0, j, str(planning.date))
            sheet.write(i, j, planning.daily_hours)
            i += 1

class DasProjectPlanningReport(models.AbstractModel):
    _name = "report.das.project_planning_xlsx"
    _description = "DAS Project Planning Report"
    _inherit = "report.report_xlsx.abstract"

    def generate_xlsx_report(self, workbook, data, lines):
        first_format = workbook.add_format({'font_size': 12, 'align': 'vcenter', 'bold': True})
        sheet = workbook.add_worksheet('Project')
        sheet_col = ['Reference', 'Centre', 'Type', 'Key', 'Jira project', 'Client', 'Category', 'Locality',
                     'BC/Account',
                     'Project Description', 'Responsible', 'Techno', 'Payment condition']

        # Add Column name
        for i in range(len(sheet_col)):
            sheet.write(0, i, sheet_col[i], first_format)
            i += 1

        i = 1
        for res_id in lines:
            # reference
            sheet.write(i, 0, res_id.account_id.reference)
            # Department
            sheet.write(i, 1, res_id.department_id.name)
            # Type
            sheet.write(i, 2, res_id.type)
            # Key
            sheet.write(i, 3, res_id.account_id.key)
            # Jira Project
            sheet.write(i, 4, res_id.project_id.display_name)
            # Customer
            sheet.write(i, 5, res_id.project_id.partner_id.display_name)
            # Category
            sheet.write(i, 6, res_id.account_id.category_id.display_name)
            # Locality
            sheet.write(i, 7, res_id.account_id.locality)
            # BC
            sheet.write(i, 8, res_id.account_id.reference_id.display_name)
            # Project Description
            sheet.write(i, 9, res_id.account_id.description)
            # Responsible
            sheet.write(i, 10, res_id.account_id.responsible_id.display_name)
            # Techno
            sheet.write(i, 11, res_id.account_id.techno)
            # Payment condition
            sheet.write(i, 12, res_id.account_id.payment_condition)
            i += 1


import sys, pygame, random, math
pygame.init()

size = width, height = 1024, 768
black = 0, 0, 0
screen = pygame.display.set_mode(size)

explosion_imgs=[]
shield=100
for i in range(1,7):
    img=pygame.image.load("images/e"+str(i)+".png")
    explosion_imgs.append(img)

imagen=pygame.image.load("images/e1.png")
fuenteLetra=pygame.font.Font('freesansbold.ttf',20)

class ship:
    def __init__(self):
        self.img = pygame.image.load("images/spaceship.png")
        self.rect = self.img.get_rect()
        self.rect.x=20
        self.rect.y=400
        self.speed=[0,0]
        self.shooting=False
        self.reload=0
    def move(self):
        self.rect = self.rect.move(self.speed)
        if self.rect.top < 5:
            self.rect.top=5
        if self.rect.bottom > height-5:
            self.rect.bottom = height-5

class enemy:
    def __init__(self):
        self.img = pygame.image.load("images/enemy.png")
        self.rect = self.img.get_rect()
        self.rect.x=1000+random.randint(0,300)
        self.ybase=random.randint(120,660)
        self.rect.y=self.ybase
        self.speed=[-3,0]
        self.cycle=random.randint(0,300)
    def move(self):
        self.rect = self.rect.move(self.speed)
        self.cycle+=1
        self.rect.y=self.ybase+140*math.cos(-self.cycle/24)
        if self.rect.left < 5:
            global shield
            shield-=10
            enemies.remove(self)

    def reset(self):
        self.ybase=random.randint(60,700)
        self.rect.x=1000+random.randint(0,300)
class bullet:
    def __init__(self,inix,iniy):
        self.speed=[20,0]
        self.img = pygame.image.load("images/bullet.png")
        self.rect = self.img.get_rect()
        self.rect.x=inix
        self.rect.y=iniy
    def move(self):
        self.rect = self.rect.move(self.speed)
        if self.rect.x > width:
            bullets.remove(self)
class explosion:
    def __init__(self,inix,iniy):
        self.cycle=0
        self.speed=[-1,0]
        self.frames=0
        self.img = explosion_imgs[0]
        self.rect = self.img.get_rect()
        self.rect.x=inix
        self.rect.y=iniy
    def move(self):
        self.img = explosion_imgs[self.cycle]
        self.rect = self.rect.move(self.speed)
        self.frames+=1
        if (self.frames>10):
            self.frames=0
            self.cycle+=1
            if self.cycle >= 6:
                explosions.remove(self)

ship = ship()
enemies=[]
SKILL=0
def enemy_burst():
    y=random.randint(120,660)
    for i in range(5):
        temp=enemy()
        temp.rect.x=1000+i*80
        temp.ybase=y
        temp.cycle=-i*4
        enemies.append(temp)

enemy_burst()
bullets=[]
explosions=[]

logo=pygame.image.load("images/logo.png")
logorect=logo.get_rect()
logorect=logorect.move(100,300)

iniciar=0

while (iniciar==0):
    for event in pygame.event.get():
        if (event.type == pygame.QUIT):
            pygame.quit()
            sys.exit()
        if (event.type == pygame.KEYDOWN):
            if (event.key== pygame.K_RETURN):
                iniciar=1

    screen.blit(logo, logorect)

    text=fuenteLetra.render('Press Enter to START',True,(255,255,255))
    pos=(300,500)
    screen.blit(text,pos)
    pygame.display.flip()
    pygame.time.delay(10)

vivo=1

while (vivo==1):
    for event in pygame.event.get():
        if (event.type == pygame.QUIT):
            pygame.quit()
            sys.exit()
        if (event.type == pygame.KEYDOWN):
            if (event.key== pygame.K_UP):
                ship.speed=[0,-20]
            if (event.key== pygame.K_DOWN):
                ship.speed=[0,20]
            if (event.key== pygame.K_SPACE):
                ship.shooting=True
            if (event.key== pygame.K_RIGHT):
                enemy_burst()
        if (event.type == pygame.KEYUP):
            if (event.key== pygame.K_UP or event.key== pygame.K_DOWN):
                ship.speed=[0,0]
            if (event.key== pygame.K_SPACE):
                ship.shooting=False
    if(random.randint(0,200)<1):
        enemy_burst()
    ship.move()
    for e in enemies:
        e.move()
        for b in bullets:
            if e.rect.colliderect(b):
                enemies.remove(e)
                bullets.remove(b)
                explosions.append(explosion(e.rect.x,e.rect.y))
                SKILL+=1

    for b in bullets:
        b.move()
    for ex in explosions:
        ex.move()
    if(ship.shooting):
        if(ship.reload<=0 and len(bullets)<500):
            bullets.append(bullet(ship.rect.x+55,ship.rect.y+45))
            ship.reload=4
        else:
            ship.reload-=1

    screen.fill(black)
    for b in bullets:
        screen.blit(b.img, b.rect)

    for e in enemies:
        screen.blit(e.img, e.rect)

    for ex in explosions:
        screen.blit(ex.img, ex.rect)
    if shield==0:
        vivo=0
    screen.blit(ship.img, ship.rect)
    letras=fuenteLetra.render('Punts= %s' %(SKILL),True,(255,255,255))
    posTexto=(300,25)
    screen.blit(letras,posTexto)
    letras2=fuenteLetra.render('Shield= %s' %(shield),True,(255,255,255))
    posTexto2=(100,25)
    screen.blit(letras2,posTexto2)
    pygame.display.flip()
    pygame.time.delay(10)

gameover=pygame.image.load("images/gameover.png")
gameoverrect=gameover.get_rect()
gameoverrect=gameoverrect.move(100,300)

finalizar=0

while (finalizar==0):
    for event in pygame.event.get():
        if (event.type == pygame.QUIT):
            pygame.quit()
            sys.exit()
        if (event.type == pygame.KEYDOWN):
            if (event.key== pygame.K_RETURN):
                finalizar=1
    screen.fill(black)
    screen.blit(gameover, gameoverrect)

    text=fuenteLetra.render('Press Enter to EXIT',True,(255,255,255))
    pos=(300,500)

    screen.blit(text,pos)
    pygame.display.flip()
    pygame.time.delay(10)

pygame.quit()
sys.exit()


"""
Implementation of **QMix**
Comments:
    * based on `vdn2.py`
    * integrate mixer in Runner & make use of it optional
    * share network between agents
    * Uses QMixer

Applied to **defense**:
    * observation is a dictionary with keys:
        'obs': the actual observation as a ndarry containing:
            * self: pos_x, pos_y, alive flag, remaining ammo, aiming (-1 if not)
            * team: same information for agents of own team
            * others: same information for agents of other team
            * obstacles: pos_x and pos_y for obstacles
        'action_mask': Box of lenght len(actions) with:
            0 : action not allowed
            1 : action allowed
"""
import copy
import random
import sys
import inspect # get source code
from collections import deque

import numpy as np

import torch
from torch import nn
from torch import optim
from torch.nn import functional as F
device = "cuda" if torch.cuda.is_available() else "cpu"

from pettingzoo.mpe import simple_v2, simple_spread_v2

from helpers import EpisodeStep, Runner
from helpers import build_network, transform_episode

from torch.utils.tensorboard import SummaryWriter

# hack to allow import of env
import sys; sys.path.insert(0, '.')
from env import defense_v0
ENVIRONMENT = defense_v0

OPTIMIZER =  optim.Adam # optim.RMSprop #
LOCAL_RATIO = 0. #3 #1 # trade-off between local (collisions) and global reward (close to landmarks)

class RandomAgent:
    def __init__(self, actions):
        self.actions = actions

    def get_action(self, observation):
        return random.choice(self.actions)

def untangle(batch):
    observations = [{agent: step[agent].observation for agent in step} for step in batch]
    next_obs     = [{agent: step[agent].next_obs for agent in step} for step in batch]
    actions      = [{agent: step[agent].action for agent in step} for step in batch]
    # rewards and dones should be the same for all agents, so choose agent_0
    # => fully cooperative setting: all agents share a common reward function
    rewards      = torch.tensor([[step[agent].reward for agent in step] for step in batch]).to(device)
    dones        = torch.tensor([[step[agent].done for agent in step] for step in batch], dtype=torch.float32).to(device)
    states       = torch.from_numpy(np.stack([[step[agent].state for agent in step] for step in batch])).to(device)
    next_states  = torch.from_numpy(np.stack([[step[agent].next_state for agent in step] for step in batch])).to(device)
    masks        = torch.from_numpy(np.stack([[step[agent].mask for agent in step] for step in batch])).to(device)
    next_masks   = torch.from_numpy(np.stack([[step[agent].next_mask for agent in step] for step in batch])).to(device)
    return observations, masks, actions, rewards.T, dones.T, next_obs, next_masks, states, next_states

class EpisodeStep:
    def __init__(self, observation, mask, action, reward, done, next_obs, next_mask, state, next_state):
        self.observation = observation
        self.mask = mask
        self.action = action
        self.reward = reward
        self.done = done
        self.next_obs = next_obs
        self.next_mask = next_mask
        self.state = state
        self.next_state = next_state

    def __iter__(self):
        all = self.__dict__.values()
        return iter(all)

    def __repr__(self):
        s  = f"observation = {self.observation}\n"
        s += f"action = {self.action}\n"
        s += f"reward = {self.reward}\n"
        s += f"done = {self.done}\n"
        s += f"next_obs = {self.next_obs}\n"
        return s

class QMixRunner(Runner):
    def __init__(self, **kwargs) -> None:
        super().__init__(**kwargs)
        self.init_params(kwargs)
        self.env = ENVIRONMENT.env(terrain="flat_5x5", max_cycles=self.episode_length, max_distance=4)
        self.env.reset()

        N = len(self.env.agents)
        dims = self.env.observation_space('blue_0')['obs'].shape[0]
        # dims = 4 + 2 * N + 2 * (N - 1) + 2 * (N-1) # vx, vy, x, y, [landmark.x, landmark.y], [other.x, other.y], comms
        n_actions = self.env.action_space('blue_0').n
        # n_actions = 5 # [no_action, move_left, move_right, move_down, move_up]
        self.net = build_network([dims, *self.layers, n_actions]).to(device)

        self.agents = {}
        for agent in self.env.agents:
            self.agents[agent] = DQNAgent(name=agent, actions=range(n_actions), net=self.net)

        self.buffer = deque(maxlen=self.buffer_size)
        if self.use_mixer:
            self.mixer = QMixer(self.n_agents, self.env.state().shape[0], self.mixer_layer_size).to(device)
            self.target_mixer = copy.deepcopy(self.mixer).to(device)

        params = list(self.net.parameters()) # shared network amongst (identical) agents
        if self.use_mixer:
            params += list(self.mixer.parameters())
        self.optimizer = OPTIMIZER(params, lr=self.lr)

        self.writer = SummaryWriter(log_dir=f"runs/qmix_{str(self.rand_idx)}_{str(self.n_agents)}")

    def init_params(self, kwargs):
        self.n_agents = kwargs.get('n_agents', 2)
        self.episode_length = kwargs.get('episode_length', 200)
        self.layers = kwargs.get('layers', [128, 128])
        self.buffer_size = kwargs.get('buffer_size', 1024)
        self.gamma = kwargs.get('gamma', 0.99)
        self.lr = kwargs.get('lr', 0.001)
        self.n_batches = kwargs.get('n_batches', 32)     # K steps
        self.sample_size = kwargs.get('sample_size', 64) # N examples from replay memory
        self.n_evals = kwargs.get('n_evals', 10)

        self.epsilon = 1.0
        self.eps_min = kwargs.get('eps_min', 0.1)
        self.eps_steps = kwargs.get('eps_steps', 200000)
        self.eps_decay = (self.epsilon - self.eps_min)/self.eps_steps

        self.sync_rate = kwargs.get('sync_rate', 1)
        self.use_mixer = kwargs.get('use_mixer', True)
        self.mixer_layer_size = kwargs.get('mixer_layer_size', 16)

        self.verbose = kwargs.get('verbose', True)
        self.rand_idx = random.randint(0, 100000)

    def __str__(self):
        s  = f"##QMix applied to {self.env}\n------------------------------------\n"
        s += f"Number of agents = {len(self.agents)}  \n"
        s += f"Episode length = {self.episode_length}  \n"
        s += f"buffer size = {self.buffer_size}  \n"
        s += f"gamma = {self.gamma}  \n"
        s += f"learning rate = {self.lr}  \n"
        s += f"layers = {self.layers}  \n"
        s += f"eps_min = {self.eps_min}  \n"
        s += f"eps_steps = {self.eps_steps}  \n"
        s += f"sync_rate = {self.sync_rate}  \n"
        s += f"n_batches = {self.n_batches}  \n"
        s += f"sample_size = {self.sample_size}  \n"
        s += f"optimizer = {OPTIMIZER.__module__.split('.')[-1]}  \n"
        s += f"{'with' if self.use_mixer else 'without'} mixer  \n"
        s += f"device = {device}  \n"
        s += f"identifier = {self.rand_idx}  \n"
        return s

    def run(self, n_iters=10):
        for indx in range(n_iters):
            for _ in range(10):
                episode = self.generate_episode()
                self.buffer.extend(episode[:-1]) # last elem of episode contains no useful information
            if len(self.buffer) < self.sample_size:
                continue

            cum_loss = 0.0
            for _ in range(self.n_batches):
                batch = random.sample(self.buffer, k=self.sample_size)
                observations, actions, rewards, dones, next_obs, states, next_states = untangle(batch)
                qs = torch.zeros((len(self.agents), len(observations))).to(device)
                next_qs = torch.zeros((len(self.agents), len(observations))).to(device)
                for idx, agent in enumerate(self.agents):
                    agent_actions = [a[agent] for a in actions]
                    observation = torch.from_numpy(np.stack([o[agent] for o in observations])).float().to(device)
                    qs[idx, :] = self.agents[agent].net(observation)[range(len(observations)), agent_actions]
                    next_o = torch.from_numpy(np.stack([o[agent] for o in next_obs])).float().to(device)
                    next_qs[idx, :] = self.agents[agent].target_net(next_o).max(dim=1)[0].detach()

                if self.use_mixer:
                    rewards = rewards[0, :]     # rewards and dones are common for all agents, (in this environment)
                    dones = dones[0, :]         # thus this just reduces size of matrix to vector
                    states = states[:, 0, :]    # => fully cooperative setting, all agents share a common reward function
                    next_states = next_states[:, 0, :]
                    qs = self.mixer(qs.T, states) # = Qtot
                    next_qs = self.target_mixer(next_qs.T, next_states).detach()

                q_target = rewards + self.gamma * (1 - dones) * next_qs
                loss = F.mse_loss(qs, q_target)

                self.optimizer.zero_grad()
                loss.backward()
                self.optimizer.step()

                cum_loss += loss.item()

            # log statistics
            self.log(indx, n_iters, cum_loss)

            # sync networks
            if indx % self.sync_rate == 0:
                for agent in self.agents.values():
                    agent.sync()
                    agent.save(self.rand_idx)
                if self.use_mixer:
                    self.target_mixer.load_state_dict(self.mixer.state_dict())

    def generate_episode(self, render=False, train=True):
        self.env.reset()
        episode = {agent: [] for agent in self.env.agents}
        done = False
        for agent in self.env.agent_iter():
            if render:
                self.env.render()
            observation, reward, done, _ =  self.env.last()
            obs = observation['obs']
            mask = observation['action_mask']
            state = self.env.state()
            # set observation, done and reward as next_obs of previous step
            if episode[agent]:
                episode[agent][-1].next_obs = obs
                episode[agent][-1].next_mask = mask
                episode[agent][-1].next_state = state
                episode[agent][-1].reward = reward
                episode[agent][-1].done = done
            action = None if done else self.agents[agent].get_action(obs, mask, epsilon=self.epsilon if train else 0.0)
            self.env.step(action)
            episode[agent].append(EpisodeStep(observation, mask, action, None, None, None, None, state, None))
            if train:
                self.epsilon = max(self.epsilon-self.eps_decay, self.eps_min)
        return transform_episode(episode)

    def log(self, indx, losses):
        avg_rwd, std_rwd, avg_length = self.eval(self.n_evals)
        for agent in self.learners:
            avg_loss, std_loss = np.mean(losses[agent]), np.std(losses[agent])
            if self.verbose:
                print(f"{indx} - {agent:11s}: loss = {avg_loss:5.4f}, avg reward = {avg_rwd[agent]:5.4f}")
            self.writers[agent].add_scalar('avg_loss', avg_loss, indx)
            self.writers[agent].add_scalar('avg_reward', avg_rwd[agent], indx)
            self.writers[agent].add_scalar('avg_length', avg_length, indx)

    def eval(self, n):
        rewards = {agent: [] for agent in self.agents}
        lengths = []
        for _ in range(n):
            episode = self.generate_episode(train=False)
            for agent in self.agents:
                rewards[agent].append(np.sum([step[agent].reward for step in episode[:-1]]))
            lengths.append(len(episode))
        means, stds = {}, {}
        for agent in self.agents:
            means[agent] = np.mean(rewards[agent]) # TODO: better solution (eg. divide by initial reward)
            stds[agent]  = np.std(rewards[agent])

        return means, stds, np.mean(lengths)

class DQNAgent:
    def __init__(self, name, actions, net):
        self.name = name
        self.actions = actions
        self.net = net
        self.target_net = copy.deepcopy(self.net)
        self.epsilon = 1.0

    def get_action(self, observation, action_mask, epsilon=0.0):
        if np.random.rand() < epsilon:
            p = action_mask/sum(action_mask)
            action = np.random.choice(range(len(action_mask)), p=p)
        else:
            with torch.no_grad():
                qs = self.net(torch.from_numpy(observation).float().to(device))
                qs[action_mask == 0.0] = -np.infty # set all invalid actions to lowest q_value possible
                action = qs.argmax(axis=0).item()
        return action

    def sync(self):
        self.target_net.load_state_dict(self.net.state_dict())

    def save(self, rand_idx):
        filename = f"{rand_idx}_{self.name}"
        torch.save(self.net.state_dict(), './nets/qmix/'+filename)

    def load(self, rand_idx):
        filename = f"{rand_idx}_{self.name}"
        self.net.load_state_dict(torch.load('./nets/qmix/'+filename))
        self.net.to(device)

class QMixer(nn.Module):
    def __init__(self, n_agents, state_dim, layer_size=16):
        super().__init__()
        self.n_agents = n_agents
        self.layer_size = layer_size
        self.W1_layer = nn.Linear(state_dim, n_agents*layer_size)
        self.W2_layer = nn.Linear(state_dim, layer_size)
        self.b1_layer = nn.Linear(state_dim, layer_size)
        self.b2_layer = nn.Linear(state_dim, 1)
        #self.W1_layer = nn.Linear(state_dim, n_agents*1)

    def forward(self, qs, state):
        W1 = torch.abs(self.W1_layer(state))
        W1 = W1.reshape(-1, self.layer_size, self.n_agents)
        W2 = torch.abs(self.W2_layer(state))
        W2 = W2.reshape(-1, 1, self.layer_size)
        b1 = self.b1_layer(state)
        b2 = torch.relu(self.b2_layer(state))

        x = F.elu(torch.bmm(W1, qs[:, :, None]) + b1[:,:, None])
        Q = torch.bmm(W2, x).squeeze() + b2
        return Q

    def forward_(self, qs, state):
        "Alternative - testing only"
        W1 = torch.abs(self.W1_layer(state))
        W1 = W1.reshape(-1, 1, self.n_agents)

        Q = torch.bmm(W1, qs[:, :, None]).squeeze()
        return Q

def train(args):
    runner = QMixRunner(episode_length=int(args.length),
                        buffer_size=int(args.buffer_size),
                        n_batches=int(args.n_batches),
                        sample_size=int(args.sample_size), # 64
                        lr=float(args.lr),
                        gamma=float(args.gamma),
                        sync_rate=int(args.sync_rate),
                        layers=[32, 32],
                        eps_steps=int(args.eps_steps),
                        eps_min=0.1,
                        use_mixer=args.use_mixer,
                        n_agents=int(args.n_agents),
                        mixer_layer_size=int(args.mixer_layer_size),
    )
    print(runner)
    runner.writer.add_text('parameters', runner.__str__())
    runner.writer.add_text('source', inspect.getsource(sys.modules[__name__])) # write source code to tensorboard - usefull for experiment tracking
    runner.run(n_iters=int(args.n_iters))

    rand_idx = random.randint(0, 100000)
    for agent in runner.agents.values():
        agent.save(runner.rand_idx)
    print(f'--- Saved with id {rand_idx} ---')
    return runner

def demo():
    """Keep default parameters but:
    * gamma = 0.9
    * learning_rate = 0.001
    * sync_rate = 100
    Gives good result for spread_v2 after 200 iterations
    """

    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--lr", default=0.001)
    parser.add_argument("--gamma", default=0.9)
    parser.add_argument("--sync_rate", default=100)
    parser.add_argument("--n_iters", default=5000)
    parser.add_argument("--buffer_size", default=2048)
    parser.add_argument("--n_batches", default=64)
    parser.add_argument("--sample_size", default=128)
    parser.add_argument("--length", default=100) # episode_length
    parser.add_argument("--eps_steps", default=int(5e6)) # how many steps before epsilon = eps_min
    parser.add_argument('--use_mixer', dest='use_mixer', action='store_true')
    parser.add_argument('--no_mixer', dest='use_mixer', action='store_false')
    parser.set_defaults(use_mixer=True)
    parser.add_argument("--mixer_layer_size", default=16)
    parser.add_argument("--n_agents", default=2)
    args = parser.parse_args()

    runner = train(args)
    while True:
        runner.generate_episode(train=False, render=True)

if __name__ == '__main__':
    demo()


#!/usr/bin/python3
# -*-coding:utf-8-*-
import csv
import os
import sys

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

script_path = os.path.dirname(os.path.realpath(__file__))
sys.path.append(script_path + '/..')

from dictionary.utils import get_end_with_jongsung

engine = None
db_session = None
base_dir = os.path.dirname(os.path.realpath(__file__))
target_dir = base_dir + '/../../seed'

def connect_db(db_url):
    global engine
    global db_session

    print('connect to ' + db_url)
    engine = create_engine(db_url)
    Session = sessionmaker(bind=engine)
    db_session = Session()
    return db_session

def generate_dictionary_csv_files():
    names = get_pos_names()
    for name in names:
        rows = select_pos_rows(name)
        create_dictionary_csv_file(rows, name)

    names = get_class_names()
    for name in names:
        rows = select_class_rows(name)
        create_dictionary_csv_file(rows, name)

def get_pos_names():
    output = []
    sql = """
    SELECT distinct(pos) FROM lexicon_2_1 WHERE class IS NULL
    """
    rows = db_session.execute(sql).fetchall()
    for row in rows:
        pos = row['pos']
        output.append(pos)
    return output

def get_class_names():
    output = []
    sql = """
    SELECT distinct(class) FROM lexicon_2_1 WHERE class IS NOT NULL
    """
    rows = db_session.execute(sql).fetchall()
    for row in rows:
        pos = row['class']
        output.append(pos)
    return output

def select_pos_rows(pos):
    sql = """
    SELECT  surface, pos, semantic_class, `read`, `type`, start_pos, end_pos,
        expression
    FROM lexicon_2_1
    WHERE pos=:pos AND class IS NULL AND is_available = 1
    ORDER BY surface, pos, semantic_class ASC
    """
    rows = db_session.execute(sql, {'pos': pos}).fetchall()
    return rows

def select_class_rows(class_name):
    sql = """
    SELECT  surface, pos, semantic_class, `read`, `type`, start_pos, end_pos,
        expression
    FROM lexicon_2_1
    WHERE class=:cls AND is_available = 1
    ORDER BY surface, pos, semantic_class ASC
    """
    rows = db_session.execute(sql, {'cls': class_name}).fetchall()
    return rows

def create_dictionary_csv_file(rows, file_name):
    file_path = target_dir + '/' + file_name + '.csv'
    with open(file_path, 'w') as csv_file:
        writer = csv.writer(csv_file, 'unix', quoting=csv.QUOTE_MINIMAL)
        for row in rows:
            write_dic(writer, row)

def write_dic(writer, row):
    writer.writerow([row['surface'],
                     '0', '0', '0',
                     row['pos'],
                     row['semantic_class'],
                     get_end_with_jongsung(row['read']),
                     row['read'],
                     row['type'],
                     row['start_pos'],
                     row['end_pos'],
                     row['expression']])

def main():
    # db_url = 'sqlite:////home/kayden/eunjeon/eunjeon_lexicon_2_1.sqlite'
    db_url = input('DB URL (ex: sqlite:///...):')
    connect_db(db_url)
    generate_dictionary_csv_files()

if __name__ == '__main__':
    main()


from typing import List, Tuple

import ethereum.frontier.trie as normal_trie
import ethereum_optimized.trie as optimized_trie
from ethereum.base_types import Bytes

operations: List[List[Tuple[Bytes, Bytes]]] = [
    [],  # Empty Trie
    [(b"1234", b"foo")],  # Add a leaf
    [(b"1234", b"bar")],  # Change the value of a leaf
    [(b"1234", b"")],  # Delete a leaf
    [(b"abcde", b"foo"), (b"abcdef", b"foo")],  # Make an extension node
    [(b"abc", b"foo")],  # Split an extension node
    [(b"abcde", b"")],  # Merge an extension and leaf
    [(b"abcde1", b"foo"), (b"abcde2", b"")],  # Split a leaf node
    [  # Delete all subnodes of an extension node but not the value at the node
        (b"abcde1", b""),
        (b"abcde2", b""),
        (b"abcdef", b""),
    ],
    [(b"abcde", b"")],
    [(b"abcde1", b"foo"), (b"abcde2", b"")],
    [  # Delete all subnodes of an extension node and the value at the node
        (b"abcde1", b""),
        (b"abcde2", b""),
        (b"abcde", b""),
    ],
    [(b"a", b"foo"), (b"aaa", b"foo"), (b"aaaa", b"foo")],
    [(b"a", b""), (b"aa", b"foo")],
    [(b"\x00\x00\x00\x00", b"foo"), (b"\x00\x00\x00\x01", b"foo")],
    [(b"\x00\x00\x00\x10", b"foo")],
    [(b"a", b"foo"), (b"b", b"foo"), (b"c", b"foo"), (b"d", b"foo")],
]

def test_trie() -> None:
    trie_normal = normal_trie.Trie(False, b"")
    trie_optimized = optimized_trie.Trie(False, b"")
    for insert_list in operations:
        for (key, value) in insert_list:
            normal_trie.trie_set(trie_normal, key, value)
            optimized_trie.trie_set(trie_optimized, key, value)
        assert normal_trie.root(trie_normal) == optimized_trie.root(
            trie_optimized
        )


"""
tag: 数组 双指针
给你两个有序整数数组nums1 和 nums2，请你将 nums2 合并到nums1中，使 nums1 成为一个有序数组。

初始化nums1 和 nums2 的元素数量分别为m 和 n 。你可以假设nums1 的空间大小等于m + n，这样它就有足够的空间保存来自 nums2 的元素。

示例 1：

输入：nums1 = [1,2,3,0,0,0], m = 3, nums2 = [2,5,6], n = 3
输出：[1,2,2,3,5,6]
示例 2：

输入：nums1 = [1], m = 1, nums2 = [], n = 0
输出：[1]

提示：

nums1.length == m + n
nums2.length == n
0 <= m, n <= 200
1 <= m + n <= 200
-109 <= nums1[i], nums2[i] <= 109

来源：力扣（LeetCode）
链接：https://leetcode-cn.com/problems/merge-sorted-array
著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
"""
from typing import List

class Solution:
    # 还有一种，把nums2贴到nums1后面然后排序
    def merge(self, nums1: List[int], m: int, nums2: List[int], n: int) -> None:
        """
        Do not return anything, modify nums1 in-place instead.
        """
        tmp = []
        i, j = 0, 0
        while i < m or j < n:
            if i >= m:
                tmp.append(nums2[j])
                j += 1
            elif j >= n:
                tmp.append(nums1[i])
                i += 1
            else:
                n1, n2 = nums1[i], nums2[j]
                if n1 < n2:
                    tmp.append(n1)
                    i += 1
                else:
                    tmp.append(n2)
                    j += 1
        for _ in range(m + n):
            nums1[_] = tmp[_]

    def merge2(self, nums1: List[int], m: int, nums2: List[int], n: int) -> None:
        """
        利用nums1的后部，双指针，从后往前移动，把大的值放在nums1的尾部
        Do not return anything, modify nums1 in-place instead.
        """
        i, j = m - 1, n - 1
        index = m + n - 1
        while i >= 0 or j >= 0:
            if i < 0:
                nums1[index] = nums2[j]
                j -= 1
                index -= 1
            elif j < 0:
                i -= 1
            else:
                n1, n2 = nums1[i], nums2[j]
                if n1 > n2:
                    nums1[index] = n1
                    i -= 1
                    index -= 1
                else:
                    nums1[index] = n2
                    j -= 1
                    index -= 1

if __name__ == '__main__':
    # nums1 = [1, 2, 3, 0, 0, 0]
    nums1 = [1]
    m = 1
    # nums2 = [2, 5, 6]
    nums2 = []
    n = 0
    s = Solution()
    s.merge2(nums1, m, nums2, n)
    print(nums1)


from time import time
from src.king import king
from src.queen import queen
import os
import json

class input_handler:
    def input(self, symbol, values, king, townhall, hut_1, hut_2, hut_3, hut_4, wall, cannon, troop, spell, state, archer, balloon, queen, wizzard):
        if(symbol != ""):
            timei = time() - values.start_time
            dict = {"time": timei, "symbol": symbol}
            values.replaydata.append(dict)
            if(symbol == 'q' or symbol == 'Q'):
                os.system('tput reset')
                if(state == 'g'):
                    if(values.leveli == 0):
                        with open('./replays/history.txt', 'a') as k:
                            song = 1
                        with open('./replays/history.txt', 'r') as f:
                            try:
                                s = json.load(f)
                            except:
                                s = []
                        k = []
                        x = {"korq": values.korq}
                        k.append(x)
                        k.append(values.replaydata)
                        s.append(k)
                        with open('./replays/history.txt', 'w') as g:
                            json.dump(s, g,
                                      ensure_ascii=False, indent=4)
                    else:
                        with open('./replays/history.txt', 'a') as k:
                            song = 1
                        with open('./replays/history.txt', 'r') as f:
                            try:
                                s = json.load(f)
                            except:
                                s = []
                        s[len(s) - 1].append(values.replaydata)
                        with open('./replays/history.txt', 'w') as g:
                            json.dump(s, g,
                                      ensure_ascii=False, indent=4)
                exit()
        if(values.korq == 'k'):
            if(symbol == ' '):
                king.king_hit(values, townhall, hut_1, hut_2,
                              hut_3, hut_4, wall, cannon, wizzard)
            if(symbol == 'w' or symbol == 'W'):
                king.king_up(values, cannon)
            if(symbol == 's' or symbol == 'S'):
                king.king_down(values, cannon)
            if(symbol == 'a' or symbol == 'A'):
                king.king_left(values, cannon)
            if(symbol == 'd' or symbol == 'D'):
                king.king_right(values, cannon)

        if(values.korq == 'q'):
            if(symbol == ' '):
                values.queen_range = 8
                values.queen_side = 5
                queen.queen_hit(values, townhall, hut_1, hut_2,
                                hut_3, hut_4, wall, cannon, wizzard)
            if(symbol == 'b'):
                values.q_b = 1
                values.queen_range = 16
                values.queen_side = 9
                values.q_time = time()
                # queen.queen_hit(values, townhall, hut_1, hut_2,
                #                 hut_3, hut_4, wall, cannon, wizzard)
            if(symbol == 'w' or symbol == 'W'):
                queen.queen_up(values, cannon)
            if(symbol == 's' or symbol == 'S'):
                queen.queen_down(values, cannon)
            if(symbol == 'a' or symbol == 'A'):
                queen.queen_left(values, cannon)
            if(symbol == 'd' or symbol == 'D'):
                queen.queen_right(values, cannon)

        if(symbol == 'f' or symbol == 'F'):
            troop.spawn(values, symbol)
        if(symbol == 'g' or symbol == 'G'):
            troop.spawn(values, symbol)
        if(symbol == 'h' or symbol == 'H'):
            troop.spawn(values, symbol)
        if(symbol == 'j' or symbol == 'J'):
            archer.spawn(values, symbol)
        if(symbol == 'k' or symbol == 'K'):
            archer.spawn(values, symbol)
        if(symbol == 'l' or symbol == 'L'):
            archer.spawn(values, symbol)
        if(symbol == 't' or symbol == 'T'):
            balloon.spawn(values, symbol)
        if(symbol == 'y' or symbol == 'Y'):
            balloon.spawn(values, symbol)
        if(symbol == 'u' or symbol == 'U'):
            balloon.spawn(values, symbol)
        if(symbol == 'e' or symbol == 'E'):
            spell.healspell(values)
        if(symbol == 'r' or symbol == 'R'):
            if(values.rage_active == 0):
                spell.ragespell(values)


import SqlInjectionIndetifier
import pandas as pd

df = pd.read_csv('dataNew.csv')

#df['SQLInjection'] = np.where(SqlInjectionIndetifier.detectRFI(df['Endpoint']), "SQLInjection", "NO")
print(df.info())
sqlInjRFI = []
sqliInj =[]
webShell =[]
xss =[]
for index, row in df.iterrows():
    sqlInjRFI.append(SqlInjectionIndetifier.detectRFI(row['Endpoint']))
    sqliInj.append(SqlInjectionIndetifier.detectSQLi(row['Endpoint']))
    webShell.append(SqlInjectionIndetifier.detectWebShell(row['Endpoint']))
    xss.append(SqlInjectionIndetifier.detectSXX(row['Endpoint']))

df['SqliInjection'] = sqliInj
df['Webshell'] = webShell
df['SQLInjectionRFI'] = sqlInjRFI
df['XSS'] = xss
print(df.info())
print(df.head())
df.to_csv(r'dataSqlInj.csv')

#df['SQLInjection'] = df.apply(lambda row: SqlInjectionIndetifier.detectRFI(row['Endpoint']))


"""
This script will add a software to a server.
Unlike the server to OS server, it will not remove any existing connection.
"""
import argparse
import logging
import sys
from lib import my_env
from lib import murcsstore
from lib import murcsrest

if __name__ == "__main__":
    # Configure command line arguments
    parser = argparse.ArgumentParser(
        description="Connect a server to software (database)"
    )
    parser.add_argument('-s', '--hostName', type=str, required=True,
                        help='Please provide hostName to identify the server.')
    parser.add_argument('-o', '--softId', type=str, required=True,
                        help='Please provide softId of the software to add.')
    parser.add_argument('-i', '--instance', type=str, required=False,
                        help='Optionally provide the instance schema. If schema used, then | is delimiter')
    args = parser.parse_args()
    cfg = my_env.init_env("bellavista", __file__)
    mdb = murcsstore.Murcs(cfg)
    r = murcsrest.MurcsRest(cfg)
    logging.info("Arguments: {a}".format(a=args))

    # Get server record
    hostName = args.hostName
    server_rec = mdb.get_server(hostName)
    if not server_rec:
        sys.exit("Server {h} not found.".format(h=hostName))
    serverId = server_rec["serverId"]

    # Get SW record, to know the softSubType. This will be used for instType attribute from SWInstance
    soft_rec = mdb.get_soft(args.softId)
    instType = soft_rec["softSubType"]

    # Then add requested OS
    params = dict(
        instType=instType
    )
    if args.instance:
        params["instSubType"] = args.instance
    r.add_softInst_calc(args.softId, serverId, **params)
    mdb.close()


from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from fastapi_cache import FastAPICache
from fastapi_cache.backends.redis import RedisBackend
from redis import asyncio as aioredis
from auth.base_config import auth_backend, fastapi_users
from auth.schemas import UserRead, UserCreate
from config import REDIS_HOST, REDIS_PORT
from operations.router import router as router_operation
from tasks.router import router as router_tasks
from pages.router import router as router_pages
from chat.router import router as router_chat

app = FastAPI(
    title="Trading App"
)

app.mount("/static", StaticFiles(directory="static"), name="static")

app.include_router(
    fastapi_users.get_auth_router(auth_backend),
    prefix="/auth",
    tags=["Auth"],
)

app.include_router(
    fastapi_users.get_register_router(UserRead, UserCreate),
    prefix="/auth",
    tags=["Auth"],
)

app.include_router(router_operation)
app.include_router(router_tasks)
app.include_router(router_pages)
app.include_router(router_chat)

async def startup_event():
    redis = aioredis.from_url(
        f"redis://{REDIS_HOST}:{REDIS_PORT}", encoding="utf8", decode_responses=True)
    FastAPICache.init(RedisBackend(redis), prefix="fastapi-cache")

origins = [
    "http://localhost:3000",
]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["GET", "POST", "OPTIONS", "DELETE", "PATCH", "PUT"],
    allow_headers=["Content-Type", "Set-Cookie", "Access-Control-Allow-Headers", "Access-Control-Allow-Origin",
                   "Authorization"],
)

# Регистрируем событие запуска приложения
app.add_event_handler("startup", startup_event)


def leer_archivos(file_name):
    print(f'intentas abrir este rchivo {file_name}')
    # Abrir open()
    #prosesar red/write
    #cerrar close()
    #with nos permite agrupar el trabajo con archivos
    with open(file_name, 'r') as file:
        lineas = file.readlines()
        for linea in lineas :
           print(linea, end='')
        #while(linea):
        #    print(linea)
         #   linea = file.readline()
    #print('archivo leido y serrado')

def agregar_equipo(file_name, equipo):
    with open(file_name, 'a') as file:
        file.write(f"\n{equipo}")
        print('equipo guardado ')

def eliminar_equipo(file_name, equipo):
    with open(file_name, 'r') as file:
        lista_equipos = file.readlines()
    try:
        lista_equipos.remove(equipo)
        print("equipo eliminado")
        with open(file_name, 'w') as file:
            file.writelines(lista_equipos)
    except ValueError:
        print("el equipo no existe")


from asyncio.log import logger
import datetime

from rest_framework.response import Response
from rest_framework.decorators import api_view
from django.utils.translation import gettext_lazy as _

from base.models import NFT
from django.contrib.auth.models import User
from base.serializer import NFT_Serializer

# Create your views here.

def Print(val):
    logger.error("*****************************************")
    logger.error(val)
    logger.error("*****************************************")

@api_view(['GET'])
def get_historical_data(request):
    # ! There is no symbol  in CSV so not implemented

    _from_date  = request.GET.get('from_date','')
    _to_date  = request.GET.get('to_date','')
    symbol  = request.GET.get('symbol','')

    from_date = datetime.datetime.strptime(_from_date, "%Y-%m-%d").date()
    to_date = datetime.datetime.strptime(_to_date, "%Y-%m-%d").date()

    # ! There is no symbol  in CSV so not implemented

    NFT_Obj = NFT.objects.filter(date__gt=from_date,date__lt=to_date)
    serializer = NFT_Serializer(NFT_Obj, many=True)
    return Response(serializer.data)

@api_view(['POST'])
def create_user(request):
    # We mostly not focusing on security.. because this is small project
    # We use email in frontend to make user use email as username
    data =request.data
    user = User.objects.create(
        username=data['username'],
        )
    user.set_password(data['password'])

    user.save()
    return Response(status=200)


import pandas as pd
import os
from get_dataframe import get_dataframe
from get_question_type import check
from matplotlib import pyplot as plt
import numpy as np

if __name__ == '__main__':
    directory = '../csv/'
    width = 0.2
    counter = 0

    files = ['EARL_question_wise_NEW_MODEL_lcquad.csv', 'FALCON_question_wise_lcquad.csv',
             'EARL_question_wise_NEW_MODEL_qald.csv', 'FALCON_question_wise_qald.csv']

    for file in files:
        counter += 1
        filename = os.fsdecode(file)
        if ('EARL' in filename):
            full_file = os.path.join(directory, filename)
            df_EARL = get_dataframe(full_file)
            df_EARL.dropna(inplace=True)
            df_EARL['question_type'] = df_EARL.apply(lambda row: check(row['params.question'], row['params.query_db']),
                                                     axis=1)
            avg_EARL = df_EARL.groupby('question_type').mean()
            count_df = df_EARL.groupby('question_type')['params.question'].count()
            avg_EARL = avg_EARL[['metrics.precision', 'metrics.recall', 'metrics.f1']]

        if ('FALCON' in filename):
            full_file = os.path.join(directory, filename)
            df_FALCON = get_dataframe(full_file)
            df_FALCON.dropna(inplace=True)
            df_FALCON['question_type'] = df_FALCON.apply(lambda row: check(row['params.question'], row['params.query_db']),
                                                     axis=1)
            avg_FACLON = df_FALCON.groupby('question_type').mean()
            count_df = df_FALCON.groupby('question_type')['params.question'].count()
            avg_FACLON = avg_FACLON[['metrics.precision', 'metrics.recall', 'metrics.f1']]

        if counter % 2 == 0:
            x = np.arange(avg_EARL.shape[0])
            fig = plt.figure(figsize=(10, 8))
            ax1 = fig.add_subplot(111)
            ax2 = ax1.twiny()
            ax1.bar(x-0.5*width, avg_EARL['metrics.f1'], width, color='cyan')
            ax1.bar(x+0.5*width, avg_FACLON['metrics.f1'], width, color='green')
            ax1.set_xticks(x, avg_EARL.index)
            ax1.set_xlabel(r"Fragetypen", fontsize=18)
            ax1.set_ylabel('F1-Score', fontsize=18)
            ax1.legend(['EARL', 'FALCON'], fontsize=18)
            if 'lcquad' in filename:
                title_name = 'LC-QuAD 1.0'
            if 'qald' in filename:
                title_name = 'QALD 9'
            # ax1.set_title(f'Vergleich Performance der Systeme für verschiedene \nFragetypen auf {title_name}')
            ax1.set_ylim(0,1)
            ax2.set_xlim(ax1.get_xlim())
            ax2.set_xticks(x, count_df)
            ax2.set_xticklabels(count_df)
            ax2.set_xlabel(r"Anzahl Fragetypen", fontsize=18)
            # ax2.legend(fontsize=18)
            ax1.tick_params(labelsize=18)
            ax2.tick_params(labelsize=18)
            plt.savefig(f'../bar plots/{title_name}.png')
            plt.close()


import socket
from _thread import *
import sys

class Server:
    def __init__(self, ip, port):
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server = ip
        port = port
        self.server_ip = socket.gethostbyname(server)
        try:
            self.socket.bind((server, port))

        except socket.error as e:
            print(str(e))

        self.positions = {}
        self.projectiles = {}
        self.current_id = 0

    def add_player(self):
        """Add player to Server and give it id. Returns the id."""
        self.positions[self.current_id] = str(server.current_id) + ":0:0:0"
        self.current_id += 1

        return self.current_id - 1

    def parse_data(self, data, player_id):
        pair = data.split('=')
        data_id = int(pair[0])

        if data_id == 0:
            self.parse_player_data(pair[1], player_id)

        elif data_id == 1:
            self.parse_projectile_data(pair[1], player_id)

    def parse_player_data(self, data, player_id):
        self.positions[player_id] = data

    def parse_projectile_data(self, data, player_id):
        for i in range(len(self.positions)):
            self.projectiles[i] = data

    def convert_player_data(self):
        """Converts player information stored in server class to data string."""
        data = '0='
        for id in self.positions:
            data += self.positions[id]
            data += '/'
        data = data[:-1]
        data += '!'

        return data

    def convert_projectile_data(self, player_id):
        """Converts projectile information stored in server class to data string."""
        data = '1='
        if player_id in self.projectiles and self.projectiles[player_id] != None:
            data += self.projectiles[player_id]
            data += '!'
            self.projectiles[player_id] = None

            return data
        else:
            return None

def threaded_client(conn):
    """"""
    global server
    conn.send(str.encode(str(server.current_id)))
    player_id = server.add_player()

    while True:
        try:
            data = conn.recv(2048).decode()
            if not data:
                break
            else:
                server.parse_data(data, player_id)

                player_data = server.convert_player_data()
                projectile_data = server.convert_projectile_data(player_id)

                #print("Data: ", player_data, " and ", projectile_data)
                conn.sendall(str.encode(player_data))
                if projectile_data != None:
                    conn.sendall(str.encode(projectile_data))
        except:
            break

    print("Connection Closed")
    conn.close()

server = Server("192.168.10.61", 5555)

server.socket.listen(4)
print("Waiting for a connections")

while True:
    conn, addr = server.socket.accept()
    print("Connected to: ", addr)

    start_new_thread(threaded_client, (conn,))