from base_test import BaseTest
from template import Template, TemplateVariable, ConvoTemplate
from unittest.mock import Mock
from difflib import SequenceMatcher
from baseline import Baseline
from util import scrubbed_equals

class TemplateTest(BaseTest):
    def setUp(self):
        self.c = {
            'unrecognized_port': 0.5,
            'nonmatching_convo': 0,
            'max_text_len': 1024,
            'score_length_weight': 0.3,
            'score_vlen_weight': 0.3,
            'score_growth_weight': 0.2,
            'score_shrink_weight': 0.2,
            'score_ratio_weight': 0.3,
            'score_specials_weight': 1.0,
            'training_treshold': 0.4,
            'detection_treshold':  0.1,
            'special_strings': Baseline.SPECIAL_STRINGS
        }

    def check_variables(self, a,b,outcome):
        self.assertEqual([TemplateVariable(*outcome)], Template(a).find_variables(b))

    def test_variables(self):
        self.check_variables('abcd', 'axcd', (1,1,1,1))
        self.check_variables('abcd', 'axycd', (1,1,1,2))
        self.check_variables('abcd', 'ad', (1,2,0,2))
        self.assertEqual([], Template('abcd').find_variables('abcd'))

    def test_similarity(self):
        # without variables
        # replace
        self.assertBetween(0.5, 0.9, Template('abcd',[], self.c).similarity('axcd'))
        # insert
        self.assertBetween(0.5, 0.9, Template('abcd',[], self.c).similarity('abxcd'))
        # delete
        self.assertBetween(0.5, 0.9, Template('abcd',[], self.c).similarity('acd'))
        # no change
        self.assertEqual(1, Template('abcd',[], self.c).similarity('abcd'))

        # with 1 variable
        # replace
        self.assertEqual(1, Template('abcd',[TemplateVariable(1,1,1,1)], self.c).similarity('axcd'))
        # insert
        self.assertBetween(0.5, 0.9, Template('abcd',[TemplateVariable(1,1,1,1)], self.c).similarity('axycd'))
        self.assertEqual(1, Template('abcd',[TemplateVariable(1,1,1,2)], self.c).similarity('axycd'))
        # delete
        self.assertEqual(1, Template('abcd',[TemplateVariable(1,1,0,1)], self.c).similarity('acd'))
        # This testcase does not seem relevant. Why should we care about deletion?
        #self.assertBetween(0.5, 0.9, Template('abcd',[TemplateVariable(1,1,1,1)], self.c).similarity('acd'))
        # insert
        self.assertEqual(1, Template('abcd',[TemplateVariable(1,1,0,5)], self.c).similarity('awxyzcd'))
        self.assertBetween(0.3, 0.9, Template('abcd',[TemplateVariable(1,1,0,3)], self.c).similarity('awxyzcd'))
        # no change
        self.assertEqual(1, Template('abcd',[TemplateVariable(1,1,1,1)], self.c).similarity('abcd'))

    def test_practical_similarity(self):
        a = (
        "set 093a802b6de10157270839be4e9257f5 53276566323039643166343731313839323666363139653438303333633061333035270a70300a2e\nget 093a802b6de10157270839be4e9257f5\nquit\n",
        "OK\n53276566323039643166343731313839323666363139653438303333633061333035270a70300a2e")
        b = (
        "set 34b4a59dbf92f6657fe944dc10db35d4 53276436613062323565386666613535386665336233303332303631356336383762270a70300a2e\nget 34b4a59dbf92f6657fe944dc10db35d4\nquit\n",
        "OK\n53276436613062323565386666613535386665336233303332303631356336383762270a70300a2e")
        self.assertBetween(0.4, 0.7, Template(a[0], [], self.c).similarity(b[0]))
        self.assertBetween(0.4, 0.7, Template(a[1], [], self.c).similarity(b[1]))

    def test_var_expansion(self):
        pass
        self.assertEqual(1, Template('xabcdx', [TemplateVariable(1, 4, 4, 4)], self.c).similarity('xdydyx'))
        #self.assertEqual(1, Template('xabcdyefghz', [TemplateVariable(1, 4, 4, 4)], self.c).similarity('xcayfgz'))

    def test_flag_similarity(self):
        a=b'FLAG{2CB2C3F4C4E07730DEDBBE5DF90F62D0}'
        b=b'FLAG{42DE58F5A636FF957623DB77DACFBBD4}'
        self.assertEqual(1, Template(a, [TemplateVariable(5, 32, 32, 32)], self.c).similarity(b))
        a=b'FLAG{2CB2C3F4C4E07730DEDBBE5DF90F62D0}'
        b=b'FLAG{352EE4908C1415B90D5971E91C2769A1}'
        self.assertEqual(1, Template(a, [TemplateVariable(5, 32, 32, 32)], self.c).similarity(b))

    def test_flag_detection(self):
        temp = Template('abcdefgh', [TemplateVariable(2,4,4,4)], self.c)
        self.assertEqual(0, temp.similarity("abFLAGgh"))

    def test_show_template(self):
        t = Template(b'abcdefghijklmnopqrstuvwxyz',[TemplateVariable(1,4,1,4), TemplateVariable(9,1,1,4) ], self.c)
        self.assertEqual('a\x1b[32mbcde\x1b[0mfghi\x1b[32mj\x1b[0mklmnopqrstuvwxyz', t.show())
        t = Template(b'abcdefghijklmnopqrstuvwxyz',[TemplateVariable(1,4,1,4), TemplateVariable(9,0,1,4) ], self.c)
        self.assertEqual('a\x1b[32mbcde\x1b[0mfghi\x1b[31m*\x1b[0mjklmnopqrstuvwxyz', t.show())

    def check_update(self, a):
        t = Template('abcdefghijklmnopqrstuvwxyz',[TemplateVariable(1,4,1,4), TemplateVariable(9,1,1,4) ], self.c)
        self.assertLess(t.similarity(a), 1)
        t.update(a)
        self.assertEqual(1, t.similarity(a))

    def test_update(self):
        self.check_update('')
        self.check_update('xyzabcdefghijklmnopqrstuvwxyz')
        self.check_update('xyzabcdefghijklmnopqrstuvwxyzxyz')
        self.check_update('defghijklmnopqrstuvw')
        self.check_update('abcdefghixyzxyklmnopqrstuvwxyz')

    def mock_var(self, pos=0, len=0, fits=True):
        mock_var = Mock()
        mock_var.fits.return_value = fits
        mock_var.pos= pos
        mock_var.len = len
        mock_var.__lt__ = lambda x,y: True
        return mock_var

    def test_var_fit(self):
        self.assertTrue(Template("abc", [self.mock_var(fits=True)], self.c).fit_variable('good'))
        self.assertTrue(Template("abc", [self.mock_var(fits=False), self.mock_var(fits=True)], self.c).fit_variable('good'))
        self.assertFalse(Template("abc", [self.mock_var(fits=False)], self.c).fit_variable('bad'))

    def check_consolidate(self, invars, checkvars):
        invars = [TemplateVariable(*v) for v in invars]
        checkvars = [TemplateVariable(*v) for v in checkvars]
        t = Template("abcdefghijklmnopqrstuvwxyz", invars, self.c)
        self.assertEqual(checkvars, t.variables)

    def test_consolidate(self):
        # variable inclusion
        self.check_consolidate([(1, 5, 5, 5), (2, 4, 4, 4)], [(1, 5, 5, 5)])
        # variable with shorter min_len
        self.check_consolidate([(1, 5, 5, 5), (2, 4, 1, 4)], [(1, 5, 2, 5)])
        # variable with longer max_len
        self.check_consolidate([(1, 5, 5, 5), (2, 4, 4, 5)], [(1, 5, 5, 6)])
        # non contiguous variable
        self.check_consolidate([(1, 5, 5, 5), (7, 1, 1, 1)], [(1, 5, 5, 5), (7, 1, 1, 1)])

class ConvoTemplateTest(BaseTest):
    def setUp(self):
        self.c = {
            'unrecognized_port': 0.5,
            'nonmatching_convo': 0,
            'max_text_len': 1024,
            'score_length_weight': 0.3,
            'score_vlen_weight': 0.3,
            'score_growth_weight': 0.2,
            'score_shrink_weight': 0.2,
            'score_ratio_weight': 0.3,
            'score_specials_weight': 1.0,
            'training_treshold': 0.4,
            'detection_treshold':  0.1,
            'special_strings': Baseline.SPECIAL_STRINGS
        }

    def check_similarity(self, sent, recv, result):
        sent_mock = Mock()
        sent_mock.similarity.return_value = sent
        recv_mock = Mock()
        recv_mock.similarity.return_value = recv
        c = ConvoTemplate("","", self.c)
        c.recv = recv_mock
        c.sent = sent_mock
        self.assertEqual(result, c.similarity('a', 'b'))
        sent_mock.similarity.assert_called_once_with('a')
        recv_mock.similarity.assert_called_once_with('b')

    def test_similarity(self):
        self.check_similarity(2, 0.5, 0.5)
        self.check_similarity(2, 0, 0)
        self.check_similarity(0, 2, 0)
        self.check_similarity(0, 0, 0)

    def test_update(self):
        sent_mock = Mock()
        recv_mock = Mock()
        c = ConvoTemplate("", "", self.c)
        c.recv = recv_mock
        c.sent = sent_mock
        c.update('a', 'b')
        sent_mock.update.assert_called_once_with('a')
        recv_mock.update.assert_called_once_with('b')

class ScrubEqualsTest(BaseTest):
    def test_equals(self):
        self.assertTrue(scrubbed_equals(b"abc", b"abc"))
        self.assertFalse(scrubbed_equals(b"abc", b"abd"))
        self.assertTrue(scrubbed_equals(b"abcFLAG{0123AB}", b"abcFLAG{0123ABDAA}"))


from itertools import islice

test_input = """
199
200
208
210
200
207
240
269
260
263
"""

def count_increases(input: str, window: int = 1):
    lines = list(map(int, filter(lambda x: x != "", input.split("\n"))))
    cur = prev = 0
    counter = -1
    for i in range(len(lines) - window + 1):
        cur = sum(lines[i : i + window])
        if cur > prev:
            counter += 1
        prev = cur
    return counter

assert count_increases(test_input) == 7

assert count_increases(test_input, 3) == 5

with open("inputs/day1.txt", "r") as f:
    input = f.read()
    print(count_increases(input))
    print(count_increases(input, 3))


from typing import List

class Solution1:
    def reverse(self, nums, left, right):
        while left < right:
            nums[left], nums[right] = nums[right], nums[left]
            left += 1
            right -= 1

    def rotate(self, nums: [int], k: int) -> None:
        """
        Time O(n)
        Space O(1)
        k 需要先除以长度，计算余数，因为k可能超过数组的长度，会出现重复循环的旋转
        """
        k %= len(nums)

        self.reverse(nums, 0, len(nums) - 1)
        self.reverse(nums, 0, k - 1)
        self.reverse(nums, k, len(nums) - 1)

        return nums

class Solution2:
    def rotate(self, nums: List[int], k: int) -> None:
        """
        Time O(n * k) 每一次insert都要移动整个数组
        Space O(1)
        弹出尾巴，然后insert进开始
        """
        for i in range(k):
            element = nums.pop()
            nums.insert(0, element)

        return nums

s = Solution2()
print(s.rotate(nums=[1, 2, 3, 4, 5, 6, 7], k=3))


import random

def play_again():
    answer = input('Dou you want to play again "Yes or No": ').lower()
    if answer == 'yes':
        play_game()
        print('\n\n')
    else:
        print('Thank you for playing the game, Bye.')

def get_word():

    words = ['Mongolia', 'tetris', 'python', 'classmate', 'India', 'Nepal']
    return random.choice(words)

def play_game():
    alphabet = 'abcdefghijklmnopqrstuvwxyz'
    word = get_word()
    guessed = False
    letter_guessed = []
    tries = 3

    print(f'Hint: The word contains {len(word)} letters')
    print(len(word) * '_')

    while guessed == False and tries > 0:
        print(f'You have {tries} tries.')
        guess = input('Enter a letter or full word(if you got it).').lower()

        # case 1 -> Only 1 letter entered:
        if len(guess) == 1:
            if guess not in alphabet:
                print('You have not entered a letter')
            elif guess in letter_guessed:
                print('You have already guessed that letter before.')
            elif guess not in word:
                print('Sorry letter not in word.')
                letter_guessed.append(guess)
                tries -= 1
            elif guess in word:
            #     print('Well done, letter is in the word.')
                letter_guessed.append(guess)
            else:
                print('No idea why this line is printing.')

        # case 2 -> User enteres the full word:
        elif len(guess) == len(word):
            if guess == word:
                print('You got the word.')
                guessed = True
            else:
                print('Sorry you got the word wrong.')
                tries -= 1

        # case 1 -> user inputs the letter which is > than total number in the word:

        else:
            print('The entered guess is > than actual word.')

        status = ''
        if guessed == False:
            for letter in word:
                if letter in letter_guessed:
                    status += letter
                else:
                    status += '_'
            print(status)

        if status == word:
            print('Congraluation, You won!')
            guessed = True
        elif tries == 0:
            print('You have run out of guess.')

    play_again()

play_game()


import asyncio
import re
from collections.abc import Sequence

from pywebio.output import *
from pywebio.session import get_info

requirejs_config_tpl = """

"""

requirejs_tpl = """
%s

"""

def load_notebook(resources=None, verbose=False, hide_banner=False, load_timeout=5000):
    """加载 Bokeh 资源

    :param resources: 目前不支持自定义静态资源的链接
    :param verbose: 开启 Bokeh 日志 并显示 Bokeh 加载标签
    :param hide_banner: 不支持
    :param load_timeout: 不支持
    :return: None
    """
    from bokeh import __version__
    from bokeh.util.serialization import make_id

    js_gists = ["console.log('Load BokehJS complete.')"]

    html = requirejs_config_tpl.replace('__version__', __version__)
    if verbose:
        element_id = make_id()
        html += """

            Loading BokehJS ...

        """.format(element_id=element_id)

        js_gists.append(
            "document.getElementById({element_id}).innerHTML = 'Load BokehJS complete.'".format(element_id=element_id))

        js_gists.append('Bokeh.set_log_level("info");')
        js_gists.append("console.log('Set bokeh log level to INFO because you set `output_notebook(verbose=True)`')")

    put_html(requirejs_tpl % (html, '\n'.join(js_gists)), sanitize=False)

def show_doc(obj, state, notebook_handle):
    """Show a document of Bokeh

    :param obj:
    :param state:
    :param notebook_handle: 不支持
    :return:
    """
    from bokeh.embed import components

    script, div = components(obj, wrap_script=False)
    if isinstance(obj, Sequence):
        div = '\n'.join(div)
    elif isinstance(obj, dict):
        div = '\n'.join(div[k] for k in obj.keys())

    put_html(requirejs_tpl % (div, script), sanitize=False)

def show_app(app, state, notebook_url, port=0, **kw):
    """Show Bokeh applications

    :param app: A Bokeh Application to embed in PyWebIO.
    :param state: ** Unused **
    :param notebook_url:  ** Unused **
    :param port: Bokeh Server 端口
    :param kw: 传给 Bokeh Server 的额外参数
    """

    from bokeh.server.server import Server
    from bokeh.io.notebook import _origin_url, uuid4, curstate, _server_url

    from pywebio.platform.tornado import ioloop
    loop = ioloop()
    if loop is None:
        toast("Currently only supports showing bokeh application in Tornado backend",
              color='error', duration=0)
        return
    loop.make_current()
    asyncio.set_event_loop(loop.asyncio_loop)
    # loop = IOLoop.current()

    info = get_info()

    allow_websocket_origin = [info.server_host]
    if info.origin:
        allow_websocket_origin.append(_origin_url(info.origin))

    server = Server({"/": app}, io_loop=loop, port=port, allow_websocket_origin=allow_websocket_origin, **kw)

    server_id = uuid4().hex
    curstate().uuid_to_server[server_id] = server

    server.start()

    url = _server_url(info.server_host, server.port)

    from bokeh.embed import server_document
    script = server_document(url, resources=None)

    script = re.sub(r'',  # lgtm [py/bad-tag-filter]
                    r"""

                    """, script, flags=re.I)

    put_html(script, sanitize=False)

def try_install_bokeh_hook():
    """尝试安装bokeh支持"""
    try:
        from bokeh.io import install_notebook_hook
    except ImportError:
        return False

    install_notebook_hook('pywebio', load_notebook, show_doc, show_app)
    return True


import timeit

def count_pairs_with_difference(k, a):
    a_sort = sorted(a)
    count = 0
    x = timeit.timeit()
    length = len(a_sort)
    for i in range(0, length):
        required = i + k
        try:
            first_index = a.index(required)
            count += 1
            for item in range(first_index + 1, length):
                if item == required:
                    count += 1
                else:
                    continue
        except ValueError:
            continue

    print(count, x - timeit.timeit())
    return count

if __name__ == '__main__':
    count_pairs_with_difference(3, [1, 6, 8, 2, 4, 9, 12])


from multiprocessing.dummy import Pool
import binascii
import hashlib

BASE_OFFSET = 0xbc050000
DUMP_LENGTH = 0x1545b8 * 4

data = bytearray(DUMP_LENGTH)

def process_line(line):
    if(line.startswith("bc")):
        groups = line.split(" ")
        offset = int(groups[0].rstrip(":"), 16) - BASE_OFFSET
        line_bytes = b''
        for x in groups[1:5]:
            assert len(x) == 8
            b = binascii.unhexlify(x)
            #data was printed as little endian 32-bit numbers, so we need to reverse the byte order
            line_bytes += b[::-1]
        data[offset:offset + 16] = line_bytes

with open("putty.log","r") as fin:
    with Pool(12) as p:
        p.map(process_line, fin.readlines())

with open("firmware.bin","wb") as fout:
    #we read 8 extra bytes
    fout.write(data[:-8])


#!/usr/bin/env python3
import setuptools, os

with open("README.md", "r") as fh:
    long_description = fh.read()

with open("./staphb_toolkit/lib/VERSION",'r') as versionFile:
    version = versionFile.readline().strip()

setuptools.setup(
    name="staphb_toolkit",
    version=version,
    author="Kelsey Florek",
    author_email="kelsey.florek@slh.wisc.edu",
    description="A ToolKit of commonly used Public Health Bioinformatics Tools",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/StaPH-B/staphb_toolkit",
    packages=setuptools.find_packages(),
    package_data={
        'staphb_toolkit' : ['lib/VERSION'],
        '' : ['../requirements.txt','../workflows.json','../staphb_toolkit/config/docker.config','../staphb_toolkit/config/singularity.config','../staphb-tk']
    },
    entry_points={
        "console_scripts": [
            'staphb-tk = staphb_toolkit.toolkit_main:main'
        ]
    },
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
        "Operating System :: OS Independent",
    ],
    install_requires=[
        "spython>=0.0.73",
        "psutil>=5.6.3",
        "docker>=4.1.0",
        "pexpect>=4.8",
        "pyfiglet>=0.8.post1",
        "rich>=12.4.4"],
    python_requires='>=3.7',
)


def solution(skill, skill_trees):
    answer = 0
    for i in range(len(skill_trees)):
        index = []
        flag = 0
        for alpha in skill_trees[i]:
            if(alpha in skill):
                idx = skill.index(alpha)
                if(len(index)==0 and idx==0):
                    index.append(idx)
                elif(len(index)==0 and idx!=0):
                    flag = 1
                    break
                else:
                    if(max(index) + 1 == idx): # 작아야함
                        index.append(idx)
                    else:
                        flag = 1
                        break
            if(flag == 1):
                break
        if(flag == 0):
            answer += 1

    return answer

skill = "CBD"
# C, CB, CD, CBD
skill_trees = ["BACDE", "CBADF", "AECB", "BDA"]
print(solution(skill, skill_trees))


from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

DOCUMENTATION = r'''
---
module: zvm_clone_disk

short_description: z/VM Clone Disk Ansible Module

version_added: "0.0.1"

description: an Ansible module for z/VM virtual machine minidisks

options:
    name:
        description: this is the target who will own the newly cloned disk
        required: true
        type: str
    diskaddr:
        description: the address of the newly cloned disk in the targets configuration
        required: true
        type: str
    srcname:
        description: source virtual machine that owns the disk to be cloned
        required: true
        type: str
    srcaddr:
        description: source virtual machine disk address to be cloned
        required: true
        type: str
    diskalloc:
        description: how should extents for the new disk be chosen? Select by Region, by Volume, by Volume Group, or a specific device?
        type: str
        choices:
            - AUTOR
            - AUTOV
            - AUTOG
            - DEVNO
    diskloc:
        description: name of the region where diskalloc managed extents should get pulled from - the specific region, volume, or group.
        type: str
    diskmode:
        description: minidisk access mode
        type: str
        choices:
            - R
            - RR
            - W
            - WR
            - M
            - MR
            - MW
    diskrpw:
        description: read-access password to access the disk
        type: str
    diskwpw:
        description: write-access password to access the disk
        type: str
    diskmpw:
        description: multi-write-access password to access the disk
        type: str

seealso:
    - name: IBM z/VM SMAPI documentation
      description: Reference for SMAPI application developers
      link: https://www.ibm.com/docs/en/zvm/7.3?topic=reference-socket-application-programming-interfaces

author:
    - Jay Brenneman (@rjbrenn)
'''

EXAMPLES = r'''
# clone GOLDMAST user disk 250 to existing NEWGUY disk 100
- name: clone NEWGUY 100 disk
  zvm_clone_disk:
    name: NEWGUY
    diskaddr: 100
    srcname: GOLDMAST
    srcaddr: 250

# Create NEWGUY 100 in BIGPOOL0 and clone GOLDMAST 250 into it
- name: make NEWGUY 100 and clone into it
  zvm_clone_disk:
    name: NEWGUY
    diskaddr: 100
    srcname: GOLDMAST
    srcaddr: 250
    diskalloc: AUTOG
    diskloc: BIGPOOL0
    diskmode: W
'''

RETURN = r'''
return_stdout:
    description: the stdout responses from smcli as a dict
    returned: always
    type: list
    elements: str
    sample: "...Done"
return_stderr:
    description: the stderr responses from smcli as a dict
    returned: always
    type: list
    elements: str
    sample: "Defining thingone in the directory...
             Failed
             Return Code: 400
             Reason Code: 8
             Description: ULGSMC5400E Image or profile name already defined
             API issued : Image_Create_DM"
return_code:
    description: The results of doing the thing
    type: int
    returned: always
    sample: 0
reason_code:
    description: The reason for the result
    type: int
    returned: always
    sample: 0
'''

from ansible.module_utils.basic import AnsibleModule

def run_module():
    module_args = dict(
        name=dict(type='str', required=True),
        diskaddr=dict(type='str', required=True),
        srcname=dict(type='str', required=True),
        srcaddr=dict(type='str', required=True),
        diskalloc=dict(type='str', required=False, choices=['AUTOR', 'AUTOV', 'AUTOG', 'DEVNO']),
        diskloc=dict(type='str', required=False),
        diskmode=dict(type='str', required=False, choices=['R', 'RR', 'W', 'WR', 'M', 'MR', 'MW']),
        diskrpw=dict(type='str', required=False),
        diskwpw=dict(type='str', required=False),
        diskmpw=dict(type='str', required=False)
    )

    result = dict(
        changed=False,
        return_stdout=[],
        return_stderr=[],
        return_code=-9,
        reason_code=-9
    )

    module = AnsibleModule(
        argument_spec=module_args,
        supports_check_mode=True
    )

    if module.check_mode:
        module.exit_json(**result)

    cmd_string = []

    cmd_string.append("/opt/zthin/bin/smcli")
    cmd_string.append("Image_Disk_Copy_DM")
    cmd_string.append("-T")
    cmd_string.append(module.params['name'])
    cmd_string.append("-t")
    cmd_string.append(module.params['diskaddr'])
    cmd_string.append("-S")
    cmd_string.append(module.params['srcname'])
    cmd_string.append("-s")
    cmd_string.append(module.params['srcaddr'])
    if module.params['diskalloc'] is not None:
        cmd_string.append("-a")
        cmd_string.append(module.params['diskalloc'])
    if module.params['diskloc'] is not None:
        cmd_string.append("-n")
        cmd_string.append(module.params['diskloc'])
    if module.params['diskmode'] is not None:
        cmd_string.append("-m")
        cmd_string.append(module.params['diskmode'])
    if module.params['diskrpw'] is not None:
        cmd_string.append("-r")
        cmd_string.append(module.params['diskrpw'])
    if module.params['diskwpw'] is not None:
        cmd_string.append("-w")
        cmd_string.append(module.params['diskwpw'])
    if module.params['diskmpw'] is not None:
        cmd_string.append("-x")
        cmd_string.append(module.params['diskmpw'])

    # check=false because some errors are expected if we are adding a disk that already exists
    # on subsequent runs of the module - idempotently
    smcli_results = module.run_command(cmd_string, check_rc=False)
    result['return_stdout'] = smcli_results[1]
    result['return_stderr'] = smcli_results[2]
    result['return_code'] = smcli_results[0]

    if result['return_code'] == 0:
        result['changed'] = True
        module.exit_json(**result)

    if result['return_code'] >= 1:
        notreally_errs = ["Image disk already defined"]
        #
        # notreally_errs is a list of VM error messages that are OK within this contect
        noterror_count = 0

        #
        # an SMAPI error message comes back like this. The important parts are the
        # return code and the reason code
        #    "return_stdout": [
        #        "Defining thingone in the directory... ",
        #        "Failed",
        #        "  Return Code: 400",
        #        "  Reason Code: 8",
        #        "  Description: ULGSMC5400E Image or profile name already defined",
        #        "  API issued : Image_Create_DM"
        #    ]

        #
        # this is gross but I looked at alternatives and they were less readable
        # scann return stdout and stderr for
        for i in result['return_stdout']:
            for j in notreally_errs:
                if i.find(j) != -1:
                    noterror_count += 1
        if noterror_count > 0:
            #
            # optimistically guessing this is not an error condition
            # it might still actually be an error condition if there is a real error among the not-errors
            # will figure that out later if it ends up happening
            # exit no error
            result['return_stdout'].append("skipping an error because its probably OK in this situation")
            module.exit_json(**result)
        # here we are in error condition
        errormsg = "failing return code from smcli is: " + str(result['return_code'])
        module.fail_json(errormsg, **result)

    errormsg = "unknown return code from smcli: " + str(result['return_code'])
    result['return_stderr'] = ["how did we get here???"]
    module.fail_json(errormsg, **result)

def main():
    run_module()

if __name__ == '__main__':
    main()


from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StructField
from pyspark.sql.types import StringType, IntegerType

spark = SparkSession.builder.master("local").appName('Read Something').getOrCreate()

schema_data = StructType([
                          StructField("Character", StringType(), False),
                          StructField("Movie", IntegerType(), False)
                        ])

data = spark.read.csv('/home/benito.ramos/Desktop/GIT/PySpark/inputs/data_movies',inferSchema=True, header=False )

data = spark.read.format("csv").options(inferSchema=True, header=False).schema(schema_data).load('inputs/data_movies')

data.show()

data.printSchema()

spark.stop()


import numpy as np
from collections import defaultdict
from gym.envs.toy_text.nchain import NChainEnv
from rl.policy.discrete_policy import RandomPolicy, GreedyPolicy
#
# env = BlackjackEnv()
env = NChainEnv()
#
print("Action Space : ", env.action_space)
print("Observation Space : ", env.observation_space)
#
def mc_control_importance_sampling(env, num_episodes, behavior_policy, discount_factor=1.0):
    Q = defaultdict(lambda : np.zeros(env.action_space.n))
    C = defaultdict(lambda : np.zeros(env.action_space.n))
    target_policy = GreedyPolicy(env, Q)

    for i_episodes in range(1, num_episodes+1):
        if i_episodes % 1000 == 0:
            print("\rEpisode {}/{}.".format(i_episodes, num_episodes), end="")
            for key, value in Q.items():
                print(" ", key, "    ", value)
            # sys.stdout.flush()
        episode = []
        state = env.reset()
        for t in range(100):
            action = behavior_policy.predict(state)
            next_state, reward, done, _ = env.step(action)
            episode.append((state, action, reward))
            if done:
                break
            state = next_state
        # Sum of discounted returns
        G = 0.0
        #
        W = 1.0
        # For each step in episode, backwards
        for t in reversed(np.arange(len(episode))):
            state, action, reward = episode[t]
            G = discount_factor * G + reward
            C[state][action] += W
            #
            alpha = W / C[state][action]
            Q[state][action] += alpha * (G - Q[state][action])
            target_policy.update(Q)
            if action != target_policy.predict(state):
                break
            W = W * 1. / behavior_policy.action_probs[action]

    return Q, target_policy

#
random_policy = RandomPolicy(env)
Q, policy = mc_control_importance_sampling(env, num_episodes=500000, behavior_policy=random_policy,discount_factor=1.0)


from django.core.management.base import BaseCommand
from rooms.models import HouseRule

NAME = "house rules"

class Command(BaseCommand):

    help = f"This command creates some {NAME}"

    def handle(self, *args, **options):
        house_rules = [
            "No smoking.",
            "No parties or events.",
            "No pets/Pets allowed.",
            "No unregistered guests.",
            "No food or drink in bedrooms.",
            "No loud noise after 11 PM.",
        ]

        for house_rule in house_rules:
            HouseRule.objects.create(name=house_rule)

        self.stdout.write(self.style.SUCCESS(f"Create {NAME}"))


import os.path as osp
from functools import partial
from typing import Callable, Sequence

import numpy as np
import rcsenv
from init_args_serializer import Serializable

from pyrado.environments.rcspysim.base import RcsSim
from pyrado.tasks.base import Task
from pyrado.tasks.desired_state import DesStateTask
from pyrado.tasks.final_reward import FinalRewMode, FinalRewTask
from pyrado.tasks.masked import MaskedTask
from pyrado.tasks.parallel import ParallelTasks
from pyrado.tasks.predefined import create_task_space_discrepancy_task
from pyrado.tasks.reward_functions import AbsErrRewFcn, ExpQuadrErrRewFcn, RewFcn
from pyrado.tasks.utils import proximity_succeeded
from pyrado.utils.data_types import EnvSpec

rcsenv.addResourcePath(rcsenv.RCSPYSIM_CONFIG_PATH)
rcsenv.addResourcePath(osp.join(rcsenv.RCSPYSIM_CONFIG_PATH, "PlanarInsert"))

def create_insert_task(env_spec: EnvSpec, state_des: np.ndarray, rew_fcn: RewFcn, success_fcn: Callable):
    # Define the indices for selection. This needs to match the observations' names in RcsPySim.
    idcs = ["Effector_X", "Effector_Z", "Effector_B", "Effector_Xd", "Effector_Zd", "Effector_Bd"]

    # Get the masked environment specification
    spec = EnvSpec(
        env_spec.obs_space, env_spec.act_space, env_spec.state_space.subspace(env_spec.state_space.create_mask(idcs))
    )

    # Create a wrapped desired state task with the goal behind the wall
    fdst = FinalRewTask(
        DesStateTask(spec, state_des, rew_fcn, success_fcn),
        mode=FinalRewMode(state_dependent=True, time_dependent=True),
    )

    # Mask selected states
    return MaskedTask(env_spec, fdst, idcs)

class PlanarInsertSim(RcsSim, Serializable):
    """
    Planar 5- or 6-link robot environment where the task is to push the wedge-shaped end-effector through a small gap
    """

    def __init__(self, task_args: dict, collision_config: dict = None, max_dist_force: float = None, **kwargs):

        """
        Constructor

        .. note::
            This constructor should only be called via the subclasses.

        :param task_args: arguments for the task construction
        :param max_dist_force: maximum disturbance force, pass `None` for no disturbance
        :param kwargs: keyword arguments forwarded to `RcsSim`
                       collisionConfig: specification of the Rcs CollisionModel
        """
        Serializable._init(self, locals())

        # Forward to RcsSim's constructor
        RcsSim.__init__(
            self,
            envType="PlanarInsert",
            task_args=task_args,
            physicsConfigFile="pPlanarInsert.xml",
            collisionConfig=collision_config,
            **kwargs,
        )

        if kwargs.get("collisionConfig", None) is None:
            collision_config = {
                "pairs": [
                    {"body1": "Effector", "body2": "Link3"},
                    {"body1": "Effector", "body2": "Link2"},
                    {"body1": "UpperWall", "body2": "Link4"},
                    {"body1": "LowerWall", "body2": "Link4"},
                    {"body1": "LowerWall", "body2": "Link3"},
                    {"body1": "LowerWall", "body2": "Link2"},
                ],
                "threshold": 0.05,
            }
        else:
            collision_config = kwargs.get("collisionConfig")

        # Setup disturbance
        self._max_dist_force = max_dist_force

    def _create_task(self, task_args: dict) -> Task:
        # Define the task including the reward function
        state_des = task_args.get("state_des", None)
        if state_des is None:
            # Get the goal position in world coordinates
            p = self.get_body_position("Goal", "", "")
            state_des = np.array([p[0], p[2], 0, 0, 0, 0])  # X, Z, B, Xd, Zd, Bd

        # Create the individual subtasks
        task_reach_goal = create_insert_task(
            self.spec,
            state_des,
            rew_fcn=ExpQuadrErrRewFcn(
                Q=np.diag([2e1, 2e1, 1e-1, 1e-2, 1e-2, 1e-2]), R=2e-2 * np.eye(self.act_space.flat_dim)
            ),
            success_fcn=partial(proximity_succeeded, thold_dist=0.07, dims=[0, 1, 2]),  # position and angle
        )
        task_ts_discrepancy = create_task_space_discrepancy_task(
            self.spec, AbsErrRewFcn(q=0.1 * np.ones(2), r=np.zeros(self.act_space.shape))
        )
        return ParallelTasks([task_reach_goal, task_ts_discrepancy])

    @classmethod
    def get_nominal_domain_param(cls):
        return dict(
            link1_mass=2.0,
            link2_mass=2.0,
            link3_mass=2.0,
            link4_mass=2.0,
            upperwall_pos_offset_x=0.0,
            upperwall_friction_coefficient=0.5,
            effector_friction_coefficient=0.7,
        )

    def _disturbance_generator(self) -> (np.ndarray, None):
        if self._max_dist_force is None:
            return None
        # Sample angle and force uniformly
        angle = np.random.uniform(-np.pi, np.pi)
        force = np.random.uniform(0, self._max_dist_force)
        return np.array([force * np.sin(angle), 0, force * np.cos(angle)])

class PlanarInsertIKActivationSim(PlanarInsertSim, Serializable):
    """Planar 5- or 6-link robot environment controlled by  setting the input to an Rcs IK-based controller"""

    name: str = "pi-ika"

    def __init__(self, state_des: np.ndarray = None, **kwargs):
        """
        Constructor

        :param state_des: desired state for the task, pass `None` to use the default goal
        :param kwargs: keyword arguments forwarded to `RcsSim`
                       graphFileName: str = 'gPlanarInsert5Link.xml' or 'gPlanarInsert6Link.xml'
                       checkJointLimits: bool = True,
                       collisionAvoidanceIK: bool = True,
                       observeForceTorque: bool = True,
                       observePredictedCollisionCost: bool = False,
                       observeManipulabilityIndex: bool = False,
                       observeCurrentManipulability: bool = True,
                       observeDynamicalSystemGoalDistance: bool = False,
                       observeDynamicalSystemDiscrepancy: bool = False,
        """
        Serializable._init(self, locals())

        dt = kwargs.get("dt", 0.01)  # 100 Hz is the default
        task_spec_ik = [
            dict(x_des=np.array([dt * 0.1])),
            dict(x_des=np.array([dt * 0.1])),
            dict(x_des=np.array([dt * 5 / 180 * np.pi])),
        ]

        # Forward to the PlanarInsertSim's constructor, nothing more needs to be done here
        PlanarInsertSim.__init__(
            self,
            task_args=dict(state_des=state_des),
            actionModelType="ik_activation",
            taskSpecIK=task_spec_ik,
            **kwargs,
        )

class PlanarInsertTASim(PlanarInsertSim, Serializable):
    """Planar 5- or 6-link robot environment controlled by setting the task activation of a Rcs control task"""

    name: str = "pi-ta"

    def __init__(self, mps: Sequence[dict] = None, state_des: np.ndarray = None, **kwargs):
        """
        Constructor

        :param mps: movement primitives for the cartesian x and z velocity and the angular velocity around y-axis
        :param state_des: desired state for the task, pass `None` to use the default goal
        :param kwargs: keyword arguments forwarded to `RcsSim`
                       graphFileName: str = 'gPlanarInsert5Link.xml' or 'gPlanarInsert6Link.xml'
                       taskCombinationMethod: str = 'sum',  # or 'mean', 'softmax', 'product'
                       checkJointLimits: bool = True,
                       collisionAvoidanceIK: bool = True,
                       observeForceTorque: bool = True,
                       observePredictedCollisionCost: bool = False,
                       observeManipulabilityIndex: bool = False,
                       observeCurrentManipulability: bool = True,
                       observeDynamicalSystemGoalDistance: bool = False,
                       observeDynamicalSystemDiscrepancy: bool = False,
        """
        Serializable._init(self, locals())

        # Define the movement primitives
        dt = kwargs.get("dt", 0.01)  # 100 Hz is the default
        if mps is None:
            mps = [
                # Xd
                {"function": "lin", "errorDynamics": 2.0, "goal": dt * 0.1},  # [m/s]
                {"function": "lin", "errorDynamics": 2.0, "goal": -dt * 0.1},  # [m/s]
                # Zd
                {"function": "lin", "errorDynamics": 2.0, "goal": dt * 0.1},  # [m/s]
                {"function": "lin", "errorDynamics": 2.0, "goal": -dt * 0.1},  # [m/s]
                # Bd
                {"function": "lin", "errorDynamics": 2.0, "goal": dt * 10.0 / 180 * np.pi},  # [rad/s]
                {"function": "lin", "errorDynamics": 2.0, "goal": -dt * 10.0 / 180 * np.pi},  # [rad/s]
            ]

        # Forward to the PlanarInsertSim's constructor, nothing more needs to be done here
        PlanarInsertSim.__init__(
            self, task_args=dict(state_des=state_des), actionModelType="ds_activation", tasks=mps, **kwargs
        )


from django.core.files import File
from app.models import Document

def update_document(document_id, new_content):
    document = Document.objects.get(id=document_id)
    document.content = new_content
    document.save()

def delete_paragraph(document_id, paragraph_id):
    document = Document.objects.get(id=document_id)
    paragraphs = document.content.split('\n')
    if paragraph_id < len(paragraphs):
        del paragraphs[paragraph_id]
        new_content = '\n'.join(paragraphs)
        update_document(document_id, new_content)

def update_paragraph(document_id, paragraph_id, new_paragraph):
    document = Document.objects.get(id=document_id)
    paragraphs = document.content.split('\n')
    if paragraph_id < len(paragraphs):
        paragraphs[paragraph_id] = new_paragraph
        new_content = '\n'.join(paragraphs)
        update_document(document_id, new_content)

def create_new_document(content, user):
    new_document = Document(content=content, user=user)
    new_document.save()
    return new_document.id


import unittest
import os
import volmdlr
import volmdlr.edges as vme
import volmdlr.faces as vmf
import volmdlr.step as vms
import volmdlr.wires as vmw
from volmdlr import surfaces

folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'objects_extrusion_tests')

class TestExtrusionSurface3D(unittest.TestCase):
    control_points = [
        volmdlr.Point3D(-0.025917292, 0.002544355, 0.0),
        volmdlr.Point3D(-0.005449685, -0.007265721, 0.0),
        volmdlr.Point3D(0.0, 0.0, 0.0),
        volmdlr.Point3D(0.014457705000000001, -0.002636091, 0.0),
        volmdlr.Point3D(0.013503079, -0.014007147, 0.0)]
    edge = vme.BSplineCurve3D(3, control_points, [4, 1, 4], [0.0, 0.5, 1.0])
    surface = surfaces.ExtrusionSurface3D(edge, -volmdlr.Z3D)

    def test_point2d_to_3d(self):
        point3d = self.surface.point2d_to_3d(volmdlr.Point2D(0.5, 0.5))
        self.assertTrue(point3d.is_close(volmdlr.Point3D(0.002252005, -0.002475453, -0.5)))

    def test_point3d_to_2d(self):
        point2d_1 = self.surface.point3d_to_2d(self.edge.start)
        self.assertEqual(point2d_1, volmdlr.Point2D(0, 0))
        point2d_2 = self.surface.point3d_to_2d(self.edge.end)
        self.assertEqual(point2d_2, volmdlr.Point2D(1.0, 0))

        surface = surfaces.ExtrusionSurface3D.load_from_file(os.path.join(folder, "point3d_to_2d_surface.json"))
        point3d = volmdlr.Point3D(-0.1322515585788849, -0.14157776310991646, 0.0)
        point2d_3 = surface.point3d_to_2d(point3d)
        self.assertTrue(point2d_3.is_close(volmdlr.Point2D(0.6406891276879707, 0.0), 1e-3))

    def test_rectangular_cut(self):
        face = vmf.ExtrusionFace3D.from_surface_rectangular_cut(self.surface, 0, 1, 0, 2)
        self.assertEqual(face.surface2d.area(), 2)

    def test_from_step(self):
        step = vms.Step.from_file(os.path.join(folder, "bspline_extruded_simple.step"))
        model = step.to_volume_model()
        extrusion_surface = model.primitives[0].primitives[0].surface3d
        self.assertEqual(extrusion_surface.direction, -volmdlr.Z3D)
        self.assertEqual(extrusion_surface.edge.degree, 3)
        self.assertEqual(extrusion_surface.edge.knot_multiplicities, [4, 1, 4])

    def test_linesegment2d_to_3d(self):
        surface = surfaces.ExtrusionSurface3D.load_from_file(
            os.path.join(folder, "extrusion_surface_undefined_direction_linesegment.json"))
        point1 = volmdlr.Point2D(0.9020984833336293, -0.08534036750789999)
        point2 = volmdlr.Point2D(0.9286913444016728, -0.07799341694)
        linesegment2d = vme.LineSegment2D(point1, point2)
        start3d = surface.point2d_to_3d(point1)
        end3d = surface.point2d_to_3d(point2)
        result = surface.linesegment2d_to_3d(linesegment2d)[0]
        self.assertTrue(result.start.is_close(start3d))
        self.assertTrue(result.end.is_close(end3d))
        surface = surfaces.ExtrusionSurface3D.load_from_file(
            os.path.join(folder, "extrusion_surface_test_linesegment2d_to_3d.json"))
        linesegment2d = vme.LineSegment2D.load_from_file(os.path.join(folder, "linesegment2d_to_linesegment3d.json"))
        result = surface.linesegment2d_to_3d(linesegment2d)[0]
        self.assertIsInstance(result, vme.LineSegment3D)

        surface = surfaces.ExtrusionSurface3D.load_from_file(
            os.path.join(folder, "extrusion_surface_test_linesegment2d_to_3d.json"))
        linesegment2d = vme.LineSegment2D.load_from_file(os.path.join(folder, "linesegment2d_to_self_edge.json"))
        result = surface.linesegment2d_to_3d(linesegment2d)[0]
        self.assertEqual(result, surface.edge)

    def test_arc3d_to_2d(self):
        surface = surfaces.ExtrusionSurface3D.load_from_file(
            os.path.join(folder, "extrusion_surface_test_arc3d_to_2d.json"))
        contour3d = vmw.Contour3D.load_from_file(os.path.join(folder, "extrusion_contour_test_arc3d_to_2d.json"))
        arc3d = contour3d.primitives[2]
        result = surface.arc3d_to_2d(arc3d)[0]
        self.assertTrue(result.start.is_close(volmdlr.Point2D(1.0, 0.0032000000499998738)))
        self.assertTrue(result.end.is_close(volmdlr.Point2D(0.13555464614559587, 0.0032000000499998738)))

    def test_frame_mapping(self):
        surface = self.surface
        new_frame = volmdlr.Frame3D(volmdlr.Point3D(0, 0, 1), volmdlr.X3D, volmdlr.Y3D, volmdlr.Z3D)
        new_surface = surface.frame_mapping(new_frame, "old")
        self.assertEqual(new_surface.edge.start.z, 1)
        self.assertTrue(new_surface.frame.origin.is_close(volmdlr.Point3D(-0.025917292, 0.002544355, 1.0)))

    def test_contour3d_to_2d(self):
        surface = surfaces.SphericalSurface3D.load_from_file(os.path.join(folder, "contour3d_to_2d_surface.json"))
        contour = vmw.Contour3D.load_from_file(os.path.join(folder, "contour3d_to_2d_contour.json"))
        contour2d = surface.contour3d_to_2d(contour)
        self.assertTrue(contour2d.is_ordered())
        self.assertAlmostEqual(contour2d.area(), 0.0036995357929908313, 2)

        surface = surfaces.SphericalSurface3D.load_from_file(os.path.join(folder, "contour3d_to_2d_surface_2.json"))
        contour = vmw.Contour3D.load_from_file(os.path.join(folder, "contour3d_to_2d_contour_2.json"))
        contour2d = surface.contour3d_to_2d(contour)
        self.assertTrue(contour2d.is_ordered())
        self.assertAlmostEqual(contour2d.area(), 0.0475, 2)

        surface = surfaces.SphericalSurface3D.load_from_file(
            os.path.join(folder, "extrusionsurface_edge_not_in_normal_plane.json"))
        contour = vmw.Contour3D.load_from_file(
            os.path.join(folder, "extrusionsurface_edge_not_in_normal_plane_contour.json"))
        contour2d = surface.contour3d_to_2d(contour)
        self.assertTrue(contour2d.is_ordered())
        self.assertAlmostEqual(contour2d.area(), 0.005, 2)

        surface = surfaces.SphericalSurface3D.load_from_file(
            os.path.join(folder, "extrusionsurface_with_small_edge.json"))
        contour = vmw.Contour3D.load_from_file(
            os.path.join(folder, "extrusionsurface_with_small_edge_contour.json"))
        contour2d = surface.contour3d_to_2d(contour)
        self.assertTrue(contour2d.is_ordered())
        self.assertAlmostEqual(contour2d.area(), 0.00120000, 4)

if __name__ == '__main__':
    unittest.main()


import cv2
import time
import serial
import serial.tools.list_ports

def send_buzzer(freq):
    sendData = f"BUZZER={freq}\n"
    my_serial.write( sendData.encode() )

def main():
    camera = cv2.VideoCapture(0)
    camera.set(3,640)
    camera.set(4,480)

    face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
    eye_casecade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_eye.xml')

    sleep_count = 0
    while( camera.isOpened() ):
        _, image = camera.read()
        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

        faces = face_cascade.detectMultiScale(gray,scaleFactor=1.1,minNeighbors=5,minSize=(50,50),flags=cv2.CASCADE_SCALE_IMAGE)
        #print("faces detected Number: " + str(len(faces)))

        if len(faces):
            for (x,y,w,h) in faces:
                cv2.rectangle(image,(x,y),(x+w,y+h),(255,0,0),2)

                face_gray = gray[y:y+h, x:x+w]
                face_color = image[y:y+h, x:x+w]

                eyes = eye_casecade.detectMultiScale(face_gray,scaleFactor=1.2,minNeighbors=10)

                if len(eyes) == 0:
                    if sleep_count < 50:
                        sleep_count = sleep_count +1

                    if sleep_count >= 50:
                        print("졸지마")
                        send_buzzer(1000)
                else:
                    if sleep_count >= 2:
                        sleep_count = sleep_count - 2
                    send_buzzer(0)

                for (ex,ey,ew,eh) in eyes:
                    cv2.rectangle(face_color, (ex, ey), (ex+ew, ey+eh), (0,255,0), 2)
        print(sleep_count)
        cv2.imshow('result', image)

        if cv2.waitKey(10) == ord('q'):
            break

    cv2.destroyAllWindows()

if __name__ == '__main__':

    ports = list(serial.tools.list_ports.comports())
    for p in ports:
        if 'Arduino Uno' in p.description:
            print(f"{p} 포트에 연결하였습니다.")
            my_serial = serial.Serial(p.device, baudrate=9600, timeout=1.0)
            time.sleep(2.0)

    main()

    my_serial.close()

한끝아파52 소스코드/02.파이썬코드/9_2.눈을 인식하여 졸음방지 알리미 만들기/main9_2_3.py


import pandas as pd
from datetime import datetime, date
import re
import os
#import numpy as np

PATH = r"C:\Users\pradeep4.kumar\Downloads\telegrambot\data_new_bot"
os.chdir(PATH)
def importing_csv(fname):
    df = pd.read_csv(fname, header=0)
    return df

AccountInfoBank = importing_csv("AccountInfoBank.csv")
AccountInfoDetails = importing_csv("AccountInfoDetails.csv")
ActiveCust = importing_csv("ActiveCust.csv")
ATMTxnFailure = importing_csv("ATMTxnFailure.csv") #Not used
CashBalnces = importing_csv("CashBalnces.csv")
FTM_ChannelTxn = importing_csv("FTM_ChannelTxn.csv")
IPFlag = importing_csv("IPFlag.csv")
NewCust = importing_csv("NewCust.csv")
onBoarding = importing_csv("onBoarding.csv") #Not used
StandaloneCustomer = importing_csv("StandaloneCustomer.csv")

# df_final.columns = [c.replace(' ', '_') for c in df_final.columns]

def change_date_string_date(dframe,dt_colname=None,delim=" "):
    dframe1 = pd.DataFrame(dframe[dt_colname].str.split(delim,1).tolist(),
                                   columns = ['month','year'])
    dframe1["date"] = pd.to_datetime("01"+"-"+dframe1["month"]+"-"+dframe1["year"])
    del dframe1["month"]
    del dframe1["year"]
    return dframe1["date"].dt.date

#---------------------------------------------------------------------------------------------
def extract_info(dframe, key="TotalAccounts", value="2017-01-01"):
    try:
        return str(round(float(dframe.loc[dframe.index == value, key]),2))
    except TypeError:
        return "Unavailable value"

def extract_info_with_grps(dframe, key="TxnCount",groups="Channel",grp_value="ATM",  value="2017-08-01"):
    try:
        return  str(float(dframe.loc[(dframe.index == value) & (dframe[groups] == grp_value), key ]))
    except (TypeError,KeyError):
        return "Unavailable value"

#extract_info_with_grps(grpby_mtd_FTM_ChannelTxn1_mix_monyy)

#extract_info(df_monyy, key="TotalAccounts", value="2016-01-01")
#Determing if the month value is in correct format of mon yyyy or monyyyy
def month_processing(string):
    try:
        string = string.lower()
        match_found = re.match("([a-zA-Z]{3})\s*(\d{4})", string)
        return " ".join(match_found.groups()).title()
    except AttributeError:
        return "Unknown month year format"

def outcome_monthyr(string):
    x = month_processing(string)
    if x.split(" ")[0] == "Unknown":
        return "Unknown"
    else:
        return datetime.strftime(datetime.strptime(x.split(" ")[-1]+ "-" + x.split(" ")[0] + "-" + "01", "%Y-%b-%d"),"%Y-%m-%d")

###fYTD LOGIC-------------------------------------------------------------------------------

def get_the_date_fytd(dframe=None ):
    if date.today().month in (1,2,3):
        range_year_min = date( date.today().year-1,4,1)
        range_year_max = date(date.today(),3,31)
        final_value = dframe.loc[(range_year_max >= dframe.date) & ( dframe.date >= range_year_min),:]
    else:
        range_year_min = date(date.today().year, 4, 1)
        range_year_max = date(date.today().year+1,3,31)
        final_value = dframe.loc[(range_year_max >= dframe.date) & ( dframe.date >= range_year_min),:]
    return final_value

###For Banks Menu---------------------------------------------------------------------------
#Total Acc, AMB, BookValue------------------------------------------------------------------

AccountInfoBank_dt = change_date_string_date(AccountInfoBank, dt_colname="MonthNam")
AccountInfoBank1 = pd.concat([AccountInfoBank.reset_index(drop=True),pd.DataFrame(AccountInfoBank_dt)], axis=1)
AccountInfoBank1.sort_values(by="date", inplace=True)
AccountInfoBank1_fytd = get_the_date_fytd(dframe=AccountInfoBank1)

fytd_totalaccounts = str(float(AccountInfoBank1_fytd.TotalAccounts.sum(0)))
fytd_EOPBalance = str(round(float(AccountInfoBank1_fytd.EOPBalance.sum(0)/(10**7)),2))
fytd_AvgMTLBalnce = str(round(float(AccountInfoBank1_fytd.AvgMTLBalnce.sum(0)/(10**7)),2))

# df_monyy_fytd = AccountInfoBank1_fytd.loc[:,["date", "TotalAccounts", "EOPBalance", "AvgMTLBalnce"]]
# df_monyy_fytd = df_monyy_fytd.set_index(df_monyy_fytd.date.astype("str")).loc[:,["TotalAccounts", "EOPBalance", "AvgMTLBalnce" ]]

#NewCust---------------------------------------------------------------------------------------

NewCust_dt = change_date_string_date(NewCust, dt_colname="AsOnYrMonName")
NewCust1 = pd.concat([NewCust.reset_index(drop=True),pd.DataFrame(NewCust_dt)], axis=1)
NewCust1.sort_values(by="date", inplace=True)
NewCust1_fytd = get_the_date_fytd(dframe=NewCust1)

fytd_NewCust = str(round(float(NewCust1_fytd.loc[:, "CustCount"].sum(0)),2))

#grpby_NewCust_monyy_fytd = pd.DataFrame(NewCust1_fytd['CustCount'].groupby([NewCust1_fytd['date']]).sum())
# grpby_NewCust_monyy_fytd.index = grpby_NewCust_monyy_fytd.index.astype("str")

#----------------------------------------------------------------------------------------------
#IPFlag---------------------------------------------------------------------------------------
IPFlag_0 = IPFlag.loc[IPFlag.IPFlag == 0, :]
IPFlag_1 = IPFlag.loc[IPFlag.IPFlag == 1, :]
IPFlag_0_dt = change_date_string_date(IPFlag_0, dt_colname="YrMon_Acc_Opn_Date")
IPFlag_01 = pd.concat([IPFlag_0.reset_index(drop=True),pd.DataFrame(IPFlag_0_dt)], axis=1)
IPFlag_01.sort_values(by="date", inplace=True)
IPFlag_01_fytd = get_the_date_fytd(dframe=IPFlag_01)

IPFlag_1_dt = change_date_string_date(IPFlag_1, dt_colname="YrMon_Acc_Opn_Date")
IPFlag_11 = pd.concat([IPFlag_1.reset_index(drop=True),pd.DataFrame(IPFlag_1_dt)], axis=1)
IPFlag_11.sort_values(by="date", inplace=True)
IPFlag_11_fytd = get_the_date_fytd(dframe=IPFlag_11)

last_value_ip_0 = list(IPFlag_01_fytd.tail(1)["date"])
last_value_ip_1 = list(IPFlag_11_fytd.tail(1)["date"])

if last_value_ip_1[0] < last_value_ip_0[0]:
    last_value_ip_0 = list(IPFlag_11_fytd.tail(1)["YrMon_Acc_Opn_Date"])
    last_value_ip_1 = list(IPFlag_11_fytd.tail(1)["YrMon_Acc_Opn_Date"])
elif last_value_ip_1[0] > last_value_ip_0[0]:
    last_value_ip_0 = list(IPFlag_01_fytd.tail(1)["YrMon_Acc_Opn_Date"])
    last_value_ip_1 = list(IPFlag_01_fytd.tail(1)["YrMon_Acc_Opn_Date"])
else:
    last_value_ip_0 = list(IPFlag_01_fytd.tail(1)["YrMon_Acc_Opn_Date"])
    last_value_ip_1 = list(IPFlag_11_fytd.tail(1)["YrMon_Acc_Opn_Date"])

#The above logic is not incorporated for fytd
fytd_IPFlag_11 = float(IPFlag_11_fytd.loc[:, "Count(ACCT_NBR)"].sum(0))
fytd_IPFlag_01 = float(IPFlag_01_fytd.loc[:, "Count(ACCT_NBR)"].sum(0))

fytd_IPFlag_percentage = str(round(fytd_IPFlag_11/(fytd_IPFlag_01+fytd_IPFlag_11),2)*100) + "%"

#----------------------------------------------------------------------------------------------
#CashBalnces----------------------------------------------------------------------------------

cashbalances_dt = change_date_string_date(CashBalnces, dt_colname="YrMon")
CashBalnces1 = pd.concat([CashBalnces.reset_index(drop=True),pd.DataFrame(cashbalances_dt)], axis=1)
CashBalnces1.sort_values(by="date", inplace=True)
CashBalnces1_fytd = get_the_date_fytd(dframe=CashBalnces1)

fytd_CashBalnces = str(round(float(CashBalnces1_fytd.loc[:, "DayEndBal"].sum(0)/10**7),2))

#----------------------------------------------------------------------------------------------

#Active Customers------------------------------------------------------------------------------

ActiveCust_dt = change_date_string_date(ActiveCust, dt_colname="YrMon_AsOnDate")
ActiveCust1 = pd.concat([ActiveCust.reset_index(drop=True),pd.DataFrame(ActiveCust_dt)], axis=1)
ActiveCust1.sort_values(by="date", inplace=True)
ActiveCust1_fytd = get_the_date_fytd(dframe=ActiveCust1)

fytd_ActiveCust_temp = ActiveCust1_fytd.loc[:, ["TxnActCusts", "CountAccs"]].sum(0)
fytd_ActiveCust = str(round(float(fytd_ActiveCust_temp.TxnActCusts / fytd_ActiveCust_temp.CountAccs)*100,2)) + "%"

#Helper function----------------------------------------------------------------------------------------------

def compress_spaces(text):
    spaces = re.compile("\s+")
    return spaces.sub(" ",text)

#compress_spaces("abc    def")

###For Branch Menu---------------------------------------------------------------------------
list_of_branches = [compress_spaces(item.upper()) for item in list(AccountInfoDetails.loc[:, "BRNCH_NME"].unique())]
AccountInfoDetails_dt = change_date_string_date(AccountInfoDetails, dt_colname="MonthNam", delim="-")
AccountInfoDetails1 = pd.concat([AccountInfoDetails.reset_index(drop=True),pd.DataFrame(AccountInfoDetails_dt)], axis=1)
AccountInfoDetails1.sort_values(by="date", inplace=True)

AccountInfoDetails1_fytd = get_the_date_fytd(dframe=AccountInfoDetails1)

#Extra Step for cleanup
AccountInfoDetails1_fytd.loc[:,"BRNCH_NME"] = AccountInfoDetails1_fytd.loc[:,"BRNCH_NME"].apply(lambda x : re.sub("\s+"," ", x)).str.upper()

# Book value and total accounts & AMB
#AccountInfoDetails1_fytd2 = AccountInfoDetails1_fytd.loc[AccountInfoDetails1_fytd["MonthNam"] == last_value_ad[0]]

AccountInfoDetails1_fytd2_mix = pd.DataFrame(AccountInfoDetails1_fytd[["TotalAccounts","EOPBalance","AvgMTLBalnce"]].groupby(AccountInfoDetails1_fytd["BRNCH_NME"]).sum())
AccountInfoDetails1_branch_account_fytd2_mix = AccountInfoDetails1_fytd2_mix.loc[:, "TotalAccounts"]
AccountInfoDetails1_branch_eop_fytd2_mix = AccountInfoDetails1_fytd2_mix.loc[:, "EOPBalance"]
AccountInfoDetails1_branch_amb_fytd2_mix = AccountInfoDetails1_fytd2_mix.loc[:, "AvgMTLBalnce"]
fytd_branch_account_dict = {iterx : str(round(float(AccountInfoDetails1_branch_eop_fytd2_mix[AccountInfoDetails1_branch_eop_fytd2_mix.index == iterx].sum(0)/1),2)) for iterx in list_of_branches}
fytd_branch_eop_dict = {iterx : str(round(float(AccountInfoDetails1_branch_eop_fytd2_mix[AccountInfoDetails1_branch_eop_fytd2_mix.index == iterx].sum(0)/10**7),2)) for iterx in list_of_branches}
fytd_branch_amb_dict = {iterx : str(round(float(AccountInfoDetails1_branch_amb_fytd2_mix[AccountInfoDetails1_branch_amb_fytd2_mix.index == iterx].sum(0)/10**7),2)) for iterx in list_of_branches}

# New Customers

NewCust_dt = change_date_string_date(NewCust, dt_colname="AsOnYrMonName")
NewCust1 = pd.concat([NewCust.reset_index(drop=True),pd.DataFrame(NewCust_dt)], axis=1)
NewCust1.sort_values(by="date", inplace=True)
NewCust1_fytd = get_the_date_fytd(dframe=NewCust1)

#Extra Step for cleanup
NewCust1_fytd["BName"] = NewCust1_fytd["BName"].apply(lambda x : re.sub("\s+"," ", x)).str.upper()

fytd_NewCust_branch_mix = NewCust1_fytd.loc[:, "CustCount"].groupby(NewCust1_fytd["BName"]).sum()

# %Active Clients

ActiveCust_dt = change_date_string_date(ActiveCust, dt_colname="YrMon_AsOnDate")
ActiveCust1 = pd.concat([ActiveCust.reset_index(drop=True),pd.DataFrame(ActiveCust_dt)], axis=1)
ActiveCust1.sort_values(by="date", inplace=True)
ActiveCust1_fytd = get_the_date_fytd(dframe=ActiveCust1)

#Extra Step for cleanup
ActiveCust1_fytd["BRNCH_NME"] = ActiveCust1_fytd["BRNCH_NME"].apply(lambda x : re.sub("\s+"," ", x)).str.upper()

fytd_ActiveCust_branch_temp = ActiveCust1_fytd.loc[:, ["TxnActCusts", "CountAccs"]].groupby(ActiveCust1_fytd["BRNCH_NME"]).sum()
fytd_ActiveCust_branch_temp["percentage"] = round(fytd_ActiveCust_branch_temp.TxnActCusts / fytd_ActiveCust_branch_temp.CountAccs*100,2)
fytd_ActiveCust_branch_temp = fytd_ActiveCust_branch_temp.loc[:, "percentage"]

# Cash Day End Balance

cashbalances_dt = change_date_string_date(CashBalnces, dt_colname="YrMon")
CashBalnces1 = pd.concat([CashBalnces.reset_index(drop=True),pd.DataFrame(cashbalances_dt)], axis=1)
CashBalnces1.sort_values(by="date", inplace=True)
CashBalnces1_fytd = get_the_date_fytd(dframe=CashBalnces1)
#Extra Step for cleanup
CashBalnces1_fytd["BRNCH_NME"] = CashBalnces1_fytd["BRNCH_NME"].apply(lambda x : re.sub("\s+"," ", x)).str.upper()

fytd_CashBalnces_branch_mix = round(CashBalnces1_fytd.loc[:, ["DayEndBal", "BRNCH_NME"]].groupby(CashBalnces1_fytd["BRNCH_NME"]).sum()/10**7,2)

###For Products------------------------------------------------------------------------------

#AccountInfoDetails --------------------------------------------------------------------------

AccountInfoDetails_dt = change_date_string_date(AccountInfoDetails, dt_colname="MonthNam", delim="-")
AccountInfoDetails1 = pd.concat([AccountInfoDetails.reset_index(drop=True),pd.DataFrame(AccountInfoDetails_dt)], axis=1)
AccountInfoDetails1.sort_values(by="date", inplace=True)
AccountInfoDetails1_fytd = get_the_date_fytd(dframe=AccountInfoDetails1)

AccountInfoDetails1_fytd_temp = pd.DataFrame(AccountInfoDetails1_fytd.loc[:, ["TotalAccounts", "EOPBalance"]].sum(0)).T
fytd_AccountInfoDetails1_bookvalue =  str(round(float(AccountInfoDetails1_fytd_temp.EOPBalance/10**7),2))
fytd_AccountInfoDetails1_noa =  str(round(float(AccountInfoDetails1_fytd_temp.TotalAccounts),2))

#--------SA--------------------------------------------------------------------------

AccountInfoDetails1_SA_temp_fytd = pd.DataFrame(AccountInfoDetails1_fytd.loc[(AccountInfoDetails1_fytd.ProdGroup == "SA"),["TotalAccounts","EOPBalance"]].sum(0)).T
fytd_AccountInfoDetails1_SA_bookvalue =  str(round(float(AccountInfoDetails1_SA_temp_fytd.EOPBalance/10**7),2))
fytd_AccountInfoDetails1_SA_noa =  str(round(float(AccountInfoDetails1_SA_temp_fytd.TotalAccounts),2))

#--------RD--------------------------------------------------------------------------

AccountInfoDetails1_RD_fytd_temp = pd.DataFrame(AccountInfoDetails1_fytd.loc[(AccountInfoDetails1_fytd.ProdGroup == "RD"),["TotalAccounts","EOPBalance"]].sum(0)).T
fytd_AccountInfoDetails1_RD_bookvalue =  str(round(float(AccountInfoDetails1_RD_fytd_temp.EOPBalance/10**7),2))
fytd_AccountInfoDetails1_RD_noa =  str(round(float(AccountInfoDetails1_RD_fytd_temp.TotalAccounts),2))

#--------FD--------------------------------------------------------------------------

AccountInfoDetails1_FD_fytd_temp = pd.DataFrame(AccountInfoDetails1_fytd.loc[(AccountInfoDetails1_fytd.ProdGroup == "FD"),["TotalAccounts","EOPBalance"]].sum(0)).T
fytd_AccountInfoDetails1_FD_bookvalue =  str(round(float(AccountInfoDetails1_FD_fytd_temp.EOPBalance/10**7),2))
fytd_AccountInfoDetails1_FD_noa =  str(round(float(AccountInfoDetails1_FD_fytd_temp.TotalAccounts),2))

#--------CA--------------------------------------------------------------------------

AccountInfoDetails1_CA_fytd_temp = pd.DataFrame(AccountInfoDetails1_fytd.loc[(AccountInfoDetails1_fytd.ProdGroup == "CA"),["TotalAccounts","EOPBalance"]].sum(0)).T
fytd_AccountInfoDetails1_CA_bookvalue =  str(round(float(AccountInfoDetails1_CA_fytd_temp.EOPBalance/10**7),2))
fytd_AccountInfoDetails1_CA_noa =  str(round(float(AccountInfoDetails1_CA_fytd_temp.TotalAccounts),2))

#Standalone --------------------------------------------------------------------------

StandaloneCustomer_dt = change_date_string_date(StandaloneCustomer, dt_colname="YrMon_AsOnDate")
StandaloneCustomer1 = pd.concat([StandaloneCustomer.reset_index(drop=True),pd.DataFrame(StandaloneCustomer_dt)], axis=1)
StandaloneCustomer1.sort_values(by="date", inplace=True)
StandaloneCustomer1_fytd = get_the_date_fytd(dframe=StandaloneCustomer1)

fytd_StandaloneCustomer = StandaloneCustomer1_fytd.loc[:, ["CustCount"]].sum(0)
fytd_StandaloneCustomer =  str(round(float(fytd_StandaloneCustomer),2))
#---SA--------------------------------

fytd_StandaloneCustomer_SA = StandaloneCustomer1_fytd.loc[(StandaloneCustomer1_fytd.ProdGroup=="SA"), ["CustCount"]].sum(0)
fytd_StandaloneCustomer_SA =  str(round(float(fytd_StandaloneCustomer_SA),2))
#---RD--------------------------------

fytd_StandaloneCustomer_RD = StandaloneCustomer1_fytd.loc[ (StandaloneCustomer1_fytd.ProdGroup=="RD"), ["CustCount"]].sum(0)
fytd_StandaloneCustomer_RD =  str(round(float(fytd_StandaloneCustomer_RD),2))
#---CA--------------------------------

fytd_StandaloneCustomer_CA = StandaloneCustomer1_fytd.loc[(StandaloneCustomer1_fytd.ProdGroup=="CA"), ["CustCount"]].sum(0)
fytd_StandaloneCustomer_CA =  str(round(float(fytd_StandaloneCustomer_CA),2))
#---FD--------------------------------

fytd_StandaloneCustomer_FD = StandaloneCustomer1_fytd.loc[ (StandaloneCustomer1_fytd.ProdGroup=="FD"), ["CustCount"]].sum(0)
fytd_StandaloneCustomer_FD =  str(round(float(fytd_StandaloneCustomer_FD),2))

#Channel--------------------------------------------------------------------------------------------

##Total Transactions--------------------------------------------------------------------------------
# FTM_ChannelTxn

FTM_ChannelTxn_dt = change_date_string_date(FTM_ChannelTxn, dt_colname="AS_ON_DATE_YrMon",delim="-")
FTM_ChannelTxn1 = pd.concat([FTM_ChannelTxn.reset_index(drop=True),pd.DataFrame(FTM_ChannelTxn_dt)], axis=1)
FTM_ChannelTxn1.sort_values(by="date", inplace=True)
FTM_ChannelTxn1_fytd = get_the_date_fytd(dframe=FTM_ChannelTxn1)

fytd_FTM_ChannelTxn = FTM_ChannelTxn1_fytd.loc[:, ["TxnCount"]].sum(0)
fytd_FTM_ChannelTxn =  str(round(float(fytd_FTM_ChannelTxn),2))

#Channel Mix----------------------------------------------------------------------------------------
FTM_ChannelTxn_dt = change_date_string_date(FTM_ChannelTxn, dt_colname="AS_ON_DATE_YrMon",delim="-")
FTM_ChannelTxn1 = pd.concat([FTM_ChannelTxn.reset_index(drop=True),pd.DataFrame(FTM_ChannelTxn_dt)], axis=1)
FTM_ChannelTxn1.sort_values(by="date", inplace=True)
FTM_ChannelTxn1_fytd = get_the_date_fytd(dframe=FTM_ChannelTxn1)

FTM_ChannelTxn_fytd_mix = pd.DataFrame(FTM_ChannelTxn1_fytd.loc[:,"TxnCount"].groupby(FTM_ChannelTxn1_fytd["Channel"]).sum())
channel_indices = list(FTM_ChannelTxn_fytd_mix.index)
fytd_channel_dict = {items : str(round(float(FTM_ChannelTxn_fytd_mix[FTM_ChannelTxn_fytd_mix.index == items].sum(0)),2)) for items in channel_indices}

#----------Added later for Transaction value------------------------------------------------------------------------------

FTM_ChannelTxn_value_mix = pd.DataFrame(FTM_ChannelTxn1.loc[:,"TxnValue"].groupby(FTM_ChannelTxn1["Channel"]).sum())
channel_indices = list(FTM_ChannelTxn_value_mix.index)
fytd_channel_value_dict = {items : str(round(float(FTM_ChannelTxn_value_mix[FTM_ChannelTxn_value_mix.index == items].sum(0))/10**7,2)) for items in channel_indices}


import configparser
import os

try:
    _CONFIG = os.environ["YM_SYS_CFG"]
except KeyError:
    print("Set YM_SYS_CFG env var")
    exit(1)

_configs = {}

if not os.path.exists(_CONFIG):
    print(f"Cannot find system config: \"{os.path.abspath(_CONFIG)}\".")
    print("Set YM_SYS_CFG env var")
    exit(1)

print("Loading Configs")
with open(_CONFIG, mode='r') as f:
    for line in f:
        line = line.strip()
        if len(line) == 0 or line.startswith('#'):
            continue
        key, val = line.split('=')
        _configs[key.strip()] = val.strip()

configFile = _CONFIG
tempDir = _configs['TempDir']
maxLogLines = int(_configs['MaxLogLines'])

botFile = os.environ['CafeYikeJar']
frontendDir = os.environ['FrontendDir']


from io import StringIO
from contextlib import contextmanager
from psycopg2.errorcodes import UNIQUE_VIOLATION
from psycopg2 import errors

# --- Reset Database ---
DROP_TABLE_COMPANIES = "DROP TABLE IF EXISTS companies;"
DROP_TABLE_BARS = "DROP TABLE IF EXISTS bars;"
DROP_TABLE_NOT_FOUND = "DROP TABLE IF EXISTS not_found;"

# --- Create Tables ---
CREATE_COMPANIES = """
    CREATE TABLE IF NOT EXISTS companies (
        ticker TEXT PRIMARY KEY,
        name TEXT,
        exchange TEXT,
        sector TEXT
    );
"""
CREATE_COMPANIES_ = """
    CREATE TABLE IF NOT EXISTS companies (
        id SERIAL PRIMARY KEY,
        ticker TEXT,
        name TEXT,
        city TEXT,
        country TEXT,
        currency TEXT,
        exchange TEXT,
        sector TEXT,
        industry TEXT,
        UNIQUE (ticker)
    );
"""
CREATE_EXCHANGES = """
    CREATE TABLE IF NOT EXISTS exchanges (
        id SERIAL PRIMARY KEY,
        name TEXT,
        UNIQUE (name)
    );
"""
CREATE_SECTORS = """
    CREATE TABLE IF NOT EXISTS sectors (
        id SERIAL PRIMARY KEY,
        name TEXT,
        UNIQUE (name)
    );
"""
CREATE_INDUSTRIES = """
    CREATE TABLE IF NOT EXISTS industries (
        id SERIAL PRIMARY KEY,
        name TEXT,
        UNIQUE (name)
    );
"""
CREATE_BARS = """
    CREATE TABLE IF NOT EXISTS bars (
        id SERIAL PRIMARY KEY,
        ticker TEXT,
        date DATE,
        open NUMERIC,
        high NUMERIC,
        low NUMERIC,
        close NUMERIC,
        adj_close NUMERIC,
        volume BIGINT,
        UNIQUE (ticker, date)
    );
"""
CREATE_NOT_FOUND = """
    CREATE TABLE IF NOT EXISTS not_found (
        id SERIAL PRIMARY KEY,
        ticker TEXT,
        UNIQUE (ticker)
    );
"""

# --- SELECT FROM DATABASE ---
SELECT_LAST_TICKER = """
    SELECT MAX(date) AS last_row FROM bars
    WHERE ticker = %s;
"""
SELECT_FIRST_TICKER = """
    SELECT MIN(date) AS first_row FROM bars
    WHERE ticker = %s;
"""

SELECT_COMPANIES = """
    SELECT ticker FROM companies
    LIMIT %s;
"""

SELECT_COMPANIES_WHERE_EXCHANGE = """
    SELECT ticker FROM companies
    WHERE exchange = %s
    LIMIT %s;
"""

SELECT_COMPANIES_WHERE_TICKER = """
    SELECT ticker FROM companies
    WHERE ticker = %s;
"""

SELECT_TICKERS_NOT_FOUND = """
    SELECT ticker from not_found;
"""

SELECT_EXCHANGES = """
    SELECT DISTINCT exchange from companies
    ORDER BY exchange ASC
"""

# --- INSERT INTO DATABASE ---
INSERT_COMPANIE_RETURN_TICKER = """
    INSERT INTO companies
    (ticker, name, exchange, sector)
    VALUES (%s, %s, %s, %s)
    RETURNING ticker;
"""
INSERT_COMPANIE_RETURN_ID = """
    INSERT INTO companies
    (ticker, name, city, country, currency, exchange, sector, industry)
    VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
    RETURNING id;
"""
INSERT_TICKER_INTO_NOT_FOUND = """
    INSERT INTO not_found
    (ticker)
    VALUES (%s);
"""

# --- DELETE FROM DATABASE ---
DELETE_TICKER_NOT_FOUND = """
    DELETE FROM not_found
    WHERE ticker = '%s';
"""

@contextmanager
def get_cursor(connection):
    with connection:
        with connection.cursor() as cursor:
            yield cursor

def create_tables(connection):
    with get_cursor(connection) as cursor:
        # cursor.execute(CREATE_COMPANIES_)
        # cursor.execute(CREATE_BARS)
        cursor.execute(CREATE_NOT_FOUND)

def drop_tables(connection):
    with get_cursor(connection) as cursor:
        # cursor.execute(DROP_TABLE_COMPANIES)
        # cursor.execute(DROP_TABLE_BARS)
        cursor.execute(DROP_TABLE_NOT_FOUND)

def get_last_ts(connection, ticker):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_LAST_TICKER, (ticker,))
        return cursor.fetchone()[0]

def get_first_ts(connection, ticker):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_FIRST_TICKER, (ticker,))
        return cursor.fetchone()[0]

def get_not_found_tickers(connection):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_TICKERS_NOT_FOUND)
        return cursor.fetchall()

def get_tickers(connection, exchange, limit):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_COMPANIES_WHERE_EXCHANGE, (exchange, limit))
        return cursor.fetchall()

def get_all_tickers(connection, limit):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_COMPANIES, (limit, ))
        return cursor.fetchall()

def get_exchanges(connection):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_EXCHANGES)
        return cursor.fetchall()

def company_ticker_exist(connection, ticker):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_COMPANIES_WHERE_TICKER, (ticker,))
        if cursor.fetchone():
            return True
    return False

def add_company(
    connection, ticker, name, city,
    country, currency, exchange, sector,
    industry
):
    with get_cursor(connection) as cursor:
        try:
            cursor.execute(
                INSERT_COMPANIE_RETURN_ID,
                (
                    ticker, name, city, country,
                    currency, exchange, sector,
                    industry
                )
            )
        except errors.lookup(UNIQUE_VIOLATION):
            return None

        return cursor.fetchone()[0]

def add_ticker_not_found(connection, ticker):
    with get_cursor(connection) as cursor:
        try:
            cursor.execute(
                INSERT_TICKER_INTO_NOT_FOUND,
                (ticker,)
            )
        except errors.lookup(UNIQUE_VIOLATION):
            print(f"Ticker: {ticker} exists")

def bulk_insert_bars(connection, df, table: str):
    # Initialize a string buffer
    sio = StringIO()
    # Write the Pandas DataFrame as a csv to the buffer
    sio.write(df.to_csv(index=None, header=None))
    # Be sure to reset the position to the start of the stream
    sio.seek(0)

    with get_cursor(connection) as cursor:
        try:
            cursor.copy_from(sio, table, columns=df.columns, sep=',')
            connection.commit()
            return True
        except errors.lookup(UNIQUE_VIOLATION):
            return 'Error: Rows already exists!'

def delete_ticker_not_found(connection, ticker):
    with get_cursor(connection) as cursor:
        cursor.execute(DELETE_TICKER_NOT_FOUND, (ticker,))

if __name__ == '__main__':
    from connection_pool import get_connection
    with get_connection() as connection:
        drop_tables(connection)
        create_tables(connection)
        # get_last_ts(connection, 'NHY.OL')


from openpyxl.reader.excel import load_workbook
from random_batch import *
import numpy as np
import torch
import random

def load_data(excel_yes, excel_no):
    """读取数据"""
    wb_yes = load_workbook(excel_yes)
    ws_yes = wb_yes.active
    wb_no = load_workbook(excel_no)
    ws_no = wb_no.active
    """读取数据"""
    list_yes = []
    for i in range(1, 2555):
        embedding_yes = ws_yes.cell(row=i, column=2).value
        embedding_yes = embedding_yes.lstrip('[')
        embedding_yes = embedding_yes.rstrip(']')
        em = str.split(embedding_yes)
        for j in range(len(em)):
            em[j] = eval(em[j])
        em = np.array(em, dtype='float32')
        em = torch.from_numpy(em)
        list_yes.append(em)
    list_no = []
    for i in range(1, 11369):
        embedding_no = ws_no.cell(row=i, column=2).value
        embedding_no = embedding_no.lstrip('[')
        embedding_no = embedding_no.rstrip(']')
        em = str.split(embedding_no)
        for j in range(len(em)):
            em[j] = eval(em[j])
        em = np.array(em, dtype='float32')
        em = torch.from_numpy(em)
        list_no.append(em)
    x_t = list_yes + list_no
    y_t = [1] * 2554 + [0] * 11368
    return x_t, y_t

net1 = torch.load('net3_26_1.pkl')
net1.eval()
x_test, y_test= load_data('Yes_Sep.xlsx', 'No_Sep.xlsx')
test_data, test_label = random_batch(x_test, y_test, len(x_test))
y_p = net1(test_data[0])
y_h = test_label[0]
summ = 0
y_p_l = torch.zeros((y_h.shape[0]))
for i in range (0, y_h.shape[0]):
    if y_p[i,0] >= 0.5:
        y_p_l[i] = 0
    else:
        y_p_l[i] = 1
    if y_p_l[i] == y_h[i]:
        summ = summ + 1
acc = summ / y_h.shape[0]
print(acc)

wb = load_workbook('Yes_Sep.xlsx')
ws = wb.active
for i in range (1,2555):
    ws.cell(i,3,value = str(y_p_l[i-1].item()))
wb.save('Yes_Sep.xlsx')

wb = load_workbook('No_Sep.xlsx')
ws = wb.active
for i in range (1,11369):
    ws.cell(i,3,value =  str(y_p_l[i - 1 + 2554].item()))
wb.save('No_Sep.xlsx')


# coding:utf-8

'''
在此定义辅助函数
'''

import hashlib

#################################################
# 获取一个Md5的字符串
#################################################
def getMd5String(data):
    m = hashlib.md5()
    b = data.encode(encoding='utf-8')
    m.update(b)
    str_md5 = m.hexdigest()
    return str_md5

#################################################
# 获取签名，传入一个map,
# （需要对传入的参数，按照key的升序排列）
#################################################
def getSignature(data,postData={}):
    sMd5 = ""
    if len(postData) == 0:
        keys = sorted(data.keys())
        s = ""
        for key in keys:
            s = s + key + "[" + str(data[key]) + "];"
        sMd5 = getMd5String(s)
    else:
        data["Data"] = postData
        keys = sorted(data.keys())
        s = ""
        for key in keys:
            s = s + key + "[" + str(data[key]) + "];"
        sMd5 = getMd5String(s)
    return sMd5

if __name__ == "__main__":
    # print(getSignature({"a":123,"b":"abc"}))
    print(getSignature({'ts': 1590406665, 'sid': '31e07c4a44ac439dac8931aea3704c62', 'start': 1590406665, 'uid': 'bb4d91ccc1f6461daec7a0daf1c8e8f2', 'cid': '49a92ac0f9b14ba59d91ee8debe8357d'}))
    # s=getMd5String('liqingpeng')
    # print(s)


# SearchPanel.py (vars-localize)
from PyQt5.QtGui import QColor

from ui.EntryTree import ImagedMomentTree
from ui.JSONTree import JSONTree
from ui.Paginator import Paginator

__author__ = "Kevin Barnard"
__copyright__ = "Copyright 2019, Monterey Bay Aquarium Research Institute"
__credits__ = ["MBARI"]
__license__ = "GPL"
__maintainer__ = "Kevin Barnard"
__email__ = "kbarnard@mbari.org"
__doc__ = '''

Dock widget used to search for concepts and select frame grabs.

@author: __author__
@status: __status__
@license: __license__
'''
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QDockWidget, QVBoxLayout, QWidget, QHBoxLayout, QSpinBox, QScrollArea, QTextEdit, QLabel, \
    QSizePolicy, QDialog, QPushButton, QDialogButtonBox

from ui.ConceptSearchbar import ConceptSearchbar

from util.requests import get_all_concepts, delete_observation, modify_concept

from ui.EntryTree import EntryTreeItem

class SearchPanel(QDockWidget):

    def __init__(self, parent=None):
        super(SearchPanel, self).__init__(parent)

        self.setFeatures(QDockWidget.DockWidgetMovable |
                         QDockWidget.DockWidgetFloatable)
        self.setAllowedAreas(Qt.LeftDockWidgetArea)

        self.contents = QWidget()
        self.contents.setMinimumSize(400, 300)
        self.contents.setLayout(QVBoxLayout())
        self.setWidget(self.contents)

        self.concept = None

        self.top_bar = QWidget()
        self.top_bar.setLayout(QHBoxLayout())

        self.search_bar = ConceptSearchbar()
        self.search_bar.set_callback(self.concept_selected)

        self.time_window = QSpinBox()
        self.time_window.setRange(0, 20000)

        self.top_bar.layout().addWidget(self.search_bar)
        self.top_bar.layout().addWidget(self.time_window)

        self.entry_tree = ImagedMomentTree()
        self.entry_tree.currentItemChanged.connect(self.parent().load_entry)
        self.entry_tree.itemDoubleClicked.connect(self.show_popup)
        self.entry_tree.time_window = 0
        self.time_window.valueChanged.connect(self.entry_tree.set_time_window)

        self.paginator = Paginator()
        self.paginator.set_limit(25)
        self.paginator.left_button.setDisabled(True)
        self.paginator.right_button.setDisabled(True)
        self.paginator.left_signal.connect(self.load_page)
        self.paginator.right_signal.connect(self.load_page)
        self.paginator.jump_signal.connect(self.load_page)

        self.association_area = QScrollArea()
        self.association_area.setWidgetResizable(True)
        self.association_text = QLabel()
        self.association_text.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Maximum)
        self.association_area.setWidget(self.association_text)
        self.association_area.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
        self.association_area.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)

        self.contents.layout().addWidget(self.top_bar)
        self.contents.layout().addWidget(self.entry_tree, stretch=1)
        self.contents.layout().addWidget(self.paginator)
        self.contents.layout().addWidget(self.association_area)

        self.observer = ''

    def concept_selected(self, concept):
        if concept in get_all_concepts():
            self.concept = concept
            self.load_results()

    def load_results(self):
        self.parent().display_panel.image_view.set_pixmap(None)
        self.paginator.set_offset(0)
        self.load_page()
        self.parent().display_panel.image_view.redraw()

    def load_page(self):
        if self.concept:
            self.entry_tree.query(self.concept, self.paginator.offset, self.paginator.limit)  # Get imaged moments
            self.paginator.set_count(len(self.entry_tree.loaded_uuids))  # Set count

    def load_imaged_moment_uuids(self, imaged_moment_uuids):
        self.entry_tree.set_results(imaged_moment_uuids)
        self.paginator.set_offset(0)
        self.paginator.set_count(len(imaged_moment_uuids))
        self.parent().display_panel.image_view.redraw()

    def select_next(self):
        self.entry_tree.setCurrentIndex(self.entry_tree.indexBelow(self.entry_tree.currentIndex()))

    def select_prev(self):
        self.entry_tree.setCurrentIndex(self.entry_tree.indexAbove(self.entry_tree.currentIndex()))

    def show_popup(self, item: EntryTreeItem, col: int):
        if item is None or item.metadata['type'] == 'imaged_moment':
            return

        observation_uuid = item.metadata['uuid']

        editable = observation_uuid in self.entry_tree.editable_uuids
        admin_mode = self.parent().admin_mode

        dialog = QDialog()
        dialog.setMinimumSize(600, 300)
        dialog.setLayout(QVBoxLayout())
        dialog.setWindowTitle('Observation Information')
        dialog.setWindowFlag(Qt.WindowCloseButtonHint, False)

        json_tree = JSONTree(item.metadata)
        concept_widget = QWidget()
        concept_widget.setLayout(QHBoxLayout())
        delete_button = QPushButton('Delete')
        delete_button.setStyleSheet('background-color: #ff9696')
        delete_button.setDisabled(not editable and not admin_mode)
        delete_lock = False

        button_box = QDialogButtonBox(QDialogButtonBox.Save)
        button_box.setStyleSheet('background-color: lightgreen')
        button_box.accepted.connect(dialog.accept)

        def do_delete_observation():
            nonlocal dialog
            nonlocal delete_lock
            dialog.close()
            delete_observation(observation_uuid)
            delete_lock = True
            self.parent().display_panel.image_view.set_entry(item.parent())
            self.parent().display_panel.image_view.reload_moment()

        delete_button.pressed.connect(do_delete_observation)

        dialog.layout().addWidget(json_tree)
        dialog.layout().addWidget(concept_widget)
        dialog.layout().addWidget(delete_button)
        dialog.layout().addWidget(button_box)

        concept_field = ConceptSearchbar()
        concept_field.setText(item.metadata['concept'])
        concept_field.setDisabled(not editable and not admin_mode)

        concept_widget.layout().addWidget(QLabel('Concept:'))
        concept_widget.layout().addWidget(concept_field)

        concept_before = concept_field.text()

        dialog.setModal(True)
        dialog.exec_()

        concept_after = concept_field.text()
        if not delete_lock and concept_after != concept_before:  # Rename the observation
            modify_concept(observation_uuid, concept_after, self.observer)
            moment = item.parent()
            self.entry_tree.load_imaged_moment_entry(moment)
            self.parent().display_panel.image_view.set_entry(moment)


# -*-: coding utf-8 -*-
""" Skeleton Snips skill. """
import os
import sys
import spotipy
from spotipy.oauth2 import SpotifyOAuth

class SpotifyController:
    """ Skeleton Snips skill. """

    def __init__(self, client_id, secret_id, redirect_uri, tts_service=None):
        """
        """
        self.client_id = client_id
        self.secret_id = secret_id
        self.redirect_url = redirect_uri
        self.tts = tts_service
        # self.hostname = hostname
        # self.light_ids = light_ids

    def __get_access_token(self):
        scope = 'user-modify-playback-state user-read-playback-state'
        auth = SpotifyOAuth(
            self.client_id, self.secret_id, self.redirect_url,
            scope=scope, cache_path='/home/pi/.cache--snipscontroller'
        )
        token = auth.get_cached_token()
        if token is None:
            auth_url = auth.get_authorize_url()
            try:
                import webbrowser
                webbrowser.open(auth_url)
                print("Opened %s in your browser" % auth_url)
            except:
                self.tts.speak("Please navigate here: %s" % auth_url)
                print("Please navigate here: %s" % auth_url)
                print

            response = raw_input("Enter the URL you were redirected to: ")
            code = auth.parse_response_code(response)
            token = auth.get_access_token(code)
        else:
            if auth.is_token_expired(token):
                token = auth.refresh_access_token(token['refresh_token'])

        return token['access_token']

    def __adjust_volume(self, volume_change):
        sp = spotipy.Spotify(auth=self.__get_access_token())
        device = sp.current_playback()['device']
        volume = device['volume_percent']
        new_volume = min(100, max(0, volume + volume_change))
        sp.volume(new_volume)

    def stop_playing(self):
        """ Turn on something. """
        sp = spotipy.Spotify(auth=self.__get_access_token())
        sp.pause_playback()

    def resume_playing(self):
        """ Turn of something. """
        sp = spotipy.Spotify(auth=self.__get_access_token())
        sp.start_playback()

    def next_track(self):
        """ Turn of something. """
        sp = spotipy.Spotify(auth=self.__get_access_token())
        sp.next_track()

    def previous_track(self):
        """ Turn of something. """
        sp = spotipy.Spotify(auth=self.__get_access_token())
        sp.previous_track()

    def decrease_volume(self):
        self.__adjust_volume(-12)

    def increase_volume(self):
        self.__adjust_volume(12)

    def give_current_track_info(self):
        sp = spotipy.Spotify(auth=self.__get_access_token())
        current_playback = sp.current_playback()
        if current_playback['item']:
            artist = current_playback['item']['artists'][0]['name'] #.encode('utf-8')
            title = current_playback['item']['name']
            self.tts.speak(u"This is {} by {}".format(title, artist))


# -*- coding: utf-8 -*-
"""
Created on Tue Dec  1 14:59:15 2020

@author: Ozaki
"""

import pygame
import sys
from hall import Hall
from livingroom import Livingroom
from privateroom import Privateroom
from item_base import Item_Base
from ending import Ending

def main():
    width = 1024
    height = 768
    pygame.init()
    screen = pygame.display.set_mode((width + 80, height))
    clock = pygame.time.Clock()

    lock_flag = [False, False, False]

    item_get = [False, False, False, False]
    item_use = [False, False, False, False]
    item_ctrl = Item_Base(screen, item_get, item_use, width, height)

    room_state = 0
    next_room_state = 0
    room_ctrl = [Hall(screen, lock_flag, item_get, item_use),
                 Livingroom(screen, lock_flag, item_get, item_use),
                 Privateroom(screen, lock_flag, item_get, item_use),
                 Ending(screen, lock_flag, item_get, item_use)]

    while True:

        """ マウスカーソル位置 """
        x, y = pygame.mouse.get_pos()
        room_ctrl[room_state].pos_event(x, y)

        """ クリックイベント，キーイベント """
        for event in pygame.event.get():
            if event.type == pygame.MOUSEBUTTONDOWN and event.button == 1:
                if x < width:
                    room_ctrl[room_state].click_event(x, y)
                else:
                    item_ctrl.click_event(int(y / 64))
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_LEFT:
                    room_state = (room_state - 1) % -3
                elif event.key == pygame.K_RIGHT:
                    room_state = (room_state + 1) % 3
                elif event.key == pygame.K_a:
                    lock_flag[0] = not lock_flag[0]
                elif event.key == pygame.K_b:
                    lock_flag[1] = not lock_flag[1]
                elif event.key == pygame.K_c:
                    lock_flag[2] = not lock_flag[2]
                elif event.key == pygame.K_0:
                    item_get[0] = not item_get[0]
                    item_use[0] = False
                elif event.key == pygame.K_1:
                    item_get[1] = not item_get[1]
                    item_use[1] = False
                elif event.key == pygame.K_2:
                    item_get[2] = not item_get[2]
                    item_use[2] = False
                elif event.key == pygame.K_3:
                    item_get[3] = not item_get[3]
                    item_use[3] = False

            if event.type == pygame.QUIT:
                pygame.quit()
                sys.exit()

        """ 処理 """
        room_ctrl[room_state].do()
        item_ctrl.do()

        """ 描画 """
        screen.fill((0, 0, 0))
        room_ctrl[room_state].draw()
        item_ctrl.draw()
        pygame.display.update()

        """ 部屋移動 """
        next_room_state = room_ctrl[room_state].next_state()
        if room_state != next_room_state:
            surface = pygame.Surface((width, height), pygame.SRCALPHA)
            alpha = 0
            for i in range(12):
                screen.fill((0,0,0))
                surface.fill((0, 0, 0, alpha))
                room_ctrl[room_state].draw()
                item_ctrl.do()
                item_ctrl.draw()
                screen.blit(surface, (0, 0))
                pygame.display.update()
                clock.tick(30)
                alpha = alpha + 20
            room_state = next_room_state
            for i in range(12):
                screen.fill((0,0,0))
                surface.fill((0, 0, 0, alpha))
                room_ctrl[room_state].draw()
                item_ctrl.do()
                item_ctrl.draw()
                screen.blit(surface, (0, 0))
                pygame.display.update()
                clock.tick(30)
                alpha = alpha - 20

        clock.tick(30)

if __name__ == '__main__':
    main()


from dataclasses import dataclass

# from smallcalc.calc_lexer import INTEGER, LITERAL

@dataclass
class IntegerNode:
    value: int

    def __init__(self, value):
        self.value = int(value)
        self.type = 'integer'

    def asdict(self):
        return {
            'type': self.type,
            'value': self.value
        }

@dataclass
class LiteralNode:
    value: str

    def __init__(self, value):
        self.value = str(value)
        self.type = 'literal'

    def asdict(self):
        return {
            'type': self.type,
            'value': self.value
        }

class ExpressionNode:
    value: str

    def __init__(self, left, right, operator):
        self.type = 'binary'
        self.left = left
        self.right = right
        self.operator = operator

    def asdict(self):
        return {
            'type': self.type,
            'left': self.left.asdict(),
            'right': self.right.asdict(),
            'operator': self.operator.asdict()
        }


import numpy
import requests
import re
import time
import sys
import csv
import pandas as pd
from collections import Counter
from lxml import html
from tqdm import *

page_cstart = 0
href = []
authors = []
year = []
citation = []
journal = []
ref_web = []
while 1:
  page_url = 'https://scholar.google.com/citations?user=__XZ88LXYAAAAJ=en&cstart=' + str(page_cstart) + '&pagesize=100'
  page = requests.get(page_url)
  tree = html.fromstring(page.content)
  m = tree.xpath('//td[@class="gsc_a_e"]/text()')
  if len(m)>0 and m[0] == 'There are no articles in this profile.':
    break
  page_cstart = page_cstart + 100
  href = href + tree.xpath('//a[@class="gsc_a_at"]/text()')
  authors_temp = tree.xpath('//div[@class="gs_gray"]')
  for i in range(len(authors_temp)):
    if i%2 == 0:
      if authors_temp[i].text:
        authors.append(authors_temp[i].text)
      else:
        authors.append('')
    else:
      if authors_temp[i].text:
        journal.append(authors_temp[i].text)
      else:
        journal.append('')
  year_temp = tree.xpath('//span[@class="gsc_a_h"]')
  year_temp = year_temp[1:]
  for x in year_temp:
    if x.text:
      year.append(x.text)
    else:
      year.append('')
  citation_temp = tree.xpath('//a[contains(@class,"gsc_a_ac")]')
  for x in citation_temp:
    if x.text:
      citation.append(x.text)
    else:
      citation.append('')
  websites = tree.findall('.//a')
  for x in websites:
    try:
      if x.attrib['class'] == 'gsc_a_at':
        ref_web.append(x.attrib['href'])
    except KeyError:
      pass

ref_authors = []
ref_description = []
for ref_page in tqdm(ref_web):
  ref_url = 'https://scholar.google.com/' + ref_page
  page = requests.get(ref_url)
  tree = html.fromstring(str(page.content).replace('<br>',' '))
  ref_div = tree.findall('.//div')
  for i in range(len(ref_div)):
    if ref_div[i].text == 'Authors':
      ref_authors.append(ref_div[i+1].text)
    if ref_div[i].text == 'Description':
      ref_description.append(ref_div[i+1].text)

#### set With missing values in citation to 0
citation_index = [i for i in range(len(citation)) if citation[i]=='\xa0']
my_citation = numpy.array(citation)
my_citation[citation_index] = 0

#### extract name of journals
my_journal = []
for j_temp in journal:
  my_journal.append(re.split(' [0-9]', j_temp)[0])
my_journal = [element.upper() for element in my_journal]

#### Get
outf = [href, authors, year, my_citation.tolist(), my_journal]
my_df = pd.DataFrame(outf)
my_df.to_csv("output.csv")

with open("output.csv", 'wb') as f:
  writer = csv.writer(f)
  writer.writerows(outf)


import json
from unittest.mock import patch

import pytest

from src.main import app as flask_app
from src.main import get_port, get_secret_key
from src.file import FileOpenMock

@pytest.fixture
def app():
    yield flask_app

@pytest.fixture
def client(app):
    return app.test_client()

def test_index(client):
    res = client.get('/')
    assert res.status_code == 200
    expected = {'hello': 'world'}
    assert expected == json.loads(res.get_data(as_text=True))

@patch("src.main.os.listdir")
@patch('src.main.os.chdir')
def test_get_all_files(mock_chdir, mock_listdir, client):
    mock_listdir.return_value = ['file.txt', 'file.doc', 'my4.txt', 'my1.txt', 'my2.txt', '.txt', 'my.txt', 'my3.txt']
    res = client.get('/files')
    """res.get_data(), res.status_code, res """
    assert res.status_code == 200

@patch("src.main.os.listdir")
@patch('src.main.os.chdir')
def test_get_all_files_exception(mock_chdir, mock_listdir, client):
    mock_listdir.side_effect = Exception()
    res = client.get('/files')
    """res.get_data(), res.status_code, res """
    assert res.status_code == 500

@patch('src.main.os.chdir')
@patch('src.main.open')
def test_get_file_content_exception(mock_file, mock_chdir, client):
    mock_file.side_effect = Exception('[Errno 2] No such file or directory: f.txt!')
    res = client.get('/files/abc')
    """res.get_data(), res.status_code, res """
    assert res.status_code == 500

@patch('src.main.os.chdir')
@patch('src.main.fnmatch.filter')
def test_get_all_files(mock_file, mock_chdir, client):
    mock_file.side_effect = Exception('Something went wrong')
    res = client.get('/files')
    """res.get_data(), res.status_code, res """
    assert res.status_code == 500

@patch("src.main.f")
@patch('src.main.open')
@patch('src.main.os.chdir')
def test_get_file_content(mock_chdir, mock_open, mock_read, client):
    mock_read.return_value = 'content of file'
    res = client.get('/files/myu')
    """res.get_data(), res.status_code, res """
    assert res.status_code == 200

@patch('src.main.os.environ.get')
def test_get_port(mock_env):
    mock_env.return_value = 4000
    tmp = get_port()
    assert tmp == 4000
    mock_env.return_value = None
    tmp = get_port()
    assert tmp == 4000

@patch('src.main.os.environ.get')
def test_get_env(mock_key, client):
    mock_key.return_value = 'SECRET_KEY_TEST'
    tmp = get_secret_key()
    res = client.get('/env')
    assert res.status_code == 200
    assert tmp == 'SECRET_KEY_TEST'

    mock_key.return_value = None
    tmp = get_secret_key()
    assert res.status_code == 200
    assert tmp == 'SECRET_KEY'

@patch('src.main.os.environ.get')
def test_get_env_exception(mock_key, client):
    mock_key.side_effect = Exception()
    res = client.get('/env')
    assert res.status_code == 500