from __future__ import absolute_import
from __future__ import unicode_literals
from __future__ import print_function

import io
import os

import attr
import click
click.disable_unicode_literals_warning = True
from license_expression import Licensing
import saneyaml

from licensedcode import cache
from licensedcode import models
from licensedcode import match_hash

"""
A script to generate license detection rules from a simple text data file.

Note that the .yml data files are validated and the script will report
errors and stop if a rule data is not valid.

Typicaly validation errors include:
- missing license expressions
- unknown license keys
- presence of multiple is_license_xxx: flags (only one is allowed)

The file buildrules-template.txt is an template of this data file.
The file buildrules-exmaples.txt is an example of this data file.
This file contains one or more block of data such as:

 ----------------------------------------
 license_expression: foo OR bar
 relevance: 100
 is_license_notice: yes
 ---
 This is licensed under a choice of foo or bar.
 ----------------------------------------

The first section (before ---) contains the data in valid YAML that
will be saved in the rule's .yml file.

The second section (after ---) contains the rule text saved in the
the .RULE text file.

If a RULE already exists, it is skipped.
"""

@attr.attrs(slots=True)
class LicenseRule(object):
    data = attr.ib()
    text = attr.ib()
    raw_data = attr.ib(default=None)

    def __attrs_post_init__(self, *args, **kwargs):
        self.raw_data = rdat = '\n'.join(self.data).strip()
        self.text = '\n'.join(self.text).strip()

        # validate YAML syntax
        try:
            self.data = saneyaml.load(rdat)
        except:
            print('########################################################')
            print('Invalid YAML:')
            print(rdat)
            print('########################################################')
            raise

def load_data(location='00-new-licenses.txt'):
    with io.open(location, encoding='utf-8') as o:
        lines = o.read().splitlines(False)

    rules = []

    data = []
    text = []
    in_data = False
    in_text = False
    last_lines = []
    for ln, line in enumerate(lines, 1):
        last_lines.append(': '.join([str(ln), line]))
        if line == '----------------------------------------':
            if not (ln == 1 or in_text):
                raise Exception('Invalid structure: #{ln}: {line}\n'.format(**locals()) +
                                '\n'.join(last_lines[-10:]))

            in_data = True
            in_text = True
            if data and ''.join(text).strip():
                rules.append(LicenseRule(data, text))
            data = []
            text = []
            continue

        if line == '---':
            if not in_data:
                raise Exception('Invalid structure: #{ln}: {line}\n'.format(**locals()) +
                                '\n'.join(last_lines[-10:]))

            in_data = False
            in_text = True
            continue

        if in_data:
            data.append(line)
            continue

        if in_text:
            text.append(line)
            continue

    return rules

def rule_exists(text):
    """
    Return the matched rule identifier if the text is an existing rule matched
    exactly, False otherwise.
    """
    idx = cache.get_index()

    matches = idx.match(query_string=text)
    if not matches:
        return False
    if len(matches) > 1:
        return False
    match = matches[0]
    if match.matcher == match_hash.MATCH_HASH:
        return match.rule.identifier

def all_rule_tokens():
    """
    Return a set of tuples of tokens, one corresponding to every existing and
    added rules. Used to avoid duplicates.
    """
    rule_tokens = set()
    for rule in models.get_rules():
        rule_tokens.add(tuple(rule.tokens()))
    return rule_tokens

def find_rule_base_loc(license_expression):
    """
    Return a new, unique and non-existing base name location suitable to create a new
    rule.
    """
    template = (license_expression
        .lower()
        .strip()
        .replace(' ', '_')
        .replace('(', '')
        .replace(')', '')
        +'_{}')
    idx = 1
    while True:
        base_name = template.format(idx)
        base_loc = os.path.join(models.rules_data_dir, base_name)
        if not os.path.exists(base_loc + '.RULE'):
            return base_loc
        idx += 1

def validate_license_rules(rules_data, licensing):
    """
    Checks all rules and return a list of errors
    """
    errors = []
    for rule in rules_data:
        is_negative = rule.data.get('is_negative')
        is_false_positive = rule.data.get('is_false_positive')
        fp_flags = [f for f in [is_negative, is_false_positive] if f]
        if len(fp_flags) > 1:
            msg = 'Invalid rule with mutually exclusive false positive/negative flags: {}'.format(rule)
            errors.append(msg)
        if is_negative:
            continue

        relevance = rule.data.get('relevance', 0) or 0
        relevance = float(relevance)
        if relevance < 0 or relevance > 100:
            msg = 'Invalid rule relevance: {}'.format(rule)
            errors.append(msg)

        minimum_coverage = rule.data.get('minimum_coverage', 0) or 0
        minimum_coverage = float(minimum_coverage)
        if minimum_coverage < 0 or minimum_coverage > 100:
            msg = 'Invalid rule minimum_coverage: {}'.format(rule)
            errors.append(msg)

        is_license_notice = rule.data.get('is_license_notice')
        is_license_text = rule.data.get('is_license_text')
        is_license_reference = rule.data.get('is_license_reference')
        is_license_tag = rule.data.get('is_license_tag')

        type_flags = [f for f in [is_license_notice, is_license_text, is_license_tag, is_license_reference] if f]
        if len(type_flags) != 1:
            msg = 'Invalid rule is_license_* flags. Only one allowed. At least one needed: {}'.format(rule)
            errors.append(msg)

        license_expression = rule.data.get('license_expression')
        if not license_expression:
            msg = 'Missing license_expression for rule: {}'.format(rule)
            errors.append(msg)

        try:
            licensing.parse(license_expression, validate=True, simple=True)
        except Exception as e:
            msg = 'Invalid license_expression for rule: {}\n{}'.format(rule, str(e))
            errors.append(msg)

    return errors

@click.command()
@click.argument('licenses_file', type=click.Path(), metavar='FILE')
@click.help_option('-h', '--help')
def cli(licenses_file):
    """
    Create rules from a text file with delimited blocks of metadata and texts.

    As an example a file would contains one of more blocks such as this:

\b
        ----------------------------------------
        license_expression: lgpl-2.1
        relevance: 100
        is_license_notice: yes
        ---
        This program is free software; you can redistribute it and/or modify
        it under the terms of the GNU Lesser General Public License
        version 2.1 as published by the Free Software Foundation;
        ----------------------------------------
    """

    rules_data = load_data(licenses_file)
    rules_tokens = all_rule_tokens()

    licenses = cache.get_licenses_db()
    licensing = Licensing(licenses.values())

    print()
    errors = validate_license_rules(rules_data, licensing)
    if errors:
        print('Invalid rules: exiting....')
        for error in errors:
            print(error)
            print()

        raise Exception('Invalid rules: exiting....')

    print()
    for rule in rules_data:
        is_negative = rule.data.get('is_negative')
        is_false_positive = rule.data.get('is_false_positive')
        existing = rule_exists(rule.text)
        if existing and not is_negative:
            print('Skipping existing non-negative rule:', existing, 'with text:\n', rule.text[:50].strip(), '...')
            continue

        if is_negative:
            base_name = 'not-a-license'
        else:
            license_expression = rule.data.get('license_expression')
            license_expression = str(licensing.parse(license_expression, validate=True, simple=True))
            base_name = license_expression
            if is_false_positive:
                base_name = 'false-positive_' + base_name

        base_loc = find_rule_base_loc(base_name)

        data_file = base_loc + '.yml'
        with io.open(data_file, 'w', encoding='utf-8') as o:
            o.write(rule.raw_data)

        text_file = base_loc + '.RULE'
        with io.open(text_file, 'w', encoding='utf-8') as o:
            o.write(rule.text)

        rulerec = models.Rule(data_file=data_file, text_file=text_file)
        rule_tokens = tuple(rulerec.tokens())
        if rule_tokens in rules_tokens:
            # cleanup
            os.remove(text_file)
            os.remove(data_file)
            print('Skipping already added rule with text for:', base_name)
        else:
            rules_tokens.add(rule_tokens)
            rulerec.dump()
            models.update_ignorables(rulerec, verbose=False)
            print('Rule added:', 'file://' + rulerec.data_file, '\n', 'file://' + rulerec.text_file,)

if __name__ == '__main__':
    cli()


from . import memory
from .. import errors

""" Map the current 'pc' address to an 'absolute' address.  The
structure of the 'absolute' address is somewhat arbitrary, but  the
idea is to be fairly sensible with the layout to make the mapping
sensible and efficient.

segments:

cartridge - ROM - 0x4000 * 64 (0x3F)
cartridge - RAM - 0x4000 * 2
system    - RAM - 0x2000

ROM - 0x000000 - 0x0FFFFF
ROM - 0x100000 - 0x1FFFFF
RAM   0x200000 - 0x207FFF
RAM   0x208000 - 0x20A000

-> Total = 3F 0x42

mapped memory:
    0x0000 - 0x03FF                     -> ROM (bank 0) (0x0000 - 0x03FF)
    0x0400 - 0x3FFF + (0xFFFD(0-0x3F))          -> ROM (bank x) (0x0400 - 0x4000)
    0x4000 - 0x7FFF + (0xFFFE(0-0x3F))          -> ROM (bank x) (0x0000 - 0x4000)
    0x8000 - 0xBFFF + (0xFFFF(0-0x3F) + 0xFFFC(0x0C)) -> ROM (bank x) (0x0000 - 0x4000) or RAM (bank x)
- 11x0
    0xC000 - 0xDFFF                     -> System RAM   (0x0000 - 0x2000)
    0xE000 - 0xFFFF                     -> System RAM   (0x0000 - 0x2000) (mirror)

    0x0000 - 0x03FF                     -> ROM (bank 0) (0x0000 - 0x03FF)
    0x0400 - 0x3FFF + (0xFFFD)          -> ROM (bank x) (0x0400 - 0x4000)

    0x4000 - 0x7FFF + (0xFFFE)          -> ROM (bank x) (0x0000 - 0x4000)
    0x8000 - 0xBFFF + (0xFFFF + 0xFFFC) -> ROM (bank x) (0x0000 - 0x4000) or RAM (bank x)

    0xC000 - 0xDFFF                     -> System RAM   (0x0000 - 0x2000)
    0xE000 - 0xFFFF                     -> System RAM   (0x0000 - 0x2000) (mirror)

    absolute = page[(address >> 13)] | address & 0x1FFF

    | 6-bit - page 0| 6-bit - page 1| 6-bit - page 2 | 1-bit - ram/rom select | 1-bit - ram select | 3 - sub-page address | 13 - address

    all bits -> 6 + 6 + 6 + 18

"""

class MemoryAbsolute(memory.MemoryBase):
    """ Map the PC to an absolute address.
    """
    BANK_SIZE     = 0x4000
    MAX_BANKS     = 64;
    NUM_PAGES     = 3;

    def __init__(self):
        self._last_page_number = 0
        self._last_bank_number = 0

        self._ram     = [0] * self.RAM_SIZE * 2

        # 'Page0' always contains bank 0 for the first 'PAGE0' bytes, then up
        # to the page boundary may contain a different bank. This array is used
        # to copy the 'mixed' banks to all possible 'page0' pages.
        self._page0_copies = [[0] * self.BANK_SIZE for x in range(self.MAX_BANKS)]

        # Remember the last bank assignments per page, and only swap if they differ
        self.MAX_ROM_SIZE = 0x100000
        self.ABSOLUTE_PAGE_0_ROM_OFFSET = 0x000000
        self.ABSOLUTE_PAGE_X_ROM_OFFSET = 0x100000
        self.ABSOLUTE_CART_RAM_OFFSET   = 0x200000
        self.ABSOLUTE_SYS_RAM_OFFSET    = 0x208000
        self.ABSOLUTE_SEGMENT_SIZE      =   0x2000

        self._page_2     = 0
        self._ram_select = 0

        self._upper_mappings = [0x0000,
                                self.ABSOLUTE_PAGE_X_ROM_OFFSET + (self.ABSOLUTE_SEGMENT_SIZE * 1),
                                self.ABSOLUTE_PAGE_X_ROM_OFFSET + (self.ABSOLUTE_SEGMENT_SIZE * 2),
                                self.ABSOLUTE_PAGE_X_ROM_OFFSET + (self.ABSOLUTE_SEGMENT_SIZE * 3),
                                self.ABSOLUTE_PAGE_X_ROM_OFFSET + (self.ABSOLUTE_SEGMENT_SIZE * 4),
                                self.ABSOLUTE_PAGE_X_ROM_OFFSET + (self.ABSOLUTE_SEGMENT_SIZE * 5),
                                self.ABSOLUTE_SYS_RAM_OFFSET, self.ABSOLUTE_SYS_RAM_OFFSET]

        # Complete memory map
        self._memory_map     = [0] * max(self.ABSOLUTE_CART_RAM_OFFSET + self.ABSOLUTE_SEGMENT_SIZE,
                                         self.ABSOLUTE_SYS_RAM_OFFSET + self.ABSOLUTE_SEGMENT_SIZE)

    def get_absolute_address(self, address):
        return self._upper_mappings[address >> 13] | address & 0x1FFF

    def get_max_absolute_instruction_address(self):
        return self.ABSOLUTE_PAGE_X_ROM_OFFSET + self.MAX_ROM_SIZE

    def read(self, address):
        """ Assumes 'address' is 'int' or accepts >> operator. """
        # Using constants here to increase speed.
        return self._memory_map[self._upper_mappings[address >> 13] | address & 0x1FFF]

    def readArray(self, address, length):

        result = [0] * length

        for i in range(length):
            result[i] = self.read(address + i)

        return result

    def read16(self, address):
        return self.read(address) + (self.read(address + 1) << 8)

    def writeMulti(self, dest, src, length):
        """ write multiple bytes to memory.
        """

        if (((dest + length) >= self.MEMMAPSIZE) or (dest < self.RAM_OFFSET)):
            errors.warning("Write out of range %x"%(dest + length))
            return

        for i in range(length):
            self.write(dest+i, self.read(src+i))

    def write(self, address, data):
        # Should check inputs, see which instructions/condition can overflow
        self._write(address, int(data) & 0xFF)

    def set_cartridge(self, cartridge):
        self.cartridge = cartridge

        self._initialise_read()

    def _initialise_read(self):
        """ Un-optimised address translation, uses paging registers.
        """

        self._populate_absolute_memory_map()
        self.write(0xFFFC, 0)
        self.write(0xFFFD, 0)
        self.write(0xFFFE, 1)
        self.write(0xFFFF, 2)

    def _populate_absolute_memory_map(self):
        for bank in range(self.cartridge.num_banks):
            # Page '0'/'1' lookup
            for address in range(self.PAGE0):
                bank_address = address & self.LOWERMASK # BANK_MASK
                self._memory_map[self.ABSOLUTE_PAGE_0_ROM_OFFSET  + bank_address + (bank * self.BANK_SIZE)] =  self.cartridge.cartridge_banks[0][bank_address]
                self._memory_map[self.ABSOLUTE_PAGE_X_ROM_OFFSET  + bank_address + (bank * self.BANK_SIZE)] =  self.cartridge.cartridge_banks[bank][bank_address]

            for address in range(self.PAGE0, self.PAGE1):
                bank_address = address & self.LOWERMASK # BANK_MASK
                self._memory_map[self.ABSOLUTE_PAGE_0_ROM_OFFSET  + bank_address + (bank * self.BANK_SIZE)] =  self.cartridge.cartridge_banks[bank][bank_address]
                self._memory_map[self.ABSOLUTE_PAGE_X_ROM_OFFSET  + bank_address + (bank * self.BANK_SIZE)] =  self.cartridge.cartridge_banks[bank][bank_address]

    def _write(self, address, data):
        address = address & self.ADDRESS_MASK # ADDRESS_MASK

        if(address >= self.RAM_OFFSET):
            if(address >= self.MEMREGISTERS):
                # Should make these conditiona,
                if (address == self.PAGE0_BANK_SELECT_REGISTER):
                    self._upper_mappings[0] = self.ABSOLUTE_PAGE_0_ROM_OFFSET + (self.BANK_SIZE * data)
                    self._upper_mappings[1] = self.ABSOLUTE_PAGE_X_ROM_OFFSET + (self.BANK_SIZE * data) + self.ABSOLUTE_SEGMENT_SIZE
                elif (address == self.PAGE1_BANK_SELECT_REGISTER):
                    self._upper_mappings[2] = self.ABSOLUTE_PAGE_X_ROM_OFFSET + (self.BANK_SIZE * data)
                    self._upper_mappings[3] = self.ABSOLUTE_PAGE_X_ROM_OFFSET + (self.BANK_SIZE * data) + self.ABSOLUTE_SEGMENT_SIZE
                elif ((address == self.RAM_SELECT_REGISTER) or (address == self.PAGE2_BANK_SELECT_REGISTER)):
                    # Set the data first, to allow read-back of the registers.
                    #self._memory_map[self._upper_mappings[address >> 13] | address & 0x1FFF] = data

                    if (address == self.RAM_SELECT_REGISTER):
                        self._ram_select = data
                    elif (address == self.PAGE2_BANK_SELECT_REGISTER):
                        self._page_2     = data

                    if (self._ram_select & self.MAPCARTRAM): # page2_is_cartridge_ram
                      # Cart RAM select.
                      if (self._ram_select & self.PAGEOFRAM):
                        self._upper_mappings[4] = self.ABSOLUTE_CART_RAM_OFFSET + (self.ABSOLUTE_SEGMENT_SIZE * 2)
                        self._upper_mappings[5] = self.ABSOLUTE_CART_RAM_OFFSET + (self.ABSOLUTE_SEGMENT_SIZE * 3)
                      else:
                        self._upper_mappings[4] = self.ABSOLUTE_CART_RAM_OFFSET
                        self._upper_mappings[5] = self.ABSOLUTE_CART_RAM_OFFSET + self.ABSOLUTE_SEGMENT_SIZE
                    else:
                        self._upper_mappings[4] = self.ABSOLUTE_PAGE_X_ROM_OFFSET + (self.BANK_SIZE * self._page_2)
                        self._upper_mappings[5] = self.ABSOLUTE_PAGE_X_ROM_OFFSET + (self.BANK_SIZE * self._page_2) + self.ABSOLUTE_SEGMENT_SIZE

        absolute_address = self.get_absolute_address(address)
        if (absolute_address >= min(self.ABSOLUTE_CART_RAM_OFFSET, self.ABSOLUTE_SYS_RAM_OFFSET)):
            self._memory_map[absolute_address] = data


#Image to pencil sketch
#Package installed: py-opencv

import cv2

image = cv2.imread("Anisha.jpg")
gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
inverted = 255-gray_image
blur = cv2.GaussianBlur(inverted, (121, 121), 0)
inverted_blur = 255-blur
sketch = cv2.divide(gray_image, inverted_blur, scale=256.0)
cv2.imwrite("sketch_image.png", sketch)


from flask import Flask, render_template, flash, redirect
from .forms import EmailForm
import os
from . import emailHunter
from . import emailProspect
from urllib.parse import urlparse
from flask import jsonify, request

app = Flask(__name__)
app.config.update(dict(
    SECRET_KEY=  os.environ.get('SECRET_KEY'),
    WTF_CSRF_SECRET_KEY=  os.environ.get('WTF_CSRF_SECRET_KEY')
))

def clean_url(domain):
    domain = urlparse(domain)
    domain = domain.netloc if domain.netloc else domain.path

    if "www." in domain:
        domain = domain[4:]
    return domain

def generate_email(full_name, domain):
    domain = clean_url(domain)

    if not emailHunter.get_remaining_calls():
        return "Need To Upgrade Account"

    if emailHunter.find_email(full_name, domain):
        return emailHunter.find_email(full_name, domain)
    elif emailProspect.find_email(full_name, domain):
        return emailProspect.find_email(full_name, domain)
    return emailHunter.guess_email(full_name, domain)

@app.route('/', methods=['GET', 'POST'])
def index():
    form = EmailForm()
    if form.validate_on_submit():
        email_result = generate_email(str(form.full_name.data), str(form.company_url.data))
        flash('{}'.format(email_result))
        return redirect('/')
    return render_template('main.html', title='Find Email', form=form)

@app.route('/api', methods=['GET'])
def api():
    if 'name' in request.args and 'url':
        email_result = generate_email(str(request.args['name']), str(request.args['url']))
        return jsonify(
            email=email_result
            )

if __name__ == "__main__":
    app.run(debug=True, use_reloader=True)


import datetime
import re
import calendar
from __future__ import division # da mi deljenje celih stevil vrne float

def decode_duration(duration=False):
    years = months = days = 0
    pattern = r"(\d{1,2})-(\d{1,2})-(\d{1,2})"
    if duration:
        m = re.match(pattern, duration)
        if m:
            g = m.groups()
            years = int(g[0])
            months = int(g[1])
            days = int(g[2])
    return [years, months, days]

def calc_date_duration(date_from=False, date_to=False, past_work=False):
    '''

    :param date_from:
    :param date_to:
    :param past_work:
    :return: list s podatki let, mesecev, dni delovne dobe
    '''
    _month_day = [31, -1, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
    _date_from = date_from             #datetime.strptime(date_from, '%Y-%m-%d')
    _date_to = date_to                 # and datetime.strptime(date_to, '%Y-%m-%d') or datetime.today()
    _increment = 0
    if _date_from.day > _date_to.day:
        _increment = _month_day[_date_from.month - 1]
    if _increment == -1:
        if calendar.isleap(_date_from.year):
            _increment = 29
        else:
            _increment = 28
    if _increment != 0:
        _days = _date_to.day + _increment - _date_from.day
        _increment = 1
    else:
        _days = _date_to.day - _date_from.day
        _increment = 0

    if _date_from.month + _increment > _date_to.month:
        _months = (_date_to.month + 12) - (_date_from.month + _increment)
        _increment = 1
    else:
        _months = _date_to.month - (_date_from.month + _increment)
        _increment = 0
    _years = _date_to.year - (_date_from.year + _increment)

    if past_work:
        _past_years, _past_months, _past_days = decode_duration(past_work)
        if _past_days + _days > 30:
            _days += _past_days - 30
            _increment = 1
        else:
            _days += _past_days
            _increment = 0
        if _past_months + _months + _increment >= 12:
            _months += _past_months + _increment - 12
            _increment = 1
        else:
            _months += _past_months + _increment
            _increment = 0
        _years += _past_years + _increment

    return [_years, _months, _days]
    # -*- coding: utf-8 -*-

def main():
    date_from = datetime.date(1998, 12, 10)
    date_to = datetime.date(2018, 2, 1)

    working_period = calc_date_duration(date_from, date_to)
    delta = date_from-date_to
    diff = abs(delta.days)
    wpdays = working_period[0] * 365 + int(working_period[1] * (365/12)) + working_period[2]
    # wpdays = working_period[0] * 365 + working_period[1] * 30 + working_period[2]
    print (working_period)
    print (diff,  wpdays)
    years = diff // 365
    r = diff % 365
    months = r // 30
    days = r % 30
    print ("{}-{}-{}")
    print ("Calculated days ", years, months, days)
    s = "{}"
    print ("Calculated days ", years, months, days)

class AllWorkingPeriods():
    def __init__(self):
        self.working_periods = []

    def add_working_period(self, date_from, date_to, working_hours):
        self.working_periods.append(date_from, date_to, working_hours)

    def calculate_work_periods(self):
        self.years = 0
        self.months = 0
        self.days = 0
        for wp in self.working_periods:
            self.years += wp.years
            self.months += wp.months
            self.days += wp.days

        self.months += self.days // 30
        self.days = self.days % 30

        self.years += self.months // 12
        self.months = self.months % 12

class WorkingPeriod:
    FULL_TIME = 8
    def __init__(self, date_from, date_to, working_hours ):
        self.date_from = date_from
        self.date_to = date_to
        self.working_hours = min(working_hours, WorkingPeriod.FULL_TIME)

        full_months = max(0, Month.count_months(d_first_full, d_last_full))
        firs_month_days = (30-d1.day) + 1

        last_day_in_month = calendar.monthrange(d2.year, d2.month)[1]
        if d2.day == last_day_in_month:
            last_month_days = 30
        else:
            last_month_days = 30 - (30 - d2.day)

        full_months += (firs_month_days + last_month_days) // 30
        days = (firs_month_days + last_month_days) % 30

        if working_hours == WorkingPeriod.FULL_TIME:
            self.years = full_months // 12
            self.months = full_months % 12
            self.days = days
        else:
            wd = int(round((full_months * 30 + days) * working_hours / WorkingPeriod.FULL_TIME,0))
            # ce je vec kot 0.6 zaokrozim navzgor sicer navzdol
            self.years = wd // 365
            wd = wd % 365
            self.months = wd // 30
            self.days = wd % 30

    def __str__(self):
        return "{:02d}-{:02d}-{:02d}".format(self.years,
                                             self.months,
                                             self.days)

class Month:
    @classmethod
    def next_month_firs_day(cls, d):
        month = d.month + 1
        day = 1
        if month == 13:
            month = 1
            year = d.year + 1
        else:
            year = d.year
        return cls(year, month)

    @classmethod
    def previous_month_last_day(cls, d):
        month = d.month - 1
        if month == 0:
            month = 12
            year = d.year = 1
        else:
            year = d.year

    def __init__(self, year, month , day=1):
        self.year = year
        self.month = month
        self.day = day

    def date(self):
        return datetime.date(self.year, self.month, self.day)

    def inc_month(self):
        self.month += 1
        if self.month == 13:
            self.month = 1
            self.year += 1

    def __str__(self):
        return "{}-{:02d}".format(self.month, self.year)

    def __eq__(self, other):
        if self.year == other.year and self.month == other.month:
            return True
        else:
            return False

    def __lt__(self, other):
        if (self.year < other.year):
            return True
        elif self.year > other.year:
            return False
        else:
            return self.month < other.month

    @staticmethod
    def count_months(_d1, _d2):
        # vrne stevilo mesecev  med d1 in d2, vkljucno z d1
        # ce sta meseca enaka vrne 1, ker se steje da je ta mesec delal
        if _d1 > _d2:
            d1 = _d2
            d2 = _d1
            sign = -1
        else:
            d1 = _d1
            d2 = _d2
            sign = 1

        if d1.year == d2.year:
            return (d2.month - d1.month) + 1

        # 12 mesecev v celih letih
        months = (d2.year - d1.year -1) * 12
        # pristejem mesece v prvem letu
        months += 12 - (d1.month - 1)
        # pristejem mesece v zadnjem letu
        months += d2.month
        return months * sign

if __name__ == "__main__":
    # main()
    d1 = datetime.date(2017, 3, 15)
    d2 = datetime.date(2019, 5, 3)

    d_first_full = Month.next_month_firs_day(d1)
    d_last_full = Month.previous_month_last_day(d2)

    full_months = max(0, Month.count_months(d_first_full, d_last_full))
    firs_month_days = (30-d1.day) + 1
    last_day_in_month = calendar.monthrange(d2.year, d2.month)[1]
    if d2.day == last_day_in_month:
        last_month_days = 30
    else:
        last_month_days = 30 - (30 - d2.day)

    full_months += (firs_month_days + last_month_days) // 30
    days = (firs_month_days + last_month_days) % 30

    print ('Worktime period',
           "{:02d}-{:02d}-{:02d}".format(full_months // 12, full_months % 12, days))

    print (d_first_full)
    print (d_last_full)
    print ("Prvi je majsi od zadnjega", d_first_full < d_last_full)

    r1 = Month.next_month_firs_day(datetime.date(2019, 6, 15))
    r2 = Month.previous_month_last_day(datetime.date(2019, 8, 3))
    print ("Meseca", r1, ' in ', r2, " sta enaka ", r1  ==  r2)
    print  ('Stevilo mesecev je', Month.count_months(d_first_full, d_last_full))


import os
from selenium import webdriver
import time

def file_upload_test(link):
	try:
		browser = webdriver.Chrome()
		browser.get(link)

		find_by_name(browser, "firstname", "Ivan")
		find_by_name(browser, "lastname", "Ivanov")
		find_by_name(browser, "email", "my.mail@post.ru")

		upload_dutton = browser.find_element_by_id("file")
		upload_dutton.send_keys(get_file_path())

		submit_button = browser.find_element_by_css_selector(".btn.btn-primary")
		submit_button.click()

		time.sleep(30)
	finally:
		browser.quit()

def find_by_name(browser, name, input_data):
	element = browser.find_element_by_name(name)
	element.send_keys(input_data)

def get_file_path():
	current_dir = os.path.abspath(os.path.dirname(__file__))
	return os.path.join(current_dir, 'file.txt')

if __name__ == "__main__":
	link = "http://suninjuly.github.io/file_input.html"
	file_upload_test(link)


'''
    https://www.hackerrank.com/challenges/triangle-quest-2
    Make a palindromic triange
'''

def main():
  for i in range(1,int(raw_input())+1):
    print ((10**i//9)**2)

if __name__ == '__main__':
  main()


# We import the necessary packages
from functions_global import *
import numpy as np
import pandas as pd
from scipy.optimize import differential_evolution

# Do we carry out the parameter fiting in log scale?
logscale = False
if logscale:
    costfn = rmse_log
    costfn_adj = rmse_adj_log
else:
    costfn = rmse
    costfn_adj = rmse_adj

# We start by loading the raw data and tidying it up.
# It contains 4 data points per vaccine configuration per time point.
dataRaw = pd.read_csv('../../data/VNA.csv')
timesData = dataRaw['days'].tolist() # List of time points
nMeasurements = 4

# We construct the arrays of data for each vaccine configuration
PBS = []  # Non-adjuvanted vaccine
MF59 = []  # Vaccine with MF59
AS03 = []  # Vaccine with AS03
Diluvac = []  #Vaccine with Diluvac
X_data = [] # List of (repeated) time points

for i in range(len(timesData)):
    for j in range(1,nMeasurements+1):
        X_data.append(timesData[i])
        PBS.append(dataRaw.T.iloc[j][i])
    for j in range(nMeasurements+1,2*nMeasurements+1):
        MF59.append(dataRaw.T.iloc[j][i])
    for j in range(2*nMeasurements+1,3*nMeasurements+1):
        AS03.append(dataRaw.T.iloc[j][i])
    for j in range(3*nMeasurements+1,4*nMeasurements+1):
        Diluvac.append(dataRaw.T.iloc[j][i])
PBS = np.column_stack((X_data, PBS))
MF59 = np.column_stack((X_data, MF59))
AS03 = np.column_stack((X_data, AS03))
Diluvac = np.column_stack((X_data, Diluvac))

# Boundary values for the parameters to be estimated in the adjuvanted case
#                betaNA       betaHA       betaAb
bounds_adj = [(1.0, 5.0), (1.0, 5.0), (1.0, 60.0)]

nTotal = len(X_data)
nDays = nTotal/nMeasurements
nRuns = 2500

# indices of the datapoints separated by day
perDay = [np.arange(nMeasurements*i, nMeasurements*(i+1)) for i in range(nDays)]

# base parameters
params_base = pd.Series.from_csv('../../params/best_fit_params_base_A.csv')
gammaNA, gammaHA, mu, dmax = params_base['gammaNA'], params_base['gammaHA'], params_base['mu'], params_base['dmax']

baseQ = vQ0(np.abs(grid), dmax) + vQ0(np.abs(1 - grid), dmax)
H = Htilde*0.5*(np.sign(grid - 0.99*dmax) + np.sign(1.0 - 0.99*dmax - grid))
base_args = [gammaNA, gammaHA, mu, dmax, baseQ, H]

def oneRun():
    # select a random sample of 1 datapoint per day
    idx_sample = [np.random.choice(x) for x in perDay]
    params = np.array([])

    sample = MF59[idx_sample]
    args = (sample[:,0], sample[:,1], base_args)
    estimation = differential_evolution(costfn_adj, bounds_adj, args=args)
    params = np.append(params, estimation.x)

    sample = AS03[idx_sample]
    args = (sample[:,0], sample[:,1], base_args)
    estimation = differential_evolution(costfn_adj, bounds_adj, args=args)
    params = np.append(params, estimation.x)

    sample = Diluvac[idx_sample]
    args = (sample[:,0], sample[:,1], base_args)
    estimation = differential_evolution(costfn_adj, bounds_adj, args=args)
    params = np.append(params, estimation.x)

    return params

betaNA_M_list = []
betaHA_M_list = []
betaAb_M_list = []
betaNA_A_list = []
betaHA_A_list = []
betaAb_A_list = []
betaNA_D_list = []
betaHA_D_list = []
betaAb_D_list = []
for run in range(nRuns):
    params = oneRun()
    betaNA_M, betaHA_M, betaAb_M, betaNA_A, betaHA_A, betaAb_A, betaNA_D, betaHA_D, betaAb_D = params
    betaNA_M_list.append(betaNA_M)
    betaHA_M_list.append(betaHA_M)
    betaAb_M_list.append(betaAb_M)
    betaNA_M_list.append(betaNA_A)
    betaHA_M_list.append(betaHA_A)
    betaAb_M_list.append(betaAb_A)
    betaNA_M_list.append(betaNA_D)
    betaHA_M_list.append(betaHA_D)
    betaAb_M_list.append(betaAb_D)

best_fit_params_adj = {'betaNA_MF59': betaNA_M_list, 'betaHA_MF59': betaHA_M_list,
                       'betaAb_MF59': betaAb_M_list,
                       'betaNA_AS03': betaNA_A_list, 'betaHA_AS03': betaHA_A_list,
                       'betaAb_AS03': betaAb_A_list,
                       'betaNA_Diluvac': betaNA_D_list, 'betaHA_Diluvac': betaHA_D_list,
                       'betaAb_Diluvac': betaAb_D_list}

best_fit_params_adj = pd.DataFrame(best_fit_params_adj)
best_fit_params_adj.to_csv('../../params/bootstrap_adj.csv', index=False)


import numpy as np
import argparse
import os

parser = argparse.ArgumentParser(description='data')

parser.add_argument('--file', type=str, default="Data")
parser.add_argument('--length', type=int, default=3)
parser.add_argument('--train_size', type=float, default=1e6)
parser.add_argument('--test_size', type=float, default=1e5)
parser.add_argument('--number_range', type=int, default=11)

args = parser.parse_args()
np.random.seed(2023)

rang = args.number_range
nums = [i for i in range(rang)]
mul = np.arange(rang).reshape(-1,1) * np.arange(rang).reshape(1, -1)
mul = mul % rang

div = np.zeros((rang, rang), dtype=np.int32)
for i in range(rang):
    pos = np.where(mul==i)
    div[i, pos[0]] = pos[1]

dic = dict()
variable = "abcde"
for i in range(len(variable)):
    dic[i] = variable[i]

def output(X, step):
    ans = []
    for x in X:
        for j in range(x.shape[0]):
            if j == x.shape[0] - 1:
                ans.pop()
                ans.append("=")
                ans.append(x[j])
                continue
            if x[j] == 0 and j < step:
                continue
            if x[j] == 1 and j < step:
                ans.append(dic[j])
            else:
                ans.append(x[j])
                ans.append(dic[j])
            ans.append("+")
        ans.append(",")
    return ans

def get(length):
    matrix = np.random.randint(0, rang, size=(length, length + 1))
    history = []
    history.append(matrix + 0)
    for j in range(length):
        if np.sum(matrix[j:, j]) == 0:
            break
        pos = np.argmax(np.where(matrix[j:,j] != 0, 1, 0)) + j
        if pos != j:
            matrix[[pos, j]] = matrix[[j, pos]]

        division = matrix[j, j]
        if division != 1:
            for k in range(length + 1):
                matrix[j, k] = div[matrix[j, k], division]

        if np.sum(matrix[:,j]) != matrix[j, j]:
            for k in range(length):
                if j == k:
                    continue
                matrix[k] = (matrix[k] - matrix[k, j] * matrix[j]) % rang
        history.append(matrix + 0)

    if np.sum(matrix[j:, j]) != 0 and len(history) != 1:
        ans_seq = []
        step = 0
        for matri in history:
            ans_seq = ans_seq + output(matri, step)
            step += 1
            ans_seq[-1] = "<sep>"
        ans_seq.pop()
        return ans_seq
    return get(length)

train_set = set()
while len(train_set) < args.train_size:
    history = get(args.length)
    train_set.add(tuple(history))

test_set = set()
while len(test_set) < args.test_size:
    history = get(args.length)
    tmp = tuple(history)
    if tmp not in train_set:
        test_set.add(tmp)

decoder = f"{args.file}/decoder"
chain = f"{args.file}/chain"
os.makedirs(f"{args.file}", exist_ok=True)
os.makedirs(decoder, exist_ok=True)
os.makedirs(chain, exist_ok=True)

maxlen = 0
with open(f"{decoder}/train_data.txt", 'w') as f1:
    for history in train_set:
        this_len = 0
        for i in history:
            print(i, end=' ', file=f1)
            this_len += 1
            if i == "<sep>":
                break
        i = len(history) - 1
        while True:
            if history[i] == "<sep>":
                break
            i -= 1
        while True:
            i += 1
            if i == len(history):
                break
            print(history[i], end=' ', file=f1)
            this_len += 1
        print("", file=f1)
        maxlen = max(maxlen, this_len)

with open(f"{decoder}/test_data.txt", 'w') as f1:
    for history in test_set:
        this_len = 0
        for i in history:
            print(i, end=' ', file=f1)
            this_len += 1
            if i == "<sep>":
                break
        i = len(history) - 1
        while True:
            if history[i] == "<sep>":
                break
            i -= 1
        while True:
            i += 1
            if i == len(history):
                break
            print(history[i], end=' ', file=f1)
            this_len += 1
        print("", file=f1)
        maxlen = max(maxlen, this_len)

max_history = 0
with open(f"{chain}/train_data.txt", 'w') as f1:
    for history in train_set:
        max_history = max(max_history, len(history))
        for i in history:
            print(i, end=' ', file=f1)
        print("", file=f1)

with open(f"{chain}/test_data.txt", 'w') as f1:
    with open(f"{chain}/test_ans.txt", 'w') as f2:
        for history in test_set:
            for i in history:
                print(i, end=' ', file=f1)
                if i == "<sep>":
                    break
            print("", file=f1)
            i = len(history) - 1
            while True:
                if history[i] == "<sep>":
                    break
                i -= 1
            while True:
                i += 1
                if i == len(history):
                    break
                print(history[i], end=' ', file=f2)
            print("", file=f2)

print(f"max direct len:{maxlen}")
print(f"max cot len:{max_history}")


# interest = ['삼성전자', 'LG전자', 'Naver', 'SK하이닉스', '미래에셋대우']
# print('/'.join(interest))

def 조인(string, iterable):
   string.join(iterable)
    #이건 잘 모르겠습니다 선생님 ㅠㅠ

리스트 = ["내", "pc", "지키미"]
a = 조인("/", 리스트)

print(a)

venv/2학기 이공미코정/15. 리스트 문자열.py

15. 리스트 문자열.py


import flask
from werkzeug.test import Client
from werkzeug.wrappers import Response

from opentelemetry import trace as trace_api
from opentelemetry.instrumentation.flask import FlaskInstrumentor
from opentelemetry.test.wsgitestutil import WsgiTestBase

# pylint: disable=import-error
from .base_test import InstrumentationTest

class TestAutomatic(InstrumentationTest, WsgiTestBase):
    def setUp(self):
        super().setUp()

        FlaskInstrumentor().instrument()

        self.app = flask.Flask(__name__)

        self._common_initialization()

    def tearDown(self):
        super().tearDown()
        with self.disable_logging():
            FlaskInstrumentor().uninstrument()

    def test_uninstrument(self):
        # pylint: disable=access-member-before-definition
        resp = self.client.get("/hello/123")
        self.assertEqual(200, resp.status_code)
        self.assertEqual([b"Hello: 123"], list(resp.response))
        span_list = self.memory_exporter.get_finished_spans()
        self.assertEqual(len(span_list), 1)

        FlaskInstrumentor().uninstrument()
        self.app = flask.Flask(__name__)

        self.app.route("/hello/<int:helloid>")(self._hello_endpoint)
        # pylint: disable=attribute-defined-outside-init
        self.client = Client(self.app, Response)

        resp = self.client.get("/hello/123")
        self.assertEqual(200, resp.status_code)
        self.assertEqual([b"Hello: 123"], list(resp.response))
        span_list = self.memory_exporter.get_finished_spans()
        self.assertEqual(len(span_list), 1)

    def test_exluded_urls_explicit(self):
        FlaskInstrumentor().uninstrument()
        FlaskInstrumentor().instrument(excluded_urls="/hello/456")

        self.app = flask.Flask(__name__)
        self.app.route("/hello/<int:helloid>")(self._hello_endpoint)
        client = Client(self.app, Response)

        resp = client.get("/hello/123")
        self.assertEqual(200, resp.status_code)
        self.assertEqual([b"Hello: 123"], list(resp.response))
        span_list = self.memory_exporter.get_finished_spans()
        self.assertEqual(len(span_list), 1)

        resp = client.get("/hello/456")
        self.assertEqual(200, resp.status_code)
        self.assertEqual([b"Hello: 456"], list(resp.response))
        span_list = self.memory_exporter.get_finished_spans()
        self.assertEqual(len(span_list), 1)

    def test_no_op_tracer_provider(self):
        FlaskInstrumentor().uninstrument()
        FlaskInstrumentor().instrument(
            tracer_provider=trace_api.NoOpTracerProvider()
        )

        self.app = flask.Flask(__name__)
        self.app.route("/hello/<int:helloid>")(self._hello_endpoint)
        # pylint: disable=attribute-defined-outside-init
        self.client = Client(self.app, Response)
        self.client.get("/hello/123")

        span_list = self.memory_exporter.get_finished_spans()
        self.assertEqual(len(span_list), 0)


#author: Riley Doyle
#date: 7/16/20
#file: calc_CO2_loss_kLa
#status: working

import numpy as np
import matplotlib.pyplot as plt
from calc_Ks import *
from calc_alphas import *

def calc_CO2_loss_kLa (pK1, pK2, Kh, alk, d, colormap, PCO2, pHin, pHend, delpH, kLain, kLaend, delkLa):
    L = np.array(['-', '--', '-.', ':', '--'])
    pH = np.arange(pHin, pHend, delpH)
    kLasteps = np.arange(kLain, kLaend, delkLa)
    nkLasteps = len(kLasteps)
    y = np.zeros((nkLasteps, len(pH)))
    i = 0
    for c in kLasteps:
        alpha0 = calc_alpha0(pH, pK1, pK2)
        alpha1 = calc_alpha1(pH, pK1, pK2)
        alpha2 = calc_alpha2(pH, pK1, pK2)
        CO2sat = PCO2*Kh*1000

        H = 10**(-pH)
        OH = 10**(-(14-pH))
        bt = (1/(alpha1 + (2*alpha2)))
        tp = (alk - OH + H)
        CT = tp * bt

        H2CO3 = alpha0*CT
        y[i,:] = c*(H2CO3 - CO2sat)*24*44
        y = y*d
        plt.plot(pH, y[i,:].T, colormap[i], linestyle=L[i])
        i += 1


# Copyright © 2019 Province of British Columbia
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Event Template resource's input validations"""
from marshmallow import fields, validate

from .base import RequestBodyParameterSchema

class EventTemplateBodyParameterSchema(RequestBodyParameterSchema):
    """EventTemplate body schema"""

    name = fields.Str(
        metadata={"description": "Name of the event"},
        required=True
    )

    parent_id = fields.Int(
        metadata={"description": "parent event template id item"},
        allow_none=True
    )

    phase_id = fields.Int(
        metadata={"description": "Phase id of the event"},
        validate=validate.Range(min=1),
        required=True
    )

    event_type_id = fields.Int(
        metadata={"description": "Event type of the event template"},
        validate=validate.Range(min=1),
        required=True
    )

    event_category_id = fields.Int(
        metadata={"description": "Event category of the event template"},
        validate=validate.Range(min=1),
        required=True
    )

    start_at = fields.Str(
        metadata={"description": "Number of days after which the event to be started"}
    )

    number_of_days = fields.Int(
        metadata={"description": "Duration of the event"},
        allow_none=True
    )

    mandatory = fields.Bool(
        metadata={"description": "Indicate the event is a mandatory one or not"}
    )

    sort_order = fields.Int(
        metadata={"description": "Sort order of the event template item"}
    )

    event_position = fields.Str(
        metadata={"description": "position of the event in the phase"}
    )

    multiple_days = fields.Bool(
        metadata={"description": "Indicate if it is a multi day event"}
    )

    visibility = fields.Str(
        metadata={"description": "Indicate whether the event to be shown in the workplan or not"}
    )


import sqlalchemy as sqlalchemy

from db.utils import generic_query
import pandas as pd

def create_model(model_name: str, col: str, table: str) -> str:
    existing_models = list(get_defined_models()["MODEL_NAME"])
    model_iteration = 1
    while model_name + str(model_iteration) in existing_models:
        model_iteration += 1
    final_model_name = model_name + str(model_iteration)
    try:
        generic_query(
            f"CREATE MODEL {final_model_name} PREDICTING ({col}) FROM hk.{table}"
        )
    except sqlalchemy.exc.ResourceClosedError:
        pass
    return final_model_name

def train_model(model: str, table: str):
    existing_models = list(get_trained_models()["MODEL_NAME"])
    model_iteration = 1
    while model + str(model_iteration) in existing_models:
        model_iteration += 1
    final_model_name = model + str(model_iteration)
    try:
        generic_query(
            f"TRAIN MODEL {model} FROM hk.{table} AS trained_{final_model_name}"
        )
    except sqlalchemy.exc.ResourceClosedError:
        pass

def validate_model(model: str = "", val_name: str = "") -> pd.DataFrame:
    last_training_run = get_trained_models().iloc[-1]
    model_name = last_training_run['MODEL_NAME']
    validation_name = "val_"+model_name
    trained_model_name = last_training_run['TRAINED_MODEL_NAME']
    return generic_query(f"VALIDATE MODEL {model_name} USE {trained_model_name} from hk.validation_new_data")

def get_validation_metrics():
    metrics = generic_query(f"select * from information_schema.ml_validation_metrics")
    metrics = metrics.iloc[:-5]
    return metrics

def get_trained_models() -> pd.DataFrame:
    return generic_query("SELECT * FROM INFORMATION_SCHEMA.ML_TRAINED_MODELS")

def get_training_runs()-> pd.DataFrame:
    return generic_query("SELECT * FROM INFORMATION_SCHEMA.ML_TRAINING_RUNS")

def get_defined_models()-> pd.DataFrame:
    return generic_query("SELECT * FROM INFORMATION_SCHEMA.ML_MODELS")

# model_predict('test1', 'hk.loan_data_some')
def model_predict(model: str, validation_set: str) -> pd.DataFrame:
    return generic_query(
        f"""
    SELECT
      PREDICT(AcceptLoan use {model}) as prediction,
      PROBABILITY(AcceptLoan use {model} ) as probability_accepted,
      *
    FROM
      {validation_set}
    """
    )


from requests import get

def download(url, file_name):
    with open(file_name, "wb") as file:
        response = get(url)
        file.write(response.content)

if __name__ == '__main__':
    url = "https://www.google.com/url?sa=i&url=https%3A%2F%2Fdullyshin.github.io%2F2018%2F08%2F30%2FHTML-imgLink%2F&psig=AOvVaw2Am95CD8acj-vE8XaArtq9&ust=1647068867834000&source=images&cd=vfe&ved=0CAsQjRxqFwoTCMDKquu_vfYCFQAAAAAdAAAAABAJ"
    download(url, "arduino.png")


from datetime import datetime, timezone, timedelta
import numpy as np
import os
import io
import re
import math
import h5py
import hdf5plugin
import netCDF4
import tarfile
from scipy.interpolate import RectBivariateSpline

import psycopg

import subprocess
import urllib.request, urllib.error
import sys

from flask import Flask, request, make_response

def get_ncdata(ts):
    ts = ts.replace(second=0)
    ts = ts - timedelta(minutes = ts.minute % 5)
    run_ts = ts - timedelta(hours = ts.hour % 6)
    for i in range(7):
        if i == 6:
            raise "tar not found"

        tar_filename = 'wfs.t%02dz.%04d%02d%02d_%02d.tar' % (
            run_ts.hour,
            ts.year, ts.month, ts.day,
            ts.hour
        )

        tar_url = 'https://nomads.ncep.noaa.gov/pub/data/nccf/com/wfs/prod/wfs.%04d%02d%02d/%02d/%s' % (
            run_ts.year, run_ts.month, run_ts.day, run_ts.hour,
            tar_filename
        )
        try:
            res = urllib.request.urlopen(urllib.request.Request(tar_url, method="HEAD"))
            res.close()
        except urllib.error.HTTPError as e:
            if e.code == 404:
                run_ts = run_ts - timedelta(hours=6)
            else:
                raise e
        else:
            break

    print(tar_url)
    disk_filename = 'run-%04d%02d%02dT%02d-target-%04d%02d%02dT%02d.tar' % (run_ts.year, run_ts.month, run_ts.day, run_ts.hour, ts.year, ts.month, ts.day, ts.hour)

    files = []
    with os.scandir('/ipe-data') as i:
        for entry in i:
            if not entry.name.startswith('.') and entry.is_file():
                m = re.match('run-(\d{4})(\d{2})(\d{2})T(\d{2})-target-(\d{4})(\d{2})(\d{2})T(\d{2}).tar', entry.name)
                if m:
                    files.append({
                        'run_key': m[1] + m[2] + m[3] + 'T' + m[4],
                        'target_key': m[5] + m[6] + m[7] + 'T' + m[8],
                        'run_ts': datetime(year=int(m[1]), month=int(m[2]), day=int(m[3]), hour=int(m[4])),
                        'target_ts': datetime(year=int(m[5]), month=int(m[6]), day=int(m[7]), hour=int(m[8])),
                        'path': entry.path,
                        })

    by_target = {}

    for f in files:
        if f['target_ts'] < datetime.utcnow() - timedelta(minutes=90):
            print("Delete %s which has target in the past" % f['path'])
            os.unlink(f['path'])
        else:
            if f['target_key'] not in by_target:
                by_target[f['target_key']] = []
            by_target[f['target_key']].append(f)

    for k, files in by_target.items():
        if len(files) > 1:
            files.sort(key=lambda f: f['run_key'])
            for i in range(len(files) - 1):
                f = files[i]
                print("Delete %s which isn't the newest run for its target time" % f['path'])
                try:
                    os.unlink(f['path'])
                except FileNotFoundError:
                    pass

    tar_path = '/ipe-data/' + disk_filename
    if not os.path.exists(tar_path):
        print("Downloading to %s" % tar_path)
        urllib.request.urlretrieve(tar_url, tar_path)

    tf = tarfile.open(tar_path, mode='r')

    member_path = 'wfs.t%02dz.ipe05.%04d%02d%02d_%02d%02d%02d.nc' % (
        run_ts.hour,
        ts.year, ts.month, ts.day,
        ts.hour, ts.minute, ts.second
    )
    print(member_path)

    try:
        ncbuf = tf.extractfile(member_path).read()
    except tarfile.ReadError:
        os.unlink(tar_path)
        raise

    ds = netCDF4.Dataset('dummy.nc', memory=ncbuf)
    return {
        'hmf2': ds['HmF2'][:].data,
        'nmf2': ds['NmF2'][:].data,
        'lat': ds['lat'][:].data,
        'lon': ds['lon'][:].data,
    }

def resample(ilat, ilon, ds):
    olat, olon = np.meshgrid(
        np.linspace(-90, 90, 181),
        np.hstack((np.linspace(180, 359, 180), np.linspace(0, 180, 181))),
        indexing='ij',
    )
    spline = RectBivariateSpline(ilat, ilon, ds)
    return spline(olat.flatten(), olon.flatten(), grid=False).reshape(olon.shape)

def get_foe(ts):
    iri = subprocess.Popen(
        ['/build/irimap', str(ts.year), str(ts.month), str(ts.day), str(ts.hour), str(ts.minute), str(ts.second), '-100'],
        stdin=subprocess.DEVNULL,
        stdout=subprocess.PIPE,
        universal_newlines=True,
    )
    foe = np.zeros((181, 361))
    for line in iri.stdout:
        lat_, lon_, nmf2_, fof2_, md_, mufd_, hmf2_, foe_ = [float(x) for x in line.split()]
        lat = round(lat_ + 90)
        lon = round(lon_ + 180)
        foe[lat, lon] = foe_

    return foe

app = Flask(__name__)
@app.route('/generate', methods=['POST'])
def generate():
    dsn = "dbname='%s' user='%s' host='%s' password='%s'" % (os.getenv("DB_NAME"), os.getenv("DB_USER"), os.getenv("DB_HOST"), os.getenv("DB_PASSWORD"))
    con = psycopg.connect(dsn)

    run_id = int(request.form.get('run_id', -1))
    tgt = int(request.form.get('target', None))
    ts = datetime.fromtimestamp(float(tgt), tz=timezone.utc)

    data = get_ncdata(ts)

    # Repair holes at the south geomagnetic pole (8, 31) and north geomagnetic pole (86, 68-72)
    for ds in (data['nmf2'], data['hmf2']):
        ds[8, 31] = (ds[7, 30] + ds[8, 30] + ds[9, 30] + ds[7, 31] + ds[9, 31] + ds[7, 32] + ds[8, 32] + ds[9, 32]) / 8

        ds[86, 68] = (ds[85, 67] + ds[85, 68] + ds[85, 69] + ds[86, 67] + ds[87, 67] + ds[87, 68] + ds[87, 69]) / 7
        ds[86, 69] = (ds[85, 68] + ds[85, 69] + ds[85, 70] + ds[87, 68] + ds[87, 69] + ds[87, 70]) / 6
        ds[86, 70] = (ds[85, 69] + ds[85, 70] + ds[85, 71] + ds[87, 69] + ds[87, 70] + ds[87, 71]) / 6
        ds[86, 71] = (ds[85, 70] + ds[85, 71] + ds[85, 72] + ds[87, 70] + ds[87, 71] + ds[87, 72]) / 6
        ds[86, 72] = (ds[85, 71] + ds[85, 72] + ds[85, 73] + ds[86, 73] + ds[87, 71] + ds[87, 72] + ds[87, 73]) / 7

    # Replicate lon=0 data at lon=360
    data['lon'] = np.append(data['lon'], 360)
    data['hmf2'] = np.column_stack(( data['hmf2'], data['hmf2'][:,0] ))
    data['nmf2'] = np.column_stack(( data['nmf2'], data['nmf2'][:,0] ))
    # Upscale to 1°x1°
    data['hmf2'] = resample(data['lat'], data['lon'], data['hmf2'])
    data['nmf2'] = resample(data['lat'], data['lon'], data['nmf2'])

    # Convert NmF2 to foF2
    data['fof2'] = np.sqrt(data['nmf2'] / 1.24e10)

    # Shimazaki/Dudeney using IRI-estimated foE
    data['M'] = 1490. / (data['hmf2'] + 176)
    data['foe'] = get_foe(ts)
    data['dM'] = 0.253/np.clip(data['fof2'] / data['foe'] - 1.215, 0.01, None) - 0.012

    data['mufd'] = data['fof2'] * (data['M'] + data['dM'])

    bio = io.BytesIO()
    h5 = h5py.File(bio, 'w')

    h5.create_dataset('/ts', data=np.array(ts.timestamp()))
    for key in ['fof2', 'mufd', 'hmf2', 'foe']:
        h5.create_dataset('maps/' + key, data=data[key], **hdf5plugin.SZ(absolute=0.001))

    h5.close()
    dataset = bio.getvalue()

    with con.cursor() as cur:
        cur.execute('insert into ipemap (time, run_id, dataset) values (%s, %s, %s) on conflict (run_id, time) do update set dataset=excluded.dataset', (ts, run_id, dataset))
        con.commit()

    con.close()
    return make_response("OK\n")

if __name__ == '__main__':
    app.run(debug=False, host='0.0.0.0', port=int(os.getenv('ESSN_PORT')))


import torch
import torch.nn as nn
import torch.nn.functional as F

class SampaddingConv1D_BN(nn.Module):
    def __init__(self,in_channels,out_channels,kernel_size):
        super(SampaddingConv1D_BN, self).__init__()
        self.padding = nn.ConstantPad1d((int((kernel_size-1)/2), int(kernel_size/2)), 0)
        self.conv1d = torch.nn.Conv1d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size)
        self.bn = nn.BatchNorm1d(num_features=out_channels)

    def forward(self, X):
        X = self.padding(X)
        X = self.conv1d(X)
        X = self.bn(X)
        return X

class build_layer_with_layer_parameter(nn.Module):
    def __init__(self,layer_parameters):
        super(build_layer_with_layer_parameter, self).__init__()
        self.conv_list = nn.ModuleList()

        for i in layer_parameters:
            conv = SampaddingConv1D_BN(i[0],i[1],i[2])
            self.conv_list.append(conv)

    def forward(self, X):

        conv_result_list = []
        for conv in self.conv_list:
            conv_result = conv(X)
            conv_result_list.append(conv_result)

        result = F.relu(torch.cat(tuple(conv_result_list), 1))
        return result

class OS_CNNMRM2(nn.Module):
    def __init__(self,layer_parameter_list,n_class,few_shot = True):
        super(OS_CNNMRM2, self).__init__()
        self.few_shot = few_shot
        self.layer_parameter_list = layer_parameter_list
        self.layer_list = []

        for i in range(len(layer_parameter_list)):
            layer = build_layer_with_layer_parameter(layer_parameter_list[i])
            self.layer_list.append(layer)

        self.net = nn.Sequential(*self.layer_list)

        self.global_average_pool = nn.AdaptiveAvgPool1d(1)

        out_put_channel_numebr = 0
        for final_layer_parameters in layer_parameter_list[-1]:
            out_put_channel_numebr = out_put_channel_numebr+ final_layer_parameters[1]

        self.linear = nn.Linear(out_put_channel_numebr, n_class)

    def forward(self, X,labels= None):

        X = self.net(X)

        # X = self.averagepool(X)
        # X = X.squeeze_(-1)

        x_embeddings = self.global_average_pool(X).squeeze(-1)
        orig_out = (self.linear(x_embeddings))  # batch * nb_class
        knn_out = None
        if labels is not None:
            # knn_labels = labels[:]
            # knn_labels[knn_labels==-1]= orig_out[knn_labels==-1] # 将验证集中的样本置为pred
            is_test = (labels == -1)
            is_train = (labels != -1)
            x_train_embeddings, x_test_embeddings, orig_out_test = x_embeddings[is_train], x_embeddings[is_test],torch.softmax(orig_out[is_test],dim = 1)
            similar_x_test = torch.matmul(x_embeddings,x_test_embeddings.T)
            similar_x_test_stand = (similar_x_test-similar_x_test.mean(1,keepdim = True))/similar_x_test.std(1,keepdim = True)
            knn_out = torch.matmul((similar_x_test_stand),orig_out_test/orig_out_test.sum(dim = 0,keepdim = True))
        return {
            'tcrm2_out': orig_out,
            'x_embeddings': x_embeddings,
            'ttrm2_out': knn_out
        }


import pytest as pt

add_incompatible_paths = ["operator/add_bool_nil.lox",
                          "operator/add_bool_num.lox",
                          "operator/add_bool_string.lox",
                          "operator/add_nil_nil.lox",
                          "operator/add_num_nil.lox",
                          "operator/add_string_nil.lox"]
add_incompatible_ids = ["bool + nil",
                        "bool + number",
                        "bool + string",
                        "2 nils",
                        "number + nil",
                        "string + nil"]

incompatible_paths = ["operator/divide_nonnum_num.lox",
                      "operator/divide_num_nonnum.lox",
                      "operator/multiply_nonnum_num.lox",
                      "operator/multiply_num_nonnum.lox",
                      "operator/subtract_nonnum_num.lox",
                      "operator/subtract_num_nonnum.lox",
                      "operator/power_nonnum_num.lox",
                      "operator/power_num_nonnum.lox",
                      "operator/modulo_nonnum_num.lox",
                      "operator/modulo_num_nonnum.lox",
                      "operator/greater_nonnum_num.lox",
                      "operator/greater_num_nonnum.lox",
                      "operator/greater_or_equal_nonnum_num.lox",
                      "operator/greater_or_equal_num_nonnum.lox",
                      "operator/less_nonnum_num.lox",
                      "operator/less_num_nonnum.lox",
                      "operator/less_or_equal_nonnum_num.lox",
                      "operator/less_or_equal_num_nonnum.lox"]
incompatible_ids = ["nonnum / num", "num / nonnum", "nonnum * num", "num * nonnum", "nonnum - num", "num - nonnum",
                    "nonnum ^ num", "num ^ nonnum", "nonnum % num", "num % nonnum",
                    "nonnum > num", "num > nonnum", "nonnum >= num", "num >= nonnum",
                    "nonnum < num", "num < nonnum", "nonnum <= num", "num <= nonnum"]

class TestOperators:
    def test_add(self, capsys, lox):
        lox.run_file("operator/add.lox")
        capture = capsys.readouterr().out
        assert capture == "\n".join(["579", "string"]) + "\n"

    def test_subtract(self, capsys, lox):
        lox.run_file("operator/subtract.lox")
        capture = capsys.readouterr().out
        assert capture == "\n".join(["1", "0"]) + "\n"

    def test_negate(self, capsys, lox):
        lox.run_file("operator/negate.lox")
        capture = capsys.readouterr().out
        assert capture == "\n".join(["-3", "3", "-3"]) + "\n"

    def test_modulo(self, capsys, lox):
        lox.run_file("operator/modulo.lox")
        capture = capsys.readouterr().out
        assert capture == "\n".join(["2", "0.33999999999999986", "10.100000000000001"]) + "\n"

    def test_divide(self, capsys, lox):
        lox.run_file("operator/divide.lox")
        capture = capsys.readouterr().out
        assert capture == "\n".join(["4", "1"]) + "\n"

    def test_multiply(self, capsys, lox):
        lox.run_file("operator/multiply.lox")
        capture = capsys.readouterr().out
        assert capture == "\n".join(["15", "3.702"]) + "\n"

    def test_power(self, capsys, lox):
        lox.run_file("operator/power.lox")
        capture = capsys.readouterr().out
        assert capture == "\n".join(["64", "28.9976173063065"]) + "\n"

    def test_compare(self, capsys, lox):
        lox.run_file("operator/comparison.lox")

        aux_1, aux_2 = [True, False, False], [True, True, False]
        aux_3, aux_4 = [not x for x in aux_1], [not x for x in aux_2]
        expected_val = aux_1 + aux_2 + aux_4 + aux_3 + [False] * 4 + [True] * 4
        expected_val = [str(x).lower() for x in expected_val]

        capture = capsys.readouterr().out
        assert capture == "\n".join(expected_val) + "\n"

    def test_not(self, capsys, lox):
        lox.run_file("operator/not.lox")

        aux_1 = [False, True, True]
        aux_2 = [not x for x in aux_1]
        expected_val = aux_1 + [False, False] + aux_2
        expected_val = [str(x).lower() for x in expected_val]

        capture = capsys.readouterr().out
        assert capture == "\n".join(expected_val) + "\n"

    def test_not_class(self, capsys, lox):
        lox.run_file("operator/not_class.lox")
        capture = capsys.readouterr().out
        assert capture == "false\n" * 2

    def test_equals(self, capsys, lox):
        lox.run_file("operator/equals.lox")

        expected_val = [True] + [True, False] * 3 + [False] * 3
        expected_val = [str(x).lower() for x in expected_val]

        capture = capsys.readouterr().out
        assert capture == "\n".join(expected_val) + "\n"

    def test_not_equals(self, capsys, lox):
        lox.run_file("operator/not_equals.lox")

        expected_val = [False] + [False, True] * 3 + [True] * 3
        expected_val = [str(x).lower() for x in expected_val]

        capture = capsys.readouterr().out
        assert capture == "\n".join(expected_val) + "\n"

    def test_class_equals(self, capsys, lox):
        lox.run_file("operator/equals_class.lox")

        expected_val = [True, False, False, True] + [False] * 4
        expected_val = [str(x).lower() for x in expected_val]

        capture = capsys.readouterr().out
        assert capture == "\n".join(expected_val) + "\n"

    def test_method_equals(self, capsys, lox):
        lox.run_file("operator/equals_method.lox")
        capture = capsys.readouterr().out
        assert capture == "\n".join(["true", "false"]) + "\n"

    @pt.mark.parametrize("path", add_incompatible_paths, ids=add_incompatible_ids)
    def test_add_incompatible(self, capsys, lox, path):
        with pt.raises(SystemExit) as exc:
            lox.run_file(path)
            assert exc.value == 70

        capture = capsys.readouterr().err
        assert capture == "Error: Operands must be two numbers or two strings.\n[line 1]\n"

    @pt.mark.parametrize("path", incompatible_paths, ids=incompatible_ids)
    def test_incompatible(self, capsys, lox, path):
        with pt.raises(SystemExit) as exc:
            lox.run_file(path)
            assert exc.value == 70

        capture = capsys.readouterr().err
        assert capture == "Error: Operands must be numbers.\n[line 1]\n"

    def test_negate_nonnum(self, capsys, lox):
        with pt.raises(SystemExit) as exc:
            lox.run_file("operator/negate_nonnum.lox")
            assert exc.value == 70

        capture = capsys.readouterr().err
        assert capture == "Error: Operand must be a number.\n[line 1]\n"


# -*- coding: utf-8 -*-

import networkx as nx
import matplotlib.pyplot as plt

from collections import deque
from heapq import heappush, heappop
from itertools import count
#-----------------------------------------
# FILE DEDICATED TO PROTOTYPING FEATURES
#-----------------------------------------

def dijkstra(G, inicial, final=None, weight='weight'):

    inicial = {inicial}
    if final in inicial:
        return (0, [final])
    weight = pesos(G, weight)
    caminhos = {inicio: [inicio] for inicio in inicial}  #Dicionario de caminhos possiveis

    G_succ = G._succ if G.is_directed() else G._adj
    pred = None
    push = heappush
    pop = heappop
    dist = {}  # Dicionario de distancias finais
    visao = {}
    #margem eh uma estrutura heapq com tres tuplas (distance,c,nodo)
    c = count()
    margem = []
    for inicio in inicial:
        if inicio not in G:
            raise nx.NodeNotFound("Inicio {} nao esta no G".format(source))
        visao[inicio] = 0
        push(margem, (0, next(c), inicio))
    while margem:
        (d, _, v) = pop(margem)
        if v in dist:
            continue  # already searched this node.
        dist[v] = d
        if v == final:
            break
        for u, e in G_succ[v].items():
            cost = weight(v, u, e)
            if cost is None:
                continue
            vu_dist = dist[v] + cost
            if u not in visao or vu_dist < visao[u]:
                visao[u] = vu_dist
                push(margem, (vu_dist, next(c), u))
                if caminhos is not None:
                    caminhos[u] = caminhos[v] + [u]
                if pred is not None:
                    pred[u] = [v]
            elif vu_dist == visao[u]:
                if pred is not None:
                    pred[u].append(v)

    if final is None:
        return (dist, caminhos)
    try:
        return (dist[final], caminhos[final])
    except KeyError:
        raise nx.NetworkXNoPath("Nao ha caminho para {}.".format(final))

def pesos(G, weight):
 #Metodo para retornar os pesos de um determinado grafo
    if G.is_multigraph():
        return lambda u, v, d: min(attr.get(weight, 1) for attr in d.values())
    return lambda u, v, data: data.get(weight, 1)

def bellman_ford(G, inicial, final=None, weight='weight'):
    if inicial == final:
        return (0, [inicial])

    weight = pesos(G, weight)

    caminhos = {inicial: [inicial]}  # dicionario de caminhos

    dist = None #Definicao da variavel de distancia para realizar o calculo da distancia entre os grafos
    pred = None  #Nao vao ser utilizados dicionarios de listas
    for s in [inicial]:
        if s not in G:
            raise nx.NodeNotFound("Objeto inicial {} nao esta em G".format(s))

    if pred is None:
        pred = {v: [] for v in [inicial]}

    if dist is None:
        dist = {v: 0 for v in [inicial]}

    G_succ = G.succ if G.is_directed() else G.adj #Ajuste de classificacao de acordo com o tipo do grafo
    inf = float('inf') #definindo infinito [utilizado para grafos inalcansaveis]
    n = len(G)

    count = {}
    q = deque([inicial])
    na_fila = set([inicial])
    while q:
        u = q.popleft()
        na_fila.remove(u)

        if all(pred_u not in na_fila for pred_u in pred[u]):
            dist_u = dist[u]
            for v, e in G_succ[u].items():
                dist_v = dist_u + weight(v, u, e)

                if dist_v < dist.get(v, inf):
                    if v not in na_fila:
                        q.append(v)
                        na_fila.add(v)
                        count_v = count.get(v, 0) + 1
                        if count_v == n:
                            raise nx.NetworkXUnbounded(
                                "Ciclo de custo negativo detectado")
                        count[v] = count_v
                    dist[v] = dist_v
                    pred[v] = [u]

                elif dist.get(v) is not None and dist_v == dist.get(v):
                    pred[v].append(u)

    if caminhos is not None:
        dsts = [final] if final is not None else pred
        for dst in dsts:

            caminho = [dst]
            cur = dst

            while pred[cur]:
                cur = pred[cur][0]
                caminho.append(cur)

            caminho.reverse()
            caminhos[dst] = caminho

    if final is None:
        return (dist, caminhos)
    try:
        return (dist[final], caminhos[final])
    except KeyError:
        msg = "Node %s not reachable from %s" % (final, inicial)
        raise nx.NetworkXNoPath(msg)

def floyd_warshall(G, weight='weight'):
    """Find all-pairs shortest path lengths using Floyd's algorithm.
    Parameters
    ----------
    G : NetworkX graph
    weight: string, optional (default= 'weight')
       Edge data key corresponding to the edge weight.
    Returns
    -------
    distance : dict
       A dictionary,  keyed by source and target, of shortest paths distances
       between nodes.
    Notes
    ------
    Floyd's algorithm is appropriate for finding shortest paths
    in dense graphs or graphs with negative weights when Dijkstra's algorithm
    fails.  This algorithm can still fail if there are negative cycles.
    It has running time $O(n^3)$ with running space of $O(n^2)$.
    See Also
    --------
    floyd_warshall_predecessor_and_distance
    floyd_warshall_numpy
    all_pairs_shortest_path
    all_pairs_shortest_path_length
    """
    # could make this its own function to reduce memory costs
    return floyd_warshall_predecessor_and_distance(G, weight=weight)[1]

def floyd_warshall_predecessor_and_distance(G, weight='weight'):
    """Find all-pairs shortest path lengths using Floyd's algorithm.
    Parameters
    ----------
    G : NetworkX graph
    weight: string, optional (default= 'weight')
       Edge data key corresponding to the edge weight.
    Returns
    -------
    predecessor,distance : dictionaries
       Dictionaries, keyed by source and target, of predecessors and distances
       in the shortest path.
    Examples
    --------
    >>> G = nx.DiGraph()
    >>> G.add_weighted_edges_from([('s', 'u', 10), ('s', 'x', 5),
    ...     ('u', 'v', 1), ('u', 'x', 2), ('v', 'y', 1), ('x', 'u', 3),
    ...     ('x', 'v', 5), ('x', 'y', 2), ('y', 's', 7), ('y', 'v', 6)])
    >>> predecessors, _ = nx.floyd_warshall_predecessor_and_distance(G)
    >>> print(nx.reconstruct_path('s', 'v', predecessors))
    ['s', 'x', 'u', 'v']
    Notes
    ------
    Floyd's algorithm is appropriate for finding shortest paths
    in dense graphs or graphs with negative weights when Dijkstra's algorithm
    fails.  This algorithm can still fail if there are negative cycles.
    It has running time $O(n^3)$ with running space of $O(n^2)$.
    See Also
    --------
    floyd_warshall
    floyd_warshall_numpy
    all_pairs_shortest_path
    all_pairs_shortest_path_length
    """
    from collections import defaultdict
    # dictionary-of-dictionaries representation for dist and pred
    # use some defaultdict magick here
    # for dist the default is the floating point inf value
    dist = defaultdict(lambda: defaultdict(lambda: float('inf')))
    for u in G:
        dist[u][u] = 0
    pred = defaultdict(dict)
    # initialize path distance dictionary to be the adjacency matrix
    # also set the distance to self to 0 (zero diagonal)
    undirected = not G.is_directed()
    for u, v, d in G.edges(data=True):
        e_weight = d.get(weight, 1.0)
        dist[u][v] = min(e_weight, dist[u][v])
        pred[u][v] = u
        if undirected:
            dist[v][u] = min(e_weight, dist[v][u])
            pred[v][u] = v
    for w in G:
        for u in G:
            for v in G:
                if dist[u][v] > dist[u][w] + dist[w][v]:
                    dist[u][v] = dist[u][w] + dist[w][v]
                    pred[u][v] = pred[w][v]
    return dict(pred), dict(dist)

G = nx.DiGraph()
'''
elist = [(1, 2, {'weight':5}), (2, 3, {'weight':3}), (3, 5, {'weight':1}), (4, 5,{'weight':7.3}),(2, 5 ,{'weight':6.2})]
G.add_edges_from(elist)

#Res = dijkstra_path(G, 1, 5)

#(length, path) = single_source_dijkstra(G, 1, target=5, weight='weight')
#Res = path

Res = dijkstra(G, 1, final=5, weight='weight')

print(Res)

#print(bellman_ford_path(G, 1, 5))
Res2 = bellman_ford(G, 1, final=5, weight='weight')

print(Res2)

print( nx.floyd_warshall_numpy(G))
'''
elist = [(1, 2, {'weight':5}), (2, 3, {'weight':3}), (3, 4, {'weight':1}), (4, 5,{'weight':7.3})]
G.add_edges_from(elist)
nx.draw(G)
plt.show()


#!/usr/bin/env python

# TODO: combine with cvae_node_predict through a flag in prediction_input.msg

from __future__ import absolute_import, division, print_function
import timeit
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"

import rospy
import numpy as np
import tensorflow as tf
from tensorflow.python.client import timeline
from traffic_weaving_prediction.msg import prediction_input, prediction_output
from numpy_ros import numpy_to_multiarray, multiarray_to_numpy
dir(tf.contrib)

# model_dir = "/data/laneswap/exported_models/1505240770"
model_dir = rospy.get_param("model", "") or "/data/trafficweaving/models/1507924360"
print("Loading model from: " + model_dir)
profile = False

def unstandardize(tensor, mean, std, include_bias=True):
    tile_ct = int(tensor.shape[-1].value / mean.shape[-1].value)
    if include_bias:
        return tensor * tf.tile(std, [tile_ct]) + tf.tile(mean, [tile_ct])
    else:
        return tensor * tf.tile(std, [tile_ct])

def score_trajectories(y, car1, car2, traj_length, car1_future):
    k, bs, _, _ = tf.unstack(tf.shape(y))
    ph = y.shape[2].value
    with tf.variable_scope("scoring"):
        curr_step = traj_length - 1
        car2_present = tf.expand_dims(tf.expand_dims(car2[:,curr_step,:], 0), 0)        # [1, 1, 1, 6]
        car2_present_p, car2_present_v, car2_present_a = tf.split(car2_present, 3, axis=3)                     # [1, 1, 1, 2]x3

        car2_future_a = y                                                                                # [k, bs, ph, 2]
        car2_present_and_future_a = tf.concat([tf.tile(car2_present_a, [k, bs, 1, 1]), car2_future_a], axis=2) # [k, bs, ph+1, 2]
        car2_future_v = car2_present_v + tf.cumsum((car2_present_and_future_a[:,:,:-1,:] +
                                                    car2_present_and_future_a[:,:,1:,:])/2, axis=2)/10         # [k, bs, ph, 2]
        car2_present_and_future_v = tf.concat([tf.tile(car2_present_v, [k, bs, 1, 1]), car2_future_v], axis=2) # [k, bs, ph+1, 2]
        car2_future_p = car2_present_p + tf.cumsum((car2_present_and_future_v[:,:,:-1,:] +
                                                    car2_present_and_future_v[:,:,1:,:])/2, axis=2)/10         # [k, bs, ph, 2]

        car1_future_p, car1_future_v, car1_future_a = tf.split(car1_future, 3, axis=2)    # [bs, ph, 2]x3
        delta_p = car2_future_p - car1_future_p                                           # [k, bs, ph, 2]

        # proximity penalty
        dx_danger, dy_danger = 8, 2
        r_danger = np.hypot(dx_danger, dy_danger)
        cars_close = tf.reduce_all(tf.abs(delta_p) < [dx_danger, dy_danger], axis=-1)              # [k, bs, ph]
        prox_penalty = -1000*tf.to_float(cars_close)*(1 + r_danger - tf.norm(delta_p, axis=-1))    # [k, bs, ph]

        # time of crossover reward (maybe delta_x should be adjusted for car length)
        delta_v = car2_future_v - car1_future_v            # [k, bs, ph, 2]
        delta_x = delta_p[:,:,:,0]
        delta_xd = delta_v[:,:,:,0]
        ntco = tf.sign(delta_x)*tf.maximum(tf.abs(delta_x) - 3, 0)*delta_xd # negative crossover time
        ntco = tf.maximum(ntco, 0)    # ignore positive crossover time, we just want to reward negative
        tco_reward = 100*tf.minimum(ntco, 1)    # [k, bs, ph]

        # car 2 extreme speed penalty (penalty for cutting off; -20 for each m/s below 24 m/s)
        car2_speed_penalty = 50*tf.minimum(car2_future_v[:,:,:,0] - 24, 0)    # [k, bs, ph]

        # car 1 extreme speed penalty
        car1_speed_penalty = (10*tf.minimum(car1_future_v[:,:,0] - 24, 0) -
                              10*tf.maximum(car1_future_v[:,:,0] - 36, 0))    # [bs, ph]

        # car 1 goal penalty
        y_top, y_bot = -1.83, -6.09
        y_mid = (y_top + y_bot)/2
        top_start_indicator = tf.to_float(car1[0,0,1] > y_mid)
        bot_start_indicator = 1 - top_start_indicator
        urgency = 500*tf.minimum(1.2 + car1_future_p[:,:,0]/150 + .6, 1)
        car1_goal_penalty = -(top_start_indicator*urgency*tf.minimum(tf.abs(car1_future_p[:,:,1] - y_bot), 2) +
                              bot_start_indicator*urgency*tf.minimum(tf.abs(car1_future_p[:,:,1] - y_top), 2))    # [bs, ph]

        # car 1 acceleration penalty
        car1_accel_penalty = -tf.square(car1_future_a[:,:,0])    # *car1_future_a[:,:,0]/10
        car1_accel_tv_penalty = -tf.reduce_sum(tf.abs(car1_future_a[:,1:,0] - car1_future_a[:,:-1,0]), -1)

        rewards = (prox_penalty + tco_reward + car2_speed_penalty +
                   car1_speed_penalty + car1_goal_penalty + car1_accel_penalty)  # [k, bs, ph]
        return tf.reduce_mean(tf.reduce_sum(rewards*np.exp(np.log(0.9)*np.arange(ph)), -1) + car1_accel_tv_penalty, 0)

class Scorer(object):

    def __init__(self):
        with tf.Graph().as_default() as g:
            self.sess = tf.Session()
            print("Loading model from: " + model_dir)
            tf.saved_model.loader.load(self.sess,
                                       [tf.saved_model.tag_constants.SERVING],
                                       model_dir)
            self.y = g.get_tensor_by_name("outputs/y:0")
            self.car1 = g.get_tensor_by_name("car1:0")
            self.car2 = g.get_tensor_by_name("car2:0")
            self.extras = g.get_tensor_by_name("extras:0")
            self.traj_lengths = g.get_tensor_by_name("traj_lengths:0")
            self.traj_length = self.traj_lengths[0]
            self.car1_future = g.get_tensor_by_name("car1_future:0")
            self.car1_future_x = g.get_tensor_by_name("car1_future_x:0")
            self.car1_future_y = g.get_tensor_by_name("car1_future_y:0")
            self.sample_ct = g.get_tensor_by_name("sample_ct:0")
            self.r = score_trajectories(self.y,
                                        self.car1,
                                        self.car2,
                                        self.traj_length,
                                        self.car1_future)
            self.c1top32 = tf.gather(self.car1_future, tf.nn.top_k(self.r, 32)[1])
            self.c1best = self.car1_future[tf.argmax(self.r),:,:]

        rospy.init_node("cvae_scoring", anonymous=True)
        rospy.Subscriber("prediction_input", prediction_input, self.prediction_callback)
        self.pub = rospy.Publisher("prediction_output", prediction_output, queue_size=10)

    def prediction_callback(self, input_msg):
        tic = timeit.default_timer()
        print("subscribed to prediction input")
        tic0 = timeit.default_timer()
        feed_dict = {self.car1: multiarray_to_numpy(input_msg.car1),
                     self.car2: multiarray_to_numpy(input_msg.car2),
                     self.extras: multiarray_to_numpy(input_msg.extras),
                     self.traj_lengths: multiarray_to_numpy(input_msg.traj_lengths),
                     self.sample_ct: [input_msg.sample_ct]}
        if input_msg.car1_future.data:
            feed_dict[self.car1_future] = multiarray_to_numpy(input_msg.car1_future)
        else:
            feed_dict[self.car1_future_x] = multiarray_to_numpy(input_msg.car1_future_x)
            feed_dict[self.car1_future_y] = multiarray_to_numpy(input_msg.car1_future_y)
        toc0 = timeit.default_timer()

        print("constructing feed_dict took: ", toc0 - tic0, " (s), running tf!")

        tic0 = timeit.default_timer()
        if profile:
            options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
            run_metadata = tf.RunMetadata()
            c1top32 = self.sess.run(self.c1top32,
                                    feed_dict=feed_dict,
                                    options=options,
                                    run_metadata=run_metadata)
            fetched_timeline = timeline.Timeline(run_metadata.step_stats)
            chrome_trace = fetched_timeline.generate_chrome_trace_format()
            with open('/home/schmrlng/Dropbox/timeline' +
                      os.environ["CUDA_VISIBLE_DEVICES"] +
                      '_0.json', 'w') as f:
                f.write(chrome_trace)

            feed_dict[self.car1_future] = c1top32
            feed_dict[self.sample_ct] = [input_msg.sample_ct*64]
            feed_dict.pop(self.car1_future_x)    # should be unnecessary
            feed_dict.pop(self.car1_future_y)    # should be unnecessary

            run_metadata = tf.RunMetadata()
            c1best, r = self.sess.run([self.c1best, self.r],
                                      feed_dict=feed_dict,
                                      options=options,
                                      run_metadata=run_metadata)
            fetched_timeline = timeline.Timeline(run_metadata.step_stats)
            chrome_trace = fetched_timeline.generate_chrome_trace_format()
            with open('/home/schmrlng/Dropbox/timeline' +
                      os.environ["CUDA_VISIBLE_DEVICES"] +
                      '_1.json', 'w') as f:
                f.write(chrome_trace)
        else:
            c1top32 = self.sess.run(self.c1top32,
                                    feed_dict=feed_dict)
            feed_dict[self.car1_future] = c1top32
            feed_dict[self.sample_ct] = [input_msg.sample_ct*64]
            feed_dict.pop(self.car1_future_x)    # should be unnecessary
            feed_dict.pop(self.car1_future_y)    # should be unnecessary
            c1best, r = self.sess.run([self.c1best, self.r],
                                      feed_dict=feed_dict)
        toc0 = timeit.default_timer()

        print("done running tf!, took (s): ", toc0 - tic0)

        tic0 = timeit.default_timer()
        output_msg = prediction_output()
        output_msg.y = numpy_to_multiarray(c1best)
        output_msg.r = numpy_to_multiarray(r)
        self.pub.publish(output_msg)
        toc0 = timeit.default_timer()
        toc = timeit.default_timer()

        print("output_msg constructed and published, took (s): ", toc0 - tic0)
        print("total time taken (s): ", toc - tic)

    def run(self):
        rospy.spin()

if __name__ == "__main__":
    s = Scorer()
    s.run()


import pandas as pd
import glob

# ########################## Train ##################################

trn_df = pd.DataFrame(columns=['file', 'label'])

folders = glob.glob('Data/train/*')
for fol in folders:
    for fil in glob.glob(fol+'/*'):
        df_tmp = pd.DataFrame(
            [(fil, fil.split('/')[::-1][1])],
            columns=['file', 'label']
            )
        trn_df = trn_df.append(df_tmp)

trn_df.to_csv('emotionTrainData.csv', index=False)

# ############################# Validation ##########################

val_df = pd.DataFrame(columns=['file', 'label'])

folders = glob.glob('Data/test/*')
for fol in folders:
    for fil in glob.glob(fol+'/*'):
        df_tmp = pd.DataFrame(
            [(fil, fil.split('/')[::-1][1])],
            columns=['file', 'label']
            )
        val_df = val_df.append(df_tmp)

val_df.to_csv('emotionValidationData.csv', index=False)


#!/usr/bin/env python
import roslib; roslib.load_manifest('quadrotorTestControl')
import rospy
from std_msgs.msg import Float32
from Phidgets.PhidgetException import *
from Phidgets.Events.Events import *
from Phidgets.Devices.InterfaceKit import *

height = 0

def talker():
    pub = rospy.Publisher('sonar', Float32)
    rospy.init_node('sonar')
    while not rospy.is_shutdown():
        rospy.loginfo(height)
        pub.publish(Float32(height))
        rospy.sleep(1.0)

def AttachHandler(event):
    attachedDevice = event.device
    serialNumber = attachedDevice.getSerialNum()
    deviceName = attachedDevice.getDeviceName()
    attachedDevice.setRatiometric(True)
    attachedDevice.setSensorChangeTrigger(5,1)
    print("Hello to Device " + str(deviceName) + ", Serial Number: " + str(serialNumber))

def DetachHandler(event):
    detachedDevice = event.device
    serialNumber = detachedDevice.getSerialNum()
    deviceName = detachedDevice.getDeviceName()
    print("Goodbye Device " + str(deviceName) + ", Serial Number: " + str(serialNumber))

def sensorChanged(e):
#    f = open('sonar.txt','a')
    height = e.value*1024.0/1000
#    print ("Sensor %i: %i" % (e.index, height))
#    f.write('Height at %.3f\n',height)
#    f.close()
    print ("Sensor %i: %i" % (e.index, height))
    if not rospy.is_shutdown():
        rospy.loginfo(height)
        pub.publish(Float32(height))
    return 0

def LocalErrorCatcher(event):
    print("Phidget Exception: " + str(e.code) + " - " + str(e.details) + ", Exiting...")
    exit(1)

if __name__ == '__main__':

#    f = open('sonar.txt','a')
#    f.write('********************************\n')
#    f.close()

    # Create
    try:
        device = InterfaceKit()
    except RuntimeError as e:
        print("Runtime Error: %s" % e.message)

    try:
        device.setOnAttachHandler(AttachHandler)
        device.setOnDetachHandler(DetachHandler)
        device.openPhidget()
        device.setOnSensorChangeHandler(sensorChanged)
    except PhidgetException as e:
        LocalErrorCatcher(e)

    try:
        pub = rospy.Publisher('sonar', Float32)
        rospy.init_node('sonar')
    except rospy.ROSInterruptException:
        pass

    print("Phidget Simple Playground (plug and unplug devices)");
    print("Press Enter to end anytime...");
    character = str(raw_input())

    print("Closing...")
 #   f = open('sonar.txt','a')
 #   f.write('********************************\n')
 #   f.close()
    try:
        device.closePhidget()
    except PhidgetException as e:
        LocalErrorCatcher(e)


import sys,getopt,threadpool,random,xlrd,os
sys.path.append('.')
sys.path.append('..')
from src.offer.baseInfoCollect.collect_email import EmailCollecter
from src.offer.baseInfoCollect.collect_browser import BrowserCollecter
from src.offer.baseInfoCollect.collect_identity_chinese import CollectCHAddr
from src.offer.baseInfoCollect.collect_identity_foregin import IdentityCollecter
from src.offer.baseInfoCollect.colllect_identity_foregin_xlsx import IdentityCollecter_US_XLSX

class BaseInfoRun(object):
    @staticmethod
    def collect_UA(threadNum=1):
        pool = threadpool.ThreadPool(threadNum)
        requests=threadpool.makeRequests(BrowserCollecter.writeOneUAToDB,BrowserCollecter.getCSVUserAgent())
        [pool.putRequest(req) for req in requests]
        pool.wait()
    @staticmethod
    def collect_email(threadNum=1):
        pool = threadpool.ThreadPool(threadNum)
        allEmail = EmailCollecter.getTXTEmail()
        data = [(index,None) for index in allEmail]
        requests=threadpool.makeRequests(EmailCollecter.writeOneEmailToDB,data)
        [pool.putRequest(req) for req in requests]
        pool.wait()
    @staticmethod
    def collect_Chinese_Identity(collctNum=None,threadNum=1):
        pool = threadpool.ThreadPool(threadNum)
        allPos = CollectCHAddr.getPosition()
        if collctNum and len(allPos)>collctNum:
            allPos=random.sample(allPos,collctNum)
        data = [((index,), None) for index in allPos]
        requests = threadpool.makeRequests(CollectCHAddr.getOneAddr, data)
        [pool.putRequest(req) for req in requests]
        pool.wait()
    @staticmethod
    def collect_Foregin_Identity(collctNum=1000,country=('us',),threadNum=1):
        def makeName(countries):
            coun=random.choice(countries)
            return coun,coun
        pool = threadpool.ThreadPool(threadNum)
        data = [(makeName(country), None) for _ in range(collctNum)]
        requests = threadpool.makeRequests(IdentityCollecter.getForeginIdentity, data)
        [pool.putRequest(req) for req in requests]
        pool.wait()
if __name__=="__main__":
    # BaseInfoRun.collect_Foregin_Identity(collctNum=1,country=('us',))
    try:
        options, args = getopt.getopt(sys.argv[1:], "",['threadNum=','email','ua','identity_chn=','identity_en=','country='])
    except getopt.GetoptError:
        sys.exit()
    runType=-1
    runThread=1
    #中文个人信息参数
    identityNum_CHN=-1
    #外文个人信息参数
    identityNum_EN=1000
    country_en=('us',)

    for name,value in options:
        if name in ('--threadNum',):
            runThread=int(value)
        elif name in ('--email',):
            runType=0
        elif name in ('--ua',):
            runType = 1
        elif name in ('--identity_chn',):
            runType = 2
            if int(value)==-1:
                identityNum_CHN=None
            else:
                identityNum_CHN =int(value)
        elif name in ('--identity_en',):
            runType = 3
            if int(value)!=-1:
                identityNum_EN =int(value)
        elif name in ('--country',):
            country_en=tuple(str(value).split(','))
    if runType==0:
        BaseInfoRun.collect_email(threadNum=runThread)
    elif runType==1:
        BaseInfoRun.collect_UA(threadNum=runThread)
    elif runType==2:
        BaseInfoRun.collect_Chinese_Identity(collctNum=identityNum_CHN)
    elif runType==3:
        # BaseInfoRun.collect_Foregin_Identity(collctNum=identityNum_EN,country=country_en,threadNum=runThread)
        # allAddr = IdentityCollecter_US_XLSX.readXLSX_ADDR()
        # for item in allAddr:
        #     print(item)
        #     address, city, state, zipcode, lastName, firstName, gender, birthday, country = IdentityCollecter_US_XLSX.getForeginIdentity(
        #         'us', 'us')
        #     IdentityCollecter_US_XLSX.writeOneIdentityToDB(lastName, firstName, gender, birthday, item[0], item[1],
        #                                                    item[2], country,
        #                                                    item[3])
        ExcelFile = xlrd.open_workbook(
            os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))),
                         'resource',
                         'liao.xls'))
        sheet = ExcelFile.sheet_by_index(0)
        accounts = []
        for i in range(1, sheet.nrows):
            birthday = str(random.randint(1980, 1990)) + '-' + str(random.randint(1, 12)).zfill(2) + '-' + str(
                random.randint(1, 28)).zfill(2)
            country='US'
            gender='female'
            firstName,lastName,zipcode,state,city,address = sheet.cell_value(i, 0), sheet.cell_value(i, 1), int(sheet.cell_value(i, 2)), sheet.cell_value(i, 3), sheet.cell_value(i, 4), sheet.cell_value(i, 5)
            IdentityCollecter_US_XLSX.writeOneIdentityToDB(lastName, firstName, gender, birthday, address, city,
                                                           state, country,
                                                           zipcode)


#oops---Multiple inheritance

class Employee:
    no_of_leaves=9
    var=10
    var=20
    var1=100
    def __init__(self,name,sal,role):
        self.name=name
        self.salary=sal
        self.role=role

    def printdetails(self):
        return f"Name is:{self.name} Salary is :{self.salary} Role is:{self.role}"

class Player:
    no_of_games=4
    var=20
    var1=200
    def __init__(self,name,sal,game):
        self.name=name
        self.sal=sal
        self.game=game

    def printdetails(self):
        return f"Name is: {self.name} Salary is:{self.sal} Game is: {self.game}"

class CoolProgrammer(Employee,Player):   #Multiple inheritance
    language="C++"
    var=30

    def printlanguage(self):
        return f"Language is :{self.language}"

class CoolProgrammer1(Player,Employee):   # Multiple inheritance
    language="Python"
    var=50

    def printlanguage(self):
        return f"Language is :{self.language}"

karan=CoolProgrammer("Karan",12000,'CoolProgrammer')   # it will go and first try to find the constructor in the
                                                       # same class then it will check in Employee class and then in
                                                       # Player class as according to the order of inheritance
print(karan.printlanguage())
print(karan.printdetails()) #here also both Employee and Player classes had printdetails() function but
                            # printdetails() of Employee class is executed

print(karan.var)        # it will go and first try to find var in the
                        # same class then it will check in Employee class and then in
                        # Player class as according to the order of inheritance

print(karan.var1)       # it will go and first try to find var1 in the
                        # same class then it will check in Employee class and then in
                        # Player class as according to the order of inheritance

harry=CoolProgrammer1('Harry',14000,'Tennis')
print(harry.printdetails())

print(harry.printlanguage())

print(harry.var)
print(harry.var1)


import random
import numpy as np
import nltk
from nltk import word_tokenize
import torch
# from transformers import get_linear_schedule_with_warmup

# Run function `preprocessing_for_bert` on the train set and the validation set
def preprocessing_for_bert(data, tokenizer):
    """Perform required preprocessing steps for pretrained BERT.
    @param      data (np.array): Array of texts to be processed.
    @return     input_ids (torch.Tensor): Tensor of token ids to be fed to a model.
    @return     attention_masks (torch.Tensor): Tensor of indices specifying which
                tokens should be attended to by the model.
    """
    # Create empty lists to store outputs
    input_ids = []
    attention_masks = []
    MAX_LEN = 64

    # For every sentence...
    for sent in data:
        # `encode_plus` will:
        #    (1) Tokenize the sentence
        #    (2) Add the `[CLS]` and `[SEP]` token to the start and end
        #    (3) Truncate/Pad sentence to max length
        #    (4) Map tokens to their IDs
        #    (5) Create attention mask
        #    (6) Return a dictionary of outputs

        encoded_sent = tokenizer.encode_plus(text=sent,  # Preprocess sentence
                                            add_special_tokens=True,        # Add `[CLS]` and `[SEP]`
                                            max_length=MAX_LEN,                  # Max length to truncate/pad
                                            pad_to_max_length=True,         # Pad sentence to max length
                                            #return_tensors='pt',           # Return PyTorch tensor
                                            return_attention_mask=True)      # Return attention mask

        # Add the outputs to the lists
        input_ids.append(encoded_sent.get('input_ids'))
        attention_masks.append(encoded_sent.get('attention_mask'))

    # Convert lists to tensors
    input_ids = torch.tensor(input_ids)
    attention_masks = torch.tensor(attention_masks)

    return input_ids, attention_masks

def tokenizer_(args, text):
    if args.model_state == 'CNN_classifier':
        token = word_tokenize(text)
        if len(token) < max(args.KERNEL_SIZE):
            for i in range(0, max(args.KERNEL_SIZE)-len(token)):
                token.append('<PAD>') # 커널 사이즈 보다 문장의 길이가 작은 경우 에러 방지
    else:
        token = word_tokenize(text)
    return token

def set_seed(seed_value=188):
    """Set seed for reproducibility.
    """
    random.seed(seed_value)
    np.random.seed(seed_value)
    torch.manual_seed(seed_value)
    torch.cuda.manual_seed_all(seed_value)