import json
import logging
import os
import re
import sys
from math import floor
from zipfile import ZipFile
import glob
from optparse import OptionParser

LOGGER = logging.getLogger(__name__)

def unzip_osz(filepath, dst):
    filename, ext = os.path.splitext(filepath)
    dstpath = os.path.join(dst, os.path.basename(filename))
    LOGGER.info(dstpath)
    with ZipFile(filepath, "r") as z:
        if not os.path.exists(dstpath):
            os.mkdir(dstpath)
        z.extractall(path=dstpath)

    return dstpath

def get_beatmap_data(filepath):
    """Reads osu file and returns metadata. Cannot use ini parser reliably as it is not a proper INI file

    Sometimes the osu file leaves comments, so it is more reliable to parse this section manually and
    get the information that we need

    Top sections contain metadata with key:val split
    Bottom sections are a list of objects with ,: separated parameters
    """
    f = open(filepath, "rb")
    f.seek(0)

    metadata = {}
    objs = {}

    # first line is osu file format
    metadata["format"] = re.match("osu file format (.+)", f.readline().decode()).group(
        1
    )
    LOGGER.info(f"Got file format {metadata['format']}")

    obj_headers = ["Events", "TimingPoints", "HitObjects"]

    # 0 = skip, 1 = keyval, 2 = parameterlist
    linefmt = 1
    section = None
    for line in f:
        line = line.decode()
        line = line.strip("\r\n")
        if re.match("\/\/", line) or not line or line == "":
            continue

        # anything with obj lists
        obj_headers = ["Events", "TimingPoints", "HitObjects"]

        # [Section] header
        matches = re.match("\[(.+)\]", line)
        if matches and matches.group(1):
            section = matches.group(1)

            if section == "Editor":
                linefmt = 0

            # everything events and below is object list
            elif section in obj_headers:
                linefmt = 2
                objs[section] = []

            else:
                linefmt = 1

        elif linefmt == 0:
            continue
        elif linefmt == 1:
            key, val = list(map(lambda x: x.strip(" "), line.split(":")))
            metadata[key] = val
        elif linefmt == 2:
            vals = list(map(lambda x: x.strip(" "), line.split(",")))
            objs[section].append(vals)
    return metadata, objs

def sanitise_metadata(m):
    for k, v in m.items():
        for fn in (int, float):
            try:
                m[k] = fn(v)
            except:
                continue

    return m

def sanitise_event(e):
    ret = {}

    if len(e) < 2:
        LOGGER.error(f"Unparseable event {e}")

    ret["startTime"] = float(e[1])

    if e[0] == "0":
        ret["eventType"] = "bg"
        ret["file"] = e[2].strip('"')
        ret["x"] = int(e[3])
        ret["y"] = int(e[4])
    else:
        LOGGER.error(f"Unsupported event {e[0]}")
        ret["eventType"] = "unsupported"

    # not supporting anything else right now
    return ret

def sanitise_timing(e):
    ret = {}

    ret["uninherited"] = True if e[6] == "1" else False

    # this can either be measure value, or SV change
    if ret["uninherited"]:
        ret["beatLength"] = float(e[1])
    else:
        ret["sv_mult"] = -1.0 / (float(e[1]) / 100.0)

    ret["time"] = int(e[0])
    ret["meter"] = int(e[2])
    ret["sampleSet"] = int(e[3])
    ret["sampleIndex"] = int(e[4])
    ret["volume"] = int(e[5])
    ret["effects"] = int(e[7])

    return ret

def sanitise_mania_hitobj(e, n):
    ret = {}

    ret["lane"] = floor(int(e[0]) * n / 512)
    ret["time"] = int(e[2])
    type_int = int(e[3])

    # note last variable is empty as .osu has trailing :
    tail = e[5].split(":")

    if (type_int >> 0) & 1:
        ret["ln"] = False
    elif (type_int >> 7) & 1:
        ret["ln"] = True
        ret["time_end"] = int(tail[0])
    ret["hitSound"] = int(e[4])
    ret["sample"] = tail[-2]

    return ret

def _measure_time_from_bpm(bpm):
    return 1 / (bpm / 60 / 1000)

def _bpm_from_measure_time(ms):
    return 1 / ms * 1000 * 60

def mania_calc_offset(timings):
    """Calculate the required offset from start for timings

    BMS gets measures from the BPM value, we can offset the measure by changing BPM
    Calculate how much offset by checking the first timing point for the BPM
    (this would be 1/beatLength * 1000 * 60)
    """

    first_timing = [x for x in timings if x["uninherited"]][0]
    initial_bpm = _bpm_from_measure_time(first_timing["beatLength"])
    LOGGER.info(f"Got first timing BPM {initial_bpm}")

    offset_ms = (first_timing["time"] / first_timing["beatLength"] % 1) * first_timing[
        "beatLength"
    ]

    # how much the audio needs to shift in order to create a full measure at start of track
    shift_ms = first_timing["beatLength"] - offset_ms
    LOGGER.info(f"Got offset of {shift_ms}")

    return initial_bpm, shift_ms

def mania_add_offset(obj, offset):
    obj["time"] += offset
    if "time_end" in obj:
        obj["time_end"] += offset
    return obj

def extract_osz(filepath):
    with ZipFile(filepath, "r") as osz:
        osz.extractall()

def _mania_ms_to_pulse(ms, measure_ms, resolution):
    """Given an osumania hitobj, convert the ms into pulse (ignoring previous timing points)"""
    measure_pulse = (ms / measure_ms) * resolution
    return measure_pulse

def bmson_gen_note(maniaobj, beat_ms, time_offset):
    """Generate bmson note from osumania objects and timings"""
    note = {}
    note["x"] = maniaobj["lane"] + 1

    note["y"] = _mania_ms_to_pulse(maniaobj["time"] - time_offset, beat_ms, 240)
    note["y"] = int(round(note["y"], 1))

    if not maniaobj["ln"]:
        note["l"] = 0
    else:
        diff = maniaobj["time_end"] - maniaobj["time"]
        note["l"] = _mania_ms_to_pulse(diff, beat_ms, 240)
        note["l"] = int(round(note["l"], 1))

    return note

def bmson_group_mania_soundchannels(hitobjs, timings, hitsounds):
    """bmson format groups notes with the same hitsounds together"""

    timings_i = iter(timings)

    # current timing reference = at x time, y beats per measure
    c_timing_ref = next(timings_i, None)
    c_next_ref = next(timings_i, None)
    if not c_timing_ref:
        LOGGER.error("No timings in list")
        return

    if not c_timing_ref["uninherited"]:
        LOGGER.error("Expected first timing point to be uninherited")
        return

    ref_measure_ms = c_timing_ref["beatLength"]
    measure_ms = ref_measure_ms

    # the total amount of pulses elapsed since beginning of timing point
    total_pulses = _mania_ms_to_pulse(c_timing_ref["time"], measure_ms, 240)

    sound_channels = []
    default_channel = {"name": "0", "notes": []}
    bpm_events = []

    for o in hitobjs:
        if c_next_ref and c_next_ref["time"] <= o["time"]:
            time_diff = c_next_ref["time"] - c_timing_ref["time"]
            total_pulses += _mania_ms_to_pulse(time_diff, measure_ms, 240)

            c_timing_ref = c_next_ref
            c_next_ref = next(timings_i, None)

            if not c_timing_ref["uninherited"]:
                target_ref_ms = ref_measure_ms
                target_ms = ref_measure_ms / c_timing_ref["sv_mult"]
            else:
                target_ref_ms = c_timing_ref["beatLength"]
                target_ms = target_ref_ms
                LOGGER.warning("Multiple bpm settings may not work")

            # check if values are sensible
            if 10 < target_ms < 9999:
                measure_ms = target_ms
                ref_measure_ms = target_ref_ms

            bpm_event = {"y": total_pulses, "bpm": _bpm_from_measure_time(measure_ms)}
            bpm_events.append(bpm_event)

        sample = o["sample"]
        if sample != "0" and hitsounds:
            channel_obj = next(
                filter(lambda x: x["name"] == sample, sound_channels), None
            )
        else:
            channel_obj = default_channel
        if not channel_obj:
            channel_obj = {"name": sample, "notes": []}
            sound_channels.append(channel_obj)

        note_obj = bmson_gen_note(o, measure_ms, c_timing_ref["time"])
        note_obj["y"] += total_pulses
        channel_obj["notes"].append(note_obj)

    sound_channels.append(default_channel)
    return sound_channels, bpm_events

def bmson_gen_main_audio_info(pulse, audiofile):
    """Generates the main audio file name in osumania at the correct offset"""
    channel_obj = {"name": audiofile, "notes": []}

    start_obj = {"x": 0, "y": pulse, "c": True}
    channel_obj["notes"].append(start_obj)

    return channel_obj

def bmson_gen_info(metadata):
    info = {}
    info["title"] = metadata["TitleUnicode"]
    info["subtitle"] = metadata["Version"]
    info["artist"] = metadata["ArtistUnicode"]
    info["subartists"] = [f'obj:{metadata["Creator"]}']
    info["genre"] = "O!M Converted"
    info["mode_hint"] = "beat-7k"
    info["level"] = 0
    info["preview_music"] = metadata["AudioFilename"]
    info["resolution"] = 240

    return info

def bmson_gen_bga(bg):
    # only images for now
    bga = {
        "bga_header": [{"id": 0, "name": bg}],
        "bga_events": [{"id": 0, "y": 0}],
        "layer_events": [],
        "poor_events": [],
    }
    return bga

def convert_mania_chart(filepath, dstpath, extra_offset, hitsounds):
    LOGGER.info(f"Converting {filepath}")
    chart_data, chart_objs = get_beatmap_data(filepath)

    # sanitise collected data
    chart_data = sanitise_metadata(chart_data)

    if chart_data["CircleSize"] != 7:
        return

    chart_events_all = list(map(sanitise_event, chart_objs["Events"]))
    chart_events = list(filter(lambda x: x["eventType"] != "unused", chart_events_all))
    chart_timings = list(map(sanitise_timing, chart_objs["TimingPoints"]))
    chart_hitobjs = list(
        map(
            lambda x: sanitise_mania_hitobj(x, chart_data["CircleSize"]),
            chart_objs["HitObjects"],
        )
    )

    # calculate offset for measures
    bpm, offset = mania_calc_offset(chart_timings)
    bpm = round(bpm, 3)
    offset = round(offset, 3)

    chart_hitobjs = list(
        map(lambda x: mania_add_offset(x, offset + extra_offset), chart_hitobjs)
    )
    chart_timings = list(
        map(lambda x: mania_add_offset(x, offset + extra_offset), chart_timings)
    )

    for i in chart_hitobjs[0:5]:
        print(i)

    """BMS Chart processing"""
    # make info dictionary
    info = bmson_gen_info(chart_data)

    # add timing data
    info["init_bpm"] = bpm
    bg = next(filter(lambda x: x["eventType"] == "bg", chart_events), "")["file"]
    info["eyecatch_image"] = bg
    info["back_image"] = bg

    print(info)

    # make sound channels
    channels, bpm_events = bmson_group_mania_soundchannels(chart_hitobjs, chart_timings, hitsounds)

    # calc pulse for first audio
    first_length = chart_timings[0]["beatLength"]
    pulse = _mania_ms_to_pulse(offset + chart_data["AudioLeadIn"], first_length, 240)
    pulse = int(pulse)

    channels.append(bmson_gen_main_audio_info(pulse, chart_data["AudioFilename"]))

    bmson = {
        "version": "1.0.0",
        "info": info,
        "bga": bmson_gen_bga(bg),
        "bpm_events": bpm_events,
        "lines": None,
        "stop_events": None,
        "sound_channels": channels,
    }

    filebase = os.path.basename(filepath)
    dstfile = os.path.join(dstpath, f"{filebase}.bmson")
    with open(dstfile, "w") as f:
        json.dump(bmson, f, indent=2)

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)

    parser = OptionParser()
    parser.add_option("-z", "--osz", dest="osz", help="mania beatmap to convert")
    parser.add_option("-d", "--dst", dest="dst", help="destination bms directory")
    parser.add_option("-o", "--offset", dest="offset", help="offset override")
    parser.add_option(
        "-s",
        "--hitsounds",
        dest="hitsounds",
        default=False,
        help="make dedicated hitsound channels, default off",
    )
    parser.add_option(
        "-p",
        "--present",
        dest="preset",
        default="beatoraja",
        help="offset preset for mp3 [beatoraja/bemuse] (defaults to beatoraja)",
    )

    opt, args = parser.parse_args()
    if not opt.osz:
        parser.error("osz not given")
    if not opt.dst:
        parser.error("dst not given")

    if opt.offset:
        try:
            offset = int(opt.offset)
        except ValueError:
            offset = 0

    elif opt.preset and opt.preset == "bemuse":
        offset = 5

    """Mania chart processing"""
    # get raw data of beatmap
    dstfolder = unzip_osz(opt.osz, opt.dst)
    LOGGER.info(dstfolder)
    for file in glob.glob(f"{dstfolder}/*.osu"):
        convert_mania_chart(file, dstfolder, offset, opt.hitsounds)


class Solution:
    def simplifyPath(self, path: str) -> str:
        path=path.replace("//","/")
        x=path.split("/")
        t=[]
        x=[i for i in x if i!=""]
        print(x)
        for i in x:
            if i==".":
                continue
            elif i=="..":
                if t!=[]:
                    t.pop()
            else:
                t.append("/"+i)
        if t==[]:
            return "/"
        return "".join(i for i in t)


from forbiddenfp import count_if

grid = r""".|...\....
|.-.\.....
.....|-...
........|.
..........
.........\
..../.\\..
.-.-/..|..
.|....-|.\
..//.|....""".splitlines()
num_rows = len(grid)
num_cols = len(grid[0])
row_range = range(num_rows)
col_range = range(num_cols)

def start(start_r: int, start_c: int, start_dr: int, start_dc: int) -> int:
    seen = set()
    energy = [[False] * num_cols for _ in grid]
    dfs = lambda r, c, dr, dc: seen.add((r, c, dr, dc)).also(energy[r].setitem(c, True)).also(
        grid[r][c].map_val({
            ".": [(dr, dc)],
            "/": [(-dc, -dr)],
            "\\": [(dc, dr)],
            "-": [(dr, dc)] if dc else [(0, -1), (0, 1)],
            "|": [(dr, dc)] if dr else [(-1, 0), (1, 0)]
        }).map_unpack(lambda ddr, ddc: (r + ddr, c + ddc, ddr, ddc)).filter_unpack(
            lambda rr, cc, *ds: rr in row_range and cc in col_range and (rr, cc, *ds) not in seen
        ).each_unpack(dfs))
    dfs(start_r, start_c, start_dr, start_dc)
    return energy.sum(count_if())

# 1
start(0, 0, 0, 1).print()
# 2
row_range.map(lambda r: (r, 0, 0, 1)).chain(
    row_range.map(lambda r: (r, num_cols - 1, 0, -1))).chain(
    col_range.map(lambda c: (0, c, 1, 0))).chain(
    col_range.map(lambda c: (num_rows - 1, c, -1, 0))
).map_unpack(start).max().print()


import random
import numpy as np
import math
import time
import scipy.stats as stats

# EX1
def monte_carlo_est(num_samples=100):

    estimations = [0] * num_samples

    for i in range(num_samples):
        rand = random.random()
        estimations[i] = math.exp(rand)

    std = np.std(estimations)
    CI = std/math.sqrt(num_samples) * stats.t.ppf(0.05, num_samples-1)

    return np.mean(estimations), CI

# EX2

def antithetic_est(num_samples=100):
    estimations = [0] * num_samples

    for i in range(num_samples):
        rand = random.random()
        exp_num = math.exp(rand)

        estimations[i] = (exp_num + math.exp(1)/exp_num) / 2

    std = np.std(estimations)
    CI = std/math.sqrt(num_samples) * stats.t.ppf(0.05, num_samples-1)

    return np.mean(estimations), CI

def control_est(num_samples=100):
    estimations = [0] * num_samples

    c = -1.69056

    for i in range(num_samples):
        rand = random.random()

        estimations[i] = math.exp(rand) + c*(rand - 1/2)

    std = np.std(estimations)
    CI = std/math.sqrt(num_samples) * stats.t.ppf(0.05/2, num_samples-1)

    return np.mean(estimations), CI

def strat_est(num_samples=100, strata=10):

    estimations = [0] * num_samples

    for i in range(num_samples):
        # sample number, divide by strata and use expon
        rand_nums = np.exp(np.random.random(10)/strata)

        # Compute expon values e^(0/10), e^(1/10)...e^(9/10)
        exp_vals = np.exp(np.arange(0, 1, 1/strata))

        sample = np.sum(rand_nums * exp_vals) / 10

        estimations[i] = sample

    std = np.std(estimations)
    CI = std/math.sqrt(num_samples) * stats.t.ppf(0.05/2, num_samples-1)

    return np.mean(estimations), CI

print(f'True Evaluation of integral: {math.exp(1) - 1}')

# TEST EX1
print('Monte Carlo estimation')
time_montecarlo = time.time()
est_montecarlo, CI_montecarlo = monte_carlo_est(num_samples=100)
print(f'Estimation: {est_montecarlo}')
print(f'Confidence Interval: {est_montecarlo}+-{CI_montecarlo}')
print(f'finish time: {time.time() - time_montecarlo}')
print('#'*5)

# TEST EX2
print('Anithetic Variables estimation')
time_antithetic = time.time()
est_antithetic, CI_antithetic = antithetic_est(num_samples=100)
print(f'Estimation: {est_antithetic}')
print(f'Confidence Interval: {est_antithetic}+-{CI_antithetic}')
print(f'finish time: {time.time() - time_antithetic}')
print('#'*5)

# TEST EX3
print('Control Variate estimation')
time_contr = time.time()
est_contr, CI_contr = control_est(num_samples=100)
print(f'Estimation: {est_contr}')
print(f'Confidence Interval: {est_contr}+-{CI_contr}')
print(f'finish time: {time.time() - time_contr}')
print('#'*5)

# TEST EX4
print('Stratified Sampling estimation')
time_strat = time.time()
est_strat, CI_strat = strat_est(num_samples=100)
print(f'Estimation: {est_strat}')
print(f'Confidence Interval: {est_strat}+-{CI_strat}')
print(f'finish time: {time.time() - time_strat}')
print('#'*5)


from uuid import uuid4

import attr
from django.test import SimpleTestCase

from casexml.apps.case.const import CASE_ATTR_ID
from casexml.apps.case.xform import get_case_ids_from_form
from casexml.apps.case.xml import V2_NAMESPACE
from casexml.apps.case.xml.parser import XMLNS_ATTR

class TestXformCaseIds(SimpleTestCase):

    def test_basic(self):
        case_id = uuid4().hex
        xform = FakeForm({
            'data': {'some': 'stuff'},
            'case': case_block(case_id),
        })
        self.assertEqual(get_case_ids_from_form(xform), {case_id})

    def test_blocks_in_list(self):
        case_ids = {uuid4().hex for x in range(3)}
        xform = FakeForm({'data': {'parent': {'parent': {
            'case': [case_block(c) for c in case_ids]
        }}}})
        self.assertEqual(get_case_ids_from_form(xform), case_ids)

    def test_blocks_in_repeat(self):
        case_ids = {uuid4().hex for x in range(3)}
        blocks = [case_block(c) for c in case_ids]
        xform = FakeForm({
            'data': {
                'parent': {
                    'repeats': [{'group': {'case': block}} for block in blocks]
                }
            }
        })
        self.assertEqual(get_case_ids_from_form(xform), case_ids)

def case_block(case_id):
    return {XMLNS_ATTR: V2_NAMESPACE, CASE_ATTR_ID: case_id}

@attr.s
class FakeForm:
    form_data = attr.ib()

    def get_xml_element(self):
        return []


from hvps.utils import get_serial_ports
from hvps import Iseg

import pytest
import sys
import logging

serial_port = "COM4"  # change this to the serial port you are using
serial_baud = 9600
timeout = 5.0

def serial_port_available():
    ports = get_serial_ports()
    return ports != []

def is_macos():
    return sys.platform == "Darwin"

serial_skip_decorator = pytest.mark.skipif(
    serial_port_available() or not is_macos(), reason="No serial ports available"
)

@serial_skip_decorator
def test_iseg_init(caplog):
    caplog.set_level("DEBUG")

    with Iseg(logging_level="DEBUG") as iseg:
        assert iseg.baudrate == 115200
        assert "Using baud rate 115200" in caplog.text
        assert "Using port " in caplog.text
        assert "Using timeout " in caplog.text

@serial_skip_decorator
def test_iseg_module_monitor():
    iseg = Iseg(
        port=serial_port,
        baudrate=serial_baud,
        connect=True,
        timeout=timeout,
        logging_level=logging.DEBUG,
    )
    iseg.connect()

    print(
        f"Serial port status: connected: {iseg.connected}, port: {iseg.port}, baudrate: {iseg.baudrate}, timeout: {iseg.timeout}"
    )
    module = iseg.module(0)

    number_of_channels = module.number_of_channels
    print(f"number_of_channels: {number_of_channels}")

    firmware_release = module.firmware_release
    print(f"firmware_release: {firmware_release}")

    module_status = module.module_status
    print(f"module_status: {module_status}")

    filter_averaging_steps = module.filter_averaging_steps
    print(f"filter_averaging_steps: {filter_averaging_steps}")

    kill_enable = module.kill_enable
    print(f"kill_enable: {kill_enable}")

    adjustment = module.adjustment
    print(f"adjustment: {adjustment}")

    module_can_address = module.module_can_address
    print(f"module_can_address: {module_can_address}")

    module_can_bitrate = module.module_can_bitrate
    print(f"module_can_bitrate: {module_can_bitrate}")

    serial_echo_enabled = module.serial_echo_enabled
    print(f"serial_echo_enabled: {serial_echo_enabled}")

    serial_echo_disabled = module.serial_echo_disabled
    print(f"serial_echo_disabled: {serial_echo_disabled}")

    module_voltage_limit = module.module_voltage_limit
    print(f"module_voltage_limit: {module_voltage_limit}")

    module_current_limit = module.module_current_limit
    print(f"module_current_limit: {module_current_limit}")

    module_voltage_ramp_speed = module.module_voltage_ramp_speed
    print(f"module_voltage_ramp_speed: {module_voltage_ramp_speed}")

    module_current_ramp_speed = module.module_current_ramp_speed
    print(f"module_current_ramp_speed: {module_current_ramp_speed}")

    module_control_register = module.module_control_register
    print(f"module_control_register: {module_control_register}")

    module_status_register = module.module_status_register
    print(f"module_status_register: {module_status_register}")

    module_event_status_register = module.module_event_status_register
    print(f"module_event_status_register: {module_event_status_register}")

    module_event_mask_register = module.module_event_mask_register
    print(f"module_event_mask_register: {module_event_mask_register}")

    module_event_channel_status_register = module.module_event_channel_status_register
    print(
        f"module_event_channel_status_register: {module_event_channel_status_register}"
    )

    module_event_channel_mask_register = module.module_event_channel_mask_register
    print(f"module_event_channel_mask_register: {module_event_channel_mask_register}")

    module_supply_voltage = module.module_supply_voltage
    print(f"module_supply_voltage: {module_supply_voltage}")

    module_supply_voltage_p24v = module.module_supply_voltage_p24v
    print(f"module_supply_voltage_p24v: {module_supply_voltage_p24v}")

    module_supply_voltage_n24v = module.module_supply_voltage_n24v
    print(f"module_supply_voltage_n24v: {module_supply_voltage_n24v}")

    module_supply_voltage_p5v = module.module_supply_voltage_p5v
    print(f"module_supply_voltage_p5v: {module_supply_voltage_p5v}")

    module_supply_voltage_p3v = module.module_supply_voltage_p3v
    print(f"module_supply_voltage_p3v: {module_supply_voltage_p3v}")

    module_supply_voltage_p12v = module.module_supply_voltage_p12v
    print(f"module_supply_voltage_p12v: {module_supply_voltage_p12v}")

    module_supply_voltage_n12v = module.module_supply_voltage_n12v
    print(f"module_supply_voltage_n12v: {module_supply_voltage_n12v}")

    module_temperature = module.module_temperature
    print(f"module_temperature: {module_temperature}")

    setvalue_changes_counter = module.setvalue_changes_counter
    print(f"setvalue_changes_counter: {setvalue_changes_counter}")

    firmware_name = module.firmware_name
    print(f"firmware_name: {firmware_name}")

    # Setter methods
    module.serial_baud_rate = 9600
    module.serial_echo_enable = 1
    with pytest.raises(ValueError):
        module.serial_echo_enable = 2

    module.filter_averaging_steps = 256
    with pytest.raises(ValueError):
        module.filter_averaging_steps = 257

    module.kill_enable = 1
    with pytest.raises(ValueError):
        module.kill_enable = 2

    module.adjustment = 0
    with pytest.raises(ValueError):
        module.adjustment = 2

    module.module_event_mask_register = 5

    module.module_can_address = 12
    with pytest.raises(ValueError):
        module.module_can_address = 250

    module.module_can_bitrate = 250000
    with pytest.raises(ValueError):
        module.module_can_bitrate = 250

    # Other methods
    module.enter_configuration_mode(12345)
    module.exit_configuration_mode()
    module.set_serial_echo_enabled()
    module.set_serial_echo_disabled()
    module.reset_module_event_status()
    module.clear_module_event_status_bits(8)

    iseg.disconnect()

@serial_skip_decorator
def test_iseg_channel_monitor():
    # TODO: make this work
    # TODO: validate the values (atleast check if they are not None)
    # TODO: Do the same for CAEN

    trip_action = channel.trip_action
    print(f"trip_action: {trip_action}")

    output_mode = channel.output_mode
    print(f"output_mode: {output_mode}")

    output_polarity = channel.output_polarity
    print(f"output_polarity: {output_polarity}")

    available_output_polarities = channel.available_output_polarities
    print(f"available_output_polarities: {available_output_polarities}")

    voltage_set = channel.voltage_set
    print(f"voltage_set: {voltage_set}")

    voltage_limit = channel.voltage_limit
    print(f"voltage_limit: {voltage_limit}")

    voltage_nominal = channel.voltage_nominal
    print(f"voltage_nominal: {voltage_nominal}")

    available_output_modes = channel.available_output_modes
    print(f"available_output_modes: {available_output_modes}")

    voltage_mode = channel.voltage_mode
    print(f"voltage_mode: {voltage_mode}")

    voltage_mode_list = channel.voltage_mode_list
    print(f"voltage_mode_list: {voltage_mode_list}")

    voltage_bounds = channel.voltage_bounds
    print(f"voltage_bounds: {voltage_bounds}")

    set_on = channel.set_on
    print(f"set_on: {set_on}")

    emergency_off = channel.emergency_off
    print(f"emergency_off: {emergency_off}")

    current_set = channel.current_set
    print(f"current_set: {current_set}")

    current_limit = channel.current_limit
    print(f"current_limit: {current_limit}")

    current_nominal = channel.current_nominal
    print(f"current_nominal: {current_nominal}")

    current_mode = channel.current_mode
    print(f"current_mode: {current_mode}")

    modes = channel.current_mode_list
    print(f"modes: {modes}")

    bounds = channel.current_bounds
    print(f"bounds: {bounds}")

    speed = channel.current_ramp_speed
    print(f"speed: {speed}")

    speed = channel.voltage_ramp_speed
    print(f"speed: {speed}")

    speed_min = channel.voltage_ramp_speed_minimum
    print(f"speed_min: {speed_min}")

    speed_max = channel.voltage_ramp_speed_maximum
    print(f"speed_max: {speed_max}")

    speed_min = channel.current_ramp_speed_minimum
    print(f"speed_min: {speed_min}")

    speed_max = channel.current_ramp_speed_maximum
    print(f"speed_max: {speed_max}")

    control = channel.channel_control
    print(f"control: {control}")

    status = channel.channel_status
    print(f"status: {status}")

    mask = channel.channel_event_mask
    print(f"mask: {mask}")

    voltage = channel.measured_voltage
    print(f"voltage: {voltage}")

    current = channel.measured_current
    print(f"current: {current}")

    speed = channel.channel_voltage_ramp_up_speed
    print(f"speed: {speed}")

    speed = channel.channel_voltage_ramp_down_speed
    print(f"speed: {speed}")

    speed = channel.channel_current_ramp_up_speed
    print(f"speed: {speed}")

    speed = channel.channel_current_ramp_down_speed
    print(f"speed: {speed}")

    # Set the action to be taken when a current trip occurs for the channel
    channel.trip_action = 1
    with pytest.raises(ValueError):
        channel.trip_action = 5

    # Set the trip timeout in milliseconds
    channel.trip_timeout = 2000
    with pytest.raises(ValueError):
        channel.trip_timeout = 0

    # Set the action to be taken when an External Inhibit event occurs for the channel
    channel.external_inhibit_action = 2
    with pytest.raises(ValueError):
        channel.external_inhibit_action = 5

    # Set the channel output mode
    channel.output_mode = 3
    with pytest.raises(ValueError):
        channel.output_mode = 5

    # Set the output polarity of the channel
    channel.output_polarity = "n"
    with pytest.raises(ValueError):
        channel.output_polarity = "x"

    # Set the channel voltage set
    channel.voltage_set = 12.5

    # Set the channel voltage bounds
    channel.voltage_bounds = 10.0

    # Clear the channel from state emergency off
    channel.clear_emergency_off()

    # Set the channel current set
    channel.current_set = 2.5

    # Set the channel current bounds
    channel.current_bounds = 2.0

    # Set the channel voltage ramp speed for up and down direction
    channel.set_channel_voltage_ramp_up_down_speed(250)

    # Set the channel voltage ramp up speed
    channel.channel_voltage_ramp_up_speed = 200

    # Set the channel voltage ramp down speed
    channel.channel_voltage_ramp_down_speed = 150.0

    # Switch on the high voltage with the configured ramp speed
    channel.switch_on_high_voltage()

    # Switch off the high voltage with the configured ramp speed
    channel.switch_off_high_voltage()

    # Shut down the channel high voltage (without ramp)
    channel.shutdown_channel_high_voltage()

    # Clear the channel from state emergency off
    channel.clear_channel_emergency_off()

    # Clear the Channel Event Status register
    channel.clear_event_status()

    # Clears single bits or bit combinations in the Channel Event Status register
    channel.clear_event_bits(3)

    # Set the Channel Event Mask register
    channel.set_event_mask(7)


#Insert = Push
#Delete = Pop

class Stack:
    def __init__(self,maxsize):
        self.stack = [None] * maxsize
        self.maxsize = maxsize
        self.top = -1

    def StackEmpty(self):
        if self.top < 0:
            return True
        else:
            return False

    def Push(self,e):
        if  self.top < self.maxsize-1:
            self.top += 1
            self.stack[self.top] = e
        else:
            raise Exception("Overflow")

    def Pop(self):
        if not self.StackEmpty():
            self.top -= 1
            return self.stack[self.top+1]
        else:
            raise Exception("Underflow")

stack = Stack(2)
stack.Push(108)
stack.Push(123)
stack.Push(123)
print(stack.Pop())
print(stack.Pop())


# The kata variable is always a tuple that contains 5 non-negative integers. The first
# integer contains up to 4 digits, the rest up to 2 digits.
kata = (2019, 9, 25, 3, 30)

def count_digits(num: int):
    count = 0
    while (num != 0):
        num //= 10
        count += 1
    return count

if (type(kata) != tuple):
    exit(print("Kata variable is not a Tuple, mais", type(kata)))

size = len(kata)
if (size == 5):
    if (count_digits(kata[0]) <= 4 and
            count_digits(kata[1]) <= 2 and
            count_digits(kata[2]) <= 2 and
            count_digits(kata[3]) <= 2 and
            count_digits(kata[4]) <= 2):
        print("%02d/%02d/%04d %02d:%02d" %( kata[1], kata[2], kata[0], kata[3], kata[4]))
        # print("{:0>2d}/{:0>2d}/{:0>4d} {:0>2d}:{:0>2d}".format(
        #     kata[1], kata[2], kata[0], kata[3], kata[4]))
    else:
        print("Kata variable does not have the right format")


####################################################################
#CLASS: mard
#FUNCTION: __init__, set_name, set_age, set_surname
#BRIEF:
#PARAMS: (self,name,surname,age)
#OUTPUT:
#RETURN: self.__name, self.__surname, self.__age
####################################################################

class mard:
    #receives arguments(self,name,surname,age)
    def __init__(self,name,surname,age):
        self.__name = name
        self.__surname = surname
        self.__age = age

    #checking if parameters are correct name
    def set_name(self,name):
        if name.isalpha():
            self.__name = name
        else:
            self.__name = "sxal anuni mutq"

    #checking if parameters are correct age
    def set_age(self,age):
        if age >= 0:
            self.__age = age
        else:
            self.__age = "sxal tariqi mutq"

    #checking if parameters are correct surname
    def set_surname(self,surname):
        if surname.isalpha():
            self.__surname = surname
        else:
            self.__surname = "sxal azganuni mutq"

    #return self.__name,self.__surname,self.__age
    def get_a(self):
        return self.__name,self.__surname,self.__age

#create a character and give parameters,display the result on the screen
person = mard("Valod","Petrosyan",45)
print(person.get_a())
person.set_name("Gabriel")
person.set_age(5)
person.set_surname("Poxosyan")
print(person.get_a())


import os
import random
print("###### WELCOME TO RPS######")
user_score=0
comp_score=0
ties=0
name=input("enter your name here:")
print("""
Winning rules
1. paper vs rock-->paper
2. rock vs scissors-->rock
3. scissors vs paper-->scissors
""")
while True:
    print(""" The choices are
    1. Rock
    2. Paper
    3. Scissors
    """)
    choice=int(input("enter your choice from 1-3:"))
    print()
    while choice >3 or choice < 1:
        choice=int(input("enter valid input"))
    if choice==1:
        user_choice="Rock"
    elif choice==2:
        user_choice="Paper"
    else :
        user_choice="Scissors"
    print(name,"'s choice is",user_choice)
    print("now it is computer's turn")
    computer= random.randint(1,3)
    if computer==1:
        comp_choice="Rock"
    elif computer==2:
        comp_choice="Paper"
    else :
        comp_choice="Scissors"
    print("computer choice is",comp_choice)
    if((user_choice=="Papaer" and comp_choice=="Rock") or( user_choice=="Rock" and comp_choice=="Paper")):
       print("Paper win")
       resuit="paper"
    elif((user_choice=="Rock" and comp_choice=="Scissors") or( user_choice=="scissors" and comp_choice=="Rock")):
        print("Rock Win")
        result="Rock"
    elif(user_choice==comp_choice):
        print("it is a tie")
        result="Ties"
    else:
        print("scissors wins")
        result="scissors"
    if result=="Ties":
        ties+=1
    elif result== user_choice:
        print(name,"WIN")
        user_score +=1
    else:
        print("COMPUTER WIN")
        comp_score +=1
    print("score are")
    print(name,"'s score is",user_score)
    print("computer's score",comp_score)
    print("ties are",ties)

    repeat=input("do you want to play again?")
    if repeat=="n" or repeat=="no":
        break
print("Game Over")
print("Thank You!")


from time import time, sleep

import pandas as pd
import numpy as np
from tqdm import tqdm
from descriptastorus.descriptors.DescriptorGenerator import MakeGenerator

from rdkit import Chem

def canonical_smiles(df):
    """
    Objective: Create list of canonical SMILES from SMILES
    Intent: While the SMILES in dataset from moleculenet.ai are all canonical, it is always good to be safe. I don't
            know if I should add in a way to detect irregular SMILES and remove the rows that contains them in the
            dataframe. However, that process should be carried out at the start of the pipeline instead of at the end.
    :param smiles_list:
    :return:
    """

    smiles = df['smiles']
    con_smiles = []
    for smile in smiles:
        mol = Chem.MolFromSmiles(smile)
        if mol is not None:
            con_smiles.append(Chem.MolToSmiles(mol))
        else:
            con_smiles.append('bad_smiles')
    df['smiles'] = con_smiles
    df = df.loc[df['smiles'] != 'bad_smiles']

    return df

def featurize(self, not_silent=True, retrieve_from_mysql=False):
    """
    Caclulate molecular features.
    Returns DataFrame, list of selected features (numeric values. i.e [0,4]),
     and time to featurize.
    Keyword arguments:
    feat_meth -- Features you want by their numerical value.  Default = None (require user input)
    """
    feat_meth = self.feat_meth
    # if self.dataset == "flashpoint.csv":
    #     self.data['flashpoint'] = [float(i) for i in list(self.data['flashpoint'])]
    df = self.data

    # available featurization options
    feat_sets = ['rdkit2d', 'rdkitfpbits', 'morgan3counts', 'morganfeature3counts', 'morganchiral3counts',
                 'atompaircounts']

    if feat_meth is None:  # ask for features
        print('   {:5}    {:>15}'.format("Selection", "Featurization Method"))
        [print('{:^15} {}'.format(*feat)) for feat in enumerate(feat_sets)];
        feat_meth = [int(x) for x in input(
            'Choose your features  by number from list above.  You can choose multiple with \'space\' delimiter:  ').split()]
    selected_feat = [feat_sets[i] for i in feat_meth]

    self.selected_feat_string = '-'.join(selected_feat) # This variable will be used later in train.py for giving classification roc graph a unique file name.

    self.feat_method_name = selected_feat

    # Get data from MySql if called
    if retrieve_from_mysql:
        print("Pulling data from MySql")
        self.featurize_from_mysql()
        return

    if not_silent:  # Add option to silence messages
        print("You have selected the following featurizations: ", end="   ", flush=True)
        print(*selected_feat, sep=', ')
        print('Calculating features...')
    sleep(0.25)
    # Start timer
    start_feat = time()

    # Use descriptastorus generator
    generator = MakeGenerator(selected_feat)
    columns = []

    # get the names of the features for column labels
    for name, numpy_type in generator.GetColumns():
        columns.append(name)
    smi = df['smiles']

    issue_row_list = []
    issue_row = 0
    for smiles in smi:
        x = Chem.MolFromSmiles(smiles)
        if x == None:
            issue_row_list.append(issue_row)
        issue_row = issue_row + 1

    rows = df.index[[issue_row_list]]
    df.drop(rows, inplace=True)
    smi.drop(rows, inplace=True)

    smi2 = tqdm(smi, desc="Featurization")  # for progress bar
    data = list(map(generator.process, smi2))
    if not_silent:
        print('Done.')
    stop_feat = time()
    feat_time = stop_feat - start_feat

    # make dataframe of all features
    features = pd.DataFrame(data, columns=columns)

    df = df[~df.index.duplicated(keep='first')]
    features = features[~features.index.duplicated(keep='first')]
    df = pd.concat([df, features], axis=1)
    df = df.dropna()

    # remove the "RDKit2d_calculated = True" column(s)
    df = df.drop(list(df.filter(regex='_calculated')), axis=1)
    df = df.drop(list(df.filter(regex='[lL]og[pP]')), axis=1)

    # store data back into the instance
    self.data = df
    self.feat_time = feat_time
    self.data.iloc[:, 1:] = self.data.iloc[:, 1:].apply(pd.to_numeric)
    # Replacing infinite with nan
    self.data.replace([np.inf, -np.inf], np.nan, inplace=True)

    # Dropping all the rows with nan values
    self.data.dropna(inplace=True)
    self.data.reset_index(drop=True, inplace=True)


# coding: utf-8

# In[ ]:

import classifierAlgo
import os
import time

def testPredictionPerMood(listText, listIndentAmbiance):
    compteur = 0
    nbText = 0
    listAmbiance = ['Administration', 'Anger', 'Attic', 'Bathroom', 'Bedroom', 'Calm', 'Car', 'Castle', 'Cave', 'Cemetery', 'Church', 'Circus', 'Company', 'CountryTown', 'Desert', 'Fear', 'Field', 'Fight', 'Fog', 'Forest', 'Garden', 'Gunshot', 'Gym', 'Happy', 'Hospital', 'Industry', 'Jail', 'Kitchens', 'Library', 'LivingRoom', 'Love', 'Meadow', 'Mine', 'Mountain', 'Ocean', 'Plane', 'Rain', 'Sad', 'School', 'Snow', 'Sun', 'Thunder', 'TrainStation', 'Wind']
    for i in listText:
        string = classifierAlgo.stringtoMood(i)
        indentAmbiance = listIndentAmbiance[nbText]
        if string == listAmbiance[indentAmbiance]:
            compteur +=1
            #listRate.append([ambiance, compteur*100/nbText])
        nbText +=1
    return compteur/nbText

def testPredictionTotal(testDataDirectoryPath):
    sumAccuracy = 0
    listRatePerMood = testPredictionPerMood(testDataDirectoryPath)
    n = len(listRatePerMood)
    for i in range(n) :
        sumAccuracy += listRatePerMood[1]
    return sumAccuracy/(n)

def getTrainingTime():
    c1 = cl.clock()
    classifierAlgo.stringtoMood('')
    c2 = cl.clock()
    return c2-c1

import os
banque = classifierAlgo.recuperation("Banque")
listText = banque[7][0][2]
listIndentAmbiance = banque[7][0][3]
accuracy = testPredictionPerMood(listText,listIndentAmbiance)
print(accuracy)
time = getTrainingTime()
print(time)


from sklearn.datasets import load_boston
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score
import time

# 1. 데이터
datasets = load_boston()
x = datasets.data
y = datasets.target
x_train, x_test, y_train, y_test =  train_test_split(x, y, train_size=0.8, shuffle=True, random_state=66)

# 2. 모델구성
model = Sequential()
model.add(Dense(5, input_dim=13))
model.add(Dense(10))
model.add(Dense(20))
model.add(Dense(20))
model.add(Dense(10))
model.add(Dense(5))
model.add(Dense(1))

# 3. 컴파일, 훈련
model.compile(loss = 'mse', optimizer='adam')

from tensorflow.python.keras.callbacks import EarlyStopping
earlyStopping = EarlyStopping(monitor='val_loss', patience=30, mode='min', verbose=1, restore_best_weights=True)
# monitor에서 제공하는 것은 val_loss, loss 정도이고 R2는 제공 안함
# mode auto면 loss계열은 자동으로 최소, accuracy계열은 자동으로 최대 찾음

start_time = time.time()
hist = model.fit(x_train, y_train, epochs=1000, batch_size=1,
                 validation_split=0.2,
                 callbacks=[earlyStopping],
                 verbose=1)
end_time = time.time()

# 4. 평가, 예측
loss = model.evaluate(x_test, y_test)
print('loss: ', loss)

y_predict = model.predict(x_test)
r2 = r2_score(y_test, y_predict)
print('r2:', r2)

print('==============================================')
print(hist)
print('==============================================')
print(hist.history)
print('==============================================')
print('loss: ',hist.history['loss'])
print('\n val_loss: ', hist.history['val_loss'])

# 그림을 그리자!
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.font_manager as fm

mpl.rcParams['font.family'] = 'malgun gothic'
mpl.rcParams['axes.unicode_minus'] = False

plt.figure(figsize=(9,6))
plt.plot(hist.history['loss'], marker='.', c='red', label='loss')
# x가 1일 때 hist.history[1], 2일때 hist.history[2] ... 인 셈
plt.plot(hist.history['val_loss'], marker='.', c='blue', label='val_loss')
plt.grid()
plt.title('보스턴//로스와 발리데이션 로스')
plt.ylabel('loss')
plt.xlabel('epochs')
# plt.legend(loc='upper right')
plt.legend()

print('걸린 시간 : ', end_time - start_time) # 시간 체킹

plt.show()

#============================================
# loss:  20.629039764404297
# r2: 0.7531907652550968


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""utils module."""

import logging
import os

DEBUG = 'debug'
FORMAT = "%(asctime)s [%(threadName)10s] [%(levelname)5s] %(message)s"
FORMAT_DEBUG = "%(asctime)s [%(name)18s] [%(threadName)10s] [%(levelname)5s] %(message)s"  # noqa
FIELD_STYLES = {
    'programname': {
        'color': 'cyan'
    },
    'name': {
        'color': 'blue'
    },
    'levelname': {
        'color': 'white'
    },
    'asctime': {
        'color': 'magenta'
    },
    'threadName': {
        'color': 'cyan'
    }
}

LEVEL_STYLES = {
    'info': {},
    'critical': {
        'color': 'red',
    },
    'error': {
        'color': 'red',
        'bold': True
    },
    'debug': {
        'color': 'green'
    },
    'warning': {
        'color': 'yellow'
    }
}

TOPO_DIR = 'topologies'
TESTS_DIR = 'scenarios_tests'

def _str_to_level(string):
    """Convert string level to actual logging.level.

    Anything that's not debug will be converted to info level.

    :string: either 'debug' or 'info'.

    """
    if string.lower() == DEBUG:
        return logging.DEBUG
    return logging.INFO

def _create_dir(directory):
    """Create directory."""
    if not os.path.isdir(directory):
        os.makedirs(directory)

def find_file(file_name, sub_dir=None):
    """Find file under cwd, cwd/sub_dir or xdg_dir/sub_dir in this order.

    :file_name: yml file name
    :sub_dir:sub directory
    :returns: file path

    """
    cwd = os.getcwd()
    for dir_name in [
            cwd,
            os.path.join(cwd, sub_dir),
            os.path.join(_get_xdg_dir(), sub_dir)
    ]:
        try:
            f_name = os.path.join(dir_name, file_name)
            with open(f_name):
                pass
            return f_name
        except OSError:
            pass
    raise OSError("File {} couldn't be found".format(file_name))

def _get_xdg_dir():
    """Get and Initialize xdg configuration directory.

    :return: xdg_dir

    """
    xdg_dir = os.path.join(os.path.expanduser('~'), '.config', 'netblow')
    return xdg_dir

def _bootstrap_xdg_dirs():
    """Bootstrap xdg dirs."""
    xdg_dir = _get_xdg_dir()
    _create_dir(xdg_dir)
    dirs = [os.path.join(xdg_dir, TOPO_DIR), os.path.join(xdg_dir, TESTS_DIR)]
    for dir_name in dirs:
        _create_dir(dir_name)


import allure
import pytest

from pages.locators import LoginPageLocators, HomePageLocators
from pages.login_page import LoginPage
from pages.page import Page
from pages.report_page import ReportPage
from pages.search_page import SearchPage
from utils.config import config
from utils.utils import get_base_url_by_job_name, get_current_function_name

@pytest.mark.usefixtures('setup')
class TestSiteCrawler:
    reruns = config.JOB_RERUNS
    reruns_delay = config.JOB_RERUNS_DELAY

    @pytest.mark.usefixtures('screenshot_on_failure')
    @pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
    @allure.title('Download sn report test')
    @allure.description('This is test of download sn report')
    def test_download_sn_report(self):
        base_url = get_base_url_by_job_name(config.JOB_LIST, get_current_function_name())
        home_page = Page(self.driver, base_url)
        home_page.open_page(wait_element=LoginPageLocators.tmo_logo_img)
        login_page = LoginPage(self.driver, base_url)
        login_page.login(user='sn', wait_element=HomePageLocators.sn_user_info_dropdown_button)
        report_page = ReportPage(self.driver, base_url)
        report_page.download_report('2d4f0fff1b987d580815a712604bcbca', 'excel')
        report_page.download_report('2d4f0fff1b987d580815a712604bcbca', 'json')

    @pytest.mark.usefixtures('screenshot_on_failure')
    @pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
    @allure.title('Download mem report test')
    @allure.description('This is test of download mem report')
    def test_download_mem_report(self):
        base_url = get_base_url_by_job_name(config.JOB_LIST, get_current_function_name())
        search_page = SearchPage(self.driver, base_url)
        search_page.open_page(wait_element=LoginPageLocators.msft_logo_img)
        login_page = LoginPage(self.driver, base_url)
        login_page.login(user='mem', wait_element=HomePageLocators.msft_user_info_button)
        search_page.open_page(url='#view/Microsoft_Intune_DeviceSettings/DevicesMenu/~/mDMDevicesPreview')
        search_page.download_report()


import sys
import os
from findAllMatchings import findAllMatchings

def getPaths(curDir):
	paths = curDir.split('/')
	print (curDir)
	exit()
	paths[-2] = 'genomasDeInput/'
	paths.pop(-1)
	inputPath = '/'.join(paths)
	inputName = sys.argv[1]

	pathToInput = inputPath+inputName
	return pathToInput, inputPath, inputName

def runFindAllMatchings(pathToInput):
	#sys.argv.extend([pathToInput, 'keyword'])
	#sys.argv[1], sys.argv[2] = pathToInput, 'keyword'
	#execfile('findAllMatchings.py')
	findAllMatchings(pathToInput)

def getFmly(results):
	fmly = {}
	# create dictionary {gene family : positions of family in genome}
	for idx, itm in enumerate(results):
	    if(itm not in fmly):
	        fmly[itm] = [idx]
	    else:
	        fmly[itm].extend([idx])

	return fmly

def getOutPath(curDir):
	paths = curDir.split('/')
	paths[-2] = 'saidas'
	paths[-1] = 'losangos/'
	outPutPath = '/'.join(paths)
	outName = sys.argv[1] + '_losango'

	return outPutPath + outName


"""Testing for Symbolic Aggregate Approximation."""

# Author: Johann Faouzi <johann.faouzi@gmail.com>
# License: BSD-3-Clause

import numpy as np
import pytest
import re
from pyts.approximation import SymbolicAggregateApproximation

X = np.arange(30).reshape(3, 10)

@pytest.mark.parametrize(
    'params, error, err_msg',
    [({'n_bins': '3'}, TypeError, "'n_bins' must be an integer."),

     ({'alphabet': 'whoops'}, TypeError,
      "'alphabet' must be None, 'ordinal' or array-like with shape (n_bins,) "
      "(got {0})".format('whoops')),

     ({'n_bins': 1}, ValueError,
      "'n_bins' must be greater than or equal to 2 and lower than "
      "or equal to 26 (got 1)."),

     ({'n_bins': 27}, ValueError,
      "'n_bins' must be greater than or equal to 2 and lower than "
      "or equal to 26 (got 27)."),

     ({'strategy': 'whoops'}, ValueError,
      "'strategy' must be either 'uniform', 'quantile' or 'normal' "
      "(got {0})".format('whoops')),

     ({'alphabet': ['a', 'b', 'c']}, ValueError,
      "If 'alphabet' is array-like, its shape must be equal to (n_bins,).")]
)
def test_parameter_check(params, error, err_msg):
    """Test parameter validation."""
    sax = SymbolicAggregateApproximation(**params)
    with pytest.raises(error, match=re.escape(err_msg)):
        sax.transform(X)

@pytest.mark.parametrize(
    'params, X, arr_desired',
    [({}, [[0, 1, 2, 3]], [['a', 'b', 'c', 'd']]),

     ({'strategy': 'uniform'}, [[0, 1, 2, 3]], [['a', 'b', 'c', 'd']]),

     ({}, [[0, 4, 2, 6]], [['a', 'c', 'b', 'd']]),

     ({}, [[-5, -8, -7, -6]], [['d', 'a', 'b', 'c']]),

     ({'alphabet': 'ordinal'}, [[0, 1, 2, 3]], [[0, 1, 2, 3]]),

     ({'alphabet': ['d', 'c', 'b', 'a']}, [[0, 1, 2, 3]],
      [['d', 'c', 'b', 'a']]),

     ({'alphabet': ['0', '1', '2', '3']}, [[0, 3, 2, 1]],
      [['0', '3', '2', '1']])]
)
def test_actual_results(params, X, arr_desired):
    """Test that the actual results are the expected ones."""
    arr_actual = SymbolicAggregateApproximation(**params).fit_transform(X)
    np.testing.assert_array_equal(arr_actual, arr_desired)


import codecs
import sys
import json
import tqdm
import os
# 处理SemanticScholar 数据的脚本
# from elasticsearch import Elasticsearch
# from elasticsearch.helpers import bulk

data_dir = "H:\\Scholar"
paper_path = os.path.join(data_dir)

bad_data_num =0

file_list = []
author_list = []
author_dict = {}

venue_list = []
venues ={}
journal_list = []
journals = {}
journal_out_list = []

in_citation_dict = {}
in_citation_list = []

for paper_file in os.listdir(paper_path):
    domain = os.path.abspath(paper_path)
    if paper_file.startswith("s2-corpus") and  not paper_file.endswith(".gz"):
        paper_file = os.path.join(domain,paper_file)
        file_list.append(paper_file)
file_list.sort()

# file_list = ["H:\\Scholar\\s2-corpus-000"]
def proc_file_list(file_list,label):
    global  bad_data_num
    file_num = 0
    for file in file_list:
        with codecs.open(file,'r','utf-8') as f:

            i = 0
            for line in f.readlines():
                if len(line.strip()) == 0:
                    break

                try:
                    dict_item = json.loads(line)
                except:
                    bad_data_num += 1
                    continue
                list_itme = list(dict_item["fieldsOfStudy"])
                if "Computer Science" not in list_itme :
                    continue
                if label == "authors":
                    for author in dict_item['authors']:
                        # print(author)
                        if (str(author['ids']) in author_dict.keys()) == True:
                            author_dict[str(author['ids'])]["citation_num"] += len(dict_item['inCitations'])
                            author_dict[str(author['ids'])]["publish_num"] += 1

                        else:
                            item = {"author_id" : list(author['ids']),"publish_num" : 1,"name":author['name'],'org':"","citation_num":len(dict_item['inCitations'])}
                            author_dict[str(author['ids'])] = item
                elif label == "journal":
                    venue,journalName,journalVolume,journalPages = dict_item['venue'],dict_item['journalName'],dict_item['journalVolume'],dict_item['journalPages']
                    journalName = str(journalName).strip()
                    if journalName != "":
                        if journalName not in journal_list:
                            item = {"name":journalName,"volume":journalVolume,"pages":journalPages,'id':len(journal_list),'venue':venue}
                            item['publish_num'],item['citation_num'] = 1,len(dict_item['inCitations'])

                            item['authors'] = []
                            for author in dict_item['authors']:
                                item['authors'].append(str(author['ids']))
                            item['authors_num'] = len(item['authors'])  #
                            journal_list.append(journalName)
                            journals[journalName] = item
                        else:
                            journals[journalName]['publish_num'], journals[journalName]['citation_num'] = journals[journalName]['publish_num']+1, journals[journalName]['citation_num']+len(dict_item['inCitations'])
                            for author in dict_item['authors']:
                                if (str(author['ids']) not in journals[journalName]["authors"] ) :
                                    journals[journalName]['authors'].append(str(author['ids']))
                            journals[journalName]['authors_num'] = len(journals[journalName]['authors'])  # 投稿人数

                elif label == "inCitations":
                    item = {"id":dict_item["id"],"inCitations":dict_item["inCitations"],"inCitationsNum":len(dict_item["inCitations"])}
                    # in_citation_dict[dict_item["id"]] = item
                    in_citation_list.append(item)
                i += 1
            file_num += 1

        if label == "inCitations":
            print(file_num,len(in_citation_dict.keys()))
            write_inCitations_list()
        elif label == "authors":
            print(file_num,len(author_dict))
        elif label == "journal":
            print(file_num, len(journal_list))

def make_author_list():
    print(len(list(author_dict.keys())))
    for key in list(author_dict.keys()):
        author_list.append(author_dict.pop(key))
def write_author_list():
    with codecs.open(data_dir+"authors.txt","w") as f:
        for author in author_list:
            f.write(json.dumps(author)+"\n")
    author_list.clear()
def make_journal_out_list():
    print(len(list(journals.keys())))
    for key in list(journals.keys()):
        journal_out_list.append(journals.pop(key))

def write_journal_out_list():
    with codecs.open(data_dir+"journal.txt","w") as f:
        for journal in journal_out_list:
            del journal['authors']
            f.write(json.dumps(journal)+"\n")
    journal_out_list.clear()
def write_inCitations_list():
    with codecs.open(data_dir+"inCitations.txt","a") as f:
        for item in in_citation_list:
            f.write(json.dumps(item)+"\n")
    in_citation_list.clear()
proc_file_list(file_list,"authors")
make_author_list()
write_author_list()

proc_file_list(file_list,"journal")
make_journal_out_list()
write_journal_out_list()

proc_file_list(file_list,"inCitations")


from multiprocessing import Pool
from w2_item_url import get_itemurl,item_links4
from w2_channel_extract import category,caturl

# db_urls = [item['url'] for item in item_links3.find()]
# index_urls = [item['url'] for item in detail1.find()]
# x = db_urls
# y = index_urls
# rest_of_urls = x-y

# print (rest_of_urls)

caturl.remove('http://bj.ganji.com/shoujihaoma/')
category.remove('手机号码')

def get_all_links(url):
     get_itemurl(url,url.split('/')[-2])

if __name__ == '__main__':

  for url in caturl:
      get_all_links(url)


from tkinter import N

l = ['a', 'b', 'c', 'd', 'e']

print(l[1:4])

dict = {'plat': 'vscode', 'lang': 'python', 'author': 'xinmu'}

print(dict['lang'])
del dict['plat']
print(0)

class Student:
    def __init__(self, age, name, number):
        self.age = age
        self.name = name
        self.number = number

    def test(self):
        print("student")

class BoyStudnet(Student):
    ...

class GirlStudnet(Student):
    def __init__(self, age, name, number):
        super(GirlStudnet, self).__init__(age, name, number)

    def test(self):
        print(3)

boy = BoyStudnet(10, "xinmu", 1)
print(boy.name)
boy.test()


from itertools import product

import numpy as np
import pytest

from pandas import DataFrame, Series
import pandas._testing as tm
from pandas.core.base import DataError

# gh-12373 : rolling functions error on float32 data
# make sure rolling functions works for different dtypes
#
# NOTE that these are yielded tests and so _create_data
# is explicitly called.
#
# further note that we are only checking rolling for fully dtype
# compliance (though both expanding and ewm inherit)

class Dtype:
    window = 2

    funcs = {
        "count": lambda v: v.count(),
        "max": lambda v: v.max(),
        "min": lambda v: v.min(),
        "sum": lambda v: v.sum(),
        "mean": lambda v: v.mean(),
        "std": lambda v: v.std(),
        "var": lambda v: v.var(),
        "median": lambda v: v.median(),
    }

    def get_expects(self):
        expects = {
            "sr1": {
                "count": Series([1, 2, 2, 2, 2], dtype="float64"),
                "max": Series([np.nan, 1, 2, 3, 4], dtype="float64"),
                "min": Series([np.nan, 0, 1, 2, 3], dtype="float64"),
                "sum": Series([np.nan, 1, 3, 5, 7], dtype="float64"),
                "mean": Series([np.nan, 0.5, 1.5, 2.5, 3.5], dtype="float64"),
                "std": Series([np.nan] + [np.sqrt(0.5)] * 4, dtype="float64"),
                "var": Series([np.nan, 0.5, 0.5, 0.5, 0.5], dtype="float64"),
                "median": Series([np.nan, 0.5, 1.5, 2.5, 3.5], dtype="float64"),
            },
            "sr2": {
                "count": Series([1, 2, 2, 2, 2], dtype="float64"),
                "max": Series([np.nan, 10, 8, 6, 4], dtype="float64"),
                "min": Series([np.nan, 8, 6, 4, 2], dtype="float64"),
                "sum": Series([np.nan, 18, 14, 10, 6], dtype="float64"),
                "mean": Series([np.nan, 9, 7, 5, 3], dtype="float64"),
                "std": Series([np.nan] + [np.sqrt(2)] * 4, dtype="float64"),
                "var": Series([np.nan, 2, 2, 2, 2], dtype="float64"),
                "median": Series([np.nan, 9, 7, 5, 3], dtype="float64"),
            },
            "sr3": {
                "count": Series([1, 2, 2, 1, 1], dtype="float64"),
                "max": Series([np.nan, 1, 2, np.nan, np.nan], dtype="float64"),
                "min": Series([np.nan, 0, 1, np.nan, np.nan], dtype="float64"),
                "sum": Series([np.nan, 1, 3, np.nan, np.nan], dtype="float64"),
                "mean": Series([np.nan, 0.5, 1.5, np.nan, np.nan], dtype="float64"),
                "std": Series(
                    [np.nan] + [np.sqrt(0.5)] * 2 + [np.nan] * 2, dtype="float64"
                ),
                "var": Series([np.nan, 0.5, 0.5, np.nan, np.nan], dtype="float64"),
                "median": Series([np.nan, 0.5, 1.5, np.nan, np.nan], dtype="float64"),
            },
            "df": {
                "count": DataFrame(
                    {0: Series([1, 2, 2, 2, 2]), 1: Series([1, 2, 2, 2, 2])},
                    dtype="float64",
                ),
                "max": DataFrame(
                    {0: Series([np.nan, 2, 4, 6, 8]), 1: Series([np.nan, 3, 5, 7, 9])},
                    dtype="float64",
                ),
                "min": DataFrame(
                    {0: Series([np.nan, 0, 2, 4, 6]), 1: Series([np.nan, 1, 3, 5, 7])},
                    dtype="float64",
                ),
                "sum": DataFrame(
                    {
                        0: Series([np.nan, 2, 6, 10, 14]),
                        1: Series([np.nan, 4, 8, 12, 16]),
                    },
                    dtype="float64",
                ),
                "mean": DataFrame(
                    {0: Series([np.nan, 1, 3, 5, 7]), 1: Series([np.nan, 2, 4, 6, 8])},
                    dtype="float64",
                ),
                "std": DataFrame(
                    {
                        0: Series([np.nan] + [np.sqrt(2)] * 4),
                        1: Series([np.nan] + [np.sqrt(2)] * 4),
                    },
                    dtype="float64",
                ),
                "var": DataFrame(
                    {0: Series([np.nan, 2, 2, 2, 2]), 1: Series([np.nan, 2, 2, 2, 2])},
                    dtype="float64",
                ),
                "median": DataFrame(
                    {0: Series([np.nan, 1, 3, 5, 7]), 1: Series([np.nan, 2, 4, 6, 8])},
                    dtype="float64",
                ),
            },
        }
        return expects

    def _create_dtype_data(self, dtype):
        sr1 = Series(np.arange(5), dtype=dtype)
        sr2 = Series(np.arange(10, 0, -2), dtype=dtype)
        sr3 = sr1.copy()
        sr3[3] = np.NaN
        df = DataFrame(np.arange(10).reshape((5, 2)), dtype=dtype)

        data = {"sr1": sr1, "sr2": sr2, "sr3": sr3, "df": df}

        return data

    def _create_data(self):
        self.data = self._create_dtype_data(self.dtype)
        self.expects = self.get_expects()

    def test_dtypes(self):
        self._create_data()
        for f_name, d_name in product(self.funcs.keys(), self.data.keys()):

            f = self.funcs[f_name]
            d = self.data[d_name]
            exp = self.expects[d_name][f_name]
            self.check_dtypes(f, f_name, d, d_name, exp)

    def check_dtypes(self, f, f_name, d, d_name, exp):
        roll = d.rolling(window=self.window)
        result = f(roll)
        tm.assert_almost_equal(result, exp)

class TestDtype_object(Dtype):
    dtype = object

class Dtype_integer(Dtype):
    pass

class TestDtype_int8(Dtype_integer):
    dtype = np.int8

class TestDtype_int16(Dtype_integer):
    dtype = np.int16

class TestDtype_int32(Dtype_integer):
    dtype = np.int32

class TestDtype_int64(Dtype_integer):
    dtype = np.int64

class Dtype_uinteger(Dtype):
    pass

class TestDtype_uint8(Dtype_uinteger):
    dtype = np.uint8

class TestDtype_uint16(Dtype_uinteger):
    dtype = np.uint16

class TestDtype_uint32(Dtype_uinteger):
    dtype = np.uint32

class TestDtype_uint64(Dtype_uinteger):
    dtype = np.uint64

class Dtype_float(Dtype):
    pass

class TestDtype_float16(Dtype_float):
    dtype = np.float16

class TestDtype_float32(Dtype_float):
    dtype = np.float32

class TestDtype_float64(Dtype_float):
    dtype = np.float64

class TestDtype_category(Dtype):
    dtype = "category"
    include_df = False

    def _create_dtype_data(self, dtype):
        sr1 = Series(range(5), dtype=dtype)
        sr2 = Series(range(10, 0, -2), dtype=dtype)

        data = {"sr1": sr1, "sr2": sr2}

        return data

class DatetimeLike(Dtype):
    def check_dtypes(self, f, f_name, d, d_name, exp):

        roll = d.rolling(window=self.window)
        if f_name == "count":
            result = f(roll)
            tm.assert_almost_equal(result, exp)

        else:
            with pytest.raises(DataError):
                f(roll)

class TestDtype_timedelta(DatetimeLike):
    dtype = np.dtype("m8[ns]")

class TestDtype_datetime(DatetimeLike):
    dtype = np.dtype("M8[ns]")

class TestDtype_datetime64UTC(DatetimeLike):
    dtype = "datetime64[ns, UTC]"

    def _create_data(self):
        pytest.skip(
            "direct creation of extension dtype "
            "datetime64[ns, UTC] is not supported ATM"
        )


#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""Load data from MongoDB.
   Note that, the ReadModelData class is not picklable,
     since MongoClient returns thread.lock objects.
    @author   : Liangjun Zhu
    @changelog: 18-01-02  - lj - separated from plot_timeseries.\n
                18-02-09  - lj - compatible with Python3.\n
"""
from __future__ import absolute_import

import os
import sys
from collections import OrderedDict

from pygeoc.utils import StringClass, text_type

if os.path.abspath(os.path.join(sys.path[0], '..')) not in sys.path:
    sys.path.insert(0, os.path.abspath(os.path.join(sys.path[0], '..')))

from preprocess.db_mongodb import ConnectMongoDB, MongoQuery
from preprocess.text import DBTableNames, ModelCfgFields, FieldNames, SubbsnStatsName, \
    DataValueFields, DataType, StationFields

class ReadModelData(object):
    def __init__(self, host, port, dbname):
        """Initialization."""
        client = ConnectMongoDB(host, port)
        conn = client.get_conn()
        self.maindb = conn[dbname]
        self.filein_tab = self.maindb[DBTableNames.main_filein]
        self._climdb_name = self.HydroClimateDBName
        self.climatedb = conn[self._climdb_name]
        self._mode = ''
        self._interval = -1
        # UTCTIME
        self._stime = None
        self._etime = None
        self._outletid = -1

    @property
    def HydroClimateDBName(self):
        climtbl = self.maindb[DBTableNames.main_sitelist]
        allitems = climtbl.find()
        if not allitems.count():
            raise RuntimeError('%s Collection is not existed or empty!' %
                               DBTableNames.main_sitelist)
        for item in allitems:
            if FieldNames.db in item:
                self._climdb_name = item.get(FieldNames.db)
                break
        return self._climdb_name

    @property
    def Mode(self):
        """Get simulation mode."""
        if self._mode != '':
            return self._mode.upper()
        mode_dict = self.filein_tab.find_one({ModelCfgFields.tag: FieldNames.mode})
        self._mode = mode_dict[ModelCfgFields.value]
        if isinstance(self._mode, text_type):
            self._mode = str(self._mode)
        return self._mode.upper()

    @property
    def Interval(self):
        if self._interval > 0:
            return self._interval
        findinterval = self.filein_tab.find_one({ModelCfgFields.tag: ModelCfgFields.interval})
        self._interval = int(findinterval[ModelCfgFields.value])
        return self._interval

    @property
    def OutletID(self):
        if self._outletid > 0:
            return self._outletid
        self._outletid = int(MongoQuery.get_init_parameter_value(self.maindb,
                                                                 SubbsnStatsName.outlet))
        return self._outletid

    @property
    def SimulationPeriod(self):
        if self._stime is not None and self._etime is not None:
            return self._stime, self._etime
        st = self.filein_tab.find_one({ModelCfgFields.tag: ModelCfgFields.stime})[
            ModelCfgFields.value]
        et = self.filein_tab.find_one({ModelCfgFields.tag: ModelCfgFields.etime})[
            ModelCfgFields.value]
        st = StringClass.get_datetime(st)
        et = StringClass.get_datetime(et)
        if self._stime is None or st > self._stime:
            self._stime = st
        if self._etime is None or et < self._etime:
            self._etime = et
        if st > self._etime > self._stime:
            self._stime = st
            self._etime = et
        return self._stime, self._etime

    def Precipitation(self, subbsn_id, start_time, end_time):
        """
        The precipitation is read according to the subbasin ID.
            Especially when plot a specific subbasin (such as ID 3).
            For the whole basin, the subbasin ID is 0.
        Returns:
            Precipitation data list with the first element as datetime.
            [[Datetime1, value1], [Datetime2, value2], ..., [Datetimen, valuen]]
        """
        pcp_date_value = list()
        sitelist_tab = self.maindb[DBTableNames.main_sitelist]
        findsites = sitelist_tab.find_one({FieldNames.subbasin_id: subbsn_id,
                                           FieldNames.mode: self.Mode})
        if findsites is not None:
            site_liststr = findsites[FieldNames.site_p]
        else:
            raise RuntimeError('Cannot find precipitation site for subbasin %d.' % subbsn_id)
        site_list = StringClass.extract_numeric_values_from_string(site_liststr)
        site_list = [int(v) for v in site_list]
        if len(site_list) == 0:
            raise RuntimeError('Cannot find precipitation site for subbasin %d.' % subbsn_id)

        pcp_dict = OrderedDict()

        for pdata in self.climatedb[DBTableNames.data_values].find(
                {DataValueFields.utc: {"$gte": start_time, '$lte': end_time},
                 DataValueFields.type: DataType.p,
                 DataValueFields.id: {"$in": site_list}}).sort([(DataValueFields.utc, 1)]):
            curt = pdata[DataValueFields.utc]
            curv = pdata[DataValueFields.value]
            if curt not in pcp_dict:
                pcp_dict[curt] = 0.
            pcp_dict[curt] += curv
        # average
        if len(site_list) > 1:
            for t in pcp_dict:
                pcp_dict[t] /= len(site_list)
        for t, v in pcp_dict.items():
            # print(str(t), v)
            pcp_date_value.append([t, v])
        print('Read precipitation from %s to %s done.' % (start_time.strftime('%c'),
                                                          end_time.strftime('%c')))
        return pcp_date_value

    def Observation(self, subbsn_id, vars, start_time, end_time):
        """Read observation data of given variables.

        Changelog:
          - 1. 2018-8-29 Use None when the observation of one variables is absent.

        Returns:
            1. Observed variable names, [var1, var2, ...]
            2. Observed data dict of selected plotted variables, with UTCDATETIME.
               {Datetime: [value_of_var1, value_of_var2, ...], ...}
        """
        vars_existed = list()
        data_dict = OrderedDict()

        coll_list = self.climatedb.collection_names()
        if DBTableNames.observes not in coll_list:
            return None, None
        isoutlet = 0
        if subbsn_id == self.OutletID:
            isoutlet = 1

        def get_observed_name(name):
            """To avoid the prefix of subbasin number."""
            if '_' in name:
                name = name.split('_')[1]
            return name

        def get_basename(name):
            """Get base variable name, e.g., SED for SED and SEDConc."""
            name = get_observed_name(name)
            if 'Conc' in name:
                name = name.split('Conc')[0]
            return name

        siteTbl = self.climatedb[DBTableNames.sites]
        obsTbl = self.climatedb[DBTableNames.observes]
        for i, param_name in enumerate(vars):
            site_items = siteTbl.find_one({StationFields.type: get_basename(param_name),
                                           StationFields.outlet: isoutlet,
                                           StationFields.subbsn: subbsn_id})

            if site_items is None:
                continue
            site_id = site_items.get(StationFields.id)
            for obs in obsTbl.find({DataValueFields.utc: {"$gte": start_time, '$lte': end_time},
                                    DataValueFields.type: get_observed_name(param_name),
                                    DataValueFields.id: site_id}).sort([(DataValueFields.utc, 1)]):

                if param_name not in vars_existed:
                    vars_existed.append(param_name)
                curt = obs[DataValueFields.utc]
                curv = obs[DataValueFields.value]
                if curt not in data_dict:
                    data_dict[curt] = [None] * len(vars)
                data_dict[curt][i] = curv
        if not vars_existed:
            return None, None
        # remove the redundant None in data_dict, in case of len(vars_existed) != len(vars)
        for i, vname in enumerate(vars):
            if vname in vars_existed:
                continue
            for dt, adata in list(data_dict.items()):
                del adata[i]

        print('Read observation data of %s from %s to %s done.' % (','.join(vars_existed),
                                                                   start_time.strftime('%c'),
                                                                   end_time.strftime('%c')))
        return vars_existed, data_dict

def main():
    """Functional tests."""
    import datetime

    host = '192.168.253.203'
    port = 27018
    dbname = 'youwuzhen10m_longterm_model'
    stime = datetime.datetime(2013, 1, 1, 0, 0)
    etime = datetime.datetime(2013, 12, 31, 0, 0)
    rd = ReadModelData(host, port, dbname)
    print(rd.HydroClimateDBName)
    print(rd.Precipitation(4, stime, etime))
    print(rd.Observation(4, ['Q'], stime, etime))

if __name__ == "__main__":
    main()


"""
Code that goes along with the Airflow located at:
http://airflow.readthedocs.org/en/latest/tutorial.html
"""
from datetime import datetime, timedelta

from airflow import DAG
from airflow.operators import BashOperator, PythonOperator

from src.example_module import example_function

default_args = {
    "owner": "airflow",
    "depends_on_past": False,
    "start_date": datetime.now(),
    "email_on_failure": False,
    "email_on_retry": False,
    "retries": 1,
    "retry_delay": timedelta(minutes=5),
    "catchup": False,
    "max_active_runs": 1
}

dag = DAG("example", default_args=default_args, schedule_interval=timedelta(1))

# t1, t2 and t3 are examples of tasks created by instantiating operators
t1 = BashOperator(task_id="print_date", bash_command="date", dag=dag)

t2 = BashOperator(task_id="sleep", bash_command="sleep 5", retries=3, dag=dag)

templated_command = """
    {% for i in range(5) %}
        echo "{{ ds }}"
        echo "{{ macros.ds_add(ds, 7)}}"
        echo "{{ params.my_param }}"
    {% endfor %}
"""

t3 = BashOperator(
    task_id="templated",
    bash_command=templated_command,
    params={"my_param": "Parameter I passed in"},
    dag=dag,
)

t4 = PythonOperator(
    task_id="python_code",
    python_callable=example_function,
    dag=dag
)

t2.set_upstream(t1)
t3.set_upstream(t1)
t4.set_upstream(t1)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat May 23 08:07:20 2020

@author: agiovann
"""
from .caiman_functions import signal_filter
import cv2
from functools import partial
import matplotlib.pyplot as plt
import numpy as np
#import scipy
#from scipy.interpolate import interp1d
from scipy import stats
from scipy.signal import argrelextrema, butter, sosfilt, sosfilt_zi

class OnlineFilter(object):
    def __init__(self, freq, fr, order=3, mode='high'):
        '''
        Object encapsulating Online filtering for spike extraction traces
        Args:
            freq: float
            cutoff frequency

        order: int
            order of the filter

        mode: str
            'high' for high-pass filtering, 'low' for low-pass filtering

        '''
        self.freq = freq
        self.fr = fr
        self.mode = mode
        self.order=order
        self.normFreq = freq / (fr / 2)
        self.filt = butter(self.order, self.normFreq, self.mode, output='sos')
        self.z_init = sosfilt_zi(self.filt)

    def fit(self, sig):
        """
        Online filter initialization and running offline

        Parameters
        ----------
        sig : ndarray
            input signal to initialize
        num_frames_buf : int
            frames to use for buffering

        Returns
        -------
        sig_filt: ndarray
            filtered signal

        """
        sig_filt = signal_filter(sig, freq=self.freq, fr=self.fr, order=self.order, mode=self.mode)
        # result_init, z_init = signal.sosfilt(b, data[:,:20000], zi=z)
        self.z_init = np.repeat(self.z_init[:,None,:], sig.shape[0], axis=1)

        #sos_all = np.zeros(sig_filt.shape)

        for i in range(0,sig.shape[-1]-1):
            _ , self.z_init = sosfilt(self.filt, np.expand_dims(sig[:,i], axis=1), zi=self.z_init)

        return sig_filt

    def fit_next(self, sig):

        sig_filt, self.z_init = sosfilt(self.filt, np.expand_dims(sig,axis=1), zi=self.z_init)
        return sig_filt.squeeze()

def rolling_window(ndarr, window_size, stride):
        """
        generates efficient rolling window for running statistics
        Args:
            ndarr: ndarray
                input pixels in format pixels x time
            window_size: int
                size of the sliding window
            stride: int
                stride of the sliding window
        Returns:
                iterator with views of the input array

        """
        for i in range(0,ndarr.shape[-1]-window_size-stride+1,stride):
            yield ndarr[:,i:np.minimum(i+window_size, ndarr.shape[-1])]

        if i+stride != ndarr.shape[-1]:
           yield ndarr[:,i+stride:]

def estimate_running_std(signal_in, win_size=20000, stride=5000,
                         idx_exclude=None, q_min=25, q_max=75):
    """
    Function to estimate ROBUST runnning std

    Args:
        win_size: int
            window used to compute running std to normalize signals when
            compensating for photobleaching

        stride: int
            corresponding stride to win_size

        idx_exclude: iterator
            indexes to exclude when computing std

        q_min: float
            lower percentile for estimation of signal variability (do not change)

        q_max: float
            higher percentile for estimation of signal variability (do not change)

    Returns:
        std_run: ndarray
            running standard deviation

    """
    if idx_exclude is not None:
        signal = signal_in[np.setdiff1d(range(len(signal_in)), idx_exclude)]
    else:
        signal = signal_in
    iter_win = rolling_window(signal[None,:],win_size,stride)
    myperc = partial(np.percentile, q=[q_min,q_max], axis=-1)
    res = np.array(list(map(myperc,iter_win))).T.squeeze()
    iqr = (res[1]-res[0])/1.35
    std_run = cv2.resize(iqr,signal_in[None,:].shape).squeeze()
    return std_run

def compute_std(peak_height):
    data = peak_height - np.median(peak_height)
    ff1 = -data * (data < 0)
    Ns = np.sum(ff1 > 0)
    std = np.sqrt(np.divide(np.sum(ff1**2), Ns))
    return std

def compute_thresh(peak_height, prev_thresh=None, delta_max=0.03, number_maxima_before=1):
    kernel = stats.gaussian_kde(peak_height)
    x_val = np.linspace(0,np.max(peak_height),1000)
    pdf = kernel(x_val)
    second_der = np.diff(pdf,2)
    mean = np.mean(peak_height)
    min_idx = argrelextrema(kernel(x_val), np.less)

    minima = x_val[min_idx]
    minima = minima[minima>mean]
    minima_2nd = argrelextrema(second_der, np.greater)
    minima_2nd = x_val[minima_2nd]

    if prev_thresh is None:
        delta_max = np.inf
        prev_thresh = mean

    thresh = prev_thresh

    if (len(minima)>0) and (np.abs(minima[0]-prev_thresh)< delta_max):
        thresh = minima[0]
        mnt = (minima_2nd-thresh)
        mnt = mnt[mnt<0]
        thresh += mnt[np.maximum(-len(mnt)+1,-number_maxima_before)]
    #else:
    #    thresh = 100

    thresh_7 = compute_std(peak_height) * 7.5

    """
    print(f'previous thresh: {prev_thresh}')
    print(f'current thresh: {thresh}')
    """
    plt.figure()
    plt.plot(x_val, pdf,'c')
    plt.plot(x_val[2:],second_der*500,'r')
    plt.plot(thresh,0, '*')
    plt.vlines(thresh_7, 0, 2, color='r')
    plt.pause(0.1)

    return thresh

def non_symm_median_filter(t, filt_window):
    m = t.copy()
    for i in range(len(t)):
        if (i > filt_window[0]) and (i < len(t) - filt_window[1]):
            m[i] = np.median(t[i - filt_window[0] : i + filt_window[1] + 1])
    return m


checkpoint_config = dict(interval=1)
# yapf:disable
log_config = dict(
    interval=50,
    hooks=[
        dict(type='CustomizedTextLoggerHook'),
        # dict(type='TensorboardLoggerHook')
    ])
# yapf:enable
custom_hooks = [dict(type='NumClassCheckHook')]

dist_params = dict(backend='nccl')
log_level = 'INFO'
load_from = None
resume_from = None
workflow = [('train', 1)]


import unittest
from unittest.mock import patch

import tweepy

from whotweeted import get_country_code, who_is_output
from whotweeted import load_cache

DATA = dict(AU='875639674244444160',
            ES='875669971954806784',
            nopb='846302762736504833',
            noloc='844092059988508673',
            badid='8756396742444441da'
            )
get_tweet = lambda x: load_cache(DATA.get(x))  # noqa E731

class WhoTweetedTestCase(unittest.TestCase):

    @patch.object(tweepy.API, 'get_status', return_value=get_tweet('AU'))
    def test_julian(self, mock_method):
        tweetid = DATA.get('AU')
        country = get_country_code(tweetid)
        who_is_out = who_is_output(country)
        self.assertEqual(country, 'AU')
        self.assertIn('Julian', who_is_out)

    @patch.object(tweepy.API, 'get_status', return_value=get_tweet('ES'))
    def test_bob(self, mock_method):
        tweetid = DATA.get('ES')
        country = get_country_code(tweetid)
        who_is_out = who_is_output(country)
        self.assertEqual(country, 'ES')
        self.assertIn('Bob', who_is_out)

    @patch.object(tweepy.API, 'get_status', return_value=get_tweet('nopb'))
    def test_no_pybites_account(self, mock_method):
        tweetid = DATA.get('nopb')
        with self.assertRaises(ValueError):
            get_country_code(tweetid)

    @patch.object(tweepy.API, 'get_status', return_value=get_tweet('noloc'))
    def test_no_location_in_tweet(self, mock_method):
        tweetid = DATA.get('noloc')
        with self.assertRaises(AttributeError):
            get_country_code(tweetid)

    # not really a return value, it crashes before decorator can cash tweet
    @patch.object(tweepy.API, 'get_status', return_value=get_tweet('nopb'))
    def test_bad_tweet_id(self, mock_method):
        tweetid = DATA.get('badid')
        print(tweetid)
        with self.assertRaises(ValueError):
            get_country_code(tweetid)

if __name__ == '__main__':
    unittest.main()


from email.MIMEMultipart import MIMEMultipart
from email.MIMEText import MIMEText
import smtplib
COMMASPACE = ', '

class AlarmEmail(object):
    def __init__(self, toAddrList=["alee@picarro.com"], fromAddr="alee@picarro.com"):
        self.smtpHostname = "woodstock.blueleaf.com"
        self.toAddrList = toAddrList
        self.fromAddr = fromAddr

    def sendMsg(self, subject="Message from CRDS", msg="This is a testing alarm message"):
        msg = MIMEText(msg)
        outer = MIMEMultipart()
        outer['Subject'] = subject
        outer['To'] = COMMASPACE.join(self.toAddrList)
        outer['From'] = self.fromAddr
        outer.attach(msg)
        mailServer = smtplib.SMTP(self.smtpHostname)
        mailServer.sendmail(self.fromAddr, self.toAddrList, outer.as_string())


# -*- coding: utf-8 -*-

"""
   程式說明請參閱14-23頁
"""

from machine import UART
from machine import Pin, I2C
import time
import ssd1306
import os

led = Pin(2, Pin.OUT, value=1)
sw = Pin(0, Pin.IN, Pin.PULL_UP)

file = None
saveFile = False
filePath = '/gps.csv'

interval = 5000
saveTime = time.ticks_ms()

i2c = I2C(scl=Pin(5), sda=Pin(4), freq=100000)
oled = ssd1306.SSD1306_I2C(128, 64, i2c)

oled.fill(0)
oled.text("GPS RUNNING...", 0, 30)
oled.show()

def callback(p):
    global saveFile, led, saveTime

    val = p.value()
    if val == 1:
        saveTime = 0
        led.value(not led.value())
        saveFile = not saveFile

sw.irq(trigger=Pin.IRQ_RISING, handler=callback)

def utcDateTime(dateStr, timeStr, timeZone=8):
    if dateStr == '' or timeStr == '':
        return None

    day = dateStr[0:2]
    month = dateStr[2:4]
    year = dateStr[4:6]
    hr = timeStr[0:2]
    min = timeStr[2:4]
    sec = timeStr[4:6]
    timeZone *= 3600

    t = time.mktime((int('20' + year), int(month), int(day),int(hr), int(min), int(sec), 0, 0))

    return time.localtime(t+timeZone)

def latitude(d, h):
    if d == '':
        return '0'

    hemi = '' if h == 'N' else '-'
    deg = int(d[0:2])
    min = str(float(d[2:]) / 60)[1:]

    return hemi + str(deg) + min

def longitude(d, h):
    if d == '':
        return '0'

    hemi = '' if h == 'E' else '-'
    deg = int(d[0:3])
    min = str(float(d[3:]) / 60)[1:]

    return hemi + str(deg) + min

def saveGPS(lat, long, today, now):
    global file, saveFile, filePath

    if saveFile:
        if file == None:
            try:
                os.stat(filePath)
                file = open(filePath, 'a')
            except OSError as e:
                if (e.args[0] == 2):
                    file = open(filePath, 'a')
                    file.write('latitude,longitude,date,time\r\n')

        file.write(lat +','+ long + ',' + today + ',' + now + '\r\n')
    else:
        if file != None:
            file.close()
            print('file closed.')
            file = None

def convertGPS(gpsStr):
    gps = gpsStr.split(b'\r\n')[0].decode('ascii').split(',')

    lat = latitude(gps[3], gps[4])  # N or S
    long = longitude(gps[5], gps[6]) # E or W
    today = ''
    now = ''
    tim = utcDateTime(gps[9], gps[1], 8)

    if tim != None:
        today = str(tim[0]) + '/' + str(tim[1]) + '/' + str(tim[2])
        now = str(tim[3]) + ':' + str(tim[4]) + ':' + str(tim[5])

    return (lat, long, today, now)

def displayGPS(lat, long, today, now):
    lat = "Lat: " + lat
    long = "Long: " + long
    oled.fill(0)
    oled.text(today, 0, 0)
    oled.text(now, 0, 10)
    oled.text(lat, 0, 20)
    oled.text(long, 0, 30)
    oled.show()

def main():
    global saveTime
    com = UART(0, 9600)
    com.init(9600)

    gpsStr = b''
    gpsReading = False

    while True:
        data = com.readline()
        if data and (gpsReading or ('$GPRMC' in data)) :
            gpsStr += data

            if '\n' in data:
                gpsReading = False
                lat, long, today, now = convertGPS(gpsStr)
                displayGPS(lat, long, today, now)

                if time.ticks_diff(time.ticks_ms(), saveTime) > interval:
                    saveGPS(lat, long, today, now)
                    saveTime = time.ticks_ms()

                gpsStr = b''
            else:
                gpsReading = True

main()


from testapp.models import Book, Author
from django.db.models import Count
from django.db import connection

def list_books():
    books = Book.objects.select_related().all()
    for book in books:
        print(f'"{book.title}".{book.author.name}')

def author_books():
    authors  = Author.objects.select_related().all()
    for author in authors:
        author_books = author.books.all()
        print(f'{author.name}: ',", ".join([f'"{book.title}"' for book in author_books]) )

def authors_rank():
    authors  = Author.objects.select_related() \
        .annotate(num_books=Count('books')) \
        .order_by('-num_books')
    for author in authors:
        author_books = author.books.all()
        print(f'{author.name}: ', f'{author.num_books}'  )


from config import db
from models import *

def read_all(id=None, sort_by=None, reverse=False, movie=None, limit=0, movie_sort=None, movie_sort_reverse=False, movie_limit=0):
    '''
    Menampilkan data Director
    :param id: menampilkan berdasarkan ID | None or int
    :param sort_by: atribut Directory yang ingin di sort | None or string
    :param reverse: tipe sort Directory, asc atau desc | None or bool
    :param movie: jenis schema data movie | None or 'half' or 'full'
    :param limit: limit data Directory yang ingin diambil | int
    :param movie_sort: atribut Movie yang ingin di sort | None or string
    :param movie_sort_reverse: tipe sort Movie, asc atau desc | None or bool
    :param movie_limit: limit data Movie yang ingin diambil | int

    :return: data Directory atau data Directory beserta Movie | object
    '''
    is_many = False

    directors = None
    if id == None:
        directors = Directors().query.all()
        is_many = True
    else:
        directors = (
            Directors.query.filter(Directors.id == id)
            .outerjoin(Movies)
            .one_or_none()
        )
        is_many = False

        if directors is None:
            return Result(
                status=False,
                message=f"Director dengan ID {id} tidak ditemukan"
            ).__dict__, 404

    directors_schame = None

    if movie != None:
        if validatorParamMovieDirector(movie) == True:
            if movie == 'full':
                directors_schame = DirectorsMoviesDetailSchame(many=is_many)
            elif movie == 'half':
                directors_schame = DirectorsMoviesSchame(many=is_many)
        else:
            return Result(
                status=False,
                message=f"Parameter movie tidak valid"
            ).__dict__, 400
    else:
        directors_schame = DirectorsSchema(many=is_many)

    data = directors_schame.dump(directors)

    if is_many:
        if sort_by != None:
            if validatorParamSortDirectorFull(sort_by) == True:
                data.sort(key=lambda x: x[sort_by], reverse=reverse)
            else:
                return Result(
                    status=False,
                    message=f"Parameter sort tidak valid"
                ).__dict__, 400

    if movie != None:
        if movie_sort != None:
            is_param_movie_valid = False
            if movie == 'half':
                is_param_movie_valid = validatorParamSortMovieHalf(movie_sort)
            elif movie == 'full':
                is_param_movie_valid = validatorParamSortMovieFull(movie_sort)

            if is_param_movie_valid == True:
                if is_many:
                    i = 0
                    while i < len(data):
                        sorted_movie = data[i].get('movies')
                        sorted_movie.sort(
                            key=lambda x: x[movie_sort], reverse=movie_sort_reverse)

                        if movie_limit > 0:
                            if len(sorted_movie) >= movie_limit:
                                sorted_movie = sorted_movie[:movie_limit]

                        data[i]['movies'] = sorted_movie
                        i += 1
                else:
                    sorted_movie = data.get('movies')
                    sorted_movie.sort(
                        key=lambda x: x[movie_sort], reverse=movie_sort_reverse)

                    if movie_limit > 0:
                        if len(sorted_movie) >= movie_limit:
                            sorted_movie = sorted_movie[:movie_limit]

                    data['movies'] = sorted_movie
            else:
                return Result(
                    status=False,
                    message=f"Parameter sort tidak valid"
                ).__dict__, 400

    if is_many:
        if limit > 0:
            if len(data) >= limit:
                data = data[:limit]

    return Result(
        status=True,
        message="Success",
        data=data
    ).__dict__, 200

def read_one(director_id, movie=None):
    '''
    Menampilkan data Director
    :param director_id: menampilkan berdasarkan ID | int
    :param movie: jenis schema data movie | None or 'half' or 'full'

    :return: data Directory atau data Directory beserta Movie | object
    '''

    director = (
        Directors.query.filter(Directors.id == director_id)
        .outerjoin(Movies)
        .one_or_none()
    )

    if director is not None:
        directors_schame = None

        if movie != None:
            if validatorParamMovieDirector(movie) == True:
                if movie == 'full':
                    directors_schame = DirectorsMoviesDetailSchame()
                elif movie == 'half':
                    directors_schame = DirectorsMoviesSchame()
            else:
                return Result(
                    status=False,
                    message=f"Parameter movie tidak valid"
                ).__dict__, 400
        else:
            directors_schame = DirectorsSchema()

        data = directors_schame.dump(director)
        return Result(
            status=True,
            message="Success",
            data=data
        ).__dict__, 200

    else:
        return Result(
            status=False,
            message=f"Director dengan ID {director_id} tidak ditemukan"
        ).__dict__, 404

def search(name, movie=None):
    '''
    Mencari data Director berdasarkan nama
    :param name: sebagai parameter pencari | string
    :param movie: jenis schema data movie | None or 'half' or 'full'

    :return: data Directory | object
    '''

    if len(name) <= 3:
        return Result(
            status=False,
            message=f"Parameter name tidak valid"
        ).__dict__, 400

    directors = (
        Directors.query.filter(Directors.name.like(f"%{name}%")).all()
    )

    if directors is None:
        return Result(
            status=True,
            message=f"Director dengan nama {name} tidak ditemukan"
        ).__dict__, 200
    else:
        if movie != None:
            if validatorParamMovieDirector(movie) == True:
                if movie == 'full':
                    directors_schame = DirectorsMoviesDetailSchame(many=True)
                elif movie == 'half':
                    directors_schame = DirectorsMoviesSchame(many=True)
            else:
                return Result(
                    status=False,
                    message=f"Parameter movie tidak valid"
                ).__dict__, 400
        else:
            directors_schame = DirectorsSchema(many=True)

        data = directors_schame.dump(directors)

        return Result(
            status=True,
            message="Success",
            data=data
        ).__dict__, 200

def create(director):
    '''
    Menyimpan data Directory
    :param director: data Directory yang ingin di disimpan | object

    :return: data Directory yang berhasil disimpan | object
    '''

    validator = ValidationDirector(director=director).isValid()

    if validator.status != True:
        return Result(
            status=False,
            message=validator.message
        ).__dict__, 400

    uid = director.get("uid")
    existring_director = (
        Directors.query.filter(Directors.uid == uid)
        .one_or_none()
    )

    if existring_director is None:
        schema = DirectorsSchema()
        new_director = schema.load(director, session=db.session)
        db.session.add(new_director)
        db.session.commit()

        data = schema.dump(new_director)
        return Result(
            status=True,
            message=f"Berhasil menambahkan Director baru",
            data=data
        ).__dict__, 201

    else:
        return Result(
            status=False,
            message=f"Director dengan User ID {uid} sudah ada"
        ).__dict__, 409

def update(director_id, director):
    '''
    Mengubah data Directory
    :param director_id: ID director yang ingin diubah | int
    :param director: data Directory yang ingin di diubah | object

    :return: data Directory yang berhasil diubah | object
    '''

    validator = ValidationDirector(director=director).isValid()

    if validator.status != True:
        return Result(
            status=False,
            message=validator.message
        ).__dict__, 400

    update_director = (
        Directors.query.filter(Directors.id == director_id)
        .one_or_none()
    )

    if update_director is None:
        return Result(
            status=False,
            message=f"Director dengan ID {director_id} tidak ditemukan"
        ).__dict__, 404

    else:
        uid = director.get("uid")

        # Cek apakah UID juga diedit
        if uid != update_director.uid:
            existing_uid = (
                Directors.query.filter(Directors.uid == uid)
                .one_or_none()
            )

            # Cek apakah UID yg diganti sudah terpakai atau belum
            if existing_uid is not None:
                return Result(
                    status=False,
                    message=f"Director dengan User ID {uid} sudah ada"
                ).__dict__, 409

        schema = DirectorsSchema()
        update = schema.load(director, session=db.session)
        update.id = update_director.id
        db.session.merge(update)
        db.session.commit()

        data = schema.dump(update_director)

        return Result(
            status=True,
            message=f"Berhasil mengubah Director",
            data=data
        ).__dict__, 200

def delete(director_id):
    '''
    Menghapus data Directory
    :param director_id: ID director yang ingin dihapus | int

    :return: pesan bahwa data berhasil dihapus | string
    '''

    director = (
        Directors.query.filter(Directors.id == director_id)
        .one_or_none()
    )

    if director is not None:
        db.session.delete(director)
        db.session.commit()

        return Result(
            status=True,
            message="Berhasil menghapus Director",
        ).__dict__, 200

    else:
        return Result(
            status=False,
            message=f"Director dengan ID {director_id} tidak ditemukan"
        ).__dict__, 404