import sys
# import network
import uasyncio as asyncio
from machine import Pin

class Stepper:
    sequence = [[1, 0, 0, 0],
                [1, 1, 0, 0],
                [0, 1, 0, 0],
                [0, 1, 1, 0],
                [0, 0, 1, 0],
                [0, 0, 1, 1],
                [0, 0, 0, 1],
                [1, 0, 0, 1]
                ]

    def __init__(self, p1, p2, p3, p4):
        self.state = 0
        self.step_angle = 375  # deg * 100
        self.min_delay = 8  # ms
        self.delay = 8 # ms
        self.pins = [Pin(p1, Pin.OUT),
                     Pin(p2, Pin.OUT),
                     Pin(p3, Pin.OUT),
                     Pin(p4, Pin.OUT)]

    def idle(self):
        for pin in self.pins:
            pin.off()

    def speed(self, s):
        self.delay = s * 1000 // self.step_angle

    def angle_to_steps(self, angle):
        return (angle * 100) // self.step_angle

    def step(self, reverse=False):
        increment = - 1 if reverse else 1
        self.state = (self.state + increment) % len(__class__.sequence)
        self.set_state(self.state)

    def set_state(self, state_number):
        for pin, state in zip(self.pins, __class__.sequence[state_number]):
            pin.value(state)

    async def turn_steps(self, steps):
        reverse = False
        if steps < 0:
            reverse = True
            steps = -steps
        for _ in range(steps):
            self.step(reverse)
            await asyncio.sleep_ms(self.delay)

    async def turn_angle(self, angle):
        await self.turn_steps(self.angle_to_steps(angle))

class Cylinder:
    def __init__(self, motor):
        self.current = 0
        self.one_step_angle = 36
        self.motor = motor
        self.ratio = 25  # ratio * 10

    async def show(self, number):
        diff1 = (number - self.current) % 10
        diff2 = (self.current - number) % 10

        diff = diff1
        if diff2 < diff1:
            diff = -diff2
        angle = diff * self.one_step_angle
        self.current = number
        await self.motor.turn_angle(angle * self.ratio // 10 * -1)

    def idle(self):
        self.motor.idle()

    def reset_state(self):
        self.current = 0

class Display:
    def __init__(self,  digit0, digit1, digit2):
        self.digit0 = Cylinder(digit0)
        self.digit1 = Cylinder(digit1)
        self.digit2 = Cylinder(digit2)
        self.state_file = "state.txt"

    async def show(self, number):
        await asyncio.gather(
            self.digit2.show((number // 100) % 10),
            self.digit1.show((number // 10) % 10),
            self.digit0.show(number % 10)
        )

    async def showf(self, number, decimal):
        await self.show(int(number * 10 ** decimal))

    async def countdown(self, number):
        await self.show(number)
        await asyncio.sleep_ms(1000)

        for i in range(number):
            await self.show(number - i - 1)
            await asyncio.sleep_ms(1000)

    def load_state(self):
        with open(self.state_file, "r") as file:
            state = file.readline().split(",")
            self.digit2.current = int(state[0])
            self.digit2.motor.state = int(state[1])
            self.digit1.current = int(state[2])
            self.digit1.motor.state = int(state[3])
            self.digit0.current = int(state[4])
            self.digit0.motor.state = int(state[5])

    def save_state(self):
        with open(self.state_file, "w") as file:
            file.write(f"{self.digit2.current},{self.digit2.motor.state}, {self.digit1.current},{self.digit1.motor.state},{self.digit0.current},{self.digit0.motor.state}")

    def reset_state(self):
        self.digit1.reset_state()
        self.digit0.reset_state()
        self.save_state()

    def idle(self):
        self.digit1.idle()
        self.digit0.idle()

    async def start(self):
        """
        Control interface commands

        SHOW (int)          - displays integer, truncates from left
        SHOWF (float) (n)   - displays float with (n) digits after floating point, truncates from left
        CONFIGURE           - start the configuration mode
        """
        reader = asyncio.StreamReader(sys.stdin)
        while True:
            command_raw = await reader.readline()
            #print(command_raw)
            command_parts = command_raw.decode().split()

            try:
                command = command_parts[0].upper()
            except IndexError:
                continue

            #print(command_parts)

            if command == "SHOW":
                try:
                    await self.show(int(command_parts[1]))
                except:
                    print("Value is not a number")
            elif command == "SHOWF":
                try:
                    await self.showf(float(command_parts[1]), int(command_parts[2]))
                except:
                    print("Value is not a number")
            elif command == "COUNTDOWN":
                try:
                    print(f"Countdown from {int(command_parts[1])}")
                    await self.countdown(int(command_parts[1]))
                except:
                    print("Value is not a number")
            elif command == "INFO":
                print(self)
            elif command == "SAVE":
                self.save_state()
                print("State saved")
            elif command == "SET":
                self.digit0.current = int(command_parts[1]) % 10
                self.digit1.current = (int(command_parts[1]) // 10) % 10
                self.digit2.current = (int(command_parts[1]) // 100) % 10
                print(self)
            elif command == "CONFIGURE":
                print("CONFIGURATION started")
                while True:
                    command = await reader.read(1)
                    #print(type(command), command)
                    if command == "1":
                        self.digit0.motor.step()
                    elif command == "2":
                        self.digit0.motor.step(reverse=True)
                    elif command == "3":
                        self.digit1.motor.step()
                    elif command == "4":
                        self.digit1.motor.step(reverse=True)
                    elif command == "5":
                        self.digit2.motor.step()
                    elif command == "6":
                        self.digit2.motor.step(reverse=True)
                    elif command == "s":
                        self.reset_state()
                        self.save_state()
                        print("Display configuration success")
                        break
                    await asyncio.sleep_ms(30)

    def __repr__(self):
        r = "Display\n"
        r += f"\tDisplayed number: {self.digit2.current}{self.digit1.current}{self.digit0.current}\n"
        r += f"\tMotor states: {self.digit2.motor.state}, {self.digit1.motor.state}, {self.digit0.motor.state}\n"
        return r

# async def connect(ssid, psk):
#     #Connect to WLAN
#     wlan = network.WLAN(network.STA_IF)
#     wlan.active(True)
#     wlan.connect(ssid, psk)
#     for _ in range(10):
#         if wlan.isconnected():
#             break
#         print('Waiting for connection...')
#         asyncio.sleep_ms(3000)
#     else:
#         print("Cannot connect to network. Resetting...")
#         machine.reset()

#     ip = wlan.ifconfig()[0]
#     print(f'Connected to {ssid} on {ip}')
#     return ip

async def main():
    try:
        display.load_state()
    except Exception as e:
        print(e)
    print(display)

    await display.start()

digit0 = Stepper(0, 1, 2, 3)
digit0.delay = 16
digit1 = Stepper(4, 5, 6, 7)
digit1.delay = 16
digit2 = Stepper(8, 9, 10, 11)
digit2.delay = 16
display = Display(digit0, digit1, digit2)

try:
    asyncio.run(main())
except:
    print("Exception occured. Saving state and exiting.")
    display.save_state()
    raise


import contextlib
import glob
import json
import nodetool
import os
import pytest
import subprocess
import tempfile
import random
import util

# To run the Scylla tools, we need to run Scylla executable itself, so we
# need to find the path of the executable that was used to run Scylla for
# this test. We do this by trying to find a local process which is listening
# to the address and port to which our our CQL connection is connected.
# If such a process exists, we verify that it is Scylla, and return the
# executable's path. If we can't find the Scylla executable we use
# pytest.skip() to skip tests relying on this executable.
@pytest.fixture(scope="module")
def scylla_path(cql):
    pid = util.local_process_id(cql)
    if not pid:
        pytest.skip("Can't find local Scylla process")
    # Now that we know the process id, use /proc to find the executable.
    try:
        path = os.readlink(f'/proc/{pid}/exe')
    except:
        pytest.skip("Can't find local Scylla executable")
    # Confirm that this executable is a real tool-providing Scylla by trying
    # to run it with the "--list-tools" option
    try:
        subprocess.check_output([path, '--list-tools'])
    except:
        pytest.skip("Local server isn't Scylla")
    return path

# A fixture for finding Scylla's data directory. We get it using the CQL
# interface to Scylla's configuration. Note that if the server is remote,
# the directory retrieved this way may be irrelevant, whether or not it
# exists on the local machine... However, if the same test that uses this
# fixture also uses the scylla_path fixture, the test will anyway be skipped
# if the running Scylla is not on the local machine local.
@pytest.fixture(scope="module")
def scylla_data_dir(cql):
    try:
        dir = json.loads(cql.execute("SELECT value FROM system.config WHERE name = 'data_file_directories'").one().value)[0]
        return dir
    except:
        pytest.skip("Can't find Scylla sstable directory")

def simple_no_clustering_table(cql, keyspace):
    table = util.unique_name()
    schema = f"CREATE TABLE {keyspace}.{table} (pk int PRIMARY KEY, v int) WITH compaction = {{'class': 'NullCompactionStrategy'}}"

    cql.execute(schema)

    for pk in range(0, 10):
        x = random.randrange(0, 4)
        if x == 0:
            # partition tombstone
            cql.execute(f"DELETE FROM {keyspace}.{table} WHERE pk = {pk}")
        else:
            # live row
            cql.execute(f"INSERT INTO {keyspace}.{table} (pk, v) VALUES ({pk}, 0)")

        if pk == 5:
            nodetool.flush(cql, f"{keyspace}.{table}")

    nodetool.flush(cql, f"{keyspace}.{table}")

    return table, schema

def simple_clustering_table(cql, keyspace):
    table = util.unique_name()
    schema = f"CREATE TABLE {keyspace}.{table} (pk1 int, pk2 int, ck1 int, ck2 int, v int, s int STATIC, PRIMARY KEY ((pk1, pk2), ck1, ck2)) WITH compaction = {{'class': 'NullCompactionStrategy'}}"

    cql.execute(schema)

    for pk in range(0, 10):
        for ck in range(0, 10):
            x = random.randrange(0, 8)
            if x == 0:
                # ttl
                cql.execute(f"INSERT INTO {keyspace}.{table} (pk1, pk2, ck1, ck2, v) VALUES ({pk}, {pk}, {ck}, {ck}, 0) USING TTL 6000")
            elif x == 1:
                # row tombstone
                cql.execute(f"DELETE FROM {keyspace}.{table} WHERE pk1 = {pk} AND pk2 = {pk} AND ck1 = {ck} AND ck2 = {ck}")
            elif x == 2:
                # cell tombstone
                cql.execute(f"DELETE v FROM {keyspace}.{table} WHERE pk1 = {pk} AND pk2 = {pk} AND ck1 = {ck} AND ck2 = {ck}")
            elif x == 3:
                # range tombstone
                l = ck * 10
                u = ck * 11
                cql.execute(f"DELETE FROM {keyspace}.{table} WHERE pk1 = {pk} AND pk2 = {pk} AND ck1 > {l} AND ck1 < {u}")
            else:
                # live row
                cql.execute(f"INSERT INTO {keyspace}.{table} (pk1, pk2, ck1, ck2, v) VALUES ({pk}, {pk}, {ck}, {ck}, 0)")

        if pk == 5:
            cql.execute(f"UPDATE {keyspace}.{table} SET s = 10 WHERE pk1 = {pk} AND pk2 = {pk}")
            nodetool.flush(cql, f"{keyspace}.{table}")

    nodetool.flush(cql, f"{keyspace}.{table}")

    return table, schema

def clustering_table_with_collection(cql, keyspace):
    table = util.unique_name()
    schema = f"CREATE TABLE {keyspace}.{table} (pk int, ck int, v1 map<int, text>, v2 set<int>, v3 list<int>, PRIMARY KEY (pk, ck)) WITH compaction = {{'class': 'NullCompactionStrategy'}}"

    cql.execute(schema)

    for pk in range(0, 10):
        for ck in range(0, 10):
            map_vals = {f"{p}: '{c}'" for p in range(0, pk) for c in range(0, ck)}
            map_str = ", ".join(map_vals)
            set_list_vals = list(range(0, pk))
            set_list_str = ", ".join(map(str, set_list_vals))
            cql.execute(f"INSERT INTO {keyspace}.{table} (pk, ck, v1, v2, v3) VALUES ({pk}, {ck}, {{{map_str}}}, {{{set_list_str}}}, [{set_list_str}])")
        if pk == 5:
            nodetool.flush(cql, f"{keyspace}.{table}")

    nodetool.flush(cql, f"{keyspace}.{table}")

    return table, schema

def clustering_table_with_udt(cql, keyspace):
    table = util.unique_name()
    create_type_schema = f"CREATE TYPE IF NOT EXISTS {keyspace}.type1 (f1 int, f2 text)"
    create_table_schema = f" CREATE TABLE {keyspace}.{table} (pk int, ck int, v type1, PRIMARY KEY (pk, ck)) WITH compaction = {{'class': 'NullCompactionStrategy'}}"

    cql.execute(create_type_schema)
    cql.execute(create_table_schema)

    for pk in range(0, 10):
        for ck in range(0, 10):
            cql.execute(f"INSERT INTO {keyspace}.{table} (pk, ck, v) VALUES ({pk}, {ck}, {{f1: 100, f2: 'asd'}})")
        if pk == 5:
            nodetool.flush(cql, f"{keyspace}.{table}")

    nodetool.flush(cql, f"{keyspace}.{table}")

    return table, "; ".join((create_type_schema, create_table_schema))

def table_with_counters(cql, keyspace):
    table = util.unique_name()
    schema = f"CREATE TABLE {keyspace}.{table} (pk int PRIMARY KEY, v counter) WITH compaction = {{'class': 'NullCompactionStrategy'}}"

    cql.execute(schema)

    for pk in range(0, 10):
        for c in range(0, 4):
            cql.execute(f"UPDATE {keyspace}.{table} SET v = v + 1 WHERE pk = {pk};")
        if pk == 5:
            nodetool.flush(cql, f"{keyspace}.{table}")

    nodetool.flush(cql, f"{keyspace}.{table}")

    return table, schema

@contextlib.contextmanager
def scylla_sstable(table_factory, cql, ks, data_dir):
    table, schema = table_factory(cql, ks)

    schema_file = os.path.join(data_dir, "..", "test_tools_schema.cql")
    with open(schema_file, "w") as f:
        f.write(schema)

    sstables = glob.glob(os.path.join(data_dir, ks, table + '-*', '*-Data.db'))

    try:
        yield (schema_file, sstables)
    finally:
        cql.execute(f"DROP TABLE {ks}.{table}")
        os.unlink(schema_file)

def one_sstable(sstables):
    assert len(sstables) > 1
    return [sstables[0]]

def all_sstables(sstables):
    assert len(sstables) > 1
    return sstables

@pytest.mark.parametrize("what", ["index", "compression-info", "summary", "statistics", "scylla-metadata"])
@pytest.mark.parametrize("which_sstables", [one_sstable, all_sstables])
def test_scylla_sstable_dump_component(cql, test_keyspace, scylla_path, scylla_data_dir, what, which_sstables):
    with scylla_sstable(simple_clustering_table, cql, test_keyspace, scylla_data_dir) as (schema_file, sstables):
        out = subprocess.check_output([scylla_path, "sstable", f"dump-{what}", "--schema-file", schema_file] + which_sstables(sstables))

    print(out)

    assert out
    assert json.loads(out)

@pytest.mark.parametrize("table_factory", [
        simple_no_clustering_table,
        simple_clustering_table,
        clustering_table_with_collection,
        clustering_table_with_udt,
        table_with_counters,
])
@pytest.mark.parametrize("merge", [True, False])
@pytest.mark.parametrize("output_format", ["text", "json"])
def test_scylla_sstable_dump_data(cql, test_keyspace, scylla_path, scylla_data_dir, table_factory, merge, output_format):
    with scylla_sstable(simple_clustering_table, cql, test_keyspace, scylla_data_dir) as (schema_file, sstables):
        args = [scylla_path, "sstable", "dump-data", "--schema-file", schema_file, "--output-format", output_format]
        if merge:
            args.append("--merge")
        out = subprocess.check_output(args + sstables)

    print(out)

    assert out
    if output_format == "json":
        assert json.loads(out)

@pytest.mark.parametrize("table_factory", [
        simple_no_clustering_table,
        simple_clustering_table,
])
def test_scylla_sstable_write(cql, test_keyspace, scylla_path, scylla_data_dir, table_factory):
    with scylla_sstable(table_factory, cql, test_keyspace, scylla_data_dir) as (schema_file, sstables):
        with tempfile.TemporaryDirectory() as tmp_dir:
            dump_common_args = [scylla_path, "sstable", "dump-data", "--schema-file", schema_file, "--output-format", "json", "--merge"]
            generation = util.unique_key_int()

            original_out = subprocess.check_output(dump_common_args + sstables)
            original_json = json.loads(original_out)["sstables"]["anonymous"]

            input_file = os.path.join(tmp_dir, 'input.json')

            with open(input_file, 'w') as f:
                json.dump(original_json, f)

            subprocess.check_call([scylla_path, "sstable", "write", "--schema-file", schema_file, "--input-file", input_file, "--output-dir", tmp_dir, "--generation", str(generation), '--logger-log-level', 'scylla-sstable=trace'])

            sstable_file = os.path.join(tmp_dir, f"me-{generation}-big-Data.db")

            actual_out = subprocess.check_output(dump_common_args + [sstable_file])
            actual_json = json.loads(actual_out)["sstables"]["anonymous"]

            assert actual_json == original_json

def script_consume_test_table_factory(cql, keyspace):
    table = util.unique_name()
    schema = f"CREATE TABLE {keyspace}.{table} (pk int, ck int, v int, s int STATIC, PRIMARY KEY (pk, ck)) WITH compaction = {{'class': 'NullCompactionStrategy'}}"

    cql.execute(schema)

    partitions = 4

    for sst in range(0, 2):
        for pk in range(sst * partitions, (sst + 1) * partitions):
            # static row
            cql.execute(f"UPDATE {keyspace}.{table} SET s = 10 WHERE pk = {pk}")
            # range tombstone
            cql.execute(f"DELETE FROM {keyspace}.{table} WHERE pk = {pk} AND ck >= 0 AND ck <= 4")
            # 2 rows
            for ck in range(0, 4):
                cql.execute(f"INSERT INTO {keyspace}.{table} (pk, ck, v) VALUES ({pk}, {ck}, 0)")

        nodetool.flush(cql, f"{keyspace}.{table}")

    return table, schema

def test_scylla_sstable_script_consume_sstable(cql, test_keyspace, scylla_path, scylla_data_dir):
    script_file = os.path.join(scylla_data_dir, "..", "test_scylla_sstable_script_consume_sstable.lua")

    script = """
wr = Scylla.new_json_writer()
i = 0

function arg(args, arg)
    ret = nil
    wr:key(arg)
    if args[arg] then
        ret = tonumber(args[arg])
        wr:int(ret)
    else
        wr:null()
    end
    return ret
end

function basename(path)
    s, e = string.find(string.reverse(path), '/', 1, true)
    return string.sub(path, #path - s + 2)
end

function consume_stream_start(args)
    wr:start_object()
    start_sst = arg(args, "start_sst")
    end_sst = arg(args, "end_sst")
    wr:key("content")
    wr:start_array()
end

function consume_sstable_start(sst)
    wrote_ps = false
    i = i + 1
    if i == start_sst then
        return false
    end
    wr:string(basename(sst.filename))
end

function consume_partition_start(ps)
    if not wrote_ps then
        wr:string("ps")
        wrote_ps = true
    end
end

function consume_sstable_end()
    if i == end_sst then
        return false
    end
end

function consume_stream_end()
    wr:end_array()
    wr:end_object()
end
"""
    with open(script_file, 'w') as f:
        f.write(script)

    with scylla_sstable(script_consume_test_table_factory, cql, test_keyspace, scylla_data_dir) as (schema_file, sstables):
        sst1 = os.path.basename(sstables[0])
        sst2 = os.path.basename(sstables[1])
        def run_scenario(script_args, expected):
            print(f"Scenario: '{script_args}'\n")
            if script_args:
                script_args = ["--script-arg", script_args]
            else:
                script_args = []
            script_args = [scylla_path, "sstable", "script", "--schema-file", schema_file, "--script-file", script_file] + script_args + sstables[0:2]
            res = json.loads(subprocess.check_output(script_args))
            assert res == expected

        run_scenario("", {'start_sst': None, 'end_sst': None, 'content': [sst1, "ps", sst2, "ps"]})
        run_scenario("start_sst=1", {'start_sst': 1, 'end_sst': None, 'content': [sst2, "ps"]})
        run_scenario("start_sst=2", {'start_sst': 2, 'end_sst': None, 'content': [sst1, "ps"]})
        run_scenario("start_sst=1:end_sst=1", {'start_sst': 1, 'end_sst': 1, 'content': []})
        run_scenario("start_sst=2:end_sst=2", {'start_sst': 2, 'end_sst': 2, 'content': [sst1, "ps"]})
        run_scenario("end_sst=1", {'start_sst': None, 'end_sst': 1, 'content': [sst1, "ps"]})
        run_scenario("end_sst=2", {'start_sst': None, 'end_sst': 2, 'content': [sst1, "ps", sst2, "ps"]})

def test_scylla_sstable_script_slice(cql, test_keyspace, scylla_path, scylla_data_dir):
    class bound:
        @staticmethod
        def unpack_value(value):
            if isinstance(value, tuple):
                return value
            else:
                return None, value

        def __init__(self, value, weight):
            self.token, self.value = self.unpack_value(value)
            self.weight = weight

        def tri_cmp(self, value):
            if self.token is None and self.value is None:
                assert(self.weight)
                return -self.weight
            token, value = self.unpack_value(value)
            if token is None:
                res = 0
            else:
                res = int(token) - int(self.token)
            if res == 0 and not value is None and not self.value is None :
                res = int(value) - int(self.value)
            return res if res else -self.weight

        def get_value(self, lookup_table, is_start):
            if self.token is None and self.value is None:
                return '-inf' if is_start else '+inf'
            if self.value is None:
                return "t{}".format(int(self.token))
            return lookup_table[self.value]

        @staticmethod
        def before(value):
            return bound(value, -1)

        @staticmethod
        def at(value):
            return bound(value, 0)

        @staticmethod
        def after(value):
            return bound(value, 1)

    class interval:
        def __init__(self, start_bound, end_bound):
            self.start = start_bound
            self.end = end_bound

        def contains(self, value):
            return self.start.tri_cmp(value) >= 0 and self.end.tri_cmp(value) <= 0

    def summarize_dump(dump):
        summary = []
        for partition in list(dump["sstables"].items())[0][1]:
            partition_summary = {"pk": partition["key"]["value"], "token": partition["key"]["token"], "frags": []}
            if "static_row" in partition:
                partition_summary["frags"].append(("sr", None))
            for clustering_fragment in partition.get("clustering_elements", []):
                type_str = "cr" if clustering_fragment["type"] == "clustering-row" else "rtc"
                partition_summary["frags"].append((type_str, clustering_fragment["key"]["value"]))
            summary.append(partition_summary)
        return summary

    def filter_summary(summary, partition_ranges, clustering_ranges):
        if not partition_ranges:
            return summary
        filtered_summary = []
        for partition in summary:
            if any(map(lambda x: interval.contains(x, (partition["token"], partition["pk"])), partition_ranges)):
                filtered_summary.append({"pk": partition["pk"], "token": partition["token"], "frags": []})
                for (t, k) in partition["frags"]:
                    if t == "rtc" or k is None or not clustering_ranges or any(map(lambda x: interval.contains(x, k), clustering_ranges)):
                        filtered_summary[-1]["frags"].append((t, k))

        return filtered_summary

    def serialize_ranges(prefix, ranges, lookup_table):
        serialized_ranges = []
        i = 0
        for r in ranges:
            s = r.start.get_value(lookup_table, True)
            e = r.end.get_value(lookup_table, False)
            serialized_ranges.append("{}{}={}{},{}{}".format(
                prefix,
                i,
                "(" if s == '-inf' or r.start.weight > 0 else "[",
                s,
                e,
                ")" if e == '+inf' or r.end.weight < 0 else "]"))
            i = i + 1
        return serialized_ranges

    scripts_path = os.path.realpath(os.path.join(__file__, '../../../tools/scylla-sstable-scripts'))
    script_file = os.path.join(scripts_path, 'slice.lua')

    with scylla_sstable(script_consume_test_table_factory, cql, test_keyspace, scylla_data_dir) as (schema_file, sstables):
        reference_summary = summarize_dump(json.loads(subprocess.check_output([scylla_path, "sstable", "dump-data", "--schema-file", schema_file, "--merge"] + sstables)))

        # same order as in dump
        pks = [(p["token"], p["pk"]) for p in reference_summary]
        cks = set()
        for p in reference_summary:
            for t, ck in p["frags"]:
                if not ck is None:
                    cks.add(ck)
        cks = sorted(list(cks))
        serialized_pk_lookup = {pk: subprocess.check_output([scylla_path, "types", "serialize", "--full-compound", "-t", "Int32Type", "--", pk]).strip().decode() for t, pk in pks}
        serialized_ck_lookup = {ck: subprocess.check_output([scylla_path, "types", "serialize", "--prefix-compound", "-t", "Int32Type", "--", ck]).strip().decode() for ck in cks}

        script_common_args = [scylla_path, "sstable", "script", "--schema-file", schema_file, "--merge", "--script-file", script_file]

        def run_scenario(scenario, partition_ranges, clustering_ranges):
            print(f"running scenario {scenario}")
            script_args = serialize_ranges("pr", partition_ranges, serialized_pk_lookup) + serialize_ranges("cr", clustering_ranges, serialized_ck_lookup)
            if script_args:
                script_args = ["--script-arg"] + [":".join(script_args)]
            print(f"script_args={script_args}")
            expected = filter_summary(reference_summary, partition_ranges, clustering_ranges)
            out = subprocess.check_output(script_common_args + script_args + sstables)
            summary = summarize_dump(json.loads(out))
            assert summary == expected

        run_scenario("no args", [], [])
        run_scenario("full range", [interval(bound.before(None), bound.after(None))], [])
        run_scenario("(pks[0], +inf)", [interval(bound.after(pks[0]), bound.after(None))], [])
        run_scenario("(-inf, pks[-3]]", [interval(bound.before(None), bound.after(pks[-3]))], [])
        run_scenario("[pks[2], pks[-2]]", [interval(bound.before(pks[2]), bound.after(pks[-2]))], [])
        run_scenario("[pks[0], pks[1]], [pks[2], pks[3]]", [interval(bound.before(pks[1]), bound.after(pks[2])), interval(bound.before(pks[3]), bound.after(pks[4]))], [])
        run_scenario("[t:pks[2], t:pks[-2]]", [interval(bound.before((pks[2][0], None)), bound.after((pks[-2][0], None)))], [])
        run_scenario("full pk range | [-inf, cks[2]]", [interval(bound.before(None), bound.after(None))], [interval(bound.before(None), bound.after(cks[2]))])
        run_scenario("[pks[0], pks[1]] | (cks[0], cks[1]], (cks[2], +inf)", [interval(bound.before(pks[1]), bound.after(pks[2]))],
                     [interval(bound.after(cks[0]), bound.after(cks[1])), interval(bound.after(cks[2]), bound.after(None))])

@pytest.mark.parametrize("table_factory", [
        simple_no_clustering_table,
        simple_clustering_table,
        clustering_table_with_collection,
        clustering_table_with_udt,
        table_with_counters,
])
def test_scylla_sstable_script(cql, test_keyspace, scylla_path, scylla_data_dir, table_factory):
    scripts_path = os.path.realpath(os.path.join(__file__, '../../../tools/scylla-sstable-scripts'))
    slice_script_path = os.path.join(scripts_path, 'slice.lua')
    dump_script_path = os.path.join(scripts_path, 'dump.lua')
    with scylla_sstable(table_factory, cql, test_keyspace, scylla_data_dir) as (schema_file, sstables):
        dump_common_args = [scylla_path, "sstable", "dump-data", "--schema-file", schema_file, "--output-format", "json"]
        script_common_args = [scylla_path, "sstable", "script", "--schema-file", schema_file]

        # without --merge
        cxx_json = json.loads(subprocess.check_output(dump_common_args + sstables))
        dump_lua_json = json.loads(subprocess.check_output(script_common_args + ["--script-file", dump_script_path] + sstables))
        slice_lua_json = json.loads(subprocess.check_output(script_common_args + ["--script-file", slice_script_path] + sstables))

        assert dump_lua_json == cxx_json
        assert slice_lua_json == cxx_json

        # with --merge
        cxx_json = json.loads(subprocess.check_output(dump_common_args + ["--merge"] + sstables))
        dump_lua_json = json.loads(subprocess.check_output(script_common_args + ["--merge", "--script-file", dump_script_path] + sstables))
        slice_lua_json = json.loads(subprocess.check_output(script_common_args + ["--merge", "--script-file", slice_script_path] + sstables))

        assert dump_lua_json == cxx_json
        assert slice_lua_json == cxx_json


import numpy as np
import matplotlib.pyplot as plt
import argparse
np.set_printoptions(suppress=True)

def kernel(X1, X2):
	'''
	Calculate the covariance matrix
	args:
		X1: n by d matrix, each row is a d-dimensional data point
		X2: m by d matrix, each row is a d-dimensional data point
	return:
		K(X1, X2): n by m covariance matrix
	'''
	lengthscale=args.lengthscale
	variance=args.variance
	# cool vectorization. note how (n, 1) is braodcast with (m, )
	squredist = np.sum(X1**2, axis=1).reshape(-1, 1) + np.sum(X2**2, axis=1) - 2*np.dot(X1, X2.T)
	return variance**2 * np.exp(-1/(2*lengthscale) * squredist)

def sample_from_gp(n_func=5, noise_variance=0.):
	'''
	Sample functions from gaussian process.
	Args:
		n_func: number of functions to sample
		noise_variance: amount of variance the algorithm assumes on the data
	It shows and saves three plots:
		- prior functions
		- posterior functions with 3-sigma confidence level and observed data
		- obervered data, true function and mean
	'''
	# this is the true unknown function we are trying to approximate
	f = lambda x: 2.5*np.sin(0.9*x).flatten()
	np.random.seed(100)
	# data to plot the true function
	n_plot = 1000
	X_plot = np.linspace(-5, 5, n_plot)
	y_plot = f(X_plot)
	# noisy training/observed data points
	n_train = 10
	X_train = np.random.uniform(-5, 5, n_train).reshape(-1, 1)
	y_train = f(X_train) + np.random.randn(n_train) * 0.1
	# test data points to sample a function from the GP
	n_test = 100
	X_test = np.linspace(-5, 5, n_test).reshape(-1, 1)

	K = kernel(X_train, X_train)
	L = np.linalg.cholesky(K + np.eye(n_train)*noise_variance)
	alpha = np.linalg.solve(L.T, np.linalg.solve(L, y_train))

	# compute mean at test points
	Ks = kernel(X_train, X_test)
	mu = np.dot(Ks.T, alpha)

	# compute covariance at test points
	# prior covariance
	Kss = kernel(X_test, X_test)
	V = np.linalg.solve(L, Ks)
	# posterior covariance
	Cov = Kss - np.dot(V.T, V)
	# compute variance at test points
	# another cool vectorization
	# np.diag(AB) = np.sum(A*B.T, axis=1)
	var = np.diag(Kss) - np.sum(V.T**2, axis=1)
	standev = np.sqrt(var)

	# sample functions from the prior
	L = np.linalg.cholesky(Kss + 1e-6*np.eye(n_test))
	f_prior = np.dot(L, np.random.randn(n_test, n_func))
	fig, ax = plt.subplots(figsize=(12, 8))
	plt.axis([-5, 5, -3, 3])
	plt.plot(X_test, f_prior, '-')
	plt.title('prior')
	plt.savefig('prior.png', bbox_inches='tight')

	# sample functions from the posterior
	L = np.linalg.cholesky(Cov + 1e-6*np.eye(n_test))
	f_post = mu.reshape(-1, 1) + np.dot(L, np.random.randn(n_test, n_func))
	fig, ax = plt.subplots(figsize=(12, 8))
	plt.axis([-5, 5, -3, 3])
	plt.plot(X_test, f_post, '-')
	plt.plot(X_train, y_train, '+', color='r', markersize=17, label='y (traing data)')
	plt.gca().fill_between(X_test.flat, mu-3*standev, mu+3*standev, color="#dddddd")
	plt.title('posterior')
	plt.savefig('posterior.png', bbox_inches='tight')

	# plot the true function and mean
	fig, ax = plt.subplots(figsize=(12, 8))
	plt.axis([-5, 5, -3, 3])
	plt.plot(X_train, y_train, '+', color='r', markersize=17, label='y (traing/observed data)')
	plt.plot(X_test, mu, '-', label='predicted mean')
	plt.plot(X_plot, y_plot, '-', color='r', label='f (true function)')
	plt.legend(loc='upper right')
	plt.gca().fill_between(X_test.flat, mu-3*standev, mu+3*standev, color="#dddddd")
	plt.title('true function and predicted mean')
	plt.savefig('true_func.png', bbox_inches='tight')
	plt.show()

def main():
	sample_from_gp(n_func=args.n_func, noise_variance=args.noise_variance)

if __name__ == "__main__":
	parser = argparse.ArgumentParser(description='Sample functions from gaussian process')
	parser.add_argument('--n_func', type=int, default=10, help='number of functions to sample')
	parser.add_argument('--lengthscale', type=float, default=1., help='lengthscale parameter of squared exponential kernel')
	parser.add_argument('--variance', type=float, default=1., help='variance parameter of squared exponential kernel')
	parser.add_argument('--noise_variance', type=float, default=0., help='variance of noise assumed on observed data')
	args = parser.parse_args()
	main()


from tensorflow.keras.models import load_model
import numpy as np

#loading the model
model = load_model('nlu\\model.h5')

#loading the output labels
labels = open('labels.txt', 'r', encoding='utf-8').read().split('\n')

#creating index to label and label to index
label2idx = {}
idx2label = {}

for k, label in enumerate(labels):
    label2idx[label] = k
    idx2label[k] = label

def classify(text):
    # Criar um array de entrada
    x = np.zeros((1, 500, 256), dtype='float32')

    # Preencher o array com dados do texto.
    for k, ch in enumerate(bytes(text.encode('utf-8'))):
        x[0, k, int(ch)] = 1.0

    # Fazer a previsão
    out = model.predict(x)
    idx = out.argmax()

    return idx2label[idx]


import aiohttp.web
import pytest

from kopf._cogs.clients.creating import create_obj
from kopf._cogs.clients.errors import APIError

async def test_simple_body_with_arguments(
        resp_mocker, aresponses, hostname, settings, resource, namespace, logger, caplog):

    post_mock = resp_mocker(return_value=aiohttp.web.json_response({}))
    aresponses.add(hostname, resource.get_url(namespace=namespace), 'post', post_mock)

    body = {'x': 'y'}
    await create_obj(
        logger=logger,
        settings=settings,
        resource=resource,
        namespace=namespace,
        name='name1',
        body=body,
    )

    assert post_mock.called
    assert post_mock.call_count == 1

    data = post_mock.call_args_list[0][0][0].data  # [callidx][args/kwargs][argidx]
    if resource.namespaced:
        assert data == {'x': 'y', 'metadata': {'name': 'name1', 'namespace': 'ns'}}
    else:
        assert data == {'x': 'y', 'metadata': {'name': 'name1'}}

async def test_full_body_with_identifiers(
        resp_mocker, aresponses, hostname, settings, resource, namespace, caplog, logger):

    post_mock = resp_mocker(return_value=aiohttp.web.json_response({}))
    aresponses.add(hostname, resource.get_url(namespace=namespace), 'post', post_mock)

    body = {'x': 'y', 'metadata': {'name': 'name1', 'namespace': namespace}}
    await create_obj(
        logger=logger,
        settings=settings,
        resource=resource,
        body=body,
    )

    assert post_mock.called
    assert post_mock.call_count == 1

    data = post_mock.call_args_list[0][0][0].data  # [callidx][args/kwargs][argidx]
    assert data == {'x': 'y', 'metadata': {'name': 'name1', 'namespace': namespace}}

# Note: 401 is wrapped into a LoginError and is tested elsewhere.
@pytest.mark.parametrize('status', [400, 403, 404, 409, 500, 666])
async def test_raises_api_errors(
        resp_mocker, aresponses, hostname, settings, status, resource, namespace, logger,
        cluster_resource, namespaced_resource):

    post_mock = resp_mocker(return_value=aresponses.Response(status=status))
    cluster_url = cluster_resource.get_url(namespace=None)
    namespaced_url = namespaced_resource.get_url(namespace='ns')
    aresponses.add(hostname, cluster_url, 'post', post_mock)
    aresponses.add(hostname, namespaced_url, 'post', post_mock)

    body = {'x': 'y'}
    with pytest.raises(APIError) as e:
        await create_obj(
            logger=logger,
            settings=settings,
            resource=resource,
            namespace=namespace,
            name='name1',
            body=body,
        )
    assert e.value.status == status


# -*- coding: utf-8 -*-
import sys
import csv

special_case = False
sc_index2 = -1
sc_index1 = -1
dicto = None

def load_data_meta(dictionary):
    fi = open("metadata.txt", "r")
    check  = False
    current_table = None
    for x in fi:
        x = x.strip()
        if check == False and x == "<begin_table>" :
            check = True
        elif check == True :
            dictionary[x.lower()] = {'attr' : [] , 'data':[], 'oattr' :[], 'oname' : x}
            current_table = x.lower()
            check = False
        elif check == False and x != "<begin_table>" and x != "<end_table>":
            dictionary[current_table]['attr'].append(x.lower())
            dictionary[current_table]['oattr'].append(x)

        elif x == "<end_table>":
            current_table = None
        else :
            sys.exit("Proper format required in metadata.txt\n")

def read_csv(dictionary):
    for table in dictionary.keys():
        with open(table+".csv") as f :
            for row in csv.reader(f):
                row = list(map(int , row))
                dictionary[table]['data'].append(row)

def cross_product(dictionary , query_from):
    merged_data_attr  = []
    merged_data = []
    try :
        for table in query_from:
            for attr in  dictionary[table]['attr']:
                merged_data_attr.append(table + "." +attr)
    except :
        sys.exit("error : in accessing the table.")

    try :
        temp_data = []
        for table in query_from:
            if len(merged_data) == 0:
                merged_data = dictionary[table]['data']
            else:
                temp_data = []
                for row1 in merged_data:
                    for row2 in dictionary[table]['data']:
                        temp_data.append(row1 + row2)
                merged_data = temp_data
    except :
        sys.exit("error : in accessing table .")

    return merged_data_attr , merged_data

def is_true_cond(cond ,attr,attr_splitted, row):
    try :
        col1 = cond[0]
        col2 = cond[2]
        col1_index = None
        col2_index = None
        op = cond[1]
        col2_is_num = False

        if col1 in attr :
            col1_index = attr.index(col1)
        elif col1 in attr_splitted and attr_splitted.count(col1) == 1:
            col1_index = attr_splitted.index(col1)
        else:
            sys.exit("error : " + col1 + " is causing issue.")

        if col2 in attr :
            col2_index = attr.index(col2)
        elif col2 in attr_splitted :
            if attr_splitted.count(col2) == 1 :
                col2_index = attr_splitted.index(col2)
        else:
            if col2.lstrip("-").isdigit() == True:
                col2_is_num = True
            else:
                sys.exit("error : " + col2 + " is causing issue.")

        if col2_is_num == False :
            if op == "=" :
                global special_case
                global sc_index1
                global sc_index2
                special_case = True
                sc_index1 = col1_index
                sc_index2 = col2_index
                return row[col1_index] == row[col2_index]
            elif op == "<=" : return row[col1_index] <= row[col2_index]
            elif op == "<" : return row[col1_index] < row[col2_index]
            elif op == ">=" : return row[col1_index] >= row[col2_index]
            elif op == ">" : return row[col1_index] > row[col2_index]
        elif col2_is_num == True :
            col2 = int(col2)
            if op == "=" :return row[col1_index] == col2
            elif op == "<=" : return row[col1_index] <= col2
            elif op == "<" : return row[col1_index] < col2
            elif op == ">=" : return row[col1_index] >= col2
            elif op == ">" : return row[col1_index] > col2
        else:
            sys.exit("error : in " + str(col2))

    except :
        sys.exit("error : error in conditional part")

def print_header(attr):
    final = []
    for row in attr :
        row = row.split(".")
        table_name = row[0]
        table_attr = row[1]
        for ele in dicto[table_name]['oattr'] :
            if ele.lower() == table_attr :
                final.append(dicto[table_name]['oname'] + "." + ele)
                break
    return ",".join(final)

def coditional_selection(attr , data , conditions):
    cond1 = conditions
    cond2 = None
    is_many_cond  = False
    refined_data = []
    attr_splitted =  []

    for ele in attr :
        attr_splitted.append(ele.split(".")[1].lower())
    try :
        if "and" in conditions:
            cond1 = conditions[0:conditions.index("and")]
            cond2 = conditions[conditions.index("and") + 1 :]
            is_many_cond = True

        elif "or" in conditions:
            cond1 = conditions[0:conditions.index("or")]
            cond2 = conditions[conditions.index("or") + 1 :]
            is_many_cond = True
    except:
        sys.exit("error : Not a proper sql statement")

    if is_many_cond == False :
        for row in data :
            if is_true_cond(cond1 ,attr , attr_splitted, row) == True:
                refined_data.append(row)
    else:
        if "and" in conditions :
            for row in data :
                if is_true_cond(cond1 ,attr,attr_splitted, row) == True and is_true_cond(cond2,attr,attr_splitted, row) == True:
                    refined_data.append(row)
        elif "or" in conditions :
            for row in data :
                if is_true_cond(cond1 ,attr, attr_splitted, row) == True or is_true_cond(cond2 ,attr,attr_splitted,  row)== True:
                    refined_data.append(row)
    return refined_data

def print_final_result(attr, data, query_items):
    try :
        if(len(query_items) < 1): sys.exit("error : incomplete sql query")
        attr_splitted = []
        for ele in attr :
            attr_splitted.append(ele.split("."))
        #=======================================================
        if len(query_items) == 1 and query_items[0] == "*":
            if special_case == True:
                del attr[sc_index2]
            print(print_header(attr))
            for row in data :
                if special_case == True:
                    del row[sc_index2]
                print(",".join(map(str,row)))
            return
       #=======================================================
        elif  len(query_items) == 1 and query_items[0].find("(") != -1 and query_items[0].find(")") != -1  :
            op = query_items[0].split("(")[0]
            col_name = query_items[0].split("(")[1].split(")")[0]
            col_index = None

            if col_name in attr :
                col_index = attr.index(col_name)
            else :
                for i in range(len(attr_splitted)):
                    li  = attr_splitted[i]
                    if li[1] == col_name:
                        col_name = ".".join(li)
                        col_index = i
                        break
            if col_index == None :
                sys.exit("column " + col_name + " not found.")

            col_arr = [row[col_index] for row in data]

            print(op + "("+ print_header([col_name]) + ")")
            if op == "sum" :
                print(sum(col_arr))
            elif op == "max":
                print(max(col_arr))
            elif op == "min":
                print(min(col_arr))
            elif op == "average" :
                print(float(sum(col_arr))/len(col_arr))
            else :
                sys.exit()
            return
    #===================================================
        elif len(query_items) >= 1 :
            cols = query_items
            distinct = False
            if query_items[0] == "distinct":
                cols = query_items[1:]
                distinct = True
            attr_name =  []
            indexs = []
            for col_name in cols :
                col_index =  None
                if col_name in attr :
                    col_index = attr.index(col_name)
                else :
                    repeat = False

                    for i in range(len(attr_splitted)):
                        li  = attr_splitted[i]
                        if li[1] == col_name:
                            if repeat == True:sys.exit()
                            temp_name = ".".join(li)
                            col_index = i
                            repeat = True
                    col_name = temp_name
                    if col_index == None :
                        sys.exit("column " + col_name + " not found.")
                attr_name.append(col_name)
                indexs.append(col_index)

            temp_index =  -1
            if special_case == True :
                if attr[sc_index1] in attr_name and attr[sc_index2] in attr_name:
                    temp_index = attr_name.index(attr[sc_index2])
                    del indexs[temp_index]
                    del attr_name[temp_index]

            print(print_header(attr_name))

            final_array = []
            for row in data:
                temp  = [row[index] for index in indexs]
                temp =  ",".join(map(str,temp))
                final_array.append(temp)
            if distinct == True : final_array = list(set(final_array))
            for row in final_array :
            	# row = map(str, row)
                print(row)
        else:
            sys.exit()

    except :
        sys.exit("error : in selection. ")

def process_query(string , dictionary):
    string = str(string).lower()

    if string[-1] != ";":
        sys.exit("error : not a proper sql format")

    string = string[:-1]
    string = string.replace(",", " ")
    tokens  = string.split(" ")
    query_items = None
    query_from = None
    query_cond = None
    condition =  False

    if tokens[0].lower() != "select" :
        sys.exit("error : not a proper sql statement")
    if "from" not in tokens :
        sys.exit("error : not a proper sql statement")

    query_items = tokens[1:tokens.index("from")]

    if "where" in tokens:
         query_from  = tokens[tokens.index("from") + 1:tokens.index("where")]
         query_cond = tokens[tokens.index("where")+1:]
         condition = True
    else:
        query_from = tokens[tokens.index("from")+1:]

    joint_data_attr , joint_data = cross_product(dictionary, query_from)
    if condition == True :
        joint_data = coditional_selection(joint_data_attr , joint_data, query_cond)
    print_final_result(joint_data_attr ,joint_data , query_items)

def main():
    dictionary = {}
    load_data_meta(dictionary)
    global dicto
    dicto = dictionary
    if len(sys.argv) > 2:
        sys.exit("given arguments length exceeds than requirment")
    read_csv(dictionary)

    process_query(sys.argv[1] , dictionary)

if __name__ == "__main__":
    main()


COLUMNS = [
    ("name", "text"),
    ("hash", "text"),
    ("mhash", "text"),
    ("remove", "integer"),
    ("mtime", "real"),
    ("exe", "real"),
    ("ignore", "integer"),
    ("ctheirs", "text"),
    ("cbase", "text"),
]

def column_index(name: str) -> int:
    """Get index of a column

    Args:
        name: column name

    Retuns:
        Index of the column

    Raise:
        IndexError: when `name` is not inside COLUMNS
    """
    for idx, column in enumerate(COLUMNS):
        if column[0] == name:
            return idx

    raise IndexError(
        f"Invalid index column '{name}'. Accept: {list(_[0] for _ in COLUMNS)}"
    )


#!/usr/bin/env python3

from contextlib import closing
from shutil import rmtree

from pathlib import Path, PurePath
from psycopg2 import connect

from qc_tool.common import CONFIG
from qc_tool.common import JOB_OUTPUT_DIRNAME
from qc_tool.common import JOB_TMP_DIRNAME
from qc_tool.common import compose_job_dir

def create_connection_manager(job_uuid):
    connection_manager = ConnectionManager(job_uuid,
                                           CONFIG["pg_host"],
                                           CONFIG["pg_port"],
                                           CONFIG["pg_user"],
                                           CONFIG["pg_database"],
                                           CONFIG["leave_schema"])
    return connection_manager

def create_jobdir_manager(job_uuid):
    job_dir = compose_job_dir(job_uuid)
    jobdir_manager = JobdirManager(job_dir, CONFIG["leave_jobdir"])
    return jobdir_manager

class ConnectionException(Exception):
    pass

class ConnectionManager():
    job_schema_name_tpl = "job_{:s}"

    def __init__(self, job_uuid, host, port, user, db_name, leave_schema):
        self.job_uuid = job_uuid
        self.host = host
        self.port = port
        self.user = user
        self.db_name = db_name
        self.leave_schema = leave_schema
        self.connection = None
        self.job_schema_name = None

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()

    def _is_connected(self):
        return self.connection is not None and self.connection.closed == 0

    def _create_connection(self):
        try:
            connection = connect(host=self.host, port=self.port, user=self.user, dbname=self.db_name)
        except Exception as ex:
            msg = "Can not make db connection for the job:{:s}.".format(self.job_uuid)
            raise ConnectionException(msg) from ex
        self.connection = connection
        self.connection.autocommit = True

    def _create_job_schema(self):
        job_uuid = self.job_uuid.lower().replace("-", "")
        job_schema_name = self.job_schema_name_tpl.format(job_uuid)
        with closing(self.connection.cursor()) as cursor:
            cursor.execute("CREATE SCHEMA {:s};".format(job_schema_name))
            self.job_schema_name = job_schema_name
            cursor.execute("SET search_path TO {:s}, public;".format(job_schema_name))

    def _drop_job_schema(self):
        with closing(self.connection.cursor()) as cursor:
            cursor.execute("DROP SCHEMA {:s} CASCADE;".format(self.job_schema_name))
            self.job_schema_name = None
            cursor.close()

    def get_dsn_schema(self):
        conn = self.get_connection()
        return (conn.dsn, self.job_schema_name)

    def get_connection(self):
        if self._is_connected():
            return self.connection
        self._create_connection()
        if self.job_schema_name is None:
            self._create_job_schema()
        return self.connection

    def close(self):
        if not self.leave_schema and self.job_schema_name is not None:
            if not self._is_connected():
                conn = self._create_connection()
            self._drop_job_schema()
        if self._is_connected():
            self.connection.close()
        self.connection = None

class JobdirManager():
    def __init__(self, job_dir, leave_dir=False):
        self.job_dir = job_dir
        self.leave_dir = leave_dir
        self.tmp_dir = None
        self.output_dir = None

    def __enter__(self):
        self.create_dirs()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.remove_tmp_dir()

    def create_dirs(self):
        tmp_dir = self.job_dir.joinpath(JOB_TMP_DIRNAME)
        tmp_dir.mkdir(parents=True)
        self.tmp_dir = tmp_dir

        output_dir = self.job_dir.joinpath(JOB_OUTPUT_DIRNAME)
        output_dir.mkdir(parents=True)
        self.output_dir = output_dir

    def remove_tmp_dir(self):
        if not self.leave_dir and self.tmp_dir is not None:
            rmtree(str(self.tmp_dir))
        self.tmp_dir = None


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from matplotlib import pyplot
import numpy as np
import os
import shutil
import operator
import caffe2.python.predictor.predictor_exporter as pe
import argparse
from caffe2.proto import caffe2_pb2

from caffe2.python import (
    brew,
    core,
    model_helper,
    net_drawer,
    optimizer,
    visualize,
    workspace,
)

# If you would like to see some really detailed initializations,
# you can change --caffe2_log_level=0 to --caffe2_log_level=-1
core.GlobalInit(['caffe2', '--caffe2_log_level=1'])
print("Necessities imported!")

class TrainCaffe(object):
    USE_LENET_MODEL = True
    data_folder = None
    root_folder = None
    train_model = None
    test_model = None
    deploy_model = None

    def __init__(self):
        print('Init train class')

    # This section preps your image and test set in a lmdb database
    def DownloadResource(self, url, path):
        '''Downloads resources from s3 by url and unzips them to the provided path'''
        import requests, zipfile, StringIO
        print("Downloading... {} to {}".format(url, path))
        r = requests.get(url, stream=True)
        z = zipfile.ZipFile(StringIO.StringIO(r.content))
        z.extractall(path)
        print("Completed download and extraction.")

    def PrepareData(self):
        # Setup the paths for the necessary directories
        current_folder = os.path.join(os.path.expanduser('~'), 'data')
        self.data_folder = os.path.join(current_folder, 'tutorial_data', 'breakhis')
        self.root_folder = os.path.join(current_folder, 'tutorial_files', 'tutorial_breakhis')
        db_missing = False

        # Check if the data folder already exists
        if not os.path.exists(self.data_folder):
            os.makedirs(self.data_folder)
            print("Your data folder was not found!! This was generated: {}".format(self.data_folder))

        # Check if the training lmdb exists in the data folder
        if os.path.exists(os.path.join(self.data_folder, "breakhis40x-train-nchw-lmdb")):
            print("lmdb train db found!")
        else:
            db_missing = True

        # Check if the testing lmdb exists in the data folder
        if os.path.exists(os.path.join(self.data_folder, "breakhis40x-test-nchw-lmdb")):
            print("lmdb test db found!")
        else:
            db_missing = True

        # Attempt the download of the db if either was missing
        if db_missing:
            print("one or both of the MNIST lmbd dbs not found!!")
            db_url = "http://download.caffe2.ai/databases/mnist-lmdb.zip"
            try:
                self.DownloadResource(db_url, self.data_folder)
            except Exception as ex:
                print("Failed to download dataset. Please download it manually from {}".format(db_url))
                print("Unzip it and place the two database folders here: {}".format(self.data_folder))
                raise ex

        # Clean up statistics from any old runs
        if os.path.exists(self.root_folder):
            print("Looks like you ran this before, so we need to cleanup those old files...")
            shutil.rmtree(self.root_folder)

        os.makedirs(self.root_folder)
        workspace.ResetWorkspace(self.root_folder)

        print("training data folder:" + self.data_folder)
        print("workspace root folder:" + self.root_folder)

    def AddInput(self, model, batch_size, db, db_type):
        ### load the data from db - Method 1 using brew
        #data_uint8, label = brew.db_input(
        #    model,
        #    blobs_out=["data_uint8", "label"],
        #    batch_size=batch_size,
        #    db=db,
        #    db_type=db_type,
        #)
        ### load the data from db - Method 2 using TensorProtosDB
        data_uint8, label = model.TensorProtosDBInput(
            [], ["data_uint8", "label"], batch_size=batch_size,
            db=db, db_type=db_type)
        # cast the data to float
        data = model.Cast(data_uint8, "data", to=core.DataType.FLOAT)
        # scale data from [0,255] down to [0,1]
        data = model.Scale(data, data, scale=float(1./256))
        # don't need the gradient for the backward pass
        data = model.StopGradient(data, data)
        return data, label

    # Function to construct a MLP neural network
    # The input 'model' is a model helper and 'data' is the input data blob's name
    def AddMLPModel(self, model, data):
        size = 28 * 28 * 1
        sizes = [size, size * 2, size * 2, 10]
        layer = data
        for i in range(len(sizes) - 1):
            layer = brew.fc(model, layer, 'dense_{}'.format(i), dim_in=sizes[i], dim_out=sizes[i + 1])
            layer = brew.relu(model, layer, 'relu_{}'.format(i))
        softmax = brew.softmax(model, layer, 'softmax')
        return softmax

    def AddLeNetModel(self, model, data):
        '''
        This part is the standard LeNet model: from data to the softmax prediction.

        For each convolutional layer we specify dim_in - number of input channels
        and dim_out - number or output channels. Also each Conv and MaxPool layer changes the
        image size. For example, kernel of size 5 reduces each side of an image by 4.

        While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides
        each side in half.
        '''
        # Image size: 28 x 28 -> 24 x 24
        conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5)
        # Image size: 24 x 24 -> 12 x 12
        pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
        # Image size: 12 x 12 -> 8 x 8
        conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5)
        # Image size: 8 x 8 -> 4 x 4
        pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
        # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size
        # Here, the data is flattened from a tensor of dimension 50x4x4 to a vector of length 50*4*4
        fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500)
        relu3 = brew.relu(model, fc3, 'relu3')
        # Last FC Layer
        pred = brew.fc(model, relu3, 'pred', dim_in=500, dim_out=10)
        # Softmax Layer
        softmax = brew.softmax(model, pred, 'softmax')

        return softmax

    def AddLeNetModelBreakHis(self, model, data):
        '''
        This part is the standard LeNet model: from data to the softmax prediction.

        For each convolutional layer we specify dim_in - number of input channels
        and dim_out - number or output channels. Also each Conv and MaxPool layer changes the
        image size. For example, kernel of size 5 reduces each side of an image by 4.

        While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides
        each side in half.
        '''
        # Image size: 28 x 28 -> 24 x 24
        # Image size: 224 x 224-> 220 x 220
        conv1 = brew.conv(model, data, 'conv1', dim_in=3, dim_out=20, kernel=5)
        # Image size: 24 x 24 -> 12 x 12
        # Image size: 220 x 220-> 110 x 110
        pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
        # Image size: 12 x 12 -> 8 x 8
        # Image size: 110 x 110 -> 106 x 106
        conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5)
        # Image size: 8 x 8 -> 4 x 4
        # Image size: 106 x 106 -> 53 x 53
        pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
        # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size
        # Here, the data is flattened from a tensor of dimension 50x4x4 to a vector of length 50*4*4
        fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 53 * 53, dim_out=500)
        relu3 = brew.relu(model, fc3, 'relu3')
        # Last FC Layer
        pred = brew.fc(model, relu3, 'pred', dim_in=500, dim_out=2)
        # Softmax Layer
        softmax = brew.softmax(model, pred, 'softmax')

        return softmax

    def AddModel(self,model, data):
        if self.USE_LENET_MODEL:
            return self.AddLeNetModelBreakHis(model, data)
        else:
            return self.AddMLPModel(model, data)

    def AddAccuracy(self, model, softmax, label):
        """Adds an accuracy op to the model"""
        accuracy = brew.accuracy(model, [softmax, label], "accuracy")
        return accuracy

    def AddTrainingOperators(self, model, softmax, label):
        """Adds training operators to the model."""
        # Compute cross entropy between softmax scores and labels
        xent = model.LabelCrossEntropy([softmax, label], 'xent')
        # Compute the expected loss
        loss = model.AveragedLoss(xent, "loss")
        # Track the accuracy of the model
        self.AddAccuracy(model, softmax, label)
        # Use the average loss we just computed to add gradient operators to the model
        model.AddGradientOperators([loss])
        # Specify the optimization algorithm
        optimizer.build_sgd(
            model,
            base_learning_rate=0.1,
            policy="step",
            stepsize=1,
            gamma=0.999,
        )

    def AddBookkeepingOperators(self, model):
        """This adds a few bookkeeping operators that we can inspect later.

        These operators do not affect the training procedure: they only collect
        statistics and prints them to file or to logs.
        """
        # Print basically prints out the content of the blob. to_file=1 routes the
        # printed output to a file. The file is going to be stored under
        #     root_folder/[blob name]
        model.Print('accuracy', [], to_file=1)
        model.Print('loss', [], to_file=1)
        # Summarizes the parameters. Different from Print, Summarize gives some
        # statistics of the parameter, such as mean, std, min and max.
        for param in model.params:
            model.Summarize(param, [], to_file=1)
            model.Summarize(model.param_to_grad[param], [], to_file=1)
        # Now, if we really want to be verbose, we can summarize EVERY blob
        # that the model produces; it is probably not a good idea, because that
        # is going to take time - summarization do not come for free. For this
        # demo, we will only show how to summarize the parameters and their
        # gradients.

    def SetDeviceOption(self, model, device_option):
        # Clear op-specific device options and set global device option.
        for net in ("net", "param_init_net"):
            net_def = getattr(model, net).Proto()
            net_def.device_option.CopyFrom(device_option)
            #for op in net_def.op:
            #    # Some operators are CPU-only.
            #    if str(op.output[0]) not in ("optimizer_iteration", "iteration_mutex", "data", "data_uint8"):
            #        op.ClearField("device_option")
            #        op.ClearField("engine")
            setattr(model, net, core.Net(net_def))
    '''
    def SetRunGPU(self, model):
        device_option = caffe2_pb2.DeviceOption()
        device_option.device_type = caffe2_pb2.CUDA
        device_option.cuda_gpu_id = 0
        self.SetDeviceOption(model, device_option)
        return device_option
    '''

    def ConstructModel(self):
        #### Train Model
        # Specify the data will be input in NCHW order
        #  (i.e. [batch_size, num_channels, height, width])
        arg_scope = {"order": "NCHW"}
        # Create the model helper for the train model
        self.train_model = model_helper.ModelHelper(name="mnist_train", arg_scope=arg_scope)
        #self.train_model.param_init_net.RunAllOnGPU()
        #self.train_model.net.RunAllOnGPU()
        # Specify the input is from the train lmdb
        data, label = self.AddInput(
            self.train_model, batch_size=64,
            db=os.path.join(self.data_folder, 'breakhis40x-train-nchw-lmdb'),
            db_type='lmdb')
        # Add the model definition (fc layers, conv layers, softmax, etc.)
        softmax = self.AddModel(self.train_model, data)
        # Add training operators, specify loss function and optimization algorithm
        self.AddTrainingOperators(self.train_model, softmax, label)
        # Add bookkeeping operators to save stats from training
        self.AddBookkeepingOperators(self.train_model)

        #### Testing model.
        # We will set the batch size to 100, so that the testing
        #   pass is 100 iterations (10,000 images in total).
        #   For the testing model, we need the data input part, the main AddModel
        #   part, and an accuracy part. Note that init_params is set False because
        #   we will be using the parameters obtained from the train model which will
        #   already be in the workspace.
        self.test_model = model_helper.ModelHelper(
            name="mnist_test", arg_scope=arg_scope, init_params=False)
        #self.test_model.param_init_net.RunAllOnGPU()
        #self.test_model.net.RunAllOnGPU()

        data, label = self.AddInput(
            self.test_model, batch_size=100,
            db=os.path.join(self.data_folder, 'breakhis40x-test-nchw-lmdb'),
            db_type='lmdb')
        softmax = self.AddModel(self.test_model, data)
        self.AddAccuracy(self.test_model, softmax, label)

        #### Deployment model.
        # We simply need the main AddModel part.
        self.deploy_model = model_helper.ModelHelper(
            name="mnist_deploy", arg_scope=arg_scope, init_params=False)
        #self.deploy_model.param_init_net.RunAllOnGPU()
        #self.deploy_model.net.RunAllOnGPU()

        self.AddModel(self.deploy_model, "data")

        print("*******train_model.net.Proto()*******\n")
        print(str(self.train_model.net.Proto())[:400] + '\n...')
        print("\n*******train_model.param_init_net.Proto()*******\n")
        print(str(self.train_model.param_init_net.Proto())[:400] + '\n...')

    def RunTraining(self):
        # The parameter initialization network only needs to be run once.
        # Now all the parameter blobs are initialized in the workspace.
        workspace.RunNetOnce(self.train_model.param_init_net)
        #for name in workspace.C.blobs():
        #    blob = workspace.FetchBlob(name)
        #    workspace.FeedBlob(name, blob, device_option=core.DeviceOption(caffe2_pb2.CUDA, 0))

        # Creating an actual network as a C++ object in memory.
        #   We need this as the object is going to be used a lot
        #   so we avoid creating an object every single time it is used.
        # overwrite=True allows you to run this cell several times and avoid errors
        workspace.CreateNet(self.train_model.net, overwrite=True)
        #device_option = self.SetRunGPU(self.train_model)

        # Set the iterations number and track the accuracy & loss
        total_iters = 200
        accuracy = np.zeros(total_iters)
        loss = np.zeros(total_iters)

        # MAIN TRAINING LOOP!
        # Now, we will manually run the network for 200 iterations.
        for i in range(total_iters):
            workspace.RunNet(self.train_model.net)
            accuracy[i] = workspace.blobs['accuracy']
            loss[i] = workspace.blobs['loss']
            # Check the accuracy and loss every so often
            if i % 25 == 0:
                print("Iter: {}, Loss: {}, Accuracy: {}".format(i, loss[i], accuracy[i]))

        # After the execution is done, let's plot the values.
        pyplot.plot(loss, 'b')
        pyplot.plot(accuracy, 'r')
        pyplot.title("Summary of Training Run")
        pyplot.xlabel("Iteration")
        pyplot.legend(('Loss', 'Accuracy'), loc='upper right')

    def VisualizeResults(self):
        ### Let's look at some of the training data
        pyplot.figure()
        pyplot.title("Training Data Sample")
        # Grab the most recent data blob (i.e. batch) from the workspace
        data = workspace.FetchBlob('data')
        # Use visualize module to show the examples from the last batch that was fed to the model
        _ = visualize.NCHW.ShowMultiple(data)

        ### Let's visualize the softmax result
        pyplot.figure()
        pyplot.title('Softmax Prediction for the first image above')
        pyplot.ylabel('Confidence')
        pyplot.xlabel('Label')
        # Grab and visualize the softmax blob for the batch we just visualized. Since batch size
        #  is 64, the softmax blob contains 64 vectors, one for each image in the batch. To grab
        #  the vector for the first image, we can simply index the fetched softmax blob at zero.
        softmax = workspace.FetchBlob('softmax')
        _ = pyplot.plot(softmax[0], 'ro')

    def VisualizeLearnedFilters(self):
        if self.USE_LENET_MODEL:
            pyplot.figure()
            pyplot.title("Conv1 Output Feature Maps for Most Recent Mini-batch")
            # Grab the output feature maps of conv1. Change this to conv2 in order to look into the second one.
            #  Remember, early convolutional layers tend to learn human-interpretable features but later conv
            #  layers work with highly-abstract representations. For this reason, it may be harder to understand
            #  features of the later conv layers.
            conv = workspace.FetchBlob('conv1')

            # We can look into any channel. Think of it as a feature model learned.
            # In this case we look into the 5th channel. Play with other channels to see other features
            conv1 = conv[:, [1], :, :]

            _ = visualize.NCHW.ShowMultiple(conv1)

            pyplot.figure()
            pyplot.title("Conv1 Output Feature Maps for Most Recent Mini-batch")
            conv2 = conv[:, [5], :, :]

            e = visualize.NCHW.ShowMultiple(conv2)

    def TestModel(self):
        # param_init_net here will only create a data reader
        # Other parameters won't be re-created because we selected init_params=False before
        workspace.RunNetOnce(self.test_model.param_init_net)
        workspace.CreateNet(self.test_model.net, overwrite=True)

        # Testing Loop
        test_accuracy = np.zeros(100)
        for i in range(100):
            # Run a forward pass of the net on the current batch
            workspace.RunNet(self.test_model.net)
            # Collect the batch accuracy from the workspace
            test_accuracy[i] = workspace.FetchBlob('accuracy')

        # After the execution is done, let's plot the accuracy values.
        pyplot.figure()
        pyplot.plot(test_accuracy, 'r')
        pyplot.title('Accuracy over test batches.')
        print('test_accuracy: %f' % test_accuracy.mean())

    def DeployModel(self):
        # construct the model to be exported
        # the inputs/outputs of the model are manually specified.
        pe_meta = pe.PredictorExportMeta(
            predict_net=self.deploy_model.net.Proto(),
            parameters=[str(b) for b in self.deploy_model.params],
            inputs=["data"],
            outputs=["softmax"],
        )

        # save the model to a file. Use minidb as the file format
        pe.save_to_db("minidb", os.path.join(self.root_folder, "mnist_model.minidb"), pe_meta)
        print("Deploy model saved to: " + self.root_folder + "/mnist_model.minidb")

    def LoadModel(self):
        # Grab and display the last data batch used before we scratch the workspace. This purely for our convenience...
        blob = workspace.FetchBlob("data")
        pyplot.figure()
        pyplot.title("Batch of Testing Data")
        _ = visualize.NCHW.ShowMultiple(blob)

        # reset the workspace, to make sure the model is actually loaded
        workspace.ResetWorkspace(self.root_folder)

        # verify that all blobs from training are destroyed.
        print("The blobs in the workspace after reset: {}".format(workspace.Blobs()))

        # load the predict net
        predict_net = pe.prepare_prediction_net(os.path.join(self.root_folder, "mnist_model.minidb"), "minidb")

        # verify that blobs are loaded back
        print("The blobs in the workspace after loading the model: {}".format(workspace.Blobs()))

        # feed the previously saved data to the loaded model
        workspace.FeedBlob("data", blob)

        # predict
        workspace.RunNetOnce(predict_net)
        softmax = workspace.FetchBlob("softmax")

        print("Shape of softmax: ", softmax.shape)

        # Quick way to get the top-1 prediction result
        # Squeeze out the unnecessary axis. This returns a 1-D array of length 10
        # Get the prediction and the confidence by finding the maximum value and index of maximum value in preds array
        curr_pred, curr_conf = max(enumerate(softmax[0]), key=operator.itemgetter(1))
        print("Prediction: ", curr_pred)
        print("Confidence: ", curr_conf)

        # the first letter should be predicted correctly
        pyplot.figure()
        pyplot.title('Prediction for the first image')
        pyplot.ylabel('Confidence')
        pyplot.xlabel('Label')
        _ = pyplot.plot(softmax[0], 'ro')

def main():
    parser = argparse.ArgumentParser(
        description="BreakHis: breast cancer classification"
    )
    parser.add_argument("--predict_net", type=str, default='models/squeezenet/predict_net.pb',
                        help="Predict net file (net with operators")
    args = parser.parse_args()

    print(args)
    tr = TrainCaffe()
    tr.PrepareData()
    tr.ConstructModel()
    tr.RunTraining()
    tr.VisualizeResults()
    tr.VisualizeLearnedFilters()
    tr.TestModel()
    tr.DeployModel()
    tr.LoadModel()

if __name__ == '__main__':
    workspace.GlobalInit(['caffe2', '--caffe2_log_level=2'])
    main()


from imutils.perspective import four_point_transform
import imutils
import cv2

# define the dictionary of digit segments so we can identify
# each digit on the thermostat
DIGITS_LOOKUP = {
    (1, 1, 1, 0, 1, 1, 1): 0,
    (0, 0, 1, 0, 0, 1, 0): 1,
    (1, 0, 1, 1, 1, 1, 0): 2,
    (1, 0, 1, 1, 0, 1, 1): 3,
    (0, 1, 1, 1, 0, 1, 0): 4,
    (1, 1, 0, 1, 0, 1, 1): 5,
    (1, 1, 0, 1, 1, 1, 1): 6,
    (1, 0, 1, 0, 0, 1, 0): 7,
    (1, 1, 1, 1, 1, 1, 1): 8,
    (1, 1, 1, 1, 0, 1, 1): 9
}

# load the example image
image = cv2.imread('arabic2.jpg')

# pre-process the image by resizing it, converting it to
# graycale, blurring it, and computing an edge map
image = imutils.resize(image, height=500)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blurred = cv2.GaussianBlur(gray, (5, 5), 0)
edged = cv2.Canny(blurred, 50, 200, 255)

#cv2.imshow("image", edged)
#cv2.waitKey()

# find contours in the edge map, then sort them by their
# size in descending order
cnts = cv2.findContours(edged.copy(), cv2.RETR_EXTERNAL,
                        cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
cnts = sorted(cnts, key=cv2.contourArea, reverse=True)
displayCnt = None

# loop over the contours
for c in cnts:
    # approximate the contour
    peri = cv2.arcLength(c, True)
    approx = cv2.approxPolyDP(c, 0.02 * peri, True)

    # if the contour has four vertices, then we have found
    # the thermostat display
    if len(approx) == 4:
        displayCnt = approx
        break

# extract the thermostat display, apply a perspective transform
# to it
warped = four_point_transform(gray, displayCnt.reshape(4, 2))
output = four_point_transform(image, displayCnt.reshape(4, 2))

thresh1 = 50
im_bw = cv2.threshold(warped, thresh1, 255, cv2.THRESH_BINARY)[1]
cv2.imwrite("outputServer_waterMeter.jpg", im_bw)

#this image goes to newOCR API to be recognized isA


#!/usr/bin/env python
# coding: utf-8

# In[1]:

#encoding=utf-8
import pandas as pd
import numpy as np
import os
import pylab
import matplotlib.pyplot as plt
from scipy import stats, fftpack
import sklearn
import sklearn.metrics
from numpy import square, sqrt, array, arange, mean, var, linspace
from scipy import stats, fftpack
from sklearn import preprocessing

# In[2]:

os.chdir('D:/UNIVERSITY_STUDY/PHM/2nd_term/NASA PHM/IMS/2nd_test/2nd_test/')
file_chdir=os.getcwd()
set1=[]
for root,dirs,files in os.walk(file_chdir):
    for file in files:
        File=pd.read_csv(file,header=None,sep='\s+',engine='python')
        set1.append(File)

# In[3]:

set1[0]

# In[4]:

set_size = 984 #数据集大小

# In[5]:

set2=[]
plt.figure(figsize=(12,5))  #利用快速傅立叶变换将2号轴承时域转化为频域
x2 = np.linspace(0, 1, 12000)
i=0
for i in range(set_size):
    y2 = set1[i][:][1]
    yy = fftpack.fft(y2)
    yf1=abs(fftpack.fft(y2))/len(x2)           #归一化处理
    xf1 = np.arange(len(y2))   # 频率
    set2.append(yf1)
plt.subplot(223)
plt.plot(xf1,yf1,'r')
plt.title('FFT of Mixed wave(normalization)',fontsize=9,color='r')
plt.show()

# In[6]:

set2

# In[7]:

import math
Rs0=1
Rs1=[]
Rs2=[]
Rs3=[]
Rs4=[]
Rs5=[]

def RS_value(data_set, freq1, freq2):
    Rs_val=[]
    fengzi=0
    qq1=0
    qq2=0
    fengmu1=0
    fengmu2=0
    fengmu=0
    for j in range(set_size):
        f0_avg=np.mean(data_set[0])
        f1_avg=np.mean(data_set[j])
        for i in range(freq1,freq2):
            qq1=data_set[0][i]-f0_avg
            qq2=data_set[j][i]-f1_avg
            fengzi=np.abs(fengzi+qq1*qq2)
            fengmu1=fengmu1+qq1*qq1
            fengmu2=fengmu2+qq2*qq2
        fengmu=fengmu1*fengmu2
        Rs_val.append(fengzi/math.sqrt(fengmu1*fengmu2))
    return Rs_val

Rs1=RS_value(set2, 0, 2560)
Rs2=RS_value(set2, 2560, 5120)
Rs3=RS_value(set2, 5120, 7680)
Rs4=RS_value(set2, 7680, 10240)
Rs5=RS_value(set2, 0, 10240)

# In[8]:

#提取2号轴承均方根RMSE---------------1
i = 0
rmse = []
x = range(20480)
xs = range(set_size)
for i in range(set_size):
    y = set1[i][:][1]
    rmse.append(sqrt(sklearn.metrics.mean_squared_error(x,y)))
plt.plot(xs,rmse,'r')
plt.xlabel(u"Time domain-RMSE") #X轴标签
plt.ylabel("RMSE/g") #Y轴标签
plt.show()

#提取2号轴承方差VAR-------------2
i = 0
var1 = []
x = range(20480)
xs = range(set_size)
for i in range(set_size):
    y = set1[i][:][1]
    var1.append(var(y))
plt.plot(xs,var1,'r')
plt.xlabel(u"Time domain-VAR") #X轴标签
plt.ylabel("VAR") #Y轴标签
plt.show()

#提取2号轴承绝对均值-----------------------------3
i = 0
j = 0
mean1 = []
x = range(20480)
xs = range(set_size)
for i in range(set_size):
    y = set1[i][:][1]
    y1 = array([y]).T
    for j in range(20480):
        y1[j]=abs(y1[j])
    mean1.append(mean(y1))
plt.plot(xs,mean1,'r')
plt.xlabel(u"Time domain-mean") #X轴标签
plt.ylabel("mean/g") #Y轴标签
plt.show()

#提取2号轴承峰度-------------------------4
i = 0
j = 0
ku_2 = []
x = range(20480)
xs = range(set_size)
for i in range(set_size):
    y = set1[i][:][1]
    ku_2.append(stats.kurtosis(y))
plt.plot(xs,ku_2,'r')
plt.xlabel(u"Time domain-ku") #X轴标签
plt.ylabel("ku") #Y轴标签
plt.show()

#提取2号轴承偏度----------5
i = 0
j = 0
skew = []
x = range(20480)
xs = range(set_size)
for i in range(set_size):
    y = set1[i][:][1]
    s = pd.Series(y)
    skew.append(s.skew())
plt.plot(xs,skew,'r')
plt.xlabel(u"Time domain-skew") #X轴标签
plt.ylabel("skew") #Y轴标签
plt.show()

#提取2号轴承峰峰值----------------6
i = 0
j = 0
ptp_2 = []
x = range(20480)
xs = range(set_size)
for i in range(set_size):
    y = set1[i][:][1]
    ptp_2.append(max(y) - min(y))
plt.plot(xs,ptp_2,'r')
plt.xlabel(u"Time domain-ptp") #X轴标签
plt.ylabel("peak-to-peak value") #Y轴标签
plt.show()

# In[9]:

#装向量
set3 = [[0] * 6 for i in range(set_size)]
rmse=preprocessing.scale(rmse)
var1=preprocessing.scale(var1)
mean1=preprocessing.scale(mean1)
ku_2=preprocessing.scale(ku_2)
skew=preprocessing.scale(skew)
ptp_2=preprocessing.scale(ptp_2)
for i in range(set_size):
    set3[i][0]=rmse[i]
    set3[i][1]=var1[i]
    set3[i][2]=mean1[i]
    set3[i][3]=ku_2[i]
    set3[i][4]=skew[i]
    set3[i][5]=ptp_2[i]
print(set3)

# In[10]:

#把向量转化为Rs向量
Rst1=RS_value(set3, 0, 6)

plt.plot(xs,Rst1,'r')
plt.xlabel(u"Time domain-RSt") #X轴标签
plt.ylabel("Rst value") #Y轴标签
plt.show()
print(Rst1)

# In[11]:

#开始尝试进行小波包分解

import pywt
import pywt.data

mode = pywt.Modes.smooth
set4=[]

def plot_signal_decomp(data, w, title):
    """Decompose and plot a signal S.
    S = An + Dn + Dn-1 + ... + D1
    """
    w = pywt.Wavelet(w)#选取小波函数
    a = data
    ca = []#近似分量
    for i in range(3):
        (a, d) = pywt.dwt(a, w, mode)#进行3阶离散小波变换
        ca.append(a)

    rec_a = []

    for i, coeff in enumerate(ca):
        coeff_list = [coeff, None] + [None] * i
        rec_a.append(pywt.waverec(coeff_list, w))#重构
    set4.append(rec_a[2])
for i in range(set_size):
    plot_signal_decomp(set1[i][1], 'sym5', "DWT: Ecg sample - Symmlets5")
print(set4)

# In[12]:

print(len(set4[0]))

# In[13]:

#计算能量比
#分为8段：0-2560；2560-5120；5120-7680；7680-10240
#10240-12800;12800-15360;15360-17920;17920-20480
Et=[]
for i in range(set_size):
    E1=0
    for j in range(2560):
        E1=E1+np.abs(set4[i][j])*np.abs(set4[i][j])

    E2=0
    for j in range(2560,5120):
        E2=E2+np.abs(set4[i][j])*np.abs(set4[i][j])

    E3=0
    for j in range(5120,7680):
        E3=E3+np.abs(set4[i][j])*np.abs(set4[i][j])

    E4=0
    for j in range(7680,10240):
        E4=E4+np.abs(set4[i][j])*np.abs(set4[i][j])

    E5=0
    for j in range(10240,12800):
        E5=E5+np.abs(set4[i][j])*np.abs(set4[i][j])

    E6=0
    for j in range(12800,15360):
        E6=E6+np.abs(set4[i][j])*np.abs(set4[i][j])

    E7=0
    for j in range(15360,17920):
        E7=E7+np.abs(set4[i][j])*np.abs(set4[i][j])

    E8=0
    for j in range(17920,20480):
        E8=E8+np.abs(set4[i][j])*np.abs(set4[i][j])
    Ez=E1+E2+E3+E4+E5+E6+E7+E8
    Et.append([E1/Ez,E2/Ez,E3/Ez,E4/Ez,E5/Ez,E6/Ez,E7/Ez,E8/Ez])

print(Et)

# In[14]:

#构建特征集
feature= [[0] * 14 for i in range(set_size)]
for i in range(set_size):
    feature[i][0]=Rs1[i]
    feature[i][1]=Rs2[i]
    feature[i][2]=Rs3[i]
    feature[i][3]=Rs4[i]
    feature[i][4]=Rs5[i]
    feature[i][5]=Rst1[i]
    feature[i][6]=Et[i][0]
    feature[i][7]=Et[i][1]
    feature[i][8]=Et[i][2]
    feature[i][9]=Et[i][3]
    feature[i][10]=Et[i][4]
    feature[i][11]=Et[i][5]
    feature[i][12]=Et[i][6]
    feature[i][13]=Et[i][7]

# In[15]:

feature[0]

# In[16]:

#corr计算
Zcorr=[]
for j in range(14):
    fengmu=0
    fengzi=0
    qq1=0
    qq2=0
    corr=0
    ss1=[]
    #计算第一个特征的corr
    for i in range(set_size):
        ss1.append(feature[i][j])
    Fb=np.mean(ss1)
    Lb=(set_size-1)/2
    for i in range(set_size):
        fengzi=fengzi+np.abs((ss1[i]-Fb)*(i-Lb))
        qq1=qq1+(ss1[i]-Fb)*(ss1[i]-Fb)
        qq2=qq2+(i-Lb)*(i-Lb)
    fengmu=math.sqrt(qq1*qq2)
    corr=fengzi/fengmu
    Zcorr.append(corr)
print(Zcorr)

# In[17]:

#计算Mon

ZMon=[]
for j in range(14):
    ss2=[]
    Zeng=0
    Fu=0
    #计算第一个特征的Mon
    for i in range(set_size):
        ss2.append(feature[i][j])
    for i in range(set_size-1):
        if(ss2[i+1]-ss2[i]>0):
            Zeng=Zeng+1
        if(ss2[i+1]-ss2[i]<0):
            Fu=Fu+1
    Mon=np.abs((Zeng-Fu)/(set_size-1))
    ZMon.append(Mon)
print(ZMon)

# In[18]:

#计算敏感特性的标准Cri=（Corr+Mon）/2
ZCri=[]
for i in range(14):
    ZCri.append((Zcorr[i]+ZMon[i])/2)
print(ZCri)

# In[19]:

#故障标签
data_lable = []
for i in range(set_size):
    data_lable.append((1/(set_size-1))*i)
data_lable

# In[22]:

#最终特征集,基于敏感指标的大小排序，丢弃了Rs3和Rs5这两个特征
feature_fin = [[0] * 13 for i in range(set_size)]
for i in range(set_size):
    feature_fin[i][0] = feature[i][0]
    feature_fin[i][1] = feature[i][1]
    feature_fin[i][2] = feature[i][3]
    feature_fin[i][3] = feature[i][5]
    feature_fin[i][4] = feature[i][6]
    feature_fin[i][5] = feature[i][7]
    feature_fin[i][6] = feature[i][8]
    feature_fin[i][7] = feature[i][9]
    feature_fin[i][8] = feature[i][10]
    feature_fin[i][9] = feature[i][11]
    feature_fin[i][10] = feature[i][12]
    feature_fin[i][11] = feature[i][13]
    feature_fin[i][12] = data_lable[i]

# In[50]:

len(feature_fin)

# In[38]:

#将时间序列数据转化为有监督学习问题
def series_to_supervised(data, n_in=1, n_out=1, dropnan=True):
    n_vars = 1 if type(data) is list else data.shape[1]
    df = pd.DataFrame(data)
    cols, names = list(), list()
    # input sequence (t-n, ... t-1)
    for i in range(n_in, 0, -1):
        cols.append(df.shift(i))
        names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]
    # forecast sequence (t, t+1, ... t+n)
    for i in range(0, n_out):
        cols.append(df.shift(-i))
        if i == 0:
            names += [('var%d(t)' % (j+1)) for j in range(n_vars)]
        else:
            names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]
    # put it all together
    agg = pd.concat(cols, axis=1)
    agg.columns = names
    # drop rows with NaN values
    if dropnan:
        agg.dropna(inplace=True)
    return agg

#转换数据集
values = np.array(feature_fin)
re_feature = series_to_supervised(values)
# drop columns we don't want to predict
re_feature.drop(re_feature.columns[[13,14,15,16,17,18,19,20,21,22,23,24]], axis=1, inplace=True)

# In[51]:

re_feature

# 开始建立预测模型

# In[69]:

#取偶数点和奇数点分别作为训练集和测试集
values = re_feature.values
train_data = []#训练集
valid_data = []#验证集
test_data = []#测试集
for i in range(set_size-1):
    if i%3 == 0:
        valid_data.append(values[i])
    elif i%3 == 1:
        test_data.append(values[i])
    else:
        train_data.append(values[i])
train_data = np.array(train_data)
valid_data = np.array(valid_data)
test_data = np.array(test_data)
train_x, train_y = train_data[:, :-1], train_data[:, -1]
valid_x, valid_y = valid_data[:, :-1], valid_data[:, -1]
test_x, test_y = test_data[:, :-1], test_data[:, -1]
#将训练集和测试集转换成符合LSTM要求的数据格式,即 [样本，时间步，特征]
train_x = train_x.reshape((train_x.shape[0], 1, train_x.shape[1]))
valid_x = valid_x.reshape((valid_x.shape[0], 1, valid_x.shape[1]))
test_x = test_x.reshape((test_x.shape[0], 1, test_x.shape[1]))
print(train_x.shape, train_y.shape, test_x.shape, test_y.shape, valid_x.shape, valid_y.shape)

# In[70]:

#通过keras框架建立LSTM网络模型并训练
from keras.models import Sequential
from keras.layers import Dense, LSTM, Dropout

model = Sequential()
model.add(LSTM(50, activation='relu',input_shape=(train_x.shape[1], train_x.shape[2])))
model.add(Dense(1, activation='linear'))
model.compile(loss='mean_squared_error', optimizer='adam')
# fit network
LSTM = model.fit(train_x, train_y, epochs=100, batch_size=72, validation_data=(valid_x, valid_y), verbose=2, shuffle=False)
# plot history
plt.plot(LSTM.history['loss'], label='train')
plt.plot(LSTM.history['val_loss'], label='test')
plt.legend()
plt.show()

# In[71]:

#带入测试集进行预测
xl=range(328)
plt.figure(figsize=(24,8))
test_predict = model.predict(test_x)
plt.plot(xl,test_y,'r')
plt.plot(xl,test_predict,'b')
plt.show()

# In[ ]:


# Runtime: 36 ms, faster than 99.32% of Python3 online submissions for Minimum Add to Make Parentheses Valid.

class Solution:
    def minAddToMakeValid(self, S):
        """
        :type S: str
        :rtype: int
        """
        # The number of '()
        openCount = 0

        i = 0
        count = 0
        while i < len(S):
            if S[i] == '(':
                openCount += 1
            elif openCount == 0:
                count += 1
            else:
                openCount -= 1
            i += 1

        return count + openCount

Python/Problem greater than 100/Problem 921 - Minimum Add to Make Parentheses Valid.py


# -*- coding: utf-8 -*-
"""
# Talk is cheap,show me the codes!

@Author billie
@Time 2020/7/27 7:14 下午
@Describe
获取系统信息：
1、cpu
2、memory
3、disk
4、net
5、pid
6、sensors

"""
import psutil

class sysInfo():
    def __init__(self):
        self.user = 'billie'
        self.cpu = self.get_cpu()
        self.memory = self.get_memory()
        self.disk = self.get_disk()
        self.net = self.get_net()
        self.pid = self.get_pid()
        self.sensors = self.get_sensors()
        self.others = self.get_others()
        self.allInfo = {'cpu':self.cpu,
                        'memory':self.memory,
                        'disk':self.disk,
                        'net':self.net,
                        'pid':self.pid,
                        'sensors':self.sensors,
                        'othors':self.others
                        }

    def get_cpu(self):
        cpu_dic = dict()
        cpu_dic['cpu_count']=psutil.cpu_count()
        cpu_dic['cpu_percent']=psutil.cpu_percent()
        cpu_dic['cpu_times']=psutil.cpu_times()
        cpu_dic['cpu_times_percent']=psutil.cpu_times_percent()
        cpu_dic['cpu_stats']=psutil.cpu_stats()
        cpu_dic['cpu_freq']=psutil.cpu_freq()
        return cpu_dic

    def get_memory(self):
        memory_dict = dict()
        memory_dict['virtual_memory']=psutil.virtual_memory()
        memory_dict['swap_memory']=psutil.swap_memory()
        return memory_dict

    def get_disk(self):
        disk_dict = dict()
        disk_dict['disk_io_counters']=psutil.disk_io_counters()
        disk_dict['disk_partitions']=psutil.disk_partitions()
        disk_dict['disk_usage']=psutil.disk_usage('/')
        return disk_dict

    def get_net(self):
        net_dict = dict()
        net_dict['net_if_addrs']=psutil.net_if_addrs()
        net_dict['net_if_stats']=psutil.net_if_stats()
        return net_dict

    def get_pid(self):
        pid_dict = dict()
        pid_dict['pids']=psutil.pids()
        return pid_dict

    def get_sensors(self):
        sensors_dict = dict()
        # sensors_dict['sensors_temperatures']=psutil.sensors_temperatures(fahrenheit=False)
        # sensors_dict['sensors_fans']=psutil.sensors_fans()
        # sensors_dict['sensors_battery']=psutil.sensors_battery()
        return sensors_dict
    def get_others(self):
        others_dict = dict()
        others_dict['users'] = psutil.users()
        others_dict['boot_time'] = psutil.boot_time()
        return others_dict