#!/usr/bin/env python3

# ---------------------------------------------------------------------
#  bak_to_git_2.py
# ---------------------------------------------------------------------

import argparse
import csv
import shutil
import subprocess
import sys

from collections import namedtuple
from datetime import datetime
from pathlib import Path

from bak_to_common import ask_to_continue, log_fmt

from btg2_stats import ProgressStats

run_dt = datetime.now()

log_path = Path.cwd() / f"log-bak_to_git_2-{run_dt:%Y%m%d_%H%M%S}.txt"

AppOptions = namedtuple(
    "AppOptions",
    "csv_path, skip_backup, log_dir, run_cmd, stats_file, do_report",
)

def write_log(msg, do_print=False):
    if do_print:
        print(msg)
    with open(log_path, "a") as log_file:
        log_file.write(f"{msg}\n")

def run_compare(run_cmd, left_file, right_file):
    print(f"\nCompare\n  L: {left_file}\n  R: {right_file}\n")

    cmds = [run_cmd, left_file, right_file]

    write_log(f"RUN: {log_fmt(cmds)}")

    result = subprocess.run(
        cmds,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        universal_newlines=True,
    )

    if 0 < len(result.stdout):
        write_log(f"STDOUT: {result.stdout.strip()}")

    assert result.returncode in [0, 1, 2, 11, 12, 13]
    #  bcompare return codes:
    #    Code  Meaning
    #       0  Success
    #       1  Binary same
    #       2  Rules-based same
    #      11  Binary differences
    #      12  Similar
    #      13  Rules-based differences
    #
    # TODO: Look at return codes for other tools (kdiff3, meld).

def get_opts(argv) -> AppOptions:

    ap = argparse.ArgumentParser(
        description="BakToGit Step 2: Run Beyond Compare (bcompare) to "
        + "review source file changes between 'wipbak' backups. This is "
        + "to be used along with manually editing the CSV file to set "
        + "the 'SKIP_Y' and 'COMMIT_MESSAGE' columns."
    )

    ap.add_argument(
        "input_csv",
        action="store",
        help="Path to CSV file created by bak_to_git_1.py.",
    )

    ap.add_argument(
        "--skip-backup",
        dest="skip_backup",
        action="store_true",
        help="Do not create a backup of the input CSV file. By default, a "
        + "backup copy is created at the start of a session.",
    )

    ap.add_argument(
        "--log-dir",
        dest="log_dir",
        action="store",
        help="Output directory for log files.",
    )

    ap.add_argument(
        "--compare-cmd",
        dest="run_cmd",
        default="bcompare",
        action="store",
        help="Alternate executable command to launch a file comparison "
        + "tool. The tool must take the names of two files to compare as "
        + "the first two command-line arguments. The default is 'bcompare' "
        + "(Beyond Compare by Scooter Software).",
    )

    ap.add_argument(
        "--report",
        dest="do_report",
        action="store_true",
        help="Print a progress report to the console instead of doing an "
        + "interactive session.",
    )

    ap.add_argument(
        "--stats-file",
        dest="stats_file",
        type=str,
        action="store",
        help="Name of the file in which to store data used for progress "
        + "estimates when running with the --report option.",
    )

    args = ap.parse_args(argv[1:])

    #  Also skip backup if running the progress (stats) report.
    skip_bak = args.skip_backup or args.do_report

    opts = AppOptions(
        Path(args.input_csv),
        skip_bak,
        args.log_dir,
        args.run_cmd,
        args.stats_file,
        args.do_report,
    )

    if not (opts.csv_path.exists() and opts.csv_path.is_file()):
        sys.stderr.write(f"ERROR: File not found '{opts.csv_path}'")
        sys.exit(1)

    if opts.log_dir is not None:
        if not Path(opts.log_dir).exists():
            sys.stderr.write(
                f"ERROR: Log directory not found '{opts.log_dir}'"
            )
            sys.exit(1)

    return opts

def get_rename(add_command):
    """
    Checks add_command for "RENAME: <old_name>".
    Returns the value of old_name if there is a match.
    Otherwise returns an empty string.
    """
    if add_command is None or len(add_command) == 0:
        return ""
    s = add_command.strip()
    if s.lower().startswith("rename:"):
        old_name = s.split(":")[1].strip().strip('"').strip("'")
        assert 0 < len(old_name)
        return old_name
    else:
        return ""

def process_row(run_cmd, row, prevs, stats: ProgressStats):
    print(f"Row sort_key = '{row['sort_key']}'")
    base_name = row["base_name"]
    no_msg = len(row["COMMIT_MESSAGE"]) == 0

    #  If the 'SKIP_Y' field is blank, then rows with an
    #  existing commit message are skipped.

    #  Setting 'SKIP_Y' to 'Y' will skip the comparison
    #  regardless of the commit message.
    if row["SKIP_Y"].lower() == "y":
        stats.count_skip()
        return True

    if not no_msg:
        stats.count_commit()

    #  If this session is for statistics report then skip the interaction.
    if stats.do_report:
        return True

    #  Setting 'SKIP_Y' to 'N' will run the comparison
    #  regardless of the commit message.
    no_skip = row["SKIP_Y"].lower() == "n"

    if no_msg or no_skip:
        base_rename = get_rename(row["ADD_COMMAND"])
        if 0 < len(base_rename):
            prev_key = base_rename
            print(f"\nRENAME: '{base_rename}'")
        else:
            prev_key = base_name

        if prev_key in prevs.keys():
            run_compare(run_cmd, prevs[prev_key], row["full_name"])
        else:
            if len(row["prev_full_name"]) == 0:
                print(f"\nNew file: {base_name}")
                prevs[base_name] = row["full_name"]
                return True
            else:
                warning = "UNEXPECTED PREVIOUS VERSION"
                print(f"\n{warning}: {base_name}")
                run_compare(run_cmd, row["prev_full_name"], row["full_name"])

        answer = ask_to_continue(
            "(k = Keep left file for next comparison) Continue [Y,n,k]? ",
            ["y", "n", "k", ""],
        )

        if answer == "n":
            print("\nStopping.\n")
            return False

        if answer == "k":
            print("\nKeeping previous Left file for comparison.\n")
            stats.log_act("skip")
        else:
            stats.log_act("commit")
            prevs[base_name] = row["full_name"]
            print("")
    else:
        prevs[base_name] = row["full_name"]
    return True

def main(argv):
    now_tag = datetime.now().strftime("%y%m%d_%H%M%S")

    opts = get_opts(argv)

    global log_path
    if opts.log_dir is not None:
        log_path = (
            Path(opts.log_dir).expanduser().resolve().joinpath(log_path.name)
        )

    write_log(f"BEGIN at {run_dt:%Y-%m-%d %H:%M:%S}")

    stats = ProgressStats(
        opts.stats_file, save_immediate=True, is_reporting=opts.do_report
    )
    stats.start_session(opts.csv_path)

    if opts.do_report:
        stats.load()

    if not opts.skip_backup:
        #  Make a backup of the source csv file in case there are problems
        #  while manually editing the file.

        bak_suffix = f".{now_tag}.bak"

        bak_path = opts.csv_path.with_name(
            f"z-{opts.csv_path.name}"
        ).with_suffix(bak_suffix)

        assert not bak_path.exists()

        print(f"\nSaving backup as '{bak_path.name}'\n")
        write_log(f"BACKUP: '{bak_path}'")
        shutil.copyfile(opts.csv_path, bak_path)

    write_log(f"READ: '{opts.csv_path}'")

    prevs = {}
    with open(opts.csv_path) as csv_file:
        reader = csv.DictReader(csv_file)
        for row in reader:
            if len(row["sort_key"]) > 0:
                stats.count_row()
                if not process_row(opts.run_cmd, row, prevs, stats):
                    break

    stats.stop_session()
    stats.save()

    if opts.do_report:
        rpt = stats.report()
        print(rpt)
        write_log(rpt)
    else:
        print("Done (bak_to_git_2.py).")

    write_log(f"END at {datetime.now():%Y-%m-%d %H:%M:%S}")

if __name__ == "__main__":
    sys.exit(main(sys.argv))


from collections import deque
import numpy as np
import random
import heapq

class MaxHeapObj(object):
    def __init__(self, val):
        self.val = val

    def __lt__(self, other):
        return self.val > other.val

    def __eq__(self, other):
        return self.val == other.val

    def __str__(self):
        return str(self.val)

class MinHeap(object):
    def __init__(self):
        self.h = []

    def heappush(self, x):
        heapq.heappush(self.h, x)

    def heappop(self):
        return heapq.heappop(self.h)

    def __getitem__(self, i):
        return self.h[i]

    def __len__(self):
        return len(self.h)

    def remove(self, v):
        self.h.remove(v)

    def heaptop(self):
        if len(self.h) > 0:
            return self.h[0]
        else:
            return None

class MaxHeap(MinHeap):
    def heappush(self, x):
        heapq.heappush(self.h, MaxHeapObj(x))

    def heappop(self):
        return heapq.heappop(self.h).val

    def __getitem__(self, i):
        return self.h[i].val

    def remove(self, v):
        self.h.remove(MaxHeapObj(v))

class ReplayBuffer:
    def __init__(self, size, gamma):
        # init deque
        self.memory = deque(maxlen=size)
        # init heaps
        self.goodReward = MinHeap()
        self.badReward = MaxHeap()

        self.gamma = gamma

    def remember(self, state, action, reward, next_state):
        element = (reward, state, action, next_state)

        if len(self.memory) == self.memory.maxlen:
            vOut = self.memory[0]
            # remove oldest deque element from heaps
            if vOut >= self.goodReward.heaptop():
                self.goodReward.remove(vOut)
            else:
                self.badReward.remove(vOut)

        # add element to the relevant heap
        if (len(self.goodReward) > 0) and (element >= self.goodReward.heaptop()):
            self.goodReward.heappush(element)
        else:
            self.badReward.heappush(element)

        # balance heaps
        if abs(len(self.goodReward) - len(self.badReward)) > 1:
            if len(self.goodReward) > len(self.badReward):
                vOut = self.goodReward.heappop()
                self.badReward.heappush(vOut)
            else:
                vOut = self.badReward.heappop()
                self.goodReward.heappush(vOut)

        # add element to deque
        self.memory.append(element)

    def replay(self, actorMainModel, actorTargetModel, actorTrainFunc, actorWolpertingerFunc, actorUpdateEpsilonFunc,
               criticMainModel, criticTargetModel, criticGradientsFunc, stateNormalizeFunc, updateTargetModelsParams,
               trainSetSize):
        # Sample trainSet from  memory
        # trainSet = random.sample(self.memory, min(trainSetSize, len(self.memory)))
        trainSet = random.sample(self.goodReward.h, min(int(trainSetSize / 2), len(self.goodReward)))
        tempSet = random.sample(self.badReward.h, min(trainSetSize - len(trainSet), len(self.badReward)))
        trainSet.extend([t.val for t in tempSet])
        del tempSet

        trainState = []
        trainAction = []
        trainNextState = []
        trainReward = []
        for i, (state, action, reward, next_state) in enumerate(trainSet):
            trainState.append(stateNormalizeFunc(state))
            trainAction.append(action)
            trainNextState.append(stateNormalizeFunc(next_state))
            trainReward.append(reward)
        # TODO: check trainAction contains action vectors rather than action ID

        trainState = np.array(trainState)
        trainAction = np.array(trainAction)
        trainReward = np.expand_dims(np.array(trainReward), axis=1)
        trainNextState = np.array(trainNextState)

        # Calculate targets
        # # predict actor target model next state preferred continuous action
        # action = actorTargetModel.predict(trainNextState)
        # predict actor target model next state preferred discrete action
        action, _, _ = actorWolpertingerFunc(trainNextState, actorTargetModel, criticTargetModel)
        # TODO: it is not clear from paper if critic model should be MAIN or TARGET ???
        # predict critic target model (nextState, nextAction) Q value, i.e. future reward
        criticTargetPrediction = criticTargetModel.predict([trainNextState, action])
        # update future **discounted** reward
        target = trainReward + (self.gamma * criticTargetPrediction)

        # train (update) critic train model
        loss = criticMainModel.train_on_batch([trainState, trainAction], target)
        # train (update) actor train model
        actions_for_grad = actorMainModel.predict(trainState)
        grads = criticGradientsFunc(trainState, actions_for_grad)
        actorTrainFunc(trainState, grads)

        # update actor & critic target models weights
        updateTargetModelsParams()

        # update epsilon value
        actorUpdateEpsilonFunc()

        return loss


from core.util import Tokenrun; import sys

def run_intepreter(files):
    if (len(sys.argv) > 1):
        if (not files.endswith(".jv")):
            print("That is not the compitable file, try another file with (.jv) ext")
        else:
            with open(files, 'r') as f:
                data = f.read()
                result, error = Tokenrun(files, data)
                if error:
                    print(error.as_string())
                if result:
                    if len(result.elements) == 1:
                        print(repr(result.elements[0]))
                    else:
                        print(repr(result))

def run_from_gui(valueN, value):
    result, error = Tokenrun(valueN, value)
    if error:
        print(error.as_string())
    if result:
        if len(result.elements) == 1:
            print(repr(result.elements[0]))
        else:
            print(repr(result))

if __name__ == "__main__":
    try:
        run_intepreter(sys.argv[1])
    except Exception as err:
        print(f"Error: {err}")


import numpy as np
from deploy.data_io.data_metaclass import DataIO

class BasicDataIO(DataIO):
    def preprocess(self, input_data):
        # return numpy array of shape [1, T, C, H, W]
        outdata = []
        for image in input_data:
            image = np.transpose(image, (2, 0, 1))  # HWC to CHW
            image = image.astype(np.float32) / 255.0
            image = np.expand_dims(image, axis=0)
            outdata.append(image)
        outdata = np.concatenate(outdata)
        outdata = np.expand_dims(outdata, axis=0)
        return outdata

    def postprocess(self, input_data):
        outdata = []
        img = input_data.squeeze(axis=0).astype(np.float32).clip(0, 1)
        img = (img * 255.0).round().astype(np.uint8)
        if len(img.shape) == 4:
            img = np.transpose(img, (0, 2, 3, 1))  # TCHW to THWC
            for i in range(img.shape[0]):
                outdata.append(img[i, ...].squeeze())
        elif len(img.shape) == 3:
            img = np.transpose(img, (1, 2, 0))  # CHW to HWC
            outdata.append(img)
        return outdata


import shutil
from pathlib import Path
import argparse

def main(args):
    get_size = lambda path: sum(f.stat().st_size for f in path.glob('**/*') if f.is_file())

    root_paths = Path(args.root_dir).glob('*')

    for root_path in root_paths:
        for path in root_path.glob('*'):
            if not path.is_dir(): continue
            kb_size = get_size(path) / (1024 ** 2)

            if kb_size < 200.:
                shutil.rmtree(path)
                print(f"{path.parent.name}/{path.name}: {kb_size:.4f} kb (deleted)")
            else:
                print(f"{path.parent.name}/{path.name}: {kb_size:.4f} kb")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='root path')

    parser.add_argument('--root-dir', type=str,
                        default='/workspace/logs/cassava-leaf-disease-classification/',
                        help='root log directory')

    args = parser.parse_args()
    main(args)


from server import add_new_element_to_store
from server import delete_element_from_store
from server import modify_element_in_store

import unittest
class TestServer(unittest.TestCase):
	def test_add_new_element_to_store(self):
		status=add_new_element_to_store(2,"hi")
		self.assertTrue(status)
	def test_delete_element_from_server(self):
		status=delete_element_from_store(2)
		self.assertTrue(status)
	def test_modify_element_from_store(self):
		status=modify_element_in_store(2,"Hello")
		self.assertTrue(status)
if __name__ == '__main__':
	unittest.main()


from django.shortcuts import render
from .models import Nodes,Data,SearchData,Commodity,Order
from rest_framework import mixins,status,filters,viewsets
from .serializers import NodesSerializer,DataSerializer,SearchDataSerializer,OrderSerializer,CommoditySerializer
from rest_framework.pagination import PageNumberPagination
from django_filters.rest_framework import DjangoFilterBackend
from .filters import NodesFilter
from .mqtt import *
from rest_framework.decorators import action
from rest_framework.response import Response
from .CountModelMixin import CountModelMixin
from django.db.models import Count,F
from rest_framework.decorators import api_view
from itertools import chain
import time
import json
import datetime

# Create your views here.
class DataListViewSet(viewsets.ModelViewSet, CountModelMixin):
    """
    接口允许被查看和修改
    """
    queryset = Data.objects.all().order_by('-recordTime')
    serializer_class = DataSerializer
    filter_backends = (filters.SearchFilter, DjangoFilterBackend,)
    search_fields = ('nodeId', 'recordTime', 'val', 'safe', 'unit')
    filter_fields = ('nodeId', 'recordTime', 'val', 'safe', 'unit')

    """
    根据时间段返回list数据
    """
    @action(detail=False)
    def get(self, request, *args, **kwargs):
        queryset = self.filter_queryset(self.get_queryset())
        start_date = self.request.query_params.get('start_date', None)
        end_date = self.request.query_params.get('end_date', None)
        if start_date or end_date:
            queryset = queryset.filter(recordTime__range=[start_date, end_date])
            serializer = DataSerializer(queryset, many=True)
            return Response(serializer.data)

    """
    根据时间返回list数据
    """
    @action(detail=False)
    def getByTimescale(self, request, *args, **kwargs):
        timescale = self.request.query_params.get('timescale', None)
        num = self.request.query_params.get('num', None)
        if timescale == 'month':
            queryset = Data.objects.filter(recordTime__month=num)
        elif timescale == 'year':
            queryset = Data.objects.filter(recordTime__year=num)
        serializer = DataSerializer(queryset, many=True)
        return Response(serializer.data)

    """
    根据时间尺度分割，并返回相应的数据，其中type有2种类型，默认为data，然后就是count
    """
    @action(detail=False)
    def segmentData(self, request, *args, **kwargs):
        timescale = self.request.query_params.get('timescale', None)
        num = self.request.query_params.get('num', None)
        type = self.request.query_params.get('type', None)
        nodeId = self.request.query_params.get('nodeId', None)
        result = []
        if timescale == 'year':
            queryset = Data.objects.filter(recordTime__year=num)
            if nodeId!='all' and nodeId:
                queryset = queryset.filter(nodeId=nodeId)
            if type == 'count':
                for i in range(1,13):
                    result.insert(i, {'x': str(i)+'月', 'y': len(DataSerializer(queryset.filter(recordTime__month=i), many=True).data)})
            else:
                for i in range(1,13):
                    result.insert(i, {str(i)+'月': DataSerializer(queryset.filter(recordTime__month=i),many=True).data})
        elif timescale == 'month':
            queryset = Data.objects.filter(recordTime__year=2019,recordTime__month=num)
            if nodeId!='all' and nodeId:
                queryset = Data.objects.filter(nodeId=nodeId)
            if type == 'count':
                for i in range(1,32):
                    result.insert(i, {'x': str(i)+'号', 'y': len(DataSerializer(queryset.filter(recordTime__day=i), many=True).data)})
            else:
                for i in range(1,32):
                    result.insert(i, {str(i)+'号': DataSerializer(queryset.filter(recordTime__day=i),many=True).data})
        return Response(result)

    """
    根据时间尺度返回安全率数值   type有三个值  count、safe、unsafe
    """
    @action(detail=False)
    def segmentSafe(self, request, *args, **kwargs):
        timescale = self.request.query_params.get('timescale', None)
        type = self.request.query_params.get('type', None)
        nodeId = self.request.query_params.get('nodeId', None)
        result = []
        if timescale == 'year':
            last_range = datetime.datetime.now() - datetime.timedelta(days=365)
            safeQueryset = Data.objects.filter(recordTime__gte=last_range,safe=True)
            unsafeQueryset = Data.objects.filter(recordTime__gte=last_range,safe=False)
            queryset = Data.objects.filter(recordTime__gte=last_range)
            if nodeId!= 'all' and nodeId:
                safeQueryset=safeQueryset.filter(nodeId=nodeId)
                unsafeQueryset=unsafeQueryset.filter(nodeId=nodeId)
                queryset=queryset.filter(nodeId=nodeId)
            if type == 'count':
                for i in range(1,13):
                    allNum = queryset.filter(recordTime__month=i).count()
                    temp = safeQueryset.filter(recordTime__month=i).count()
                    y=temp/allNum*100 if allNum != 0 else 0
                    result.insert(i,{'x': str(i)+'月','y': y})
            elif type == 'safe':
                for i in range(1,13):
                    result.insert(i, {str(i)+'月': DataSerializer(safeQueryset.filter(recordTime__month=i),many=True).data})
            else:
                for i in range(1,13):
                    result.insert(i, {str(i)+'月': DataSerializer(unsafeQueryset.filter(recordTime__month=i),many=True).data})
        elif timescale == 'month':
            last_range = datetime.datetime.now() - datetime.timedelta(days=31)
            safeQueryset = Data.objects.filter(recordTime__gte=last_range,safe=True)
            unsafeQueryset = Data.objects.filter(recordTime__gte=last_range,safe=False)
            queryset = Data.objects.filter(recordTime__gte=last_range)
            if nodeId!= 'all' and nodeId:
                safeQueryset=safeQueryset.filter(nodeId=nodeId)
                unsafeQueryset=unsafeQueryset.filter(nodeId=nodeId)
                queryset=queryset.filter(nodeId=nodeId)
            if type == 'count':
                for i in range(1,32):
                    allNum = queryset.filter(recordTime__day=i).count()
                    temp = safeQueryset.filter(recordTime__day=i).count()
                    y=temp/allNum*100 if allNum != 0 else 0
                    result.insert(i,{'x': str(i)+'号','y': y})
            elif type == 'safe':
                for i in range(1,32):
                    result.insert(i, {str(i)+'号': DataSerializer(safeQueryset.filter(recordTime__day=i),many=True).data})
            else:
                for i in range(1,32):
                    result.insert(i, {str(i)+'号': DataSerializer(unsafeQueryset.filter(recordTime__day=i),many=True).data})
        elif timescale == 'today':
            last_range = datetime.datetime.now() - datetime.timedelta(days=1)
            safeQueryset = Data.objects.filter(recordTime__gte=last_range,safe=True)
            unsafeQueryset = Data.objects.filter(recordTime__gte=last_range,safe=False)
            queryset = Data.objects.filter(recordTime__gte=last_range)
            if nodeId!= 'all' and nodeId:
                safeQueryset=safeQueryset.filter(nodeId=nodeId)
                unsafeQueryset=unsafeQueryset.filter(nodeId=nodeId)
                queryset=queryset.filter(nodeId=nodeId)
            if type == 'count':
                for i in range(1,25):
                    allNum = queryset.filter(recordTime__hour=i).count()
                    temp = safeQueryset.filter(recordTime__hour=i).count()
                    y=temp/allNum*100 if allNum != 0 else 0
                    result.insert(i,{'x': str(i)+'点','y': y})
            elif type == 'safe':
                for i in range(1,25):
                    result.insert(i, {str(i)+'点': DataSerializer(safeQueryset.filter(recordTime__hour=i),many=True).data})
            else:
                for i in range(1,25):
                    result.insert(i, {str(i)+'点': DataSerializer(unsafeQueryset.filter(recordTime__hour=i),many=True).data})
        return Response(result)

    """
    返回三天的图标，用于linechart表格的展示

    order_by  '-' 从大到小,不加则为从小到大
    """
    @action(detail=False)
    def lineChartData(self, request, *args, **kwargs):
        nodeId = self.request.query_params.get('nodeId', None)
        today_range = datetime.datetime.now() - datetime.timedelta(days=1)
        yesterday_range = datetime.datetime.now() - datetime.timedelta(days=2)
        today_queryset = Data.objects.filter(nodeId=nodeId,recordTime__range=[today_range,datetime.datetime.now()])
        yesterday_queryset = Data.objects.filter(nodeId=nodeId,recordTime__range=[yesterday_range,today_range])
        today_count = today_queryset.count()
        yesterday_count = yesterday_queryset.count()
        today_safe = today_queryset.filter(safe=True).values(x=F('recordTime'),y1=F('val'))
        today_unsafe = today_queryset.filter(safe=False).values(x=F('recordTime'),y2=F('val'))
        yesterday_safe = yesterday_queryset.filter(safe=True).values(x=F('recordTime'),y1=F('val'))
        yesterday_unsafe = yesterday_queryset.filter(safe=False).values(x=F('recordTime'),y2=F('val'))
        result= [{'name':'今天','cvr':today_count},{'name':'昨天','cvr':yesterday_count}]
        for i in today_safe:
            i['y2']=0
        for i in today_unsafe:
            i['y1']=0
        for i in yesterday_safe:
            i['y2']=0
        for i in yesterday_unsafe:
            i['y1']=0
        today = list(chain(today_safe,today_unsafe))
        yesterday = list(chain(yesterday_safe,yesterday_unsafe))
        #today = today_safe | today_unsafe
        #yesterday = yesterday_safe | yesterday_unsafe
        result[0]['shop']=sorted(today, key=lambda x:x['x'])
        result[1]['shop']=sorted(yesterday, key=lambda x:x['x'])
        return Response(result)

    @action(detail=False)
    def realTimeSafe(self, request, *args, **kwargs):
        """实时安全率

        根据时间尺度返回实时安全率数据
        gte greater      lt  less then

        Args:
            timescale: 时间尺度,可为hour,day,week,month,year

        Returns:
            返回实时安全率数值，以dict的形式

            {'safeRate': 15}
        """
        timescale = self.request.query_params.get('timescale',None)
        last_range = ''
        if timescale == 'hour':
            last_range = datetime.datetime.now() - datetime.timedelta(hours=1)
        elif timescale == 'day':
            last_range = datetime.datetime.now() - datetime.timedelta(days=1)
        elif timescale == 'week':
            last_range = datetime.datetime.now() - datetime.timedelta(weeks=7)
        elif timescale == 'month':
            last_range = datetime.datetime.now() - datetime.timedelta(days=30)
        else:
            last_range = datetime.datetime.now() -datetime.timedelta(days=365)
        safeNum = Data.objects.filter(recordTime__gte=last_range,safe=True).count()
        allNum = Data.objects.filter(recordTime__gte=last_range).count()
        result= round(safeNum/allNum*100,2) if allNum!=0 else 0
        return Response({'safeRate': result})

    @action(detail=False)
    def countByTimescale(self, request, *args, **kwargs):
        """根据时间返回count值

        timescale可为年月类型，然后根据查询的时间数字，返回需要的count数据

        Args:
            timescale: 时间尺度可为年月
            num: 查询时间

        Returns:
            返回一个dict类型，count数值
            示例：

            {'count': 12}
        """
        timescale = self.request.query_params.get('timescale', None)
        num = self.request.query_params.get('num', None)
        count = 0
        if timescale == 'month':
            count = Data.objects.filter(recordTime__year=2019,recordTime__month=num).count()
        elif timescale == 'year':
            count = Data.objects.filter(recordTime__year=num).count()
        elif timescale == 'today':
            last_range = datetime.datetime.now() - datetime.timedelta(days=1)
            count = Data.objects.filter(recordTime__gte=last_range).count()
        elif timescale == 'hour':
            last_range = datetime.datetime.now() - datetime.timedelta(hours=1)
            count = Data.objects.filter(recordTime__gte=last_range).count()
        return Response({'count': count})

    """
    返回数据收集排名榜，做个名字加数字总和
    """
    @action(detail=False)
    def countRank(self, request, *args, **kwargs):
        limit = self.request.query_params.get('limit', None)
        type = self.request.query_params.get('type', None)
        result= []
        if type == 'data':
            queryset = Nodes.objects.annotate(num=Count('data__nodeId')).order_by('-num')[:int(limit)]
            for i in queryset:
                result.append({'title': i.node_name,'total': i.num})
        elif type == 'safe':
            queryset = Nodes.objects.annotate(num=Count('data__nodeId')).order_by('-num')[:int(limit)]
            safeQueryset = Nodes.objects.filter(data__safe=True).annotate(num=Count('data__nodeId')).order_by('-num')[:int(limit)]
            for i in range(len(safeQueryset)):
                safeRate=safeQueryset[i].num/queryset[i].num*100 if queryset[i].num!=0 else 0
                result.append({'title': queryset[i].node_name,'total': safeRate})
        return Response(result)

    """
    返回指定id的所有count数据
    """
    @action(detail=False)
    def getCount(self, request, *args, **kwargs):
        nodeId = self.request.query_params.get('nodeId', None)
        totalCount = Data.objects.filter(nodeId = nodeId).count()
        daily_range = datetime.datetime.now() - datetime.timedelta(days=1)
        dailyCount = Data.objects.filter(nodeId = nodeId, recordTime__gte=daily_range).count()
        return Response({'totalCollect': totalCount,'dailyCollect': dailyCount})

    """
    返回指定id的safe数据
    """
    @action(detail=False)
    def getSafeCount(self, request, *args, **kwargs):
        nodeId = self.request.query_params.get('nodeId', None)
        totalCount = Data.objects.filter(nodeId=nodeId).count()
        totalSafeCount = Data.objects.filter(nodeId=nodeId, safe=True).count()
        daily_range = datetime.datetime.now() - datetime.timedelta(days=1)
        dailySafeCount = Data.objects.filter(nodeId=nodeId, recordTime__gte=daily_range).count()
        totalSafe = round(totalSafeCount / totalCount *100,2) if totalCount!= 0 else 0
        dailySafe = round(dailySafeCount / totalCount *100,2) if totalCount!=0 else 0
        return Response({'totalSafe': totalSafe,'dailySafe': dailySafe})

class OrderListViewSet(viewsets.ModelViewSet, CountModelMixin):
    """
    接口说明
    """
    queryset = Order.objects.all().order_by('-amount')
    serializer_class = OrderSerializer
    filter_backends = (filters.SearchFilter, DjangoFilterBackend,)
    search_fields = ('id', 'commodityId', 'amount')
    filter_fields = ('id', 'commodityId', 'amount')

class CommodityListViewSet(viewsets.ModelViewSet, CountModelMixin):
    """
    接口说明
    """
    queryset = Commodity.objects.all().order_by('-y')
    serializer_class = CommoditySerializer
    filter_backends = (filters.SearchFilter, DjangoFilterBackend,)
    search_fields = ('id', 'x', 'type','y')
    filter_fields = ('id', 'x', 'type','y')

    @action(detail=False)
    def getSalesTypeData(self, request, *args, **kwargs):
        queryset = Commodity.objects.values('x','y','type')
        return Response(queryset)

class SearchDataListViewSet(viewsets.ModelViewSet, CountModelMixin):
    """
    接口说明
    """
    queryset = SearchData.objects.all().order_by('-keyword')
    serializer_class = SearchDataSerializer
    filter_backends = (filters.SearchFilter, DjangoFilterBackend,)
    search_fields = ('keyword', 'count', 'status')
    filter_fields = ('keyword', 'count', 'status')

    @action(detail=False,methods=['POST'])
    def addHistory(self, request, *args, **kwargs):
        searchName = self.request.data.get('keyword')
        try:
            searchHistory = SearchData.objects.filter(keyword=searchName)
            if searchHistory.count() > 0:
                count = SearchData.objects.get(keyword=searchName).count + 1
                searchHistory.update(count=count)
                return Response({'msg:':'add history count successfully'},status=status.HTTP_200_OK)
            else:
                searchHistory = SearchData(keyword=searchName)
                searchHistory.save()
                return Response({'msg': 'add history data successfully'},status= status.HTTP_200_OK)
        except:
            searchHistory = SearchData(keyword=searchName)
            searchHistory.save()
            return Response({'msg': searchHistory},status= status.HTTP_200_OK)

 #           return Response({'msg':'error'},status= status.HTTP_400_BAD_REQUEST)

# 可选用的模型mixins.ListModelMixin, viewsets.GenericViewSet 自定义型
class NodesListViewSet(viewsets.ModelViewSet, CountModelMixin):
    """
    接口说明
    """
    queryset = Nodes.objects.all().order_by('-node_type')
    serializer_class = NodesSerializer
    filter_backends = (filters.SearchFilter, DjangoFilterBackend,)
    search_fields = ('node_name', 'node_type', 'minVal', 'maxVal', 'adcode','nodeId','subscribe')
    filter_fields = ('node_name', 'node_type', 'minVal', 'maxVal', 'adcode','nodeId','subscribe')

    @action(detail=True,methods=['POST'])
    def subscribe(self,request,*args,**kwargs):
        nodeId = self.request.data.get('nodeId')
        subscribe =self.request.data.get('subscribe')
        try:
            subscribeNode =Nodes.objects.get(nodeId=nodeId)
            subscribeNode.subscribe=subscribe
            subscribeNode.save()
            topics=list(Nodes.objects.filter(subscribe=True).values_list('nodeId',flat=True))
            for i,item in enumerate(topics):
                topics[i] = ("application/1/device/"+topics[i]+"/rx",0)
            topics.append(("gateway/+/rx",0))
            print(topics)
            #client.subscribe([("gateway/+/rx",0),("application/1/device/+/rx",0)])
            client.subscribe(topics)
            print("subscribing done")
            #client.loop_stop()
            #time.sleep(5)
            client.loop_start()
            return Response({'msg':'successfully change'},status=status.HTTP_200_OK)
        except:
            return Response({'msg':'error with nodeId'},status=status.HTTP_404_NOT_FOUND)

    @action(detail=True,methods=['POST'])
    def test(self,request,*args,**kwargs):
        tx = self.request.data.get('tx')
        nodeId = self.request.data.get('nodeId')
        msg=''
        topic= "application/1/device/"+nodeId+"/tx"
        print("tx with:"+topic)
        if tx == True:
            msg={"reference":"abcd34","confirmed":False,"fPort":10,"data":"MQEAAQE="}
        #else:
        #    msg={"reference":"abcd1234","confirmed":False,"fPort":10,"data":"MQEAAQA="}
        str_msg=json.dumps(msg)
        client.publish(topic,str_msg,0)
        node = Nodes.objects.get(nodeId=nodeId)
        node.tx=tx
        node.save()
        return Response({'msg':'tx change successfully'},status=status.HTTP_200_OK)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jul  1 14:58:18 2020

@author: jaisi8631
"""

# ------------------
# REGULAR IMPORTS
# ------------------
import pickle
import pandas as pd
import numpy as np
import math
from scipy import signal
import matplotlib.pyplot as plt

from sklearn.metrics import confusion_matrix
from sklearn.metrics import precision_recall_fscore_support
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import StandardScaler
from sklearn.utils import class_weight
from sklearn import decomposition
from sklearn.model_selection import GridSearchCV

import keras
from keras.utils.vis_utils import plot_model
from keras.models import Model
from keras.optimizers import Adam
from keras.models import load_model
from keras.callbacks import ModelCheckpoint
from keras.callbacks import EarlyStopping
from keras.wrappers.scikit_learn import KerasClassifier

# ------------------
# LSTM IMPORTS
# ------------------
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM

# ------------------
# CNN IMPORTS
# ------------------
from keras.optimizers import SGD
from keras.layers import Reshape
from keras.layers import Conv1D
from keras.layers import InputLayer
from keras.layers import Dropout
from keras.layers import Flatten
from keras.layers import MaxPooling1D
from keras.layers import GlobalAveragePooling1D

# ------------------
# CONSTANTS
# ------------------
START_TIME_INTERICTAL = 407898590000
END_TIME_INTERICTAL = 407998350000
START_TIME_ICTAL = 416039606029
END_TIME_ICTAL = 416112464960
FS = 1024
DOWN_SAMPLE_FACTOR = 10
STEP_SIZE = 256
SEQUENCE_LEN = 1024
SEQUENCE_PCA = 1000

TRAIN_SIZE = 0.8
TEST_SIZE = 0.1
VAL_SIZE = 0.1

EPOCHS = 5
BS = 128
LR = 0.01
NUM_CLASSES = 2
MOMENTUM = 0.1
DECAY = 1e-6

GS_EPOCHS = [1, 2]
GS_BS = [32, 64]
GS_OPTIMIZERS = ['adam', 'rmsprop']

ADAM_DEFAULT = 'adam'
SGD_DEFAULT = 'sgd'
ADAM_CUSTOM = Adam(lr = LR)
SGD_CUSTOM = SGD(lr = LR, momentum = MOMENTUM, decay = DECAY, nesterov = True)

PATH_INTERICTAL = "../datasets/hup138-interictal.pickle"
PATH_ICTAL = "../datasets/hup138-ictal.pickle"

# ------------------
# LOSS HISTORY
# ------------------
class LossHistory(keras.callbacks.Callback):
    def on_train_begin(self, logs={}):
        self.history = {'loss':[],'acc':[]}

    def on_batch_end(self, batch, logs={}):
        self.history['loss'].append(logs.get('loss'))
        self.history['acc'].append(logs.get('acc'))

# ------------------
# LOAD DATA
# ------------------
def get_data(path):
    with open(path, 'rb') as f: data, fs = pickle.load(f)
    labels = pd.read_csv("labels/hup138-labels.csv", header = None)
    labels_list = labels[0].tolist()
    data = data[data.columns.intersection(labels_list)]
    return data

def iEEG_data_filter(data, fs, cutoff1, cutoff2, notch):
	column_names = data.columns
	data = np.array(data)
	number_of_channels = data.shape[1]

    # Cut-off frequency of the filter
	fc = np.array([cutoff1, cutoff2])
    # Normalize the frequency
	w = fc / np.array([(fs / 2), (fs / 2)])

	b, a = signal.butter(4, w, 'bandpass')
	filtered = np.zeros(data.shape)
	for i in np.arange(0,number_of_channels):
		filtered[:,i] = signal.filtfilt(b, a, data[:,i])
	filtered = filtered + (data[0] - filtered[0])
	f0 = notch
	q = 30
	b, a = signal.iirnotch(f0, q, fs)
	notched = np.zeros(data.shape)
	for i in np.arange(0, number_of_channels):
		notched[:, i] = signal.filtfilt(b, a, filtered[:, i])
	notched_df = pd.DataFrame(notched, columns=column_names)
	return notched_df

# ------------------
# DATASET OUTLINE
# ------------------
def create_timestamps(data):
    timestamps = []
    for i in range(0, data.shape[0]):
        timestamps.append(START_TIME_ICTAL + (i * FS))
    return timestamps

# ------------------
# PROCESS DATA
# ------------------
def create_merge_dataset(data, split_point, start_time_interictal,
                         end_time_interictal, start_time_ictal, end_time_ictal):
    dataset = []
    dataset_targets = []
    labels = pd.read_csv("labels/hup138-labels.csv", header = None)

    for column in data:

        print("Reading data for electrode " + column)

        col_list = data[column].tolist()
        col_data = labels.loc[labels[0] == column]

        col_start_time = col_data.iat[0, 1]
        col_end_time = col_data.iat[0, 2]

        if(col_start_time == '-' or col_end_time == '-'):
            col_start_time = int(start_time_ictal + 1)
            col_end_time = int(start_time_ictal + 1)
        else:
            col_start_time = int(col_start_time)
            col_end_time = int(col_end_time)

        # first process interictal data
        for index in range(SEQUENCE_LEN, split_point, STEP_SIZE):
            sequence = col_list[(index - SEQUENCE_LEN) : index]
            sequence = [[i] for i in sequence]
            dataset.append(sequence)

            sequence_end_time = start_time_interictal + (index * FS * 10)

            if(sequence_end_time >= col_start_time and
               sequence_end_time <= col_end_time):
                dataset_targets.append(1)
            else:
                dataset_targets.append(0)

        # then process ictal data
        for index in range(SEQUENCE_LEN, data.shape[0] - split_point, STEP_SIZE):
            new_index = index + split_point
            sequence = col_list[(new_index - SEQUENCE_LEN) : new_index]
            sequence = [[i] for i in sequence]
            dataset.append(sequence)

            sequence_end_time = start_time_ictal + (index * FS * 10)

            if(sequence_end_time >= col_start_time and
               sequence_end_time <= col_end_time):
                dataset_targets.append(1)
            else:
                dataset_targets.append(0)

    dataset = np.array(dataset)
    dataset_targets = np.array(dataset_targets)

    return dataset, dataset_targets

def scale_data(dataset):
    scaler = StandardScaler()
    dataset = scaler.fit_transform(dataset.reshape(-1, dataset.shape[-1])).reshape(dataset.shape)
    return dataset

def apply_pca(dataset, components):
    pca = decomposition.PCA(n_components = components)
    pca.fit(components)
    dataset = pca.transform(dataset)
    return dataset

def create_class_weights(y_train):
    class_weights = class_weight.compute_class_weight('balanced',
                                                      classes = np.unique(y_train),
                                                      y = y_train)
    class_weight_dict = dict(enumerate(class_weights))
    return class_weight_dict

# ------------------
# LSTM MODEL
# ------------------
def create_lstm_model():
    model = Sequential()
    model.add(LSTM(256, input_shape = (SEQUENCE_LEN, 1), dropout = 0.2, recurrent_dropout = 0.2, return_sequences = True))
    model.add(LSTM(32, dropout = 0.2, recurrent_dropout = 0.2, return_sequences = True))
    model.add(LSTM(32, return_sequences = False))
    model.add(Dense(1, activation = 'sigmoid'))
    print(model.summary())
    plot_model(model, to_file = 'figures/lstm_diagram.png',
               show_shapes = True, show_layer_names = True)
    return model

def train_lstm_model(model, model_name, x_train, y_train, x_val, y_val):
    chk = ModelCheckpoint('models/' + model_name + '.pkl',
                          monitor = 'val_accuracy',
                          save_best_only = True,
                          mode = 'max',
                          verbose = 1)
    lstm_history = LossHistory()
    callbacks_list = [
        chk,
        lstm_history
    ]

    model.compile(loss = 'binary_crossentropy',
                  optimizer = ADAM_CUSTOM,
                  metrics = ['accuracy'])

    class_weights = create_class_weights(y_train)

    model.fit(x_train, y_train,
              class_weight = class_weights,
              epochs = EPOCHS,
              batch_size = BS,
              callbacks = callbacks_list,
              validation_data = (x_val, y_val))

    return model, lstm_history

# ------------------
# CNN MODEL
# ------------------
def create_custom_cnn_model():
    model = Sequential()
    model.add(Conv1D(500, 100, activation='relu', input_shape = (SEQUENCE_LEN, 1)))
    model.add(Dropout(0.5))
    model.add(MaxPooling1D(3))
    model.add(Conv1D(100, 10, activation='relu'))
    model.add(Dropout(0.2))
    model.add(Conv1D(100, 10, activation='relu'))
    model.add(Dropout(0.2))
    model.add(Conv1D(10, 10, activation='relu'))
    model.add(GlobalAveragePooling1D())
    model.add(Dropout(0.5))
    model.add(Dense(1, activation = 'sigmoid'))
    print(model.summary())
    plot_model(model, to_file = 'figures/cnn_custom_diagram.png',
               show_shapes = True, show_layer_names = True)
    return model

def create_wavenet_cnn_model():
    model = Sequential()
    model.add(Conv1D(1, 100, activation='relu', input_shape = (SEQUENCE_LEN, 1)))
    for rate in (1, 2, 4, 8) * 2:
        model.add(Conv1D(filters = 20, kernel_size = 2, padding = 'causal',
                         activation = 'relu', dilation_rate = rate))
        model.add(Dropout(0.1))
    model.add(Conv1D(filters = 10, kernel_size = 1))
    model.add(GlobalAveragePooling1D())
    model.add(Dense(1, activation = 'sigmoid'))
    print(model.summary())
    plot_model(model, to_file = 'figures/cnn_wavenet_diagram.png',
               show_shapes = True, show_layer_names = True)
    return model

def train_cnn_model(model, model_name, x_train, y_train, x_val, y_val):
    chk = ModelCheckpoint('models/' + model_name + '.pkl',
                          monitor = 'val_accuracy',
                          save_best_only = True,
                          mode = 'max',
                          verbose = 1)
    cnn_history = LossHistory()
    callbacks_list = [
        chk,
        cnn_history
    ]

    model.compile(loss = 'binary_crossentropy',
                  optimizer = ADAM_CUSTOM,
                  metrics = ['accuracy'])

    class_weights = create_class_weights(y_train)

    model.fit(x_train, y_train,
              class_weight = class_weights,
              batch_size = BS,
              epochs = EPOCHS,
              callbacks = callbacks_list,
              validation_data = (x_val, y_val))

    return model, cnn_history

# ------------------
# GRID SEARCH
# ------------------
def lstm_gridsearch(optimizer = 'adam'):
    model = Sequential()
    model.add(LSTM(256, input_shape = (SEQUENCE_LEN, 1), dropout = 0.2, recurrent_dropout = 0.2, return_sequences = True))
    model.add(LSTM(32, dropout = 0.2, recurrent_dropout = 0.2, return_sequences = True))
    model.add(LSTM(32, return_sequences = False))
    model.add(Dense(1, activation = 'sigmoid'))
    model.compile(loss = 'binary_crossentropy', optimizer = optimizer, metrics = ['accuracy'])
    return model

def cnn_gridsearch(optimizer = 'adam'):
    model = Sequential()
    model.add(Conv1D(500, 100, activation='relu', input_shape = (SEQUENCE_LEN, 1)))
    model.add(Dropout(0.5))
    model.add(MaxPooling1D(3))
    model.add(Conv1D(100, 10, activation='relu'))
    model.add(Dropout(0.2))
    model.add(Conv1D(10, 10, activation='relu'))
    model.add(GlobalAveragePooling1D())
    model.add(Dropout(0.5))
    model.add(Dense(1, activation = 'sigmoid'))
    model.compile(loss = 'binary_crossentropy', optimizer = optimizer, metrics = ['accuracy'])
    return model

def build_gridsearch(build_function, x_train, y_train, model_name):
    grid = {'epochs': GS_EPOCHS,
            'batch_size': GS_BS,
            'optimizer': GS_OPTIMIZERS}
    load_model = KerasClassifier(build_fn = build_function)

    # train using GridSearch
    validator = GridSearchCV(load_model,
                             param_grid = grid,
                             scoring = 'accuracy',
                             n_jobs = 1,
                             verbose = 10)
    validator.fit(x_train, y_train)

    # retain best model
    print('Best Parameters: ')
    model = validator.best_estimator_.model
    print(validator.best_params_)
    model.save('models/' + model_name + '.pkl')

    return model

# ------------------
# EVALUATE MODEL
# ------------------
def plot_batch_losses(history, plot_name):
    y1 = history.history['loss']
    y2 = history.history['acc']

    y3 = []
    for index, value in enumerate(y1):
        if value > 100:
            y3.append(index)
    for i in reversed(y3):
        del y1[i]

    x1 = np.arange(len(y1))
    fig, ax = plt.subplots()
    line1, = ax.plot(x1, y1, label = 'loss')
    ax.set_xlabel('Batch')
    ax.set_ylabel('Loss')
    plt.savefig('figures/' + plot_name + '.png')
    plt.show()

def model_acc(model_name):
    pickle_name = 'models/' + model_name + '.pkl'
    model = load_model(pickle_name)
    y_preds = model.predict_classes(x_test)
    acc = accuracy_score(y_test, y_preds)
    cm = confusion_matrix(y_test, y_preds)
    scores = precision_recall_fscore_support(y_test, y_preds, average = 'macro')
    return acc, cm, scores

# ------------------
# MAIN METHOD
# ------------------
if __name__=="__main__":

    # --------------------
    # DATA WRANGLING
    # --------------------
    # interictal
    data = get_data(PATH_INTERICTAL)
    timestamps = create_timestamps(data)
    data_filtered = iEEG_data_filter(data, FS, 0.16, 200, 60)
    fs_downSample = FS / DOWN_SAMPLE_FACTOR
    data_filtered_tmp = signal.decimate(data_filtered, DOWN_SAMPLE_FACTOR, axis = 0)
    data_filtered = pd.DataFrame(data_filtered_tmp, columns = data_filtered.columns); del data_filtered_tmp
    data_interictal = data_filtered

    # ictal
    data = get_data(PATH_ICTAL)
    timestamps = create_timestamps(data)
    data_filtered = iEEG_data_filter(data, FS, 0.16, 200, 60)
    fs_downSample = FS / DOWN_SAMPLE_FACTOR
    data_filtered_tmp = signal.decimate(data_filtered, DOWN_SAMPLE_FACTOR, axis = 0)
    data_filtered = pd.DataFrame(data_filtered_tmp, columns = data_filtered.columns); del data_filtered_tmp
    data_ictal = data_filtered

    # concatenate
    data = pd.concat([data_interictal, data_ictal], ignore_index = True)
    dataset, dataset_targets = create_merge_dataset(data, data_interictal.shape[0],
                                              START_TIME_INTERICTAL, END_TIME_INTERICTAL,
                                              START_TIME_ICTAL, END_TIME_ICTAL)

    # --------------------
    # DATASET PROCESSING
    # --------------------
    # optional preprocessing methods
    """
    dataset = scale_data(dataset)
    dataset = apply_pca(dataset, SEQUENCE_PCA)
    """

    # random split:
    """
    x_train, x_test, y_train, y_test = train_test_split(dataset,
                                                        dataset_targets,
                                                        test_size = 0.2)
    x_test, x_val, y_test, y_val = train_test_split(x_test,
                                                    y_test,
                                                    test_size = 0.5)
    print("Number of seizing instances in training set: " +
          str(np.count_nonzero(y_train == 1)))
    print("Number of non-seizing instances in training set: " +
          str(np.count_nonzero(y_train == 0)))
    print("Number of seizing instances in testing set: " +
          str(np.count_nonzero(y_test == 1)))
    print("Number of non-seizing instances in testing set: " +
          str(np.count_nonzero(y_test == 0)))
    """

    # per-electrode split:
    rows, cols, depth = dataset.shape
    num_train = int(TRAIN_SIZE * rows)
    num_test = int(TEST_SIZE * rows)
    num_val = int(VAL_SIZE * rows)

    x_train = dataset[ : num_train, :]
    x_test = dataset[num_train : num_train + num_test, :]
    x_val = dataset[num_train + num_test : , :]

    y_train = dataset_targets[ : num_train]
    y_test = dataset_targets[num_train : num_train + num_test]
    y_val = dataset_targets[num_train + num_test : ]
    print("Number of seizing instances in training set: " +
          str(np.count_nonzero(y_train == 1)))
    print("Number of non-seizing instances in training set: " +
          str(np.count_nonzero(y_train == 0)))
    print("Number of seizing instances in testing set: " +
          str(np.count_nonzero(y_test == 1)))
    print("Number of non-seizing instances in testing set: " +
          str(np.count_nonzero(y_test == 0)))

    # --------------------
    # OPTION 1: Models
    # --------------------
    # train wavenet CNN
    cnn_model = create_wavenet_cnn_model()
    cnn_model_name = 'eeg-model-cnn-wavenet'
    cnn_model, cnn_history = train_cnn_model(cnn_model, cnn_model_name,
                                 x_train, y_train, x_val, y_val)
    plot_batch_losses(cnn_history, 'cnn-wavenet-history')

    # test wavenet CNN
    cnn_acc, cnn_cm, cnn_scores = model_acc(cnn_model_name)
    print("CNN WaveNet Test Set Accuracy: ")
    print("%.4f" % round(cnn_acc, 4))
    print("CNN WaveNet Test Set Confusion Matrix: ")
    print(cnn_cm)

    """
    # train LSTM
    lstm_model = create_lstm_model()
    lstm_model_name = 'eeg-model-lstm'
    lstm_model, lstm_history = train_lstm_model(lstm_model, lstm_model_name,
                                  x_train, y_train, x_val, y_val)
    plot_batch_losses(lstm_history, 'lstm-history')

    # test LSTM
    lstm_acc, lstm_cm, lstm_scores = model_acc(lstm_model_name)
    print("LSTM Test Set Accuracy: ")
    print("%.4f" % round(lstm_acc, 4))
    print("LSTM Test Set Confusion Matrix: ")
    print(lstm_cm)

    # train custom CNN
    cnn_model = create_custom_cnn_model()
    cnn_model_name = 'eeg-model-cnn'
    cnn_model, cnn_history = train_cnn_model(cnn_model, cnn_model_name,
                                 x_train, y_train, x_val, y_val)
    plot_batch_losses(cnn_history, 'cnn-history')

    # test CNN
    cnn_acc, cnn_cm, cnn_scores = model_acc(cnn_model_name)
    print("CNN Test Set Accuracy: ")
    print("%.4f" % round(cnn_acc, 4))
    print("CNN Test Set Confusion Matrix: ")
    print(cnn_cm)
    """

    # --------------------
    # OPTION 2: GridSearch
    # --------------------
    """
    # train and test LSTM
    gs_lstm_name = 'egg-lstm-gridsearch'
    gs_lstm = build_gridsearch(lstm_gridsearch, x_train, y_train, gs_lstm_name)
    gs_lstm_acc = model_acc(gs_lstm_name)
    print("GridSearch LSTM Test Set Accuracy: ")
    print("%.4f" % round(gs_lstm_acc, 4))

    # train and test CNN
    gs_cnn_name = 'eeg-cnn-gridsearch'
    gs_cnn = build_gridsearch(cnn_gridsearch, x_train, y_train, gs_cnn_name)
    gs_cnn_acc = model_acc(gs_cnn_name)
    print("GridSearch CNN Test Set Accuracy: ")
    print("%.4f" % round(gs_cnn_acc, 4))
    """


from turtle import Turtle
ALIGN = "center"
FONT = ("Arial", 15, "normal")

class ScoreBoard(Turtle):

    def __init__(self):
        super().__init__()
        self.score = 0
        with open("data.txt") as file:
            self.highest_score = int(file.read())
        self.color("white")
        self.penup()
        self.goto(0, 270)
        self.hideturtle()
        self.show_score()

    def show_score(self):
        self.write(f"Score: {self.score}  Highest Score: {self.highest_score}", align=ALIGN, font=FONT)

    def reset_score(self):
        if self.score > self.highest_score:
            self.highest_score = self.score
            with open("data.txt", mode="w") as file:
                file.write(f"{self.highest_score}")

        self.score = 0
        self.clear()
        self.show_score()

    def increase_score(self):
        self.score += 1
        self.clear()
        self.show_score()


import os
from datetime import datetime
from users import *

def log_conversation(user_input, jarvis_response, log_dir='./Jarvis_Memory/conversation_history'):
    current_date = datetime.now().strftime("%Y-%m-%d")
    current_time = datetime.now().strftime("%H:%M:%S")

    log_file = f"{current_date}_conversation.log"
    log_file_path = os.path.join(log_dir, log_file)

    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    with open(log_file_path, 'a', encoding='utf-8') as f:
        f.write("---- " + current_time + " ----\n")
        f.write(user_manager.current_user.name + ": " + user_input + "\n")
        f.write("Jarvis: " + jarvis_response + "\n\n")

def read_conversation(date, log_dir):
    log_file = f"{date}_conversation.log"
    log_file_path = os.path.join(log_dir, log_file)

    if not os.path.exists(log_file_path):
        return f"No conversation history found for {date}."

    with open(log_file_path, 'r', encoding='utf-8') as f:
        return f.read()


# -*- coding: utf-8 -*-

import os
import json
import copy
import random

from rich import print as rprint
from faker import Faker
from s3pathlib import S3Path, context
from mpire import WorkerPool
import pandas as pd
import awswrangler as wr

from learn_aws_glue_etl_programming.config import config
from learn_aws_glue_etl_programming.boto_ses import boto_ses

context.attach_boto_session(boto_ses)

s3path_prefix = S3Path(config.s3path_prefix, "ds02_nested_parquet")

fake = Faker()

n_files = 3
n_rows_per_file = 1000
n_message_lower = 1
n_message_upper = 3
n_tag_lower = 0
n_tag_upper = 5

def create_one(nth_file: int):
    start_id = 1 + (nth_file - 1) * n_rows_per_file
    end_id = start_id + n_rows_per_file
    data = [
        {
            "id": _id,
            "messages": [
                f"message {id + 1}"
                for id in range(random.randint(n_message_lower, n_message_upper))
            ],
            "tags": [
                {"key": "Name", "value": fake.name()}
                for _ in range(random.randint(n_tag_lower, n_tag_upper))
            ]
        }
        for _id in range(start_id, end_id)
    ]
    for dct in data:
        dct["data"] = copy.deepcopy(dct)

    df = pd.DataFrame(data, columns=["id", "messages", "tags", "data"])
    s3path = S3Path(s3path_prefix, f"{str(nth_file).zfill(3)}.parquet")
    print(f"dump to {s3path.console_url}")
    wr.s3.to_parquet(
        df,
        path=s3path.uri,
        index=False,
        dataset=False,
    )

    return data

def create_dataset():
    print(f"--- creating dataset {s3path_prefix.basename} ---")
    kwargs = [
        {"nth_file": ith_file}
        for ith_file in range(1, 1 + n_files)
    ]
    with WorkerPool(n_jobs=4) as pool:
        pool.map(create_one, kwargs)

if __name__ == "__main__":
    # create_one(nth_file=1)
    create_dataset()
    pass


# Definition for singly-linked list.
class ListNode:
    def __init__(self, x):
        self.val = x
        self.next = None

class Solution:

    # 기본적인 아이디어는 두 개의 리스트를 연결했을 때
    # a+b와 b+a의 길이는 같아지기 때문에 비교를 하기 편해진다.
    # 또한 이렇게 둘을 연결했을 때 어디서 교차지점이 생기는지 확인하기 편하다.
    #
    # 예시1
    # A + B : 41845561845
    # B + A : 56184541845
    def getIntersectionNode(self, headA: ListNode, headB: ListNode) -> Optional[ListNode]:

        # a와 b는 리스트의 시작점
        a = headA
        b = headB

        # a == b일 경우 둘 다 같은 노드에 있다는 것
        #  1. 해당 위치가 교차지점임을 의미. 해당 위치를 반환한다. (a 또는 b 둘 다 가능)
        #  2. 둘 다 null값을 가지는 경우로 교차지점을 찾지 못하고 마지막 노드까지 왔다는 것을 의미.
        #     이 경우 a 또는 b는 null 값을 보유하고 있기 때문에 해당 값을 return하면 된다.
        while a != b:
            # 만약 a가 null이라면(a의 끝에 도달했다면) b의 시작점을 가리켜야 한다.
            # 그렇지 않다면 그 다음을 가리킨다.
            if a != None:
                a = a.next
            else:
                a = headB

            if b != None:
                b = b.next
            else:
                b = headA

        return a

    # Hash table 개념을 활용하지만 딕셔너리가 아닌 set을 활용한 풀이(Hash Set)
    def getIntersectionNode(self, headA: ListNode, headB: ListNode) -> Optional[ListNode]:

        hash_set = set()

        while headA != None:
            hash_set.add(headA)
            headA = headA.next
        while headB != None:
            if headB in hash_set:
                return headB
            headB = headB.next
        return None


import re
from collections import Counter

class Ingredient:
    quantity = None
    material = None

    def __init__(self, quantity, material):
        self.quantity = quantity
        self.material = material

    def get_material(self):
        return self.material

    def get_quantity(self):
        return self.quantity

class Reaction:
    inputs = []
    output = None
    INGREDIENT_PATTERN = r'[0-9]+ [A-Z]+'

    def __init__(self, string):
        (inputs, output) = string.split(' => ')
        inputs = re.findall(self.INGREDIENT_PATTERN, inputs)
        output = re.findall(self.INGREDIENT_PATTERN, output)[0]

        (output_quantity, output_material) = output.split(' ')
        self.output = Ingredient(int(output_quantity), output_material)

        inputs = [
            input.split(' ') for input in inputs
        ]
        self.inputs = [
            Ingredient(int(input[0]), input[1]) for input in inputs
        ]

    def get_output(self):
        return self.output

    def get_inputs(self):
        return self.inputs

    def consume_ingredient(self, ingredient, available_materials):
        if ingredient in self.inputs:
            available_materials.update({
                ingredient.get_material(): -1 * ingredient.get_quantity()
            })
        else:
            raise Exception(
                'Unrecognized ingredient: {} {}'.format(ingredient.get_quantity(), ingredient.get_material())
            )

    def produce(self, available_materials):
        available_materials.update({
            self.output.get_material(): self.output.get_quantity()
        })

class Production:
    available_reaction_dict = None
    available_materials = None
    raw_material = None
    raw_material_count = None

    def __init__(self, reactions, raw_material):
        self.available_reaction_dict = {
            reaction.get_output().get_material(): reaction
            for reaction in reactions
        }
        self.raw_material = raw_material
        self.available_materials = Counter()
        self.raw_material_count = 0

    def get_raw_material_count(self):
        return self.raw_material_count

    def get_available_material_count(self):
        return sum(self.available_materials.values())

    def get_available_materials(self):
        return self.available_materials

    def reset(self):
        self.available_materials = Counter()
        self.raw_material_count = 0

    def add_or_consume_material(self, quantity, material):
        self.available_materials.update({
            material: quantity
        })

    def multiply_all_available_materials(self, multiplier):
        self.available_materials = {
            key: value * multiplier
            for key, value in self.available_materials.items()
        }

    def produce_material(self, quantity, material, allow_raw_material_production=True):
        quantity_produced = 0
        while quantity_produced < quantity:
            reaction = self.available_reaction_dict[material]
            inputs = reaction.get_inputs()
            for ingredient in inputs:
                ingredient_quantity = ingredient.get_quantity()
                ingredient_material = ingredient.get_material()
                if self.available_materials[ingredient_material] < ingredient_quantity:
                    if ingredient_material == self.raw_material:
                        if allow_raw_material_production:
                            self.available_materials.update({
                                ingredient_material: ingredient_quantity
                            })
                            self.raw_material_count += ingredient_quantity
                        else:
                            return False
                    else:
                        success = self.produce_material(
                            ingredient_quantity - self.available_materials[ingredient_material],
                            ingredient_material,
                            allow_raw_material_production=allow_raw_material_production
                        )
                        if not success:
                            return False

                reaction.consume_ingredient(ingredient, self.available_materials)

            reaction.produce(self.available_materials)
            quantity_produced += reaction.get_output().get_quantity()

        return True

FUEL = 'FUEL'
ORE = 'ORE'
TOTAL_ORE_COUNT = 1000000000000

with open('input.txt', 'r') as f:
    reaction_strings = f.readlines()

reactions = [
    Reaction(reaction_string) for reaction_string in reaction_strings
]

# Part 1
p = Production(reactions, ORE)
p.produce_material(1, FUEL)
ore_to_produce_one_fuel = p.get_raw_material_count()
print(ore_to_produce_one_fuel)

# Part 2
min_fuel_produced = TOTAL_ORE_COUNT // ore_to_produce_one_fuel
p.reset()
p.produce_material(min_fuel_produced, FUEL)
print(p.get_raw_material_count())

# Attempt 2:
# leftover_ore = TOTAL_ORE_COUNT % ore_to_produce_one_fuel
# p.multiply_all_available_materials(min_fuel_produced)
# p.add_or_consume_material(leftover_ore, ORE)
#
# additional_fuel_produced = 0
# while True:
#     success = p.produce_material(1, FUEL, allow_raw_material_production=False)
#     if not success:
#         break
#     additional_fuel_produced += 1
#     print(additional_fuel_produced)
# print(min_fuel_produced)
# print(min_fuel_produced + additional_fuel_produced)

# Attempt 1:
# p = Production(reactions, ORE)
# fuel_per_period = 0
# while True:
#     p.produce_material(1, FUEL)
#     p.add_or_consume_material(-1, FUEL)
#     fuel_per_period += 1
#     if p.get_available_material_count() == 0:
#         break
#
# ore_per_period = p.get_raw_material_count()
# number_of_periods = (ORE_COUNT // ore_per_period)
# fuel_produced = number_of_periods * fuel_per_period
# leftover_ores = ORE_COUNT % ore_per_period
#
# p.reset()
# leftover_fuel_produced = 0
# while True:
#     p.produce_material(1, FUEL)
#     if p.raw_material_count > leftover_ores:
#         break
#     leftover_fuel_produced += 1
#
# fuel_produced += leftover_fuel_produced
# print(fuel_produced)


import pygame
import random

pygame.init()

black = (0, 0, 0)
blue = (65, 167, 198)
red = (255, 0, 0)
orange = (249, 102, 63)
grey = (109, 107, 107)
width, height = 600, 400

game_display = pygame.display.set_mode((width, height))

pygame.display.set_caption('Sanik Game')

snake_size = 10
snake_speed = 15

font_style = pygame.font.SysFont('ubuntu', 30)
'''def message(msg,color):
    mesg = font_style.render(msg, True, color)
    game_display.blit(mesg, [width/2, height/2])'''

clock = pygame.time.Clock()

def our_snake(snake_size, snake_List, color):
    if color==orange:
        for x in snake_List:
            pygame.draw.rect(game_display, color, [x[0], x[1], snake_size, snake_size])
    elif color==blue:
        for x in snake_List:
            pygame.draw.rect(game_display, color, [x[0], x[1], snake_size, snake_size])

#pygame.draw.rect(game_display, orange, [fuudx, fuudy, snake_size, snake_size])
#ygame.draw.rect(game_display, blue, [foodx, foody, snake_size, snake_size])

def main_game():

    global color,y,snake_speed
   # global foodx,foody,fuudx,fuudy

    foodx = round(random.randrange(0, width - snake_size) / 10.0) * 10.0
    foody = round(random.randrange(0, height - snake_size) / 10.0) * 10.0

    fuudx = round(random.randrange(0, width - snake_size) / 10.0) * 10.0
    fuudy = round(random.randrange(0, height - snake_size) / 10.0) * 10.0

    food_color = ['blue', 'orange']

    color=blue
    game_over = False
    game_close = False
    x1 = width / 2
    y1 = height / 2

    x1_new = 0
    y1_new = 0

    snake_List = []
    Length_of_snake = 1

    x = random.randrange(0, 2)
    y = food_color[x]

    while not game_over:
        while game_close == True:
            game_display.fill(black)

            mesg = font_style.render('You Lost! or Press R-Play Again or Press Q-Quit', True, grey)
            game_display.blit(mesg, [width / 11, height / 3])
            pygame.display.update()
            for event in pygame.event.get():
                if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_q:
                        game_over = True
                        game_close = False
                    elif event.key == pygame.K_r:
                        main_game()
                if event.type == pygame.QUIT:
                    pygame.quit()
                    quit()

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                game_over = True
            if event.type == pygame.KEYDOWN:
                '''if Length_of_snake>3:
                    print(Length_of_snake)

                    if snake_List[0][0] > snake_List[1][0]:# moving left
                        if event.key == pygame.K_DOWN:
                            x1_new = 0
                            y1_new = 10
                        elif event.key == pygame.K_LEFT:
                            x1_new = -10
                            y1_new = 0
                        elif event.key == pygame.K_UP:
                            x1_new = 0
                            y1_new = -10
                        elif event.key == pygame.K_RIGHT:
                             pass

                    elif snake_List[0][0] < snake_List[1][0]:# moving right
                        if event.key == pygame.K_DOWN:
                            x1_new = 0
                            y1_new = 10
                        elif event.key == pygame.K_RIGHT:
                            x1_new = 10
                            y1_new = 0
                        elif event.key == pygame.K_UP:
                            x1_new = 0
                            y1_new = -10
                        elif event.key == pygame.K_LEFT:
                            pass

                    elif snake_List[0][1] < snake_List[1][1]:# moving up
                        if event.key == pygame.K_DOWN:
                            pass
                        elif event.key == pygame.K_LEFT:
                            x1_new = -10
                            y1_new = 0
                        elif event.key == pygame.K_RIGHT:
                            x1_new = 10
                            y1_new = 0
                        elif event.key == pygame.K_UP:
                            pass

                    elif snake_List[0][1] > snake_List[1][1]:# moving down
                        if event.key == pygame.K_UP:
                            x1_new = 0
                            y1_new = 10
                        elif event.key == pygame.K_LEFT:
                            x1_new = -10
                            y1_new = 0
                        elif event.key == pygame.K_RIGHT:
                            x1_new = 10
                            y1_new = 0
                        elif event.key == pygame.K_DOWN:
                            pass
                else:'''
                if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_LEFT:
                            x1_new = -10
                            y1_new = 0
                    elif event.key == pygame.K_RIGHT:
                            x1_new = 10
                            y1_new = 0
                    elif event.key == pygame.K_UP:
                            x1_new = 0
                            y1_new = -10
                    elif event.key == pygame.K_DOWN:
                            x1_new = 0
                            y1_new = 10

        if x1 >= width or x1 < 0 or y1 > height or y1 < 0:
            game_close = True
            print('Your score:', Length_of_snake-1)

        x1 += x1_new
        y1 += y1_new
        game_display.fill(black)

        if y == 'orange':
            orange_snake=pygame.draw.rect(game_display, orange, [fuudx, fuudy, snake_size, snake_size])
        elif y=='blue':
            pygame.draw.rect(game_display, blue, [foodx, foody, snake_size, snake_size])

        snake_Head = []
        snake_Head.append(x1)
        snake_Head.append(y1)
        snake_List.append(snake_Head)
        if len(snake_List) > Length_of_snake:
            del snake_List[0]

        '''for x in snake_List[:-1]:
            if x[0] == snake_Head[0] and x[1]==snake_Head[1]:
                print(x, snake_Head)
                game_close = True

            if x == snake_Head:
                print('final hit', x, snake_Head)
                game_close = True
                print('Your score:', Length_of_snake-1)'''
        if x1 == foodx and y1 == foody:
            print(x1, y1)
            foodx = round(random.randrange(0, width - snake_size) / 10.0) * 10.0
            foody = round(random.randrange(0, height - snake_size) / 10.0) * 10.0
            x = random.randrange(0, 2)
            y = food_color[x]
            Length_of_snake += 1
            color=blue

        elif x1 == fuudx and y1 == fuudy:
            if snake_List[0][0]> snake_List[1][0]:
                x1_new = 10                             #moving left
                y1_new = 0
                x1 += x1_new
                y1 += y1_new
            elif snake_List[0][0]< snake_List[1][0]:
                x1_new = -10                            #moving right
                y1_new = 0
                x1 += x1_new
                y1 += y1_new
            elif snake_List[0][1]< snake_List[1][1]:
                x1_new = 0                              #moving up
                y1_new = -10
                x1 += x1_new
                y1 += y1_new
            elif snake_List[0][1]> snake_List[1][1]:
                x1_new = 0                               #moving down
                y1_new = 10
                x1 += x1_new
                y1 += y1_new
            print('orange food',x1, y1)
            fuudx = round(random.randrange(0, width - snake_size) / 10.0) * 10.0
            fuudy = round(random.randrange(0, height - snake_size) / 10.0) * 10.0
            print(snake_List)
            Length_of_snake += 1
            color = orange

            x = random.randrange(0, 2)
            y=food_color[x]

        our_snake(snake_size, snake_List, color)
        pygame.display.update()

        clock.tick(15)

    pygame.quit()
    quit()

main_game()


# -*- coding: utf-8 -*-
# __author__ = "qingchen.zhang"
# 20190908

import os
import json
from biocluster.core.exceptions import OptionError
from biocluster.module import Module

class GenePredictGffModule(Module):
    """
    细菌比较基因组seq+gff序列预测模块
    分析内容：编码基因预测、tRNA预测、rRNA预测，及其相关信息统计
    这部分的基因预测不再区分染色体和质粒，因为主要是基因预测的软件不同，如果基因预测的结果cds数目为0，前端直接报错，不会传给后台；
    """

    def __init__(self, work_id):
        super(GenePredictGffModule, self).__init__(work_id)
        option = [
            {"name": "genome", "type": "infile", "format": "sequence.fasta"},  # 染色体序列文件
            {"name": "genome_plas", "type": "infile", "format": "sequence.fasta"},  # 质粒序列文件
            {"name": "gene_prefix", "type": "string"},  # 基因前缀，自定义前缀,默认gene
            # 质粒前缀,字典形式含一个或多个质粒，"{‘plassmidA’: ‘plasa", ‘plassmidB’: ‘plasb"}"
            {"name": "genome_name", "type": "string"}, # 基因组名称
            {"name": "genome_type", "type": "string"}, #基因组类型是属于完成图还是扫描图
            {"name": "sample", "type": "string"}, #样品名称
            {"name": "plasmid_prefix", "type": "string"},
            # 质粒前缀,字典形式含一个或多个质粒，"{‘plassmidA’: ‘plasa", ‘plassmidB’: ‘plasb"}"
            {"name": "input_gff", "type": "infile", "format": "gene_structure.gff3"},  # 输入的gff文件
            {"name": "trna", "type": "string", "default": "-"},  # tRNA输入数目，如果没有填写就默认为-，如果为0或其他数字不执行预测
            {"name": "rrna", "type": "string", "default": "-"},  # rRNA输入数目，如果没有填写就默认为-，如果为0或其他数字不执行预测
            {"name": "cds_fnn", "type": "outfile", "format": "sequence.fasta"},  # 预测出的染色体序列
            {"name": "seq_plas", "type": "outfile", "format": "sequence.fasta"},  # 完成图质粒预测的序列
            {"name": "sample_gene_gff", "type": "outfile", "format": "gene_structure.gff3"},  # 样品编码基因预测gff文件
            {"name": "sample_trna_gff", "type": "outfile", "format": "gene_structure.gff3"},  # tRNA预测结果
            {"name": "sample_rrna_gff", "type": "outfile", "format": "gene_structure.gff3"},  # rRNA预测结果
            {"name": "sample_gene_fnn", "type": "outfile", "format": "sequence.fasta"},
            {"name": "sample_gene_faa", "type": "outfile", "format": "sequence.fasta"},
        ]
        self.add_option(option)
        self.genome_tools = []  # 一个基因组运行编码基因预测、tRNA预测、rRNA预测
        #self.predict_tidy = [self.dna_predict_tidy]
        self.step.add_steps("trna", "trnap", "cds", "predict_tidy")
        self.tools = []
        self.database = ""

    def set_step(self, event):
        if "start" in event["data"].keys():
            event["data"]["start"].start()
        if "end" in event["data"].keys():
            event["data"]["end"].finish()
        self.step.update()

    def check_options(self):
        """
        检查参数
        :return:
        """
        if (not self.option("genome").is_set) and (not self.option("genome_plas").is_set):
            raise OptionError("必须提供基因组文件")
        if not self.option("genome_name"):
            raise OptionError("必须提供样品名")
        if not self.option("input_gff").is_set:
            raise OptionError("必须提供所需的gff文件")
        return True

    def run_cds_predict(self):
        """
        进行cds基因预测,从gff文件中提取出所需要的信息
        :return:
        """
        self.cds_predict = self.add_tool("bac_comp_genome.get_seq_from_gff")
        opts = {
            "sample": self.option("sample"),
            "genome": self.option("genome_name"),
            "input_gff": self.option("input_gff"),
            "genome_type": self.option("genome_type"),
            "type": "cds"
        }
        if self.option("gene_prefix"):
            opts["gene_tag"] = self.option("gene_prefix")
        else:
            opts["gene_tag"] = self.option("plasmid_prefix")
        self.cds_predict.set_options(opts)
        self.genome_tools.append(self.cds_predict)

    def run_trna_predict(self):
        """
        进行tRNA功能预测，先对tRNA个数进行判断，然后选择方式方法
        :return:
        """
        if self.option("trna") != "-":
            self.trna_predict = self.add_tool("bac_comp_genome.get_seq_from_gff")
            opts = {
                "sample": self.option("sample"),
                "genome": self.option("genome_name"),
                # "gene_tag": self.option("gene_prefix"),
                "input_gff": self.option("input_gff"),
                "genome_type": self.option("genome_type"),
                "type": "tRNA"
            }
        else:
            self.trna_predict = self.add_tool("bac_comp_genome.trnascanse")
            opts = {
                "input_genome": self.option("genome"),
                "genome_name": self.option("genome_name"),
                "genome_type": self.option("genome_type"),
                # "gene_tag": self.option("gene_prefix")
            }
        if self.option("gene_prefix"):
            opts["gene_tag"] = self.option("gene_prefix")
        else:
            opts["gene_tag"] = self.option("plasmid_prefix")
        self.trna_predict.set_options(opts)
        self.genome_tools.append(self.trna_predict)

    def run_rrna_predict(self):
        """
        进行rRNA功能预测，先对rRNA个数进行判断，然后选择方法去做
        :return:
        """
        if self.option("rrna") != "-":
            self.rrna_predict = self.add_tool("bac_comp_genome.get_seq_from_gff")
            opts = {
                "sample": self.option("sample"),
                "genome": self.option("genome_name"),
                # "gene_tag": self.option("gene_prefix"),
                "input_gff": self.option("input_gff"),
                "genome_type": self.option("genome_type"),
                "type": "rRNA"
            }
        else:
            self.rrna_predict = self.add_tool("bac_comp_genome.barrnap")
            opts = {
                "input_genome": self.option("genome"),
                "genome_name": self.option("genome_name"),
                "genome_type": self.option("genome_type"),
                # "gene_tag": self.option("gene_prefix")
            }
        if self.option("gene_prefix"):
            opts["gene_tag"] = self.option("gene_prefix")
        else:
            opts["gene_tag"] = self.option("plasmid_prefix")
        self.rrna_predict.set_options(opts)
        self.genome_tools.append(self.rrna_predict)

    def run_dna_predict_tidy(self):
        """
        对染色体的预测结果重新矫正，统计基础信息
        :return:
        """
        self.dna_predict_tidy = self.add_tool("bac_comp_genome.dna_predict_tidy")
        opts = {
            "input_genome": self.option("genome"),
            "gene_prefix": self.option("gene_prefix"),
            "gene_gff": self.cds_predict.option("gff"),
            "trna_gff": self.trna_predict.option("gff"),
            "rrna_gff": self.rrna_predict.option("gff"),
            "genome_name": self.option("genome_name"),
            "genome_type": self.option("genome_type"),
            "sample": self.option("sample"),
            "is_gff": "gff+seq"
        }
        self.dna_predict_tidy.set_options(opts)
        self.dna_predict_tidy.on("start", self.set_step, {"start": self.step.predict_tidy})
        self.dna_predict_tidy.on("end", self.set_step, {"end": self.step.predict_tidy})
        self.dna_predict_tidy.on("end", self.set_output, "predict_tidy")
        self.dna_predict_tidy.run()

    def run_dna_predict_tidy_pla(self):
        """
        根据质粒预测结果进行重新矫正，统计基础信息
        :return:
        """
        self.dna_predict_tidyp = self.add_tool("bac_comp_genome.dna_predict_tidy")
        opts = {
            "input_genome": self.option("genome_plas"),
            "plasmid_prefix": json.dumps({self.option("genome_name"): self.option("plasmid_prefix")}),
            "gene_gff": self.cds_predict.option("gff"),
            "trna_gff": self.trna_predict.option("gff"),
            "rrna_gff": self.rrna_predict.option("gff"),
            "genome_name": self.option("genome_name"),
            "genome_type": self.option("genome_type"),
            "is_gff": "gff+seq"
        }
        self.dna_predict_tidyp.set_options(opts)
        self.dna_predict_tidyp.on("start", self.set_step, {"start": self.step.predict_tidy})
        self.dna_predict_tidyp.on("end", self.set_step, {"end": self.step.predict_tidy})
        self.dna_predict_tidyp.on("end", self.set_output, "predict_tidy")
        self.dna_predict_tidyp.run()

    def run(self):
        self.logger.info("开始运行啦")
        super(GenePredictGffModule, self).run()
        if self.option("genome").is_set or self.option("genome_plas").is_set:
            self.logger.info("正在对染色体进行基因预测")
            self.run_cds_predict()
            self.run_trna_predict()
            self.run_rrna_predict()
            if self.option("plasmid_prefix"):
                self.on_rely(self.genome_tools, self.run_dna_predict_tidy_pla)
            else:
                self.on_rely(self.genome_tools, self.run_dna_predict_tidy)
            for tool in self.genome_tools:
                tool.run()

    def set_output(self, event):
        """
        设置结果目录
        :param event:
        :return:
        """
        self.logger.info("正在设置结果目录")
        if event["data"] == "predict_tidy":
            if self.option("genome").is_set:
                dir_path = self.dna_predict_tidy.output_dir
                for file in os.listdir(dir_path):
                    file_path = os.path.join(self.dna_predict_tidy.output_dir, file)
                    new_file = os.path.join(self.output_dir, file)
                    if os.path.exists(new_file):
                        os.remove(new_file)
                    os.link(file_path, new_file)
            elif self.option("genome_plas").is_set:
                dir_path = self.dna_predict_tidyp.output_dir
                for file in os.listdir(dir_path):
                    file_path = os.path.join(self.dna_predict_tidyp.output_dir, file)
                    new_file = os.path.join(self.output_dir, file)
                    if os.path.exists(new_file):
                        os.remove(new_file)
                    os.link(file_path, new_file)
            else:
                self.logger.info("没有正确的生成结果文件")
            self.logger.info("设置结果目录文件完成")
            self.end()

    def end(self):
        super(GenePredictGffModule, self).end()


class ListNode(object):
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

def isPalindrome(head):
    """
    :type head: ListNode
    :rtype: bool
    """
    og = []
    reversed = []

    # 1) convert the list into an array
    while head:
        og.append(head.val)
        head = head.next

    # 2) reverse the array
    for i in range(len(og), -1, -1):
        reversed.append(og[i])
    # 3) compare
    if og == reversed:
        return True

    return False

head = [1, 2, 2, 1]
print(isPalindrome(head))


import numpy as np

from popsynth.distribution import DistributionParameter
from popsynth.distributions.spherical_distribution import SphericalDistribution

class SpiralGalaxyDistribution(SphericalDistribution):
    _distribution_name = "SpiralGalaxyDistribution"

    rho = DistributionParameter(vmin=0)
    a = DistributionParameter()
    b = DistributionParameter()
    R1 = DistributionParameter()
    R0 = DistributionParameter()

    def __init__(
        self,
        seed: int = 1234,
        name: str = "spiral_galaxy",
        form: str = None,
    ):
        """
        A spiral galaxy spatial distribution.

        :param seed: Random seed
        :type seed: int
        :param name: Name of the distribution
        :type name: str
        :param form: Mathematical description of distribution
        :type form: str
        :param rho: Local density
        :type rho: :class:`DistributionParameter`
        :param a: Shape parameter
        :type a: :class:`DistributionParameter`
        :param b: Shape parameter
        :type b: :class:`DistributionParameter`
        :param R1: Scale parameter
        :type R1: :class:`DistributionParameter`
        :param R0: Scale parameter
        :type R0: :class:`DistributionParameter`
        """

        spatial_form = r"\rho \Big(\frac{r + R_1}{R_0+R_1}\Big)^a "
        spatial_form += r"\exp\Big[-b \frac{r-R_0}{R_0 + R_1}\Big]"

        super(SpiralGalaxyDistribution, self).__init__(
            seed=seed,
            name=name,
            form=form,
        )

    def dNdV(self, r):

        return (
            self.rho
            * np.power(
                (r + self.R1) / (self.R0 + self.R1),
                self.a,
            )
            * np.exp(-self.b * (r - self.R0) / (self.R0 + self.R1))
        )

    def draw_sky_positions(self, size):
        """
        Based on Wainscoat 1992 and
        Faucher-Giguere 2007.

        Code thanks to Mortiz Pleintinger.
        """

        # Parameters for possun [-8.5,0,0], from Wainscoat 1992
        k = np.array([4.25, 4.25, 4.89, 4.89])
        r0 = np.array([3.48, 3.48, 4.90, 4.90])
        theta0 = np.array([0.0, 3.141, 2.525, 5.666])  # Wainscoat 1992
        height = 0.3  # kpc

        idx = np.random.randint(4, size=size)

        ks = k[idx]
        r0s = r0[idx]
        theta0s = theta0[idx]

        tet = ks * np.log(self._distances / r0s) + theta0s

        winklcor = np.random.uniform(0.0, 2 * np.pi, size=size)

        corrtet = winklcor * np.exp(
            -0.35 * self._distances
        )  # Faucher-Giguere 2007

        spiraltheta = tet + corrtet  # Faucher-Giguere 2007

        zpos = np.random.exponential(height, size=size)

        zpos *= np.random.choice([-1, 1], size=size)

        self._distances = self._distances + np.random.normal(
            0, scale=0.07 * np.abs(self._distances), size=size
        )

        phi = np.arccos(zpos / np.sqrt(self._distances ** 2 + zpos ** 2))

        self._theta = spiraltheta

        self._phi = phi


import base64
import json
import re
import socket
import sys
from urllib.parse import urlparse

import fitz
import pytesseract as tess
import requests
from pdf2image import convert_from_bytes

from batch.SpanishSpellChecker import SpanishSpellChecker
from common.TextractService import TextractService
from common.commonsLib import loggerElk

class ImageToTxt:
    ENCODING = 'utf-8'
    TIMEOUT_SEC = 160
    MIN_TEXT_LENGTH = 10

    def __init__(self, s3service):
        self.logger = loggerElk(__name__)
        self.spell_checker_service = SpanishSpellChecker()
        self.s3_service = s3service
        self.textract = TextractService(self.s3_service)

    def get_ocr_from_pdf2readable(self, s3_service, key):
        self.logger.Information("ImageToTxt::forced native OCR")
        decoded_pages = self.__get_text_by_pdf2readable(
            s3_service=s3_service, bucket=s3_service.real_bucket, key=key,
            force=1, skip=0, force_no_native=0, disable=0)
        sum_native = sum(map(lambda x: len(self.clean(x)), decoded_pages))
        if sum_native < self.MIN_TEXT_LENGTH:
            self.logger.Information(f"ImageToTxt::forced readable OCR")
            decoded_pages = self.__get_text_by_pdf2readable(
                s3_service=s3_service, bucket=s3_service.real_bucket, key=key,
                force=1, skip=sys.maxsize, disable=1)

        return decoded_pages

    def get_ocr_from_pdf2readable_with_cache(self, s3_service, key):
        decoded_pages = self.get_cache_from_pdf2readable(s3_service, key)
        if not decoded_pages:
            self.get_ocr_from_pdf2readable(s3_service, key)

        return decoded_pages

    def get_max_ocr_from_pdf2readable(self, s3_service, key):
        txt_decoded_list = list()
        txt_decoded = self.get_native_from_pdf2readable(s3_service, key)
        txt_decoded_list.append(txt_decoded)
        txt_decoded = self.get_native_ocr_from_pdf2readable(s3_service, key)
        txt_decoded_list.append(txt_decoded)
        txt_decoded = self.get_readable_from_pdf2readable(s3_service, key)
        txt_decoded_list.append(txt_decoded)

        try:
            txt_decoded_length = list(map(lambda x: len(' '.join(x)), txt_decoded_list))
            txt_decoded_max_id = txt_decoded_length.index(max(txt_decoded_length))
            txt_decoded = txt_decoded_list[txt_decoded_max_id]
        except Exception:
            pass

        return txt_decoded

    def get_cache_from_pdf2readable(self, s3_service, key):
        self.logger.Information("ImageToTxt::cached")
        decoded_pages = self.__get_text_by_pdf2readable(
            s3_service=s3_service, bucket=s3_service.real_bucket, key=key,
            force=0, skip=0, force_no_native=0, disable=1)

        return decoded_pages

    def get_native_from_pdf2readable(self, s3_service, key, skip=0):
        self.logger.Information("ImageToTxt::forced native")
        decoded_pages = self.__get_text_by_pdf2readable(
            s3_service=s3_service, bucket=s3_service.real_bucket, key=key,
            force=1, skip=skip, force_no_native=0, disable=1)

        return decoded_pages

    def get_native_ocr_from_pdf2readable(self, s3_service, key):
        self.logger.Information("ImageToTxt::forced native + OCR")
        decoded_pages = self.__get_text_by_pdf2readable(
            s3_service=s3_service, bucket=s3_service.real_bucket, key=key,
            force=1, skip=0, force_no_native=0, disable=0)

        return decoded_pages

    def get_readable_from_pdf2readable(self, s3_service, key):
        self.logger.Information("ImageToTxt::forced readable OCR")
        decoded_pages = self.__get_text_by_pdf2readable(
            s3_service=s3_service, bucket=s3_service.real_bucket, key=key,
            force=1, skip=sys.maxsize, force_no_native=0, disable=1)

        return decoded_pages

    def get_text_by_tesseract(self, doc_image=None):
        full_text = ''

        if doc_image is not None:
            full_text = self.__get_string(doc_image)

        return full_text

    def get_text_by_textract(self, doc_image=None):
        full_text = ''

        if doc_image is not None:
            full_text = self.__get_string(doc_image)

        return full_text

    def __get_string(self, file_input):
        texto = self.__img_to_string(file_input)
        return texto

    @staticmethod
    def __img_to_string(img):
        lang = 'spa+cat'
        config = ('-l ' + lang + ' --oem 1 --psm 3')
        tess.pytesseract.tesseract_cmd = r"tesseract"
        text_string = tess.image_to_string(img, config=config)
        return text_string

    def __get_text_by_pdf2readable(self,
                                   s3_service,
                                   bucket, key,
                                   force=0,
                                   force_no_native=0,
                                   disable=1,
                                   skip=MIN_TEXT_LENGTH):
        pages = list()
        full_text = ''

        if bucket and key:
            full_text = self.__ocr_pdf2readable(
                s3_service, key, force, force_no_native, disable, skip)

        if len(full_text) > 0:
            pages = re.compile(r'\[\[\[\d+\]\]\]').split(full_text)
            if pages:
                pages = [page for page in pages if page]

        return pages

    def __ocr_pdf2readable(self, s3_service, key, force, force_no_native, disable, skip):
        full_text = ''

        headers = {'content-type': 'application/json'}
        payload_ocr = {
            'persistence': 'S3',
            'source': 'S3',
            'bucket': s3_service.bucket,
            'key': key,
            'data': key,
            'lang': 'spa',
            'forcescan': force,
            'disableocr': disable,
            "forcenonative": force_no_native,
            "max_pages": 99,
            "txtLenForSkip": skip,
            "kb_limit_size": 0,
            "queue": ""
        }

        try:
            r_ocr = None
            self.logger.Information(f'ImgToTxt::Real bucket:{s3_service.bucket}')
            self.logger.Information(f'ImgToTxt::Calling ocr_pdf2readable: '
                                    f'{s3_service.readable_url} with: {payload_ocr}')
            if self.ping(s3_service.readable_url):
                r_ocr = requests.post(s3_service.readable_url,
                                      data=json.dumps(payload_ocr),
                                      headers=headers,
                                      timeout=self.TIMEOUT_SEC)
        except Exception as e:
            if str(e) == "('Connection aborted.', RemoteDisconnected(" \
                         "'Remote end closed connection without response'))":
                self.logger.Error(f'ImgToTxt::Exception {e} after {self.TIMEOUT_SEC} seconds.')
            else:
                self.logger.Error(f"ImgToTxt::{e}")
            return ''

        if r_ocr is None or len(r_ocr.text) == 0:
            self.logger.Information(f'ImgToTxt::Error when calling pdf2readable')
            return ''

        try:
            try:
                json_ocr = json.loads(r_ocr.text)
                if not json_ocr or not len(json_ocr) or not isinstance(json_ocr, dict):
                    raise ValueError("JSON result malformed")
            except Exception as _:
                self.logger.Error('ImgToTxt::JSON format error')
                return ''

            if ('status' in json_ocr and json_ocr['status'] == 'False') or \
                    ('statusCode' in json_ocr and json_ocr['statusCode'] == 500):
                self.logger.Error('ImgToTxt::status False or 500')
                return ''
            rt_len = 0
            if json_ocr and 'resultText' in json_ocr and json_ocr['resultText']:
                rt_len = len(json_ocr['resultText'])
            rtn_len = 0
            if json_ocr and 'resultTextNonNative' in json_ocr and json_ocr['resultTextNonNative']:
                rtn_len = len(json_ocr['resultTextNonNative'])
            if rt_len or rtn_len:
                full_text += json_ocr['resultText'] if rt_len >= rtn_len else json_ocr['resultTextNonNative']
        except Exception as ex:
            self.logger.Error(f"ImgToTxt::ocr_pdf2readable service response -> [{str(ex)}]")
            return ''

        decoded_text = self.load_document(full_text)
        if len(decoded_text.strip()) > self.MIN_TEXT_LENGTH:
            self.logger.Information(f'ImgToTxt::ocr_pdf2readable got some text -> '
                                    f'{self.clean(decoded_text)[:self.MIN_TEXT_LENGTH]}')
        elif 0 < len(decoded_text.strip()) <= self.MIN_TEXT_LENGTH:
            self.logger.Information(f'ImgToTxt::ocr_pdf2readable got little text -> '
                                    f'{len(decoded_text.strip())} characters')
            return ''
        else:
            self.logger.Error(f'ImgToTxt::Error when calling pdf2readable. No text returned. '
                              f'Check that the bucket and credentials '
                              f'are in the pdf2readable config')
            return ''

        return decoded_text

    @staticmethod
    def clean(data):
        if data is None:
            return ''

        if type(data) == int or type(data) == float:
            return data

        data = data.strip()
        data = data.replace('\r', '')
        data = data.replace('\n', '')
        data = data.replace('\t', '')

        data = data.replace('.', '')
        data = data.replace(':', '')
        data = data.replace(';', '')
        data = data.replace('\'', '')
        data = data.lower()

        return data

    @staticmethod
    def check_doc_format(key, file_data, txt_decoded):
        try:
            fitz.Document(filetype='pdf', stream=file_data)
            convert_from_bytes(file_data, dpi=1)
        except Exception:
            raise ValueError(key)

        return key, file_data, txt_decoded

    @staticmethod
    def ping(srv_url):
        address = urlparse(srv_url)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.settimeout(2)
        try:
            sock.connect((address.hostname, address.port))
        except Exception:
            return False
        else:
            sock.close()
            return True

    @staticmethod
    def load_document(content):
        try:
            full_text = content.decode(ImageToTxt.ENCODING, errors='replace')
        except (UnicodeDecodeError, AttributeError):
            full_text = content.encode(ImageToTxt.ENCODING, errors="replace").decode(
                ImageToTxt.ENCODING, errors='replace')
        try:
            data_encoded = full_text.encode(ImageToTxt.ENCODING, errors="replace")
            b64_encoded = base64.b64encode(data_encoded)
            b64_decoded = base64.b64decode(b64_encoded)
            full_text = b64_decoded.decode(ImageToTxt.ENCODING, errors='replace')
            try:
                if base64.b64encode(base64.b64decode(data_encoded)) == data_encoded:
                    b64_decoded = base64.b64decode(b64_decoded)
                    full_text = b64_decoded.decode(ImageToTxt.ENCODING, errors='replace')
            except Exception as _:
                pass
        except Exception as _:
            pass

        return full_text