#!/usr/bin/env python3
import argparse
import json
import multiprocessing
import os
import re
import shlex
import shutil
import subprocess
import sys
import time

# noinspection PyBroadException
try:
    import mutagen
except:
    mutagen = None

# Your unique announce URL
announce = ''

# Where to output .torrent files
torrent_output = '.'

# Where to save transcoded albums
transcode_output = '.'

# The default formats to transcode to
default_formats = '320,v0'

# Whether or not to transcode by default
default_transcode = True

# Whether or not to make .torrent files by default
default_torrent = True

# The maximum number of threads to maintain. Any number less than 1 means the
# script will use the number of CPU cores in the system. This is the default
# value for the -c (--cores) option.
max_threads = 0

# I prefix torrents I download as FL for Freeleech, UL for Upload, etc. Any
# prefix in this set will be removed from any transcoded albums and from the
# resulting torrent files created.
ignored_prefixes = {
}

# torrent_commands is the set of all ways to create a torrent using various
# torrent clients. These are the following replacements:
# {0}: Source directory to create a torrent from
# {1}: Output .torrent file
# {2}: Your announce URL
torrent_commands = {
    'transmission-create -p -o {1} -t {2} {0}',
    'mktorrent -p -o {1} -a {2} {0}'
}
torrent_command = None

# transcode_commands is the map of how to transcode into each format. The
# replacements are as follows:
# {0}: The input file (*.flac)
# {1}: The output file (*.mp3 or *.m4a)
# {2}: Song title
# {3}: Artist
# {4}: Album
# {5}: date
# {6}: track number
ffmpeg = 'ffmpeg -threads 1 '
transcode_commands = {
    '16-48': ffmpeg + '-i {0} -acodec flac -sample_fmt s16 -ar 48000 {1}',
    '16-44': ffmpeg + '-i {0} -acodec flac -sample_fmt s16 -ar 44100 {1}',
    'alac': ffmpeg + '-i {0} -acodec alac {1}',
    '320': ffmpeg + '-i {0} -acodec libmp3lame -ab 320k {1}',
    'v0': 'flac --decode --stdout {0} | lame -V 0 -q 0 --add-id3v2 --tt {2} --ta {3} --tl {4} --ty {5} --tn {6} - {1}',
    'v1': 'flac --decode --stdout {0} | lame -V 1 -q 0 --add-id3v2 --tt {2} --ta {3} --tl {4} --ty {5} --tn {6} - {1}',
    'v2': 'flac --decode --stdout {0} | lame -V 2 -q 0 --add-id3v2 --tt {2} --ta {3} --tl {4} --ty {5} --tn {6} - {1}'
}

# extensions maps each codec type to the extension it should use
extensions = {
    '16-48': 'flac',
    '16-44': 'flac',
    'alac': 'm4a',
    '320': 'mp3',
    'v0': 'mp3',
    'v2': 'mp3'
}

# codecs is use in string matching. If, in naming an album's folder name, you
# would use [FLAC] or [ALAC] or [320], then the lowercase contents of the
# brackets belongs in codecs so it can be matched and replaced with the
# transcode codec type.
codecs = {
    'wav',
    'flac', 'flac 24bit', 'flac 16-44', 'flac 16-48', 'flac 24-44', 'flac 24-48', 'flac 24-96', 'flac 24-196',
    '16-44', '16-48', '24-44', '24-48', '24-96', '24-196',
    'alac',
    '320', '256', '224', '192',
    'v0', 'apx', '256 vbr', 'v1', '224 vbr', 'v2', 'aps', '192 vbr'
}

# The list of lossless file extensions. While m4a can be lossy, it's up to you,
# the user, to ensure you're only transcoding from a lossless source material.
LOSSLESS_EXT = {'flac', 'wav', 'm4a'}

# The list of lossy file extensions
LOSSY_EXT = {'mp3', 'aac', 'opus', 'ogg', 'vorbis'}

# The version number
__version__ = '0.7'

exit_code = 0
FILE_NOT_FOUND = 1 << 0
ARG_NOT_DIRECTORY = 1 << 1
NO_TORRENT_CLIENT = 1 << 2
TRANSCODE_AGAINST_RULES = 1 << 3
TRANSCODE_DIR_EXISTS = 1 << 4
UNKNOWN_TRANSCODE = 1 << 5
NO_ANNOUNCE_URL = 1 << 6
NO_TRANSCODER = 1 << 7
TORRENT_ERROR = 1 << 8
TRANSCODE_ERROR = 1 << 9

def enumerate_contents(directory):
    has_lossy = False
    lossless_files = []
    data_files = []
    directories = []

    for root, _, files in os.walk(directory):
        root = root[len(directory):].lstrip('/')

        if len(root) > 0:
            directories.append(root)

        for file in files:
            extension = file[file.rfind('.') + 1:]
            if len(root) > 0:
                file = root + '/' + file

            if extension in LOSSLESS_EXT:
                lossless_files.append(file)
            else:
                if extension in LOSSY_EXT:
                    has_lossy = True
                data_files.append(file)

    return directories, data_files, has_lossy, lossless_files

def format_command(command, *args):
    safe_args = [quote(arg) for arg in args]
    return command.format(*safe_args)

def command_exists(command):
    return which(shlex.split(command)[0]) is not None

def find_torrent_command(commands):
    for command in commands:
        if command_exists(command):
            return command

    return None

def to_str(data):
    if type(data) is str:
        return to_str(data.encode('utf-8', 'surrogateescape'))
    else:
        return data.decode('utf-8', 'ignore')

def copy_contents(src, dst, dirs, files):
    os.mkdir(dst)

    for subdir in dirs:
        os.mkdir(dst + '/' + subdir)

    for file in files:
        shutil.copy(src + '/' + file, dst + '/' + file)

def make_torrent(directory, output, announce_url):
    global torrent_command, exit_code
    print('Making torrent for ' + directory)

    if torrent_command is None:
        torrent_command = find_torrent_command(torrent_commands)
        if torrent_command is None:
            print('No torrent client found, can\'t create a torrent')
            exit_code |= NO_TORRENT_CLIENT
            return

    command = format_command(torrent_command, directory, torrent_output + '/' + output, announce_url)
    torrent_status = subprocess.call(command, shell=True)
    if torrent_status != 0:
        print('Making torrent file exited with status {}!'.format(torrent_status))
        exit_code |= TORRENT_ERROR

def get_tags(filename):
    command = 'ffprobe -v 0 -print_format json -show_format'.split(' ') + [filename]
    info = json.loads(to_str(subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0]))

    if 'format' not in info or 'tags' not in info['format']:
        return '', '', '', '', ''

    tags = info['format']['tags']
    tags = {key.lower(): tags[key] for key in tags}
    parsed = {'title': '', 'artist': '', 'album': '', 'date': '', 'track': ''}

    for key in tags:
        if key in parsed:
            parsed[key] = tags[key]

    if len(parsed['track']) > 0 and 'tracktotal' in tags and len(tags['tracktotal']) > 0:
        parsed['track'] += '/' + tags['tracktotal']

    return parsed['title'], parsed['artist'], parsed['album'], parsed['date'], parsed['track']

def copy_album_art(source, dest):
    if mutagen is None:
        return

    flac = mutagen.File(source)

    if len(flac.pictures) > 0:
        # noinspection PyUnresolvedReferences
        apic = mutagen.id3.APIC(mime=flac.pictures[0].mime, data=flac.pictures[0].data)

        mp3 = mutagen.File(dest)
        mp3.tags.add(apic)
        mp3.save()

# noinspection PyUnresolvedReferences
def transcode_files(src, dst, files, command, extension):
    global exit_code
    remaining = files[:]
    transcoded = []
    threads = [None] * max_threads
    filenames = []

    transcoding = True

    while transcoding:
        transcoding = False

        for i in range(len(threads)):
            if threads[i] is None or threads[i].poll() is not None:
                if threads[i] is not None:
                    if threads[i].poll() != 0:
                        print('Error transcoding, process exited with code {}'.format(threads[i].poll()))
                        print('stderr output...')
                        print(to_str(threads[i].communicate()[1]))
                    # noinspection PyBroadException
                    try:
                        threads[i].kill()
                    except Exception as _:
                        pass

                threads[i] = None

                if len(remaining) > 0:
                    transcoding = True
                    file = remaining.pop()
                    transcoded.append(dst + '/' + file[:file.rfind('.') + 1] + extension)
                    threads[i] = subprocess.Popen(
                        format_command(command, src + '/' + file, transcoded[-1], *get_tags(src + '/' + file)),
                        stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,
                        universal_newlines=True
                    )
                    filenames.append((src + '/' + file, transcoded[-1]))
                    print(to_str('Transcoding {} ({} remaining)'.format(file, len(remaining))))
            else:
                transcoding = True

        time.sleep(0.05)

    for file in transcoded:
        if not os.path.isfile(file):
            print('An error occurred and {} was not created'.format(file))
            exit_code |= TRANSCODE_ERROR
        elif os.path.getsize(file) == 0:
            print('An error occurred and {} is empty'.format(file))
            exit_code |= TRANSCODE_ERROR

    try:
        for pair in filenames:
            copy_album_art(*pair)
    except:
        pass

def transcode_album(source, directories, files, lossless_files, formats, explicit_transcode, mktorrent):
    global exit_code

    codec_regex = r'\[(' + '|'.join([codec for codec in codecs]) + r')\](?!.*\/.*)'
    dir_has_codec = re.search(codec_regex, source, flags=re.IGNORECASE) is not None

    for transcode_format in formats:
        if not command_exists(transcode_commands[transcode_format]):
            command = shlex.split(transcode_commands[transcode_format])[0]
            print('Cannot transcode to ' + transcode_format + ', "' + command + '" not found')
            exit_code |= NO_TRANSCODER
            continue

        print('\nTranscoding to ' + transcode_format)

        if dir_has_codec:
            transcoded = re.sub(codec_regex, '[{}]'.format(transcode_format.upper()), source, flags=re.IGNORECASE)
        else:
            transcoded = source.rstrip() + ' [{}]'.format(transcode_format.upper())

        transcoded = transcoded[transcoded.rfind('/') + 1:]

        for prefix in ignored_prefixes:
            if transcoded.startswith(prefix):
                transcoded = transcoded[len(prefix):]
                break

        transcoded = transcode_output + '/' + transcoded

        if os.path.exists(transcoded):
            if explicit_transcode:
                exit_code |= TRANSCODE_DIR_EXISTS

            print('Directory already exists: ' + transcoded)
            continue

        copy_contents(source, transcoded, directories, files)
        transcode_files(source, transcoded, lossless_files, transcode_commands[transcode_format],
                        extensions[transcode_format])

        if mktorrent:
            make_torrent(transcoded, transcoded[transcoded.rfind('/'):] + '.torrent', announce)

def is_transcode_allowed(has_lossy, lossless_files, explicit_transcode):
    global exit_code

    if has_lossy > 0:
        if len(lossless_files) == 0:
            print('Cannot transcode lossy formats, exiting')
            exit_code |= TRANSCODE_AGAINST_RULES
            return False
        elif not explicit_transcode:
            print('Found mixed lossy and lossless, you must explicitly enable transcoding')
            exit_code |= TRANSCODE_AGAINST_RULES
            return False

    if len(lossless_files) == 0:
        print('Nothing to transcode!')
        exit_code |= TRANSCODE_AGAINST_RULES
        return False

    return True

def check_main_args(directory, transcode_formats, explicit_torrent):
    global exit_code
    code = 0

    if not os.path.exists(directory):
        print('The directory "{}" doesn\'t exist'.format(directory))
        code |= FILE_NOT_FOUND
    elif os.path.isfile(directory):
        print('The file "{}" is not a directory'.format(directory))
        code |= ARG_NOT_DIRECTORY

    for i in range(len(transcode_formats)):
        transcode_formats[i] = transcode_formats[i].lower()

        if transcode_formats[i] not in transcode_commands.keys():
            print('No way of transcoding to ' + transcode_formats[i])
            code |= UNKNOWN_TRANSCODE

    if explicit_torrent and (announce is None or len(announce) == 0):
        print('You cannot create torrents without first setting your announce URL')
        code |= NO_ANNOUNCE_URL

    exit_code |= code

    return code == 0

def process_album(directory, do_transcode, explicit_transcode, transcode_formats, do_torrent, explicit_torrent,
                  original_torrent):
    global exit_code
    directory = os.path.abspath(directory)

    if not (check_main_args(directory, transcode_formats, explicit_torrent)):
        return

    if original_torrent:
        make_torrent(directory, directory[directory.rfind('/'):] + '.torrent', announce)

    if do_transcode:
        directories, data_files, has_lossy, lossless_files = enumerate_contents(directory)

        if is_transcode_allowed(has_lossy, lossless_files, explicit_transcode):
            transcode_album(directory, directories, data_files, lossless_files, transcode_formats, explicit_transcode,
                            do_torrent)

def parse_args():
    description = '(Version {}) Transcode albums and create torrents in one command. Default behavior can be changed ' \
                  'by opening %(prog)s with a text editor and changing the variables at the top of the file.' \
        .format(__version__)

    parser = argparse.ArgumentParser(description=description)
    transcode_group = parser.add_mutually_exclusive_group()
    torrent_group = parser.add_mutually_exclusive_group()

    parser.add_argument('album', help='The album to process', nargs='+')

    parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__)

    announce_postfix = ' (Usable URL set)' if len(announce) > 0 else ''
    parser.add_argument('-a', '--announce', action='store', default=announce,
                        help='The torrent announce URL to use' + announce_postfix)

    postfixes = {
        't': ' (default)' if default_transcode else '',
        'T': ' (default)' if not default_transcode else '',
        'm': ' (default)' if default_torrent else '',
        'M': ' (default)' if not default_torrent else ''
    }
    transcode_group.add_argument('-t', '--transcode', action='store_true',
                                 help='Transcode the given album into other formats' + postfixes['t'])
    transcode_group.add_argument('-T', '--no-transcode', action='store_true',
                                 help='Ensures the given album is NOT transcoded' + postfixes['T'])

    torrent_group.add_argument('-m', '--make-torrent', action='count', default=0,
                               help='Creates a torrent of any transcoded albums. Specify more than once to also create '
                                    'a torrent of the source album (e.g. -mm).' + postfixes['m'])
    torrent_group.add_argument('-M', '--no-torrent', action='store_true',
                               help='Ensures no .torrent files are created' + postfixes['M'])

    parser.add_argument('-f', '--formats', action='store', default=default_formats,
                        help='The comma-separated formats to transcode to (can be of 16-48,16-44,alac,320,v0,v1,v2) '
                             '(default: %(default)s)')
    parser.add_argument('-c', '--cores', action='store', type=int, default=max_threads,
                        help='The number of cores to transcode on. Any number below 1 means to use the '
                             'number of CPU cores in the system (default: %(default)s)')

    parser.add_argument('-o', '--torrent-output', action='store', default=torrent_output,
                        help='The directory to store any created .torrent files (default: %(default)s)')
    parser.add_argument('-O', '--transcode-output', action='store', default=transcode_output,
                        help='The directory to store any transcoded albums in (default: %(default)s)')

    return parser.parse_args()

def main(args):
    global exit_code, announce, torrent_output, transcode_output, max_threads

    announce = args.announce

    do_transcode = default_transcode and not args.no_transcode
    explicit_transcode = args.transcode
    formats = args.formats.split(',')

    do_torrent = default_torrent and not args.no_torrent
    explicit_torrent = args.make_torrent
    original_torrent = args.make_torrent == 2

    if not explicit_torrent and len(announce) == 0:
        do_torrent = False
    if mutagen is None and 'v0' in formats:
        print('Mutagen is not installed; album art won\'t be copied to VBR transcodes')
        print('To keep album art, install mutagen (try "sudo python3 -m pip install mutagen")')
        if sys.version_info[1] < 4:
            print('Your python version is <3.4, you must install pip yourself before mutagen.')

    torrent_output = args.torrent_output
    transcode_output = args.transcode_output

    max_threads = args.cores
    if max_threads < 1:
        max_threads = multiprocessing.cpu_count()

    if not os.path.isdir(torrent_output):
        print('The given torrent output dir ({}) is not a directory'.format(torrent_output))
        exit_code |= ARG_NOT_DIRECTORY
    elif not os.path.isdir(transcode_output):
        print('The given transcode output dir ({}) is not a directory'.format(transcode_output))
        exit_code |= ARG_NOT_DIRECTORY

    if exit_code != 0:
        return

    first_print = True

    for album in args.album:
        if not first_print:
            print('\n\n')
        first_print = False

        print('Processing ' + album)
        process_album(album, do_transcode, explicit_transcode, formats, do_torrent, explicit_torrent, original_torrent)

#
# The following functions are copied for use in older version of Python. They
# are standard library functions in Python >3.2 that don't exist in 3.2 itself.
#
_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search

def quote(s):
    """Return a shell-escaped version of the string *s*."""
    if not s:
        return "''"
    if _find_unsafe(s) is None:
        return s

    return "'" + s.replace("'", "'\"'\"'") + "'"

def which(cmd, mode=os.F_OK | os.X_OK, path=None):
    def _access_check(fn, _mode):
        return (os.path.exists(fn) and os.access(fn, _mode)
                and not os.path.isdir(fn))

    if os.path.dirname(cmd):
        if _access_check(cmd, mode):
            return cmd
        return None

    if path is None:
        path = os.environ.get("PATH", os.defpath)
    if not path:
        return None
    path = path.split(os.pathsep)

    if sys.platform == "win32":
        if os.curdir not in path:
            path.insert(0, os.curdir)

        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
            files = [cmd]
        else:
            files = [cmd + ext for ext in pathext]
    else:
        files = [cmd]

    seen = set()
    for _dir in path:
        normdir = os.path.normcase(_dir)
        if normdir not in seen:
            seen.add(normdir)
            for thefile in files:
                name = os.path.join(_dir, thefile)
                if _access_check(name, mode):
                    return name
    return None

main(parse_args())
if exit_code != 0:
    print('An error occurred, exiting with code {}'.format(exit_code))
sys.exit(exit_code)


# -*- coding=UTF-8 -*-

from django import forms
from django.contrib.auth.models import User
from data.models import BrandGroup, Brand, Area
from lib.dynamicforms import Form

def users():
    users = [(x.id, str(x)) for x in User.objects.filter(is_staff=False).order_by('username')]
    users.insert(0, ('', 'выбрать',))
    return users

def brandgroups():
    g = [(x.id, "%s :: %s" % (x.direction, x.title)) \
             for x in BrandGroup.objects.all().order_by('-direction__title')]
    g.insert(0, ('', 'выбрать'))
    return g

class OrderItemForm(Form):
    TEMPLATE = 'cp/ordereditem_form.html'
    CORE = ('part_number',)

    supplier = forms.CharField(widget=forms.Select(choices=(),attrs={'onchange': 'changeDir(this);'}), label=u'DIR', required=True)
    area = forms.CharField(widget=forms.TextInput(attrs={'size':15}), label=u'Поставщик', required=True)
    brand = forms.CharField(widget=forms.TextInput(attrs={'size':15}), label=u'Бренд', required=True)
    part_number = forms.CharField(widget=forms.TextInput(attrs={'size':15}), label=u'PART #',required=True)
    comment_customer = forms.CharField(widget=forms.Textarea(attrs={'cols':15, 'rows':3}), label=u'COMMENT 1', required=False)
    comment_supplier = forms.CharField(widget=forms.Textarea(attrs={'cols':15, 'rows':3}), label=u'COMMENT 2', required=False)
    quantity = forms.IntegerField(min_value=1, widget=forms.TextInput(attrs={'size':5, 'class':'quantity'}), label=u'Q', required=True)
    client = forms.CharField(widget=forms.Select(choices=users()), label=u'CL', required=True)
    description_ru = forms.CharField(widget=forms.Textarea(attrs={'cols':15, 'rows':3}), label=u'RUS', required=False)
    description_en = forms.CharField(widget=forms.Textarea(attrs={'cols':15, 'rows':3}), label=u'ENG', required=False)
    price_base = forms.FloatField(widget=forms.TextInput(attrs={'size':5}), label=u'RETAIL', required=True)
    price_sale = forms.FloatField(widget=forms.TextInput(attrs={'size':5, 'class': 'priceSale'}), label=u'PRICE', required=True)

    def __init__(self, *args, **kwargs):
        super(OrderItemForm, self).__init__(*args, **kwargs)
        self.fields['client'].widget.choices = users()
        self.fields['supplier'].widget.choices = brandgroups()

    def clean_area(self):
        if 'area' in self.cleaned_data and self.cleaned_data['area']:
            area = self.cleaned_data['area']
            try:
                area = Area.objects.get(title__iexact=area)
            except Area.DoesNotExist:
                raise forms.ValidationError(u"Такого поставщика не существует")
            else:
                if 'supplier' in self.cleaned_data and self.cleaned_data['supplier']:
                    brandgroup = BrandGroup.objects.get(id = self.cleaned_data['supplier'])
                    if area not in brandgroup.area.all():
                        raise forms.ValidationError(u"Этот поставщик не входит в выбранное направление")

            return area

    def clean_brand(self):
        if 'brand' in self.cleaned_data and self.cleaned_data['brand']:
            brand = self.cleaned_data['brand']
            try:
                brand = Brand.objects.get(title__iexact=brand)
            except Brand.DoesNotExist:
                raise forms.ValidationError(u"Такого бренда не существует")
            else:
                if 'area' in self.cleaned_data and self.cleaned_data['area']:
                    if brand not in self.cleaned_data['area'].brands.all():
                        raise forms.ValidationError(u"Этот бренд не входит в выбранное направление")

            return brand

    def clean_client(self):
        if 'client' in self.cleaned_data and self.cleaned_data['client']:
            client = self.cleaned_data['client']
            try:
                client = User.objects.get(id = client)
            except User.DoesNotExist:
                raise forms.ValidationError(u"Такого пользователя не существует")

            return client

class SearchForm(forms.Form):
    def __init__(self, *args, **kwargs):
        maker_choices = kwargs.pop("maker_choices")
        super(SearchForm, self).__init__(*args, **kwargs)
        self.fields['maker'].widget.choices = maker_choices
    maker = forms.CharField(widget=forms.Select(choices=()), label=u'MAKE', required=True)
    part_number = forms.CharField(widget=forms.TextInput(attrs={'size':15}), label=u'Part Number', required=True)


# -*- coding: utf-8 -*-
from PyQt5 import QtWidgets
from PyQt5 import QtGui
from PyQt5 import QtCore
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import *

# import Fuct_QThreadUI
from functools import partial
import urllib
import socket
import Fuct_Global

# 设置全局socket超时2秒
socket.setdefaulttimeout(4)

"""
其他窗口
"""
# 日线窗口
class DayLines(QtWidgets.QWidget):
    def __init__(self,code, parent = None):
        super(DayLines,self).__init__(parent)
        self.setWindowTitle(u'日线')
        self.days = 30
        self.Status = True
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(":/Imag/Imag/icon.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.setWindowIcon(icon)
        self.setWindowFlags(Qt.Qt.WindowStaysOnTopHint|Qt.Qt.WindowMinimizeButtonHint)
        self.mainlayout = QGridLayout(self)
        self.mainlayout.setContentsMargins(0,0,0,0)
        self.myLabelEx = myLabel()
        self.mainlayout.addWidget(self.myLabelEx)
        self.File = "./data/tmp/Quote.png"
        if code[0] == "6":
            self.code = "0" + code
        else:
            self.code = "1" + code
        self.setPng()
        self.connect(self.myLabelEx, QtCore.SIGNAL('setpng'), self.CalcDays)

    def setPng(self):
        # 获取并设置图片
        url = "http://img1.money.126.net/chart/hs/kline/day/%d/%s.png" %(self.days,self.code)
        for i in range(5):
            content = urllib.urlopen(url).read()
            if len(content) > 10000:
                with open(self.File,"wb") as F:
                        F.write(content)
                self.myLabelEx.setPixmap(QPixmap(self.File))#####设置标签图片
                self.tmpCode = self.code
                self.tmpDays = self.days
                break

    def CalcDays(self):
        # 计算天数
        if self.days == 30:
            self.days = 90
        elif self.days == 90:
            self.days = 180
        else:
            self.days = 30
        self.setPng()

    def closeEvent(self, QCloseEvent):
        self.Status = False

# 重写QLabel，加入点击事件，用来显示日线数据
class myLabel(QLabel):
    def __init__(self, parent=None):
        super(myLabel, self).__init__(parent)

    def mousePressEvent(self, e):
        # 重载点击信号
        self.emit(QtCore.SIGNAL("setpng"))

# 龙虎榜日期控件
class dateWindow(QWidget):
    def __init__(self, parent=None):
        super(dateWindow, self).__init__(parent)
        self.setWindowTitle(u'日期查询')
        self.resize(300, 350)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(":/Imag/Imag/icon.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.setWindowIcon(icon)
        self.setWindowFlags(Qt.Qt.WindowStaysOnTopHint)
        self.cal = QtGui.QCalendarWidget(self)
        self.cal.setGridVisible(True)
        self.label = QtGui.QLabel(self)
        self.button = QtGui.QPushButton(self)
        self.button.setText(u"查   询")
        date = self.cal.selectedDate()
        self.label.setText(str(date.toPyDate()))
        vbox = QtGui.QVBoxLayout()
        vbox.addWidget(self.label)
        vbox.addWidget(self.cal)
        vbox.addWidget(self.button)
        self.setLayout(vbox)
        self.connect(self.cal, QtCore.SIGNAL('selectionChanged()'), self.showDate)
        self.button.clicked.connect(partial(self.button_clicked))

    def button_clicked(self):
        date = self.cal.selectedDate()
        date = str(date.toPyDate())
        self.emit(QtCore.SIGNAL("RankChice_dateWindow"), date)

    def showDate(self):
        date = self.cal.selectedDate()
        self.label.setText(str(date.toPyDate()))

# 涨停预测，自定义时间
class limitWindow(QWidget):
    def __init__(self, parent=None):
        super(limitWindow, self).__init__(parent)
        self.setWindowTitle(u'涨停预测')
        self.resize(300, 100)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(":/Imag/Imag/icon.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.setWindowIcon(icon)
        self.setWindowFlags(Qt.Qt.WindowStaysOnTopHint)
        # 定义控件
        self.label1 = QtGui.QLabel(self)
        self.label2 = QtGui.QLabel(self)
        self.edit = QtGui.QLineEdit()
        self.button = QtGui.QPushButton(self)
        # 设置控件
        self.label1.setText(u"*时间格式：2017-06-28 15:00")
        self.label1.setStyleSheet("color:rgb(255, 0, 0);")
        self.label2.setText(u"开始时间：")
        self.edit.setText(Fuct_Global.lastdayDateTime("%Y-%m-%d"+" 15:00"))
        self.button.setText(u"开始检索")

        vbox1 = QtGui.QVBoxLayout()
        hbox1 = QtGui.QHBoxLayout()

        hbox1.addWidget(self.label2)
        hbox1.addWidget(self.edit)

        vbox1.addWidget(self.label1)
        vbox1.addLayout(hbox1)
        vbox1.addWidget(self.button)
        self.setLayout(vbox1)
        self.button.clicked.connect(partial(self.button_clicked))

    def button_clicked(self):
        date = self.edit.text()
        date = date.replace("：",":").replace("  "," ").strip()
        self.close()
        self.emit(QtCore.SIGNAL("limit_dateWindow"), date)

# 涨跌幅追踪，自定义时间
class fuctuationWindow(QWidget):
    def __init__(self, parent=None):
        super(fuctuationWindow, self).__init__(parent)
        self.setWindowTitle(u'涨跌幅追踪')
        # self.resize(300, 100)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(":/Imag/Imag/icon.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.setWindowIcon(icon)
        self.setWindowFlags(Qt.Qt.WindowStaysOnTopHint)
        # 定义控件
        self.label1 = QtGui.QLabel(self)
        self.label2 = QtGui.QLabel(self)
        self.editStart = QtGui.QLineEdit()
        self.editEnd = QtGui.QLineEdit()
        self.QToolButtonStart = QtGui.QToolButton(self)
        self.QToolButtonEnd = QtGui.QToolButton(self)
        self.button = QtGui.QPushButton(self)
        # 定义控件
        day1 = QtGui.QPushButton(self)
        day1.setText(u"近一天")
        day2 = QtGui.QPushButton(self)
        day2.setText(u"近两天")
        day3 = QtGui.QPushButton(self)
        day3.setText(u"近三天")

        week1 = QtGui.QPushButton(self)
        week1.setText(u"近一周")
        week2 = QtGui.QPushButton(self)
        week2.setText(u"近两周")
        week3 = QtGui.QPushButton(self)
        week3.setText(u"近三周")

        month1 = QtGui.QPushButton(self)
        month1.setText(u"近一月")
        month2 = QtGui.QPushButton(self)
        month2.setText(u"近两月")
        month3 = QtGui.QPushButton(self)
        month3.setText(u"近三月")

        # 设置图标
        icon1 = QtGui.QIcon()
        icon1.addPixmap(QtGui.QPixmap(":/Imag/Imag/calendar.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.QToolButtonStart.setIcon(icon1)
        self.QToolButtonStart.setAutoRaise(True)
        self.QToolButtonEnd.setIcon(icon1)
        self.QToolButtonEnd.setAutoRaise(True)
        # 设置控件
        self.label1.setText(u"开始日期:")
        self.label2.setText(u"结束日期:")
        self.button.setText(u"开始检索")

        grid = QtGui.QGridLayout()
        grid.addWidget(day1, 0, 0)
        grid.addWidget(day2, 0, 1)
        grid.addWidget(day3, 0, 2)

        grid.addWidget(week1, 1, 0)
        grid.addWidget(week2, 1, 1)
        grid.addWidget(week3, 1, 2)

        grid.addWidget(month1, 2, 0)
        grid.addWidget(month2, 2, 1)
        grid.addWidget(month3, 2, 2)

        vbox1 = QtGui.QVBoxLayout()
        hbox1 = QtGui.QHBoxLayout()
        hbox2 = QtGui.QHBoxLayout()

        hbox1.addWidget(self.label1)
        hbox1.addWidget(self.editStart)
        hbox1.addWidget(self.QToolButtonStart)

        hbox2.addWidget(self.label2)
        hbox2.addWidget(self.editEnd)
        hbox2.addWidget(self.QToolButtonEnd)

        vbox1.addLayout(grid)
        vbox1.addLayout(hbox1)
        vbox1.addLayout(hbox2)
        vbox1.addWidget(self.button)
        self.setLayout(vbox1)
        self.button.clicked.connect(partial(self.button_clicked))

    def button_clicked(self):
        start = self.editStart.text()
        end = self.editEnd.text()
        self.emit(QtCore.SIGNAL("fuctuation_dateWindow"), start, end)

if __name__ == '__main__':
    pass


from sedona.core.SpatialRDD import SpatialRDD
from sedona.core.spatialOperator.join_params import JoinParams
from sedona.core.spatialOperator.rdd import SedonaPairRDDList, SedonaPairRDD
from sedona.utils.decorators import require

class JoinQueryRaw:

    @classmethod
    @require(["JoinQuery"])
    def SpatialJoinQuery(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool, considerBoundaryIntersection: bool) -> SedonaPairRDDList:
        jvm = spatialRDD._jvm
        sc = spatialRDD._sc

        srdd = jvm.JoinQuery.SpatialJoinQuery(
            spatialRDD._srdd,
            queryRDD._srdd,
            useIndex,
            considerBoundaryIntersection
        )

        return SedonaPairRDDList(srdd, sc)

    @classmethod
    @require(["JoinQuery"])
    def DistanceJoinQuery(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool, considerBoundaryIntersection: bool) -> SedonaPairRDDList:

        jvm = spatialRDD._jvm
        sc = spatialRDD._sc
        srdd = jvm.JoinQuery.DistanceJoinQuery(
            spatialRDD._srdd,
            queryRDD._srdd,
            useIndex,
            considerBoundaryIntersection
        )
        return SedonaPairRDDList(srdd, sc)

    @classmethod
    @require(["JoinQuery"])
    def spatialJoin(cls, queryWindowRDD: SpatialRDD, objectRDD: SpatialRDD, joinParams: JoinParams) -> SedonaPairRDD:

        jvm = queryWindowRDD._jvm
        sc = queryWindowRDD._sc

        jvm_join_params = joinParams.jvm_instance(jvm)

        srdd = jvm.JoinQuery.spatialJoin(queryWindowRDD._srdd, objectRDD._srdd, jvm_join_params)

        return SedonaPairRDD(srdd, sc)

    @classmethod
    @require(["JoinQuery"])
    def DistanceJoinQueryFlat(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool, considerBoundaryIntersection: bool) -> SedonaPairRDD:

        jvm = spatialRDD._jvm
        sc = spatialRDD._sc

        spatial_join = jvm.JoinQuery.DistanceJoinQueryFlat
        srdd = spatial_join(
            spatialRDD._srdd,
            queryRDD._srdd,
            useIndex,
            considerBoundaryIntersection
        )
        return SedonaPairRDD(srdd, sc)

    @classmethod
    @require(["JoinQuery"])
    def SpatialJoinQueryFlat(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool,
                             considerBoundaryIntersection: bool) -> SedonaPairRDD:

        jvm = spatialRDD._jvm
        sc = spatialRDD._sc

        spatial_join = jvm.JoinQuery.SpatialJoinQueryFlat
        srdd = spatial_join(
            spatialRDD._srdd,
            queryRDD._srdd,
            useIndex,
            considerBoundaryIntersection
        )
        return SedonaPairRDD(srdd, sc)


from SumoSupervisor import SumoSupervisor

import argparse
import os
import re
import shutil
import subprocess
import sys
import tempfile

def sumoImportError():
    sys.stderr.write("SUMO not found.\n")
    if sys.platform.startswith('linux'):
        if os.getenv('SNAP_NAME'):
            sys.stderr.write("Please set the 'SumoInterface.externController' field to TRUE and "
                             "launch the controller as extern controller.\n"
                             "When launching the controller as extern, you must specify the SUMO options like --no-netconvert "
                             "or --no-gui manually.\n"
                             "Install SUMO ")
        else:
            sys.stderr.write("Please install it ")
        sys.stderr.write("with `sudo apt install sumo sumo-tools` and reboot.\n")
    else:
        sys.stderr.write("Please install it following the instructions at https://sumo.dlr.de/docs/Installing/.\n")
    sys.exit("Or check that the SUMO_HOME environment variable points to the directory of your SUMO installation.")

# we need to import python modules from the $SUMO_HOME/tools directory
try:
    WEBOTS_HOME = os.path.normpath(os.environ.get('WEBOTS_HOME'))
    if 'SUMO_HOME' in os.environ:
        sumoPath = os.environ['SUMO_HOME']
        print('Using SUMO from %s' % sumoPath)
    else:
        sumoImportError()

    if sys.platform.startswith('darwin'):
        libraryVariablePath = 'DYLD_LIBRARY_PATH'
    elif sys.platform.startswith('linux'):
        libraryVariablePath = 'LD_LIBRARY_PATH'
    else:
        libraryVariablePath = 'PATH'
    path = os.environ.get(libraryVariablePath)
    addToPath = os.path.join(sumoPath, 'bin')
    if sys.platform.startswith('linux'):
        addToPath = os.path.join(WEBOTS_HOME, 'lib') + ';' + os.path.join(sumoPath, 'bin')
    if path is None:
        os.putenv(libraryVariablePath, addToPath)
    else:
        os.putenv(libraryVariablePath, addToPath + ';' + path)
    sys.path.append(os.path.join(sumoPath, 'tools'))
    import traci
    import sumolib
except ImportError:
    sumoImportError()

def get_options():
    """Parse the controler arguments."""
    parser = argparse.ArgumentParser()
    parser.add_argument("--no-gui", dest="noGUI", action="store_true", default=False,
                        help="runs the command line version of sumo")
    parser.add_argument("--verbose", dest="verbose", action="store_true", default=False,
                        help="prints sumo output in Webots console")
    parser.add_argument("--no-netconvert", dest="noNetconvert", action="store_true", default=False,
                        help="does not run netconvert before launching sumo")
    parser.add_argument("--disable-traffic-lights", dest="disableTrafficLights", action="store_true", default=False,
                        help="disables the update of the traffic lights state in Webots")
    parser.add_argument("--step", type=int, dest="step", default=200, help="specifies the time step of sumo [ms]")
    parser.add_argument("--max-vehicles", type=int, dest="maxVehicles", default=100,
                        help="specifies the maximum vehicles to add on Webots side")
    parser.add_argument("--rotate-wheels", dest="rotateWheels", action="store_true", default=False,
                        help="enables the wheels rotation.")
    parser.add_argument("--radius", type=int, dest="radius", default=-1,
                        help="specifies the visibility radius of the vehicles in meters (-1 means no limit)")
    parser.add_argument("--enable-height", dest="enableHeight", action="store_true", default=False,
                        help="specifies if height information should be extracted from the edge name")
    parser.add_argument("--directory", dest="directory", default="",
                        help="specifies the directory where are located the files defining the network")
    parser.add_argument("--port", type=int, dest="port", default=8873, help="specifies which port to use")
    parser.add_argument("--seed", type=int, dest="seed", default=1,
                        help="specifies the seed of the SUMO random number generator (0 for the '--random' option of SUMO)")
    parser.add_argument("--use-display", dest="useDisplay", action="store_true", default=False,
                        help="displays the gui view of SUMO in a Webots display (only working in gui mode)")
    parser.add_argument("--display-refresh-rate", type=int, dest="displayRefreshRate", default=1000,
                        help="specifies the refresh rate of the SUMO display in Webots")
    parser.add_argument("--display-zoom", type=float, dest="displayZoom", default=1.0,
                        help="specifies the initial zoom of the SUMO display in Webots (100 means no scaling)")
    parser.add_argument("--display-fit-size", dest="displayFitSize", action="store_true", default=False,
                        help="specifies if the image should be resized to fit the SUMO display size or not")
    parser.add_argument("--maximum-lateral-speed", type=float, dest="maximumLateralSpeed", default=2.5,
                        help="specifies the maximal lateral speed of any vehicle in meter per second.")
    parser.add_argument("--maximum-angular-speed", type=float, dest="maximumAngularSpeed", default=3,
                        help="specifies the maximal angular speed of any vehicle in radian per second.")
    parser.add_argument("--lane-change-delay", type=float, dest="laneChangeDelay", default=3,
                        help='specifies the time required to change lane (during this period position in Webots and SUMO may '
                        'not be perfectly synchronized anymore).')
    parser.add_argument("--sumo-arguments", dest="sumoArguments", default="", help="specifies additional SUMO arguments.")
    args = parser.parse_args()
    return args

# The main program starts from here

# Start sumo as a server and then connect and run
controller = SumoSupervisor()
options = get_options()

useDisplay = False
if options.noGUI:
    sumoBinary = os.path.join(sumoPath, 'bin', 'sumo')
else:
    if sys.platform.startswith('darwin') and not os.path.exists(os.path.join(os.sep, 'opt', 'X11')):
        sys.stderr.write("X11 is not installed and is required to launch the gui version of SUMO.\n")
        sys.stderr.write("You can easily install X11 following this link: https://support.apple.com/en-us/HT201341\n")
        sys.stderr.write("Starting command line version of SUMO instead.\n")
        sumoBinary = os.path.join(sumoPath, 'bin', 'sumo')
    else:
        sumoBinary = os.path.join(sumoPath, 'bin', 'sumo-gui')
        if options.useDisplay:
            useDisplay = True

# check if the target directory is in the WEBOTS_HOME path or not set, and adjust path if it is the case
directory = options.directory if options.directory == '' else os.path.normpath(options.directory)
if directory.startswith('WEBOTS_HOME'):
    directory = directory.replace('WEBOTS_HOME', WEBOTS_HOME)
elif directory == "":  # no directory set, use standard directory (same name of the world ending with '_net')
    directory = re.sub(r'.wbt$', '_net', controller.getWorldPath())
if not os.path.isdir(directory):
    sys.exit("You should specify in which directory are stored the network files associated to this world with the "
             "'--directory' argument or put them in the '%s' directory." % directory)

tmpDirectory = None
# generate the net file with the 'netconvert' utility
if not options.noNetconvert:
    # generate temporary directory and move network file in it
    tmpDirectory = tempfile.mkdtemp()
    for item in os.listdir(directory):
        s = os.path.join(directory, item)
        d = os.path.join(tmpDirectory, item)
        if os.path.isdir(s):
            shutil.copytree(s, d, True)
        else:
            shutil.copy2(s, d)
    directory = tmpDirectory
    print("Temporary network files generated in " + tmpDirectory + "\n")
    # check if default configuration file exist
    netConfigurationFile = os.path.join(directory, 'sumo.netccfg')
    if not os.path.isfile(netConfigurationFile):
        fileFound = ''
        # no default configuration file, try to find another one
        for file in os.listdir(directory):
            if file.endswith('.netccfg'):
                if fileFound == "":
                    netConfigurationFile = os.path.join(directory, file)
                    fileFound = file
                else:
                    print("More than one NETCONVERT configuration file found, using: " + fileFound + "\n")
                    break
    if not os.path.isfile(netConfigurationFile) and tmpDirectory is not None:
        shutil.rmtree(tmpDirectory)
        sys.exit("Could not find any NETCONVERT configuration file (*.netccfg).")
    if subprocess.call([os.path.join(sumoPath, 'bin', 'netconvert'), "-c", netConfigurationFile, "--xml-validation", "never"],
                       stdout=sys.stdout, stderr=sys.stderr) != 0:
        sys.exit("NETCONVERT failed to generate the network file.")

# this is the normal way of using traci. sumo is started as a
# subprocess and then the python script connects and runs
FNULL = open(os.devnull, 'w')
# check if default configuration file exist
configurationFile = os.path.join(directory, 'sumo.sumocfg')
if not os.path.isfile(configurationFile):
    fileFound = ""
    for file in os.listdir(directory):
        # no default configuration file, try to find another one
        if file.endswith('.sumocfg'):
            if fileFound == '':
                configurationFile = os.path.join(directory, file)
                fileFound = file
            else:
                print("More than one SUMO configuration file found, using: " + fileFound + "\n")
                break
if not os.path.isfile(configurationFile) and tmpDirectory is not None:
    shutil.rmtree(tmpDirectory)
    sys.exit("Could not find any SUMO configuration file (*.sumocfg).")

arguments = [sumoBinary, "-c", configurationFile, "--start",
             "--quit-on-end=true", "--step-length=" + str(options.step / 1000.0), "--remote-port", str(options.port)]

if options.seed == 0:
    arguments.append("--random")
else:
    arguments.append("--seed=" + str(options.seed))

if options.verbose:
    arguments.append("--verbose")

if os.path.isfile(os.path.join(directory, 'gui-settings.cfg')) and not options.noGUI:
    arguments.extend(["--gui-settings-file", os.path.join(directory, 'gui-settings.cfg')])

if options.sumoArguments != "":
    arguments.extend(options.sumoArguments.split())

sumoProcess = subprocess.Popen(arguments, stdout=FNULL, stderr=subprocess.STDOUT)
controller.run(options.port, options.disableTrafficLights, directory,
               options.step, options.rotateWheels, options.maxVehicles,
               options.radius, options.enableHeight, useDisplay,
               options.displayRefreshRate, options.displayZoom,
               options.displayFitSize, options.maximumLateralSpeed,
               options.maximumAngularSpeed, options.laneChangeDelay, traci, sumolib)
sumoProcess.terminate()

# remove temporary folder
if tmpDirectory is not None:
    print("Removing temporary network files in " + tmpDirectory + "\n")
    shutil.rmtree(tmpDirectory)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
from abc import abstractmethod

VGG_MEAN = [103.939, 116.779, 123.68]
CIFAR10_MEAN = [125.30690002, 122.95014954, 113.86599731]  #not RGB! already switched!
IMAGENET_MODES = ['eval', 'strong', 'weak']
CIFAR10_MODES = ['train', 'eval']

INPUT_QUEUE_MEMORY_FACTOR=16

class ImagePreprocessor(object):

    def __init__(self, mode, dataset, image_size, batch_size, num_preprocess_threads=4, num_readers=None):
        self.mode = mode
        self.dataset = dataset
        self.image_size = image_size
        self.batch_size = batch_size
        self.num_preprocess_threads = num_preprocess_threads
        self.num_readers = num_readers

    @abstractmethod
    def get_batch_input_tensors(self):
        pass

    @abstractmethod
    def preprocess_fn(self, image_size, image_buffer, bbox, mode, thread_id=0):
        pass

class CIFAR10Preprocessor(ImagePreprocessor):

    def __init__(self, mode, dataset, image_size, batch_size, num_preprocess_threads=4, num_readers=None):
        super(CIFAR10Preprocessor, self).__init__(mode, dataset, image_size, batch_size, num_preprocess_threads, num_readers)
        assert self.mode in CIFAR10_MODES
        if self.num_readers is None:
            if mode == 'eval':
                self.num_readers = 1
            else:
                self.num_readers = 4

    def get_batch_input_tensors(self):
        with tf.name_scope('batch_processing'):
            data_files = self.dataset.data_files()
            if data_files is None:
                raise ValueError('No data files found for this dataset')

            # Create filename_queue
            if self.mode == 'train':
                filename_queue = tf.train.string_input_producer(data_files,
                                                                shuffle=True,
                                                                capacity=16)
            else:
                filename_queue = tf.train.string_input_producer(data_files,
                                                                shuffle=False,
                                                                capacity=1)

            if self.num_preprocess_threads % 4:
                raise ValueError('Please make num_preprocess_threads a multiple '
                                 'of 4 (%d % 4 != 0).', self.num_preprocess_threads)

            if self.num_readers < 1:
                raise ValueError('Please make num_readers at least 1')

            # Approximate number of examples per shard.
            examples_per_shard = 1024
            # Size the random shuffle queue to balance between good global
            # mixing (more examples) and memory use (fewer examples).
            # 1 image uses 299*299*3*4 bytes = 1MB
            # The default input_queue_memory_factor is 16 implying a shuffling queue
            # size: examples_per_shard * 16 * 1MB = 17.6GB
            min_queue_examples = examples_per_shard * INPUT_QUEUE_MEMORY_FACTOR
            if self.mode == 'train':
                examples_queue = tf.RandomShuffleQueue(
                    capacity=min_queue_examples + 3 * self.batch_size,
                    min_after_dequeue=min_queue_examples,
                    dtypes=[tf.string])
            else:
                examples_queue = tf.FIFOQueue(
                    capacity=examples_per_shard + 3 * self.batch_size,
                    dtypes=[tf.string])

            # Create multiple readers to populate the queue of examples.
            if self.num_readers > 1:
                enqueue_ops = []
                for _ in range(self.num_readers):
                    reader = self.dataset.reader()
                    _, value = reader.read(filename_queue)
                    enqueue_ops.append(examples_queue.enqueue([value]))

                tf.train.queue_runner.add_queue_runner(
                    tf.train.queue_runner.QueueRunner(examples_queue, enqueue_ops))
                example_serialized = examples_queue.dequeue()
            else:
                reader = self.dataset.reader()
                _, example_serialized = reader.read(filename_queue)

            images_and_labels = []
            for thread_id in range(self.num_preprocess_threads):
                # Parse a serialized Example proto to extract the image and metadata.
                image, label_index = self.cifar10_parse_example_proto(example_serialized)
                # image = self.cifar10_preprocess(image_buffer)
                images_and_labels.append([image, label_index])

            images, label_index_batch = tf.train.batch_join(
                images_and_labels,
                batch_size=self.batch_size,
                capacity=2 * self.num_preprocess_threads * self.batch_size)

            # Reshape images into these desired dimensions.
            height = self.image_size
            width = self.image_size
            depth = 3

            images = tf.cast(images, tf.float32)
            images = tf.reshape(images, shape=[self.batch_size, height, width, depth])

            # Display the training images in the visualizer.
            tf.summary.image('images', images)

            return images, tf.reshape(label_index_batch, [self.batch_size])

    def cifar10_distort(self, image):
        image = tf.image.resize_image_with_crop_or_pad(image, 38, 38)
        image = tf.random_crop(image, [self.image_size, self.image_size, 3])
        image = tf.image.random_flip_left_right(image)
        return image

    def cifar10_parse_example_proto(self, example_serialized):
        features = tf.parse_single_example(
            example_serialized,
            features={
                'image': tf.FixedLenFeature([], tf.string),
                'label': tf.FixedLenFeature([], tf.int64),
            })
        label = tf.cast(features['label'], dtype=tf.int32)
        image = tf.decode_raw(features['image'], tf.uint8)
        image.set_shape([3 * self.image_size * self.image_size])
        # CHW -> HWC
        image = tf.cast(tf.transpose(tf.reshape(image, [3, self.image_size, self.image_size]), [1, 2, 0]), tf.float32)

        red, green, blue = tf.split(value=image, num_or_size_splits=[1, 1, 1], axis=2)
        assert red.get_shape().as_list() == [32, 32, 1]
        assert green.get_shape().as_list() == [32, 32, 1]
        assert blue.get_shape().as_list() == [32, 32, 1]
        image = tf.concat(values=[
            blue - CIFAR10_MEAN[0],
            green - CIFAR10_MEAN[1],
            red - CIFAR10_MEAN[2],
        ], axis=2)

        if self.mode == 'train':
            print('distort image')
            image = self.cifar10_distort(image)

        # image = tf.subtract(image, tf.constant(np.array(CIFAR10_MEAN, dtype=np.float32), dtype=tf.float32))
        return image, label

    # def parser(self, serialized_example):
    #     """Parses a single tf.Example into image and label tensors."""
    #     # Dimensions of the images in the CIFAR-10 dataset.
    #     # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the
    #     # input format.
    #     features = tf.parse_single_example(
    #         serialized_example,
    #         features={
    #             'image': tf.FixedLenFeature([], tf.string),
    #             'label': tf.FixedLenFeature([], tf.int64),
    #         })
    #     image = tf.decode_raw(features['image'], tf.uint8)
    #     image.set_shape([3 * self.image_size * self.image_size])
    #
    #     # Reshape from [depth * height * width] to [depth, height, width].
    #     image = tf.cast(
    #         tf.transpose(tf.reshape(image, [3, self.image_size, self.image_size]), [1, 2, 0]),
    #         tf.float32)
    #     label = tf.cast(features['label'], tf.int32)
    #
    #     image = tf.subtract(image, tf.constant(np.array(CIFAR10_MEAN, dtype=np.float32),dtype=tf.float32))
    #
    #     # Custom preprocessing.
    #     if self.mode == 'train':
    #         image = self.distort(image)
    #
    #     return image, label
    #
    # def get_batch_input_tensors(self):
    #     filenames = self.dataset.data_files()
    #     num_examples_per_epoch = self.dataset.num_examples_per_epoch()
    #     tfrecord_dataset = tf.contrib.data.TFRecordDataset(filenames).repeat()
    #     tfrecord_dataset = tfrecord_dataset.map(
    #         self.parser, num_threads=self.batch_size, output_buffer_size=2 * self.batch_size)
    #
    #     if self.mode == 'train':
    #         min_queue_examples = int(num_examples_per_epoch * 0.4)
    #         # Ensure that the capacity is sufficiently large to provide good random
    #         # shuffling.
    #         tfrecord_dataset = tfrecord_dataset.shuffle(buffer_size=min_queue_examples + 3 * self.batch_size)
    #
    #     # Batch it up.
    #     tfrecord_dataset = tfrecord_dataset.batch(self.batch_size)
    #     iterator = tfrecord_dataset.make_one_shot_iterator()
    #     image_batch, label_batch = iterator.get_next()
    #
    #     return image_batch, label_batch
    #
    #

class ImageNetPreprocessor(ImagePreprocessor):

    def __init__(self, mode, dataset, image_size, batch_size, num_preprocess_threads=4, num_readers=None):
        super(ImageNetPreprocessor, self).__init__(mode, dataset, image_size, batch_size, num_preprocess_threads, num_readers)
        assert self.mode in IMAGENET_MODES
        if self.num_readers is None:
            if mode == 'eval':
                self.num_readers = 1
            else:
                self.num_readers = 4

    def get_batch_input_tensors(self):
        if self.mode == 'eval':
            return inputs(self.dataset, batch_size=self.batch_size, image_size=self.image_size, num_preprocess_threads=self.num_preprocess_threads)
        elif self.mode == 'strong':
            return distorted_inputs(self.dataset, batch_size=self.batch_size, image_size=self.image_size, num_readers=self.num_readers)
        elif self.mode == 'weak':
            return inputs_with_weak_augment(self.dataset, batch_size=self.batch_size, image_size=self.image_size, num_readers=self.num_readers)
        else:
            assert False

def inputs(dataset, batch_size, image_size, num_preprocess_threads=4):
    """Generate batches of ImageNet images for evaluation.

    Use this function as the inputs for evaluating a network.

    Note that some (minimal) image preprocessing occurs during evaluation
    including central cropping and resizing of the image to fit the network.

    Args:
      dataset: instance of Dataset class specifying the dataset.
      batch_size: integer, number of examples in batch
      image_size
      num_preprocess_threads: integer, total number of preprocessing threads but
        None defaults to FLAGS.num_preprocess_threads.

    Returns:
      images: Images. 4D tensor of size [batch_size, image_size,
                                         image_size, 3].
      labels: 1-D integer Tensor of [batch_size].
    """

    # Force all input processing onto CPU in order to reserve the GPU for
    # the forward inference and back-propagation.
    with tf.device('/cpu:0'):
        images, labels = batch_inputs(
            dataset, batch_size=batch_size, image_size=image_size, train=False,
            num_preprocess_threads=num_preprocess_threads,
            num_readers=1, mode='eval')
    return images, labels

def distorted_inputs(dataset, image_size, batch_size, num_readers):
    with tf.device('/cpu:0'):
        images, labels = batch_inputs(
            dataset, batch_size=batch_size, image_size=image_size, train=True,
            num_readers=num_readers, mode='strong')
    return images, labels

def inputs_with_weak_augment(dataset, image_size, batch_size, num_readers):
    with tf.device('/cpu:0'):
        images, labels = batch_inputs(
            dataset, batch_size=batch_size, image_size=image_size, train=True,
            num_readers=num_readers, mode='weak')
    return images, labels

def decode_jpeg(image_buffer, scope=None):
    """Decode a JPEG string into one 3-D float image Tensor.

    Args:
      image_buffer: scalar string Tensor.
      scope: Optional scope for name_scope.
    Returns:
      3-D float Tensor with values ranging from [0, 1).
    """
    with tf.name_scope(values=[image_buffer], name=scope,
            default_name='decode_jpeg'):
        # Decode the string as an RGB JPEG.
        # Note that the resulting image contains an unknown height and width
        # that is set dynamically by decode_jpeg. In other words, the height
        # and width of image is unknown at compile-time.
        image = tf.image.decode_jpeg(image_buffer, channels=3)

        # After this point, all image pixels reside in [0,1)
        # until the very end, when they're rescaled to (-1, 1).  The various
        # adjust_* ops all require this range for dtype float.
        image = tf.image.convert_image_dtype(image, dtype=tf.float32)
        return image

def distort_color(image, thread_id=0, scope=None):
    """Distort the color of the image.

    Each color distortion is non-commutative and thus ordering of the color ops
    matters. Ideally we would randomly permute the ordering of the color ops.
    Rather then adding that level of complication, we select a distinct ordering
    of color ops for each preprocessing thread.

    Args:
      image: Tensor containing single image.
      thread_id: preprocessing thread ID.
      scope: Optional scope for name_scope.
    Returns:
      color-distorted image
    """
    with tf.name_scope(values=[image], name=scope, default_name='distort_color'):
        color_ordering = thread_id % 2

        if color_ordering == 0:
            image = tf.image.random_brightness(image, max_delta=32. / 255.)
            image = tf.image.random_saturation(image, lower=0.5, upper=1.5)
            image = tf.image.random_hue(image, max_delta=0.2)
            image = tf.image.random_contrast(image, lower=0.5, upper=1.5)
        elif color_ordering == 1:
            image = tf.image.random_brightness(image, max_delta=32. / 255.)
            image = tf.image.random_contrast(image, lower=0.5, upper=1.5)
            image = tf.image.random_saturation(image, lower=0.5, upper=1.5)
            image = tf.image.random_hue(image, max_delta=0.2)

        # The random_* ops do not necessarily clamp.
        image = tf.clip_by_value(image, 0.0, 1.0)
        return image

def distort_image(image, height, width, bbox, thread_id=0, scope=None):
    """Distort one image for training a network.

    Distorting images provides a useful technique for augmenting the data
    set during training in order to make the network invariant to aspects
    of the image that do not effect the label.

    Args:
      image: 3-D float Tensor of image
      height: integer
      width: integer
      bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]
        where each coordinate is [0, 1) and the coordinates are arranged
        as [ymin, xmin, ymax, xmax].
      thread_id: integer indicating the preprocessing thread.
      scope: Optional scope for name_scope.
    Returns:
      3-D float Tensor of distorted image used for training.
    """
    with tf.name_scope(values=[image, height, width, bbox], name=scope,
            default_name='distort_image'):
        # Each bounding box has shape [1, num_boxes, box coords] and
        # the coordinates are ordered [ymin, xmin, ymax, xmax].

        # Display the bounding box in the first thread only.
        if not thread_id:
            image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0),
                bbox)
            tf.summary.image('image_with_bounding_boxes', image_with_box)

            # A large fraction of image datasets contain a human-annotated bounding
            # box delineating the region of the image containing the object of interest.
            # We choose to create a new bounding box for the object which is a randomly
            # distorted version of the human-annotated bounding box that obeys an allowed
            # range of aspect ratios, sizes and overlap with the human-annotated
            # bounding box. If no box is supplied, then we assume the bounding box is
            # the entire image.
        sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
            tf.shape(image),
            bounding_boxes=bbox,
            min_object_covered=0.1,
            aspect_ratio_range=[0.75, 1.33],
            area_range=[0.05, 1.0],
            max_attempts=100,
            use_image_if_no_bounding_boxes=True)
        bbox_begin, bbox_size, distort_bbox = sample_distorted_bounding_box
        if not thread_id:
            image_with_distorted_box = tf.image.draw_bounding_boxes(
                tf.expand_dims(image, 0), distort_bbox)
            tf.summary.image('images_with_distorted_bounding_box',
                image_with_distorted_box)

        # Crop the image to the specified bounding box.
        distorted_image = tf.slice(image, bbox_begin, bbox_size)

        # This resizing operation may distort the images because the aspect
        # ratio is not respected. We select a resize method in a round robin
        # fashion based on the thread number.
        # Note that ResizeMethod contains 4 enumerated resizing methods.
        resize_method = thread_id % 4
        distorted_image = tf.image.resize_images(distorted_image, [height, width],
            method=resize_method)
        # Restore the shape since the dynamic slice based upon the bbox_size loses
        # the third dimension.
        distorted_image.set_shape([height, width, 3])
        if not thread_id:
            tf.summary.image('cropped_resized_image',
                tf.expand_dims(distorted_image, 0))

        # Randomly flip the image horizontally.
        distorted_image = tf.image.random_flip_left_right(distorted_image)

        # Randomly distort the colors.
        distorted_image = distort_color(distorted_image, thread_id)

        if not thread_id:
            tf.summary.image('final_distorted_image',
                tf.expand_dims(distorted_image, 0))
        return distorted_image

def eval_image(image, height, width, scope=None):
    """Prepare one image for evaluation.

    Args:
      image: 3-D float Tensor
      height: integer
      width: integer
      scope: Optional scope for name_scope.
    Returns:
      3-D float Tensor of prepared image.
    """
    with tf.name_scope(values=[image, height, width], name=scope,
            default_name='eval_image'):
        # Crop the central region of the image with an area containing 87.5% of
        # the original image.
        image = tf.image.central_crop(image, central_fraction=0.875)

        # Resize the image to the original height and width.
        image = tf.expand_dims(image, 0)
        image = tf.image.resize_bilinear(image, [height, width],
            align_corners=False)
        image = tf.squeeze(image, [0])
        return image

def random_crop_and_flip_image(image, height, width, thread_id=0, scope=None):
    with tf.name_scope(values=[image, height, width], name=scope,
            default_name='distort_image'):

        distorted_image = tf.image.resize_images(image, [255, 255])
        distorted_image = tf.random_crop(distorted_image, [height, width, 3])

        distorted_image.set_shape([height, width, 3])
        if not thread_id:
            tf.summary.image('cropped_resized_image',
                tf.expand_dims(distorted_image, 0))

        # Randomly flip the image horizontally.
        distorted_image = tf.image.random_flip_left_right(distorted_image)
        if not thread_id:
            tf.summary.image('final_distorted_image',
                tf.expand_dims(distorted_image, 0))
        return distorted_image

def image_preprocessing(image_size, image_buffer, bbox, mode, thread_id=0):

    if bbox is None:
        assert mode == 'weak'

    image = decode_jpeg(image_buffer)
    height = image_size
    width = image_size

    if mode == 'strong':
        image = distort_image(image, height, width, bbox, thread_id)
    elif mode == 'eval':
        image = eval_image(image, height, width)
    elif mode == 'weak':
        image = random_crop_and_flip_image(image, height, width, thread_id)

    # Finally, rescale to [-1,1] instead of [0, 1)
    #Shawn: important!
    # image = tf.subtract(image, 0.5)
    # image = tf.multiply(image, 2.0)
    image = tf.multiply(image, 255.)
    red, green, blue = tf.split(value=image, num_or_size_splits = [1,1,1], axis=2)
    assert red.get_shape().as_list() == [224, 224, 1]
    assert green.get_shape().as_list() == [224, 224, 1]
    assert blue.get_shape().as_list() == [224, 224, 1]
    bgr = tf.concat(values=[
        blue - VGG_MEAN[0],
        green - VGG_MEAN[1],
        red - VGG_MEAN[2],
    ], axis=2)
    return bgr

def parse_example_proto(example_serialized):
    """Parses an Example proto containing a training example of an image.

    The output of the build_image_data.py image preprocessing script is a dataset
    containing serialized Example protocol buffers. Each Example proto contains
    the following fields:

      image/height: 462
      image/width: 581
      image/colorspace: 'RGB'
      image/channels: 3
      image/class/label: 615
      image/class/synset: 'n03623198'
      image/class/text: 'knee pad'
      image/object/bbox/xmin: 0.1
      image/object/bbox/xmax: 0.9
      image/object/bbox/ymin: 0.2
      image/object/bbox/ymax: 0.6
      image/object/bbox/label: 615
      image/format: 'JPEG'
      image/filename: 'ILSVRC2012_val_00041207.JPEG'
      image/encoded: <JPEG encoded string>

    Args:
      example_serialized: scalar Tensor tf.string containing a serialized
        Example protocol buffer.

    Returns:
      image_buffer: Tensor tf.string containing the contents of a JPEG file.
      label: Tensor tf.int32 containing the label.
      bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]
        where each coordinate is [0, 1) and the coordinates are arranged as
        [ymin, xmin, ymax, xmax].
      text: Tensor tf.string containing the human-readable label.
    """
    # Dense features in Example proto.
    feature_map = {
        'image/encoded': tf.FixedLenFeature([], dtype=tf.string,
            default_value=''),
        'image/class/label': tf.FixedLenFeature([1], dtype=tf.int64,
            default_value=-1),
        'image/class/text': tf.FixedLenFeature([], dtype=tf.string,
            default_value=''),
    }
    sparse_float32 = tf.VarLenFeature(dtype=tf.float32)
    # Sparse features in Example proto.
    feature_map.update(
        {k: sparse_float32 for k in ['image/object/bbox/xmin',
                                     'image/object/bbox/ymin',
                                     'image/object/bbox/xmax',
                                     'image/object/bbox/ymax']})

    features = tf.parse_single_example(example_serialized, feature_map)
    label = tf.cast(features['image/class/label'], dtype=tf.int32)

    xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0)
    ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 0)
    xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 0)
    ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 0)

    # Note that we impose an ordering of (y, x) just to make life difficult.
    bbox = tf.concat(axis=0, values=[ymin, xmin, ymax, xmax])

    # Force the variable number of bounding boxes into the shape
    # [1, num_boxes, coords].
    bbox = tf.expand_dims(bbox, 0)
    bbox = tf.transpose(bbox, [0, 2, 1])

    return features['image/encoded'], label, bbox, features['image/class/text']

def batch_inputs(dataset, batch_size, image_size, train, num_preprocess_threads=4,
                 num_readers=1, input_queue_memory_factor=16, mode='strong'):
    """Contruct batches of training or evaluation examples from the image dataset.

    Args:
      dataset: instance of Dataset class specifying the dataset.
        See dataset.py for details.
      batch_size: integer
      train: boolean
      num_preprocess_threads: integer, total number of preprocessing threads
      num_readers: integer, number of parallel readers

    Returns:
      images: 4-D float Tensor of a batch of images
      labels: 1-D integer Tensor of [batch_size].

    Raises:
      ValueError: if data is not found
    """
    with tf.name_scope('batch_processing'):
        data_files = dataset.data_files()
        if data_files is None:
            raise ValueError('No data files found for this dataset')

        # Create filename_queue
        if train:
            filename_queue = tf.train.string_input_producer(data_files,
                shuffle=True,
                capacity=16)
        else:
            filename_queue = tf.train.string_input_producer(data_files,
                shuffle=False,
                capacity=1)

        if num_preprocess_threads % 4:
            raise ValueError('Please make num_preprocess_threads a multiple '
                             'of 4 (%d % 4 != 0).', num_preprocess_threads)

        if num_readers < 1:
            raise ValueError('Please make num_readers at least 1')

        # Approximate number of examples per shard.
        examples_per_shard = 1024
        # Size the random shuffle queue to balance between good global
        # mixing (more examples) and memory use (fewer examples).
        # 1 image uses 299*299*3*4 bytes = 1MB
        # The default input_queue_memory_factor is 16 implying a shuffling queue
        # size: examples_per_shard * 16 * 1MB = 17.6GB
        min_queue_examples = examples_per_shard * input_queue_memory_factor
        if train:
            examples_queue = tf.RandomShuffleQueue(
                capacity=min_queue_examples + 3 * batch_size,
                min_after_dequeue=min_queue_examples,
                dtypes=[tf.string])
        else:
            examples_queue = tf.FIFOQueue(
                capacity=examples_per_shard + 3 * batch_size,
                dtypes=[tf.string])

        # Create multiple readers to populate the queue of examples.
        if num_readers > 1:
            enqueue_ops = []
            for _ in range(num_readers):
                reader = dataset.reader()
                _, value = reader.read(filename_queue)
                enqueue_ops.append(examples_queue.enqueue([value]))

            tf.train.queue_runner.add_queue_runner(
                tf.train.queue_runner.QueueRunner(examples_queue, enqueue_ops))
            example_serialized = examples_queue.dequeue()
        else:
            reader = dataset.reader()
            _, example_serialized = reader.read(filename_queue)

        images_and_labels = []
        for thread_id in range(num_preprocess_threads):
            # Parse a serialized Example proto to extract the image and metadata.
            image_buffer, label_index, bbox, _ = parse_example_proto(
                example_serialized)
            image = image_preprocessing(image_size, image_buffer, bbox, mode, thread_id)
            images_and_labels.append([image, label_index])

        images, label_index_batch = tf.train.batch_join(
            images_and_labels,
            batch_size=batch_size,
            capacity=2 * num_preprocess_threads * batch_size)

        # Reshape images into these desired dimensions.
        height = image_size
        width = image_size
        depth = 3

        images = tf.cast(images, tf.float32)
        images = tf.reshape(images, shape=[batch_size, height, width, depth])

        # Display the training images in the visualizer.
        tf.summary.image('images', images)

        return images, tf.reshape(label_index_batch, [batch_size])


"""Sensor platform for Veolia."""
import datetime

from .const import COORDINATOR, DAILY, DOMAIN, HOURLY
from .entity import VeoliaEntity

async def async_setup_entry(hass, entry, async_add_devices):
    """Set up sensor platform."""
    coordinator = hass.data[DOMAIN][COORDINATOR]
    sensors = [
        VeoliaDailyUsageSensor(coordinator, entry),
        VeoliaMonthlyUsageSensor(coordinator, entry),
    ]
    # HOURLY array is empty when hourly report is not enabled
    if coordinator.data[HOURLY]:
        sensors.append(VeoliaHourlyUsageSensor(coordinator, entry)),

    async_add_devices(sensors)

class VeoliaHourlyUsageSensor(VeoliaEntity):
    """Monitors the hourly water usage."""

    _attr_name = "veolia_hourly_consumption"

    @property
    def state(self):
        """Return the state of the sensor."""
        hour = datetime.datetime.now().hour
        return self.coordinator.data[HOURLY][hour - 1]

class VeoliaDailyUsageSensor(VeoliaEntity):
    """Monitors the daily water usage."""

    _attr_name = "veolia_daily_consumption"

    @property
    def state(self):
        """Return the state of the sensor."""
        state = self.coordinator.data[DAILY][-1]

        if state > 0:
            return state

        return None

class VeoliaMonthlyUsageSensor(VeoliaEntity):
    """Monitors the monthly water usage."""

    _attr_name = "veolia_monthly_consumption"

    @property
    def state(self):
        """Return the state of the sensor."""
        state = sum(self.coordinator.data[DAILY])
        if state > 0:
            return state

        return None


LETTER_COUNT = 26
LOWEST_ORD = ord('a')

class TrieNode:
    def __init__(self):
        self.isWord = False
        self.mapping = [-1] * LETTER_COUNT

class Trie:
    def __init__(self):
        self.root = TrieNode()

    def insert(self, word: str) -> None:
        curr = self.root

        for c in word:
            a = ord(c) - LOWEST_ORD
            if curr.mapping[a] == -1:
                curr.mapping[a] = TrieNode()

            curr = curr.mapping[a]

        curr.isWord = True

    def _search(self, word: str) -> TrieNode:
        curr = self.root

        for c in word:
            a = ord(c) - LOWEST_ORD
            if curr.mapping[a] == -1:
                return None

            curr = curr.mapping[a]

        if curr == self.root:
            return None

        return curr

    def search(self, word: str) -> bool:
        node = self._search(word)

        return node and node.isWord

    def startsWith(self, prefix: str) -> bool:
        node = self._search(prefix)

        return node is not None

# Your Trie object will be instantiated and called as such:
# obj = Trie()
# obj.insert(word)
# param_2 = obj.search(word)
# param_3 = obj.startsWith(prefix)


"""
for loop

summary
product
"""

# summary of  1...100
# 1 + 2 + 3 + 4 + 5 + 6 + 7 + ... + 100

# goal?
# initial value?
# conditions of termination
# what job to repeat

summary = 0

# (current)summary <- (last)summary + i
#
# summary 1 <- 0 + 1
# summary 3 <- 1 + 2
# summary 6 <- 3 + 3
# summary 10 <- 6 + 4
# ...
# summary ? <- summary + 100

for i in range(1, 101):
    summary = summary + i

print("summary of 1 to 100 is {}".format(summary))

# homework - 1
# product of 1...20
# 1 * 2 * 3 * 4 * 5 * 6 * 7 * ... * 100
# homework - 2
# quiz 9 (don't do for loop)


"""
Каждый из N школьников некоторой школы знает Mi языков. Определите, какие
языки знают все школьники и языки, которые знает хотя бы один из
школьников.
Первая строка входных данных содержит количество школьников N. Далее
идет N чисел Mi, после каждого из чисел идет Mi строк, содержищих названия
языков, которые знает i-й школьник. Длина названий языков не превышает 1000й школьник. Длина названий языков не превышает 1000
символов, количество различных языков не более 1000. 1≤N≤1000, 1≤Mi≤500.
"""
language = []
some = set()
obshee = set()
for a in range(int(input())):
    v = int(input())
    a = {input() for j in range(v)}
    obshee.update(a)
    if len(some) ==0:
        some.update(a)
    else:
        some&=a
print(len(some))
print('\n'.join(sorted(some)))
print(len(obshee))
print('\n'.join(sorted(obshee)))


# %%
from typing import List, Tuple

import numpy as np
import pandas as pd

class Processor(object):
    """Class with all parsing and clearning related functions"""

    def __init__(self, logger) -> None:
        self.logger = logger
        pass

    def read_data(self, path: str) -> pd.DataFrame:
        self.logger.info("Loading the data")
        df = pd.read_csv(path, sep=",")
        return df

    def save_data(self, df, path: str, nrows: int) -> None:
        self.logger.info(f"Writing data sample of size {nrows} to {path}")
        df[:nrows].to_csv(path, index=False)
        pass

    def process(self, df) -> pd.DataFrame:
        """DataFrame processing pipeline of basic operations"""
        self.logger.info("Processing the data")
        df_processed = (
            df
            .pipe(self._parse_dates)
            .pipe(self._drop_cols_logic)
            .pipe(self._drop_cols_comp)
            .pipe(self._fill_zeros)
        )
        self.logger.info("Processing finished")
        return df_processed

    def split_data(
        self, df, train_size: float = 0.7, val_size: float = 0.2
    ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
        """Create datasets for train, val, test
        Full dataset contains 200k searches,
        len(df.srch_id.unique()) -> 199,795.
        Searches are on chronological order so split can be done
        by cutting off based on srch_id.

        Args:
            df ([pd.DataFrame]): Full dataset unsplitted

        Returns:
            Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: [
                Train set containing first 70% (if default),
                Validation set containing next 20% (if default),
                Test set containing remaining data (10% default)
            ]
        """
        self.logger.info("Splitting into train, val, test")

        split1, split2 = self._calculate_split_ids(df, train_size, val_size)

        df_train = df[df["srch_id"] <= split1]
        df_val = df[(df["srch_id"] > split1) & (df["srch_id"] <= split2)]
        df_test = df[df["srch_id"] > split2]

        self.logger.info("length of train set: " + str(len(df_train)))
        self.logger.info("length of validation set: " + str(len(df_val)))
        self.logger.info("length of test set: " + str(len(df_test)))

        return df_train, df_val, df_test

    def split_Xy(self, df) -> Tuple[pd.DataFrame, pd.Series]:
        """Split the dataframe into X and y
        In the original kaggle competition the bookings had
        a weight of 4 and clicks of 1 for the ndcg_at_k metric
        These weights are handled in the xgb model
        by setting the label_gain
        """
        self.logger.info("Splitting DataFrame into X, y")
        y = df["srch_id"].to_frame()
        y = y.assign(r=df["click_bool"].add(df["booking_bool"]))
        y = y.drop(columns=["srch_id"])["r"]

        X = df.drop(columns=["click_bool", "booking_bool"])
        return X, y

    def numeric_and_complete_cols(self, df) -> List[str]:
        """create a list of columns which are complete and numeric"""
        missing_values = df.isna().sum()
        no_missing = missing_values[missing_values == 0]
        col_list = [
            x
            for x in no_missing.index
            if not x.endswith("id") and not x.endswith("bool")
        ]
        return col_list

    def _parse_dates(self, df) -> pd.DataFrame:
        """Transforms string to datetime object
        Followed by calculating the day of week,
        month and quarter for seasonal features
        """
        self.logger.info("\tparsing the date_time")
        df_dt = df.assign(date_time=pd.to_datetime(df["date_time"]))
        df_parsed = df_dt.assign(
            day_of_week=df_dt["date_time"].dt.dayofweek + 1,
            month=df_dt["date_time"].dt.month,
        )
        return df_parsed

    def _drop_cols_logic(self, df) -> pd.DataFrame:
        """The competition belonging to this data
        has a formal test set which does not contain
        the position or gross booking.
        Kaggle Expedia reccommender competition.
        """
        self.logger.info("\tdropping columns not usable")

        drop_list = ["date_time", "position", "gross_bookings_usd"]
        df_dropped = df.drop(columns=drop_list)
        return df_dropped

    def _drop_cols_comp(self, df) -> pd.DataFrame:
        """The competitor columns have no added value
        in predictive performance and are half of the
        columns

        The second set of columns is dropped to
        reduce dimensionality without a major
        loss in performance. This saves training
        time and makes shap faster.
        """
        self.logger.info("\tdropping columns competitors")

        for col in df.columns:
            if col.startswith("comp"):
                df = df.drop(columns=col)
        return df

    def _fill_zeros(self, df) -> pd.DataFrame:
        """By defenition the zeros represent missing values in this dataset
        This functions replaces the zeros with np.nan
        """
        self.logger.info("\tFilling zeros with NaNs")

        fill_names = [
            "prop_starrating",
            "prop_review_score",
            "prop_log_historical_price",
        ]
        for col_name in fill_names:
            df[col_name] = df[col_name].replace(to_replace=0, value=np.nan)
        return df

    def _calculate_split_ids(
        self, df, train_size: float, val_size: float
    ) -> Tuple[int, int]:
        srches = df["srch_id"].unique()
        split1 = srches[round(len(srches) * train_size)]
        split2 = srches[round(len(srches) * (train_size + val_size))]
        return split1, split2


# torch
import torch
from torch import nn
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

# utils
import utilities
import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import confusion_matrix

class AutoEncoder(nn.Module):
    def __init__(self, inp_size, hidden_dim, out_dim):
        super().__init__()

        current_dim = inp_size

        self.autoencoder = nn.Sequential()

        i = 0

        for dim in hidden_dim:
            self.autoencoder.add_module(name='Linear_module_' + str(i), module=nn.Linear(current_dim, dim, bias=True))
            self.autoencoder.add_module(name='ReLU_module_' + str(i), module=nn.LeakyReLU(negative_slope=0.4))
            self.autoencoder.add_module(name='Dropout_module' + str(i), module=nn.Dropout(p=0.2))
            current_dim = dim
            i = i + 1

        self.autoencoder.add_module(name='Linear_module_out', module=nn.Linear(current_dim, out_dim))
        self.autoencoder.add_module(name='module_LReLU_out', module=nn.Sigmoid())

        init_weights(self)

    def forward(self, x):
        return self.autoencoder(x)

def init_weights(my_module):
    for sub_module in my_module.modules():
        if isinstance(sub_module, nn.Linear):
            nn.init.kaiming_normal_(sub_module.weight, nonlinearity='leaky_relu')
            nn.init.constant_(sub_module.bias, 0)

def batch_preparation(data, label, batch_size):
    X = data[data.select_dtypes('number').columns.tolist()]
    y = data[label]
    X = X.drop([label], axis=1)

    sc = MinMaxScaler()
    X = pd.DataFrame(data=sc.fit_transform(X), columns=X.columns)

    X_torch = torch.from_numpy(X.values).float()

    data_loader = DataLoader(X_torch, batch_size=batch_size)

    return X_torch, y, data_loader

def train_ae(X_torch, data_loader, model, n_epochs, tb, lr):
    log_interval = 10000

    loss_function = nn.L1Loss(reduction='mean')
    optim = torch.optim.Adam(model.parameters(), lr=lr)

    data = torch.autograd.Variable(X_torch)

    reconstruction_loss_all = 0

    model.train()

    for epoch in range(n_epochs):
        i = 1
        loss_on_epoch = 0
        for mini_batch in data_loader:
            optim.zero_grad()
            batch_recon = model(mini_batch.float())
            loss = loss_function(batch_recon, mini_batch)

            loss_on_epoch += loss.item()

            loss.backward()
            optim.step()

            if (i + 1) % log_interval == 0:
                print("Epoch {} - batch {}. \n The error in this mini_bach was: {:.3f}".format(epoch + 1,
                                                                                i + 1,
                                                                                loss.item()))

            i += 1
        print("Epoch {}. \n The mean error on this epoch was: {:.3f}".format(epoch + 1,
                                                                             loss_on_epoch / len(data_loader)))

        print("... {:.0%} of train completed".format((epoch + 1) / n_epochs))

        tb.add_scalar('Loss', loss_on_epoch, epoch+1)

    model.eval()
    reconstruction = model(data)
    test_loss = nn.L1Loss(reduction='none')
    reconstruction_loss_all = test_loss(reconstruction, data)
    model.train()

    return reconstruction_loss_all

def prediction_quality(reconstruction_loss, y, params, tb):
    reconstruction_loss_np = reconstruction_loss.detach().numpy().sum(axis=1)

    # utilities.plot_outlier_scores(y, reconstruction_loss_np, bw=0.1,
    #                              title='Fraud, Personalized Autoencoder. (epochs={})'.format(params.torch_epochs))

    contaminated_record = int(reconstruction_loss_np.shape[0] * 0.02)

    loss_df = pd.DataFrame(reconstruction_loss_np, columns=['loss'])
    threshold = loss_df.sort_values(ascending=False, by='loss').head(contaminated_record).tail(1).values
    loss_df['label'] = loss_df['loss'].map(lambda x: 1 if x >= threshold[0] else 0)

    report = pd.concat([pd.Series(y, name='true', dtype='int32'),
                        pd.Series(loss_df['label'], name='label', dtype='int32')],
                       axis=1)

    report['true positive'] = np.where((report['true'] == 1) & (report['label'] == 1), 1, 0)
    report['false negative'] = np.where((report['true'] == 1) & (report['label'] == 0), 1, 0)
    report['false positive'] = np.where((report['true'] == 0) & (report['label'] == 1), 1, 0)
    report['true negative'] = np.where((report['true'] == 0) & (report['label'] == 0), 1, 0)

    coverage = report['true positive'].sum() / report['true'].sum()

    print('Checking {} case, the coverage on frauds is about {:.0%}'
          .format(report['label'].sum(),
                  coverage))

    tb.add_text(tag='Report', text_string='Checking {} case, the coverage on frauds is about {:.0%}'
                .format(report['label'].sum(),
                        coverage))

    cm = confusion_matrix(y_true=report['true'].values, y_pred=report['label'].values)

    tb.add_text(tag='Report', text_string='True Negative: {};\n True Positive: {};'
                                          '\n False Positive {};\n False Negative {}.'
                .format(cm[0][0], cm[1][1], cm[0][1], cm[1][0]), global_step=None, walltime=None)

    tb.add_hparams(hparam_dict={'learning rate': params.torch_lr,
                                'batch size': params.torch_batch,
                                'hidden_dimensions': str(params.torch_hidden_dim),
                                'epochs': params.torch_epochs,
                                'loss': 'L1',
                                'activation output': 'sigmoid',
                                'dropout': 'active'
                                }, metric_dict={})

    tb.close()

    return report

def personalized_autoencoder(data, params):
    comment = "Run's ID = {}".format(params.torch_id)
    tb = SummaryWriter(comment=comment)

    X_torch, y,  data_loader = batch_preparation(data, params.label, params.torch_batch)
    model = AutoEncoder(params.torch_input_dim,
                        params.torch_hidden_dim,
                        params.torch_output_dim).float()

    tb.add_graph(model=model, input_to_model=next(iter(data_loader)))

    reconstruction_loss = train_ae(X_torch, data_loader, model, params.torch_epochs, tb, params.torch_lr)
    report = prediction_quality(reconstruction_loss, y, params, tb)

    return report


def re_arrange(array):
    arr_len = len(array)
    k = 0
    for i in range(arr_len):
        if array[i]:
            array[k] = array[i]
            k = k + 1

def merge(array_big, array_small):
    len_big = len(array_big)
    len_small = len(array_small)

    i = len_big - len_small - 1
    j = len_small - 1
    k = len_big - 1

    while i >= 0 and j >= 0:
        if array_big[i] >= array_small[j]:
            array_big[k] = array_big[i]
            i -= 1
        if array_big[i] < array_small[j]:
            array_big[k] = array_small[j]
            j -= 1
        k -= 1

    while j >= 0:
        array_big[j] = array_small[j]
        j -= 1

if __name__ == '__main__':
    arr_big = [0, 2, 0, 3, 0, 5, 1]
    arr_small = [0, 1, 9]

    re_arrange(arr_big)
    merge(arr_big, arr_small)

    print(arr_big)


import unittest
from BestTimeToBuyAndSellStock_final import bestTimeToBuyAndSellStock

test_cases = [
    ([22,1,5], 4),
    ([7,1,5,3,6,4], 5),
    ([7,6,4,3,1], 0),
    ([5,10,1,3,2,1,2], 5),
    ([21,22,1,5,7,2], 6),
    ([21,22,10,15,1,3], 5)
]

class TestValidClockTimes(unittest.TestCase):
    def test_all(self):
        for stock, expected in test_cases:
            with self.subTest(case=stock, expected=expected):
                result = bestTimeToBuyAndSellStock(stock)
                self.assertEqual(result, expected)

if __name__ == '__main__':
    unittest.main()


import numpy as np
import cv2
import glob
import pickle

# チェスボードの行数、列数、サイズを定義
rows = 5
cols = 7
square_size = 0.1

# キャリブレーション用のオブジェクトを定義
objp = np.zeros((rows * cols, 3), np.float32)
objp[:, :2] = np.mgrid[0:cols, 0:rows].T.reshape(-1, 2) * square_size

print("必要なものをインポートしました。")

# 画像が入っているフォルダのパスを定義
path_list = ['./img0/cam0/.png', './img0/cam1/.png', './img0/cam2/*.png']

for i, path in enumerate(path_list):
    print(f"カメラ{i}の画像を処理します。")

# フォルダから画像を読み込み、画像のリストを作成
images = []
image_points = []
for filename in glob.glob(path):
    img = cv2.imread(filename)
    ret, corners = cv2.findChessboardCorners(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY), (cols, rows), None)
    if ret == True:
        images.append(img)
        image_points.append(corners)
    else:
        print(f"Failed to detect chessboard corners in {filename}")

print(f"カメラ{i}の画像から、画像のリストを作成しました。")

# キャリブレーションを行い、内部パラメータと外部パラメータを求める
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera([objp]*len(images), image_points, cv2.cvtColor(images[0], cv2.COLOR_BGR2GRAY).shape[::-1], None, None)

print(f"カメラ{i}のキャリブレーションを行い、内部パラメータと外部パラメータを求めました。")

# 内部パラメータと外部パラメータをprintする
print(f"カメラ{i}の内部パラメータ:")
print(mtx)
print(f"カメラ{i}の外部パラメータ:")
print(rvecs)
print(tvecs)
print("------------------------")


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(

     name='sasa_phys',

     version='0.1',

     author="Tim Luca Turan, Max Bräuer",

     author_email="timturan@web.de",

     description="Semi Analytic Stacking Algorithm for Meta Surface stacks",

     long_description=long_description,

   long_description_content_type="text/markdown",

     url="https://github.com/TimLucaTuran/SASA",

     packages=setuptools.find_packages(),

     classifiers=[

         "Programming Language :: Python :: 3",

         "License :: OSI Approved :: MIT License",

         "Operating System :: OS Independent",

     ],

 )


"""
Write functions that accept 'dirty' string input, and ouputs a human readable value.
Use a combination of python string methods and regular expressions.

Write, test, and refactor as you go.

>>> scrub_numbers('Be9autiful9 i4s be2tter th4an ug42ly')
'Beautiful is better than ugly.'

>>> gentle_clean('Explicit_is-better_than -implicit')
'Explicit is better than implicit.'

>>> clean_data('  42Simple-is_better_than-compl9ex   ')
'Simple is better than complex.'

>>> some_scrubber('F l a t   i s   b e t t e r   t h a n   n e s t e d . ')
'Flat is better than nested.'

>>> mr_clean('Sparse is better than dense')
' S p a r s e   i s   b e t t e r   t h a n   d e n s e '

>>> ms_clean('Readability counts')
'R9y c4s'

>>> strong_cleaner('Err@#%$ors sho@#$@#$uld nev1!$#@er pass sile&I&&*(ntly')
'Errors should never pass silently.'

# >>> extracto('1S2pe3cia4l ca5ses ar6en't sp7ecial en8ough to b9reak the r0ules.')
#  45
#
# >>> extracto('2S4pe6cia8l ca0ses ar2en't sp4ecial en6ough to b8reak the r0ules.')
#  40
#
# >>> extracto('3S6pe9cia2l ca5ses ar8en't sp1ecial en4ough to b7reak the r0ules.')
# 45
# """
import re

#
def scrub_numbers(something):
    """
     Find the int and replace it with nothing
    """
    result = re.sub(r'\d+', '', something)    #?
    result2 = result + '.'

    return result2

# scrub_numbers('99gwhwe')

def gentle_clean(something):

    item = something.replace('_', ' ').replace('-', ' ')
    a_dopo = ' '.join(item.split()) + '.'
    return a_dopo

def clean_data(something):

    result = re.sub(r'\d+', '', something)                  # \d+ find one or more digits
    result2 = result.replace('_', ' ').replace('-', ' ')    # set variable; for result, replace _ with ' ' empty and replace '-' with ' '.
    result3 = ' '.join(result2.split()) + '.'
    return result3

def some_scrubber(something):

    result = re.sub(r'(?<=\w)\s(?=\w)', '', something)    # REVIEW regex here.
    result2 = result.replace('   ', ' ')
    result3 = result2.replace(' .', '.').strip()
    return result3

def mr_clean(something):

    result = list()
    for item in something:
        result.extend([item, ' '])            # review extend

    result = ''.join(result)
    result = ' ' + result
    return result

def ms_clean(something):
    """
    Replace characters after the first and last with the len of those characters' count.
    # find the first and last characters of each word; count the number of char between the first and last of the words
    """
    something.split()
    result1 = list(something[0:11:10] + something[12:18:5])
    result1.insert(1, '9')
    result1.insert(4, '4')
    result2 = ''.join(result1[0:3])
    result3 = ''.join(result1)[3:7]
    print(repr(result2 + ' ' + result3))

def strong_cleaner(something):

    result = re.sub(r'[*?@#%$!&(I1]', '', something)+'.'       # re.sub(something,'r(?<=\w)\s(?=\w)', something)
    return result

#
# def special_cases(dada):

    #  result = re.sub(r'\s{3}', ' ', something)
    # result = something.replace('  ', ' ')
    #
    # if '   ' in result:
    #     result
    # else:
    #     ' ' in result
    #     result.replace(' ', '')
    #
    # return result

    # result2 = result.replace(' ', '')
    # result2 = result.split()
    # result3 = result2.strip()


import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable

class CNN(nn.Module):

    def __init__(self):
        super().__init__()

        D = 50
        C = 2
        Ci = 1
        Co = 200
        Ks = [2, 3, 4, 5]

        self.convs = nn.ModuleList([nn.Conv2d(Ci, Co, (K, D)) for K in Ks])
        self.dropout = nn.Dropout(0.2)
        self.fc1 = nn.Linear(len(Ks) * Co, Co)
        self.fc2 = nn.Linear(Co, C)

    def forward(self, x):
        #(128, max_sent, 50)
        x = x.unsqueeze(1)  # (N, Ci, W, D)

        x = [F.relu(conv(x)).squeeze(3) for conv in self.convs]  # [(N, Co, W), ...]*len(Ks)

        x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x]  # [(N, Co), ...]*len(Ks)

        x = torch.cat(x, 1)

        x = self.dropout(x)  # (N, len(Ks)*Co)
        x = self.fc1(x)  # (N, C)
        logit = self.fc2(x)
        return logit


import matplotlib.pyplot as plt
import sys, os
import numpy as np

benchmark = sys.argv[1]
num_bins = int(sys.argv[2])
min_val = float(sys.argv[3])
max_val = float(sys.argv[4])

filenames = ["cpython_capi","cpython_hpy","graalpython_capi", "graalpython_hpy"]
data = [[],[],[],[]]
bins = []
labels = filenames

datapath = "../" + benchmark + "/output/"
datafiles = []
for file in filenames:
    datafiles.append(open(datapath+file))

for i in range(4):
    for line in datafiles[i]:
        data[i].append(float(line))

for i in range(num_bins):
    bins.append(round(min_val+i/num_bins*(max_val-min_val),8))

plt.hist(data[0], bins, color='gray', label=labels[0], alpha=0.8)
plt.hist(data[1], bins, color='blue', label=labels[1], alpha=0.5)
plt.hist(data[2], bins, color='orange', label=labels[2], alpha=0.8)
plt.hist(data[3], bins, color='red', label=labels[3], alpha=0.5)
plt.legend()
plt.xlabel("Time [s]")
plt.ylabel("Frequency")
plt.show()