import gzip
from graph import DeBrujinGraph
import random
import string

# Classe main
#Auteur: Julien Kiang, Dina Benkirane
#Question 3 et 4 incomplete
with gzip.open('GCF_000002985.6_WBcel235_rna.fna.gz', 'rt') as f:
    #fonction pour read le fichier fasta
    def read_fasta(path):
        with gzip.open(path, 'rt') as f:
            accession, description, seq = None, None, None
            for line in f:
                if line[0] == '>':
                    # yield current record
                    if accession is not None:
                        yield accession, description, seq

                    # start a new record
                    accession, description = line[1:].rstrip().split(maxsplit=1)
                    seq = ''
                else:
                    seq += line.rstrip()

next(read_fasta('GCF_000002985.6_WBcel235_rna.fna.gz'))

#Creation fichier contigs
contigs=open("contigs.fa","w+")
#Ecrire dans le fichier contigs
def write_contig(str):
    # Identifiant aléatoire
    random.seed(123)  # recommendé pour des résultats reproduisibles!
    contigs.write('>'.join(random.choices(string.ascii_uppercase, k=10)), str)
    contigs.close()

write_contig(read_fasta("GCF_000002985.6_WBcel235_rna.fna.gz"))

with gzip.open('reads.fastq.gz', 'rt') as f:
    #Fonction pour read les fichier fastq
    def read_fastq(path):
        with gzip.open(path, 'rt') as f:
            for line in f:
                sequence = f.readline().rstrip()
                #Question 2a
                question2=DeBrujinGraph(sequence)
                if sequence[0] is not '~':
                    yield sequence

next(read_fastq('reads.fastq.gz'))

k=21

def kmers():
    for _ in read_fastq('reads.fastq.gz'):
        kmers = [_[i:i + k] for i in range(len(_) - k + 1)]
        yield (kmers)

#Creation fichier occurrence.bed
occurrence=open("occurrences.bed","w+")
def read_bed(path):
    with open(path) as f:
        ref, start, end, name = f.readline().rstrip().split('\t')
        yield ref, int(start) - 1, int(end), name

def write_bed():
    pass
#next(read_bed('example.bed'))
#next(read_bed('occurences.bed'))

def separation():
    for i in kmers():
        for j in i:
            yield j

c = DeBrujinGraph(separation(), 21)
print(c.noPredecessors())


from pathlib import Path
from urllib.request import urlopen, Request

import click
import yaml

# Default site configs from GitHub
DEFAULT_SITE_CONFIGS = (
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/100IT.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/BIFI.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/CESGA.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/CESNET-MCC.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/CETA-GRID.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/CLOUDIFIN.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/CYFRONET-CLOUD.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/DESY-HH.yaml",
    'https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/IFCA-LCG2.yaml',
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/IISAS-FedCloud-cloud.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/IISAS-GPUCloud.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/IN2P3-IRES.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/INFN-CATANIA-STACK.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/INFN-PADOVA-STACK.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/Kharkov-KIPT-LCG2.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/NCG-INGRID-PT.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/RECAS-BARI.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/SCAI.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/TR-FC1-ULAKBIM.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/UA-BITP.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/UNIV-LILLE.yaml",
    "https://raw.githubusercontent.com/EGI-Foundation/fedcloud-catchall-operations/master/sites/fedcloud.srce.hr.yaml",
)

LOCAL_CONFIG_DIR = ".fedcloud-site-config/"

site_config_data = []

def read_site_config():
    """
    Read site configurations from local config dir if exist, otherwise from default GitHub location. Storing
    site configurations in global variable, that will be used by other functions. Call read_local_site_config()
    or read_default_site_config()

    :return: None
    """
    if len(site_config_data) > 0:
        return
    config_dir = Path.home() / LOCAL_CONFIG_DIR
    if config_dir.exists():
        read_local_site_config(config_dir)
    else:
        read_default_site_config()

def read_default_site_config():
    """
    Read default site configurations from GitHub.  Storing
    site configurations in a global variable, that will be used by other functions

    :return: None
    """
    site_config_data.clear()
    for filename in DEFAULT_SITE_CONFIGS:
        if filename.lower().startswith('http'):
            req = Request(filename)
        else:
            raise ValueError from None

        # URLs already checked, so ignore bandit test
        with urlopen(req) as yaml_file:       # nosec
            site_info = yaml.safe_load(yaml_file)
            site_config_data.append(site_info)

def read_local_site_config(config_dir):
    """
    Read site configurations from local directory specified in config_dir. Storing
    site configurations in global variable, that will be used by other functions

    :param config_dir: path to directory containing site configuration

    :return: None
    """
    site_config_data.clear()
    config_dir = Path(config_dir)
    for f in sorted(config_dir.glob('*.yaml')):
        yaml_file = f.open()
        site_info = yaml.safe_load(yaml_file)
        site_config_data.append(site_info)

def save_site_config(config_dir):
    """
    Save site configs to local directory specified in config_dir. Overwrite local configs if exist

    :param config_dir: path to directory containing site configuration

    :return: None
    """
    config_dir = Path(config_dir)
    config_dir.mkdir(parents=True, exist_ok=True)
    for site_info in site_config_data:
        config_file = config_dir / (site_info["gocdb"] + ".yaml")
        with config_file.open("w", encoding="utf-8") as f:
            yaml.dump(site_info, f)

def list_sites():
    """
    List of all sites IDs in site configurations

    :return: list of site IDs
    """
    read_site_config()
    result = []
    for site_info in site_config_data:
        result.append(site_info["gocdb"])
    return result

def find_site_data(site_name):
    """
    Return configuration of the correspondent site with site_name

    :param site_name: site ID in GOCDB
    :return: configuration of site if found, otherwise None
    """
    read_site_config()

    for site_info in site_config_data:
        if site_info["gocdb"] == site_name:
            return site_info
    return None

def find_endpoint_and_project_id(site_name, vo):
    """
    Return Keystone endpoint and project ID from site name and VO according to site configuration

    :param site_name: site ID in GOCDB
    :param vo: VO name. None if finding only site endpoint

    :return: endpoint, project_id, protocol if the VO exist on the site, otherwise None, None, None
    """
    site_info = find_site_data(site_name)
    if site_info is None:
        return None, None, None

    protocol = site_info.get("protocol")
    # If only site name is given, return endpoint without project ID
    if vo is None:
        return site_info["endpoint"], None, protocol

    for vo_info in site_info["vos"]:
        if vo_info["name"] == vo:
            return site_info["endpoint"], vo_info["auth"]["project_id"], protocol

    # Return None, None if VO not found
    return None, None, None

@click.group()
def site():
    """
    Site command group for manipulation with site configurations
    """
    pass

@site.command()
@click.option(
    "--site",
    help="Name of the site",
    required=True,
    envvar="EGI_SITE",
)
def show(site):
    """
    Print configuration of the specified site
    """
    site_info = find_site_data(site)
    if site_info:
        print(yaml.dump(site_info, sort_keys=True))
    else:
        print("Site %s not found" % site)

@site.command()
@click.option(
    "--site",
    help="Name of the site",
    required=True,
    envvar="EGI_SITE",
)
@click.option(
    "--vo",
    help="Name of the VO",
    required=True,
    envvar="EGI_VO",
)
def show_project_id(site, vo):
    """
    Printing Keystone endpoint and project ID of the VO on the site
    """
    endpoint, project_id, protocol = find_endpoint_and_project_id(site, vo)
    if endpoint:
        print(" Endpoint : %s \n Project ID : %s" % (endpoint, project_id))
    else:
        print("VO %s not found on site %s" % (vo, site))

@site.command()
def show_all():
    """
    Print all site configuration
    """
    read_site_config()
    for site_info in site_config_data:
        site_info_str = yaml.dump(site_info, sort_keys=True)
        print(site_info_str)

@site.command()
def save_config():
    """
    Read default site configs from GitHub and save them to local folder in home directory
    Overwrite local configs if exist
    """
    read_default_site_config()
    config_dir = Path.home() / LOCAL_CONFIG_DIR
    print("Saving site configs to directory %s" % config_dir)
    save_site_config(config_dir)

@site.command()
def list():
    """
    List site IDs
    """
    read_site_config()
    for site_info in site_config_data:
        print(site_info["gocdb"])


# 导入手写字体加载器
from sklearn.datasets import load_digits
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.svm import LinearSVC
from sklearn.metrics import classification_report

'''
博文：http://www.cnblogs.com/Lin-Yi/p/8970520.html

支持向量机
根据训练样本的分布，搜索所有可能的线性分类器最佳的一个。
从高纬度的数据中筛选最有效的少量训练样本。
节省数据内存，提高预测性能
但是付出更多的cpu和计算时间
'''

'''
1 获取数据
'''
# 通过数据加载器获得手写字体数字的数码图像数据并存储在digits变量中
digits = load_digits()
# 查看数据的特征维度和规模
# print(digits.data.shape)  # (1797, 64)

'''
2 分割训练集合和测试集合
'''
x_train, x_test, y_train, y_test = train_test_split(digits.data,
                                                    digits.target,
                                                    test_size=0.25,
                                                    random_state=33)

'''
3 使用支持向量机分类模型对数字图像进行识别
'''
# 对训练数据和测试数据进行标准化
ss = StandardScaler()
x_train = ss.fit_transform(x_train)
x_test = ss.fit_transform(x_test)

# 初始化线性假设的支持向量机分类器
lsvc = LinearSVC()
# 进行训练
lsvc.fit(x_train, y_train)
# 利用训练好的模型对测试集合进行预测 测试结果存储在y_predict中
y_predict = lsvc.predict(x_test)

'''
4 支持向量机分类器 模型能力评估
'''
print("准确率：", lsvc.score(x_test, y_test))
print("其他评估数据：\n", classification_report(y_test, y_predict, target_names=digits.target_names.astype(str)))
'''
准确率： 0.9488888888888889
其他评估数据：  精确率      召回率  f1指标     数据个数
              precision    recall  f1-score   support

          0       0.92      0.97      0.94        35
          1       0.95      0.98      0.96        54
          2       0.98      1.00      0.99        44
          3       0.93      0.93      0.93        46
          4       0.97      1.00      0.99        35
          5       0.94      0.94      0.94        48
          6       0.96      0.98      0.97        51
          7       0.90      1.00      0.95        35
          8       0.98      0.83      0.90        58
          9       0.95      0.91      0.93        44

avg / total       0.95      0.95      0.95       450
'''

01_监督学习_分类预测/02_(支持向量机分类器)手写数字识别.py

02_(支持向量机分类器)手写数字识别.py


#!/usr/bin/env python
import numpy as np
import matplotlib.pyplot as plt
import cv2

from dataclasses import dataclass
from typing import Any, List

@dataclass
class Filter:
  psi: float
  filter: Any

def gabor_patch(size, psi):
    theta = 0.0
    sigma = 4.0
    lambd = 31
    kern = cv2.getGaborKernel(
      (size, size),
      sigma, theta, lambd, 0.7, psi, ktype=cv2.CV_32F
    )

    kern /= 1.5*kern.sum()
    return kern

def build_filters():
  filters = []
  ksize = 31

  start = 0
  end = np.pi

  for psi in np.arange(start, end, (end - start) / 16):
    filters.append(Filter(psi, gabor_patch(ksize, psi)))
  return filters

if __name__ == '__main__':
  filters: List[Filter] = build_filters()
  fig, axes = plt.subplots(ncols=4, nrows=4)

  for ida, row in enumerate(axes):
    for idr, ax in enumerate(row):
      f = filters[ida*4+idr]
      ax.set_title("Phase: {:2f}".format(f.psi))
      ax.imshow(f.filter)

  plt.show()


"""
A field that serializes/loads data using msgpack, an ultra-fast and compact binary
serialization format with more native data types than JSON.
"""

try:
    from ._field import MsgPackField
except ImportError:
    __all__ = []
else:
    __all__ = ["MsgPackField"]

__version__ = "0.1.0"


import logging
import os
import boto3
import azure.functions as func

def main(req: func.HttpRequest) -> func.HttpResponse:

    logging.info('Python HTTP trigger function processing a S123 Submit.')

    ses_iam_sk = os.environ['ses_iam_sk']
    ses_iam_ak = os.environ['ses_iam_ak']

    recipients = ['']
    sender = ''
    subject = 'Response Submitted!'
    body = 'A response was successfully submitted for your survey!'
    send_email(ses_iam_ak, ses_iam_sk, sender, recipients, subject, body)

    message = "HTTP Trigger processed"
    return func.HttpResponse(message)

def send_email(ses_ak, ses_sk, sender, recipients, subject, body):
    CHARSET = "UTF-8"

    client = boto3.client(
        'sesv2',
        aws_access_key_id=ses_ak,
        aws_secret_access_key=ses_sk,
        region_name='us-east-1'
    )

    response = client.send_email(
        FromEmailAddress=sender,
        Destination={
            'ToAddresses': recipients
        },
        Content={
            'Simple': {
                'Body': {
                    'Text': {
                        'Charset': CHARSET,
                        'Data': body
                    }
                },
                'Subject': {
                    'Charset': CHARSET,
                    'Data': subject
                }
            }
        }
    )
    logging.info(response)
    logging.info("Email Response ")

    return

Integrating Azure Functions with ArcGIS/HTTP Trigger/EmailHTTPTrigger/__init__.py


import cv2

vidcap = cv2.VideoCapture('BadApple.mp4')
success,image = vidcap.read()
count = 0

AsciVid = []

while success:
    image = cv2.resize(image,(150,50))
    chars = []
    for i in image:
        for j in i:
            if(sum(j)<191):
                chars.append(' ')
            elif (sum(j)<382):
                chars.append('.')
            elif (sum(j)<573):
                chars.append('*')
            else:
                chars.append('@')
        chars.append('\n')

    AsciVid.append(''.join(chars))
    f = open("./Frames/Frame%d.txt"%(count), "w")
    f.write(''.join(chars))
    f.close()
    success,image = vidcap.read()
    count += 1
    print("converting..........")


"""SIMD-oriented Fast Mersenne Twister 19937 Pseudo Random Number Generator"""

from __future__ import annotations
import numpy as np
from ..compilation import (
    optional_jitclass,
    optional_ir_function,
    array_type,
)
from ..options import USE_NUMBA

def sfmt_shuffle(state: np.ndarray):
    """SFMT shuffling function"""
    # the contents of this function are only ever used when USE_NUMBA is set to False
    # otherwise, it is overloaded with the ir version

    # compute the first two states manually

    x_0 = state[0] << 8
    x_1 = (state[1] << 8) | (state[0] >> 24)
    x_2 = (state[2] << 8) | (state[1] >> 24)
    x_3 = (state[3] << 8) | (state[2] >> 24)

    y_0 = (state[617] << 24) | (state[616] >> 8)
    y_1 = (state[618] << 24) | (state[617] >> 8)
    y_2 = (state[619] << 24) | (state[618] >> 8)
    y_3 = state[619] >> 8

    b_0 = (state[488] >> 11) & 0xDFFFFFEF
    b_1 = (state[489] >> 11) & 0xDDFECB7F
    b_2 = (state[490] >> 11) & 0xBFFAFFFF
    b_3 = (state[491] >> 11) & 0xBFFFFFF6

    d_0 = state[620] << 18
    d_1 = state[621] << 18
    d_2 = state[622] << 18
    d_3 = state[623] << 18

    state[0] ^= x_0 ^ y_0 ^ b_0 ^ d_0
    state[1] ^= x_1 ^ y_1 ^ b_1 ^ d_1
    state[2] ^= x_2 ^ y_2 ^ b_2 ^ d_2
    state[3] ^= x_3 ^ y_3 ^ b_3 ^ d_3

    x_0 = state[4] << 8
    x_1 = (state[5] << 8) | (state[4] >> 24)
    x_2 = (state[6] << 8) | (state[5] >> 24)
    x_3 = (state[7] << 8) | (state[6] >> 24)

    y_0 = (state[621] << 24) | (state[620] >> 8)
    y_1 = (state[622] << 24) | (state[621] >> 8)
    y_2 = (state[623] << 24) | (state[622] >> 8)
    y_3 = state[623] >> 8

    b_0 = (state[492] >> 11) & 0xDFFFFFEF
    b_1 = (state[493] >> 11) & 0xDDFECB7F
    b_2 = (state[494] >> 11) & 0xBFFAFFFF
    b_3 = (state[495] >> 11) & 0xBFFFFFF6

    d_0 = state[0] << 18
    d_1 = state[1] << 18
    d_2 = state[2] << 18
    d_3 = state[3] << 18

    state[4] ^= x_0 ^ y_0 ^ b_0 ^ d_0
    state[5] ^= x_1 ^ y_1 ^ b_1 ^ d_1
    state[6] ^= x_2 ^ y_2 ^ b_2 ^ d_2
    state[7] ^= x_3 ^ y_3 ^ b_3 ^ d_3

    for i in range(8, 136, 4):
        x_0 = state[i] << 8
        x_1 = (state[i + 1] << 8) | (state[i] >> 24)
        x_2 = (state[i + 2] << 8) | (state[i + 1] >> 24)
        x_3 = (state[i + 3] << 8) | (state[i + 2] >> 24)

        b_0 = (state[i + 488] >> 11) & 0xDFFFFFEF
        b_1 = (state[i + 489] >> 11) & 0xDDFECB7F
        b_2 = (state[i + 490] >> 11) & 0xBFFAFFFF
        b_3 = (state[i + 491] >> 11) & 0xBFFFFFF6

        d_0 = state[i - 4] << 18
        d_1 = state[i - 3] << 18
        d_2 = state[i - 2] << 18
        d_3 = state[i - 1] << 18

        y_0 = (state[i - 7] << 24) | (state[i - 8] >> 8)
        y_1 = (state[i - 6] << 24) | (state[i - 7] >> 8)
        y_2 = (state[i - 5] << 24) | (state[i - 6] >> 8)
        y_3 = state[i - 5] >> 8

        state[i] ^= x_0 ^ b_0 ^ d_0 ^ y_0
        state[i + 1] ^= x_1 ^ b_1 ^ d_1 ^ y_1
        state[i + 2] ^= x_2 ^ b_2 ^ d_2 ^ y_2
        state[i + 3] ^= x_3 ^ b_3 ^ d_3 ^ y_3

    for i in range(136, 624, 4):
        x_0 = state[i] << 8
        x_1 = (state[i + 1] << 8) | (state[i] >> 24)
        x_2 = (state[i + 2] << 8) | (state[i + 1] >> 24)
        x_3 = (state[i + 3] << 8) | (state[i + 2] >> 24)

        b_0 = (state[i - 136] >> 11) & 0xDFFFFFEF
        b_1 = (state[i - 135] >> 11) & 0xDDFECB7F
        b_2 = (state[i - 134] >> 11) & 0xBFFAFFFF
        b_3 = (state[i - 133] >> 11) & 0xBFFFFFF6

        d_0 = state[i - 4] << 18
        d_1 = state[i - 3] << 18
        d_2 = state[i - 2] << 18
        d_3 = state[i - 1] << 18

        y_0 = (state[i - 7] << 24) | (state[i - 8] >> 8)
        y_1 = (state[i - 6] << 24) | (state[i - 7] >> 8)
        y_2 = (state[i - 5] << 24) | (state[i - 6] >> 8)
        y_3 = state[i - 5] >> 8

        state[i] ^= x_0 ^ b_0 ^ d_0 ^ y_0
        state[i + 1] ^= x_1 ^ b_1 ^ d_1 ^ y_1
        state[i + 2] ^= x_2 ^ b_2 ^ d_2 ^ y_2
        state[i + 3] ^= x_3 ^ b_3 ^ d_3 ^ y_3

if USE_NUMBA:
    import numba
    from numba.core.imputils import impl_ret_untracked
    from llvmlite import ir

    I8 = ir.IntType(8)
    I32 = ir.IntType(32)
    I64 = ir.IntType(64)
    I64_ZERO = ir.Constant(I64, 0)
    I64_4 = ir.Constant(I64, 4)
    VI32X4 = ir.VectorType(I32, 4)
    VI32X4_18 = ir.Constant(VI32X4, 18)
    VI32X4_11 = ir.Constant(VI32X4, 11)
    VI32X16 = ir.VectorType(I32, 16)
    VI8X16 = ir.VectorType(I8, 16)
    VI8X16_ZERO = ir.Constant(VI8X16, None)
    MASK = ir.Constant(VI32X4, (0xDFFFFFEF, 0xDDFECB7F, 0xBFFAFFFF, 0xBFFFFFF6))
    SHIFT_LEFT_MASK = ir.Constant(
        VI32X16, [16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
    )
    SHIFT_RIGHT_MASK = ir.Constant(
        VI32X16, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
    )

    @optional_ir_function(sfmt_shuffle, numba.void(numba.uint32[::1]))
    def sfmt_shuffle_ir(context, builder: ir.IRBuilder, signature, arguments):
        """LLVM IR implementation of sfmt_shuffle"""
        # sfmt_shuffle_ir assumes that arguments[0] is a (624,) np.array of dtype np.uint32
        # it does not do any bounds checking or verifying the shape/size of the array or integers

        # state_obj is a reference to an np.array object
        # np.array objects are stored as follows:
        # {
        #     meminfo: i8*,
        #     parent: i8*,
        #     nitems: i64,
        #     itemsize: i64,
        #     data: dtype*,
        #     shape: [ndim x i64],
        #     strides: [ndim x i64]
        # }
        (state_obj,) = arguments
        # the actual pointer to the data of an np.array is stored at index 4
        state_array_pointer = builder.extract_value(state_obj, 4)

        # store index in a pointer to avoid needing to rewrite variable in loop
        # TODO: overwriting variable is probably faster than accessing memory over and over
        # but it wasn't obvious how to do this
        index_ptr = builder.alloca(I64)
        builder.store(I64_ZERO, index_ptr)

        # allocated memory for temporarily storing a <i32 x 4>/<i8 x 16> vector
        temp_vi32x4 = builder.alloca(VI32X4)
        temp_vi8x16 = builder.bitcast(temp_vi32x4, VI8X16.as_pointer())

        # equivalent to _mm_slli_si128(vec, 1)/_mm_srli_si128(vec, 1) depending on mask
        def shift_bytes(vec_ptr, mask):
            vec_ptr_vi8x16 = builder.bitcast(vec_ptr, VI8X16.as_pointer())
            vec_vi8x16 = builder.shuffle_vector(
                builder.load(vec_ptr_vi8x16), VI8X16_ZERO, mask
            )
            builder.store(vec_vi8x16, temp_vi8x16)
            return builder.load(temp_vi32x4)

        def mm_recursion(ofs):
            index = builder.load(index_ptr)
            index_b = builder.add(index, ir.Constant(I64, ofs[0]))
            index_c = builder.add(index, ir.Constant(I64, ofs[1]))
            index_d = builder.add(index, ir.Constant(I64, ofs[2]))

            vec_a_ptr = builder.bitcast(
                builder.gep(state_array_pointer, (index,)), VI32X4.as_pointer()
            )
            vec_b_ptr = builder.bitcast(
                builder.gep(state_array_pointer, (index_b,)), VI32X4.as_pointer()
            )
            vec_c_ptr = builder.bitcast(
                builder.gep(state_array_pointer, (index_c,)), VI32X4.as_pointer()
            )
            vec_d_ptr = builder.bitcast(
                builder.gep(state_array_pointer, (index_d,)), VI32X4.as_pointer()
            )

            vec_a = builder.load(vec_a_ptr)
            vec_b = builder.load(vec_b_ptr)
            vec_d = builder.load(vec_d_ptr)

            vec_x = shift_bytes(vec_a_ptr, SHIFT_LEFT_MASK)
            vec_y = shift_bytes(vec_c_ptr, SHIFT_RIGHT_MASK)

            vec_b1 = builder.and_(builder.lshr(vec_b, VI32X4_11), MASK)
            vec_d1 = builder.shl(vec_d, VI32X4_18)

            vec_a = builder.xor(
                builder.xor(builder.xor(builder.xor(vec_a, vec_x), vec_b1), vec_y),
                vec_d1,
            )

            builder.store(vec_a, vec_a_ptr)

            index = builder.add(index, I64_4)
            builder.store(index, index_ptr)

            return index

        def loop_mm_recursion(ofs, last_index, block=None):
            # first block needs to be created and branched to
            if block is None:
                block = builder.append_basic_block()
                builder.branch(block)
            # create next block to allow jumping to it
            next_block = builder.append_basic_block()
            with builder.goto_block(block):
                # do mm_recursion and check current index
                index = mm_recursion(ofs)
                # if index == last_index: jump to next_block else: jump back to the start of loop
                builder.cbranch(
                    builder.icmp_unsigned("==", index, ir.Constant(I64, last_index)),
                    next_block,
                    block,
                )
            # return the next block
            return next_block

        # compute first two states manually
        mm_recursion((488, 616, 620))
        mm_recursion((488, 616, -4))
        # split the loop to avoid modulo
        next_block = loop_mm_recursion((488, -8, -4), 136)
        final_block = loop_mm_recursion((-136, -8, -4), 624, block=next_block)

        builder.position_at_end(final_block)

        # return void as the array is changed in-place
        return impl_ret_untracked(context, builder, signature.return_type, None)

# TODO: jump tables
# TODO: staticmethod const functions
# TODO: reverse next
@optional_jitclass
class SIMDFastMersenneTwister:
    """SIMD-oriented Fast Mersenne Twister Pseudo Random Number Generator"""

    state: array_type(np.uint32)  # contiguous array
    index: np.uint16

    def __init__(self, seed: np.uint32) -> None:
        seed = np.uint32(seed)
        self.state = np.empty(624, dtype=np.uint32)
        self.index = np.uint16(624)  # ensures shuffle after initialization
        self.re_init(seed)

    def re_init(self, seed: np.uint32) -> None:
        """Reinitialize without creating a new object"""
        seed = np.uint32(seed)
        self.state[0] = seed
        self.index = np.uint16(624)  # ensures shuffle after initialization
        inner = np.uint32(seed & np.uint32(1))

        for i in range(1, 624):
            seed = np.uint32(
                np.uint32(0x6C078965) * (seed ^ (seed >> np.uint32(30))) + np.uint32(i)
            )
            self.state[i] = seed

        inner ^= self.state[3] & np.uint32(0x13C9E684)
        inner ^= inner >> 16
        inner ^= inner >> 8
        inner ^= inner >> 4
        inner ^= inner >> 2
        inner ^= inner >> 1

        self.state[0] ^= ~inner & np.uint32(1)

    def advance(self, adv: np.uint32) -> None:
        """Advance SIMD-oriented Fast Mersenne Twister sequence by adv"""
        adv = np.uint32(adv)
        adv = (adv * np.uint32(2)) + np.uint32(self.index)
        while adv >= np.uint32(624):
            self.shuffle()
            adv -= np.uint32(624)
        self.index = np.uint16(adv)

    def next(self) -> np.uint64:
        """Access and return the next 64-bit rand"""
        if self.index == np.uint16(624):
            self.shuffle()

        low = self.state[self.index]
        self.index += 1
        high = self.state[self.index]
        self.index += 1
        return np.uint64(low) | (np.uint64(high) << np.uint64(32))

    def shuffle(self) -> None:
        """Advance and shuffle the entire state (624 advances)"""
        state = self.state

        sfmt_shuffle(state)

        self.index = 0

    def next_rand(self, maximum: np.uint64) -> np.uint64:
        """Generate and return the next [0, maximum) random uint via modulo distribution"""
        return self.next() % np.uint64(maximum)


# Learner: Nguyen Truong Thinh
# Contact me: nguyentruongthinhvn2020@gmail.com || +84393280504
#
# Topic: Supervised Learning: Classifications in Action.
#           The Classifier's Answers

import numpy as np
from ml.util import forward, gradient

def classify(x, w):
    """Predict the value of an unlabeled image"""
    y_hat = forward(x, w)
    labels = np.argmax(y_hat, axis=1)
    return labels.reshape(-1, 1)

def loss(x, y, w):
    y_hat = forward(x, w)
    first_term = y * np.log(y_hat)
    second_term = (1 - y) * np.log(1 - y_hat)
    return -np.sum(first_term + second_term) / x.shape[0]

def report(iteration, x_train, y_train, x_test, y_test, w):
    matches = np.count_nonzero(classify(x_test, w) == y_test)
    n_test_examples = y_test.shape[0]
    matches = matches * 100.0 / n_test_examples
    training_loss = loss(x_train, y_train, w)
    print("%d - Loss: %.20f, %.2f%%" % (iteration, training_loss, matches))

def base_train(x_train, y_train, x_test, y_test, iterations, lr):
    w = np.zeros((x_train.shape[1], y_train.shape[1]))
    for i in range(iterations):
        report(i, x_train, y_train, x_test, y_test, w)
        w -= gradient(x_train, y_train, w) * lr
    report(iterations, x_train, y_train, x_test, y_test, w)
    return w


import os
import cv2 as cv
import json
import requests
import pandas as pd

class BaselineFunctions:
    @staticmethod
    def load_all_images_of_folder(folder):
        images = []
        file_names = []
        for file_name in os.listdir(folder):
            img = cv.imread(os.path.join(folder, file_name))
            if img is not None:
                images.append(img)
                file_names.append(file_name)
        return images, file_names

    @staticmethod
    def delete_all_files_in_folder(folder):
        for file_name in os.listdir(folder):
            os.remove(os.path.join(folder, file_name))

    @staticmethod
    def download_image(path, phone_src, model_name, web_prefix=''):
        path_to_file = path + str(model_name) + ".webp"
        if not os.path.exists(path_to_file):
            try:
                phone_link = web_prefix + str(phone_src)
                print("Downloading: " + phone_link)
                phone_img = requests.get(phone_link, stream=True).content
                with open(path_to_file, "wb+") as f:
                    f.write(phone_img)
            except:
                print("Error: Download of " + str(model_name) + " did not succeed")
                pass
        else:
            print("Image already exists at : " + path_to_file)

    @staticmethod
    def check_if_data_base_entry_exists(path, db_file_name, model_name):
        path_to_file = path + str(db_file_name)
        try:
            with open(path_to_file, 'r') as f:
                data_entries = json.load(f)
            for data_entry in data_entries:
                if data_entry["marketingName"] == str(model_name):
                    return True
            return False
        except:
            print("Error: DataBaseFile " + str(db_file_name) + " could not be opened")
            pass

    @staticmethod
    def nfc_chip_coordinates_in_percent(x0_nfc, y0_nfc, x1_nfc, y1_nfc, x0_edge, y0_edge, x1_edge, y1_edge):
        # x0-phoneEdge is 0
        x0_nfc = x0_nfc - x0_edge
        x1_nfc = x1_nfc - x0_edge
        x1_edge = x1_edge - x0_edge
        # y0-phoneEdge is 0
        y0_nfc = y0_nfc - y0_edge
        y1_nfc = y1_nfc - y0_edge
        y1_edge = y1_edge - y0_edge
        # NFC Chip Coordinates in Percent of Phone Screen
        x0_nfc = BaselineFunctions.normalize(float(x0_nfc / x1_edge))
        y0_nfc = BaselineFunctions.normalize(float(y0_nfc / y1_edge))
        x1_nfc = BaselineFunctions.normalize(float(x1_nfc / x1_edge))
        y1_nfc = BaselineFunctions.normalize(float(y1_nfc / y1_edge))
        # mirror x-axis
        x0_nfc = 1 - x0_nfc
        x1_nfc = 1 - x1_nfc
        return x1_nfc, y0_nfc, x0_nfc, y1_nfc

    @staticmethod
    def normalize(x):
        if x < 0.0:
            return 0.0
        elif x > 1.0:
            return 1.0
        else:
            return x

    @staticmethod
    def format_coordinates(manufacturer, name, model_names, x0, y0, x1, y1):
        name = name.split(".")[0]
        nfc_chip_location = {
            "manufacturer": manufacturer,
            "marketingName": name,
            "modelNames": model_names,
            "nfcPos": {
                "x0": x0,
                "y0": y0,
                "x1": x1,
                "y1": y1
            }
        }
        return nfc_chip_location

    @staticmethod
    def write_to_json_file(nfc_chip_locations, path, filename):
        full_path = path + filename
        try:
            if os.path.exists(full_path):
                data = []
                with open(full_path, 'r') as f:
                    data_entries = json.load(f)
                for data_entry in data_entries:
                    data.append(data_entry)
                for nfc_chip_location in nfc_chip_locations:
                    if not nfc_chip_location["marketingName"] in data:
                        data.append(nfc_chip_location)
                with open(full_path, 'w', encoding='utf-8') as f:
                    json.dump(data, f, ensure_ascii=False, indent=5)
            else:
                with open(full_path, 'w', encoding='utf-8') as f:
                    json.dump(nfc_chip_locations, f, ensure_ascii=False, indent=5)
        except:
            print("Error: Path " + str(full_path) + " could not be reached")
            pass

    @staticmethod
    def load_google_play_device_list():
        url = "https://storage.googleapis.com/play_public/supported_devices.csv"
        return pd.read_csv(url, encoding="utf-16")


"""Drop tip in place command request, result, and implementation models."""
from __future__ import annotations
from pydantic import Field, BaseModel
from typing import TYPE_CHECKING, Optional, Type
from typing_extensions import Literal

from .pipetting_common import PipetteIdMixin
from .command import AbstractCommandImpl, BaseCommand, BaseCommandCreate

if TYPE_CHECKING:
    from ..execution import TipHandler

DropTipInPlaceCommandType = Literal["dropTipInPlace"]

class DropTipInPlaceParams(PipetteIdMixin):
    """Payload required to drop a tip in place."""

    homeAfter: Optional[bool] = Field(
        None,
        description=(
            "Whether to home this pipette's plunger after dropping the tip."
            " You should normally leave this unspecified to let the robot choose"
            " a safe default depending on its hardware."
        ),
    )

class DropTipInPlaceResult(BaseModel):
    """Result data from the execution of a DropTipInPlace command."""

    pass

class DropTipInPlaceImplementation(
    AbstractCommandImpl[DropTipInPlaceParams, DropTipInPlaceResult]
):
    """Drop tip in place command implementation."""

    def __init__(
        self,
        tip_handler: TipHandler,
        **kwargs: object,
    ) -> None:
        self._tip_handler = tip_handler

    async def execute(self, params: DropTipInPlaceParams) -> DropTipInPlaceResult:
        """Drop a tip using the requested pipette."""
        await self._tip_handler.drop_tip(
            pipette_id=params.pipetteId, home_after=params.homeAfter
        )

        return DropTipInPlaceResult()

class DropTipInPlace(BaseCommand[DropTipInPlaceParams, DropTipInPlaceResult]):
    """Drop tip in place command model."""

    commandType: DropTipInPlaceCommandType = "dropTipInPlace"
    params: DropTipInPlaceParams
    result: Optional[DropTipInPlaceResult]

    _ImplementationCls: Type[
        DropTipInPlaceImplementation
    ] = DropTipInPlaceImplementation

class DropTipInPlaceCreate(BaseCommandCreate[DropTipInPlaceParams]):
    """Drop tip in place command creation request model."""

    commandType: DropTipInPlaceCommandType = "dropTipInPlace"
    params: DropTipInPlaceParams

    _CommandCls: Type[DropTipInPlace] = DropTipInPlace


from flask import Flask, request, render_template, jsonify
from chatbot.chatbot import get_response, predict_class
import json

app= Flask(__name__)

intents = json.loads(open('Project/chatbot/intents.json').read())

@app.route('/', methods=['GET', 'POST'])
def home():
    chatbot_response = ''

    if request.method == 'POST':
        user_input = request.form['user_input']
        chatbot_response = get_response(predict_class(user_input), intents)
        return jsonify({'response': chatbot_response})

    return render_template('index.html')

if __name__ == '__main__':
    app.run(debug=True)


import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris

iris = load_iris()

def short(x):
    if x < 0.6:
        return -(x/0.6)+1
    else:
        return 0

def medium(x):
    if x < 0.6:
        return x/0.6
    else:
        return (x-1)/(0.6-1)

def tall(x):
    if x > 0.6:
        return (x-0.6)/(1-0.6)
    else:
        return 0

# Returns a list where every element is the probability of each species
def rule_eval(SL, SW, PL, PW):
    rule1 = min(min(min( max(SL[0], SL[2]), max(SW[1], SW[2] )), max(PL[1], PL[2])), PW[1])
    rule2 = min(max(PL[0], PL[1]), PW[0])
    rule3 = min(min(max(SW[0], SW[1]), PL[2]), PW[2])
    rule4 = min(min(min(SL[1], max(SW[0], SW[1])), PL[0]), PW[2])

    return [rule2, rule1 + rule4, rule3]

correct_gusses = 0

for x, y in enumerate(iris['data']):

    inp1 = y[0]
    inp2 = y[1]
    inp3 = y[2]
    inp4 = y[3]

    an = ((inp1 - 4.3) / (7.9 - 4.3))
    bn = ((inp2 - 2) / (4.4 - 2))
    cn = ((inp3 - 1) / (6.9 - 1))
    dn = ((inp4 - 0.1) / (2.50 - 0.1))

    SL = [short(an), medium(an), tall(an)]
    SW = [short(bn), medium(bn), tall(bn)]
    PL = [short(cn), medium(cn), tall(cn)]
    PW = [short(dn), medium(dn), tall(dn)]

    result = rule_eval(SL, SW, PL, PW)

    species = result.index(max(result))
    if species == 0:
        print(x, '\tSetosa\t\t', iris['target'][x], '\t', result)

    elif species == 1:
        print(x, '\tVersicolor\t', iris['target'][x], '\t', result)

    elif species == 2:
        print(x, '\tVirginica\t', iris['target'][x], '\t', result)

    if species == iris['target'][x]:
        correct_gusses = correct_gusses + 1

print('Predictability: ', correct_gusses/150)


import spacy
import requests

def main():
    nlp = spacy.load('en_core_web_md')
    nlp.add_pipe('entityLinker', last=True)
    #question = input('Please ask a question:\n')
    parse = nlp("What movie won best picture in the 2020 Academy Awards?")
    for sent in parse.sents:
        sent._.linkedEntities.pretty_print()
    #entities = parse._.linkedEntities.pretty_print()
    verb_dict = {'play': ('actor', 'role'), 'won': ('award', 'winner'), 'release': ('release date', 'publication place')}
    if len(entities) == 2:
        for word in parse:
            if word.dep_ == 'root' and word.lemma_ in verb_dict:
                prop = verb_dict[word.lemma_]
    # Get property uri
    url = 'https://www.wikidata.org/w/api.php'
    params = {'action': 'wbsearchentities',
              'language': 'en',
              'format': 'json',
              'type': 'property'}
    prop_ids = list()
    for p in prop:
        params['search'] = p
        json = requests.get(url, params).json()
        if json['search']:
            prop_ids.append(json['search'][0]['id'])
        else:
            return 'No answer found'

    # create SPARQL query for each key-word
    answers = list()
    entity1 = 'Q' + entities[0]
    entity2 = 'Q' + entities[1]
    query = "SELECT ?answerLabel WHERE { wd:" + entity2 + " p:" + prop_ids[0] \
            + " [ ps:" + prop_ids[0] + " ?answer; pq:" + prop_ids[1] + "wd:" + entity1 \
            + "]. SERVICE wikibase:label { bd:serviceParam wikibase:language 'en' }}"
    url = 'https://query.wikidata.org/sparql'
    results = requests.get(url, params={'query': query,
                                        'format': 'json'}).json()
    if results['results']['bindings']:
        for result in results['results']['bindings']:
            answers.append(result['answerLabel']['value'])
    return '\n'.join(set(answers))

if __name__ == '__main__':
    main()


import random
import json
class car(object):
    def __init__(self,  hp, wd, price, seats,
    running, dtp, type, v_engine,
    engine_type, years, fuel_economy, acceleration, top_speed,
     v_trunk, box):
        self.hp=hp
        self.wd=wd
        self.price=price
        self.seats=seats
        self.running=running
        self.dtp=dtp
        self.type=type
        self.v_engine=v_engine
        self.v_trunk=v_trunk
        self.engine_type=engine_type
        self.years=years
        self.fuel_economy=fuel_economy
        self.acceleration=acceleration
        self.box=box
        self.top_speed=top_speed
def how_many_seats(type):
    if type in ['SUV', 'Sedan', 'Hatcback', 'Crossover', 'Pickup truck']:
        return 5
    elif type in ['Coupe', 'Sports car', 'Racecar', 'hypercar', 'muscle car', 'gt']:
        return 2
    elif type=='Minivan':
        return 7

def generate_car():
    hp=random.randint(50, 2000)
    wd=random.choice(['AWD', 'FWD', 'RWD'])
    price=random.randint(50, 100000)*1000
    running=random.randint(1, 500)*1000*random.choice([0, 1])
    dtp=random.randint(0, 10)*random.choice([0, 0, 0, 0, 0, 0, 1, 1, 0])
    type=random.choice(['SUV', 'Sedan', 'Hatcback', 'Crossover', 'Coupe', 'Minivan', 'Sports car', 'Racecar', 'Pickup truck', 'hypercar', 'muscle car', 'gt'])
    seats = how_many_seats(type)
    v_engine=round(random.uniform(1.6, 10.), 1)
    v_trunk=random.randint(100, 1000)
    engine_type=random.choice(['disel', 'gasoline', 'electro', 'hybrid'])
    years=random.randint(0, 40)
    fuel_economy=round(random.uniform(5., 20.), 1)
    acceleration=round(random.uniform(1.9,10.), 1)
    box=random.choice(['auto', 'robot', 'mannual'])
    top_speed=random.randint(150, 500)
    return car( hp, wd, price, seats,
    running, dtp, type, v_engine,
    engine_type, years, fuel_economy, acceleration, top_speed,
     v_trunk, box)
lists_of_cars={}
v=generate_car()
lis=[]
for i in range(1000):
    v=generate_car()
    lis=[v.hp, v.wd, v.price, v.seats,
    v.running, v.dtp, v.type, v.v_engine,
    v.engine_type, v.years, v.fuel_economy, v.acceleration, v.top_speed,
     v.v_trunk, v.box]
    lists_of_cars.update({'car #'+str(i) : lis})

print (lists_of_cars)

with open("cars.json", "w") as ile:
     json.dump(lists_of_cars, ile)


#!/usr/bin/python3
"""
Prototype: def minOperations(n)
Returns an integer
If n is impossible to achieve, return 0
Example:

n = 9

H => Copy All => Paste => HH => Paste =>HHH => Copy All =>
Paste => HHHHHH => Paste => HHHHHHHHH

Number of operations: 6
"""

def minOperations(n):
    """
    Calculates the fewest number of operations needed to
    result in exactly n H characters in the file.
    """
    if n <= 1:
        return 0
    else:
        for i in range(2, n + 1):
            if n % i == 0:
                return minOperations(int(n / i)) + i
        return n


import logging
import jsonpickle
from core.EmailEnvelope import EmailEnvelope
from core.filtering.filters.PastFilter import PastFilter
from typing import Sequence

class AIFilter(PastFilter):
    checks: Sequence[str] = ["get_count_urls", "get_count_images", "get_from_return_path", "get_from_reply_to",
                             "get_id_email_client"]
    all_content_types = ["text/html", "text/plain", "multipart/mixed", "application/octet-stream",
                         "multipart/alternative", "multipart/related", "image/jpeg", "image/gif", "message/rfc822",
                         "text/plain charset=us-ascii", "image/png", "text/x-vcard", "image/bmp",
                         "application/x-zip-compressed", "multipart/signed", "application/pgp-signature",
                         "text/enriched", "application/ms-tnef", "video/mng", "application/x-pkcs7-signature",
                         "multipart/report", "message/delivery-status", "text/rfc822-headers",
                         "application/x-java-applet", "application/x-patch"]
    all_extension_types = ["txt", "html", "jpg", "png", "gif", "lst", "JPG", "htm", "doc", "GIF", "JPE", "b64", "BIN",
                           "Jpg", "zip", "dat", "rar", "bmp", "jpe", "gz", "PDF.html", "url", "ng", "spec.patch",
                           "spec", "patch", "p7s", "am", "jpeg"]

    def set_initial_data(self, data):
        """
        This method initialize the data which was read from the respective model's file
        :param data: data read of the model's file
        """
        self.data = jsonpickle.decode(data)

    def filter(self, envelope: EmailEnvelope) -> bool:
        """
        This method return the classification of the email, if 1 is Spa,
        :param envelope: email
        :return: if 1 then is spam and return true, else false
        """
        email_params = envelope.ai_matrix_for_email()
        is_spam = True if self.data.predict([email_params]) == [1] else False
        if is_spam:
            logging.info(f"An AIFilter has detected the email as spam")
        return is_spam


import sqlite3
import time
import random
import json
import requests

try:
    import telepot
except(ImportError, IOError):
    print('not found a telegram api - require telepot' + '\n'
          'run command: "pip install telepot"')
    exit('Closing...')

token = input('Введите ваш токен ключ: ')

"""создание шаблона базы данных"""
DataBaseName = 'userd.db'
conn = sqlite3.connect(DataBaseName)
c = conn.cursor()
c.execute("CREATE TABLE IF NOT EXISTS cities (names)")
c.execute("CREATE TABLE IF NOT EXISTS users (id, msg)")
j = requests.get('http://api.hh.ru/areas')
foo = json.loads(j.content.decode('utf-8'))

"""проводит проверку на заполненность поля names в таблице
   cities и если нет заполняет их именами городов"""
c.execute('SELECT * FROM cities')
if len(c.fetchall()) == 0:
    for i in foo:
        f = i['areas']
        for j in f:
            d = j['areas']
            for e in d:
                g = [(e['name'])]
                c.execute("INSERT INTO cities VALUES (?)", g)
    conn.commit()
else:
    print('-' * 70)

def handle(msg):
    text_type, chat_type, chat_id = telepot.glance(msg)
    answer_names = []
    g = msg
    ask = g['text']

    """открытие бд для чтение внутри функции"""
    DataBaseName = 'userd.db'
    conn = sqlite3.connect(DataBaseName)
    c = conn.cursor()
    c.execute('SELECT * FROM cities')

    """получение все имен для отбора из них по первой букве"""
    all_names = c.fetchall()
    if ask != '/start':
        bot.sendChatAction(chat_id, 'typing')
        answer_names.clear()
        c.execute("INSERT INTO users VALUES (?,?)", (chat_id, ask))
        conn.commit()
        for row in all_names:
            h = row[0]
            if h[0].title() == ask[-1].title():
                answer_names.append(row[0])
        g = len(answer_names)
        if g == 0:
            bot.sendMessage(chat_id, 'Воав, потише братишка. Я не '
                            'знаю городов на такую букву. '
                            'Давай на другую букву.')
        else:
            choice = random.randrange(0, g)
            time.sleep(1)
            bot.sendMessage(chat_id, answer_names[choice])

bot = telepot.Bot(token)
bot.message_loop(handle)
print('Listening ...')
while True:
    f = input('input yes for closing ==> ')
    print('-' * 72)
    if f == 'yes':
        conn.close()
        break


from __future__ import absolute_import

from concurrent import futures
import time
import json
import functools

import grpc

from coconut import registry
from coconut.rpc_server import Runner
from coconut.protos.runner import runner_pb2_grpc
from coconut.protos.tasks import task_pb2
from coconut.backend import Backend
from coconut.registry.service import Service
from coconut.task import Task
from coconut.utils import get_lan_ip, new_uuid

_ONE_DAY_IN_SECONDS = 60 * 60 * 24

_DEFAULT_SERVICE_NAME = 'coconut-python-runner'

class Coconut(object):
    def __init__(
            self,
            name=_DEFAULT_SERVICE_NAME,
            host=None,
            port=6666,
            registry_host='127.0.0.1',
            registry_port=8500):

        self.name = name
        if host is None:
            self.host = get_lan_ip()
        else:
            self.host = host
        self.port = port
        self.kind = 'coconut-runner'

        self.registry = registry.Consul(registry_host, registry_port)
        self.backend = Backend(self.registry)
        self.funcs = {}

    def __process(self, request, context):
        if request.key not in self.funcs:
            context.set_code(500)
            context.set_details("Task not found")
            return task_pb2.Reply()

        f = self.funcs[request.key]

        try:
            task = Task(f, request)
            results = task.call()

            return task_pb2.Reply(results=[task_pb2.Result(type=result.type, value=result.encode_value()) for result in results])
        except Exception as e:
            print("Call task failed {}".format(e))
            context.set_code(500)
            context.set_details(str(e))
            return task_pb2.Reply()

    def serve(self):
        """
        serve task request
        """
        server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
        runner_pb2_grpc.add_RunnerServicer_to_server(
            Runner(self.__process), server)
        server.add_insecure_port('{}:{}'.format(self.host, self.port))
        server.start()

        self.server = server

        print('grpc Server started on {}:{}'.format(self.host, self.port))
        self.__register_service()

        try:
            while True:
                time.sleep(_ONE_DAY_IN_SECONDS)
        except KeyboardInterrupt:
            self.__deregister_service()
            server.stop(0)

    def publish(self, service_name, key, args=None, countdown=0):
        response = self.backend.publish(service_name, key, args, countdown)
        return response.uuid

    def task(self, *args, **kwargs):
        """
        register task function
        """
        def wrapper(f):

            key = kwargs.get('key') or f.__name__
            self.funcs[key] = f

            @functools.wraps(f)
            def __func(*args, **kwargs):
                return f(*args, **kwargs)

            return __func
        return wrapper

    def __register_service(self):
        service = Service(
            self.name,
            new_uuid(),
            self.kind,
            self.host,
            self.port
        )
        print("Registering {}: {}".format(service.kind, service.service_id))
        self.registry.register(service)
        self.service = service

    def __deregister_service(self):
        if self.service is not None:
            print("Deregistering {}...".format(self.service.kind))
            self.registry.deregister(self.service)


from pyspark.sql import SparkSession
from datetime import datetime
import sys
import configparser
import pyspark.sql.functions as F
from pyspark.sql.types import *
import pandas as pd

config = configparser.ConfigParser()
config.read('/app/mount/parameters.conf')
data_lake_name=config.get("general","data_lake_name")
data_path=config.get("general","data_path")
username=config.get("general","username")

cloudPath=data_lake_name+data_path

print("Running as Username: ", username)

spark = SparkSession \
    .builder \
    .appName("ICEBERG LOAD") \
    .config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog")\
    .config("spark.sql.catalog.spark_catalog.type", "hive")\
    .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions")\
    .config("spark.kubernetes.access.hadoopFileSystems", data_lake_name)\
    .getOrCreate()

#---------------------------------------------------
#               INSERT DATA
#---------------------------------------------------

# STORE TIMESTAMP BEFORE INSERTS
now = datetime.now()
timestamp = datetime.timestamp(now)
print("PRE-INSERT TIMESTAMP: ", timestamp)

print("PRE-INSERT TABLE SNAPSHOTS AND HISTORY")
spark.sql("SELECT * FROM spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_{0}.history".format(username)).show(20, False)
spark.sql("SELECT * FROM spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_{0}.snapshots".format(username)).show(20, False)

# PRE-INSERT COUNT
print("PRE-INSERT COUNT")
spark.sql("SELECT COUNT(*) FROM spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_{0}".format(username)).show()

# INSERT DATA APPROACH 1 - APPEND FROM DATAFRAME
temp_df = spark.sql("SELECT * FROM spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_{0}".format(username)).sample(fraction=0.1, seed=3)
temp_df.writeTo("spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_{0}".format(username)).append()

# INSERT DATA APPROACH 2 - INSERT VIA SQL
spark.sql("DROP TABLE IF EXISTS spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_SAMPLE_{0}".format(username))
temp_df.writeTo("spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_SAMPLE_{0}".format(username)).create()

print("INSERT DATA VIA SPARK SQL")
query_5 = """INSERT INTO spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_{0} SELECT * FROM spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_SAMPLE_{0}""".format(username)
print(query_5)
spark.sql(query_5)

#---------------------------------------------------
#               TIME TRAVEL
#---------------------------------------------------

# NOTICE SNAPSHOTS HAVE BEEN ADDED
#spark.read.format("iceberg").load("spark_catalog.{}_CAR_DATA.CAR_SALES.history".format(username)).show(20, False)
spark.sql("SELECT * FROM spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_{0}.history".format(username)).show(20, False)
spark.sql("SELECT * FROM spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_{0}.snapshots".format(username)).show(20, False)

# POST-INSERT COUNT
print("POST-INSERT COUNT")
spark.sql("SELECT COUNT(*) FROM spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_{0}".format(username)).show()

# TIME TRAVEL AS OF PREVIOUS TIMESTAMP
df = spark.read.option("as-of-timestamp", int(timestamp*1000)).format("iceberg").load("spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_{0}".format(username))

# POST TIME TRAVEL COUNT
print("POST-TIME TRAVEL COUNT")
print(df.count())

#---------------------------------------------------
#               INCREMENTAL REPORT
#---------------------------------------------------

# ICEBERG TABLE HISTORY (SHOWS EACH SNAPSHOT AND TIMESTAMP)
spark.sql("SELECT * FROM CDE_WORKSHOP_{0}.CAR_SALES_{0}.history;".format(username)).show()

# ICEBERG TABLE SNAPSHOTS (USEFUL FOR INCREMENTAL QUERIES AND TIME TRAVEL)
spark.sql("SELECT * FROM CDE_WORKSHOP_{0}.CAR_SALES_{0}.snapshots;".format(username)).show()

# STORE FIRST AND LAST SNAPSHOT ID'S FROM SNAPSHOTS TABLE
snapshots_df = spark.sql("SELECT * FROM CDE_WORKSHOP_{0}.CAR_SALES_{0}.snapshots;".format(username))

print("SHOWING SNAPSHOTS DF")
#snapshots_df.show()

snapshots = snapshots_df.toPandas()

print("SHOWING ICEBERG SNAPSHOTS DF")
print(snapshots.head())

last_snapshot = str(snapshots[["snapshot_id"]].iloc[-1,0])
second_last_snapshot = str(snapshots[["snapshot_id"]].iloc[-2,0])

print("PICKING LAST AND SECOND LAST ICEBERG SNAPSHOTS FOR REPORT \n")

print(last_snapshot)
print(second_last_snapshot)

#last_snapshot = snapshots_df.select("snapshot_id").tail(1)[0][0]
#first_snapshot = snapshots_df.select("snapshot_id").head(1)[0][0]
print("INCREMENTAL REPORT\n")
print("SELECT ALL UPDATES BETWEEN THE SELECTED SNAPSHOTS\n")
spark.read\
    .format("iceberg")\
    .option("start-snapshot-id", second_last_snapshot)\
    .option("end-snapshot-id", last_snapshot)\
    .load("spark_catalog.CDE_WORKSHOP_{0}.CAR_SALES_{0}".format(username)).show()

#---------------------------------------------------
#               ANALYTICAL QUERIES
#---------------------------------------------------

"""reports_df = spark.sql("SELECT * FROM CDE_WORKSHOP_{0}.SALES_REPORT_{0}".format(username))

print("GROUP TOTAL SALES BY MODEL")
model_sales_df = reports_df.groupBy("Model").sum("Saleprice").na.drop().sort(F.asc('sum(Saleprice)')).withColumnRenamed("sum(Saleprice)", "sales_by_model")
model_sales_df = model_sales_df.withColumn('total_sales_by_model', model_sales_df.sales_by_model.cast(DecimalType(18, 2)))
model_sales_df.select(["Model", "total_sales_by_model"]).sort(F.asc('Model')).show()

print("GROUP TOTAL SALES BY GENDER")
gender_sales_df = reports_df.groupBy("Gender").sum("Saleprice").na.drop().sort(F.asc('sum(Saleprice)')).withColumnRenamed("sum(Saleprice)", "sales_by_gender")
gender_sales_df = gender_sales_df.withColumn('total_sales_by_gender', gender_sales_df.sales_by_gender.cast(DecimalType(18, 2)))
gender_sales_df.select(["Gender", "total_sales_by_gender"]).sort(F.asc('Gender')).show()
"""


from os.path import expanduser

import torch
from joblib import Memory

def torch_cached(func):
    mem = Memory(expanduser('~/cache'))

    def cached_func(*args, **kwargs):
        def false_func(*processed_args, **processed_kwargs):
            return func(*args, **kwargs)
        processed_args = []
        for arg in args:
            if isinstance(arg, torch.Tensor):
                arg = arg.cpu().numpy()
            processed_args.append(arg)
        processed_kwargs = {}
        for k, kwarg in kwargs.items():
            if isinstance(kwarg, torch.Tensor):
                kwarg = kwarg.cpu().numpy()
            processed_kwargs[k] = kwarg
        return mem.cache(false_func)(*processed_args, **processed_kwargs)

    return cached_func


# common
import threading
# selenium
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from crawler_log import *

class Sem():
    def __init__(self, num = 2):
        self.sem = threading.Semaphore(value = num)
        self.flag = False
    def init(self, num):
        if not self.flag:
            self.sem = threading.Semaphore(value = num)
            self.flag = True
    def acquire(self):
        self.sem.acquire()
    def release(self):
        self.sem.release()
sem = Sem()

def getChromeDriver(options):
    opts = webdriver.ChromeOptions()
    for opt in options:
        opts.add_argument(opt)
    driver = webdriver.Chrome(options = opts)
    return driver

def isElementPresent(driver, by, value):
        """
        用来判断元素标签是否存在，
        """
        try:
            driver.find_element(by=by, value=value)
        # 原文是except NoSuchElementException, e:
        except NoSuchElementException as e:
            # 发生了NoSuchElementException异常，说明页面中未找到该元素，返回False
            return False
        else:
            # 没有发生异常，表示在页面中找到了该元素，返回True
            return True

# 点击元素并打开新标签页
def clickOpenWindow(driver, element, index, by, value):
    LOGI('switch to window {}'.format(index))
    driver.switch_to.window(index)
    LOGI('click {}'.format(value))
    element.find_element(by, value).click()
    newWindow= driver.window_handles[len(driver.window_handles)-1]
    return newWindow


from anndata import AnnData
import numpy as np
from gssnng.smoothing import get_smoothing_matrix
from scipy import sparse

def test_get_smoothing_matrix():
    """
    assert the the rows of the smoothing matrix are normalized to 1
    """
    ncells = 4
    ngenes = 3

    adjacency = np.array([
        [0, 0.1, 0, 0],
        [0, 0, 0, 0],  # weird edge case where one cell doesnt have any neighbors!
        [0, 0, 0, 1],
        [0, 0, 1, 0]])

    adata = AnnData(np.random.rand(ncells,ngenes))
    adata.obsp['distances'] = sparse.csr_matrix(adjacency)

    # adjacency has to ignore the weightings of the neighbors
    S = get_smoothing_matrix(adata, mode='adjacency', add_diag=False)
    np.testing.assert_allclose(S.A.sum(1), [1, 0, 1, 1])

    adata.obsp['connectivities'] = sparse.csr_matrix(adjacency)
    S = get_smoothing_matrix(adata, mode='connectivity', add_diag=False)
    np.testing.assert_allclose(S.A.sum(1), [1, 0, 1, 1])

    # if there's diagnoal elements the results are differnt
    S = get_smoothing_matrix(adata, mode='adjacency', add_diag=True)
    np.testing.assert_allclose(S.A.sum(1), [1, 1, 1, 1])

    adata.obsp['connectivities'] = sparse.csr_matrix(adjacency)
    S = get_smoothing_matrix(adata, mode='connectivity', add_diag=True)
    np.testing.assert_allclose(S.A.sum(1), [1, 1, 1, 1])

test_get_smoothing_matrix()
print('test done')


from lib import util
from lib.targetop import TargetOp

CS = "CS"
SCK = "SCK"
SI = "SI"
SO = "SO"
HOLD = "HOLD"
WP = "WP"

SIZE = 512
CHUNK = 64

Tcss = Tcsd = 500e-9
Tsu = 50e-9
Thd = 100e-9
Ths = 200e-9
Thi = Tlo = 475e-9
Twc = 5e-3

READ = "0000a011 aaaaaaaa"  # iiiiiiii*[0,512]
WRITE = "0000a010 aaaaaaaa"  # dddddddd*[1,16]
DATA_BYTE = "dddddddd"
# WRDI = "00000100"
WREN = "00000110"
RDSR = "00000101" + "xxxxxxxx"
# WRSR = "00000001 0000ddrr"

class Ee25lc040:
    """25LC040 SPI EEPROM
    CS SCK SI SO WP HOLD"""

    def __init__(self, pinproxy, progressbar):
        self.pinproxy = pinproxy
        self.progressbar = progressbar

    def _open(self):
        self.pinproxy.set_as_input(SO)
        for pin in (CS, SCK, SI, HOLD, WP):
            self.pinproxy.set_as_output(pin)
            self.pinproxy.reset_pin(pin)
        self.pinproxy.set_pin(HOLD)
        self.pinproxy.set_pin(WP)

    def read(self):
        self._open()
        for addr in range(0, SIZE, CHUNK):
            self.progressbar.update(addr, SIZE)
            cmd = util.cmd(READ, a=addr) + [0] * 8 * CHUNK
            self._pump(cmd, 16)
        return dict(enumerate(self.pinproxy.pop_fetched(SO)))

    def erase(self):
        self._open()
        self.write({})

    def write(self, mem):
        self._open()
        for page in range(0, SIZE, 16):
            data = [mem.get(page + address, 0xff) for address in range(16)]
            self.progressbar.update(page, SIZE)
            self._wren()
            self.rdsr()
            self._write_page(page, data)
        sr_stats = self.pinproxy.pop_fetched(SO)
        # status_register = _, _, _, _, BP1, BP0, WEL, WIP
        if not all(map(lambda sr: sr & 2, sr_stats)):
            raise Exception("Write failed.")

    def _write_page(self, address, data):
        assert 1 <= len(data) <= 16
        cmd = util.cmd(WRITE, a=address)
        for byte in data:
            cmd += util.cmd(DATA_BYTE, d=byte)
        self._pump(cmd)
        self.pinproxy.wait(Twc)

    def _wren(self):  # enable write
        self._pump(util.cmd(WREN))

#    def _wrdi(self):
#        self._pump(util.cmd(WRDI))

    def rdsr(self):  # read status register
        self._pump(util.cmd(RDSR), 8)

#    def _wrsr(self, bp1, bp0):  # write status register
#        self._pump(util.cmd(WRSR, d=bp1, r=bp0))

    def _pump(self, sequence, read_after=None):
        p = self.pinproxy
        p.reset_pin(CS)
        p.wait(Tcss)

        for i, bit in enumerate(sequence):
            p.set_pin(SI, bit)
            p.wait(Tsu)
            p.set_pin(SCK)
            p.wait(Thi)
            if read_after and i >= read_after:
                p.fetch_pin(SO)
            p.wait(Thd)
            p.reset_pin(SCK)
            p.wait(Tlo)

        p.wait(Tcsd)
        p.set_pin(CS)

@TargetOp
def read(pinproxy, progressbar):
    return Ee25lc040(pinproxy, progressbar).read()

@TargetOp
def write(pinproxy, progressbar, mem):
    return Ee25lc040(pinproxy, progressbar).write(mem)

@TargetOp
def erase(pinproxy, progressbar):
    return Ee25lc040(pinproxy, progressbar).erase()


import random

listnum = [str(val) for val in range(1, 5)]
print(listnum)
print([int(val) for val in listnum])
print(list(map(int, listnum)))

def fun_add(x, y, z):
    return int(x) + int(y) + int(z)

gen = lambda len: [str(random.randrange(30, 100)) for val in range(len)]
list1 = gen(4)
list2 = gen(4)
list3 = gen(4)
print(list1, list2, list3)
print(list(map(fun_add, list1, list2, list3)))
print(list(map(lambda *args: args, list1, list2, list3)))
print(list(map(lambda *args: sum([int(val) for val in args]), list1, list2, list3)))
print(list(map(lambda *args: sum(map(int, args)), list1, list2, list3)))

fpath = 'D:\\important\\pythonCode\\function\\moviePlay.txt'
f = open(fpath,encoding='UTF-8-sig')
result = map(lambda line: line.strip().split()[:2], f.readlines())
result = list(result)
result.sort(key=lambda vals: vals[1])
for val in result:
    print(val)

listn1 = [1, 2, 3]
listn2 = [1, 2]
print(list(map(lambda x, y: x + y, listn1, listn2)))


import numpy as np
from scipy import stats
import seaborn as sns
import statsmodels.stats.api as sms

"""
Bir fabrikada rassal olarak seçilen 100 ürünün ortalama ağırlığı 1040 gr,
std sapması 25 gr olarak bulunmuştur. Bu fabrikada üretilen tüm ürünlerin
ortalama oğırlıkları %95 güven ile kaçtır?
"""
n = 100
x = 1040
s = 25
conf = 0.95
interval = stats.norm.interval(alpha=conf, loc=x, scale=s/np.sqrt(n))
print(interval)

"""
85 ev sahibi ile yapılan bir ankette, ev bakımına aylık ortalama 67$
(std sapma 14$) harcadıkları tespit edilmiştir. Tüm ev sahiplerinin
aylık ev bakım harcamaları için %95 güven aralığı oluşturunuz.
"""
interval = stats.norm.interval(alpha=0.95, loc=67, scale=14/np.sqrt(85))
print(interval)

"""
Piyasaya yeni sürülen bir ürünün uzunluğunun std sapması 2 cm'dir.
Rastgele seçilen 16 ürünün ortalama uzunluğu 4 cm olarak hesaplanmıştır.
%95 güvenle anakütle ortalamasını tahmin ediniz.(n<30 fakat pop. std sapması biliniyor)
"""
interval = stats.norm.interval(alpha=0.95, loc=4, scale=2/np.sqrt(16))
print(interval)

"""
Bir fabrikada üretilen margarin paketlerinin ağırlığının varyansı 100 gr'dır.
Rastgele seçilen 25 paketin ağırlığının ortalaması 120 gr'dır. Ana kütle ortalamasını
%90 ve %99 güvenle tahmin ediniz.
"""
interval = stats.norm.interval(alpha=0.9, loc=120, scale=np.sqrt(100)/np.sqrt(25))
print(interval)
interval = stats.norm.interval(alpha=0.99, loc=120, scale=np.sqrt(100)/np.sqrt(25))
print(interval)

"""
Bir firmanın ürettiği ürünlerin 100 tanesi rassal örneklem olarak seçilmiştir.
Ortalama ağırlık 385 gr ve std sapması 12 gr olarak hesaplanmıştır.
Üretilen ürünlerin ortalama ağırlığını %95 güven düzeyi ile tahmin ediniz.
"""
interval = stats.norm.interval(alpha=0.95, loc=385, scale=12/np.sqrt(100))
print(interval)

###########################################################################
# T Dist.
n = 30
x = 140
s = 25
conf = 0.95
df = n - 1
interval = stats.t.interval(alpha=conf, loc=x, df=df, scale=s/np.sqrt(n))
print(interval)

"""
Rassal olarak seçilen 20 bilgisayarın tamirat maliyetleri kaydedilmiştir.
Seçilen 20 bilgisayarın maliyet ortalamasının 216.53$ olduğu tespit edilmiştir.
Bu örneklemin std sapması 15.86$ olarak hesaplanmıştır.
Tüm bilgisayarların ortalama tamirat maliyetini %95 güven ile tahmin ediniz
"""
n = 20
x = 216.53
s = 15.86
conf = 0.95
df = n - 1
interval = stats.t.interval(alpha=conf, loc=x, df=df, scale=s/np.sqrt(n))
print(interval)

##########################################################################################
# tips veri setindeki sayısal değişkenler için güven aralığı
# confidence interval for numeric variables in tips dateset
df = sns.load_dataset("tips")
df.describe().T
sms.DescrStatsW(df["total_bill"]).tconfint_mean()
sms.DescrStatsW(df["tip"]).tconfint_mean()

# titanic veri setindeki sayısal değişkenler için güven aralığı
# confidence interval for numeric variables in titanic dateset
df = sns.load_dataset("titanic")
df.describe().T
sms.DescrStatsW(df["age"].dropna()).tconfint_mean()
sms.DescrStatsW(df["fare"].dropna()).tconfint_mean()


class Solution:
    def isLongPressedName(self, name, typed):
        """
        :type name: str
        :type typed: str
        :rtype: bool
        """
        i, j = 0, 0
        while i < len(name) and j < len(typed):
            if name[i] == typed[j]:
                i += 1
                j += 1
            else:
                if j > 0 and typed[j] == typed[j - 1]:
                    j += 1
                else:
                    return False
        if i == len(name) and j == len(typed):
            return True
        elif i == len(name) and j < len(typed):
            for m in range(j, len(typed)):
                if typed[m] != name[-1]:
                    return False
            return True
        return False

if __name__ == '__main__':
    solution = Solution()
    name = "vtkgn"
    typed = "vttkgnn"
    print(solution.isLongPressedName(name, typed))


from math import cos, sin, pi

def input():
    with open("12/input.txt", "r") as f:
        return [line.strip() for line in f.readlines()]

def instruction1(line, pos):
    a = line[0]
    if a == "N":
        pos[2] += int(line[1:])
    if a == "S":
        pos[2] -= int(line[1:])
    if a == "E":
        pos[1] += int(line[1:])
    if a == "W":
        pos[1] -= int(line[1:])
    if a == "L":
        pos[0] += int(line[1:])
    if a == "R":
        pos[0] -= int(line[1:])
    if a == "F":
        pos[1] += int(line[1:])*cos(pi / 180 * pos[0])
        pos[2] += int(line[1:])*sin(pi / 180 * pos[0])
    return pos

def rotate(pos, angle):
    angle = angle * pi / 180
    px, py = pos[0], pos[1]

    qx = cos(angle) * px - sin(angle) * py
    qy = sin(angle) * px + cos(angle) * py
    return [qx, qy]

def instruction2(line, ship, wp):
    a = line[0]
    if a == "N":
        wp[1] += int(line[1:])
    if a == "S":
        wp[1] -= int(line[1:])
    if a == "E":
        wp[0] += int(line[1:])
    if a == "W":
        wp[0] -= int(line[1:])
    if a == "L":
        wp = rotate(wp, int(line[1:]))
    if a == "R":
        wp = rotate(wp, -int(line[1:]))
    if a == "F":
        ship[0] += wp[0] * int(line[1:])
        ship[1] += wp[1] * int(line[1:])
    return ship, wp

def manhattan(x, y):
    return abs(x) + abs(y)

def sol1():
    lines = input()

    pos = [0,0,0]

    for line in lines:
        pos = instruction1(line, pos)

    return round(manhattan(pos[1],pos[2]))

def sol2():
    lines = input()

    ship = [0,0]
    wp = [10,1]

    for line in lines:
        ship, wp = instruction2(line, ship, wp)

    return round(manhattan(ship[0],ship[1]))

print(sol1())
print(sol2())


import math

n = int(input())
x = [0]*n
y = [0] * n
ans = 0

for i in range(n):
    x[i], y[i] = map(int, input().split())  # 複数数値入力

for i in range(n - 1):
    for j in range(i+1, n):
        if ((x[i] - x[j]) ** 2 + (y[i] - y[j]) ** 2) > ans:
            ans = (x[i] - x[j]) ** 2 + (y[i] - y[j]) ** 2

print(math.sqrt(ans))


def factorial_num(n1, n2):
    value_numb1 = 1
    value_numb2 = 1
    for i in range(1, n1 + 1):
        value_numb1 *= i
    for y in range(1, num2 + 1):
        value_numb2 *= y
    result = value_numb1 / value_numb2
    print(f'{result:.2f}')

num1, num2 = int(input()), int(input())
factorial_num(num1, num2)


import json
from logging import getLogger
from time import sleep

from celery import task, current_task
from celery.result import AsyncResult
from configobj import ConfigObj
from django import template
from django.contrib.auth.decorators import login_required
from django.db import connection
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from django.template import loader
from django.urls import reverse

from app.models import Satellites, InterfaceProprieties
from .utils.generic import GenericUtils

logger = getLogger(__name__)

register = template.Library()

# page: index
def index(request):
    """

    :param request:
    :return: HttpResponse
    """
    context = {}
    template = loader.get_template('app/home.html')
    return HttpResponse(template.render(context, request))

# all html templates (for dev) TODO: remove
def all_html(request):
    context = {}
    # The template to be loaded as per gentelella.
    # All resource paths for gentelella end in .html.

    # Pick out the html file name from the url. And load that template.
    load_template = request.path.split('/')[-1]
    template = loader.get_template('app/' + load_template)
    return HttpResponse(template.render(context, request))

# page: channels TODO: remove hardcoded path from here
@login_required(login_url="login/")
def channels(request):
    freq_list = []
    obj = ConfigObj("app/utils/astra.conf")
    dict = {}
    i = 0
    for key, value in obj.items():
        try:
            # affectation of audiopid only to raise exception
            audiopid = (value['AUDIO_PID'])

            # Add the name of the channel
            value.update({'NAME': key})

            # Get the list of frequencies
            freq_list.append(value['FREQUENCY'])

            dict[i] = i
            dict[i] = value
            i += 1
        except:
            pass
    # Sort and remove duplicates
        freq_list = sorted(list(set(freq_list)))

    #print (dict)

    context = {
        'title': 'Channels list',
        'channels': dict,
        'frequencies': freq_list
    }
    template = loader.get_template('app/channels.html')
    return HttpResponse(template.render(context, request))

# page: satellites
@login_required(login_url="login/")
def sats(request):
    """

    :param request:
    :return: HttpResponse
    """
    context = {
        'sats': Satellites.objects.all(),
    }
    template = loader.get_template('app/satellites.html')
    return HttpResponse(template.render(context, request))

# page: 404
def error_404(request):
    """
    error_404
    :param request:
    :return: HttpResponse
    """
    context = {}
    template = loader.get_template('app/page_404.html')
    return HttpResponse(template.render(context, request))

# page: profile
@login_required(login_url="login/")
def profile(request):
    """

    :param request:
    :return:
    """
    context = {}
    template = loader.get_template('app/profile.html')
    return HttpResponse(template.render(context, request))

@task()
def do_work():
    """ Get some rest, asynchronously, and update the state all the time """
    for i in range(100):
        sleep(0.1)
        current_task.update_state(state='PROGRESS',
            meta={'current': i, 'total': 100})

def poll_state(request):
    """ A view to report the progress to the user """
    if 'job' in request.GET:
        job_id = request.GET['job']
    else:
        return HttpResponse('No job id given.')

    job = AsyncResult(job_id)
    data = job.result or job.state
    return HttpResponse(json.dumps(data), content_type='application/json')

def init_work(request):
    """ A view to start a background job and redirect to the status page """
    job = do_work.delay()
    return HttpResponseRedirect(reverse('poll_state') + '?job=' + job.id)

@login_required(login_url="login/")
def sats_ajax(request):
    """

    :param request:
    :return: HttpResponse
    """
    if request.method == 'POST':
        if request.is_ajax():
            data = request.POST['mydata']
            if data == "update":
                # get the list from /usr/share/dvb/dvb-s
                gen = GenericUtils()
                transpon_from_files = gen.sats_list()

                # Delete all entries in the db
                cursor = connection.cursor()
                cursor.execute("TRUNCATE TABLE `app_satellites`")

                # Populate the table with all file names and file content
                for tr in transpon_from_files:
                    Satellites(name=tr, config=gen.read_sat_config(tr)).save()
                    print(tr)
            elif data == "delete":
                transponder_name = request.POST['element']
                # Delete
                #print(transponder_name)
                Satellites.objects.filter(name=transponder_name).delete()
            return HttpResponse(data)
    return render(request)

@login_required(login_url="login/")
def post_config_ajax(request):
    """
    post_config_ajax
    :param request:
    :return: HttpResponse
    """
    if request.method == 'POST':
        if request.is_ajax():
            sat = request.POST['satellitesinput']
            delivery = request.POST['deliverysystemselect']
            freq = request.POST['frequencyinput']
            symrate = request.POST['symrateinput']
            polarization = request.POST['polarizationselect']
            modulation = request.POST['modulationselect']
            fec = request.POST['fecselect']
            rolloff = request.POST['rolloffselect']
            pilot = request.POST['pilotselect']

            print(sat)

        return HttpResponse(request)
    else:
        context = {}
        tpl = loader.get_template('app/page_404.html')
        return HttpResponse(tpl.render(context, request))

@login_required(login_url="login/")
def post_interface_config_ajax(request):
    """
    post_interface_config_ajax
    :param request:
    :return: HttpResponse
    """
    if request.method == 'POST':
        if request.is_ajax():
            interfacenum = request.POST['interfacenum']
            satselect = request.POST['satellitesselect']

            lnbtype = request.POST['lnbtypeselect']
            lnblofstd = request.POST['lnblofstandardinput']
            lnblof = request.POST['lnblofinput']
            lnbloflow = request.POST['lnbloflowinput']
            lnblofhigh = request.POST['lnblofhighinput']

            print(satselect)
            print("from:" + interfacenum)
        return HttpResponse(request)
    else:
        context = {}
        tpl = loader.get_template('app/page_404.html')
        return HttpResponse(tpl.render(context, request))

# page: interfaces
@login_required(login_url="login/")
def interfaces(request):
    """

    :param request:
    :return: HttpResponse
    """

    dvb_int = InterfaceProprieties.objects.all()
    satellites = Satellites.objects.all()

    context = {
        'satellites': satellites,
        'dvb_int': dvb_int,
    }

    tpl = loader.get_template('app/interfaces.html')
    return HttpResponse(tpl.render(context, request))


#!/usr/bin/env python
#-*- coding:utf-8 -*-

import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.font_manager as fm
import matplotlib as mpl

mpl.rcParams['axes.unicode_minus'] = False

## 한글 폰트
path = "/home/gsdc/.fonts/NanumGothicCoding.ttf"
font_name = fm.FontProperties(fname=path, size=50).get_name()
print(font_name)
plt.rc('font', family=font_name)

data = pd.read_csv("output.csv", header=None, names=["date", "journal","keyword","count"])

data.head()

dg =data.groupby(["keyword"]).agg({"count":sum})
dg_sort = dg.sort_values(['count'],ascending=False)

dgg = dg_sort.reset_index()
#dgg_filtered = dgg[ ~dgg.keyword.isin(["있다.", "수","등","있는"])].head(20)
#print(dgg_filtered.head(20))

print("주요 키워드?")
print(dgg.head(20))
### 특정 단어를

dgg[:20].plot(x="keyword",y="count")
plt.show()


import inspect
import re

from fastapi import FastAPI
from fastapi.openapi.utils import get_openapi
from fastapi.routing import APIRoute
from fastapi_jwt_auth import AuthJWT
from starlette.middleware.cors import CORSMiddleware

from app.api.api import api_routers
from app.config.settings import Settings

app = FastAPI()

origins = [
    "http://localhost.tiangolo.com",
    "https://localhost.tiangolo.com",
    "http://localhost",
    "http://127.0.0.1:3000",
    "http://localhost:3000",
]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

def custom_openapi():
    if app.openapi_schema:
        return app.openapi_schema

    openapi_schema = get_openapi(
        title="Social Network API",
        version="1.0",
        description="An API for a Social Network.",
        routes=app.routes,
    )

    openapi_schema["components"]["securitySchemes"] = {
        "Bearer Auth": {
            "type": "apiKey",
            "in": "header",
            "name": "Authorization",
            "description": "Enter: **'Bearer <JWT>'**, where JWT is the access token",
        }
    }

    # Get all routes where jwt_optional() or jwt_required
    api_router = [route for route in app.routes if isinstance(route, APIRoute)]

    for route in api_router:
        path = getattr(route, "path")
        endpoint = getattr(route, "endpoint")
        methods = [method.lower() for method in getattr(route, "methods")]

        for method in methods:
            # access_token
            if (
                re.search("jwt_required", inspect.getsource(endpoint))
                or re.search("fresh_jwt_required", inspect.getsource(endpoint))
                or re.search("jwt_optional", inspect.getsource(endpoint))
            ):
                openapi_schema["paths"][path][method]["security"] = [
                    {"Bearer Auth": []}
                ]

    app.openapi_schema = openapi_schema
    return app.openapi_schema

app.openapi = custom_openapi

@app.get("/")
async def read_main():
    return {"mgs": "Hello World"}

@AuthJWT.load_config
def get_config():
    return Settings()

app.include_router(api_routers)


from urllib.request import urlopen
from urllib.error import HTTPError
from bs4 import BeautifulSoup

def get_title(url):
    try:
        html = urlopen(url)
    except HTTPError as e:
        print("We can't take a page")
        return None

    try:
        bsObj = BeautifulSoup(html.read(), "html.parser")
        title = bsObj.body.h1
    except AttributeError as e:
        print("We can't find a tag")
        return None

    return title

title = get_title("http://www.pythonscraping.com/exercises/exercise1.html")
if title is None:
    print("We have a problem")
else:
    print(title)

projects from book/base_examples/first scraper.py


"""The Authorization service.

This module is to handle authorization related queries.
"""

from flask_restx import abort

from api.utils import TokenInfo
from api.utils.roles import Membership
from api.models import Staff as StaffModel
from api.models import StaffWorkRole as StaffWorkRoleModel

# pylint: disable=unused-argument,inconsistent-return-statements
def check_auth(**kwargs):
    """Check if user is authorized to perform action on the service."""
    token_roles = set(TokenInfo.get_roles())
    permitted_roles = set(kwargs.get('one_of_roles', []))
    has_valid_roles = token_roles & permitted_roles
    if has_valid_roles:
        return

    matching_memberships = {membership.name for membership in Membership} & permitted_roles

    if matching_memberships and _has_team_membership(kwargs, matching_memberships):
        return True

    abort(403)

def _has_team_membership(kwargs, team_permitted_roles) -> bool:
    work_id = kwargs.get('work_id')

    if not work_id:
        return False

    email = TokenInfo.get_user_data()['email_id']
    staff_model: StaffModel = StaffModel.find_by_email(email)

    work_roles = StaffWorkRoleModel.find_by_params(
        {"work_id": work_id, "staff_id": staff_model.id}
    )
    if not work_roles:
        return False

    if Membership.TEAM_MEMBER in team_permitted_roles:
        return bool(work_roles)

    membership_ids = {membership.value for membership in Membership}

    return any(role.id in membership_ids for role in work_roles)


import pygame
import events

from scores import Scores
from hero import Hero
from sound import bg_music
from pearl import Pearl

import background as bg

def run():
    pygame.init()
    pygame.display.set_caption("Underwater Adventure")
    window = pygame.display.set_mode((bg.WIDTH, bg.HEIGHT))
    clock = pygame.time.Clock()
    hero = Hero(window)

    enemies = pygame.sprite.Group()
    friends = pygame.sprite.Group()
    group_pearls = pygame.sprite.Group()

    background = bg.Background()
    score = Scores(window)
    pygame.time.set_timer(pygame.USEREVENT, 10000)

    bg_music()
    while True:
        pearl = Pearl()  # создаем в цикле новый объект perl для разных картинок

        events.event(enemies, score, group_pearls, pearl, window)

        if score.game == True:
            background.update()
            background.render(window)
            hero.update()
            score.show_health(hero)
            score.draw_photo()
            score.finish(hero)
            events.make_friend(friends, window)
            events.make_fish(enemies, window, friends)
            events.move_pearl(window, group_pearls)
            events.collide(hero, enemies, group_pearls)
        pygame.display.update()
        clock.tick(60)

if __name__ == '__main__':
    run()


from .ast import *
from .token import *

class Parser(object):

    def __init__(self, lexer):
        self.lexer = lexer
        self.current_token = self.lexer.get_next_token()

    def error(self):
        raise Exception('Invalid syntax')

    def eat(self, token_type):
        if self.current_token.type == token_type:
            self.current_token = self.lexer.get_next_token()
        else:
            self.error()

    def program(self):
        """program : compound_statement DOT"""
        node = self.compound_statement()
        self.eat(DOT)
        return node

    def compound_statement(self):
        """
        compound_statement: BEGIN statement_list END
        """
        self.eat(BEGIN)
        nodes = self.statement_list()
        self.eat(END)
        root = Compound()
        for node in nodes:
            root.children.append(node)

        return root

    def statement_list(self):
        """
        statement_list : statement
                       | statement SEMI statement_list
        """
        node = self.statement()
        results = [
         node]
        while self.current_token.type == SEMI:
            self.eat(SEMI)
            results.append(self.statement())

        if self.current_token.type == ID:
            self.error()
        return results

    def statement(self):
        """
        statement : compound_statement
                  | assignment_statement
                  | empty
        """
        if self.current_token.type == BEGIN:
            node = self.compound_statement()
        else:
            if self.current_token.type == ID:
                node = self.assignment_statement()
            else:
                node = self.empty()
        return node

    def assignment_statement(self):
        """
        assignment_statement : variable ASSIGN expr
        """
        left = self.variable()
        token = self.current_token
        self.eat(ASSIGN)
        right = self.expr()
        node = Assign(left, token, right)
        return node

    def variable(self):
        """
        variable : ID
        """
        node = Var(self.current_token)
        self.eat(ID)
        return node

    def empty(self):
        """An empty production"""
        return NoOp()

    def expr(self):
        """
        expr : term ((PLUS | MINUS) term)*
        """
        node = self.term()
        while self.current_token.type in (PLUS, MINUS):
            token = self.current_token
            if token.type == PLUS:
                self.eat(PLUS)
            else:
                if token.type == MINUS:
                    self.eat(MINUS)
            node = BinOp(left=node, op=token, right=(self.term()))

        return node

    def term(self):
        """term : factor ((MUL | DIV) factor)*"""
        node = self.factor()
        while self.current_token.type in (MUL, DIV):
            token = self.current_token
            if token.type == MUL:
                self.eat(MUL)
            else:
                if token.type == DIV:
                    self.eat(DIV)
            node = BinOp(left=node, op=token, right=(self.factor()))

        return node

    def factor(self):
        """factor : PLUS factor
                  | MINUS factor
                  | INTEGER
                  | LPAREN expr RPAREN
                  | variable
        """
        token = self.current_token
        if token.type == PLUS:
            self.eat(PLUS)
            node = UnaryOp(token, self.factor())
            return node
        if token.type == MINUS:
            self.eat(MINUS)
            node = UnaryOp(token, self.factor())
            return node
        if token.type == INTEGER:
            self.eat(INTEGER)
            return Num(token)
        if token.type == LPAREN:
            self.eat(LPAREN)
            node = self.expr()
            self.eat(RPAREN)
            return node
        node = self.variable()
        return node

    def parse(self):
        """
        program : compound_statement DOT

        compound_statement : BEGIN statement_list END

        statement_list : statement
                       | statement SEMI statement_list

        statement : compound_statement
                  | assignment_statement
                  | empty

        assignment_statement : variable ASSIGN expr

        empty :

        expr: term ((PLUS | MINUS) term)*

        term: factor ((MUL | DIV) factor)*

        factor : PLUS factor
               | MINUS factor
               | INTEGER
               | LPAREN expr RPAREN
               | variable

        variable: ID
        """
        node = self.program()
        if self.current_token.type != EOF:
            self.error()
        return node


import tensorflow as tf
from tensorflow import keras

import numpy as np
import matplotlib.pyplot as plt

# import fashion MNIST dataset
fashion_mnist = keras.datasets.fashion_mnist

(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()

# Class names
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

# Displaying an image
'''plt.figure()
plt.imshow(train_images[4])
plt.colorbar()
plt.grid(False)
plt.show()'''

train_images, test_images = train_images/255.0, test_images/255.0

# First 25 images from the training set
'''plt.figure(figsize=(10, 10))
for i in range(25):
    plt.subplot(5, 5, i+1)
    plt.xticks([])
    plt.yticks([])
    plt.grid(False)
    plt.imshow(train_images[i], cmap=plt.cm.binary)
    plt.xlabel(class_names[train_labels[i]])
plt.show()'''

# Building the model
model = keras.Sequential([
    keras.layers.Flatten(input_shape=(28, 28)),
    keras.layers.Dense(128, activation='relu'),
    keras.layers.Dense(10)
])
# Compile the Model
model.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])

# Feed the model
model.fit(train_images, train_labels, epochs=10)

# Print accuracy
test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)
print('\nTest accuracy:', test_acc)

# Make Predictions
probability_model = tf.keras.Sequential([model,
                                         tf.keras.layers.Softmax()])
predictions = probability_model.predict(test_images)

# Function to look at 10 class predictions
def plot_image(i, predictions_array, true_label, img):
    true_label, img = true_label[i], img[i]
    plt.grid(False)
    plt.xticks([])
    plt.yticks([])

    plt.imshow(img, cmap=plt.cm.binary)

    predicted_label = np.argmax(predictions_array)
    if predicted_label == true_label:
        color = 'blue'
    else:
        color = 'red'

    plt.xlabel("{} {:2.0f}% ({})".format(class_names[predicted_label],
                                100*np.max(predictions_array),
                                class_names[true_label]),
                                color=color)

def plot_value_array(i, predictions_array, true_label):
    true_label = true_label[i]
    plt.grid(False)
    plt.xticks(range(10))
    plt.yticks([])
    this_plot = plt.bar(range(10), predictions_array, color="#777777")
    plt.ylim([0, 1])
    predicted_label = np.argmax(predictions_array)

    this_plot[predicted_label].set_color('red')
    this_plot[true_label].set_color('blue')

# Verify Predictions
'''i = 0
plt.figure(figsize=(6, 3))
plt.subplot(1, 2, 1)
plot_image(i, predictions[i], test_labels, test_images)
plt.subplot(1, 2, 2)
plot_value_array(i, predictions[i], test_labels)
plt.show()

i = 12
plt.figure(figsize=(6,3))
plt.subplot(1,2,1)
plot_image(i, predictions[i], test_labels, test_images)
plt.subplot(1,2,2)
plot_value_array(i, predictions[i],  test_labels)
plt.show()'''

# Use the trained model
# Grab an image from the test data set
# Use the trained model
# Grab an image from the test data set
def trained_model():
    classify = input("What image do you want to classify?(Pick a number from 1-9999)")
    classify = int(classify)
    img = test_images[classify]
    print(img.shape)
    img = (np.expand_dims(img, 0))
    predictions_single = probability_model.predict(img)
    print(predictions_single)
    plot_value_array(1, predictions_single[0], test_labels)
    _ = plt.xticks(range(10), class_names, rotation=45)
    predictions_single = np.argmax(predictions_single[0])
    i = classify
    plt.figure(figsize=(6, 3))
    plt.subplot(1, 2, 1)
    plot_image(i, predictions[i], test_labels, test_images)
    plt.show()
    print(
        "This image most likely is a {}"
        .format(class_names[(predictions_single)]))

trained_model()


import tensorflow as tf
tf.set_random_seed(337)

#1. 데이터
x = [1, 2, 3, 4, 5]
y = [2, 4, 6, 8, 10]

w = tf.Variable(tf.random_normal([1]), dtype=tf.float32) #random_uniform 균등 분포(N빵), random_normal 정규 분포 [1]은 1개짜리
b = tf.Variable(tf.random_normal([1]), dtype=tf.float32) #random_uniform 균등 분포(N빵), random_normal 정규 분포 [1]은 1개짜리
# w = tf.random_normal([1])
# b = tf.random_normal([1])

#################동일한 코드
# sess = tf.compat.v1.Session()
# sess.run(tf.global_variables_initializer())
# print(sess.run(w)) #[-0.4121612]

with tf.compat.v1.Session() as sess :
    sess.run(tf.global_variables_initializer())
    print(sess.run(w))
##################

#####[실습]#####

hypothesis = x * w + b

# #3-1 컴파일
loss = tf.reduce_mean(tf.square(hypothesis-y))

optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)

#3-2 훈련
with tf.compat.v1.Session() as sess :
# sess = tf.compat.v1.Session()
    sess.run(tf.global_variables_initializer())

    #model.fit()
    epochs = 20001
    for step in range(epochs) :
        sess.run(train)
        if step %20 == 0 :
            print(step, 'loss :', sess.run(loss), 'w :', sess.run(w), 'b :', sess.run(b))

    # sess.close()
    #with문은 자동으로 close() 해줌.


import time
import grpc
from concurrent import futures
import addreply_pb2, addreply_pb2_grpc
import comment

class addreplyservice(addreply_pb2_grpc.addreplyServicer):
    def add(self, request, ctx):
        message = comment.addreply(request.token, request.news_id, request.comment_id, request.content)
        return addcoment_pb2.output(result = message)

def main():
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=4))
    addreply_pb2_grpc.add_addreplyServicer_to_server(addreplyservice(),server)
    server.add_insecure_port('127.0.0.1:7777')
    server.start()
    try:
        print("running...")
        time.sleep(1000)
    except KeyboardInterrupt:
        print("stopping...")
        server.stop(0)

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Feb 16 09:23:16 2019

@author: rushikeshtapdiya
"""

from __future__ import print_function
import keras
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization
from keras.layers import Conv2D, MaxPooling2D
from keras.layers.advanced_activations import ELU
from keras.preprocessing.image import ImageDataGenerator
import os

num_classes = 20
img_rows, img_cols = 32, 32
batch_size = 16

train_data_dir = 'segmented/'
train_datagen = ImageDataGenerator(
      rescale=1./255,
      rotation_range=20,
      width_shift_range=0.2,
      height_shift_range=0.2,
      horizontal_flip=True,
      fill_mode='nearest',
      validation_split=0.3)

# Change the batchsize according to your system RAM
train_batchsize = 16
val_batchsize = 10

train_generator = train_datagen.flow_from_directory(
        train_data_dir,
        target_size=(img_rows, img_cols),
        batch_size=train_batchsize,
        class_mode='categorical',
        subset='training')

validation_generator = train_datagen.flow_from_directory(
        train_data_dir,
        target_size=(img_rows, img_cols),
        batch_size=val_batchsize,
        class_mode='categorical',
        shuffle=False,
        subset='validation')


"""
将remove_bad.csv数据集分为训练集和测试集，本次直接按照比例分配
"""
import pandas as pd
import numpy as np
import random
import csv
# 读取文件，转换格式
file_name = './dataset/remove_bad.csv'

df = pd.read_csv(file_name)
ls = list(df.values)
# 初始化格式，使用字典存储每个row
data = dict()
new_data = dict()
data[0] = []
data[1] = []
data[2] = []
data[3] = []
data[4] = []
new_data[0] = []
new_data[1] = []
new_data[2] = []
new_data[3] = []
new_data[4] = []
# 将每个row放到对应的桶
# w mi

for l in ls:
    data[l[1]].append(l)

# 将数据集分开为训练集和测试集
# num_trian = [1225, 700, 980, 315, 280]
train_list = []
test_list = []

# 遍历5个类型
for k, v in data.items():
    sample = random.sample(range(0, len(v)), round(len(v) * 0.1))
    sample.sort()
    j = 0
    for i in range(len(v)):
        if j < len(sample) and i == sample[j]:
            test_list.append(v[i])
            j += 1
        elif j == len(sample):
            train_list.append(v[i])
        elif i != sample[j]:
            train_list.append(v[i])

# 保存train_list_jpeg和test_list_jpeg
with open('./dataset/32000train.csv', 'w', newline='') as fp:
    writer = csv.writer(fp)
    writer.writerow(['image', 'level'])
    writer.writerows(train_list)
with open('./dataset/32000test.csv', 'w', newline='') as fp:
    writer = csv.writer(fp)
    writer.writerow(['image', 'level'])
    writer.writerows(test_list)