import io
import os
import random
from unittest.mock import patch, MagicMock

import pytest

from peek import __file__ as package_root
from peek.es_api_spec.kspec_js import JsSpecParser, JsSpecEvaluator, build_js_specs

package_root = os.path.dirname(package_root)
kibana_dir = os.path.join(package_root, 'specs', 'kibana-7.8.1')

@pytest.mark.skipif(not os.path.exists(kibana_dir), reason='Kibana directory not available')
def test_parse_specs():
    spec_parser = JsSpecParser(kibana_dir)
    spec_parser.parse()

@pytest.mark.skipif(not os.path.exists(kibana_dir), reason='Kibana directory not available')
def test_eval_specs():
    spec_parser = JsSpecParser(kibana_dir)
    nodes = spec_parser.parse()
    spec_evaluator = JsSpecEvaluator()
    specs = spec_evaluator.visit(nodes)
    assert 'GLOBAL' in specs
    assert 'search' in specs
    assert 'put_mapping' in specs

def test_build_js_spec_will_respect_cache_config():
    kibana_dir = 'kibana_dir'
    mock_parser = MagicMock()
    mock_parser_cls = MagicMock(name='JsSpecParser', return_value=mock_parser)
    mock_evaluator = MagicMock(name='JsSpecEvaluator', return_value=MagicMock())
    cache_io = io.StringIO("source")
    use_cache_file = random.choice((True, False))
    cache_file_exists = random.choice((True, False))
    with patch('os.path.exists', MagicMock(return_value=cache_file_exists)), patch(
        'builtins.open', MagicMock(return_value=cache_io)
    ), patch('peek.es_api_spec.kspec_js.JsSpecParser', mock_parser_cls), patch(
        'peek.es_api_spec.kspec_js.JsSpecEvaluator', mock_evaluator
    ):
        build_js_specs(kibana_dir, use_cache_file)

        if not use_cache_file:
            mock_parser_cls.assert_called_with(kibana_dir, source=None)
            mock_parser.parse.assert_called_once()
            mock_parser.save.assert_not_called()
        else:
            if cache_file_exists:
                mock_parser_cls.assert_called_with(kibana_dir, source='source')
                mock_parser.save.assert_not_called()
            else:
                mock_parser_cls.assert_called_with(kibana_dir, source=None)
                mock_parser.save.assert_called()

def test_parser_will_not_read_spec_files_when_source_is_provided():
    kibana_dir = None  # intentionally have it as None so if used will trip
    spec_parser = JsSpecParser(kibana_dir, source='let x = 42')
    nodes = spec_parser.parse()
    assert len(nodes) == 1


#!/usr/bin/env python3

import subprocess
import os
import sys
import glob

REPO_PATH = 'git-repo'

def git_clone(url):
    r = subprocess.run(['git', 'clone', url, REPO_PATH])

    if r.returncode == 0:
        return True
    else:
        print("[COUT] Git clone error: Invalid argument to exit",
              file=sys.stderr)
        print("[COUT] CO_RESULT = false")
        return False

def setup(path):
    file_name = os.path.basename(path)
    dir_name = os.path.dirname(path)
    r = subprocess.run('cd {}; python3 {} install'.format(dir_name, file_name),
                       shell=True)

    if r.returncode != 0:
        print("[COUT] install dependences failed: {}".format(path), file=sys.stderr)
        return False

    return True

def pip_install(file_name):
    r = subprocess.run(['pip3', 'install', '-r', file_name])

    if r.returncode != 0:
        print("[COUT] install dependences failed: {}".format(path), file=sys.stderr)
        return False

    return True

def upload_file(upload):
    r1 = subprocess.run(['curl', '-XPUT', '-d', '@/tmp/output.tar.bz2', upload])
    print()
    if r1.returncode != 0:
        print("[COUT] upload error", file=sys.stderr)
        return False
    return True

def pynsist(file_name):
    r = subprocess.run(['pynsist', '{}/{}'.format(REPO_PATH, file_name)])

    if r.returncode != 0:
        print("[COUT] pynsist error", file=sys.stderr)
        return False

    return True

def compress(file_name):
    dirname = os.path.dirname(file_name)
    r = subprocess.run('cd {}/{}/build/nsis; tar cjvf /tmp/output.tar.bz2 .'.format(REPO_PATH, dirname), shell=True)

    if r.returncode != 0:
        print("[COUT] compress error", file=sys.stderr)
        return False

    return True

def parse_argument():
    data = os.environ.get('CO_DATA', None)
    if not data:
        return {}

    validate = ['git-url', 'entry-file', 'upload']
    ret = {}
    for s in data.split(' '):
        s = s.strip()
        if not s:
            continue
        arg = s.split('=')
        if len(arg) < 2:
            print('[COUT] Unknown Parameter: [{}]'.format(s))
            continue

        if arg[0] not in validate:
            print('[COUT] Unknown Parameter: [{}]'.format(s))
            continue

        ret[arg[0]] = arg[1]

    return ret

def main():
    argv = parse_argument()
    git_url = argv.get('git-url')
    if not git_url:
        print("[COUT] The git-url value is null", file=sys.stderr)
        print("[COUT] CO_RESULT = false")
        return

    entry_file = argv.get('entry-file')
    if not entry_file:
        print("[COUT] The entry-file value is null", file=sys.stderr)
        print("[COUT] CO_RESULT = false")
        return

    upload = argv.get('upload')
    if not upload:
        print("[COUT] The upload value is null", file=sys.stderr)
        print("[COUT] CO_RESULT = false")
        return

    if not git_clone(git_url):
        return

    if not pynsist(entry_file):
        print("[COUT] CO_RESULT = false")
        return

    if not compress(entry_file):
        print("[COUT] CO_RESULT = false")
        return

    if not upload_file(upload):
        print("[COUT] CO_RESULT = false")
        return

    print("[COUT] CO_RESULT = true")

main()


# -*- encoding: utf-8 -*-

from odoo import api, models
import logging

_logger = logging.getLogger(__name__)
class ReporteCompras(models.AbstractModel):
    _name = 'report.l10n_gt_extra.reporte_compras'

    def lineas(self, datos):
        totales = {}

        totales['num_facturas'] = 0
        totales['compra'] = {'exento': 0, 'neto': 0, 'iva': 0, 'total': 0}
        totales['servicio'] = {'exento': 0, 'neto': 0, 'iva': 0, 'total': 0}
        totales['importacion'] = {'exento': 0, 'neto': 0, 'iva': 0, 'total': 0}
        totales['combustible'] = {'exento': 0, 'neto': 0, 'iva': 0, 'total': 0}
        totales['compras'] = {'bienes': 0}
        totales['total'] = 0
        totales['resumen'] = {'exento': 0, 'neto': 0, 'iva': 0, 'total': 0}
        totales['pequenio_contribuyente'] = 0

        journal_ids = [x for x in datos['diarios_id']]
        facturas = self.env['account.invoice'].search([
            ('state', 'in', ['open', 'paid', 'in_payment']),
            ('journal_id', 'in', journal_ids),
            ('date', '<=', datos['fecha_hasta']),
            ('date', '>=', datos['fecha_desde']),
        ], order='date_invoice, reference')

        lineas = []
        for f in facturas:
            totales['num_facturas'] += 1

            tipo_cambio = 1
            if f.currency_id.id != f.company_id.currency_id.id:
                total = 0
                for l in f.move_id.line_ids:
                    if l.account_id.id == f.account_id.id:
                        total += l.credit - l.debit
                tipo_cambio = abs(total / f.amount_total)

            tipo = 'FACT'
            if f.type != 'in_invoice':
                tipo = 'NC'
            if f.partner_id.pequenio_contribuyente:
                tipo += ' PEQ'

            linea = {
                'estado': f.state,
                'tipo': tipo,
                'fecha': f.date_invoice,
                #'serie': f.journal_id.code,
                'serie': f.provider_invoice_serial or '',
                # 'numero': f.reference or '',
                #'numero': f.move_name or '',
                'numero': f.provider_invoice_number or '',
                'proveedor': f.partner_id,
                'compra': 0,
                'compra_exento': 0,
                'servicio': 0,
                'servicio_exento': 0,
                'combustible': 0,
                'combustible_exento': 0,
                'importacion': 0,
                'importacion_exento': 0,
                'importacion_iva': 0,
                'compra_iva': 0,
                'servicio_iva': 0,
                'combustible_iva': 0,
                'base': 0,
                'iva': 0,
                'total': 0
            }
            is_compra = False
            is_service = False
            is_mix = False
            is_import = False
            is_gas = False
            flag_gas = False

            for linea_factura in f.invoice_line_ids:
                precio = (linea_factura.price_unit *
                          (1-(linea_factura.discount or 0.0)/100.0)) * tipo_cambio
                if tipo == 'NC':
                    precio = precio * -1

                tipo_linea = f.tipo_gasto

                if linea_factura.invoice_line_tax_ids:
                    for tax in linea_factura.invoice_line_tax_ids:
                        if tax.sat_tax_type == 'gas':
                            if is_compra or is_service:
                                is_mix = True
                                flag_gas = True
                            else:
                                is_gas = True
                                flag_gas = True
                    if flag_gas:
                        flag_gas = False

                if f.tipo_gasto == 'mixto':

                    if linea_factura.product_id.type == 'product' or linea_factura.product_id.type == 'consu':
                        tipo_linea = 'compra'
                    if linea_factura.product_id.type == 'service':
                        tipo_linea = 'servicio'
                    if is_gas:
                        tipo_linea = 'combustible'

                if f.tipo_gasto == 'combustible':
                    tipo_linea = 'combustible'

                r = linea_factura.invoice_line_tax_ids.compute_all(
                    precio, currency=f.currency_id, quantity=linea_factura.quantity, product=linea_factura.product_id, partner=f.partner_id)

                linea['base'] += r['base']
                totales[tipo_linea]['total'] += r['base']
                #totales[tipo_linea]['total'] += precio * linea_factura.quantity

                if len(linea_factura.invoice_line_tax_ids) > 0:
                    linea[tipo_linea] += r['base']
                    if tipo_linea == 'compra':
                        totales['compras']['bienes'] += r['base']
                    totales[tipo_linea]['neto'] += r['base']
                    totales['resumen']['neto'] += r['base']
                    for i in r['taxes']:
                        if i['id'] == datos['impuesto_id'][0]:
                            linea['iva'] += i['amount']
                            linea[tipo_linea+'_iva'] += i['amount']
                            totales[tipo_linea]['iva'] += i['amount']
                            totales['resumen']['iva'] += r['base']
                            totales[tipo_linea]['total'] += i['amount']
                        elif i['amount'] > 0:

                            if tipo_linea == 'combustible':
                                linea['compra_exento'] += i['amount']
                                totales[tipo_linea]['exento'] += i['amount']
                                totales[tipo_linea]['total'] += i['amount']
                            else:
                                linea[tipo_linea+'_exento'] += i['amount']
                                totales[tipo_linea]['exento'] += i['amount']
                                totales[tipo_linea]['total'] += i['amount']
                            totales['resumen']['exento'] += i['amount']

                else:
                    linea[tipo_linea+'_exento'] += r['base']
                    totales[tipo_linea]['exento'] += r['base']
                    totales['resumen']['exento'] += r['base']
                    totales[tipo_linea]['total'] += r['base']

                linea['total'] += precio * linea_factura.quantity

                totales['total'] += precio * linea_factura.quantity

            if f.partner_id.pequenio_contribuyente:
                totales['pequenio_contribuyente'] += linea['base']

            lineas.append(linea)

        return {'lineas': lineas, 'totales': totales}

    @api.model
    def _get_report_values(self, docids, data=None):
        return self.get_report_values(docids, data)

    @api.model
    def get_report_values(self, docids, data=None):
        model = self.env.context.get('active_model')
        docs = self.env[model].browse(self.env.context.get('active_ids', []))

        diario = self.env['account.journal'].browse(data['form']['diarios_id'][0])

        return {
            'doc_ids': self.ids,
            'doc_model': model,
            'data': data['form'],
            'docs': docs,
            'lineas': self.lineas,
            'direccion': diario.direccion and diario.direccion.street,
        }


"""
============================
Author:柠檬班-木森
Time:2020/3/3   21:17
E-mail:3247119728@qq.com
Company:湖南零檬信息技术有限公司
============================
"""

class TestData:
    url = "http://www.baidu.com"
    method = "get"

# 在类外面获取类属性
# print(TestData.url)

# getattr:获取类属性
# 参数1：类
# 参数2：属性名
# 参数3: 如果属性不存在，则返回该值
# res = getattr(TestData, "url1", "None")
# print(res)

# 动态获取属性的含义：
# name = input("请输入要获取的属性名:")
# res = getattr(TestData, name, "None")
# print(res)

# 类外面定义类属性
# 方式一：类.属性名= 属性值
# TestData.attr = 100
# print(TestData.attr)

# 方式二：使用setattr
# 动态设置属性：setattr
# 参数一：类
# 参数二：属性名
# 参数三：属性值
# setattr(TestData,"attr2",999)
# print(TestData.attr2)

# 需求：把下面两个列表中的数据设为属性和对应的属性值
title = ["name", "age", "gender"]
data = ["木森", 18, "男"]

# TestData.name = data[0]
# 设置属性
# setattr(TestData, title[0], data[0])
# setattr(TestData, title[1], data[1])
# setattr(TestData, title[2], data[2])

# for i in range(len(title)):
#     setattr(TestData, title[i], data[i])
# print(TestData.name)
# print(TestData.age)
# print(TestData.gender)

# 动态删除属性 delattr
# 参数一：类
# 参数二 ：属性名
# 方式一：关键字：del
del TestData.url
print(TestData.url)

# name = input("请输入你要删除的属性:")
# delattr(TestData,name)
#
# print(TestData.url)

# 扩展：使用del删除变量
# a = 100
# del a
# print(a)

# 判断属性是否存在：存在返回Ture,不存在返回False
res = hasattr(TestData, "url1")
print(res)

07动态设置和获取属性.py


import datetime
class Task():

    def __init__(self, task, priority, deadline=0):
        self.task = task
        self.priority = priority
        self.deadline = deadline
        self.date = datetime.datetime.now()
        self.id = int(self.get_id()) +1

    def get_id(self):
        with open('../homework 4/id_counter', 'w') as id_file:
            return id_file.read().strip()

    def update_id(self):
        with open('../homework 4/id_counter', 'w') as id_file:
            id_file.write(self.id)
if __name__ == '__main__':
    t1 = Task(1,1)
    t2 = Task(2,2)
    print(Task.ID)


# -*- coding: utf-8 -*-

import pandas as pd
import numpy as np
import re
import config
import logging

logger = logging.getLogger('QQ')

def changeFace(ans):
	ans = str(ans)
	val = re.sub("Face Amount: ", ' ', ans)
	val = re.sub(",", ' ', val)
	val = str(re.sub(" ", '', val))
	index = (val.find('$'))
	if(index != -1):
		val = val[1:]

	#val = int(val)
	return (val)

def changeWt(ans):
	ans = str(ans)
	val = re.sub("#", 'lb', ans)
	val = re.sub("lbs", 'lb', val)
	val = re.sub("Pounds", 'lb', val)
	val = re.sub("pounds", 'lb', val)
	val = str(re.sub(" ", '', val))
	if(re.search('KG|Kg|kg', val, re.I | re.U)):
		x = re.findall(r'\d+', ans)
		val = str(int(int(x[0]) * 2.2)) + 'lb'
	#val = int(val)
	return (val)

def changeHeight(ans):
	ans = str(ans)
	val = re.sub("Feet", '.', ans)
	val = str(re.sub(" ", '', val))
	if(re.search('inches|Inches', val, re.I | re.U) or val=='nan'):
		val = re.sub("Inches", '', val)
		return (val)
	else:
		val = re.sub("Inches", '', val)
		val = re.sub("Inch", '', val)
		val = val + '0'
		return (val)

def changePT(ans):
	ans = str(ans)
	val = re.sub("Product Type", '', ans)
	val = re.sub(":", '', val)
	val = str(re.sub(" ", '', val))
	return (val)

def emailfirst(doc):
	if(type(doc) == str):
		mails = doc.split(";")
		return str(mails[0])
	else:
		doc = ""
		return doc
def human_format(num):
    num = float('{:.3g}'.format(num))
    magnitude = 0
    while abs(num) >= 1000:
        magnitude += 1
        num /= 1000.0
    return '{}{}'.format('{:f}'.format(num).rstrip('0').rstrip('.'), ['', 'K', 'M', 'B', 'T'][magnitude])

def standardize_famnt(doc):
	if(doc.isdigit()):
		number = int(doc)
		num = human_format(number)
		return(num)
	else:
		return(doc)

def preprocess_main(file):

	logger.info(">> Start - Preprocessing. Standardize Face amount, Weight... etc")
	logger.debug("Opening -" + config.regex_processed_csv)
	df = pd.read_csv(config.regex_processed_csv, encoding='UTF-8')
	df = df.drop(columns=['Year_of_Birth'])
	of = pd.read_csv(file, encoding='UTF-8')
	of['recepientemail'] = of['recepientemail'].apply(emailfirst)
	df['recepientemail'] = of['recepientemail'].values

	logger.info("Standardize Weight")
	df['Weight'] = df['Weight'].apply(changeWt)

	logger.info("Standardize Height")
	df['Height'] = df['Height'].apply(changeHeight)

	logger.info("Standardize Face amount")
	df['Face Amount'] = df['Face Amount'].apply(changeFace)

	logger.info("Standardize Product Type")
	df['Product Type'] = df['Product Type'].apply(changePT)
	df['Face Amount'] = df['Face Amount'].apply(standardize_famnt)
	# print(df['Face Amount'])

	#print(df)
	df.to_csv(config.preprocessed_csv, index=False, encoding="utf-8")
	logger.info("<< End - Preprocess")

preprocess_main(config.raw_data_csv)


import csv

from django.conf import settings
from django.core.management.base import BaseCommand

from reviews.models import Comment, Review
from users.models import User

class Command(BaseCommand):
    def handle(self, *args, **options) -> None:
        print('Importing data from:', settings.DATA_IMPORT_LOCATION)

        with open(
            f'{settings.DATA_IMPORT_LOCATION}/comments.csv',
            'r',
            encoding='utf-8-sig',
        ) as csv_file:
            csv_file = csv.reader(csv_file)
            next(csv_file)

            for counter, line in enumerate(csv_file):
                id = line[0]
                review = Review.objects.get(id=line[1])
                text = line[2]
                author = User.objects.get(id=line[3])
                pub_date = line[4]

                if not Comment.objects.filter(
                    review=review,
                    text=text,
                    author=author,
                    pub_date=pub_date,
                ).exists():
                    obj = Comment()
                    obj.id = id
                    obj.review = review
                    obj.text = text
                    obj.author = author
                    obj.pub_date = pub_date
                    obj.save()

                    print(obj)

            print(f'Import complete, imported {counter} products')


import argparse
import ast
import os, sys
import os.path as osp
import shutil

from tqdm import tqdm
import numpy as np
import pandas as pd

sys.path.append("/home/data1/yw/competetion_tools/iterative-stratification")
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold

def analyze_xml(file_name, max_col):
    '''
    从xml文件中解析class，对象位置
    :param file_name: xml文件位置
    :return: class，每个类别的矩形位置
    '''

    fp = open(file_name)

    class_name = []
    for p in fp:
        if '<object>' in p:
            bndbox = next(fp).split('>')[1].split('<')[0]  # 标准VOC会有这个字段
            name = next(fp).split('>')[1].split('<')[0]
            if name in class_name_dict.keys():
                class_name.append(class_name_dict[name])
            # class_name.append(next(fp).split('>')[1].split('<')[0])

    fp.close()

    "没有标签"
    if len(class_name) > 0:
        max_col = max(max_col, len(class_name))
        return [os.path.basename(file_name).split(".")[0]], class_name, max_col
    else:
        return [], [], []

def analyze_classes_proportition(xml_dir, xmls=None):
    '''
    从xml文件中解析class，对象位置
    :param file_name: xml文件位置
    :return: class，每个类别的矩形位置
    '''

    if not xmls:
        xmls = os.listdir(xml_dir)
    class_num_dict = {"holothurian": 0, "echinus": 0, "scallop": 0, "starfish": 0, "waterweeds": 0, }

    for xml in tqdm(xmls):
        xml_path = os.path.join(xml_dir, xml)

        fp = open(xml_path)

        for p in fp:
            if '<object>' in p:

                name = next(fp).split('>')[1].split('<')[0]
                if name in class_num_dict.keys():
                    class_num_dict[name] += 1

        fp.close()

    print(class_num_dict)

    # "没有标签"
    # if len(class_name) > 0:
    #     return  [os.path.basename(file_name).split(".")[0]],class_name
    # else:
    #     return [], []

def analyze_classes_wh_ratio(xml_dir, xmls=None, all_bbox=False):
    '''
    从xml文件中解析class，对象位置
    :param file_name: xml文件位置
    :return: class，每个类别的矩形位置
    '''
    import matplotlib.pyplot as plt
    from matplotlib import font_manager
    plt.rcParams['font.sans-serif'] = ['SimHei']  # ÉèÖÃ×ÖÌåÎªºÚÌå
    plt.rcParams['font.family'] = 'sans-serif'  # ÉèÖÃ×ÖÌåÑùÊ½
    plt.rcParams['figure.figsize'] = (10.0, 10.0)  # ÉèÖÃ×ÖÌå´óÐ¡

    if not xmls:
        xmls = os.listdir(xml_dir)
    class_ratio_dict = {"holothurian": [], "echinus": [], "scallop": [], "starfish": []}

    for xml in tqdm(xmls):
        xml_path = os.path.join(xml_dir, xml)

        fp = open(xml_path)

        name = None
        for p in fp:
            if '<object>' in p:
                name = next(fp).split('>')[1].split('<')[0]

            if '<bndbox>' in p:
                rectangle = [round(eval(next(fp).split('>')[1].split('<')[0])) for _ in range(4)]

                rectangle_w = rectangle[2] - rectangle[0]
                rectangle_h = rectangle[3] - rectangle[1]
                if name in class_ratio_dict.keys():
                    class_ratio_dict[name].append(round(rectangle_w / rectangle_h, 1))
        fp.close()

    if not all_bbox:
        for key in class_ratio_dict.keys():
            unique_ratio = set(class_ratio_dict[key])
            unique_ratio = sorted(unique_ratio)
            bbox_unique_count = [class_ratio_dict[key].count(i) for i in unique_ratio]

            wh_dataframe = pd.DataFrame(bbox_unique_count, index=unique_ratio, columns=["%s_wh_ratio" % key])
            wh_dataframe.plot(kind='bar', color="#55aacc")
            plt.show()
    else:
        all_bbox = []
        for key in class_ratio_dict.keys():
            all_bbox.extend(class_ratio_dict[key])
        unique_ratio = set(all_bbox)
        unique_ratio = sorted(unique_ratio)
        bbox_unique_count = [all_bbox.count(i) for i in unique_ratio]

        wh_dataframe = pd.DataFrame(bbox_unique_count, index=unique_ratio, columns=["%s_wh_ratio" % "all_classes"])
        wh_dataframe.plot(kind='bar', color="#55aacc")
        plt.show()

def get_imagesname_labels(xml_dir, xmls):
    if not xmls:
        xmls = os.listdir(xml_dir)

    X = []
    _Y = []

    max_col = 0
    for ind in tqdm(range(len(xmls))):
        xml_path = os.path.join(xml_dir, xmls[ind])
        x, y, _max_col = analyze_xml(xml_path, max_col)

        if len(x) > 0 and len(y) > 0:
            X.append(x)
            _Y.append(y)

            max_col = max(max_col, _max_col)

    Y = np.zeros(shape=(len(X), max_col), dtype=np.uint8)
    for _ in range(len(_Y)):
        Y[_][:len(_Y[_])] = _Y[_]

    return np.array(X), Y

def split_folds(folds_dir, annotation_dir, target_xml_names=None):
    X, Y = get_imagesname_labels(annotation_dir, xmls=target_xml_names)
    mskf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=3)

    os.makedirs(folds_dir, exist_ok=True)

    fold_num = 1
    for train_index, test_index in mskf.split(X, Y):
        # print("TRAIN:", train_index, "TEST:", test_index)
        X_train, X_test = X[train_index], X[test_index]
        # Y_train, Y_test = Y[train_index], Y[test_index]

        save_fold_to_csv(X_train, X_test, save_dir=os.path.join(folds_dir, "fold_v%s" % fold_num))
        fold_num += 1

def save_fold_to_csv(train_numpy_ob, test_numpy_ob, save_dir):
    os.makedirs(save_dir, exist_ok=True)

    pd_data_train = pd.DataFrame(train_numpy_ob, columns=['filename'])
    save_path_train = os.path.join(save_dir, "train.csv")
    pd_data_train.to_csv(save_path_train)

    pd_data_test = pd.DataFrame(test_numpy_ob, columns=['filename'])
    save_path = os.path.join(save_dir, "test.csv")
    pd_data_test.to_csv(save_path)

def example_for_iterstrat():
    X = np.array([[1, 2], [3, 4], [1, 2], [3, 4], [1, 2], [3, 4], [1, 2], [3, 4]])
    y = np.array([[0, 0], [0, 0], [0, 1], [0, 1], [1, 1], [1, 1], [1, 0], [1, 0]])

    mskf = MultilabelStratifiedKFold(n_splits=3, shuffle=True, random_state=0)

    for train_index, test_index in mskf.split(X, y):
        print("TRAIN:", train_index, "TEST:", test_index)
        X_train, X_test = X[train_index], X[test_index]
        y_train, y_test = y[train_index], y[test_index]

def check_class_balance():
    xml_dir = r"/home/data1/yw/water_detection/train/Annotations"

    for v_num in range(1, 6):
        xml_list = []
        test_xml = []

        csv_path = r"/home/data1/yw/water_detection/split_folds/fold_v%s/train.csv" % v_num
        test_csv = r"/home/data1/yw/water_detection/split_folds/fold_v%s/test.csv" % v_num
        with open(csv_path, "r") as f:
            for line_record in f.readlines():
                xml_name = line_record.strip("\n").split(',')[1] + ".xml"
                xml_list.append(xml_name)
            xml_list.pop(0)

        with open(test_csv, "r") as f:
            for line_record in f.readlines():
                xml_name = line_record.strip("\n").split(',')[1] + ".xml"
                test_xml.append(xml_name)
            test_xml.pop(0)

        print("fold_v%s" % v_num)
        analyze_classes_proportition(xml_dir, xmls=xml_list)
        analyze_classes_proportition(xml_dir, xmls=test_xml)
        print("\n")

def check_class_balance_txt():
    xml_dir = r"/home/data1/yw/water_detection/train/Annotations"

    for v_num in range(2, 3):
        xml_list = []
        test_xml = []

        csv_path = r"/home/data1/yw/water_detection/split_folds/fold_v2/fold_v%s/train.txt" % v_num
        test_csv = r"/home/data1/yw/water_detection/split_folds/fold_v2/fold_v%s/test.txt" % v_num
        with open(csv_path, "r") as f:
            for line_record in f.readlines():
                xml_name = line_record.strip("\n") + ".xml"
                xml_list.append(xml_name)

        with open(test_csv, "r") as f:
            for line_record in f.readlines():
                xml_name = line_record.strip("\n") + ".xml"
                test_xml.append(xml_name)

        print("fold_v%s" % v_num)
        analyze_classes_proportition(xml_dir, xmls=xml_list)
        analyze_classes_proportition(xml_dir, xmls=test_xml)
        print("\n")

def edit_csv(csv_dir):
    # csv_dir=r"/home/data1/yw/water_detection/split_folds/fold_v2"

    with open(os.path.join(csv_dir, "train.csv"), "r") as f:
        lines_record = f.readlines()
        lines_record.pop(0)

    with open(os.path.join(csv_dir, "train.txt"), "w") as f:
        for line in lines_record:
            f.write(line.strip("\n").split(',')[1] + "\n")

    with open(os.path.join(csv_dir, "test.csv"), "r") as f:
        lines_record = f.readlines()
        lines_record.pop(0)

    with open(os.path.join(csv_dir, "test.txt"), "w") as f:
        for line in lines_record:
            f.write(line.strip("\n").split(',')[1] + "\n")

    os.remove(os.path.join(csv_dir, "train.csv"))
    os.remove(os.path.join(csv_dir, "test.csv"))

def split_train_val(vfold_path, xml_dir, dirty_txt=None):
    train_val = os.path.join(vfold_path, "train.csv")  # 把要处理的xml文件名写入csv
    if os.path.exists(train_val):
        with open(train_val, 'r') as f:
            records = f.readlines()
            records.pop(0)

        target_xml_names = []
        for cord in records:
            target_xml_names.append(cord.strip("\n").split(',')[1] + '.xml')
    else:

        target_xml_names = os.listdir(xml_dir)
        target_xml_names = list(filter(lambda x: x.endswith('xml'), target_xml_names))

    if dirty_txt:  # there are dirty data in the given data, then filter them from txt
        with open(dirty_txt, 'r') as dt:
            dirty_list = dt.readlines()
        dirty_list = [x.strip('\n').split('.')[0] + '.xml' for x in dirty_list]

        target_xml_names = [x for x in target_xml_names if x not in dirty_list]

    split_folds(vfold_path, xml_dir, target_xml_names)

    # edit_csv(os.path.join(vfold_path,"fold_v2"))
    # shutil.copy(train_val,os.path.join(vfold_path,"fold_v2","trainval.txt"))

if __name__ == '__main__':
    # analyze_classes_proportition("/home/data1/yw/water_detection/train/Annotations")

    # split all dataset
    # split_folds(r"/home/data1/yw/mmdetection/data/water_detection/train_implmt/split_folds",r"/home/data1/yw/mmdetection/data/water_detection/train_implmt/Annotations")

    # class_name_dict={"holothurian":0 ,"echinus": 1,"scallop": 2,"starfish": 3,"waterweeds": 4,}
    # class_name_dict = {"echinus": 1, "scallop": 2, "starfish": 3, "holothurian": 4, }
    class_name_dict = {"Airport": 1, "Port": 2}

    # split train val dataset
    xml_dir = r"/home/data1/yw/data/mmdetection_data/airport_port_det_kdxf/train/Annotations"
    out_dir = r"/home/data1/yw/data/mmdetection_data/airport_port_det_kdxf/k-fold-v2"
    dirty_txt = r"/home/data1/yw/github_projects/personal_github/code_aculat/data_operation/image_pre_none.txt"
    split_train_val(out_dir, xml_dir, dirty_txt)
    # check_class_balance()
    # check_class_balance_txt()
    # edit_csv()
    # example_for_iterstrat()

    # analyse_image_wh("/home/data1/yw/water_detection/train/Annotations",
    #                  csv_path=["/home/data1/yw/water_detection/split_folds/fold_v1/train.csv",
    #                            "/home/data1/yw/water_detection/split_folds/fold_v1/test.csv"])


def twin(val):
    if type(val) != tuple and val != None:
        val = (val, val)
    return val

class ConvSettings:

    def __init__(self,
                 in_channels,
                 out_channels,
                 kernel_size,
                 padding=0,
                 dilation=1,
                 stride=1):
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.kernel_size = kernel_size
        self.padding = padding
        self.dilation = dilation
        self.stride = stride

    def compute_internal(self):
        return 2 * self.padding - self.dilation * (self.kernel_size - 1)

    def compute_output(self, I):
        output = (I + self.compute_internal() - 1) / self.stride + 1
        return int(output)

    def copy(self, kernel_size, padding, dilation):
        return ConvSettings(self.in_channels, self.out_channels, kernel_size,
                            padding, dilation, self.stride)

    def is_complete_equivalent(self, other):
        for (self_prop, other_prop) in [
            (self.in_channels, other.in_channels),
            (self.out_channels, other.out_channels),
            (self.kernel_size, other.kernel_size),
            (self.padding, other.padding),
            (self.dilation, other.dilation),
            (self.stride, other.stride),
        ]:
            if self_prop != other_prop:
                return False

        return True

    def is_type_equivalent(self, other):
        if self.stride != other.stride:
            return False

        if self.compute_internal() != other.compute_internal():
            return False

        return True

    def is_instant_equivalent(self, other, I):
        return self.compute_output(I) == other.compute_output(I)

    def generate_settings_type_eq(self,
                                  k_limit=None,
                                  p_limit=None,
                                  d_limit=None):
        settings = []
        if [k_limit, p_limit, d_limit].count(None) != 1:
            print("ERROR: UNBOUNDED HYPERPARAMETERS")
            return settings

        def add_settings(k, p, d):
            new = self.copy(int(k), p, d)
            if self.is_type_equivalent(new):
                settings.append(new)

        k_limit, p_limit, d_limit = twin(k_limit), twin(p_limit), twin(d_limit)
        if k_limit is None:
            for p in range(p_limit[0], p_limit[1] + 1):
                for d in range(d_limit[0], d_limit[1] + 1):
                    k = (2 * p - self.compute_internal()) / d + 1
                    add_settings(k, p, d)
        elif p_limit is None:
            for k in range(k_limit[0], k_limit[1] + 1):
                for d in range(d_limit[0], d_limit[1] + 1):
                    p = int(d * (k - 1) / 2)
                    add_settings(k, p, d)
        elif d_limit is None:
            for k in range(k_limit[0], k_limit[1] + 1):
                for p in range(p_limit[0], p_limit[1] + 1):
                    d = int((2 * p) / (k - 1))
                    add_settings(k, p, d)

        return settings

if __name__ == "__main__":
    settings = ConvSettings(0, 0, 3, 1, 1, 1)

    ranges = {
        "kd04": ((1, 4), None, (1, 4)),
        "kd08": ((1, 8), None, (1, 8)),
        "kd12": ((1, 12), None, (1, 12)),
        "kd16": ((1, 16), None, (1, 16)),
        "kp04": ((2, 4), (1, 4), None),
        "kp08": ((2, 8), (1, 8), None),
        "kp12": ((2, 12), (1, 12), None),
        "kp16": ((2, 16), (1, 16), None),
        "pd04": (None, (1, 4), (1, 4)),
        "pd08": (None, (1, 8), (1, 8)),
        "pd12": (None, (1, 12), (1, 12)),
        "pd16": (None, (1, 16), (1, 16)),
    }
    for (name, (K, P, D)) in ranges.items():
        new_settings = settings.generate_settings_type_eq(K, P, D)
        content = ""

        for ns in new_settings:
            content += f"({ns.kernel_size},{ns.padding},{ns.dilation})\n"

        f = open(f"{name}.txt", "w")
        f.write(content)
        f.close()


class SNode:
    def __init__(self, data):
        self.data = data
        self.next = None

    def __repr__(self):
        return self.data

class SLinkedList:
    def __init__(self):
        self.head = None

    def __repr__(self):
        node = self.head
        nodes = []
        while node is not None:
            nodes.append(node.data)
            node = node.next
        nodes.append("None")
        return " -> ".join(nodes)

    def insert(self, data):
        node = SNode(data)
        if not self.head:
            self.head = node
            return

        current_node = self.head
        while True:
            if not current_node.next:
                current_node.next = node
                break
            current_node = current_node.next

    def isEmpty(self):
        return self.head is None


#!/usr/bin/env python3

import os
import os.path
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.base import MIMEBase
from email import encoders

email = ''
password = ''
to = '@kindle.com'
group = 5

def send_email(s, i, paths):
    msg = MIMEMultipart()
    msg['Subject'] = 'Kindle docs ' + str(i)
    print(msg['Subject'])
    msg['From'] = email
    msg['To'] = to

    for path in paths:
        msg.attach(new_attachment(path))

    msg = msg.as_string()
    s.sendmail(email, to, msg)
    print()

def new_attachment(path):
    with open(path, 'rb') as book:
        attachment = MIMEBase('application', 'x-mobipocket-ebook')
        attachment.set_payload(book.read())
        fname = filename(path)
        print('  ', fname)
        attachment.add_header('Content-Disposition', 'attachment', filename=fname)
        encoders.encode_base64(attachment)
    return attachment

def filename(path):
    basename = os.path.basename(path)
    name, ext = os.path.splitext(basename)
    title, author = name.rsplit('-', 1)
    return title.strip() + ext

def main():
    s = smtplib.SMTP_SSL('smtp.gmail.com', 465)
    s.login(email, password)

    paths = []
    for root, dirs, files in os.walk('books'):
        for file in files:
            paths.append(os.path.join(root, file))

    i = 1
    for start in range(0, len(paths), group):
        send_email(s, i, paths[start:start+group])
        i += 1

    s.quit()

if __name__ == '__main__':
    main()


import os, multiprocessing

PORT = os.getenv("PORT", '7020')
HOST = os.getenv("HOST", '0.0.0.0')

errorlog = '-'
loglevel = 'info'
accesslog = '-'
access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'

bind = HOST+":"+PORT
workers = multiprocessing.cpu_count() * 2 + 1
threads = 2 * multiprocessing.cpu_count()


import discord
from discord.ext import commands

import database
import utils

class Misc(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    @commands.command(aliases=["p"])
    async def ping(self, ctx):
        await ctx.send(
            embed=utils.generate_embed("Pong", f"{self.bot.latency * 1000:2.3f} ms")
        )

async def setup(bot):
    await bot.add_cog(Misc(bot))


from loctext.learning.evaluations import is_in_swissprot, is_child_of_swissprot_annotation, accept_relation_uniprot_go, are_go_parent_and_child, get_localization_name

def parse(filepath, previous_annotations=None):
    ret = {}
    with open(filepath) as f:
        header = next(f)
        if previous_annotations:
            print(header.strip())

        for line in f:
            line = line.strip()
            typ, uniprot_ac, go, loc_name, inSwissProt, childSwissProt, confirmed, num_docs, *docs = line.split("\t")

            rel_key = uniprot_ac, go

            if previous_annotations is None:
                if not confirmed:
                    break
                else:
                    ret[rel_key] = confirmed
            else:
                confirmed = previous_annotations.get(rel_key, "")
                inSwissProt = str(is_in_swissprot(uniprot_ac, go))
                childSwissProt = childSwissProt
                row = [typ, uniprot_ac, go, loc_name, inSwissProt, childSwissProt, confirmed, num_docs] + docs
                print('\t'.join(row))
    return ret

if __name__ == "__main__":
    import sys

    main_filepath = sys.argv[1]

    # Order matters
    previous_results_filepaths = sys.argv[2:]
    assert len(previous_results_filepaths) == 2
    previous_annotations = {}
    for filepath in previous_results_filepaths:
        previous_annotations.update(parse(filepath, None))

    parse(main_filepath, previous_annotations)


# -*- coding: utf-8 -*-

import os.path

here = os.path.abspath(os.path.dirname(__file__))
base = os.path.abspath(os.path.join(here, '..'))

DEBUG_TB_INTERCEPT_REDIRECTS = False

DEBUG = True
SECRET_KEY = 'skylines'

SMTP_SERVER = 'localhost'
EMAIL_FROM = 'no-reply@skylines.aero'

"""
# Logging handlers (disabled in DEBUG mode)

ADMINS = [
    'tobias.bieniek@gmx.de'
]

mail_handler = (
    'ERROR', 'SMTPHandler',
    ('localhost', 'error@skylines.aero', ADMINS, 'SkyLines Error Report'))

file_handler = (
    'INFO', 'RotatingFileHandler',
    ('/home/turbo/skylines.log', 'a', 10000, 4))

LOGGING_HANDLERS = [mail_handler, file_handler]
"""

# This should probably be changed for a multi-threaded production server
CACHE_TYPE = 'simple'

SQLALCHEMY_DATABASE_URI = 'postgresql:///skylines'

ASSETS_DEBUG = False
ASSETS_AUTO_BUILD = True
ASSETS_DIRECTORY = os.path.join(base, 'webassets')
ASSETS_URL = '/assets'
ASSETS_LOAD_DIR = os.path.join(base, 'skylines', 'frontend', 'static')
ASSETS_LOAD_URL = '/'

SKYLINES_FILES_PATH = os.path.join(base, 'htdocs', 'files')
SKYLINES_ELEVATION_PATH = os.path.join(base, 'htdocs', 'srtm')
SKYLINES_MAPSERVER_PATH = os.path.join(base, 'mapserver')
SKYLINES_BACKEND_PATH = os.path.join(base, 'backend')

SKYLINES_TEMPORARY_DIR = '/tmp'

# how many entries should a list have?
SKYLINES_LISTS_DISPLAY_LENGTH = 50

SKYLINES_API_URL = 'http://localhost:5001'
SKYLINES_MAP_TILE_URL = 'https://www.skylines.aero/mapproxy'

BROKER_URL = 'redis://localhost:6379/0'
CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'
CELERYD_LOG_LEVEL = 'INFO'

# limits for AnalyseFlight
SKYLINES_ANALYSIS_ITER = 10e6 # iteration limit, should be around 10e6 to 50e6
SKYLINES_ANALYSIS_MEMORY = 256 # approx memory limit in MB

# List of airspace types to check for infringements
SKYLINES_AIRSPACE_CHECK = ('RESTRICT', 'DANGER', 'PROHIBITED', 'CTR',
                           'CLASSA', 'CLASSB', 'CLASSC', 'CLASSD',
                           'NOGLIDER', 'TMZ', 'MATZ')

SKYLINES_CESIUM_PATH = os.path.join(base, 'skylines', 'frontend', 'static', 'vendor', 'openlayers', 'cesium')


import streamlit as st
from langchain.callbacks.base import BaseCallbackHandler
from langchain.chains import RetrievalQAWithSourcesChain
from langchain.chains import RetrievalQA
from langchain.chains import ConversationalRetrievalChain,LLMChain, RetrievalQA

import os
from dotenv import load_dotenv
import qdrant_client

load_dotenv()

openai_api_type = os.getenv("OPENAI_API_TYPE")
openai_api_key = os.getenv("OPENAI_API_KEY")
openai_api_base = os.getenv("OPENAI_API_BASE")
openai_api_version = os.getenv("OPENAI_API_VERSION")
deployment_name = os.getenv("OPENAI_COMPLETION_DEPLOYMENT_NAME")
completion_model = os.getenv("OPENAI_COMPLETION_MODEL")
embedding_name = os.getenv("OPENAI_EMBEDDING_DEPLOYMENT_NAME")
qdrant_url = os.getenv("QDRANT_URL")
qdrant_collection = os.getenv("QDRANT_COLLECTION")

st.set_page_config(page_title="Movie expert AI", page_icon="ðŸ“º")

def settings():

    # Vectorstore
    from langchain.vectorstores import Qdrant
    from langchain.embeddings import OpenAIEmbeddings
    embeddings_model = OpenAIEmbeddings(
        deployment=embedding_name,
        chunk_size=1
    )

    client = qdrant_client.QdrantClient(
        qdrant_url
    )

    vector_store = Qdrant(
        client=client, collection_name=qdrant_collection,
        embeddings=embeddings_model,
    )

    # LLM
    from langchain.llms import AzureOpenAI
    from langchain.chat_models import AzureChatOpenAI
    llm = AzureChatOpenAI(
        openai_api_type = openai_api_type,
        openai_api_version = openai_api_version,
        openai_api_base = openai_api_base,
        openai_api_key = openai_api_key,
        deployment_name = deployment_name,
        model_name=completion_model
    )

    retriever = vector_store.as_retriever(search_type="mmr")

    return retriever, llm

class StreamHandler(BaseCallbackHandler):
    def __init__(self, container, initial_text=""):
        self.container = container
        self.text = initial_text

    def on_llm_new_token(self, token: str, **kwargs) -> None:
        self.text += token
        self.container.info(self.text)

class PrintRetrievalHandler(BaseCallbackHandler):
    def __init__(self, container):
        self.container = container.expander("Context Retrieval")

    def on_retriever_start(self, query: str, **kwargs):
        self.container.write(f"**Question:** {query}")

    def on_retriever_end(self, documents, **kwargs):
        # self.container.write(documents)
        for idx, doc in enumerate(documents):
            source = doc.metadata["source"]
            self.container.write(f"**Results from {source}**")
            self.container.text(doc.page_content)

st.sidebar.image("movieagent.jpg")
st.header("`Movie AI Agent`")
st.info("`I am an AI that can answer questions about all the movies you told me earlier.`")

# Make retriever and llm
if 'retriever' not in st.session_state:
    st.session_state['retriever'], st.session_state['llm'] = settings()
retriever = st.session_state.retriever
llm = st.session_state.llm

# User input
question = st.text_input("`Ask a question:`")

if question:
    qa_sources_chain = RetrievalQAWithSourcesChain.from_chain_type(llm, retriever=retriever)
    retrieval_streamer_cb = PrintRetrievalHandler(st.container())
    answer = st.empty()
    stream_handler = StreamHandler(answer, initial_text="`Answer:`\n\n")
    result = qa_sources_chain({"question": question},callbacks=[retrieval_streamer_cb, stream_handler])
    print(result)
    answer.info('`Answer:`\n\n' + result['answer'])
    st.info('`Sources:`\n\n' + result['sources'])


import json
from pytils.translit import slugify

from django.core.management.base import BaseCommand

from my import models

def load_json():
    with open('./delivery/data/data.txt', encoding='utf-8') as f:
        data = json.loads(f.read())
    return data

class Command(BaseCommand):
    help = 'load data from old site'

    def handle(self, *args, **options):
        data = load_json()
        for item in data:
            cat = models.Categories.objects.create(name=item['cat'], slug=slugify(item['cat']))
            for menu in item['menu']:
                sc = models.SubCategories.objects.filter(name=menu['subcategory'].replace('  ', ' ')).first()
                if menu['subcategory']=='гр.':
                    menu['subcategory'] =None
                else:
                    menu['subcategory'] = sc or\
                                        models.SubCategories.objects.create(
                                                                        name=menu['subcategory'].replace('  ',' '),
                                                                        slug=slugify(menu['subcategory']))

                if menu['image']:
                    image =  f"media/old/{menu['image']}"
                else:
                    image = None
                m = models.Products(
                    name=menu['name'],
                    slug=slugify(menu['name']+menu['subcategory'].__str__()),
                    category=cat,
                    description=menu['description'],
                    price=menu['price'],
                    image=image,
                    subcategory = menu['subcategory']
                )
                m.save()


# Definition for an interval.
# class Interval(object):
#     def __init__(self, s=0, e=0):
#         self.start = s
#         self.end = e

# 贪心算法的代表情况之一，贪心算法的成立性见书中证明
# 若要求最少的删除，则要保证当前情况下选择最早结束的interval ，则局部可以保证容纳最多数量的interval
# 由于贪心性质，则总体可以保证容纳数量最多的interval
class Solution(object):
    def eraseOverlapIntervals(self, intervals):
        """
        :type intervals: List[Interval]
        :rtype: int
        """
        intervals.sort(key = lambda x:x.end)
        #for i in range(len(intervals)):
        #    print(intervals[i].start, ' ', intervals[i].end)
        current, res = -float('inf'), 0
        for i in intervals:
            if current > i.start :
                res += 1
            else:
                current = i.end
        return res


from django.shortcuts import render
from itertools import chain
from author.models import Author
from .models import Book
from django.contrib.auth.decorators import login_required
from operator import attrgetter

@login_required(login_url='login')
def searchBooks(request):
    query = request.GET.get('query')
    if not query:
        return showBooks(request)
    by_id = None
    try:
        by_id = Book.objects.all().filter(id=int(query))
    except:
        print('no int')
    by_name = Book.objects.all().filter(name__icontains=query)
    by_description = Book.objects.all().filter(description__icontains=query)
    by_author_name = Book.objects.all().filter(authors__name__icontains=query)
    by_author_surname = Book.objects.all().filter(authors__surname__icontains=query)
    if by_id:
        result_list = list(chain(by_id, by_name, by_description, by_author_name, by_author_surname))
    else:
        result_list = list(chain(by_name, by_description, by_author_name, by_author_surname))
    book_author_zip = zip(result_list, get_authors(result_list))
    return render(request, 'book/book.html', {'book_author_zip': book_author_zip})

@login_required(login_url='login')
def showBooks(request):
    books_query = Book.objects.order_by('id')
    book_author_zip = zip(books_query, get_authors(books_query))
    return render(request, 'book/book.html', {'book_author_zip': book_author_zip})

def get_authors(books):
    authors_list = []
    if type(books) == Book:
        print(True)
        book = books
        print(book.authors)
        if list(book.authors.all()):
            for author in book.authors.all():
                authors_list.append(author)
        else:
            authors_list.append(None)
    else:
        for book in books:
            temp = []
            if list(book.authors.all()):
                for author in book.authors.all():
                    temp.append(author)
                authors_list.append(temp)
            else:
                authors_list.append(None)
    return authors_list

def showOrderedBooks(request, pk):
    books_query = Book.objects.order_by(pk)
    book_author_zip = zip(books_query, get_authors(books_query))
    return render(request, 'book/book.html', {'book_author_zip': book_author_zip})

def showSpecificBook(request, id):
    book_obj = Book.objects.get(id=id)
    return render(request, 'book/specificBook.html', {'book': book_obj, 'authors': get_authors(book_obj)})

@login_required(login_url='login')
def createBook(request):
    if request.user.role != 1:
        text = 'You have no permission to view this page'
        return render(request, 'authentication/denied.html', {'text': text})
    if request.method == 'POST':
        name = request.POST.get('name')
        description = request.POST.get('description')
        count = request.POST.get('count')
        Book.create(name=name, description=description, count=count)
    return render(request, 'book/create_book.html')


import calendar
from datetime import datetime as dt
from datetime import timedelta as td

"""
Python utility library for payroll calendars - weekly, biweekly, semimonthly
and monthly

https://payroll.unca.edu/sites/default/files/2016%20Payroll%20Calendar.pdf`
"""

def add_one_month(t):
    """Return a `datetime.date` or `datetime.datetime` (as given) that is
    one month earlier.

    Note that the resultant day of the month might change if the following
    month has fewer days:

        >>> add_one_month(datetime.date(2010, 1, 31))
        datetime.date(2010, 2, 28)
    """
    one_day = td(days=1)
    one_month_later = t + one_day
    while one_month_later.month == t.month:  # advance to start of next month
        one_month_later += one_day
    target_month = one_month_later.month
    while one_month_later.day < t.day:  # advance to appropriate day
        one_month_later += one_day
        if one_month_later.month != target_month:  # gone too far
            one_month_later -= one_day
            break
    return one_month_later

def next_sunday(date):
    if date.weekday() == 6:
        next_sunday = date
    else:
        day = date
        while day < date + td(days=7):
            day = day + td(days=1)

            if day.weekday()== 6:
                break
        next_sunday = day
    return next_sunday

def previous_monday(date):
    if date.weekday() == 0:
        return date
    else:
        day = date
        while day > date - td(days=7):
            day = day - td(days=1)

            if day.weekday()== 0:
                break
        previous_monday = day
    return previous_monday

def current_week(date):
    period_start = previous_monday(date)
    period_end = next_sunday(date)
    return period_start, period_end

def current_month(date):
    period_start = dt(year=date.year,
                      month=date.month,
                      day=1)

    period_end = add_one_month(dt(year=date.year, month=date.month, day=1)) - td(days=1)
    return period_start, period_end

def last_monday_previous_year(date):
    jan1 = dt(year=date.year, month=1, day=1)
    mon = jan1
    while mon.weekday() != 0:

        mon = mon - td(days=1)
    return mon

def first_biweek_of_year(date):
    return last_monday_previous_year(dt.now()), \
        last_monday_previous_year(dt.now()) + td(days=13)

def next_biweek(start, end):
    return start + td(14), end + td(14)

def next_week(start, end):
    return start + td(7), end + td(7)

def next_month(start, end):
    if start.month != end.month:
        print('start and end month are different')
    if start.month < 12:
        return (dt(year=start.year, month=start.month + 1, day=start.day),
                dt(year=end.year, month=end.month + 1, day=calendar.monthrange(end.year, end.month + 1)[1]))
    else:
        return (dt(year=start.year + 1, month=1, day=start.day),
                dt(year=end.year + 1, month=1, day=calendar.monthrange(end.year, 1)[1]))

def next_semimonth(start, end):
    if end.day == 15:

        if end.month < 12:
            return (dt(year=start.year, month=start.month, day=16),
                    dt(year=end.year, month=end.month + 1 if end.month < 12 else 1, day=1) -
                    td(days=1))
        else:
            if end.day < 16:
                return (dt(year=end.year, month=end.month, day=16),
                        dt(year=end.year, month=end.month, day=calendar.monthrange(end.year, end.month)[1]))
    else:
        if end.month < 12:
            return (dt(year=start.year, month=start.month + 1, day=1),
                    dt(year=end.year, month=end.month + 1, day=15))
        else:
            return (dt(year=start.year + 1, month=1, day=1),
                    dt(year=start.year + 1, month=1, day=15))

def current_semimonth(date):
    if date.day < 16:
        return (dt(year=date.year, month=date.month, day=1),
                dt(year=date.year, month=date.month, day=15))
    else:
        return (dt(year=date.year, month=date.month, day=16),
                dt(year=date.year, month=date.month, day=calendar.monthrange(date.year, date.month)[1]))

def current_biweek(date):
    start, end = first_biweek_of_year(date)
    if date >= start and date <= end + td(days=1):
        return start, end

    else:
        while date >= start and date >= end + td(days=1):
            start, end = next_biweek(start, end)

    return start, end

# todo: write unittest for weeks_between_dates


#encoding=utf8
import nltk,json
from tool import ner_stanford,cut_stanford

def get_stanfrod_ner_nodes(parent):
    date = ''
    org = ''
    loc = ''
    for node in parent:
        if type(node) is nltk.Tree:
            if node.label() == 'DATE':
                date = date+' '+''.join([i[0] for i in node])
            elif node.label() == 'ORGANIZATIONL':
                org = org+' '+''.join([i[0] for i in node])
            elif node.label() == 'LOCATION':
                loc = loc+' '+''.join([i[0] for i in node])

    if len(date) > 0 or len(org) > 0 or len(loc) > 0:
        return {'date':date,'org':org,'loc':loc}
    else:
        return {}

def grammer_parse(raw_sentence=None,file_object=None):
    if len(raw_sentence.strip()) < 2:
        return False
    grammer_dict = {
        'stanford_ner_drop':r"""
        DATE:{<DATE>+<MISC>?<DATE>*}
        {<DATE>+}
        {<TIME>+}
        ORGANIZATIONL:{<ORGANIZATION>+}
        LOCATION:{<LOCATION|STATE_OR_PROVINCE|CITY|COUNTRY>+}
        """
    }

    stanford_ner_drop_rp = nltk.RegexpParser(grammer_dict['stanford_ner_drop'])
    try:
        stanford_ner_drop_result = stanford_ner_drop_rp.parse(ner_stanford(raw_sentence))
        # 通过stanfordnlp的ner之后，再通过nltk的parse进行构建语法树
    except:
        print('the error sentence is {}'.format(raw_sentence))
    # 如果try里面的语句没有问题，就执行else语句
    else:
        stanford_keep_drop_dict = get_stanfrod_ner_nodes(stanford_ner_drop_result)
        if len(stanford_keep_drop_dict) > 0:
            file_object.write(json.dumps(stanford_keep_drop_dict,skipkeys=False,
            ensure_ascii=False,
            check_circular=True,
            allow_nan=True,
            cls=None,
            indent=4,
            separators=None,
            default=None,
            sort_keys=False
            ))

分词、词性标注以及命名实体识别/命名实体识别/rules.py


from flask_restful import Resource
from flask import request, jsonify, make_response
from api.models import User
from api.models import Project
from api.models import Task
from api.maiin import app
from flask_login import LoginManager
from flask_login import login_user, current_user,login_required,logout_user
from flask_restful import Api
from api.validators import *

login_manager = LoginManager()
login_manager.app=app
login_manager.init_app(app)
login_manager.login_view = 'login'

"""define the user_loader method"""
@login_manager.user_loader
def load_user(user_id):
    return User.query.get(int(user_id))

"""Class to handle the / requests, its possible use GET methods"""
class Initial(Resource):
    @login_required
    def get(self):
        return app.send_static_file('dashboard.html')

"""Class to handle the /login/ requests, its possible use GET methods"""
class Login(Resource):
    def get(self):
        return app.send_static_file('login.html')

"""Class to handle the /dashboard/ requests, its possible use GET methods"""
class Dashboard(Resource):
    @login_required
    def get(self):
        return app.send_static_file('dashboard.html')

"""Class to handle the /api/user/login/ requests, its possible use POST methods"""
class LoginAPI(Resource):
    def post(self):
        data = request.get_json()
        user = User.get_user(data["username"])
        if user:
            if user.password == data["password"]:
                login_user(user, remember=True)
                return make_response(jsonify({'redirect':'/dashboard'}),200)
        return make_response(jsonify({'credentials': 'Credenciais erradas'}),404)

"""Class to handle the /api/user/logout/ requests, its possible use GET methods"""
class Logout(Resource):
    @login_required
    def get(self):
        logout_user()
        return make_response(jsonify({'redirect':'/login'}),200)

"""Class to handle the /api/user/ requests,its possible use PUT/GET methods"""
class UserAPI(Resource):
    @login_required
    def get(self):
        user = load_user(current_user.id)
        return make_response(jsonify(user.serialize))

    @login_required
    def put(self):
        data = request.get_json()
        user = User.get_user(data["username"])
        if user == None or user.id==current_user.id:
            if user.id==current_user.id:
                if not validateEmail(data["email"]):
                    return make_response(jsonify({'email': 'Email inválido!'}), 400)
                result=User.update_user(current_user.id,data)
                if result:
                    return make_response(jsonify({'result': 'Informacao alterada'}),200)
                return make_response(jsonify({'error': 'Ocorreu um erro! Tente de outra vez!'}), 500)
            return make_response(jsonify({'account': 'Não tem acesso a esta conta'}), 403)
        return make_response(jsonify({'username': 'Username ja utilizado!'}), 400)

"""Class to handle the /api/user/register requests, its possible use POST methods"""
class RegisterAPI(Resource):
    def post(self):
        data = request.get_json()
        user = User.get_user(data["username"])
        if user==None:
            if not validateEmail(data["email"]):
                return make_response(jsonify({'email': 'Email inválido!'}), 400)
            else:
                result = User.add_user(data)
                if result :
                    return make_response(jsonify({'redirect': '/login'}), 200)
                return make_response(jsonify({'error': 'Ocorreu um erro no registo. Tente outra vez.'}), 500)
        return make_response(jsonify({'username': 'Username já utilizado!'}), 400)

"""Class to handle the /api/projects/ requests, its possible use POST/GET methods"""
class ProjectsAPI(Resource):
    @login_required
    def get(self):
        projects=Project.getProjects(current_user.id)
        return make_response(jsonify(projects), 200)

    @login_required
    def post(self):
        data = request.get_json()
        project = Project.getProjectByName(data["name"],current_user.id)
        id=current_user.id
        if project == None:
            result = Project.add_project(current_user.id, data)
            if result:
                return make_response(jsonify({'result': 'Projeto criado com sucesso!'}), 201)
            return make_response(jsonify({'error': 'Ocorreu um erro! Tente outra vez!'}), 200)
        return make_response(jsonify({'name': 'Ja tem um nome de projeto igual a este!'}), 400)

"""Class to handle the /api/project/<id> requests, its possible use PUT/GET/DELETE methods"""
class ProjectAPI(Resource):
    @login_required
    def get(self,idProject):
        project = Project.getProject(idProject,current_user.id)
        if project != None :
            return make_response(jsonify(project), 200)
        return make_response(jsonify({'project': 'Projeto nao encontrado!'}), 404)#meter no site

    @login_required
    def put(self,idProject):
        data = request.get_json()
        project = Project.getProject(idProject,current_user.id)
        if project != None :
            result = Project.update_project(idProject, data)
            if result :
                return make_response(jsonify({'result': 'Projeto atualizado com sucesso!'}), 200)
            return make_response(jsonify({'error': 'Ocorreu um erro! Tente outra vez!'}), 500)
        return make_response(jsonify({'project': 'Projeto nao encontrado!'}), 404)

    @login_required
    def delete(self,idProject):
        project = Project.getProject(idProject,current_user.id)
        if project != None :
            result =Project.delete_project(idProject)
            if result:
                return make_response(jsonify({'result': 'Projeto removido com sucesso!'}), 200)
            return make_response(jsonify({'error': 'Ocorreu um erro! Tente outra vez!'}), 500)
        return make_response(jsonify({'project': 'Projeto nao encontrado!'}), 404)

"""Class to handle the /api/project/<id>/tasks/ requests, its possible use POST/GET methods"""
class TasksAPI(Resource):
    @login_required
    def get(self,idProject):
        project = Project.getProject(idProject,current_user.id)
        if (project != None):
            tasks=Task.getTasks(idProject)
            return make_response(jsonify(tasks), 200)
        return make_response(jsonify({'task': 'Task nao encontrada!'}), 404)

    def post(self,idProject):
        data = request.get_json()
        task=validateTask(data,idProject)
        if task==None:
            return make_response(jsonify({'fields': 'Existem campos mal preenchidos!'}), 400)
        else:
            result = Task.add_task(idProject,data)
            if result:
                return make_response(jsonify({'result': 'Task adicionada com sucesso!'}), 200)
            return make_response(jsonify({'error': 'Ocorreu um erro! Tente de outra vez!'}), 500)

"""Class to handle the /api/project/<id>/tasks/<id> requests, its possible use PUT/GET/DELETE methods"""
class TaskAPI(Resource):
    @login_required
    def get(self,idProject,idTask):
        project = Project.getProject(idProject,current_user.id)
        task = Task.getTask(idProject, idTask)
        if project != None and task!=None:
            return make_response(jsonify(task), 200)
        return make_response(jsonify({'task': 'Task nao encontrada!'}), 403)

    @login_required
    def put(self,idProject,idTask):
        project = Project.getProject(idProject,current_user.id)
        if project != None :
            data = request.get_json()
            task = validateTask(data, idProject)
            if task == None:
                return make_response(jsonify({'fields': 'Existem campos mal preenchidos!'}), 400)
            else:
                task = Task.getTask(idProject, idTask)
                if task != None:
                    result=Task.update_task(idTask,data,idProject)
                    if result:
                        return make_response(jsonify({'result': 'Task atualizada com sucesso!'}), 200)
                    return make_response(jsonify({'error': 'Ocorreu um erro! Tente de outra vez!'}), 500)
        return make_response(jsonify({'task': 'Task nao encontrada!'}), 404)

    @login_required
    def delete(self,idProject,idTask):
        project = Project.getProject(idProject,current_user.id)
        if project != None :
            task = Task.getTask(idProject, idTask)
            if task != None:
                result = Task.delete_task(idProject,idTask)
                if result:
                    return make_response(jsonify({'result': 'Task removida com sucesso!'}), 200)
                return make_response(jsonify({'error': 'Ocorreu um erro! Tente de outra vez!'}), 500)
        return make_response(jsonify({'task': 'Task nao encontrada!'}), 404)

"""Define the routes"""
api = Api(app)
api.add_resource(Initial, '/', endpoint = 'initial')
api.add_resource(Login, '/login', endpoint = 'login')
api.add_resource(Dashboard, '/dashboard', endpoint = 'dashboard')
api.add_resource(UserAPI, '/api/user/', endpoint = 'user')
api.add_resource(LoginAPI, '/api/user/login/', endpoint = 'loginapi')
api.add_resource(Logout, '/api/user/logout/', endpoint = 'logout')
api.add_resource(RegisterAPI, '/api/user/register/', endpoint = 'register')
api.add_resource(ProjectsAPI, '/api/projects/', endpoint = 'projects')
api.add_resource(ProjectAPI, '/api/projects/<idProject>/', endpoint = 'project')
api.add_resource(TasksAPI, '/api/projects/<idProject>/tasks/', endpoint = 'tasks')
api.add_resource(TaskAPI, '/api/projects/<idProject>/tasks/<idTask>/', endpoint = 'task')


from sqlalchemy import between

from nanohttp import context, HTTPBadRequest

class FilteringMixin:
    @classmethod
    def filter_by_request(cls, query=None):
        # noinspection PyUnresolvedReferences
        query = query or cls.query

        # noinspection PyUnresolvedReferences
        for c in cls.iter_dict_columns():
            # noinspection PyUnresolvedReferences
            json_name = cls.get_dict_key(c)
            if json_name in context.query:
                value = context.query[json_name]
                query = cls._filter_by_column_value(query, c, value)

        return query

    @classmethod
    def _filter_by_column_value(cls, query, column, value):
        import_value = getattr(cls, 'import_value')
        if not isinstance(value, str):
            raise HTTPBadRequest()

        if value.startswith('^') or value.startswith('!^'):
            value = value.split(',')
            not_ = value[0].startswith('!^')
            first_item = value[0][2 if not_ else 1:]
            items = [first_item] + value[1:]
            items = [i for i in items if i.strip()]
            if not len(items):
                raise HTTPBadRequest('Invalid query string: %s' % value)
            expression = column.in_([import_value(column, j) for j in items])
            if not_:
                expression = ~expression

        elif value.startswith('~'):
            values = value[1:].split(',')
            start, end = [import_value(column, v) for v in values]
            expression = between(column, start, end)

        elif value == 'null':
            expression = column.is_(None)
        elif value == '!null':
            expression = column.isnot(None)
        elif value.startswith('!'):
            expression = column != import_value(column, value[1:])
        elif value.startswith('>='):
            expression = column >= import_value(column, value[2:])
        elif value.startswith('>'):
            expression = column > import_value(column, value[1:])
        elif value.startswith('<='):
            expression = column <= import_value(column, value[2:])
        elif value.startswith('<'):
            expression = column < import_value(column, value[1:])
        elif value.startswith('%~'):
            expression = column.ilike('%%%s%%' % import_value(column, value[2:]))
        elif value.startswith('%'):
            expression = column.like('%%%s%%' % import_value(column, value[1:]))
        else:
            expression = column == import_value(column, value)

        return query.filter(expression)


# import module sys to get the type of exception
import sys

randomList = ['a', 0, 2]

for entry in randomList:
    try:
        print("The entry is", entry)
        r = 1/int(entry)
        break
    except Exception as e:
        print("Oops!", e.__class__, "occurred.")
        print("Next entry.")
        print()
print("The reciprocal of", entry, "is", r, "\n\n")

#
# In Python, exceptions are raised when errors occur at run-time.
# We can manually raise an exception using the "raise" keyword.
try:
    a = int(input("Enter a positive integer: "))
    if a <= 0:
        raise ValueError("That is not a positive number!")
except ValueError as ve:
    print(ve)


import turtle

t=turtle.Turtle()

sc = turtle.Screen()
sc.bgcolor('black')

t.pencolor('cyan')
t.speed(0)
t.width(10)

def circle():
    for i in range(7):
        t.circle(150-(i*8))
        t.right(60)

for j in range(8):
    circle()

t.ht()
turtle.done()


size = 0
q = []

def process(v):
    while len(q) > 0 and q[0] <= v[0]:
        q.pop(0)

    if len(q) < size:
        if len(q) == 0:
            q.append(v[0] + v[1])
            return v[0]
        else:
            st = max(v[0], q[-1])
            q.append(st + v[1])
            return st
    else:
        return -1

def request_process(A):
    response = []
    for v in A:
        response.append(process(v))

    return response

if __name__ == "__main__":
    size, n = map(int, input().split())

    A = []
    for _ in range(n):
        s, p = map(int, input().split())
        A.append([s, p])

    response_list = request_process(A)

    for response in response_list:
        print(response)


#!/usr/bin/env python
'Explore project results'

##   Copyright (C) 2010 University of Texas at Austin
##
##   This program is free software; you can redistribute it and/or modify
##   it under the terms of the GNU General Public License as published by
##   the Free Software Foundation; either version 2 of the License, or
##   (at your option) any later version.
##
##   This program is distributed in the hope that it will be useful,
##   but WITHOUT ANY WARRANTY; without even the implied warranty of
##   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##   GNU General Public License for more details.
##
##   You should have received a copy of the GNU General Public License
##   along with this program; if not, write to the Free Software
##   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
import sys

try:
    import wx
except:
    sys.stderr.write('Please install wx!\n\n')
    sys.exit(1)

from signal import *
import os, commands

class TestFrame(wx.Frame):
    def __init__(self):
        cwd = os.getcwd()
        proj = os.path.join(
            os.path.basename(os.path.dirname(os.path.dirname(cwd))),
            os.path.basename(os.path.dirname(cwd)),
            os.path.basename(cwd))
        wx.Frame.__init__(self,None,-1,proj)
        self.pid = 0

        signal(SIGINT,self.handler)

        sizer = wx.BoxSizer(wx.VERTICAL)

        panel = wx.Panel(self,-1)
        buttons = self.set_buttons(panel)
        panel.SetSizer(buttons)
        buttons.Fit(self)

        sizer.Add(panel,0,wx.ALL|wx.EXPAND,5)

        self.results = commands.getoutput("scons -s results").split()
        c = self.results[-1:][0]
        if (c < 'A' or c > 'z'):
            self.results.pop() # remove scons junk

        self.flip = {}

        panel = wx.Panel(self,-1)
        rsizer = wx.FlexGridSizer(rows=len(self.results),cols=2,vgap=5)

        for fig in self.results:
            self.flip[fig] = False

            title = self.get_title(fig)
            if title:
                title = '%s [%s]' % (fig,title)
            else:
                title = fig
            cb = wx.CheckBox(panel,-1,title)
            cb.Bind(wx.EVT_CHECKBOX, self.set_flip)
            rsizer.Add(cb,0,0)

            b = wx.Button(panel,-1,'show')
            self.Bind(wx.EVT_BUTTON,self.show(fig),b)
            rsizer.Add(b,0,0)

        panel.SetSizer(rsizer)
        rsizer.Fit(self)

        sizer.Add(panel,0,wx.ALL|wx.EXPAND,15)

        self.SetSizer(sizer)
        sizer.Fit(self)

    def set_flip(self, event):
        sender = event.GetEventObject()
        fig = sender.GetLabel()
        self.flip[fig] = sender.GetValue()

    def set_buttons(self,panel):
        bsizer = wx.BoxSizer(wx.HORIZONTAL)

        bcycl = wx.Button(panel,-1,'Cycle')
        bcycl.SetBackgroundColour('light yellow')
        self.Bind(wx.EVT_BUTTON,self.showall,bcycl)
        bsizer.Add(bcycl,0,wx.ALL,5)

        bflip = wx.Button(panel,-1,'Flip')
        bflip.SetBackgroundColour('light green')
        self.Bind(wx.EVT_BUTTON,self.flipit,bflip)
        bsizer.Add(bflip,0,wx.ALL,5)

        bquit = wx.Button(panel,-1,'Quit')
        bquit.SetBackgroundColour('pink')
        self.Bind(wx.EVT_BUTTON,self.quit,bquit)
        bsizer.Add(bquit,0,wx.ALL,5)

        return bsizer

    def quit(self,event):
        sys.exit(0)

    def handler(self,signum,frame):
        sys.stdout.write("\n%s: aborting...\n" % sys.argv[0])
        if self.pid:
            os.kill(self.pid,SIGINT)
        sys.exit(1)

    def showall(self,event):
        self.pid = os.fork()
        if not self.pid:
            for fig in self.results:
                os.system ("pscons %s.view" % fig)
        sys.exit()

    def flipit(self,event):
        figs = map(lambda x: 'Fig/%s.vpl' % x,
                   filter(lambda y: self.flip[y],self.results))
        if figs:
            self.pid = os.fork()
            if not self.pid:
                command = "sfpen %s" % ' '.join(figs)
                sys.stderr.write(command+'\n')
                os.system (command)
                sys.exit()

    def show(self,fig):
        def showfig(event):
            os.system("pscons %s.view" % fig)
        return showfig

    def get_title(self,fig):
        vpl = os.path.join('Fig',fig+'.vpl')
        if not os.path.isfile(vpl):
            return None
        txt = os.path.join('Fig',fig+'.txt')
        os.system('sfpldb < %s > %s' % (vpl,txt))
        titles = []
        try:
            plot = open(txt,'r')
            for line in plot:
                if line[0] != '[':
                    continue
                line2 = plot.next()
                if line2[:5] != 'title':
                    continue
                while line2:
                    line2 = plot.next()
                    if line2[0] == 'G':
                        title = plot.next()
                        titles.append(title.rstrip())
                        break
            title = ','.join(titles)
        except:
            title = None

        plot.close()
        os.unlink(txt)
        return title

if __name__=='__main__':
    app = wx.App()
    frame = TestFrame()
    frame.Show()

    app.MainLoop()


'''
Created on 12 de abr de 2017

@author: Thiago
'''
from analytics import minhash_util
from math import sqrt

def check(dirToScreens):
    import os
    from os import path
    files = []
    for f in os.listdir(dirToScreens):
        if f.endswith(".csv"):
            files.append(f)
    return files

def classifica(att1,att2,gabarito,rev=False):
    for resp in gabarito:
        resposta = resp
        if (rev):
            resposta = [resp[1],resp[0]]

        if (att1 == resposta[0]) and (att2 == resposta[1]):
            return 'TM'
    return 'FM'

def corrigeLinha(att1,dt1,att2,dt2,gabarito):
    if dt1 == 'ncvoters' or dt2 == 'ncvoters':
        if dt1 == 'ncvoters':
            return classifica(h1,h2,gabarito),False
        else:
            return classifica(h1,h2,gabarito,rev=True),True

    if dt1 == 'medicare' or dt2 == 'medicare':
        if dt1 == 'medicare':
            return classifica(h1,h2,gabarito),False
        else:
            return classifica(h1,h2,gabarito,rev=True),True

def buscaTrueNonMatch(file1,data_length1,dt1,file2,data_length2,dt2,espelho_gabartio,gabarito,inv_flag):
    a = set()
    for g in gabarito:
        tupla = tuple(g)
        if inv_flag:
            tupla = (g[1],g[0])

        a.add(tupla)
    z = a - espelho_gabarito

    r = []
    if len(z) != 0:
        for e in z:
            tnm = [p1['filename'],data_length1,dt1,e[0],p2['filename'],data_length2,dt2,e[1],0,'TNM']
            r.append(tnm)

    return r

if __name__ == '__main__':
    profile_dir = 'data/profile/plano_bigram/'
    outfile_prefix =  'data/profile/resumo_'

    filenames = check(profile_dir)
    first = True
    dt_set = set()

    for fname in filenames:
        dt_set.add(minhash_util.getDataType(fname))

    outfiles = {}
    headers = {}
    for element in dt_set:
        file = outfile_prefix + str(element) +'_plano_bigram.csv'
        outfiles[element] =  open(file,'w')
        headers[element] = True

    for fname in filenames:
        dt = minhash_util.getDataType(fname)
        outfile = outfiles[dt]

        file = profile_dir + fname
        with open(file) as infile:
            header = infile.readline()
            data = infile.readline()
            if headers[dt]:
                headers[dt] = False
                outfile.write(header)
            else:
#                 print(data)
                outfile.write(data)

    print("Done profile")
    profile_dir = 'data/profile/'
    filenames = check(profile_dir)

    profiles = []
    for fname in filenames:
        file = profile_dir + fname
#        p = minhash_util.readProfile2Dict(file)
        pp = minhash_util.readFullProfile2Dict(file)
        for p in pp:
            profiles.append(p)

    from util import config

    #####debug
    count_debug = 0

    result = []
    result_enchaced = []

    for p1 in profiles:
        dt1 = minhash_util.getDataType(p1['filename'])
        hs1 = config.getHeaders(dt1)
        data_length1 = p1['filename'].split('_')[3].split('.')[0]
#         print(p1['filename'])

        for p2 in profiles:
            dt2 = minhash_util.getDataType(p2['filename'])
            if dt1 != dt2:
                hs2 = config.getHeaders(dt2)
                data_length2 = p2['filename'].split('_')[3].split('.')[0]
                #gabarito
                gabarito = minhash_util.getGoldStandard(dt1,dt2)
                espelho_gabarito = set()

                for h1 in hs1[1:]:
                    for h2 in hs2[1:]:
                        u1 = float(p1['min_num_bigram_'+str(h1)])
                        u2 = float(p1['max_num_bigram_'+str(h1)])
#                         unique_vals_column_
                        w1 = float(p2['min_num_bigram_'+str(h2)])
                        w2 = float(p2['max_num_bigram_'+str(h2)])
                        #     min_num_1_
                        #     max_num_1_
                        u = (u1,u2)
                        w = (w1,w2)
                        u_inter_w = minhash_util.intercetion(u, w)

                        d = (minhash_util.vector_lenght(u) + minhash_util.vector_lenght(w))
                        if u_inter_w == 0:
                            sim_dl = 0
                        else:
                            if d == 0:
                                #alterado 1/
                                sim_dl = u_inter_w
                            else:
                                sim_dl = (2*u_inter_w)/d

                        if ( minhash_util.vector_lenght(u) == 0 ) or (minhash_util.vector_lenght(w) == 0):
                            if (minhash_util.vector_lenght(u) == minhash_util.vector_lenght(w) ):
                                cos_sim = 1
                            else:
                                cos_sim = u_inter_w / sqrt(minhash_util.vector_lenght(u) + minhash_util.vector_lenght(w))
                        else:
                            cos_sim = u_inter_w / sqrt(minhash_util.vector_lenght(u) * minhash_util.vector_lenght(w))

                        classificacao,inv_boolean = corrigeLinha(h1,dt1,h2,dt2, gabarito)
                        if sim_dl > 0:
                            #debug

                            if classificacao == 'TM':
                                espelho_gabarito.add( (h1,h2) )
                            rl = [p1['filename'],data_length1,dt1,h1,p2['filename'],data_length2,dt2,h2,sim_dl,classificacao]
                            result.append(rl)

                            ####debug
                            ######## enhaced
                            max_sim = max([sim_dl,cos_sim])
                            min_sim = min([sim_dl,cos_sim])

                            if (1 - min_sim/max_sim < 0.6):
                                count_debug += 1
                                erl = [p1['filename'],data_length1,dt1,h1,p2['filename'],data_length2,dt2,h2,((sim_dl+cos_sim)/2),classificacao]
                                result_enchaced.append(erl)
#                                 print("-----------")
#                                 print(h1,u)
#                                 print(h2,w)
#                                 print("sim_dl : " , sim_dl)
#                                 print("sim_cos : ", cos_sim)

                    #adiciona o True Non Match
                    tnm_list = buscaTrueNonMatch(p1['filename'],data_length1,dt1, p2['filename'],data_length2,dt2,
                                                 espelho_gabarito, gabarito , inv_boolean)

                #  adiciona os tnm ao resultado
                for tnm in tnm_list:
                    result.append(tnm)
                    result_enchaced.append(tnm)

    hs = ['file_1','data_length1','data_type_1','attribut_1','file_2','data_length2','data_type_2','attribut_2','sim_dl','classificacao']
    ofile = profile_dir + 'analytic.txt'
    config.write(ofile, hs, result)

    eofile = profile_dir + 'enchaced-analytic.txt'
    config.write(eofile, hs, result_enchaced)
    print("Done", count_debug)
#     print(result)


# -*- coding: utf-8 -*-
from crypt import methods
from os import access
from posixpath import split
from unicodedata import category
import requests
import time
import json
import numpy as np
import json
import ast
import os
import fnmatch
import sys
from flask_cors import *
from flask import Flask,render_template,request,Response,redirect,url_for
app = Flask(__name__)
CORS(app, supports_credentials=True)

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/blk_trace')
def blk_trace():
    return render_template('blk_trace.html')

@app.route('/dio_simple')
def dio_simple():
    return render_template('dio_simple.html')

@app.route('/dio_info')
def dio_info():
    return render_template('dio_info.html')

@app.route("/io_per_pmc")
def io_per_pmc():
    f = open(path+"/sum_io_pmc.json")
    data = json.load(f)
    name = "io_per_pmc"
    seq = data["seq_data"]
    f.close()
    return render_template('avg_with_mark.html',graph_name = "io_per_pmc",data=" ".join(str(i) for i in seq))

@app.route("/blk_merge_rate")
def blk_merge_rate():
    f = open(path+"/merge_rate.json")
    data = json.load(f)
    name = "blk_merge_rate"
    seq = data["rate"]
    f.close()
    return render_template('avg_with_mark.html',graph_name = "blk_merge_rate",data=" ".join(str(i) for i in seq))

@app.route("/blk_heatmap/<length>")
def blk_heatmap(length):
    drive_json = []
    for f_name in os.listdir(path):
        if(fnmatch.fnmatch(f_name,"*_heatmap.json")):
            print(f_name)
            drive_json.append(f_name.rstrip("_heatmap.json"))
    print(drive_json)
    return render_template('drive_heatmap_list.html',drives = drive_json,length=length)

@app.route("/query_avg/<stage>",methods=['GET'])
def query_avg(stage):
    f = open(path+"/"+stage+'_avg.json')
    data = json.load(f)
    name = data["name"]
    avgs = data["avgs"]
    f.close()
    return render_template('avg_with_mark.html',graph_name = stage+'_avg',data=" ".join(str(i) for i in avgs))

@app.route("/query_dis/<stage>",methods=['GET'])
def query_dis(stage):
    f = open(path+"/"+stage+'_dis.json')
    data = json.load(f)
    name = data["name"]
    dis = data["distribution"]
    f.close()
    return render_template('dis_with_mark.html',title = data["name"], mode = stage+"_distribution",bucket = " ".join(str(i) for i in dis))

@app.route('/dio_throughput')
def dio_throughput():
    f = open(path+'/dio_throughput.json')
    data = json.load(f)
    f.close()
    return render_template('pmc_aq.html',graph_name = data["name"],data = data["seq_data"],inv = 100)

@app.route('/dio_time_stramp')
def dio_time_stramp():
    f = open(path+'/dio_record_time.json')
    data = json.load(f)
    x_data = data["x_data"]
    y_data = data["y_data"]
    x_str = ','.join(str(x) for x in x_data)
    y_str = ','.join(str(y) for y in y_data)
    f.close()
    return render_template('record_time.html',title = data["name"], mode = "ext4_simple_dio",x_data = x_str,y_data = y_str)

@app.route('/dio_time_stramp_bucket/<acc>',methods=['GET'])
def dio_time_stramp_bucket(acc):
    f = open(path+'/dio_record_time.json')
    data = json.load(f)
    x_data = data["x_data"]
    sec_acc = float(acc)*1e9
    last_tp = int((x_data[-1])/sec_acc)+10
    bucket = [0 for i in range(last_tp)]
    for tp in x_data:
        index = int((tp)/sec_acc)
        bucket[index] += 1
    f.close()
    bucket_str =  ','.join(str(x) for x in bucket)
    return render_template('record_bucket.html',title = data["name"], mode = "ext4_info_dio",bucket = bucket_str)

@app.route('/ext4_simple_dio_latency/<split_page>')
def ext4_simple_dio_latency(split_page):
    type = ["kernel_crossing","file_system","block_io","common","file"]
    f = open(path+"/dio_event_latency.json")
    data = json.load(f)
    type_data = [data[key] for key in type]
    type_data_str = []
    for data in type_data:
        type_data_str.append(','.join(str(x) for x in data))
    all_str = ' '.join(type_data_str)
    return render_template('latency.html',data = all_str,type = ','.join(type),split_count = split_page)

@app.route('/ext4_info_dio_latency')
def ext4_info_dio_latency():
    type = ["kernel_crossing","file_system","block_io","common","file"]
    f = open(path+"/ext4_dio_info_event_latency.json")
    data = json.load(f)
    type_data = [data[key] for key in type]
    type_data_str = []
    for data in type_data:
        type_data_str.append(','.join(str(x) for x in data))
    all_str = ' '.join(type_data_str)
    return render_template('latency_info.html',data = all_str,type = ','.join(type))

# json format
# [common_name+op_tag,count]
io_operation = ["direct_read","direct_write","fsync"]
@app.route('/file_event_count')
def file_event_count():
    f = open(path+"/file_dio_op_counter.json")
    data = json.load(f)
    event_str = data["seq_data"].replace("[", "").replace("]", "")
    file_list = event_str.split(",")
    data_map = {}
    for i in range(0,len(file_list),2):
        file_name,op = file_list[i].split("+")
        count = int(file_list[i+1])
        value = []
        if op == "dr":
            value = [count,0,0]
        elif op == "dw":
            value = [0,count,0]
        elif op == "fy":
            value = [0,0,count]
        else:
            value = [0,0,0]
        if file_name in data_map:
            for i in range(len(value)):
                data_map[file_name][i] += value[i]
        else:
            data_map[file_name] = value
    category = []
    values = [[],[],[]]
    for item in data_map.items():
        category.append(item[0])
        values[0].append(item[1][0])
        values[1].append(item[1][1])
        values[2].append(item[1][2])
    value_str = [",".join(str(i) for i in value) for value in values]
    print(category)
    return render_template('func_event.html',event_name = " ".join(io_operation),event_count = " ".join(value_str),type = " ".join(category))

@app.route('/process_event_count')
def process_event_count():
    f = open(path+"/process_dio_op_counter.json")
    data = json.load(f)
    event_str = data["seq_data"].replace("[", "").replace("]", "")
    file_list = event_str.split(",")
    data_map = {}
    for i in range(0,len(file_list),2):
        file_name,op = file_list[i].split("+")
        count = int(file_list[i+1])
        value = []
        if op == "dr":
            value = [count,0,0]
        elif op == "dw":
            value = [0,count,0]
        elif op == "fy":
            value = [0,0,count]
        else:
            value = [0,0,0]
        if file_name in data_map:
            for i in range(len(value)):
                data_map[file_name][i] += value[i]
        else:
            data_map[file_name] = value
    category = []
    values = [[],[],[]]
    for item in data_map.items():
        category.append(item[0])
        values[0].append(item[1][0])
        values[1].append(item[1][1])
        values[2].append(item[1][2])
    value_str = [",".join(str(i) for i in value) for value in values]
    print(value_str)
    return render_template('func_event.html',event_name = " ".join(io_operation),event_count = " ".join(value_str),type = " ".join(category))

@app.route("/query_flow/<index>",methods=['GET'])
def query_flow(index):
    type = ["block_sched","nvme_execute","nvme_verify","scsi_execute","scsi_verify","account_io"]
    f = open(path+"/rq_result/"+str(index)+".json")
    data = json.load(f)
    rq_indexs = []
    latencys = [[] for i in type]
    for key in data.keys():
        if key != "dio_index":
            for i in range(len(type)):
                latencys[i].append(data[key]["latency"][i])
            rq_indexs.append(key)
    data = []
    for latency in latencys:
        latency_str = ",".join(str(i) for i in latency)
        data.append(latency_str)
    index_str = ",".join(rq_indexs)
    all_str = " ".join(data)
    print(all_str)
    return render_template('rq_latency.html',data = all_str,type = ','.join(type))

@app.route("/heatmap/<drive>/<length>",methods=['GET'])
def io_heatmap(drive,length):
    # lsblk
    f = open(path+"/"+drive+"_heatmap.json")
    data = json.load(f)
    partition = int(data["partition"])
    height = int(partition/int(length))+1
    # (length,height)
    seq = data["seq_data"]
    return render_template('heatmap.html',height = height,length = length,seq_data = seq,drive = data["name"])
#
if __name__ == "__main__":
    path = sys.argv[1].lstrip("/")
    ip = sys.argv[2]
    port = sys.argv[3]
    app.run(host=ip, port=port,debug=True)


class User():
    def __init__(self,name,age,gender):
        self.name= name
        self.age = age
        self.gender = gender

    def show_details(self):
        print(f"Informations\n\nName = {self.name} \nAge = {self.age} \nGender = {self.gender}")

vini = User("Vini Muchulski",19,"Male")

vini.show_details()

#child class

class Bank(User):
    def __init__(self, name, age, gender):
        super().__init__(name, age, gender)
        self.balance = 0

    def deposit(self, valor):
        self.balance = self.balance + valor
        print("Balance has been updated: $ ", self.balance)

    def withdraw(self, valor):
        if self.balance - valor <0:
            print(f"Insufficient Funds! Balance = $ {self.balance}")

        else:
            self.balance =self.balance -valor
            print("Balance has been updated: $ ", self.balance)

    def view_balance(self):
        self.show_details()
        print("Balance: $ ", self.balance)

apeiron = Bank("Apeiron",19,"RocketMan")
apeiron.show_details()
apeiron.deposit(500)
apeiron.withdraw(1000)
apeiron.withdraw(100)
apeiron.view_balance()