from datasets import sythtextprovider
import tensorflow as tf
import numpy as np
import cv2
import os
import time
from nets import txtbox_384
from processing import ssd_vgg_preprocessing

os.environ['CUDA_VISIBLE_DEVICES'] = '0'
slim = tf.contrib.slim

tf.logging.set_verbosity(tf.logging.INFO)

show_pic_sum = 10
save_dir = 'pic_test_dataset'
tf.app.flags.DEFINE_string(
    'dataset_dir', 'tfrecord_train', 'The directory where the dataset files are stored.')

tf.app.flags.DEFINE_integer(
    'num_readers', 2,
    'The number of parallel readers that read data from the dataset.')

tf.app.flags.DEFINE_integer(
    'batch_size', 2, 'The number of samples in each batch.')

FLAGS = tf.app.flags.FLAGS
if not os.path.exists(save_dir):
    os.makedirs(save_dir)

def draw_polygon(img,x1,y1,x2,y2,x3,y3,x4,y4, color=(255, 0, 0)):
    # print(x1, x2, x3, x4, y1, y2, y3, y4)
    x1 = int(x1)
    x2 = int(x2)
    x3 = int(x3)
    x4 = int(x4)

    y1 = int(y1)
    y2 = int(y2)
    y3 = int(y3)
    y4 = int(y4)
    cv2.line(img,(x1,y1),(x2,y2),color,2)
    cv2.line(img,(x2,y2),(x3,y3),color,2)
    cv2.line(img,(x3,y3),(x4,y4),color,2)
    cv2.line(img,(x4,y4),(x1,y1),color,2)
    # cv2_im = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    # cv2_im = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    # cv2.imwrite('test.png', img)
    return img

def run():
    if not FLAGS.dataset_dir:
        raise ValueError('You must supply the dataset directory with --dataset_dir')

    print('-----start test-------')
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    with tf.device('/GPU:0'):
        dataset = sythtextprovider.get_datasets(FLAGS.dataset_dir)
        print(dataset)
        provider = slim.dataset_data_provider.DatasetDataProvider(
                        dataset,
                        num_readers=FLAGS.num_readers,
                        common_queue_capacity=20 * FLAGS.batch_size,
                        common_queue_min=10 * FLAGS.batch_size,
                        shuffle=True)
        print('provider:',provider)
        [image, shape, glabels, gbboxes, x1, x2, x3, x4, y1, y2, y3, y4] = provider.get(['image', 'shape',
                                                                 'object/label',
                                                                 'object/bbox',
                                                                 'object/oriented_bbox/x1',
                                                                 'object/oriented_bbox/x2',
                                                                 'object/oriented_bbox/x3',
                                                                 'object/oriented_bbox/x4',
                                                                 'object/oriented_bbox/y1',
                                                                 'object/oriented_bbox/y2',
                                                                 'object/oriented_bbox/y3',
                                                                 'object/oriented_bbox/y4'
                                                                 ])
        print('image:',image)
        print('shape:',shape)
        print('glabel:',glabels)
        print('gboxes:',gbboxes)

        gxs = tf.transpose(tf.stack([x1,x2,x3,x4])) #shape = (N,4)
        gys = tf.transpose(tf.stack([y1, y2, y3, y4]))

        image = tf.identity(image, 'input_image')
        text_shape = (384, 384)
        image, glabels, gbboxes, gxs, gys= ssd_vgg_preprocessing.preprocess_image(image,  glabels,gbboxes,gxs, gys,
                                                            text_shape,is_training=True,
                                                            data_format='NHWC')

        x1, x2 , x3, x4 = tf.unstack(gxs, axis=1)
        y1, y2, y3, y4 = tf.unstack(gys, axis=1)

        text_net = txtbox_384.TextboxNet()
        text_anchors = text_net.anchors(text_shape)
        e_localisations, e_scores, e_labels = text_net.bboxes_encode( glabels, gbboxes, text_anchors, gxs, gys)

    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7)

    config = tf.ConfigProto(log_device_placement=False, gpu_options=gpu_options, allow_soft_placement=True)
    with tf.Session(config=config) as sess:
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess, coord)
        j = 0
        all_time = 0
        try:
            while not coord.should_stop() and j < show_pic_sum:
                start_time = time.time()
                image_sess, label_sess, gbbox_sess,  x1_sess, x2_sess, x3_sess, x4_sess, y1_sess, y2_sess, y3_sess, y4_sess,p_localisations, p_scores, p_labels = sess.run([
                    image, glabels, gbboxes, x1, x2, x3, x4, y1, y2, y3, y4,e_localisations , e_scores, e_labels])
                end_time = time.time() - start_time
                all_time += end_time
                image_np = image_sess
                # print(image_np)
                # print('label_sess:',label_sess)

                p_labels_concat = np.concatenate(p_labels)
                p_scores_concat = np.concatenate(p_scores)
                debug = False
                if debug is True:
                    print(p_labels)
                    print('l_labels:',  len(p_labels_concat[p_labels_concat.nonzero()]),p_labels_concat[p_labels_concat.nonzero()] )
                    print('p_socres:', len(p_scores_concat[p_scores_concat.nonzero()]), p_scores_concat[p_scores_concat.nonzero()])
                    # print(img_np.shape)

                    print('label_sess:', np.array(list(label_sess)).shape, list(label_sess))
                img_np = np.array(image_np)
                cv2.imwrite('{}/{}.png'.format(save_dir, j), img_np)
                img_np = cv2.imread('{}/{}.png'.format(save_dir, j))

                h, w, d = img_np.shape

                label_sess = list(label_sess)
                # for i , label in enumerate(label_sess):
                i = 0
                num_correct = 0

                for label in label_sess:
                    # print(int(label) == 1)
                    if int(label) == 1:
                        num_correct += 1
                        img_np = draw_polygon(img_np,x1_sess[i] * w, y1_sess[i]*h, x2_sess[i]*w, y2_sess[i]*h, x3_sess[i]*w, y3_sess[i]*h, x4_sess[i]*w, y4_sess[i]*h)
                    if int(label) == 0:
                        img_np = draw_polygon(img_np,x1_sess[i] * w, y1_sess[i]*h, x2_sess[i]*w, y2_sess[i]*h, x3_sess[i]*w, y3_sess[i]*h, x4_sess[i]*w, y4_sess[i]*h, color=(0, 0, 255))
                    i += 1
                img_np = cv2.cvtColor(img_np, cv2.COLOR_BGR2RGB)
                cv2.imwrite('{}'.format(os.path.join(save_dir, str(j)+'.png')), img_np)
                j+= 1
                print('correct:', num_correct)
        except tf.errors.OutOfRangeError:
            print('done')
        finally:
            print('done')
            coord.request_stop()
        print('all time:', all_time, 'average:', all_time / show_pic_sum)
        coord.join(threads=threads)

if __name__ == '__main__':
    run()


a=int(input('introduceti primul numar'))
b=int(input('introduceti al doilea numar'))
if(a%2==0)and(b%2==0):
    if(a>b):
        print(a)
    elif(a==b):
        print(a,' ',b)
    else:
        print(b)
elif (a%2==0)and(b%2!=0):
    print(a)
elif(a%2!=0)and(b%2==0):
    print(b)
else:
    print('nu sunt numere pare')


from tensorboard.plugins.hparams import api as hp

HP_TOKEN_TYPE = hp.HParam('token_type', hp.Discrete(['word-piece', 'character']))
HP_VOCAB_SIZE = hp.HParam('vocab_size', hp.Discrete([2**12]))

# Preprocessing Hparams
HP_MEL_BINS = hp.HParam('mel_bins', hp.Discrete([80]))
HP_FRAME_LENGTH = hp.HParam('frame_length', hp.Discrete([0.025]))
HP_FRAME_STEP = hp.HParam('frame_step', hp.Discrete([0.01]))
HP_HERTZ_LOW = hp.HParam('hertz_low', hp.Discrete([125.0]))
HP_HERTZ_HIGH = hp.HParam('hertz_high', hp.Discrete([7600.0]))
HP_DOWNSAMPLE_FACTOR = hp.HParam('downsample_factor', hp.Discrete([3]))

# Model Hparams
HP_EMBEDDING_SIZE = hp.HParam('embedding_size', hp.Discrete([500]))
HP_ENCODER_LAYERS = hp.HParam('encoder_layers', hp.Discrete([8]))
HP_ENCODER_SIZE = hp.HParam('encoder_size', hp.Discrete([2048]))
HP_PROJECTION_SIZE = hp.HParam('projection_size', hp.Discrete([640]))
HP_TIME_REDUCT_INDEX = hp.HParam('time_reduction_index', hp.Discrete([1]))
HP_TIME_REDUCT_FACTOR = hp.HParam('time_reduction_factor', hp.Discrete([2]))
HP_PRED_NET_LAYERS = hp.HParam('pred_net_layers', hp.Discrete([2]))
HP_PRED_NET_SIZE = hp.HParam('pred_net_size', hp.Discrete([2048]))
HP_JOINT_NET_SIZE = hp.HParam('joint_net_size', hp.Discrete([640]))
HP_DROPOUT = hp.HParam('dropout', hp.Discrete([0]))

# HP_EMBEDDING_SIZE = hp.HParam('embedding_size', hp.Discrete([32]))
# HP_ENCODER_LAYERS = hp.HParam('encoder_layers', hp.Discrete([4]))
# HP_ENCODER_SIZE = hp.HParam('encoder_size', hp.Discrete([20]))
# HP_PROJECTION_SIZE = hp.HParam('projection_size', hp.Discrete([50]))
# HP_TIME_REDUCT_INDEX = hp.HParam('time_reduction_index', hp.Discrete([1]))
# HP_TIME_REDUCT_FACTOR = hp.HParam('time_reduction_factor', hp.Discrete([2]))
# HP_PRED_NET_LAYERS = hp.HParam('pred_net_layers', hp.Discrete([2]))
# HP_PRED_NET_SIZE = hp.HParam('pred_net_size', hp.Discrete([100]))
# HP_JOINT_NET_SIZE = hp.HParam('joint_net_size', hp.Discrete([50]))
# HP_DROPOUT = hp.HParam('dropout', hp.Discrete([0.2]))

HP_LEARNING_RATE = hp.HParam('learning_rate', hp.Discrete([1e-4]))

METRIC_TRAIN_LOSS = 'train_loss'
METRIC_TRAIN_ACCURACY = 'train_accuracy'
METRIC_EVAL_LOSS = 'eval_loss'
METRIC_EVAL_ACCURACY = 'eval_accuracy'
METRIC_EVAL_CER = 'eval_cer'
METRIC_EVAL_WER = 'eval_wer'
METRIC_ACCURACY = 'accuracy'
METRIC_CER = 'cer'
METRIC_WER = 'wer'


import pickle
import sys
from collections import deque

sys.setrecursionlimit(1000000)

class Graph(object):

    def __init__(self, size):
        self.nodes = {}
        self.size = size

    def __repr__(self):
        return '<Graph with {} nodes>'.format(self.size)

    def get_node(self, id):
        node = self.nodes.get(id)
        if node is None:
            node = Node(id)
            self.nodes[id] = node
        return node

    def add_data(self, u, v):
        N = self.size
        if u > 0:
            if v > 0:
                o = self.get_node(N+u)
                d = self.get_node(v)
                o.set_out(d.id)
                d.set_in(o.id)
                o = self.get_node(N+v)
                d = self.get_node(u)
                o.set_out(d.id)
                d.set_in(o.id)
            else:
                o = self.get_node(N+u)
                d = self.get_node(N-v)
                o.set_out(d.id)
                d.set_in(o.id)
                o = self.get_node(-v)
                d = self.get_node(u)
                o.set_out(d.id)
                d.set_in(o.id)
        else:
            if v > 0:
                o = self.get_node(-u)
                d = self.get_node(v)
                o.set_out(d.id)
                d.set_in(o.id)
                o = self.get_node(N+v)
                d = self.get_node(N-u)
                o.set_out(d.id)
                d.set_in(o.id)
            else:
                o = self.get_node(-u)
                d = self.get_node(N-v)
                o.set_out(d.id)
                d.set_in(o.id)
                o = self.get_node(-v)
                d = self.get_node(N-u)
                o.set_out(d.id)
                d.set_in(o.id)

    def calc_finish(self):
        print('First pass')
        ft = 0
        q = deque()
        first = list(sorted(self.nodes.values(), key=lambda x: x.id, reverse=True))
        for node in first:
            if node.bck:
                continue
            self.dfs(node, q, False)
            while q:
                ft += 1
                n = q.pop()
                n.set_finish(ft)

        print('Second pass')
        second = list(sorted(self.nodes.values(), key=lambda x: x.finish, reverse=True))
        for node in second:
            if node.fwd:
                continue
            q = deque()
            scc = {node.id}
            self.dfs(node, q, True, scc)

    def dfs(self, node, queue, fw=True, scc=None):
        queue.append(node)
        if fw:
            node.fwd = True
            for n in node.outgoing:
                n = self.get_node(n)
                if n.fwd:
                    continue
                if n.id > self.size and (n.id - self.size) in scc:
                    print(scc)
                    raise ValueError('unsatisfied')
                elif n.id < self.size and (n.id + self.size) in scc:
                    print(scc)
                    raise ValueError('unsatisfied')
                scc.add(n.id)
                self.dfs(n, queue, fw, scc)
        else:
            node.bck = True
            for n in node.incoming:
                n = self.get_node(n)
                if n.bck:
                    continue
                self.dfs(n, queue, fw)

class Node(object):

    def __init__(self, id):
        assert id > 0
        self.id = id
        self.finish = None
        self.incoming = set()
        self.outgoing = set()
        self.fwd = False
        self.bck = False

    def __repr__(self):
        return '<Node {}. in:{} out:{}>'.format(self.id, self.incoming, self.outgoing)

    def set_finish(self, value):
        self.finish = value

    def set_in(self, id):
        self.incoming.add(id)

    def set_out(self, id):
        self.outgoing.add(id)

def main():
    count = None
    with open(sys.argv[1], 'rt') as f:
        for r in f.read().split('\n'):
            if not r:
                continue
            if count is None:
                count = int(r)
                graph = Graph(count)
            else:
                graph.add_data(*tuple(map(int, r.split())))
    try:
        graph.calc_finish()
    except ValueError:
        print('unsatisfied')
    else:
        print('satisfied')

if __name__ == '__main__':
    main()


import FWCore.ParameterSet.Config as cms

import os

process = cms.PSet()

process.fwliteInput = cms.PSet(
    fileNames = cms.vstring(),
    maxEvents = cms.int32(-1),
    outputEvery = cms.uint32(100000)
)

process.fwliteOutput = cms.PSet(
    fileName = cms.string('comp_jetToTauFakeRate.root')
)

process.comp_jetToTauFakeRate = cms.PSet(

    looseRegion = cms.string(""),
    tightRegion = cms.string(""),

    processData = cms.string("data_obs"),
    processesToSubtract = cms.vstring(),

    processMC = cms.string("TTj"),

    hadTauSelections = cms.vstring(),
    trigMatchingOption = cms.string("woTriggerMatching"),

    absEtaBins = cms.vdouble(),
    ptBins = cms.vdouble(),

    isMC = cms.bool(False),

    histogramsToFit = cms.vstring("hadTaus/pt"),

    fitFunction = cms.string("[0] + [1]*x"),
    xMin = cms.double(0.),
    xMax = cms.double(200.),
    yScale = cms.string('linear'),
    initialParameters = cms.PSet(
        p0 = cms.double( 0.95),
        p1 = cms.double(-0.01),
    ),

    outputFileName = cms.string("")
)


#!/opt/anaconda3.7/bin/python3.7
import numpy as np, matplotlib.pyplot as plt, os, sys
import regex as re, ast
import argparse
from setup_functions import *
from subroutines.parse_input import *
from subroutines.scatter import *
from subroutines.bar import *
from subroutines.dev import *

if len(sys.argv) == 1:
    print(f"\n\n\tERROR: This script must be called with an additional argument, ie: "
          f"\n\n\t\t\t plotpro input.txt (--suppress)\n\n")
    sys.exit()

filenames = sys.argv[1:]

# Set up the ArgumentParser to interpret the command line arguments
parser = argparse.ArgumentParser()
parser.add_argument("file", help="Name(s) of the file(s) to plot. If more than one input, all files will be processed.", nargs='+')
parser.add_argument("-s", "--suppress", help="Suppress the display of figures. Default: False")
args = parser.parse_args()
boolsuppress = False    # Default: Don't suppress the showing of figures.
if args.suppress:
    if args.suppress.lower() in ("y", "j", "yes", "ja", "on", "true") or args.suppress == True:
        boolsuppress = True

    filenames = filenames[:-2]  # Remove the "--suppress" and "arg" from the list of input arguments

(save_list, keys_list, keys, keys_default, cosmetics_list, cosmstr, cosmval, cosmcol, cosmetic_defaults,
 hatch_list) = definitions()

# print_string = "\n\nRunning PlotPro"
print_string = initiate_output()
sep2 = "-"*35 + "="*50 + "-"*35 + "\n"*2
sep = "\n"*2 + sep2 + "\n"*2

for filename in filenames:
    if os.path.isfile(filename):
        print("Working on file: "+filename)
    else:
        print(filename+" is not a filename.")
        continue

    output_name = f"{os.path.splitext(filename)[0]}.out"

    print_string += f"\n\n{'Input file':30} {filename:25}\n{'Current working directory':30} {os.getcwd()}{sep}"

    # PPPPPPP     AAAAAA    RRRRRRR      SSSSSSS    EEEEEEEE
    # P      P   A      A   R      R    S           E
    # P      P  A        A  R      R    S           E
    # PPPPPPP   AAAAAAAAAA  RRRRRRR      SSSSS      EEEEE
    # P         A        A  R R                S    E
    # P         A        A  R   R               S   E
    # P         A        A  R    R              S   E
    # P         A        A  R     R            S    E
    # P         A        A  R      R    SSSSSSS     EEEEEEEE

    data_table, keywords, cosmetics, plot_name, print_string = read_input(
        filename,
        save_list,
        keys_list,
        keys,
        cosmetics_list,
        print_string
    )

    if "devcolour" not in cosmetics.keys():
        if "bar" in keywords["plot"]:
            if "legend" not in cosmetics.keys():
                cosmetics["legend"] = ""

            if len(cosmetics["legend"].split(";")) <= 4:
                cosmetics["devcolour"] = update_default_bar_colours(legends=cosmetics["legend"].split(";"))

        elif "dev" in keywords["plot"]:
            if "legend" not in cosmetics.keys():
                ref = ""
                for i in keywords["deviation"]:
                    if i not in ("scatter", "ref", "type"):
                        ref += i+";"

                cosmetics["legend"] = ref[:-1]  # Remove the trailing ";"

            if sum([i not in ("scatter", "ref", "type") for i in keywords["deviation"]]) <= 4:
                cosmetics["devcolour"] = update_default_bar_colours(legends=cosmetics["legend"].split(";"))

    """
    if "bar" in keywords["plot"]:
        if "devcolour" not in cosmetics.keys():
            if "legend" not in cosmetics.keys():
                cosmetics["legend"] = ""
                cosmetics["devcolour"] = update_default_bar_colours(legends=cosmetics["legend"].split(";"))

            elif len(cosmetics["legend"].split(";")) <= 4:
                cosmetics["devcolour"] = update_default_bar_colours(legends=cosmetics["legend"].split(";"))

    elif "dev" in keywords["plot"]:
        if "devcolour" not in cosmetics.keys():
            if "legend" not in cosmetics.keys():
                ref = ""
                for i in keywords["deviation"]:
                    if i not in ("scatter", "ref"):
                        ref += i+";"

                cosmetics["legend"] = ref[:-1]

            if sum([i != "scatter" and i != "ref" for i in keywords["deviation"]]) <= 4:
                cosmetics["devcolour"] = update_default_bar_colours(legends=cosmetics["legend"].split(";"))
    """

    # Input default values

    cosmetics, keywords, gridbool, print_fit, figsize, legloc, print_string = add_defaults(
        cosmetics,
        cosmetic_defaults,
        cosmetics_list,
        keywords,
        keys_default,
        keys_list,
        hatch_list,
        print_string
    )

    ref = keywords["ref"]       # put the reference column in a variable for easy use in arrays

    # Sanitate data for bad reference values - only if fractional deviations are to be made

    if keywords["frac"]:
        data_backup = np.array(data_table)
        data_table, print_string = sanitate_data(
            data_table,
            keywords["ref"],
            output_name,
            print_string
        )
    else:
        data_table = np.array(data_table)

    # Print all parameters to output (that way, it is clear exactly which keywords exist, and which values have been used)
    print_string += "Used parameters".center(len(sep2))
    print_string += f"\n{sep2}".ljust(len(sep2))
    print_string += "routecard:\n"
    # for key in sorted(list(keywords.keys())):
    for key in sorted(keys_list):
        if type(keywords[key]) == type(keys_list):
            if len(keywords[key]) == 1:
                print_string += "\t{0:<20} {1}\n".format(key, keywords[key][0])
            elif len(keywords[key]) > 1:
                print_string += "\t{0:<20} ".format(key)
                for i in np.arange(len(keywords[key])):
                    if i == len(keywords[key])-1:
                        print_string += f"{keywords[key][i]}\n"
                    else:
                        print_string += f"{keywords[key][i]}, "
            else:
                continue
        else:
            print_string += "\t{0:<20} {1}\n".format(key, keywords[key])

    print_string += "\n\ntitlecard:\n"
    # for key in cosmetics:
    for key in sorted(cosmetics_list):
        if (key == "devpattern") and cosmetics[key] == "on":
            print_string += "\t{0:<20} {1}\t\t\"".format(key, cosmetics[key])
            for i, hat in enumerate(cosmetics[key]):
                if not i == len(cosmetics[key]) - 1:
                    print_string += "{0};".format(hat)
                else:
                    print_string += "{0}\"\n".format(hat)
        else:
            print_string += "\t{0:<20} {1}\n".format(key, cosmetics[key])
    print_string += sep

    data_table = rearrange_table(data_table, ref)

    # PPPPPPP   L           OOOO    TTTTTTTT
    # P      P  L         OO    OO     TT
    # P      P  L         O      O     TT
    # PPPPPPP   L         O      O     TT
    # P         L         O      O     TT
    # P         L         O      O     TT
    # P         L         O      O     TT
    # P         L         OO    OO     TT
    # P         LLLLLLLL    OOOO       TT

    if "plot" in keywords.keys():
        spac, dec = cosmetics["outdec"] + 5, cosmetics["outdec"]
        format_dict = {"spac": spac, "dec": dec, "sep2": sep2, "figsize": figsize, "legloc": legloc, "print_fit": print_fit}
        bool_dict = {"gridbool": gridbool, "diagonal": cosmetics["diagonal"], "show_legend": cosmetics["show_legend"]}

        if "scatter" in keywords["plot"]:
            print_string += "Scatter plot".center(len(format_dict["sep2"]))
            print_string += f"\n{format_dict['sep2']}".ljust(len(format_dict["sep2"]))
            fig, ax, print_string = plot_scatter(
                data_table,
                keywords,
                cosmetics,
                print_string,
                format_dict,
                bool_dict,
                plot_name
            )
            fig.patch.set_facecolor(cosmetics["background"])
            ax.set_facecolor(cosmetics["background"])

        if "dev" in keywords["plot"]:
            print_string += "Deviations".center(len(sep2))
            print_string += f"\n{sep2}".ljust(len(sep2))
            fig, ax, print_string = plot_dev(
                data_table,
                keywords,
                cosmetics,
                print_string,
                format_dict,
                bool_dict,
                plot_name,
                output_name
            )
            fig.patch.set_facecolor(cosmetics["background"])
            for a in list(ax):
                a.set_facecolor(cosmetics["background"])

        if "bar" in keywords["plot"]:
            print_string += "Pre-calculated bar plots".center(len(sep2))
            print_string += f"\n{sep2}".ljust(len(sep2))
            print_string += "This bar plot is made directly as a bar plot of the input data.\n" \
                            "The user is therefore responsible for the quality of the calculations.\n\n"
            fig, ax, print_string = plot_bars(
                data_table,
                keywords,
                cosmetics,
                print_string,
                format_dict,
                bool_dict,
                plot_name
            )
            fig.patch.set_facecolor(cosmetics["background"])
            ax.set_facecolor(cosmetics["background"])

    # for name in plot_name:
        # name, ext = name.split(sep='.')
        # plt.savefig(f"{name}.{ext}", dpi=cosmetics["dpi"])
    #     plt.savefig(name, dpi=cosmetics["dpi"])

    print_string += sep
    print_string += "Printing content of input file:\n\n"

    with open(filename, 'r') as inFile:
        for line in inFile:
            print_string += f"{line}"

    print_string += sep

    print_string += "\n\n\nExiting PlotPro. Have a nice day :)"

    with open(output_name, 'w') as outFile:
        outFile.write(print_string)

if boolsuppress:
    print("Output figures have been suppressed by the '--suppress' argument")
else:
    plt.show()


# -*- coding: utf-8 -*-
from .base import AwsBase

class Route53Service(AwsBase):
    '''
    Class belonging to the Route 53 DNS Service
    '''

    def get_domains(self):
        '''
        Get hosted zones and its records

        Returns:
            (lst): List of Hosted Zones with Records
        '''
        domains = []
        for hz in self.list_hosted_zones():
            hz['Records'] = self.list_records(hz['Id'])
            domains.append(hz)
        return domains

    def list_hosted_zones(self):
        '''
        List all hosted zones

        Returns:
            List of hosted zones
        '''
        return self.inject_client_vars(self.client.list_hosted_zones()['HostedZones'])

    def list_records(self, hosted_zone_id):
        '''
        List all records for a hosted zone

        Args:
            hosted zone (str): The ID of the hosted zone that contains the resource record sets that you want to list

        Returns:
            List of DNS records
        '''
        records = []

        config = self.get_client_vars()
        paginator = self.client.get_paginator('list_resource_record_sets')
        for response in paginator.paginate(HostedZoneId=hosted_zone_id):
            for x in response['ResourceRecordSets']: records.append(x)

        return self.inject_client_vars(records, config)

    def list_records_by_domain(self, domain):
        '''
        List all records of a hosted-zone domain

        Args:
            domain (str): The DOMAIN name of the hosted zone that contains the resource record sets that you want to list

        Returns:
            List of DNS records
        '''
        hosted_zone = filter(lambda x: domain  + "." == x['Name'], self.list_hosted_zones())
        if hosted_zone:
            return self.list_records(hosted_zone[0]['Id'])
        return None

    def __init__(self):
        AwsBase.__init__(self, 'route53')
        self.change_region('us-east-1')


"""
File contains all tests related to the '/encode' endpoint API.
"""

import secrets

def test_get_shorten_url(test_api):
    """
    Test to make GET request to the '/encode'
    endpoint when only POST requests are
    allowed.
    Args:
        test_api: Client for making requests.
    """

    response = test_api.get("/encode")

    # Should not be able to make a GET request.
    assert response.status_code == 405
    assert response.headers["Content-Type"] == "application/json"
    assert response.json() == {"detail": "Method Not Allowed"}

def test_bad_url(test_api):
    """
    Test to ensure request to the '/encode'
    endpoint sending bad urls are handled
    appropriately.
    Args:
        test_api: Client for making requests.
    """

    request_payload = {"url": "kajsdnkfaasd"}

    response = test_api.post(url="/encode", json=request_payload)

    assert response.status_code == 400
    assert response.headers["Content-Type"] == "application/json"
    assert response.json() == {"detail": "Bad URL sent."}

def test_post_shorten_url(test_api, monkeypatch):
    """
    Test to make a valid POST request to the
    '/encode' API endpoint.
    Args:
        test_api: Client for making requests.
        monkeypatch: Monkey patch fixture of pytest.
    """

    request_payload = {"url": "https://www.google.com"}

    # mock the `token_urlsafe` method of secrets module
    # to return the token we want.
    def mock_token_urlsafe(nbytes):
        return "jflew9"

    # hijack the method with out mocked method
    monkeypatch.setattr(secrets, "token_urlsafe", mock_token_urlsafe)

    response = test_api.post(url="/encode", json=request_payload)

    assert response.status_code == 201
    assert response.headers["Content-Type"] == "application/json"
    assert "shortened_url" in response.json()
    assert "http://127.0.0.1:8000/" in response.json()["shortened_url"]
    assert response.json() == {"shortened_url": "http://127.0.0.1:8000/jflew9"}

def test_invalid_post_shorten_url(test_api):
    """
    Test to make an invalid POST request to the
    '/encode' API endpoint.
    Args:
        test_api: Client for making requests.
    """

    request_payload = {"something_else": "https://www.google.com"}
    response_payload = {
        "detail": [
            {
                "loc": ["body", "url"],
                "msg": "field required",
                "type": "value_error.missing",
            }
        ]
    }

    response = test_api.post(url="/encode", json=request_payload)

    assert response.status_code == 422
    assert response.headers["Content-Type"] == "application/json"
    assert response.json() == response_payload


from functools import reduce

class Solution:
    def subtractProductAndSum(self, n: int) -> int:

        n = list(map(int, str(n)))

        s = sum(n)
        m = 1

        for i in n:
            m *= i

        return m - s


import torch as tr
import torch.nn as nn
from models.attentions import MultiHeadAttention, PositionwiseFeedForward

class TransformerDecoderBlock(nn.Module):
    def __init__(
            self,
            d_model: int,
            n_heads: int,
            d_ff: int,
            dropout=0.1
    ) -> None:
        super().__init__()
        self.d_model = d_model
        self.attention = MultiHeadAttention(d_model, n_heads)
        self.masked_attention = MultiHeadAttention(d_model, n_heads)
        self.norm_1 = nn.LayerNorm(d_model)
        self.norm_2 = nn.LayerNorm(d_model)
        self.norm_3 = nn.LayerNorm(d_model)
        self.ff = PositionwiseFeedForward(d_model, d_ff)
        self.dropout_1 = nn.Dropout(dropout)
        self.dropout_2 = nn.Dropout(dropout)
        self.dropout_3 = nn.Dropout(dropout)

    def forward(
            self,
            enc_out: tr.Tensor,
            dec_out: tr.Tensor,
            enc_mask: tr.Tensor,
            dec_mask: tr.Tensor,
    ) -> tr.Tensor:
        """

        Args:
            enc_out (tr.Tensor): [batch_size, src_len, d_model]
            dec_out (tr.Tensor): [batch_size, tgt_len, d_model]
                the `dec_out` is the token embedding during the training stage
                (i.e. embedding of `<bos> hello world <eos>`), and the `dec_out`
                is the output of the decoder during the inference stage.
            enc_mask (tr.Tensor, optional): [batch_size, 1, src_len].
            dec_mask (tr.Tensor, optional): [batch_size, tgt_len, tgt_len].
            cache (tr.Tensor, optional): [batch_size, tgt_len-1, d_model]. Defaults to None.

        Returns:
            tr.Tensor: [batch_size, tgt_len, d_model]
        """
        q = dec_out + self.dropout_1(
            self.masked_attention(dec_out, dec_out, dec_out, dec_mask))
        q = self.norm_1(q)
        x = q + self.dropout_2(
            self.attention(q, enc_out, enc_out, enc_mask))
        x = self.norm_2(x)
        x = x + self.dropout_3(self.ff(x))
        x = self.norm_3(x)
        return x

class TransformerDecoder(nn.Module):
    def __init__(
            self,
            d_model: int,
            n_heads: int,
            d_ff: int,
            n_layers: int,
            dropout=0.1
        ) -> None:
        super().__init__()
        self.d_model = d_model
        self.n_heads = n_heads
        self.d_ff = d_ff
        self.n_layers = n_layers
        self.dropout = dropout
        self.residual_blocks = nn.ModuleList([
            TransformerDecoderBlock(d_model, n_heads, d_ff, dropout) for _ in range(n_layers)
        ])

    def forward(
            self,
            enc_out: tr.Tensor,
            dec_out: tr.Tensor,
            src_mask: tr.Tensor,
            tgt_mask: tr.Tensor
        ):
        """
        Args:
            enc_out (tr.Tensor): [batch_size, src_len, d_model]
            dec_out (tr.Tensor): [batch_size, tgt_len, d_model]
                the `dec_out` is the token embedding during the training stage
                (i.e. embedding of `<bos> hello world <eos>`), and the `dec_out`
                is the output of the decoder during the inference stage.
            enc_mask (tr.Tensor, optional): [batch_size, 1, src_len].
            dec_mask (tr.Tensor, optional): [batch_size, tgt_len, tgt_len]
            caches (List, optional): [n_layers, batch_size, tgt_len-1, d_model]

        Returns:
            tr.Tensor: [batch_size, tgt_len, token_size]
        """
        for _, layer in enumerate(self.residual_blocks):
            dec_out = layer(enc_out, dec_out, src_mask, tgt_mask)
        return dec_out


from BotApiTelegram import TelegramBot, filters

bot_token = '' # Your Bot Token from @BotFather.

bot = TelegramBot('bot_db', bot_token=bot_token)

@bot.on_update(filters.command('start'))
def on_start(message):
    message.reply('Hello.')

bot.start_polling()


import numpy as np

from . import imagesim

import flare.plots.image

def idealised(self, field, detection_filter = None, detection_threshold = 10, size_error = None):

    # return idealised photometry based simply on the depth

    # --- define the "observed" flux error in each band. In this case it is simply the depth.
    flux_err = {f: np.ones(self.N)*(field.depths[f]) for f in field.filters}

    # --- define the "observed" flux in each band. In this case it is simply the intrinsic flux + gaussian noise.
    flux = {f: self.o[f'intrinsic/flux/{f}'] + field.depths[f] * np.random.randn(self.N) for f in field.filters}

    # ---
    if not detection_filter:
        detection_filter = field.filters[-1]

    sn = flux[detection_filter]/flux_err[detection_filter]
    self.o['observed/sn'] = sn

    detected = sn > detection_threshold
    self.o['observed/detected'] = detected

    for k, v in flux.items():
        self.o[f'observed/{k}/flux'] = v

    for k, v in flux_err.items():
        self.o[f'observed/{k}/flux_err'] = v

def idealisedimage(self, field, detection_filters = None, width_pixels = 51, include_psf = False, verbose = False):

    # --- create idealised image creator
    image_creator = {f:imagesim.Idealised(f, field, verbose) for f in field.filters}

    # --- now call image
    image(self, field, image_creator, detection_filters = detection_filters, width_pixels = width_pixels, include_psf = include_psf, verbose = verbose)

def realimage(self, field, detection_filters = None, width_pixels = 51, include_psf = False, verbose = False):

    # --- create real image creator
    image_creator = {f:imagesim.Real(f, field, verbose) for f in field.filters}

    # --- now call image
    image(self, field, image_creator, detection_filters = detection_filters, width_pixels = width_pixels, include_psf = include_psf, verbose = verbose)

def image(self, field, image_creator, detection_filters = None, width_pixels = 51, include_psf = False, verbose = False):

    # --- create output quantities

    self.o[f'observed/detected'] = np.zeros(self.N, dtype=bool)

    quantities = ['r_segment_arcsec','r_eff_kron_arcsec']

    for q in ['sn'] + quantities:
        self.o[f'observed/detection/{q}'] = np.zeros(self.N)

    for f in field.filters:
        for q in ['kron_flux','kron_fluxerr','segment_flux','segment_fluxerr']:
            self.o[f'observed/{f}/{q}'] = np.zeros(self.N)

    # --- create PSF
    if include_psf:
        PSFs = imagesim.create_PSFs(field, width_pixels, final_filter = True)
    else:
        PSFs = None

    for i in range(self.N):

        if verbose: print('-'*10, i)

        ob = self.i(i=i)

        # --- create image object
        imgs = imagesim.create_image(image_creator, field, ob, width_pixels = 51, PSFs = PSFs)

        # --- create detection image
        detection_image = imagesim.create_detection_image(imgs, detection_filters)

        if verbose: flare.plots.image.make_significance_plot(detection_image)

        # --- detect sources
        detected, detection_cat, segm_deblended = imagesim.detect_sources(detection_image)

        if detected:

            if verbose: flare.plots.image.make_segm_plot(segm_deblended)

            x, y = detection_cat.xcentroid, detection_cat.ycentroid

            # --- determine distance from the centre of the image
            r = np.sqrt((x-(width_pixels-1)/2)**2 + (y-(width_pixels-1)/2)**2)

            # --- determine closest object to the centre of the image
            j = np.where(r==np.min(r))[0]

            if verbose: print(r)

            # --- only count as detected if within 3 pixels of the centre. 3 is somewhat arbitrary here.
            if r[j]<3:

                self.o[f'observed/detected'][i] = True
                self.o[f'observed/detection/sn'][i] = detection_cat.kron_flux[j]/detection_cat.kron_fluxerr[j]

                self.o[f'observed/detection/r_segment_arcsec'][i] = detection_cat.equivalent_radius[j].value * field.pixel_scale
                self.o[f'observed/detection/r_eff_kron_arcsec'][i] = detection_cat.fluxfrac_radius(0.5)[j] * field.pixel_scale

                # --- photometer sources
                source_cat = imagesim.perform_photometry(detection_cat, segm_deblended, imgs)

                for f in field.filters:
                    self.o[f'observed/{f}/kron_flux'][i] = source_cat[f].kron_flux[j]/imgs[f].nJy_to_es
                    self.o[f'observed/{f}/kron_fluxerr'][i] = source_cat[f].kron_fluxerr[j]/imgs[f].nJy_to_es
                    self.o[f'observed/{f}/segment_flux'][i] = source_cat[f].segment_flux[j]/imgs[f].nJy_to_es
                    self.o[f'observed/{f}/segment_fluxerr'][i] = source_cat[f].segment_fluxerr[j]/imgs[f].nJy_to_es

                if self.verbose: print(i, np.int(self.o[f'intrinsic/flux/{field.filters[-1]}'][i]), np.int(self.o[f'observed/{field.filters[-1]}/kron_flux'][i]))

            else:
                detected = False # --- if objects, but none within 3 pixels of the centre then set to undetected

    print('number detected:', len(self.o[f'observed/detected'][self.o[f'observed/detected']==True]))

    # --- set kron photometry as default photometry
    for f in field.filters:
        self.o[f'observed/{f}/flux'] = self.o[f'observed/{f}/kron_flux']
        self.o[f'observed/{f}/flux_err'] = self.o[f'observed/{f}/kron_fluxerr']


"""
Main Functon that initializes the User Interface class to run the chess
Game on pygame
"""
import pygame
from board import ChessBoard
from userInterface import UserInterface
if __name__ == "__main__":
    pygame.init()
    surface = pygame.display.set_mode([600, 600], 0, 0)
    pygame.display.set_caption('pyChess')
    Board = ChessBoard()
    UI = UserInterface(surface, Board)
    UI.playGame()
    pygame.quit()


# -*- coding:utf-8 -*-
"""

"""
from functools import partial

import numpy as np
import pandas as pd
from dask import dataframe as dd, array as da

from hypernets.utils import logging
from ..data_cleaner import DataCleaner, _CleanerHelper

logger = logging.get_logger(__name__)

class DaskDataCleaner(DataCleaner):
    @staticmethod
    def get_helper(X, y):
        if isinstance(X, (dd.DataFrame, dd.Series, da.Array)):
            return _DaskCleanerHelper()
        else:
            return DataCleaner.get_helper(X, y)

class _DaskCleanerHelper(_CleanerHelper):
    @staticmethod
    def reduce_mem_usage(df, excludes=None):
        raise NotImplementedError('"reduce_mem_usage" is not supported for Dask DataFrame.')

    @staticmethod
    def _get_duplicated_columns(df):
        duplicates = df.reduction(chunk=lambda c: pd.DataFrame(c.T.duplicated()).T,
                                  aggregate=lambda a: np.all(a, axis=0)).compute()
        return duplicates

    @staticmethod
    def _detect_dtype(dtype, df):
        result = {}
        df = df.copy()
        for col in df.columns.to_list():
            try:
                df[col] = df[col].astype(dtype)
                result[col] = [True]  # as-able
            except:
                result[col] = [False]
        return pd.DataFrame(result)

    def _correct_object_dtype_as(self, X, df_meta):
        for dtype, columns in df_meta.items():
            columns = [c for c in columns if str(X[c].dtype) != dtype]
            if len(columns) == 0:
                continue

            correctable = X[columns].reduction(chunk=partial(self._detect_dtype, dtype),
                                               aggregate=lambda a: np.all(a, axis=0),
                                               meta={c: 'bool' for c in columns}).compute()
            correctable = [i for i, v in correctable.items() if v]
            # for col in correctable:
            #     X[col] = X[col].astype(dtype)
            if correctable:
                X[correctable] = X[correctable].astype(dtype)
            logger.info(f'Correct columns [{",".join(correctable)}] to {dtype}.')

        return X


__all__ = [
    "DoccanoTask",
    "DoccanoClientConfig",
    "DoccanoInputConverter",
    "DoccanoOutputConverter",
]

import dataclasses
import enum
import json
import logging
import tempfile
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union
from typing_extensions import Self
from zipfile import ZipFile

from medkit.core import Attribute, OperationDescription, ProvTracer
from medkit.core.id import generate_id, generate_deterministic_id
from medkit.core.text import Entity, Relation, Span, TextDocument, span_utils

from medkit.io._common import get_anns_by_type

logger = logging.getLogger(__name__)

class DoccanoTask(enum.Enum):
    """Supported doccano tasks. The task defines
    the type of document to convert.

    Attributes
    ----------
    TEXT_CLASSIFICATION
        Documents with a category
    RELATION_EXTRACTION
        Documents with entities and relations (including IDs)
    SEQUENCE_LABELING
        Documents with entities in tuples
    """

    TEXT_CLASSIFICATION = "text_classification"
    RELATION_EXTRACTION = "relation_extraction"
    SEQUENCE_LABELING = "sequence_labeling"

@dataclasses.dataclass
class DoccanoClientConfig:
    """A class representing the configuration in the doccano client.
    The default values are the default values used by doccano.

    Attributes
    ----------
    column_text:
        Name or key representing the text
    column_label:
        Name or key representing the label
    """

    column_text: str = "text"
    column_label: str = "label"

# FIXME: datamodels to factorize in _doccano_utils
@dataclasses.dataclass()
class _DoccanoEntity:
    id: int
    start_offset: int
    end_offset: int
    label: str

    def to_dict(self) -> Dict[str, Any]:
        entity_dict = dict(
            id=self.id,
            start_offset=self.start_offset,
            end_offset=self.end_offset,
            label=self.label,
        )
        return entity_dict

@dataclasses.dataclass()
class _DoccanoEntityTuple:
    start_offset: int
    end_offset: int
    label: str

    def to_tuple(self) -> Tuple[int, int, str]:
        return (self.start_offset, self.end_offset, self.label)

@dataclasses.dataclass()
class _DoccanoRelation:
    id: int
    from_id: int
    to_id: int
    type: str

    def to_dict(self) -> Dict[str, Any]:
        relation_dict = dict(
            id=self.id,
            from_id=self.from_id,
            to_id=self.to_id,
            type=self.type,
        )
        return relation_dict

@dataclasses.dataclass()
class _DoccanoDocRelationExtraction:
    text: str
    entities: List[_DoccanoEntity]
    relations: List[_DoccanoRelation]
    metadata: Dict[str, Any]

    @classmethod
    def from_dict(
        cls, doc_line: Dict[str, Any], client_config: DoccanoClientConfig
    ) -> Self:
        text: str = doc_line.pop(client_config.column_text)
        entities = [_DoccanoEntity(**ann) for ann in doc_line.pop("entities")]
        relations = [_DoccanoRelation(**ann) for ann in doc_line.pop("relations")]
        # in doccano, metadata is what remains after removing key fields
        metadata = doc_line
        return cls(text=text, entities=entities, relations=relations, metadata=metadata)

    def to_dict(self) -> Dict[str, Any]:
        doc_dict = dict(text=self.text)
        doc_dict["entities"] = [ent.to_dict() for ent in self.entities]
        doc_dict["relations"] = [rel.to_dict() for rel in self.relations]
        doc_dict.update(self.metadata)
        return doc_dict

@dataclasses.dataclass()
class _DoccanoDocSeqLabeling:
    text: str
    entities: List[_DoccanoEntityTuple]
    metadata: Dict[str, Any]

    @classmethod
    def from_dict(
        cls, doc_line: Dict[str, Any], client_config: DoccanoClientConfig
    ) -> Self:
        text = doc_line.pop(client_config.column_text)
        entities = [
            _DoccanoEntityTuple(*ann)
            for ann in doc_line.pop(client_config.column_label)
        ]
        # in doccano, metadata is what remains after removing key fields
        metadata = doc_line
        return cls(text=text, entities=entities, metadata=metadata)

    def to_dict(self) -> Dict[str, Any]:
        doc_dict = dict(text=self.text)
        doc_dict["label"] = [ent.to_tuple() for ent in self.entities]
        doc_dict.update(self.metadata)
        return doc_dict

@dataclasses.dataclass()
class _DoccanoDocTextClassification:
    text: str
    label: str
    metadata: Dict[str, Any]

    @classmethod
    def from_dict(
        cls, doc_line: Dict[str, Any], client_config: DoccanoClientConfig
    ) -> Self:
        text = doc_line.pop(client_config.column_text)
        label = doc_line.pop(client_config.column_label)[0]

        if not isinstance(label, str):
            raise TypeError(
                "The label must be a string. Please check if the document corresponds"
                " to a text classification task rather than sequence labeling"
            )
        # in doccano, metadata is what remains after removing key fields
        metadata = doc_line
        return cls(text=text, label=label, metadata=metadata)

    def to_dict(self) -> Dict[str, Any]:
        doc_dict = dict(text=self.text, label=[str(self.label)])
        doc_dict.update(self.metadata)
        return doc_dict

class DoccanoInputConverter:
    """Convert doccano files (.JSONL) containing annotations for a given task.

    For each line, a :class:`~.core.text.TextDocument` will be created.
    The doccano files can be loaded from a directory with zip files or from a jsonl file.

    The converter supports custom configuration to define the parameters used by doccano
    when importing the data (c.f. :class:`~.io.doccano.DoccanoClientConfig`)

    .. warning::
        If the option *Count grapheme clusters as one character*  was selected
        when creating the doccano project, the converted documents are
        likely to have alignment problems; the converter does not support this option.
    """

    def __init__(
        self,
        task: DoccanoTask,
        client_config: Optional[DoccanoClientConfig] = None,
        attr_label: str = "doccano_category",
        uid: Optional[str] = None,
    ):
        """
        Parameters
        ----------
        task:
            The doccano task for the input converter
        client_config:
            Optional client configuration to define default values in doccano interface.
            This config can change, for example, the name of the text field or labels.
        attr_label:
            The label to use for the medkit attribute that represents the doccano category.
            This is related to :class:`~.io.DoccanoTask.TEXT_CLASSIFICATION` projects.
        uid:
            Identifier of the converter.
        """
        if uid is None:
            uid = generate_id()

        if client_config is None:
            client_config = DoccanoClientConfig()

        self.uid = uid
        self.client_config = client_config
        self.task = task
        self.attr_label = attr_label
        self._prov_tracer: Optional[ProvTracer] = None

    def set_prov_tracer(self, prov_tracer: ProvTracer):
        """Enable provenance tracing.

        Parameters
        ----------
        prov_tracer:
            The provenance tracer used to trace the provenance.
        """
        self._prov_tracer = prov_tracer

    @property
    def description(self) -> OperationDescription:
        """Contains all the input converter init parameters."""
        return OperationDescription(
            uid=self.uid,
            name=self.__class__.__name__,
            class_name=self.__class__.__name__,
            config=dict(task=self.task.value),
        )

    def load_from_directory_zip(self, dir_path: Union[str, Path]) -> List[TextDocument]:
        """Create a list of TextDocuments from zip files in a directory.
        The zip files should contain a JSONL file coming from doccano.

        Parameters
        ----------
        dir_path:
            The path to the directory containing zip files.

        Returns
        -------
        List[TextDocument]
            A list of TextDocuments
        """
        documents = []
        for path_zip in sorted(Path(dir_path).glob("*.zip")):
            documents.extend(self.load_from_zip(path_zip))

        if len(documents) == 0:
            logger.warning(f"No .zip nor .jsonl found in '{dir_path}'")

        return documents

    def load_from_zip(self, input_file: Union[str, Path]) -> List[TextDocument]:
        """
        Create a list of TextDocuments from a zip file containing a JSONL file
        coming from doccano.

        Parameters
        ----------
        input_file:
            The path to the zip file containing a docanno JSONL file

        Returns
        -------
        List[TextDocument]
            A list of TextDocuments
        """

        with tempfile.TemporaryDirectory() as tmpdir:
            with ZipFile(input_file, mode="r") as zip_file:
                filename = zip_file.namelist()[0]
                unzipped_file = Path(tmpdir) / filename
                zip_file.extract(filename, tmpdir)
            return self.load_from_file(unzipped_file)

    def load_from_file(self, input_file: Union[str, Path]) -> List[TextDocument]:
        """Create a list of TextDocuments from a doccano JSONL file.

        Parameters
        ----------
        input_file:
            The path to the JSONL file containing doccano annotations

        Returns
        -------
        List[TextDocument]
            A list of TextDocuments
        """
        documents = []
        with open(Path(input_file), encoding="utf-8") as fp:
            for line in fp:
                doc_line = json.loads(line)
                doc = self._parse_doc_line(doc_line)
                documents.append(doc)

        self._check_crlf_character(documents)
        return documents

    def _check_crlf_character(self, documents: List[TextDocument]):
        """Check if the list of converted documents contains the CRLF character.
        This character is the only indicator available to warn
        if there are alignment problems in the documents"""
        if (
            self.task == DoccanoTask.RELATION_EXTRACTION
            or self.task == DoccanoTask.SEQUENCE_LABELING
        ):
            nb_docs_with_warning = sum(
                document.text.find("\r\n") != -1 for document in documents
            )

            if nb_docs_with_warning > 0:
                logger.warning(
                    f"{nb_docs_with_warning}/{len(documents)} documents contain"
                    " '\\r\\n' characters. If you have selected 'Count grapheme"
                    " clusters as one character' when creating the doccano project,"
                    " converted documents are likely to have alignment problems.\n"
                    " Please ignore this message if you did not select this option when"
                    " creating the project."
                )

    def _parse_doc_line(self, doc_line: Dict[str, Any]) -> TextDocument:
        """Parse a doc_line into a TextDocument depending on the task

        Parameters
        ----------
        doc_line:
            A dictionary representing an annotation from doccano

        Returns
        -------
        TextDocument
            A document with parsed annotations.
        """
        if self.task == DoccanoTask.RELATION_EXTRACTION:
            return self._parse_doc_line_relation_extraction(doc_line=doc_line)
        if self.task == DoccanoTask.TEXT_CLASSIFICATION:
            return self._parse_doc_line_text_classification(doc_line=doc_line)
        if self.task == DoccanoTask.SEQUENCE_LABELING:
            return self._parse_doc_line_seq_labeling(doc_line=doc_line)

    def _parse_doc_line_relation_extraction(
        self, doc_line: Dict[str, Any]
    ) -> TextDocument:
        """Parse a dictionary and return a TextDocument with entities and relations

        Parameters
        ----------
        doc_line:
            Dictionary with doccano annotation

        Returns
        -------
        TextDocument
            The document with annotations
        """
        try:
            doccano_doc = _DoccanoDocRelationExtraction.from_dict(
                doc_line, client_config=self.client_config
            )
        except Exception as err:
            raise Exception(
                "Impossible to convert the document. Please check the task"
                " or the client configuration of the converter"
            ) from err

        ents_by_doccano_id = dict()
        relations = []
        for doccano_entity in doccano_doc.entities:
            text = doccano_doc.text[
                doccano_entity.start_offset : doccano_entity.end_offset
            ]
            entity = Entity(
                text=text,
                label=doccano_entity.label,
                spans=[Span(doccano_entity.start_offset, doccano_entity.end_offset)],
                metadata=dict(doccano_id=doccano_entity.id),
            )
            ents_by_doccano_id[doccano_entity.id] = entity

            if self._prov_tracer is not None:
                self._prov_tracer.add_prov(
                    entity, self.description, source_data_items=[]
                )

        for doccano_relation in doccano_doc.relations:
            relation = Relation(
                label=doccano_relation.type,
                source_id=ents_by_doccano_id[doccano_relation.from_id].uid,
                target_id=ents_by_doccano_id[doccano_relation.to_id].uid,
                metadata=dict(doccano_id=doccano_relation.id),
            )
            relations.append(relation)

            if self._prov_tracer is not None:
                self._prov_tracer.add_prov(
                    relation, self.description, source_data_items=[]
                )

        anns = list(ents_by_doccano_id.values()) + relations
        doc = TextDocument(
            text=doccano_doc.text,
            anns=anns,
            metadata=doccano_doc.metadata,
        )

        return doc

    def _parse_doc_line_seq_labeling(self, doc_line: Dict[str, Any]) -> TextDocument:
        """Parse a dictionary and return a TextDocument with entities

        Parameters
        ----------
        doc_line:
            Dictionary with doccano annotation.

        Returns
        -------
        TextDocument
            The document with annotations
        """
        try:
            doccano_doc = _DoccanoDocSeqLabeling.from_dict(
                doc_line, client_config=self.client_config
            )
        except Exception as err:
            raise Exception(
                "Impossible to convert the document. Please check the task"
                " or the client configuration of the converter"
            ) from err

        entities = []
        for doccano_entity in doccano_doc.entities:
            text = doccano_doc.text[
                doccano_entity.start_offset : doccano_entity.end_offset
            ]
            entity = Entity(
                text=text,
                label=doccano_entity.label,
                spans=[Span(doccano_entity.start_offset, doccano_entity.end_offset)],
            )
            entities.append(entity)

            if self._prov_tracer is not None:
                self._prov_tracer.add_prov(
                    entity, self.description, source_data_items=[]
                )

        doc = TextDocument(
            text=doccano_doc.text,
            anns=entities,
            metadata=doccano_doc.metadata,
        )
        return doc

    def _parse_doc_line_text_classification(
        self, doc_line: Dict[str, Any]
    ) -> TextDocument:
        """Parse a dictionary and return a TextDocument with an attribute.
        The attribute will be in its raw segment.

        Parameters
        ----------
        doc_line:
            Dictionary with doccano annotation.

        Returns
        -------
        TextDocument
            The document with its category
        """
        try:
            doccano_doc = _DoccanoDocTextClassification.from_dict(
                doc_line, client_config=self.client_config
            )
        except Exception as err:
            raise Exception(
                "Impossible to convert the document. Please check the task"
                " or the client configuration of the converter"
            ) from err

        attr = Attribute(label=self.attr_label, value=doccano_doc.label)

        if self._prov_tracer is not None:
            self._prov_tracer.add_prov(attr, self.description, source_data_items=[])

        doc = TextDocument(text=doccano_doc.text, metadata=doccano_doc.metadata)
        # FIXME: related to issue #39
        # the attribute is added to the 'raw_segment', as doc attributes are not supported
        doc.raw_segment.attrs.add(attr)
        return doc

class DoccanoOutputConverter:
    """Convert medkit files to doccano files (.JSONL) for a given task.

    For each :class:`~medkit.core.text.TextDocument` a jsonline will be created.
    """

    def __init__(
        self,
        task: DoccanoTask,
        anns_labels: Optional[List[str]] = None,
        attr_label: Optional[str] = None,
        ignore_segments: bool = True,
        include_metadata: Optional[bool] = True,
        uid: Optional[str] = None,
    ):
        """
        Parameters
        ----------
        task:
            The doccano task for the input converter
        anns_labels:
            Labels of medkit annotations to convert into doccano annotations.
            If `None` (default) all the entities or relations will be converted.
            Useful for :class:`~.io.DoccanoTask.SEQUENCE_LABELING` or
            :class:`~.io.DoccanoTask.RELATION_EXTRACTION` converters.
        attr_label:
            The label of the medkit attribute that represents the text category.
            Useful for :class:`~.io.DoccanoTask.TEXT_CLASSIFICATION` converters.
        ignore_segments:
            If `True` medkit segments will be ignored. Only entities will be
            converted to Doccano entities.  If `False` the medkit segments will
            be converted to Doccano entities as well.
            Useful for :class:`~.io.DoccanoTask.SEQUENCE_LABELING` or
            :class:`~.io.DoccanoTask.RELATION_EXTRACTION` converters.
        include_metadata:
            Whether include medkit metadata in the converted documents
        uid:
            Identifier of the converter.
        """
        if uid is None:
            uid = generate_id()

        self.uid = uid
        self.task = task
        self.anns_labels = anns_labels
        self.attr_label = attr_label
        self.ignore_segments = ignore_segments
        self.include_metadata = include_metadata

    @property
    def description(self) -> OperationDescription:
        return OperationDescription(
            uid=self.uid,
            name=self.__class__.__name__,
            class_name=self.__class__.__name__,
            config=dict(task=self.task.value),
        )

    def save(self, docs: List[TextDocument], output_file: Union[str, Path]):
        """Convert and save a list of TextDocuments into a doccano file (.JSONL)

        Parameters
        ----------
        docs:
            List of medkit doc objects to convert
        output_file:
            Path or string of the JSONL file where to save the converted documents
        """

        output_file = Path(output_file)

        with open(output_file, mode="w", encoding="utf-8") as fp:
            for medkit_doc in docs:
                doc_line = self._convert_doc_by_task(medkit_doc)
                fp.write(json.dumps(doc_line, ensure_ascii=False) + "\n")

    def _convert_doc_by_task(self, medkit_doc: TextDocument) -> Dict[str, Any]:
        """Convert a TextDocument into a dictionary depending on the task

        Parameters
        ----------
        medkit_doc:
            Document to convert

        Returns
        -------
        Dict[str,Any]
            Dictionary with doccano annotation
        """
        if self.task == DoccanoTask.RELATION_EXTRACTION:
            return self._convert_doc_relation_extraction(medkit_doc=medkit_doc)
        if self.task == DoccanoTask.TEXT_CLASSIFICATION:
            return self._convert_doc_text_classification(medkit_doc=medkit_doc)
        if self.task == DoccanoTask.SEQUENCE_LABELING:
            return self._convert_doc_seq_labeling(medkit_doc=medkit_doc)

    def _convert_doc_relation_extraction(
        self, medkit_doc: TextDocument
    ) -> Dict[str, Any]:
        """Convert a TextDocument to a doc_line compatible
        with the doccano relation extraction task

        Parameters
        ----------
        medkit_doc:
            Document to convert, it may contain entities and relations

        Returns
        -------
        Dict[str,Any]
            Dictionary with doccano annotation. It may contain
            text, entities and relations
        """
        doccano_ents_by_medkit_uid = dict()
        doccano_relations = []

        anns_by_type = get_anns_by_type(medkit_doc, self.anns_labels)
        medkit_segments = anns_by_type["entities"]
        if not self.ignore_segments:
            medkit_segments += anns_by_type["segments"]

        for medkit_segment in medkit_segments:
            spans = span_utils.normalize_spans(medkit_segment.spans)
            ann_id = generate_deterministic_id(medkit_segment.uid)
            entity = _DoccanoEntity(
                id=ann_id.int,
                start_offset=spans[0].start,
                end_offset=spans[-1].end,
                label=medkit_segment.label,
            )
            doccano_ents_by_medkit_uid[medkit_segment.uid] = entity

        for medkit_relation in anns_by_type["relations"]:
            subj = doccano_ents_by_medkit_uid.get(medkit_relation.source_id)
            obj = doccano_ents_by_medkit_uid.get(medkit_relation.target_id)

            if subj is None or obj is None:
                logger.warning(
                    f"Ignore relation {medkit_relation.uid}. Entity source/target was"
                    " no found"
                )
                continue

            ann_id = generate_deterministic_id(medkit_relation.uid)
            relation = _DoccanoRelation(
                id=ann_id.int,
                from_id=subj.id,
                to_id=obj.id,
                type=medkit_relation.label,
            )
            doccano_relations.append(relation)

        metadata = medkit_doc.metadata if self.include_metadata else {}

        doccano_doc = _DoccanoDocRelationExtraction(
            text=medkit_doc.text,
            entities=list(doccano_ents_by_medkit_uid.values()),
            relations=doccano_relations,
            metadata=metadata,
        )

        return doccano_doc.to_dict()

    def _convert_doc_seq_labeling(self, medkit_doc: TextDocument) -> Dict[str, Any]:
        """Convert a TextDocument to a doc_line compatible
        with the doccano sequence labeling task

        Parameters
        ----------
        medkit_doc:
            Document to convert, it may contain entities and relations

        Returns
        -------
        Dict[str,Any]
            Dictionary with doccano annotation. It may contain
            text ans its label (a list of tuples representing entities)
        """
        anns_by_type = get_anns_by_type(medkit_doc, self.anns_labels)
        medkit_segments = anns_by_type["entities"]
        if not self.ignore_segments:
            medkit_segments += anns_by_type["segments"]
        doccano_entities = []
        for medkit_segment in medkit_segments:
            spans = span_utils.normalize_spans(medkit_segment.spans)
            entity = _DoccanoEntityTuple(
                start_offset=spans[0].start,
                end_offset=spans[-1].end,
                label=medkit_segment.label,
            )
            doccano_entities.append(entity)

        metadata = medkit_doc.metadata if self.include_metadata else {}
        doccano_doc = _DoccanoDocSeqLabeling(
            text=medkit_doc.text,
            entities=doccano_entities,
            metadata=metadata,
        )

        return doccano_doc.to_dict()

    def _convert_doc_text_classification(
        self, medkit_doc: TextDocument
    ) -> Dict[str, Any]:
        """Convert a TextDocument to a doc_line compatible with
        the doccano text classification task. The attribute to add as a label
        should be in its raw segment.

        Parameters
        ----------
        doc_line:
            Dictionary with doccano annotation.

        Returns
        -------
        Dict[str,Any]
            Dictionary with doccano annotation. It may contain
            text ans its label (a category(str))
        """
        attributes = medkit_doc.raw_segment.attrs.get(label=self.attr_label)

        if not attributes:
            raise KeyError(
                "The attribute with the corresponding text class was not found. Check"
                f" the 'attr_label' for this converter, {self.attr_label} was provided."
            )

        metadata = medkit_doc.metadata if self.include_metadata else {}
        doccano_doc = _DoccanoDocTextClassification(
            text=medkit_doc.text,
            label=attributes[0].value,
            metadata=metadata,
        )
        return doccano_doc.to_dict()


# -*- coding: utf-8 -*-
from collections import OrderedDict

import django_filters
from django.db import transaction
from rest_framework import serializers

from ralph.api import RalphAPISerializer, RalphAPIViewSet, router
from ralph.api.serializers import RalphAPISaveSerializer
from ralph.networks.models.networks import IPAddress
from ralph.security.models import any_exceeded, SecurityScan, Vulnerability

class VulnerabilitySerializer(RalphAPISerializer):

    class Meta:
        model = Vulnerability
        depth = 1

class VulnerabilityViewSet(RalphAPIViewSet):
    queryset = Vulnerability.objects.all()
    serializer_class = VulnerabilitySerializer
    filter_fields = [
        'external_vulnerability_id',
    ]

class SecurityScanSerializer(RalphAPISerializer):
    vulnerabilities = VulnerabilitySerializer(many=True)

    class Meta:
        model = SecurityScan

class SaveSecurityScanSerializer(RalphAPISaveSerializer):

    class Meta:
        model = SecurityScan

    def to_internal_value(self, data):
        errors = OrderedDict()

        # external_id to local_id
        if 'external_vulnerabilities' in data:
            external_ids = set(data.get('external_vulnerabilities', []))
            converted = Vulnerability.objects.filter(
                external_vulnerability_id__in=external_ids)
            if len(converted) != len(external_ids):
                unknown = external_ids - set(
                    [str(v.external_vulnerability_id) for v in converted]
                )
                msg = "Unknow external_vulnerabilities: {}".format(
                    ', '.join(unknown)
                )
                errors['external_vulnerability'] = msg
            merged_vulnerabilities = data.get('vulnerabilities', [])
            merged_vulnerabilities.extend([c.id for c in converted])
            data['vulnerabilities'] = merged_vulnerabilities

        host_ip = data.get('host_ip', None)
        if host_ip:
            base_object = None
            # try to get base object by IPAddress
            try:
                ip_address = IPAddress.objects.get(address=host_ip)
            except IPAddress.DoesNotExist:
                pass
            else:
                base_object = ip_address.base_object
            if not base_object:
                errors['host_ip'] = "IP is not assigned to any host"
        else:
            errors['host_ip'] = "Host IP is required"

        if errors:
            raise serializers.ValidationError(errors)
        data['base_object'] = base_object.pk
        data['is_patched'] = not any_exceeded(
            Vulnerability.objects.filter(id__in=data['vulnerabilities'])
        )
        result = super().to_internal_value(data)
        return result

class IPFilter(django_filters.FilterSet):
    ip = django_filters.CharFilter(
        name='base_object__ethernet_set__ipaddress__address'
    )

class SecurityScanViewSet(RalphAPIViewSet):
    queryset = SecurityScan.objects.all()
    serializer_class = SecurityScanSerializer
    save_serializer_class = SaveSecurityScanSerializer

    additional_filter_class = IPFilter

    @transaction.atomic
    def create(self, request, *args, **kwargs):
        ip = IPAddress.objects.filter(address=self.request.data.get(
            'host_ip', None)
        ).first()
        if ip:
            # one scan can exist for ip (because there are linked by onetoone)
            # so this removes old scan to allow updates by post
            SecurityScan.objects.filter(base_object=ip.base_object.id).delete()
        return super().create(request, *args, **kwargs)

router.register(r'vulnerabilities', VulnerabilityViewSet)
router.register(r'security-scans', SecurityScanViewSet)
urlpatterns = []


# Pacakge Imports
from utils import *
import importlib
import utils
importlib.reload(utils)
import os.path
from os import path
import matplotlib.pyplot as plt
import math
import pandas as pd

class Visualization :
    def __init__(self) :
        """ Constructor. Creates a path to the Maxtrix Asset file for visualization functions to use. """

        file_path = pathlib.Path().absolute().joinpath('../QGIS/MileMarker/Matrix Files')
        self.file_name = file_path.joinpath('total_asset_matrix.csv')

    def plot_graph(self, start_loc, end_loc) :
        """ Creates a visual representation graph of all assets on the highway.

        Args :
            start_loc : Starting point of the graph. Any number from 0 ~ 30.
            end_loc : Ending point of the graph. Any number from 0 ~ 30.
        """
        df = read_data(self.file_name, start_loc, end_loc)

        fig, ax = plt.subplots(figsize=(15,2))
        plt.xlim(start_loc + 52, end_loc + 52)
        plt.ylim(0, 120)
        ax.set_facecolor('grey')
        ax.axhline(y=60, color='yellow')

        rds_x, rds_y = get_asset_xy_coords(df, 'rds')
        dms_x, dms_y = get_asset_xy_coords(df, 'dms')
        cctv_x, cctv_y = get_asset_xy_coords(df, 'cctv')
        rds = plt.scatter(rds_x, rds_y, color='red', label='RDS')
        dms = plt.scatter(dms_x, dms_y, color='green', label='DMS')
        cctv = plt.scatter(cctv_x, cctv_y, color='yellow', label='CCTV')

        plt.title(f'Assets from Miles {start_loc+52} to {end_loc+52}', fontdict={'fontsize':'large','fontweight':'bold'}, pad=20)
        plt.legend(loc='upper center', bbox_to_anchor=(1, 1.55), fancybox=True, shadow=True, ncol=1)
        plt.xlabel('Miles')
        plt.ylabel('ft')
        plt.savefig('../Output Graphs/asset_graph.jpg', bbox_inches='tight')

def read_data(file_name, start_loc, end_loc) :
    """ Reads the Matrix CSV file and querys only the assets that are within the given range.

    Args :
        file_name : The location / name of the Matrix CSV file.
        start_loc : Starting point of the query. Any number from 0 ~ 30.
        end_loc : Ending point of the query. Any number from 0 ~ 30.

    Return :
        The queryed DataFrame.
    """
    df = pd.read_csv(file_name, skiprows=0, error_bad_lines=False, index_col=False)
    df = df.query('Direction != "other"')
    df = df.loc[df['Milemarker Number'] >= (start_loc + 52.0)]
    df = df.loc[df['Milemarker Number'] <= (end_loc + 52.0)]
    df = df.reset_index(drop = True)

    return df

def findYVal(direction) :
    """ Helper function that acts like a dictionary to find the Y value of an asset depedning on
    its direction.

    Args :
        direction : One of three possible placements on the highway : east, center, or west.

    Return :
        The corresponding Y coordinate value of the asset.
    """
    if(direction == 'east') :
        return 10;
    if(direction == 'center') :
        return 60;
    if(direction == 'west') :
        return 110;

def get_asset_xy_coords(df, asset_type) :
    """ From the total DataFrame, chooses one type of asset and saves its x,y coordinates into
    a list for plotting purposes.

    Args :
        df : Total DataFrame.
        asset_type : The type of asset that we wish to obtain the coordinate from.

    Return :
        2 lists : first contains the X coordinates of the assets, second contains Y coordinates.
    """
    df = df[df.Type == asset_type]
    df = df.reset_index(drop = True)

    xcoord, ycoord = [], []

    for i in range(len(df)) :
        xcoord.append(df.at[i, 'Milemarker Number'])
        ycoord.append(findYVal(df.at[i, 'Direction']))

    return xcoord, ycoord


import json
from pathlib import Path

import click
from tqdm import tqdm

from aspects.data_io.parsers import parse_gzip
from aspects.utilities.settings import AMAZON_REVIEWS_DATASETS_PATH

@click.command()
@click.option('--dataset-path', required=False, type=str, help='Path to the amazon domain gz file')
@click.option('--column', required=False, type=str, default='reviewText')
@click.option('--n-reviews', required=False, type=int, default=None)
def amazon_domain_gz_to_json(dataset_path, column='reviewText', n_reviews: int = None):
    reviews = {}
    dataset_name = Path(dataset_path).name.replace('.gz', '')
    json_output = AMAZON_REVIEWS_DATASETS_PATH / dataset_name
    for idx, d in tqdm(enumerate(parse_gzip(dataset_path)), desc=f'JSON is saving to: {json_output.as_posix()}'):
        reviews[idx] = d[column]
        if n_reviews is not None and idx > n_reviews:
            break
    with open(json_output.as_posix(), 'w') as j:
        json.dump(reviews, j)

if __name__ == '__main__':
    amazon_domain_gz_to_json()


from src.parsers.parser import Parser
from src.dependency_parse import DependencyParse
import spacy
from spacy.tokens import Doc

class SpacyParser(Parser):

    def __init__(self, model_name: str):
        self.model = spacy.load(model_name)

    def parse(self,sentence: str, tokens: list): #-> DependencyParse:
        """Use the specified spaCy model to parse the sentence.py.

        The function should return the parse in the Dependency format.

        You should refer to the spaCy documentation for how to use the model for dependency parsing.
        """
        custom_tokenize = Doc(self.model.vocab,tokens)
        spacy_op = self.model(custom_tokenize)
        heads = []
        deps = []
        for token in spacy_op:
            head_id = tokens.index(token.head.text)+1
            token_dep = token.dep_
            if token_dep=="ROOT":
                token_dep = "root"
                head_id = 0
            deps.append(token_dep)
            heads.append(str(head_id))
        return DependencyParse(sentence,tokens,heads,deps)


import torch.nn as nn

def _real_init_weights(m, nonlinearity='relu'):
    if isinstance(m, list):
        for mini_m in m:
            _real_init_weights(mini_m)
    else:
        if isinstance(m, nn.Conv2d):
            # print('initializing Convolution layer...')
            nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity=nonlinearity)
            if m.bias is not None:
                nn.init.zeros_(m.bias)
        elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm, nn.InstanceNorm2d)):
            # print('initializing BN layer...')
            nn.init.ones_(m.weight)
            if m.bias is not None:
                nn.init.zeros_(m.bias)
        elif isinstance(m, nn.Linear):
            # print('initializing Linear layer...')
            nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity=nonlinearity)
            if m.bias is not None:
                nn.init.zeros_(m.bias)
        elif isinstance(m, nn.Module):
            for mini_m in m.children():
                _real_init_weights(mini_m)
        else:
            raise NotImplementedError('Unknown init module.')

def init_weights(*models, nonlinearity='relu'):
    """
    Initialize model's modules.
    """
    print('initializing modules...')
    for model in models:
        _real_init_weights(model, nonlinearity=nonlinearity)


from random import randint, choice

START_MESSAGE = 'What is the result of the expression?'
MIN_LIMIT = 0
MAX_LIMIT = 50

def calculate_expression(number_1: int, number_2: int, math_operator: str):
    '''Converts string to math expression with end result'''
    action_list = {'+': int(number_1) + int(number_2),
                   '-': int(number_1) - int(number_2),
                   '*': int(number_1) * int(number_2)}
    return action_list.get(math_operator)

def make_question_and_answer() -> list:
    '''Generates random math expression and it's result'''
    number_1 = randint(MIN_LIMIT, MAX_LIMIT)
    number_2 = randint(MIN_LIMIT, MAX_LIMIT)
    math_actions = ['+', '-', '*']
    math_operator = choice(math_actions)
    question = f'{number_1} {math_operator} {number_2}'
    answer = str(calculate_expression(number_1, number_2, math_operator))
    return [question, answer]


import json
import MapReduce
import sys

mr = MapReduce.MapReduce()

def mapper(book_input):
	doc_id = book_input[0]
	text = book_input[1]
	for word in text.split():
		mr.emit_intermediate(word,doc_id)

def reducer(word, list_of_docs):
	uniques = []
	for doc in list_of_docs:
		if doc not in uniques:
			uniques.append(doc)
	mr.emit((word,uniques))

if __name__ == '__main__':
  # the code loads the json file and executes
  # the MapReduce query which prints the result to stdout.
  inputdata = open(sys.argv[1])
  mr.execute(inputdata, mapper, reducer)


import logging
import os.path

formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')

class Logger:
    __logger = None
    __logging = None
    __error_logging = None

    @staticmethod
    def get_instance():

        if Logger.__logger is None:
            Logger()
        return Logger.__logger

    def __init__(self):

        if Logger.__logger is not None:
            raise Exception("Unauthorized access to Logger constructor")
        else:
            Logger.__logger = self

    @staticmethod
    def start_logger(log_file_path=os.path.dirname(__file__) + "\\LogFiles\\Event_Log.txt",
                     error_file_path=os.path.dirname(__file__) + "\\LogFiles\\Error_Log.txt"):
        # logging.basicConfig(filename=log_file_path, format=formatter)
        handler = logging.FileHandler(log_file_path)
        handler.setFormatter(formatter)
        Logger.__logging = logging.getLogger('main')
        Logger.__logging.setLevel(logging.DEBUG)
        Logger.__logging.addHandler(handler)

        handler = logging.FileHandler(error_file_path)
        handler.setFormatter(formatter)
        Logger.__error_logging = logging.getLogger('error_log')
        Logger.__error_logging.setLevel(logging.DEBUG)
        Logger.__error_logging.addHandler(handler)

    """ Adds a information log to the log file with the given message"""

    @staticmethod
    def info_log(message: str):
        if Logger.__logging is None:
            raise Exception('Logger not initiated')
        Logger.__logging.info('Info: ' + message)

    """ Adds a warning log to the log file with the given message"""

    @staticmethod
    def warning_log(message: str):
        if Logger.__logging is None:
            raise Exception('Logger not initiated')
        Logger.__error_logging.warning('Warning: ' + message)

    """ Adds a error log to the log file with the given message"""

    @staticmethod
    def error_log(message: str):
        if Logger.__logging is None:
            raise Exception('Logger not initiated')
        Logger.__error_logging.error('Error: ' + message)

    """ Adds a critical event log to the log file with the given message"""

    @staticmethod
    def critical_log(message: str):
        if Logger.__logging is None:
            raise Exception('Logger not initiated')
        Logger.__error_logging.critical('Critical: ' + message)


phrase = "Bonjour tout le monde"
phrase = phrase.lower()
phrase = phrase.split()
phrase.reverse()
print(" ".join(phrase).capitalize())

phrase = "Bonjour tout le monde"
phrase_split = phrase.split()
resultat = []
for mot in reversed(phrase_split):
    resultat.append(mot)
resultat_formate = " ".join(resultat).capitalize()
print(resultat_formate)


import os
import json
THIS_FILE = os.path.dirname(os.path.abspath(__file__))
PROJ_FILE = os.path.abspath(os.path.join(THIS_FILE, "../"))
db_path = PROJ_FILE + '/commentry/db.json'
class GetComment:
    def __init__(self) -> None:
        pass
    def checkdb_commentry(self):
        f = open(db_path,'r')
        l = json.load(f)
        f.close()
        if l["1"] == []:
            return 0
        else:
            path = l["1"][0]
            l["1"].remove(l["1"][0])
            f = open(db_path,'w')
            json.dump(l,f)
            f.close()
            return  1,path
f = GetComment()
print(f.checkdb_commentry())


# -*- coding: utf-8 -*-
"""Simple classical lambda-based let.

It works, but does not support assignment, so it can't be used for things like
let-over-lambda, or indeed letrec.  But it's simple, and creates real lexical
variables.

These are here mainly for documentation purposes; the other macros are designed
to work together with the regular ``let``, ``letseq``, ``letrec`` from the module
``unpythonic.syntax``, not the ones defined here.

This module is a copy of `demo/anaphoric_if_revisited/let.py` from `mcpyrate`.
"""

# Unlike the other submodules, this module contains the macro interface;
# these macros are not part of the top-level ``unpythonic.syntax`` interface.

__all__ = ["let", "letseq"]

from mcpyrate.quotes import macros, q, a, t  # noqa: F811, F401

from ast import arg

from mcpyrate import parametricmacro
from mcpyrate.quotes import capture_as_macro

@parametricmacro
def let(tree, *, args, syntax, **kw):
    """[syntax, expr] Bind expression-local variables.

    Usage::

        let[[k0, v0], ...][expr]

    `let` expands into a `lambda`::

        let[[x, 1], [y, 2]][print(x, y)]
        # --> (lambda x, y: print(x, y))(1, 2)
    """
    if syntax != "expr":
        raise SyntaxError("`let` is an expr macro only")  # pragma: no cover
    if not args:
        raise SyntaxError("expected at least one binding")  # pragma: no cover

    # args: `list` of `ast.List`. Each sublist is [ast.Name, expr].
    names = [k.id for k, _ in (a.elts for a in args)]
    values = [v for _, v in (a.elts for a in args)]
    if len(set(names)) < len(names):
        raise SyntaxError("binding names must be unique in the same `let`")  # pragma: no cover

    lam = q[lambda: a[tree]]
    lam.args.args = [arg(arg=x) for x in names]
    return q[a[lam](a[values])]

@parametricmacro
def letseq(tree, *, args, syntax, expander, **kw):
    """[syntax, expr] Sequential let, like `let*` in Scheme.

    Usage::

        letseq[[k0, v0], ...][expr]

    The difference to `let` is that in `letseq`, on the RHS
    of each new binding, the previous bindings are in scope.

    Expands to a sequence of nested `let`.

    Example::

        letseq[[x, 21], [x, 2 * x]][x]  # --> 42
    """
    if syntax != "expr":
        raise SyntaxError("`letseq` is an expr macro only")  # pragma: no cover
    if not args:
        return tree
    first, *rest = args
    body = q[a[_our_letseq][t[rest]][a[tree]]]
    return q[a[_our_let][a[first]][a[body]]]

# for hygienic macro recursion
_our_let = capture_as_macro(let)
_our_letseq = capture_as_macro(letseq)


import django
# from django.conf import settings
# from django.contrib.auth import get_user_model
# from django.http import request
from django.shortcuts import render
from django.urls import reverse_lazy

from .models import Patient, RadStaff, RadioUser
from django.views import View
from django.contrib.auth.decorators import login_required
from django.views.generic.base import TemplateView
from django.views.generic.list import ListView
from django.views.generic.edit import FormView, CreateView
from django.contrib.auth.mixins import LoginRequiredMixin

# Create your views here.
'''
class RadioWelcomeView(LoginRequiredMixin, View):
    login_url = reverse_lazy('authentication:login')
    allRadUser = RadioUser.objects.all().order_by('username_id')
    first = allRadUser[0]
    second = allRadUser[1]
    last = allRadUser.last()
    # firstRadUser = RadioUser.objects.all().order_by('username_id').first()
    # radUserUsername = radUser.username
    if user == first or user == second or user == last:

        def get(self, request):
            context = {
                "title": 'Medical Radiology Department',
                "page_title": 'Welcome to "RADIOLOGY"'
            }
            return render(request, "radio/radio_welcome.html", context)

    else:
        def get(self, request):
            context = {
                "title": 'Medical Radiology Department',
                "page_title": 'Welcome to "RADIOLOGY"'
            }
            return render(request, "root/welcome.html", context) '''

def regUserList(request):
    allUser = RadioUser.objects.all().order_by('username_id')
    user = request.user
    if user in allUser:
        return True
    else:
        return False

@login_required(login_url="/login/")
def radioWelcomeView(request):
    check = regUserList(request)
    if check == True:
        print("logged in")
        context = {
            "title": 'Medical Radiology Department',
            "page_title": 'Welcome to "RADIOLOGY"'
        }
        return render(request, 'radio/radio_welcome.html', context)
    else:
        print("Not Department User")
        return render(request, 'root/welcome.html')

class EchoHomeView(View):

    def get(self, request):
        context = {
            "title": 'Ultrasound',
            'page_title': 'ULTRASOUND',
            'subtitle': 'HOME PAGE',
            # "short_title": 'RH'
        }
        return render(request, 'echo/echo_home.html', context)

class XrayHomeView(View):

    def get(self, request):
        context = {
            "title": 'XRAY',
            # "short_title": 'PH',
            'page_title': 'XRAY',
            'subtitle': 'HOME PAGE',
        }
        return render(request, 'xray/xray_home.html', context)


import matplotlib.pyplot as plt

def generar_grafica():
    etiquetas =['A','B','C']
    valores = [200, 56, 156]

    fig, ax = plt.subplots()
    ax.pie(valores, labels=etiquetas)
    plt.savefig('pie.png')
    plt.close()


from sqlalchemy import Column, ForeignKey, Integer, BigInteger, DateTime, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql import func

Base = declarative_base()

class Wallet(Base):
    __tablename__ = 'wallets'

    id = Column(Integer, primary_key=True)
    balance = Column(BigInteger)

class Refill(Base):
    __tablename__ = 'refills'

    id = Column(Integer, primary_key=True)
    wallet_id = Column(Integer, ForeignKey('wallets.id'))
    amount = Column(BigInteger)
    datetime = Column(DateTime, server_default=func.now())

class Transaction(Base):
    __tablename__ = 'transactions'

    id = Column(Integer, primary_key=True)
    wallet_id = Column(Integer, ForeignKey('wallets.id'))
    amount = Column(BigInteger)
    datetime = Column(DateTime, server_default=func.now())
    ip = Column(String)


from pathlib import Path

from linebot import LineBotApi
from linebot.models import RichMenu, RichMenuArea, RichMenuBounds, RichMenuSize
from linebot.models.actions import PostbackAction, URIAction

from kb_2315.config import conf

line_bot_api = LineBotApi(conf.line_channel_access_token)

def create_rich_menu() -> None:
    cal_url = (
        "https://larrybolt.github.io/online-ics-feed-viewer/"
        + f"#feed={conf.host_url}/api/calendar/"
        + "&cors=false&title=Shoe%20History&hideinput=true"
    )

    rich_menu_to_create = RichMenu(
        size=RichMenuSize(width=1200, height=800),
        selected=True,
        name="richmenu",
        chat_bar_text="メニュー",
        areas=[
            # カレンダー
            RichMenuArea(
                bounds=RichMenuBounds(x=0, y=0, width=600, height=800),
                action=URIAction(
                    uri=cal_url,
                    label="カレンダーの表示",
                ),
            ),
            # データ
            RichMenuArea(
                bounds=RichMenuBounds(x=600, y=0, width=600, height=400),
                action=URIAction(uri=f"{conf.host_url}/analyze/", label="データの表示"),
            ),
            # 靴一覧
            RichMenuArea(
                bounds=RichMenuBounds(x=600, y=400, width=600, height=400),
                action=PostbackAction(data="shoes_list:"),
            ),
        ],
    )

    richMenuId = line_bot_api.create_rich_menu(rich_menu=rich_menu_to_create)

    with open(Path(__file__).parent / "image/menu.png", "rb") as f:
        line_bot_api.set_rich_menu_image(richMenuId, "image/png", f)

    line_bot_api.set_default_rich_menu(richMenuId)


import pytest
import urllib

import numpy as np
from matplotlib import pyplot as plt

from wave_unify import WaveUnifyData
from freq_detect import FreqDetect
from note import NoteName, Note

show_plots = False
normalize_plots = True
plot_length = 5000

# region helper functions

def plot_show():
    if show_plots:
        plt.show()

def create_wav_files():
    file_links = {
        'sine1.wav': "https://www3.nd.edu/~dthain/courses/cse20211/fall2013/wavfile/sine.wav",
        'sine2.wav': "https://www3.nd.edu/~dthain/courses/cse20211/fall2013/wavfile/sine2.wav",
        'sine3.wav': "https://www3.nd.edu/~dthain/courses/cse20211/fall2013/wavfile/sine3.wav",
    }
    for name, link in file_links.items():
        # check if file exists,
        try:
            file = open(name)
        except IOError:
            file_exists = False
        else:
            file_exists = True
        finally:
            file.close()
        if not file_exists:
            urllib.request.urlretrieve(link, name)
    return file_links.keys()

def get_waves():
    files = create_wav_files()
    wave_dict = {}
    for file in files:
        wavdat = WaveUnifyData(file)
        wave_dict.update({file: wavdat})
    return wave_dict

def get_freqs():
    freq_dict ={}
    for name, wave in get_waves().items():
        freq_det = FreqDetect(wave.samples_mono, wave.sample_rate)
        freq_dict.update({name: freq_det})
    return freq_dict

def get_notes():
    notes_list = [
        {'freq': 440,    'note_name': NoteName.A, 'octave': 4, 'error': +0},
        {'freq': 8.1854, 'note_name': NoteName.C, 'octave':-1, 'error': +2},
        {'freq': 200,    'note_name': NoteName.G, 'octave': 3, 'error': +35},
        {'freq': 27.2,   'note_name': NoteName.A, 'octave': 0, 'error': -19},
    ]
    return notes_list

# endregion

# region fixtures

@pytest.fixture()
def wav_files():
    return create_wav_files()

@pytest.fixture()
def notes_list():
    return get_notes()

# endregion

# region tests

def test_plot_signals(wav_files):
    for filename in wav_files:
        fmi = WaveUnifyData(filename)
        fmi.end = plot_length
        plot_args = {
            'title': "Signal " + filename,
            'normalize': normalize_plots,
        }
        if fmi.mono:
            fmi.plot_mono(**plot_args)
        else:
            stereo_sum = fmi.stereo_sum
            signals = [
                fmi.stereo_diff,
                stereo_sum,
                np.diff(stereo_sum),
            ]
            fmi.plot_signals(
                signals,
                **plot_args,
            )
    plot_show()

def test_plot_fft(wav_files):
    for filename in wav_files:
        wave_i = WaveUnifyData(filename)
        fd1 = FreqDetect(wave_i.samples_mono, wave_i.sample_rate)
        fft_i = fd1.fft()
        fd1.plot_fft(
            fft_i,
            title="FFT "+filename,
            normalize=normalize_plots,
        )
    plot_show()

def test_notes(notes_list):
    # print("Frequency C0: ", Note._freq_note_0())
    assert Note._freq_note_0() == 440 * 2**(3/12) / 2**6
    octave_offset = 1  # offset from numeric octave to notation
    for note_i in notes_list:
        freq = note_i['freq']
        note = note_i['note_name']
        octv = note_i['octave']
        errv = note_i['error']
        name = str(note).split('.')[-1] + str(octv)
        str_expected = name
        if errv:
            str_expected += " %+d cent" % errv
        str_expected += " %.3f Hz" % freq
        # print("expected: ", str_expected)
        ref1 = Note(freq)
        note_num = note + (octv + octave_offset) * 12 + errv / 100
        ref2 = Note.create(note_num)
        for ref in (ref1, ref2):
            # print(ref)
            assert round(ref.num) == note + 12 * (octv + octave_offset)
            assert int(ref.octave) == octv + octave_offset
            assert ref.name == name, str_expected
            assert repr(ref) == str_expected

def test_note_detect(wav_files):
    for filename in wav_files:
        wav_i = WaveUnifyData(filename)
        fdi = FreqDetect(wav_i.samples_mono, wav_i.sample_rate)
        fft1 = fdi.fft()
        peaks = fdi.get_peaks(fft1, -20)
        for freq in peaks:
            root_note = Note(freq)
            print(filename, ": ", root_note)

# endregion

# region exports

files_list = list(create_wav_files())
file1 = files_list[0]

waves_dict = get_waves()
waves_list = list(waves_dict.values())
wave1 = waves_list[0]

freqs_dict = get_freqs()
freqs_list = list(freqs_dict.values())
freq1 = freqs_list[0]

# endregion

# region main

if __name__ == '__main__':
    show_plots = True
    normalize_plots = 2 ** 15
    plot_length = 5000

    pytest.main()


# we will change the BGR value using a trackbar
import numpy as np
import cv2 as cv

'''
def nothing(x):
    # x is the value of current position of trackbar
    print(x)

# create a black image, a window
img = np.zeros((300, 512, 3), np.uint8)
cv2.namedWindow('image')

# create a trackbar which will change B channel
cv2.createTrackbar('B', 'image', 0, 255, nothing)
cv2.createTrackbar('G', 'image', 0, 255, nothing)
cv2.createTrackbar('R', 'image', 0, 255, nothing)

# create a switch trackbar
switch = '0: OFF\n 1:ON'
cv2.createTrackbar(switch, 'image', 0, 1, nothing)

while (1):
    cv2.imshow('image', img)
    k = cv2.waitKey(1) & 0xFF
    if k == 27:
        break

    # get the initial trackbar values
    b = cv2.getTrackbarPos('B', 'image')
    g = cv2.getTrackbarPos('G', 'image')
    r = cv2.getTrackbarPos('R', 'image')
    s = cv2.getTrackbarPos(switch, 'image')

    # use switch to control the BGR trackbar
    if s == 0:
        img[:] = 0  # nothing will change in image
    else:
        img[:] = [b, g, r]

cv2.destroyAllWindows()

'''

def nothing(x):
    print(x)

# Create a black image, a window
cv.namedWindow('image')

cv.createTrackbar('CP', 'image', 10, 400, nothing)

switch = 'color/gray'
cv.createTrackbar(switch, 'image', 0, 1, nothing)

while (1):
    img = cv.imread('lena.jpg')
    pos = cv.getTrackbarPos('CP', 'image')
    font = cv.FONT_HERSHEY_SIMPLEX
    cv.putText(img, str(pos), (50, 150), font, 6, (0, 0, 255), 10)

    k = cv.waitKey(1) & 0xFF
    if k == 27:
        break

    s = cv.getTrackbarPos(switch, 'image')

    if s == 0:
        pass
    else:
        img = cv.cvtColor(img, cv.COLOR_BGR2GRAY)

    img = cv.imshow('image', img)

cv.destroyAllWindows()


# -*- coding: utf-8 -*-
"""
Created on Thu Oct 26 13:39:38 2017

@author: Amira AYADI
"""
import os
import numpy as np
from math import *
from scipy.misc import imread
import matplotlib.pyplot as plt

img = imread('TP1.jpg')
plt.imshow(img, cmap='Greys_r')
transform = np.log(np.abs(np.fft.fft2(img))) # on prend le log que pour pouvoir voir l'image, en effet si on ne prend pas le log les valeurs seront trop grande et donc on ne verra rien
plt.imshow(transform, cmap='Greys_r' )

# Question 1 : ca donne la direction du texte

# Question 2 : creation d'une fonction descripteur

def descripteur(image):
    descripteur = []
    image = imread(image, mode ="L")
    # Trans de fourier
    image = np.abs(np.fft.fft2(image))
    shape_x = np.shape(image)[0]
    shape_y = np.shape(image)[1]
    pasdex = 3
    pasdey = 3*2
    liste_x = np.arange(0,int(shape_x/2),int(shape_x/pasdex))
    liste_y = np.arange(0,int(shape_y/2),int(shape_y/pasdey))
    # Creation descripteur
    for x in liste_x :
        for y in liste_y:
            bloc_energ=np.log(np.mean(image[x:x+int(shape_x/pasdex),y:y+int(shape_y/pasdey)]**2))
            descripteur.append(bloc_energ)
    return descripteur

# Question 3 :

def indexation(path):
    tot_descri = {}
    rep = os.listdir(path)
    for s in rep :
        #plt.imshow(image, cmap = 'Greys_r')
        image = path + s
        tot_descri[s] = descripteur(image)
    return tot_descri

path = "./BE/"
indexation(path)

def distance_manhattan(x,y):
    return sum(abs(a-b) for a,b in zip(x,y))

def image_plus_proche():
    liste=[]
    liste_nom=[]
    a_comparer = int(input(" Quelle est l'image que vous voulez comparer ? Veuillez entrer son numero \n"))
    path = "./BE/"
    if a_comparer in [1,2,3,4,5,6,7,8,9]:
        image_comp = path+"0"+str(a_comparer)+'.jpg'
    else:
        image_comp = path+str(a_comparer)+'.jpg'
    descripteur_image_comp = descripteur(image_comp)
    dico_descri = indexation(path)
    for index, key in dico_descri.items():
        liste.append(distance_manhattan(key,descripteur_image_comp))
        liste_nom.append(index)
    position=liste.index(0.0)
    del liste[position]
    del liste_nom[position]
    print("les images les plus proches sont : \n")
    for i in range(10):
        print(str(liste_nom[liste.index(min(liste))]))
        min_supp = liste.index(min(liste))
        del liste[min_supp]
        del liste_nom[min_supp]

image_plus_proche()