# Author: Jakub Szwedowicz
# Date: 09.06.2022
# e-mail: kuba.szwedowicz@gmail.com

from PyQt6.QtWidgets import QWidget, QGridLayout, QPushButton
from PyQt6.QtGui import QCursor
from PyQt6.QtCore import Qt
from typing import Dict

def create_button(text: str) -> QPushButton:
    button = QPushButton(text)
    button.setCursor(QCursor(Qt.CursorShape.PointingHandCursor))
    button.setStyleSheet("*{Border: 4px solid '#BC006C';" +
                         "Border-radius: 15px;" +
                         "font-size: 35px;" +
                         "color: 'white';" +
                         "padding: 25px 0;" +
                         "margin: 100px 200px;}" +
                         "*:hover{background: '#BC006C';}")
    return button

class MenuScreen(QWidget):
    _DEFAULT_SCREEN_SIZE = [700, 400]

    def __init__(self, parent=None):
        super(MenuScreen, self).__init__(parent)
        self.setWindowTitle('Pong')
        self.setFixedSize(MenuScreen._DEFAULT_SCREEN_SIZE[0], MenuScreen._DEFAULT_SCREEN_SIZE[1])
        self.setStyleSheet('background: #161219;')

        self._grid = QGridLayout()
        self._buttons = {'Start': create_button('Start')}
        self._grid.addWidget(self._buttons['Start'], 1, 0)

        self.setLayout(self._grid)

    def get_start_button_name(self) -> str:
        return 'Start'

    def connect_button(self, button: str, function):
        self._buttons[button].clicked.connect(function)


def remove_all_whitespaces(_str: str) -> str:
    return ''.join(_str.split())

def load_map(file_path: str, key_value_delimiter: str = '=', comment_token='#',
             remove_whitespaces: bool = True, key_filter=None, value_filter=None) -> dict:
    """
    Load key-value pairs from a given file

    :param file_path: path of input file
    :param key_value_delimiter: delimiter for each entry, line format = key{delimiter}value
    :param comment_token: signifies starting of a comment
    :param remove_whitespaces: whether to remove whitespaces from each line
    :param key_filter: a predicate to decide whether an entry corresponding to key must be included
    :param value_filter: a predicate to decide whether an entry corresponding to value must be included

    :return: dict containing key-value pairs (all as strings)
    """
    _map = {}

    try:
        with open(file_path, "r+") as f:
            for line in f.readlines():
                if comment_token:
                    comment_token_index = line.find(comment_token)
                    if comment_token_index >= 0:
                        line = line[:comment_token_index]
                if remove_whitespaces:
                    line = remove_all_whitespaces(line)
                line = line.replace('\n', '')
                if line:
                    _dat = line.split(key_value_delimiter)
                    if len(_dat) == 2:
                        k, v = _dat[0], _dat[1]
                        if k and (not key_filter or key_filter(k)) and (not value_filter or value_filter(v)):
                            _map[_dat[0]] = _dat[1]
    except Exception as e:
        print(f"Error while loading data from file {file_path}: {e}")

    return _map


#!/usr/bin/env python3

import sys, os, re
from get_file_data import get_file_data

# Extract annotation and sequence sections using regexps

if len(sys.argv) != 2:
  sys.stderr.write('Usage: {} <genbankfile>\n'.format(sys.argv[0]))
  exit(1)

filename = sys.argv[1]
record = get_file_data(filename,'//\n')[0] # get first record
annotation = None
dna = None
m = re.search('^(LOCUS.*ORIGIN\s*\n)(.*)',record, flags=re.M|re.S)
if m:
  annotation = m.group(1)
  dna = m.group(2)
else:
  sys.stderr.write('{}: Cannot separate annot./seq. in {}\n'
                    .format(sys.argv[0],filename))
  exit(1)
print('annotation:\n{}DNA:\n{}'.format(annotation,dna),end='')


import numpy as np
import pylab as pl
import math
data=np.loadtxt('WHM_Results_with_drugs.txt')
b=0
#Loop to check how long episode lasts, and get max parasitaemia
for t in range(0, 399):
    if data[t,1] > 0.00001:
    	T = t;
    if data[t,2] > b:
    	b = data[t,2]
    if data[t,1] < 0.00001:
    	break

#now load PK data
dataPK=np.loadtxt('WHM_Results_with_PKPD.txt')
dimm=dataPK.shape[0] - 1 #I've placed some useful variables in the last row
#print(dimm)

fev = dataPK[dimm,3] #Timing of first fever, in increments of dt
dt = dataPK[dimm,2] #dt (in hours)
delta = dt * (1/24.0) * dataPK[dimm,4] #Start of treatment, converted into days
#print(fev)
#print(delta)
pl.plot(data[:,0],data[:,1], marker ='o')
pl.plot(data[:,0],data[:,2], marker ='o')

#Microscopy threshold, higher (more realistic?) than used in Molineaux et al., Parasitology 122 379 (2001)
pl.plot([0,(2 * T) + 6],[40,40])
pl.plot([0,(2 * T) + 6],[fev,fev],linestyle='-.')#Fever Threshold
pl.plot([delta,delta],[0.00001 , b * 1.4])#Treatment starts
pl.plot([delta+1,delta+1],[0.00001 , b * 1.4])#24 hrs after treatment starts
pl.plot([delta+28,delta+28],[0.00001 , b * 1.4])#28 day check, if we get that far!
pl.plot(dataPK[0:dimm,0],dataPK[0:dimm,1])#Artemether
pl.plot(dataPK[0:dimm,0],dataPK[0:dimm,3])#Lumefantrine

pl.xlabel('Time (Days)')
pl.ylabel('Parasitaemia [PRBCs / $\mu L$]')
pl.title('Falciparum infection treated with Artemether & Lumefantrine')
pl.yscale('log')

pl.xlim(0.0, 6 + (2 * T))
pl.ylim(0.00001, b * 1.5) #Scaling of 1.5 just provides some space above 1st peak

pl.savefig("WHM_one_run_withPK.pdf", format='pdf')
pl.show()


from tf.transformations import euler_from_quaternion
import rospy
from gazebo_msgs.srv import GetModelState
from std_msgs.msg import Float32

_model_state = rospy.ServiceProxy('/gazebo/get_model_state', GetModelState)

def get_model_state():
    rospy.wait_for_service("/gazebo/get_model_state")
    try:
        return _model_state('jackal', 'world')
    except (rospy.ServiceException):
        rospy.logwarn("/gazebo/get_model_state service call failed")

rospy.init_node("transform_euler", anonymous=True)
theta_publisher = rospy.Publisher('/theta', Float32, queue_size=1)

r = rospy.Rate(50) # define rate here

while not rospy.is_shutdown():

    ori= get_model_state().pose.orientation
    eul=euler_from_quaternion([ori.x,ori.y,ori.z,ori.w])
    # print(eul[2])
    msg=Float32()
    msg.data=eul[2]
    theta_publisher.publish(msg)
    r.sleep()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2019/8/31 17:07
# @Author  : SeniorZhu1994
# @Email   : SeniorZhu1994@
# @Site    :
# @File    : 求字符串子串中个数最多的子串.py
# @Software: PyCharm
# @Describe:

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2019/8/29 10:40
# @Author  : SeniorZhu1994
# @Email   : SeniorZhu1994@foxmail.com
# @Site    :
# @File    : 求字符串所有子串-连续的.py
# @Software: PyCharm
# @Describe:

def cutStr2(str):
    ls_cut = []
    for i in range(len(str)): # 从字符串的第 i个元素开始向后截取
        for j in range(len(str)-i): # 以第 i个元素开头的截取所有字符串的个数为 len(str)-i个
            ls_cut.append(str[i:i+j+1])
    # ls_cut = list(set(ls_cut))

    # print(ls_cut)
    return ls_cut

def cutStr(str):
    ls_cut = []
    for i in range(len(str)):
        for j in range(i + 1, len(str)+1):
            sub = str[i:j]
            ls_cut.append(sub)
    return ls_cut

if __name__ == '__main__':
    str_in = input()
    ls_cut = cutStr(str_in)
    count = 0
    for item in ls_cut:
        if ls_cut.count(item)>count:
            count = ls_cut.count(item)
            # print(item)
    print(count)

求字符串子串中个数最多的子串.py

求字符串子串中个数最多的子串.py


import torch.nn as nn

from utils import ConvolutionalBlock, Downscaling, Upscaling

class UNet(nn.Module):
    """
    Implementation of the UNet network architecture. This is a modification of the
    network proposed https://arxiv.org/pdf/1505.04597.pdf in the context of
    semantic segmentation.

    This implementation is adapted to work as a surrogate model for simulating
    fluid flow against an airfoil depending on the airfoil's shape.

    :params:
    n_channels (int): number of channels in the input data.
    output_dim (int): dimension of the network's output.
    """

    def __init__(self, n_channels: int = 1, output_dim: int = 3):

        super(UNet, self).__init__()

        self.n_channels = n_channels
        self.output_dim = output_dim

        self.input_conv = ConvolutionalBlock(n_channels, 16)
        self.downscaling_1 = Downscaling(16, 32)
        self.downscaling_2 = Downscaling(32, 64)
        self.downscaling_3 = Downscaling(64, 128)
        self.downscaling_4 = Downscaling(128, 256)
        self.upscaling_1 = Upscaling(256, 128)
        self.upscaling_2 = Upscaling(128, 64)
        self.upscaling_3 = Upscaling(64, 32)
        self.upscaling_4 = Upscaling(32, 16)
        self.output_conv = nn.Conv2d(16, self.output_dim, kernel_size=1)

    def forward(self, x):

        # Downscaling part of UNet
        x1 = self.input_conv(x)
        x2 = self.downscaling_1(x1)
        x3 = self.downscaling_2(x2)
        x4 = self.downscaling_3(x3)
        x5 = self.downscaling_4(x4)

        # Upscaling part of UNet
        x6 = self.upscaling_1(x5, x4)
        x7 = self.upscaling_2(x6, x3)
        x8 = self.upscaling_3(x7, x2)
        x9 = self.upscaling_4(x8, x1)
        x10 = self.output_conv(x9)

        output = x10 * (1 - x)

        return output


import requests
from flask import Flask, Response, request
from bot_parser import parse_message

app = Flask(__name__)

TOKEN = '1023120359:AAHSXx5ayrjLbJRehpozKJSECOBV4ZQNTzI'

@app.route('/sanity')
def sanity():
    return "Server is running YAZAN ;)"

def establish_connection():
    TELEGRAM_INIT_WEBHOOK_URL = 'https://api.telegram.org/bot{}/setWebhook?url=https://1e22489e.ngrok.io/message'.format(TOKEN)
    requests.get(TELEGRAM_INIT_WEBHOOK_URL)

@app.route('/message', methods=["POST"])
def handle_message():
    req = request.get_json()
    try:
        message = req['message']['text']

        response = parse_message(message)

        chat_id = req['message']['chat']['id']
        res = requests.get("https://api.telegram.org/bot{}/sendMessage?chat_id={}&text={}"
                           .format(TOKEN, chat_id, response))
        return Response("success")
    except:
        return Response("Error")

if __name__ == '__main__':
    establish_connection()
    app.run(port=5002)


import requests
from bs4 import BeautifulSoup

url="https://search.naver.com/search.naver?where=nexearch&sm=tab_jum&query=%EC%BD%94%EB%A1%9C%EB%82%98"
r=requests.get(url)

soup=BeautifulSoup(r.text, "html.parser")
data=soup.select("a.news_tit")

for i in data:
    print(i.get_text())


import numpy as np
import os
from sklearn.datasets import fetch_openml
import matplotlib.pyplot as plt

# Create the directory for output images
directory = './fig/'
if not os.path.isdir(directory):
    os.makedirs(directory)

# Fetch the datasets from openml
mnist = fetch_openml('mnist_784')

"""
Part1: PCA
Q2. ~ Q4.
"""
# Q2.
# Obtain the indices of all "5".
indices = np.where(mnist['target'].to_numpy() == '5')[0]
data_5 = mnist['data'].to_numpy()[indices]
mean_5 = np.mean(data_5, axis=0)
# Transpose the dataset to the form: X = [x1, x2, x3, ..., xN]
data_5_centered = (data_5 - mean_5).T
# Calculate the covariance matrix
covariance_mat = np.cov(data_5_centered)
# Calculate eigenvalue and eigenvector
eigen_val, eigen_vec = np.linalg.eig(covariance_mat)
# Eigenvector with each rows
eigen_vec = eigen_vec.T
# Sorting the eigenvector by its eigenvalue in descending order.
eigen_val_des = eigen_val[np.argsort(eigen_val)[::-1]]
eigen_vec_des = eigen_vec[np.argsort(eigen_val)[::-1]]
# Show the first 3 largest eigenvectors.
fig = plt.figure(figsize=(9, 3))
for i in range(3):
    plt.subplot(131 + i)
    plt.imshow((eigen_vec_des[i]).real.reshape(28, 28), 'gray')
    plt.title("λ = {0}".format(eigen_val_des[i].real))
    plt.axis('off')
plt.savefig(os.path.join(directory, 'Q2.png'))

# Q3.
origin_first_5 = data_5[0]
origin_first_5_mean = origin_first_5 - mean_5
images_5 = [origin_first_5]
bases = [3, 10, 30, 100]
fig = plt.figure(figsize=(15, 3))
plt.subplot(151)
plt.imshow(origin_first_5.reshape(28, 28), 'gray')
plt.title('Original 5')
plt.axis('off')
for i, n in enumerate(bases):
    plt.subplot(152 + i)
    # Obtain the first n largest eigenvectors and corresponding coefficients.
    eigen_vec_des_top_n = eigen_vec_des[:n]
    coef_5_top_n = np.dot(eigen_vec_des_top_n, origin_first_5_mean)
    # Reconstruct the signal by the eigenvectors.
    reconstruct_n_base = np.dot(eigen_vec_des_top_n.T, coef_5_top_n)
    plt.imshow((reconstruct_n_base + mean_5).real.reshape(28, 28), 'gray')
    plt.title("5 with {0}d".format(n))
    plt.axis('off')
plt.savefig(os.path.join(directory, 'Q3.png'))

# Q4.
first_10000_target = mnist['target'].to_numpy()[:10000]
first_10000_data = mnist['data'].to_numpy()[:10000]
# Obtain the indices of all "1", "3", "6".
indices = np.where((first_10000_target == '1') | (first_10000_target == '3') | (first_10000_target == '6'))[0]
data = first_10000_data[indices]
mean = np.mean(data, axis=0)
# Transpose the dataset to the form: X = [x1, x2, x3, ..., xN]
data_centered = (data - mean).T
# Calculate eigenvalue and eigenvector
covariance_mat = np.cov(data_centered)
# Calculate eigenvalue and eigenvector
eigen_val, eigen_vec = np.linalg.eig(covariance_mat)
# Eigenvector with each rows
eigen_vec = eigen_vec.T
# Sorting the eigenvector by its eigenvalue in descending order.
eigen_val_des = eigen_val[np.argsort(eigen_val)[::-1]]
eigen_vec_des = eigen_vec[np.argsort(eigen_val)[::-1]]
# Store the projection points (x, y) to the first 2 largest eigenvectors.
points_x = {'1': [], '3': [], '6': []}
points_y = {'1': [], '3': [], '6': []}
colors = {'1': 'r', '3': 'g', '6': 'b'}
for i, dc in enumerate(data_centered.T):
    eigen_vec_des_top_2 = eigen_vec_des[:2]
    coef_top_2 = np.dot(eigen_vec_des_top_2, dc)
    points_x[first_10000_target[indices][i]].append(coef_top_2[0].real)
    points_y[first_10000_target[indices][i]].append(coef_top_2[1].real)
fig = plt.figure(figsize=(6, 4))
for key,values in colors.items():
    plt.scatter(points_x[key], points_y[key], color=values, label=key)
plt.legend()
plt.savefig(os.path.join(directory, 'Q4.png'))

"""
Part2: OMP
Q5. ~ Q6.
"""
# Q5.
# Obtain training basis(pre-normalized)
basis = mnist['data'].to_numpy()[:10000]
length = np.linalg.norm(basis, axis=1)
basis = (basis.T / length).T
# Define the sparsity is 5.
sparsity = 5
origin_signal = mnist['data'].to_numpy()[10000]
# Bases and coefficients we choose.
sparse_basis, coefficients = [], []
# Residual sidgnal equals to original signal(Need deep copy)
residue_signal = origin_signal.copy()
for k in range(sparsity):
    max_product, max_base, max_index = 0, [], 0
    # Find the base vector that cause maximum product with residual signal.
    for i, base in enumerate(basis):
        product = abs(np.dot(base, residue_signal))
        if product > max_product:
            max_product, max_base, max_index = product, base, i
    # Calculate the coefficient vector by the pseudo inerse: (M^T*M)^(-1)M^T*x
    sparse_basis.append(max_base)
    M = np.array(sparse_basis).T
    pseudo_inverse = np.linalg.pinv(M.T @ M)
    coefficients = (pseudo_inverse @ M.T) @ origin_signal
    # Update residual signal.
    residue_signal = origin_signal - (M @ coefficients)
    # Delete the basis we choose.
    basis = np.delete(basis, max_index, axis=0)
# Show the first 5 largest bases we choose.
fig = plt.figure(figsize=(15, 3))
for i, base in enumerate(sparse_basis):
    plt.subplot(151 + i)
    plt.imshow(base.reshape(28, 28), 'gray')
    plt.title("Base {0}".format(i + 1))
    plt.axis('off')
plt.savefig(os.path.join(directory, 'Q5.png'))

# Q6.
origin_signal = mnist['data'].to_numpy()[10001]
# Define the sparsity is 5, 10, 40 and 200.
sparsity = [5, 10, 40, 200]
fig = plt.figure(figsize=(15, 3))
plt.subplot(151)
plt.imshow(origin_signal.reshape(28, 28), 'gray')
plt.title('L-2=0')
plt.axis('off')
for index, s in enumerate(sparsity):
    # Obtain training basis(pre-normalized)
    basis = mnist['data'].to_numpy()[:10000]
    length = np.linalg.norm(basis, axis=1)
    basis = (basis.T / length).T
    # Bases and coefficients we choose.
    sparse_basis, coefficients = [], []
    # Residual sidgnal equals to original signal(Need deep copy)
    residue_signal = origin_signal.copy()
    for k in range(s):
        max_product, max_base, max_index = 0, [], 0
        # Find the base vector that cause maximum product with residual signal.
        for i, base in enumerate(basis):
            product = abs(np.dot(base, residue_signal))
            if product > max_product:
                max_product, max_base, max_index = product, base, i
        # Calculate the coefficient vector by the pseudo inerse: (M^T*M)^(-1)M^T*x
        sparse_basis.append(max_base)
        M = np.array(sparse_basis).T
        pseudo_inverse = np.linalg.pinv(M.T @ M)
        coefficients = (pseudo_inverse @ M.T) @ origin_signal
        # Update residual signal.
        residue_signal = origin_signal - (M @ coefficients)
        # Delete the basis we choose.
        basis = np.delete(basis, max_index, axis=0)
    L2_norm = np.linalg.norm(residue_signal)
    plt.subplot(152 + index)
    plt.imshow((np.array(sparse_basis).T @ np.array(coefficients)).reshape(28, 28), 'gray')
    plt.title("L-2={0}".format(L2_norm))
    plt.axis('off')
plt.savefig(os.path.join(directory, 'Q6.png'))


from typing import List

class Solution:
    def removeCoveredIntervals(self, intervals: List[List[int]]) -> int:
        # sort the list by start time
        # then traverse linearly
        intervals = sorted(intervals, key=lambda s: (s[0], -s[1]))
        print(intervals)
        count = 0
        prev = -1

        # since all intervals are sorted by earliest start and latest end,
        # the interval that comes before must be able to covered the next
        # as long as the end time of the earlier is later than the end time
        # of the next

        for int in intervals:
            if int[1] > prev:
                # Only count ones that aren't covered
                prev = int[1]
                count += 1

        return count

sol = Solution()
intervals = [[3, 10], [4, 10], [5, 11]]

print(sol.removeCoveredIntervals(intervals))


from __future__ import print_function
import sys

from optparse import OptionParser

from androguard.cli import androdis_main

option_0 = {
    'name': ('-i', '--input'),
    'help': 'file : use this filename (DEX/ODEX)',
    'nargs': 1
}
option_1 = {
    'name': ('-o', '--offset'),
    'help': 'offset to disassemble',
    'nargs': 1
}
option_2 = {'name': ('-s', '--size'), 'help': 'size', 'nargs': 1}

options = [option_0, option_1, option_2]

def main(options, arguments):
    if options.input and options.offset and options.size:
        offset = int(options.offset, 0)
        size = int(options.size, 0)
        androdis_main(offset, size, options.input)

if __name__ == "__main__":
    parser = OptionParser()
    for option in options:
        param = option['name']
        del option['name']
        parser.add_option(*param, **option)

    options, arguments = parser.parse_args()
    sys.argv[:] = arguments
    main(options, arguments)


import sys
from scipy.ndimage import gaussian_filter1d

num_1 = sys.argv[1]
num_2 = sys.argv[2]
num_3 = sys.argv[3]

out_1 = gaussian_filter1d([1, 2, 3], 3)

print(out_1)


import rospy
import time
from std_msgs.msg import Int32

def talker():
    pub = rospy.Publisher('test/int', Int32, queue_size=1)
    pub2 = rospy.Publisher('test/nint', Int32, queue_size=1)
    rospy.init_node('talker', anonymous=True)
    rate = rospy.Rate(4) # 10hz
    i = 0
    while not rospy.is_shutdown():

        # hello_str = "hello world %s" % rospy.get_time()
        # rospy.loginfo(hello_str)
        pub.publish(i)
        pub2.publish(-1 * i)
        i += 1
        if i > 10:
            i = 0
        rate.sleep()

if __name__ == '__main__':
    try:
        talker()
    except rospy.ROSInterruptException:
        pass


import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D, concatenate, Conv2DTranspose, \
    BatchNormalization, Dropout, Lambda
from matplotlib import pyplot as plt

from src.image_loading import sample_image_generator
from src.training_data_generator import sample_generator

def encoder_block(x, filters, kernel_size=3):
    x = Conv2D(filters=filters, kernel_size=kernel_size, activation='relu', kernel_initializer='he_normal',
               padding='same')(x)
    x = Conv2D(filters=filters, kernel_size=kernel_size, activation='relu', kernel_initializer='he_normal',
               padding='same')(x)
    p = MaxPooling2D(2)(x)
    return p, x

def decoder_block(x, skip_connection_features, filters, kernel_size=3):
    x = Conv2DTranspose(filters=filters, kernel_size=3, strides=2, padding='same')(x)
    # Blog about deconvolution vs conv_transpose https: // distill.pub / 2016 / deconv - checkerboard /
    x = concatenate([x, skip_connection_features])
    x = Conv2D(filters=filters, kernel_size=kernel_size, activation='relu', kernel_initializer='he_normal',
               padding='same')(x)
    x = Conv2D(filters=filters, kernel_size=kernel_size, activation='relu', kernel_initializer='he_normal',
               padding='same')(x)
    return x

def unet(img_channels):
    # Shallow U-Net which is enough for detecting angle brackets as they are a relatively small feature
    inputs = Input((None, None, img_channels))
    x = inputs

    p_1, skip_1 = encoder_block(x, 16, 3)
    p_2, skip_2 = encoder_block(p_1, 32, 3)

    # bottleneck
    b = Conv2D(filters=64, kernel_size=3, activation='relu', kernel_initializer='he_normal', padding='same')(p_2)
    b = Conv2D(filters=64, kernel_size=3, activation='relu', kernel_initializer='he_normal', padding='same')(b)

    up_2 = decoder_block(b, skip_2, 32, 3)
    up_1 = decoder_block(up_2, skip_1, 16, 3)

    out = Conv2D(filters=2, kernel_size=1, activation='softmax', padding='same')(up_1)
    model = Model(inputs=inputs, outputs=out)
    return model

if __name__ == '__main__':
    keras.backend.clear_session()

    # Build model
    img_size = (512, 512)
    num_classes = 2
    epochs = 2
    steps_per_epoch = 5  # 15
    # model_name = "model_2_large_augmented_noise_15"
    # model_name = "model_shallow_30"
    # model_name = "model_shallow_flexible_input"
    model_name = "refactor"
    path_trained_model = f"../data/trained_models/{model_name}.h5"
    model = unet(1)
    model.summary()

    train_dataset = tf.data.Dataset.from_generator(sample_generator, output_signature=(
        tf.TensorSpec(shape=(*img_size, 1), dtype=tf.float32), tf.TensorSpec(shape=(*img_size, 2), dtype=tf.float32)))
    train_dataset = train_dataset.batch(4)
    train_dataset = train_dataset.prefetch(buffer_size=tf.data.AUTOTUNE)

    val_dataset = tf.data.Dataset.from_generator(sample_image_generator, output_signature=(
        tf.TensorSpec(shape=(*img_size, 1), dtype=tf.float32), tf.TensorSpec(shape=(*img_size, 2), dtype=tf.float32)))
    val_dataset = val_dataset.batch(4)
    val_dataset = val_dataset.prefetch(buffer_size=tf.data.AUTOTUNE)

    model.compile(optimizer="adam", loss="binary_crossentropy",
                  metrics=[tf.keras.metrics.Accuracy(), tf.keras.metrics.IoU(num_classes=2, target_class_ids=[0])])

    # model = keras.models.load_model(path_trained_model)
    model_history = model.fit(train_dataset, epochs=epochs, steps_per_epoch=steps_per_epoch,
                              validation_data=val_dataset)
    model.save(path_trained_model)

    if True:
        print(model_history.history.keys())
        fig, ax = plt.subplots(3, 1, figsize=(10, 10))
        ax[0].set_title('Loss')
        ax[0].plot(model_history.history['loss'], label='train')
        ax[0].plot(model_history.history['val_loss'], label='validation')
        ax[0].set_xlabel('Epoch')
        ax[0].set_ylabel('Binary Crossentropy Loss')
        ax[0].legend()

        ax[1].set_title('IoU')
        ax[1].plot(model_history.history['io_u'], label='train')
        ax[1].plot(model_history.history['val_io_u'], label='validation')
        ax[1].set_xlabel('Epoch')
        ax[1].set_ylabel('IoU')
        ax[1].legend()

        ax[2].set_title('Accuracy')
        ax[2].plot(model_history.history['accuracy'], label='train')
        ax[2].plot(model_history.history['val_accuracy'], label='validation')
        ax[2].set_xlabel('Epoch')
        ax[2].set_ylabel('Accuracy')
        ax[2].legend()
        plt.savefig(f"../data/plots/{model_name}_loss.png")
        plt.show()

    fig, ax = plt.subplots(3, 3, figsize=(10, 10))
    for img, label in train_dataset.take(2):
        pred = model.predict(img)
        for ix in range(2):
            ax[ix][0].imshow(img[ix, :, :], cmap="gray")
            ax[ix][0].set_title('photo')
            ax[ix][1].imshow(label[ix, :, :, 0], cmap='gray')
            ax[ix][1].set_title('label')
            ax[ix][2].imshow(pred[ix, :, :, 0] >= pred[ix, :, :, 1], cmap='gray')
            # ax[ix][2].imshow(pred[ix, :, :, 0], cmap='gray')
            ax[ix][2].set_title('prediction')

    for img, label in val_dataset.take(2):
        pred = model.predict(img)
        for ix in range(0, 1):
            ax[ix + 2][0].imshow(img[ix, :, :], cmap="gray")
            ax[ix + 2][0].set_title('photo')
            ax[ix + 2][1].imshow(label[ix, :, :, 0], cmap='gray')
            ax[ix + 2][1].set_title('label')
            ax[ix + 2][2].imshow(pred[ix, :, :, 0] >= pred[ix, :, :, 1], cmap='gray')
            # ax[ix + 2][2].imshow(pred[ix, :, :, 0], cmap='gray')
            ax[ix + 2][2].set_title('prediction')
    fig.tight_layout()
    plt.savefig(f"../data/plots/{model_name}_samples.png")
    plt.show()


#import sys
#input = sys.stdin.readline

#import sys
#input = sys.stdin.readline
def main():
    N = int( input())
    P = list( map( int, input().split()))
    ANS = []
    Z = [0]*(N+1)
    for i, p in enumerate(P):
        Z[p] = i
    permuted = [False]*N
    for n in range(1,N+1):
        if Z[n] == n-1:
            continue
        K = Z[n]
        for i in range(K-1,n-2,-1):
            if permuted[i]:
                print(-1)
                return
            ANS.append(i+1)
            Z[P[i+1]] = i
            Z[P[i]] = i+1
            P[i], P[i+1] = P[i+1], P[i]
            permuted[i] = True

    s = 0
    for p in permuted:
        if p:
            s += 1

    if s < N-1:
        print(-1)
        return
    print("\n".join(map(str, ANS)))

if __name__ == '__main__':
    main()


from dataclasses import dataclass, field
from typing import List
from .blacklist import Blacklist
from .blacklist_ref import BlacklistRef
from .containment_aggregation_structure import ContainmentAggregationStructure

__NAMESPACE__ = "http://www.netex.org.uk/netex"

@dataclass
class BlacklistsRelStructure(ContainmentAggregationStructure):
    class Meta:
        name = "blacklists_RelStructure"

    blacklist_ref_or_blacklist: List[object] = field(
        default_factory=list,
        metadata={
            "type": "Elements",
            "choices": (
                {
                    "name": "BlacklistRef",
                    "type": BlacklistRef,
                    "namespace": "http://www.netex.org.uk/netex",
                },
                {
                    "name": "Blacklist",
                    "type": Blacklist,
                    "namespace": "http://www.netex.org.uk/netex",
                },
            ),
        }
    )


#!/usr/bin/env python

from Bio import SeqIO
import sys
import os
import csv

usage = 'python filter_by_b6.py align_R1.b6 align_R2.b6 orig_R1.fastq orig_R2.fastq filtered_prefix'

# Will add '_R1' and '_R2' to the end of the filtered prefix

if len(sys.argv) < 5:
	print('You must supply the R1 and R2 alignment files, original FASTQs, and a prefix for the filtered output\n\n%s\n' % usage)
	sys.exit()

inb6_R1 = sys.argv[1]
inb6_R2 = sys.argv[2]
infastq_R1 = sys.argv[3]
infastq_R2 = sys.argv[4]
outfq = sys.argv[5]

host_read_ids = []
with open(inb6_R1, 'r') as inf1, open(inb6_R2, 'r') as inf2:
	tabreader1 = csv.reader(inf1, delimiter='\t')
	for line in tabreader1:
		read_id = str(line[0])
#		print(read_id)
		host_read_ids.append(read_id)
	tabreader2 = csv.reader(inf2, delimiter='\t')
	for line in tabreader2:
		read_id = str(line[0])
		host_read_ids.append(read_id)
host_read_set = set(host_read_ids)
print('\n...filtering %s read pairs from FASTQ files...\n' % len(host_read_set))
# Filter the FASTQs to exclude reads from either b6 files (to maintain paired reads)
outfq1 = '_'.join([outfq, 'R1.fastq'])
outfq2 = '_'.join([outfq, 'R2.fastq'])
with open(outfq1, 'w') as outf1, open(infastq_R1, 'r') as infq1:
#	keep_seqs1 = [record for record in SeqIO.parse(infq1, 'fastq') if str(record.id).split(' ')[0] not in host_read_set]
#	SeqIO.write(keep_seqs1, outf1, 'fastq')
	# Previous two lines work the same as below line but consume massive RAM
	[SeqIO.write(record, outf1, 'fastq') for record in SeqIO.parse(infq1, 'fastq') if str(record.id).split(' ')[0] not in host_read_set]
with open(outfq2, 'w') as outf2, open(infastq_R2, 'r') as infq2:
#	keep_seqs2 = [record for record in SeqIO.parse(infq2, 'fastq') if str(record.id).split(' ')[0] not in host_read_set]
#	SeqIO.write(keep_seqs2, outf2, 'fastq')
	# Previous two lines work the same as below line but consume massive RAM
	[SeqIO.write(record, outf2, 'fastq') for record in SeqIO.parse(infq2, 'fastq') if str(record.id).split(' ')[0] not in host_read_set]


# https://www.geeksforgeeks.org/k-smallest-elements-order-using-o1-extra-space/
# You need to use constant space.

def kSmallestElements(a, k):
    n = len(a)
    for i in range(k, n):
        maxIndex = 0
        for j in range(1, k):
            if a[j] > a[maxIndex]:
                maxIndex = j
        if a[maxIndex] > a[i]:
            x = a[maxIndex]
            for j in range(maxIndex, k):
                a[j] = a[j+1]
            a[k], a[i] = a[i], x
    return a[:k]

a = [9, 3, 1, 7, 2, 4]
k = 3
print(kSmallestElements(a, k))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jun 21 18:34:51 2018

@author: tomtop
"""

import sys, os;
from natsort import natsorted;
from PyQt5.QtWidgets import QApplication, QWidget, QPushButton,\
                            QGroupBox, QDialog,\
                            QVBoxLayout, QGridLayout, QLabel,\
                            QLineEdit, QTabWidget;
from PyQt5.QtCore import QCoreApplication;
import ClearMap.Tomek_Utilities as ut;
import Parameters_Raw as p;

class createObject() :

    def __init__(self) :
        pass;

class selectionMenu(QDialog):

   def __init__(self, folders, parent=None):
      super(selectionMenu, self).__init__(parent)

      self.title = 'Clearmap GUI v1.0'
      self.right = 10;
      self.left = 10;
      self.width = 320;
      self.height = 100;
      self.toggle = None;

      self.folders = folders;
      self.number = len(self.folders);

      self.buttons2Create = ["Stiching","Resamp_Auto","Resamp_cFOS",\
                            "Align_Auto","Align_Template","Cell_Detect"\
                            ,"Heatmap","cFOS_for_Annotation","all","none"];

      self.textboxes2Create = ["orient","bgthresh","csthresh","cfosX","cfosY","cfosZ"];

      self.initUI();

   def initUI(self) :

      self.setWindowTitle(self.title);
      self.setGeometry(self.left,self.right,self.width,self.height);

      self.mainLayout = QVBoxLayout();

      self.tabs = QTabWidget();

      self.tab1 = QWidget();
      self.tabs.addTab(self.tab1,"Operations");

      self.createTab1Layout();
      self.tab1.layout = QVBoxLayout();
      self.tab1.layout.addWidget(self.GroupBoxTab1);
      self.tab1.setLayout(self.tab1.layout)

      self.tab2 = QWidget();
      self.tabs.addTab(self.tab2,"Parameters");

      self.createTab2Layout();
      self.tab2.layout = QVBoxLayout();
      self.tab2.layout.addWidget(self.GroupBoxTab2);
      self.tab2.setLayout(self.tab2.layout)

      self.mainLayout.addWidget(self.tabs);
      self.setLayout(self.mainLayout);

      self.show();

   def createTab1Layout(self) :

      self.GroupBoxTab1 = QGroupBox("Folders in the Working Directory");
      layout = QGridLayout();

      self.textTab1 = {};
      self.buttons = {};

      self.checked = {};

      y = 0;

      for n,folder in natsorted(enumerate(self.folders)) :

          self.textTab1[folder] = createObject();
          self.textTab1[folder].text = QLabel(folder);
          layout.addWidget(self.textTab1[folder].text, n, y);
          self.buttons[folder] = createObject();
          self.buttons[folder].row = n;

          for button in self.buttons2Create :

              setattr(self.buttons[folder], button, QPushButton(button));
              attr = getattr(self.buttons[folder], button);
              attr.setAutoDefault(False);
              attr.setCheckable(True);

              if button == 'all' :
                  attr.clicked.connect(lambda state, x=folder, y=button\
                                       : self.toggleAllHandler(x,y));

              elif button == 'none' :
                  attr.clicked.connect(lambda state, x=folder, y=button\
                                       : self.toggleAllHandler(x,y));

              layout.addWidget(attr, n, y+1);

              y+=1;

          y = 0;

      self.ApplyButton = QPushButton("Apply");
      self.ApplyButton.setAutoDefault(False);
      self.ApplyButton.setCheckable(True);
      layout.addWidget(self.ApplyButton, self.number+1, len(self.buttons2Create));
      self.ApplyButton.clicked.connect(self.applyChanges);

      self.QuitButton = QPushButton("Quit");
      self.QuitButton.setAutoDefault(False);
      self.QuitButton.setCheckable(True);
      layout.addWidget(self.QuitButton, self.number+1, len(self.buttons2Create)-1);
      self.QuitButton.clicked.connect(self.quitApplication);

      self.GroupBoxTab1.setLayout(layout);

   def createTab2Layout(self) :

      self.GroupBoxTab2 = QGroupBox("Folders in the Working Directory");
      layout = QGridLayout();

      self.textTab2 = {};
      self.textbox = {};

      self.bgthresholds = {};
      self.csthresholds = {};
      self.orients = {};
      self.cfosX = {};
      self.cfosY = {};
      self.cfosZ = {};

      for n,t in enumerate(self.textboxes2Create) :

          label = QLabel(t);

          layout.addWidget(label, 0, n+1);
          layout.setColumnStretch(0, 1);

      y = 0;

      for n,folder in natsorted(enumerate(self.folders)) :

          self.textTab2[folder] = createObject();
          self.textTab2[folder].text = QLabel(folder);
          layout.addWidget(self.textTab2[folder].text, n+1, y);
          self.textbox[folder] = createObject();
          self.textbox[folder].row = n+1;

          for textbox in self.textboxes2Create :

              setattr(self.textbox[folder], textbox, QLineEdit());
              attr = getattr(self.textbox[folder], textbox);

              if textbox == "bgthresh" :

                  attr.setFixedWidth(40);
                  attr.setText("{0}".format(p.detectCellShapeParameter["threshold"]));
                  attr.setToolTip("Background Threshold");

              elif textbox == "csthresh" :

                  attr.setFixedWidth(60);
                  attr.setText("{0}".format(p.thresholdPointParameter["threshold"]));
                  attr.setToolTip("Cell Size Threshold");

              elif textbox == "orient" :

                  attr.setFixedWidth(60);
                  attr.setText("{0}".format(p.FinalOrientation));
                  attr.setToolTip("Brain Orientation");

              elif textbox == "cfosX" :

                  attr.setFixedWidth(100);
                  attr.setText("{0}".format(p.cFosDetectionRange["x"]));
                  attr.setToolTip("cFOS X");

              elif textbox == "cfosY" :

                  attr.setFixedWidth(100);
                  attr.setText("{0}".format(p.cFosDetectionRange["y"]));
                  attr.setToolTip("cFOS Y");

              elif textbox == "cfosZ" :

                  attr.setFixedWidth(100);
                  attr.setText("{0}".format(p.cFosDetectionRange["z"]));
                  attr.setToolTip("cFOS Z");

              layout.addWidget(attr, n+1, y+1);

              y+=1;

          y = 0;

          layout.setColumnStretch(n+1, 1);

      self.GroupBoxTab2.setAlignment(0);
      self.GroupBoxTab2.setLayout(layout);

   def toggleAllHandler(self,folder,button) :

      attr = getattr(self.buttons[folder],button);
      row = self.buttons[folder].row;

      if attr.isChecked():

          if button == "all" :
              self.toggle = True;
          elif button == "none" :
              self.toggle = False;
          self.toggleAll(row);

      else:
          pass;

   def toggleAll(self,row) :

      for key in self.buttons.keys() :
          if self.buttons[key].row == row :
              for button in self.buttons2Create :
                  attr = getattr(self.buttons[key],button);
                  if button != "all" and button != "none" :
                       attr.setChecked(self.toggle);
                  if button == "all" or button == "none" :
                       attr.setChecked(False);

   def applyChanges(self) :

      if self.ApplyButton.isChecked() :

          for key,value in zip(self.textbox.keys(),self.textbox.values()) :

              attr = getattr(value,"bgthresh");
              bgthresh = attr.text();
              self.bgthresholds[key] = eval(bgthresh);

              attr = getattr(value,"csthresh");
              csthresh = attr.text();
              self.csthresholds[key] = eval(csthresh);

              attr = getattr(value,"orient");
              orient = attr.text();
              self.orients[key] = eval(orient);

              attr = getattr(value,"cfosX");
              cfosX = attr.text();
              if cfosX == "<built-in function all>" :
                  self.cfosX[key] = "all";
              else :
                  self.cfosX[key] = eval(cfosX);

              attr = getattr(value,"cfosY");
              cfosY = attr.text();
              if cfosY == "<built-in function all>" :
                  self.cfosY[key] = "all";
              else :
                  self.cfosY[key] = eval(cfosY);

              attr = getattr(value,"cfosZ");
              cfosZ = attr.text();
              if cfosZ == "<built-in function all>" :
                  self.cfosZ[key] = "all";
              else :
                  self.cfosZ[key] = eval(cfosZ);

          for key,value in zip(self.buttons.keys(),self.buttons.values()) :
              temp = "";
              for buttons in self.buttons2Create :
                  if buttons != "all" and buttons != "none" and buttons != "thresh" :
                      attr = getattr(value,buttons);
                      if attr.isChecked() :
                          temp+="T";
                      else :
                          temp+="F";
              self.checked[key] = temp;
              temp = "";

          QCoreApplication.instance().quit();

   def quitApplication(self) :

      if self.QuitButton.isChecked() :
          QCoreApplication.instance().quit();

def main(folders):

    if not QApplication.instance():
        app = QApplication(sys.argv)
    else:
        app = QApplication.instance()

    GUI = selectionMenu(folders);

    clickedFolders = GUI.checked;
    bgthresholds = GUI.bgthresholds;
    csthresholds = GUI.csthresholds;
    orients = GUI.orients;
    cfosX = GUI.cfosX;
    cfosY = GUI.cfosY;
    cfosZ = GUI.cfosZ;

    parameters = {
            "operations" : clickedFolders,
            "bgthresholds" : bgthresholds,
            "csthresholds" : csthresholds,
            "orients" : orients,
            "cfosX" : cfosX,
            "cfosY" : cfosY,
            "cfosZ" : cfosZ,
            };

    return app.exec_(), parameters;
#    sys.exit(app.exec_());

def launchGUI(fd,experiment) :

    print(ut.coloredMessage('[INFO] Automatic mode selected, GUI_Clearmap used','darkgreen'));

    folders = [];

    for folder in natsorted(os.listdir(fd)) :
        if folder.split('-')[0] == experiment or folder.split('_')[0] == experiment :
            if os.path.isdir(os.path.join(fd,folder)) :
                folders.append(folder);

    _,parameters = main(folders);

    return parameters;

#if __name__ == '__main__':
#    main();


import copy
import os.path

import numpy as np
import torch
import torch.nn.functional as F

from networks import Actor, Critic_MATD3

class Coach_MATD3(object):
    def __init__(self, args, agent_id, writer):
        self.agent_id = agent_id
        self.max_action = args.coach_max_action
        self.action_dim = args.coach_action_dim
        self.lr_a = args.lr_a
        self.lr_c = args.lr_c
        self.gamma = args.gamma
        self.tau = args.tau
        self.use_grad_clip = args.use_grad_clip
        self.policy_noise = args.policy_noise
        self.noise_clip = args.noise_clip
        self.policy_update_freq = args.goal_update_freq
        self.actor_pointer = 0
        # Create an individual actor and critic for each agent according to the 'agent_id'
        self.actor = Actor(args, -1, True)
        self.critic = Critic_MATD3(args, True)
        self.actor_target = copy.deepcopy(self.actor)
        self.critic_target = copy.deepcopy(self.critic)

        self.actor_optimizer = torch.optim.Adam(self.actor.parameters(), lr=self.lr_a)
        self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=self.lr_c)
        self.writer = writer

    # Each agent selects actions based on its own local observations(add noise for exploration)
    def choose_action(self, obs, noise_std):
        obs = torch.unsqueeze(torch.tensor(obs, dtype=torch.float), 0)
        a = self.actor(obs).data.numpy().flatten()
        a = (a + np.random.normal(0, noise_std, size=self.action_dim)).clip(-self.max_action, self.max_action)
        return a

    def train(self, replay_buffer):
        self.actor_pointer += 1
        batch_obs, batch_goal, batch_reward, batch_obs_next = replay_buffer.sample()

        # Compute target_Q
        with torch.no_grad():  # target_Q has no gradient
            # Trick 1:target policy smoothing
            batch_a_next = self.actor_target(batch_obs_next)
            noise = (torch.randn_like(batch_a_next) * self.policy_noise).clamp(-self.noise_clip, self.noise_clip)
            batch_a_next = (batch_a_next + noise).clamp(-self.max_action, self.max_action)

            # Trick 2:clipped double Q-learning
            Q1_next, Q2_next = self.critic_target(batch_obs_next, batch_a_next)
            target_Q = batch_reward[self.agent_id] + self.gamma * torch.min(Q1_next, Q2_next)  # shape:(batch_size,1)

        # Compute current_Q
        current_Q1, current_Q2 = self.critic(batch_obs, batch_goal)  # shape:(batch_size,1)
        critic_loss = F.mse_loss(current_Q1, target_Q) + F.mse_loss(current_Q2, target_Q)
        self.writer.add_scalar('critic_loss_coach'.format(self.agent_id), critic_loss,
                               global_step=self.actor_pointer)

        # Optimize the critic
        self.critic_optimizer.zero_grad()
        critic_loss.backward()
        if self.use_grad_clip:
            torch.nn.utils.clip_grad_norm_(self.critic.parameters(), 10.0)
        self.critic_optimizer.step()

        # Trick 3:delayed policy updates
        if self.actor_pointer % self.policy_update_freq == 0:
            # Reselect the actions of the agent corresponding to 'agent_id', the actions of other agents remain unchanged
            batch_goal = self.actor(batch_obs)
            actor_loss = -self.critic.Q1(batch_obs, batch_goal).mean()  # Only use Q1
            self.writer.add_scalar('actor_loss_coach'.format(self.agent_id), actor_loss,
                                   global_step=self.actor_pointer)
            # Optimize the actor
            self.actor_optimizer.zero_grad()
            actor_loss.backward()
            if self.use_grad_clip:
                torch.nn.utils.clip_grad_norm_(self.actor.parameters(), 10.0)
            self.actor_optimizer.step()

            # Softly update the target networks
            for param, target_param in zip(self.critic.parameters(), self.critic_target.parameters()):
                target_param.data.copy_(self.tau * param.data + (1 - self.tau) * target_param.data)

            for param, target_param in zip(self.actor.parameters(), self.actor_target.parameters()):
                target_param.data.copy_(self.tau * param.data + (1 - self.tau) * target_param.data)

    def save_model(self, env_name, algorithm, number, total_steps, agent_id):
        if not os.path.exists("./model/{}".format(env_name)):
            os.mkdir("./model/{}".format(env_name))
        torch.save(self.actor.state_dict(),
                   "./model/{}/{}_actor_number_{}_step_{}k_coach.pth".format(env_name, algorithm, number,
                                                                             int(total_steps / 1000), agent_id))


def cipher(text, shift, encrypt=True):

    """
    The Caesar cipher is one of the simplest and most widely known encryption techniques.
    This cipher takes string data and decode it by shifting each letter by the number of positions specified in the alphabet.

    Args:
        text: Input string data to be ciphered.
        shift: Number of digits in the alphabet to shift in the right direction.
        encrypt: True/False indicator defining whether or not the string is to be encrypted by this function.

    Returns:
      Ciphered text with input string data 'text' shifted the number of units defined by 'shift' input.

    Example of Usage:
        >>> print([cipher('Chef',3)])
        'Fkhi'
    """

    alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
    new_text = ''
    for c in text:
        index = alphabet.find(c)
        if index == -1:
            new_text += c
        else:
            new_index = index + shift if encrypt == True else index - shift
            new_index %= len(alphabet)
            new_text += alphabet[new_index:new_index+1]
    return new_text


def merge(left, right):
	A = []
	i = j = 0
	while i < len(left) and j < len(right):
		if left[i] < right[j]:
			A.append(left[i])
			i += 1
		else:
			A.append(right[j])
			j += 1
	A.extend(left[i:])
	A.extend(right[j:])
	return A

def mergesort(A):
	if len(A) < 2:
		return A
	i = 0
	for i in range(1, len(A)):
		if(A[i - 1] > A[i]):
			break
	return merge(A[:i], mergesort(A[i:]))

A = []
n = int(input("Enter array size: "))
print("Enter ", n, " elements")
for i in range(n):
	A. append(int(input()))
A = mergesort(A)
print("Sorted array = ", A)


# import numpy as np
import torch as T
import torch.nn as nn
import torch.nn.functional as F
# import torch.distributions as D
# from .Optimizer import RAdam
from adabelief_pytorch import AdaBelief
from torch.optim.lr_scheduler import MultiStepLR
from app.APN.model.aux.attention import attention
from app.APN.model.aux.mapping import mapping

class network(nn.Module):

    def __init__(
        self,
        features,  # number features
        outcomes,  # number outcomes
        d_m,
        N=None,
        d_l=4,
        sample=True,
        lr=1e-3,
        scheduler=None,
        regr=1e-10,
        loss=nn.BCELoss(),
        init_min=0.5,
        init_max=0.5
    ):
        super(network, self).__init__()

        self.features = features
        self.outcomes = outcomes
        self.d_m = d_m

        if N is not None:
            self.N = nn.Parameter(
                T.tensor(N),
                requires_grad=False
                )
        else:
            self.N = nn.Parameter(
                T.randint(100, (8, 2)),
                requires_grad=False
                )

        # set up for concrete distribution training mask
        self.weight_regulariser = 1e-6
        self.dropout_regulariser = 1e-7
        self.p_logit = nn.Parameter(
            T.empty(outcomes, features, 1).uniform_(init_min, init_max))
        self.p = T.sigmoid(self.p_logit)
        self.conc_reg = 0.0

        self.xmap = mapping(
            self.features,
            self.outcomes,
            d_m,
            layers=[10, 10, 10]
            )
        self.x_attn = nn.Parameter(T.randn(1, self.features, self.d_m*2))

        self.xattn = attention(
            d_q=d_m,
            d_k=d_m,
            d_v=d_m*2,
            d_m=d_m*2
        )

        self.ffn = nn.Sequential(
            nn.Linear(d_m, d_m),
            nn.Softplus(),
            nn.Linear(d_m, d_m),
        )
        self.attn = attention(
            d_q=d_m,
            d_k=d_m,
            d_v=d_m,
            d_m=d_m)

        self.y_attn = nn.Parameter(T.randn(1, self.outcomes, self.d_m))

        self.scaling = nn.Parameter(T.randn(self.outcomes, self.features))

        self.decoder_uni = nn.Sequential(
            nn.Linear(d_m, 10),
            nn.Softplus(),
            nn.Linear(10, 10),
            nn.Softplus(),
            nn.Linear(10, 2),
            nn.Softplus(),
        )
        self.decoder_full = nn.Sequential(
            nn.Linear(d_m, 10),
            nn.Softplus(),
            nn.Linear(10, 10),
            nn.Softplus(),
            nn.Linear(10, 2),
            nn.Softplus(),
        )

        self.sample = sample
        self.loss_fn = loss
        # self.optim = RAdam(self.parameters(), lr=lr)
        self.optim = AdaBelief(
            self.parameters(),
            lr=lr,
            eps=1e-16,
            betas=(0.9, 0.999),
            weight_decouple=True,
            weight_decay=1e-10,
            rectify=True,
            fixed_decay=False,
            amsgrad=False
            )
        self.regr = regr
        self.scheduler1 = MultiStepLR(self.optim, milestones=[15], gamma=0.1)
        self.kld = 0
        self.reg = 0
        self.alt = [1, 1]
        self.loss_1 = 0
        self.loss_1_alt = 0
        self.loss_2 = 0
        self.scaler_max = 1

    def forward(self, xv, yv=None, training=False):
        # find mask of missing vars, reduce y_attn, run attn
        x_mask = (xv != -1).unsqueeze(1)  # (batch, 1, x)

        xs = xv.shape
        if yv is not None:
            y_mask = (yv != -1)
            y_mask_ = y_mask.unsqueeze(2).repeat(1, 1, xs[1]) & x_mask
        # (batch, y, x)

        # find count values for beta prior
        N = self.N.unsqueeze(0)
        N = N.repeat(xs[0], 1, 1)

        # initialize beta prior with baseline
        prior_params = (N / N[:, :, 1:2])  # (b, y, 2)

        # map each feature to latent space
        xm_ = self.xmap(xv)  # (b,x,embed)
        xm = xm_.detach()

        x_attn = self.x_attn.repeat(xs[0], 1, 1)
        xshift = x_attn[:, :, :self.d_m] + xm*x_attn[:, :, self.d_m:]
        # concrete distribution training mask on mapped values
        if training:
            _, conc_mask = self._concrete_mask(xm_, y_mask)  # (b, f, 1)
            # xm = xm_.detach()
            x_mask_ = x_mask.to(T.long) * conc_mask.permute(0, 2, 1).to(T.long)

            # concrete distribution regularisation terms
            sum_of_squares = xm_.pow(2).sum(-1).unsqueeze(1)  # (b, 1, x)
            weights_reg = (self.weight_regulariser * sum_of_squares
                           / (1.0 - self.p.permute(2, 0, 1)))\
                .sum(2).sum(1).mean()
            dropout_reg = self.p * T.log(self.p)
            dropout_reg += (1.0 - self.p) * T.log(1.0 - self.p)
            dropout_reg *= self.dropout_regulariser * self.d_m
            self.regularisation = weights_reg + dropout_reg.sum()  # (1, d_m)
            xshift = xshift * conc_mask
        else:
            x_mask_ = x_mask

        # pass input data through attn network

        x_X, attn_x, _ = self.xattn(
            xshift, xshift, x_mask_,
            apply_v=True,
            v_alt=x_attn,
            )  # (batch, x, d_m)

        h = x_X[:, :, :self.d_m] + xshift*x_X[:, :, self.d_m:]
        h = self.ffn(h)  # (b, x, d_m)

        y_attn = self.y_attn.repeat(xs[0], 1, 1)

        # encode outcome dependent latent space with bayesian attn network
        if training:
            h = h * conc_mask

        z, attn, _ = self.attn(
            y_attn, h, x_mask_,
            v_alt=None,
            apply_v=True,
            )  # (batch, y, d_m), (batch, y, x), (batch, x, d_m)
        # posterior probability parameters
        # (batch, y, 2)

        multiplier = ((F.softplus(self.scaling)) * x_mask_).sum(
            -1, keepdims=True)  # (b, y, 1)
        z = (z * multiplier) + y_attn  # residual connection to y (b, y, d_m)
        beta_params = (1 + self.decoder_full(z))  # (b, y, 2)
        beta = T.distributions.Beta(
            beta_params[:, :, 0], beta_params[:, :, 1]
        )

        # use the commented out prior if not a rebalanced dataset
        # beta_prior = T.distributions.Beta(
        #     prior_params[:, :, 0], prior_params[:, :, 1]
        # )
        beta_prior = T.distributions.Beta(
            T.ones_like(prior_params[:, :, 0]),
            T.ones_like(prior_params[:, :, 1])
        )

        single_prob = beta.mean

        # univariate distributions
        values_ = xm_.unsqueeze(1) + y_attn.unsqueeze(2)  # (b, y, x, d_m)
        univariate_params = 1 + self.decoder_uni(values_)  # (b, y, x, d_m)
        univariate_beta = T.distributions.Beta(
            univariate_params[:, :, :, 0], univariate_params[:, :, :, 1]
        )
        uni_prior = T.distributions.Beta(
            T.ones_like(univariate_params[:, :, :, 0]),
            T.ones_like(univariate_params[:, :, :, 1])
        )
        probs = univariate_beta.mean  # (b, y, x)

        if yv is not None:
            self.loss_1 = self.UCE(
                yv.to(T.long), beta_params, y_mask
            )

            yv_ = yv.unsqueeze(2).repeat(1, 1, xs[1])  # (batch, y, x)
            self.loss_2 = self.UCE(
                yv_.to(T.long),
                univariate_params,
                y_mask_
            )

            # regularisation
            self.reg = (
                # self.alt[1] * xm.pow(2).sum(-1).mean()
                self.regularisation
                + 1e-5 * self.alt[0] * self.scaling.pow(2).sum()
                + self.regr * self.alt[1] * y_attn.pow(2).sum(-1).mean()
                + self.regr * self.alt[1] * x_attn.pow(2).sum(-1).mean()
                )

            self.beta_kld = T.distributions.kl_divergence(
                beta_prior, beta
            ).masked_select(y_mask).abs().mean()

            self.uni_kld = T.distributions.kl_divergence(
                uni_prior, univariate_beta
            ).masked_select(y_mask_).abs().mean()

            self.kld = (
                self.alt[0] * 1e-3 * self.beta_kld
                + self.alt[1] * 1e-3 * self.uni_kld
                )

            self.grad_loss =\
                self.alt[0] * self.loss_1\
                + self.alt[1] * self.loss_2\
                + self.kld\
                + self.reg

        return single_prob, probs, N, (attn, attn_x, beta_params)

    def step(self, step):
        self.optim.zero_grad()
        self.grad_loss.backward()
        # T.nn.utils.clip_grad_norm_(self.parameters(), 1, 2)
        self.optim.step()

    def loss(self):
        return {
            'Loss_1': float(self.loss_1),
            'Loss_2': float(self.loss_2),
            'Loss': (
                self.alt[0] * float(self.loss_1) +
                self.alt[1] * float(self.loss_2)),
            'KLD': float(self.kld),
            "REG": float(self.reg)
        }

    def UCE(self, yv, post_params, mask):
        # prep indicators of outcomes
        yv_ = F.one_hot(yv + 1, 3)[..., 1:]
        alpha_0 = post_params.sum(-1, keepdims=True)
        label = (yv_ * -1.0) + 1.0
        loss = (label * (T.digamma(alpha_0) - T.digamma(post_params))).sum(-1)
        return loss.masked_select(mask).mean()

    def _concrete_mask(self, x, ymask):
        eps = 1e-7
        tmp = 0.1

        self.p = T.sigmoid(self.p_logit)  # (outcomes, features, 1)

        # build p|y matrix for each sample in batch
        indices = ymask.nonzero()[:, 1]  # (batch)

        p = T.index_select(self.p, 0, indices)  # (b, f, 1)

        u_noise = T.rand(x.shape[0], self.features, 1)

        drop_prob = (T.log(p + eps) -
                     T.log(1 - p + eps) +
                     T.log(u_noise + eps) -
                     T.log(1 - u_noise + eps))

        drop_prob = T.sigmoid(drop_prob / tmp)

        random_tensor = 1 - drop_prob
        retain_prob = 1 - p

        x = (x * random_tensor) / retain_prob

        return x, drop_prob  # (b, features, 1)


import  os
import  sys

import barcode
from barcode.writer import ImageWriter

# Importing inner modules, so that it works as a module and as a program
try:
    from . import LogProvider  as	log_provider
except:
    import LogProvider  as	log_provider

def __create(barcode_type, data, file_path):
    """__create is a private method which creates a 'PNG' image of a barcode with the given data
        on the specified file_path

    Arguments:
        barcode_type {str} -- A valid barcode type eg. EAN13
        data {str} -- A valid data based on barcode type
        file_path {str} -- A vavlid path where the image has to be stored
    """

    try:
        barcode_class = barcode.get_barcode_class(barcode_type)
        barcode_object = barcode_class(repr(data), writer = ImageWriter())
        barcode_object.save(filename=file_path)
    except Exception as ex:
        process_data.message = 'Exception in __create - ' + str(ex)
        log_provider.insert_error_log(process_data)

def createBarcode(data = None, file_path = None, barcode_type = None, is_folder = False):
    """createBarcode will create a barcode image method which creates a 'PNG' image of a barcode with the given data
        on the specified file_path

    Keyword Arguments:
        data {str} -- A valid data based on barcode type
        file_path {str} -- A vavlid path where the image has to be stored
        barcode_type {str} -- A valid barcode type eg. EAN13
        is_folder {bool} -- Specifies whether the given path is a folder, used for manual overriding (default: {False})

    Returns:
        ProcessData -- ProcessData used for interprocess communication
    """

    ALL_PROVIDED_BARCODES = barcode.PROVIDED_BARCODES
    process_data.message = 'Error'
    process_data.status = -1
    process_data.data = data, file_path, barcode_type
    can_skip = False
    if(file_path):
        file_name, extension = os.path.splitext(os.path.basename(file_path))
    else:
        process_data.message = 'Exception in Writing - No file_path provided'
        log_provider.insert_error_log(process_data)
        return process_data
    if(not data):
        process_data.message = 'Exception in Writing - No Data provided'
        log_provider.insert_error_log(process_data)
        return process_data
    if(barcode_type):
       ALL_PROVIDED_BARCODES.clear()
       ALL_PROVIDED_BARCODES.append(barcode_type)
    else:
        if(not file_path):
            return process_data
    try:
        if(not is_folder):
            if(barcode_type):
                __create(barcode_type= barcode_type, data= data, file_path= file_path)
            else:
                can_skip = True
                for CURRENT_BARCODE in ALL_PROVIDED_BARCODES:
                    file_path = os.path.join(file_path, CURRENT_BARCODE)
                    __create(barcode_type= barcode_type, data= data, file_path= file_path)
        elif(not extension):
            process_data.message = 'Exception in Writing - No extension is provided for folder'
            log_provider.insert_error_log(process_data)
            return process_data
        else:
            if(barcode_type):
                file_path = os.path.join(file_path, barcode_type)
                __create(barcode_type= barcode_type, data= data, file_path= file_path)
    except Exception as e:
        if(not can_skip):
            process_data.message = 'Exception in Writing ' + str(e)
            log_provider.insert_error_log(process_data)
            print('Writer Exception for image ' + file_path)
            input()
            return process_data

    process_data.status = 1
    process_data.message = 'Success'
    process_data.data = file_path
    return process_data

# The Actual Module Code
try:
    bot_name, extention =   os.path.splitext(os.path.basename(__file__))
    process_data        =   log_provider.generate_process_data(bot_name)
    # Import safety block - createbarcode(), only when run as a script
    if(__name__ == '__main__'):
        createBarcode(file_path = 'D:\\Guru\\TEST_FOLDERS\\Barcode\\sample', barcode_type = 'EAN')

except Exception as exception:
	log_provider.insert_log(process_data)
	process_data.message  = '@ BarcodeWriter - ' + format(exception)
	log_provider.insert_error_log(process_data)


import tensorflow as tf

def create_padding_mask(seq):
    seq = tf.cast(tf.math.equal(seq, 0), tf.float32)
    return seq[:, tf.newaxis, tf.newaxis, :]

def create_look_ahead_mask(size):
    """
    Copyright 2018- The Hugging Face team. All rights reserved.
    """

    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)
    return mask

def tf_bernoulli(shape, probability):
    """
    Copyright 2018- The Hugging Face team. All rights reserved.
    """

    prob_matrix = tf.fill(shape, probability)
    return tf.cast(prob_matrix - tf.random.uniform(shape, 0, 1) >= 0, tf.bool)

def tf_mask_tokens(inputs, vocab_size, mask_token_id, special_tokens_mask, mlm_probability):
    """
    Copyright 2018- The Hugging Face team. All rights reserved.
    Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
    """

    input_shape = tf.shape(inputs)
    # 1 for a special token, 0 for a normal token in the special tokens mask
    # We sample a few tokens in each sequence for MLM training (with probability `self.mlm_probability`)
    masked_indices = tf_bernoulli(input_shape, mlm_probability) & ~special_tokens_mask
    # Replace unmasked indices with -100 in the labels since we only compute loss on masked tokens
    labels = tf.where(masked_indices, inputs, -100)

    # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])
    indices_replaced = tf_bernoulli(input_shape, 0.8) & masked_indices

    inputs = tf.where(indices_replaced, mask_token_id, inputs)

    # 10% of the time, we replace masked input tokens with random word
    indices_random = tf_bernoulli(input_shape, 0.1) & masked_indices & ~indices_replaced
    random_words = tf.random.uniform(input_shape, maxval=vocab_size, dtype=tf.int64)
    inputs = tf.where(indices_random, random_words, inputs)

    # The rest of the time (10% of the time) we keep the masked input tokens unchanged
    return inputs, labels


from const      import *
from test_cases import *

from pycrate_asn1dir            import RRCLTE
from pycrate_mobile.NAS         import *
from pycrate_asn1rt.codecs      import _with_json
from pycrate_asn1rt.asnobj_ext  import *
from CryptoMobile               import CM

import json

class Pdu:
    sep_char = "|  "

    def __init__(self, _pdu, _is_sdu, _msg_number):
        # type: (Pdu, bytes, bool, int) -> Pdu

        # The Packet Data Unit (as bytes)
        self.pdu = _pdu
        # The Packet Data Unit (as struct)
        self.pdu_dict = 0
        # Decoded message as String
        self.decoded_msg = ""
        # True if is UE -> eNB (uplink)
        self.is_sdu = _is_sdu
        # Number of the message (starting from 0)
        self.msg_number = _msg_number
        # Counter of the Struct raw (used as index of access)
        self.counter = 0
        # True if PDU has a NAS message inside
        self.has_nas = False
        # NAS messages (as bytes)
        self.nas = []
        # NAS messages (as json)
        self.nas_json = []
        if self.is_sdu:
            self.msg_type = RRCLTE.EUTRA_RRC_Definitions.UL_DCCH_Message
        else:
            self.msg_type = RRCLTE.EUTRA_RRC_Definitions.DL_DCCH_Message

    # fuzz indicates if a field has to be changed, fuzz_index means there's a fuzzing procedure (test all fields)
    def decode(self, print_info=True, decodeNAS=True, fuzz=False, row=0, new_val=0, fuzz_index=None, new_pdu=True):
        self.msg_type.from_uper(self.pdu)
        if new_pdu:
            self.pdu_dict = self.msg_type()

        self.decoded_msg = ""
        self.counter = 0
        self.nas = []
        self.nas_json = []
        # TODO: reimposta il Correct_test e Correct_ws_test a True e metti delf.debug_extra sotto print(self.decoded_mesg) e vedi se fallisce con il Radio link failure
        self.debug_extra()

        self.parse_dict(0, 0, self.nas, initial_dict=self.pdu_dict, print_info=print_info, fuzz=fuzz, row=row, content=new_val, fuzz_index=fuzz_index)
        if print_info:
            self.decoded_msg += '\n'
            print(self.decoded_msg)

        if fuzz_index is not None:
            if not fuzz_index.has_next_input and self.counter <= fuzz_index.field + 1:
                fuzz_index.has_next_field = False

        if len(self.nas) != 0:
            self.has_nas = True
            print("NAS Messages: ", self.nas)
        else:
            print("No NAS messages")
        if self.has_nas and decodeNAS:
            for e in self.nas:
                self.decode_nas(e, self.is_sdu, print_info=True)

    def encode(self, asHexString=False):
        try:
            if asHexString:
                return self.msg_type.to_uper(self.pdu_dict).hex()
            else:
                return self.msg_type.to_uper(self.pdu_dict)
        except Exception:
            # NOTE: Raising an exception if the encoding fails is wrong! It would bring a lot of
            # "False" exception which should not be reported
            print(FZ_PREFIX, "Could not encode PDU, returning original PDU")
            if asHexString:
                return self.pdu.hex()
            else:
                return self.pdu

    # Utilities
    def parse_dict(self, container, container_key, nas_info_list, index=0, initial_dict=None, print_info=True, fuzz=False, row=0, content=0, fuzz_index=None):
        # type: (Pdu, object, object, list, int, dict, bool, bool, int, object, object) -> object
        if initial_dict is None:
            initial_dict = container[container_key]
        for key in initial_dict:
            if print_info:
                # print(index * self.sep_char + "KEY " + key, end=' ')
                self.decoded_msg += str(index * self.sep_char) + "KEY " + str(key) + ' '
            self.recursor(initial_dict, key,  nas_info_list, index, print_info=print_info, fuzz=fuzz, row=row, content=content, fuzz_index=fuzz_index)
            if key == "dedicatedInfoNAS":
                nas_info_list.append(initial_dict[key])
            elif key == "dedicatedInfoType":
                nas_info_list.append(initial_dict[key][1])
            elif key == "dedicatedInfoNASList":
                for e in initial_dict[key]:
                    nas_info_list.append(e)

        return

    def parse_tuple(self, container, container_key, nas_info_list, index=0, print_info=True, fuzz=False, row=0, content=0, fuzz_index=None):
        # type: (Pdu, object, object, list, int, bool,  bool, int, object, object) -> object

        # Necessary conversion to enable modification to the tuple
        container[container_key] = list(container[container_key])

        for j in range(0, len(container[container_key])):
            if print_info:
                # print(index * self.sep_char, end='')
                self.decoded_msg += str(index * self.sep_char)
            self.recursor(container[container_key], j, nas_info_list, index, print_info=print_info, fuzz=fuzz, row=row, content=content, fuzz_index=fuzz_index)

        container[container_key] = tuple(container[container_key])
        return

    def recursor(self, container, container_key, nas_info_list, index, print_info=True, fuzz=False, row=0, content=None, fuzz_index=None):
        if print_info:
            elem_type = str.upper(type(container[container_key]).__name__)
            # print(elem_type, end=' ')
            self.decoded_msg += elem_type + ' '
        if isinstance(container[container_key], dict):
            if print_info:
                # print("")
                self.decoded_msg += '\n'
            self.parse_dict(container, container_key, nas_info_list, index + 1, print_info=print_info, fuzz=fuzz, row=row, content=content, fuzz_index=fuzz_index)
        elif isinstance(container[container_key], tuple) or isinstance(container[container_key], list):
            if print_info:
                # print("")
                self.decoded_msg += '\n'
            self.parse_tuple(container, container_key, nas_info_list, index + 1, print_info=print_info, fuzz=fuzz, row=row, content=content, fuzz_index=fuzz_index)
        else:
            if print_info:
                if elem_type == "BYTES":
                    # print(container[container_key].hex(), "(", self.counter, ")")
                    self.decoded_msg += container[container_key].hex() + "(" + str(self.counter) + ")\n"
                else:
                    # print(container[container_key], "(", self.counter, ")")
                    self.decoded_msg += str(container[container_key]) + "(" + str(self.counter) + ")\n"

            # Change single value (no fuzzing)
            if fuzz and row == self.counter:
                print("CHANGED")
                container[container_key] = content

            # Fuzz multiple values
            if fuzz_index is not None:
                if self.msg_number == fuzz_index.message and self.counter == fuzz_index.field:
                    print("FUZZED")
                    # If the element type is not in the defined ones (BYTE, STRING, Ecc..) raise exception and switch to the next field
                    if elem_type not in TEST_VALUES:
                        print(FZ_PREFIX, "Unspecified type:", elem_type, "at position", self.counter)
                        fuzz_index.has_next_input = False
                        raise Exception("Unspecified type: " + elem_type)

                    # Assign the test values according to the element type
                    test_values = TEST_VALUES[elem_type]

                    container[container_key] = test_values[fuzz_index.test_input_index]
                    # If last value to be tested
                    if fuzz_index.test_input_index >= len(test_values) - 1:
                        fuzz_index.has_next_input = False

            self.counter += 1

        return

    def decode_nas(self, data, is_uplink, recursive=True, print_info=True):
        pdu = data
        m = 0
        e = 0
        try:
            if is_uplink:
                m, e = parse_NAS_MO(pdu)
            else:
                m, e = parse_NAS_MT(pdu)

            v = m.get_val()
            t = m.to_json() + "\n"
            self.nas_json.append(t)

            if print_info:
                print("[DEBUG] Json NAS:", str.upper(type(t).__name__), t.replace("\n", "\n\t") + "\n")
            if recursive:
                self.parse_nas(t, is_uplink, print_info=print_info)
            if CORRECT_NAS_TEST:
                assert (e == 0)
                m.reautomate()
                assert (m.get_val() == v)
                m.set_val(v)
                assert (m.to_bytes() == pdu)
                m.from_json(t)
                assert (m.get_val() == v)
        # TODO: Handle Assertion error to raise another exception
        except AssertionError as error:
            print("Assertion Error")
            raise error
        except Exception:
            print(FZ_PREFIX, "Could not Decode Nas (Probably modified by fuzzer)")

    def parse_nas(self, t, is_uplink, print_info=True):
        j = json.loads(t)
        keys = list(j.keys())
        for e in j[keys[0]]:
            inner_keys = list(e.keys())
            if inner_keys == ["NASMessage"]:
                print("\nInner NAS Message:")
                self.decode_nas(unhexlify(e["NASMessage"]), is_uplink, False, print_info=print_info)
        if CORRECT_NAS_TEST:
            assert (keys == ["EMMSecProtNASMessage"] or
                    keys == ["EMMIdentityRequest"] or
                    keys == ["EMMAuthenticationRequest"] or
                    keys == ["EMMAuthenticationResponse"] or
                    keys == ["EMMAttachRequest"] or
                    keys == ["EMMIdentityResponse"] or
                    keys == ["EMMDetachRequestMO"])

    def debug_extra(self):
        if DEBUG_PDU_PRINT:
            print("[DEBUG] JSON Format\n" + self.msg_type.to_jer())
            print("[DEBUG] Dict print\n", self.msg_type())

        if CORRECT_TEST:
            ret = self.msg_type.to_uper()
            # TODO: in some cases, those two bytes sequence are different even if the decode is the same
            # TODO: instead of comparing bytes, compare the resulting struct (should be the same)
            assert (ret == self.pdu)

        if CORRECT_WS_TEST:
            val = self.msg_type()
            self.msg_type.from_uper_ws(self.pdu)
            val_ws = self.msg_type()
            struct = self.msg_type._struct()
            ret = self.msg_type.to_uper_ws()
            assert (ret == self.pdu)
            assert (val == val_ws)
            assert (self.msg_type._struct() == struct)
            txt = self.msg_type.to_asn1()
            self.msg_type.from_asn1(txt)
            assert (self.msg_type() == val)


#Write a script that calculates how many vowels are in a string.

example = "Count number of vowels in a String in Python"
count=0
vowels = ["a", "e", "i", "o", "u"]

for i in range(len(example)):
    if example[i] in vowels:
        count+=1

print(count)


# Script extracting data from yahoo finances

# Requirements:
# - pip3 install yfinance
# - pip3 install yahoo_fin
# - pip3 install requests_html
# - pip3 install django

def ExtractStocksData():

    import yfinance as yf1
    from multiprocessing import Process, Manager
    # import yahoo_fin.stock_info as yf2

    # stockSymbols = yf2.tickers_nasdaq()  # Get all stocks (over 3000!)
    # print(f"Number of stocks: {len(stockSymbols)}")
    # print(f"Stocks: {stockSymbols}")
    # Etoro stocks (https://etoro-cdn.etorostatic.com/e-marketing/AUM/FulllistofavailableStocksoneToro.pdf)
    # stockSymbols = ['AAPL', 'GOOG', 'FB', 'MSFT', 'AMZN', 'AABA', 'ZNGA', 'AA',
    #                 'AXP', 'BA', 'BAC', 'CAT', 'CSCO', 'CVX', 'DD', 'DIS', 'GE',
    #                 'HD', 'HPQ', 'IBM', 'INTC', 'JNJ', 'JPM', 'KO', 'MCD', 'MMM',
    #                 'MRK', 'PFE', 'PG', 'T', 'TRV', 'UNH', 'UTX', 'VZ', 'WMT',
    #                 'XOM', 'C', 'HMC', 'HBC', 'MANU', 'MA', 'NKE', 'PEP', 'SNE',
    #                  'TM', 'V', 'VOD', 'TWX', 'DB', 'CL', 'EBAY', 'SI', 'YNDX',
    #                  'QIWI', 'CHL', 'RENN', 'TEO', 'TX', 'PBR', 'VALE', 'SAN',
    #                  'TEF', 'AMX', 'CX', 'DTV', 'TV', 'SNDK', 'TSLA', 'F',
    #                  'LVS', 'LNKD', 'NOK', 'POT', 'FSLR', 'BRK.B', 'WU', 'GPS',
    #                  'RL', 'HOG', 'TRIP', 'EA', 'URBN', 'ADBE', 'NFLX', 'MSI',
    #                  'WDC', 'MU', 'ANF', 'DEM', 'AGNC', 'ADSK', 'ORCL', 'LMT',
    #                  'NVDA', 'FDX', 'NBL', 'CAR', 'BIDU', 'SBUX', 'AMGN',
    #                  'ATVI', 'CHKP', 'MOS', 'PETM', 'KDP', 'STZ', 'TWTR',
    #                  'KING', 'BABA', 'SPOT', 'DBX', 'TCEHY', 'ETSY', 'WIX',
    #                  'RYAAY', 'DLPH', 'PCRFY', 'ACB', 'SMG', 'CARA', 'INSY',
    #                  'WP', 'ZYNE', 'CRBP', 'CRON', 'PEGI', 'CVA', 'EVA', 'NEP',
    #                  'ALB', 'ENS', 'APHA', 'XI', 'OSTK', 'TLRY', 'LYFT', 'PINS',
    #                  'LEVI', 'UBER', 'BYND', 'SIE', 'BAS', 'SAP', 'BAYN', 'ALV',
    #                  'EOAN', 'DAI', 'DTE', 'LNA', 'VOW3', 'BMW', 'MUV2', 'RWE',
    #                  'FME', 'ADS', 'DPW', 'FRE', 'HEN3', 'DB1', 'TKA', 'IFX',
    #                  'SDF', 'CBK', 'MRK', 'BEI', 'HEI', 'LHA', 'CON', 'LXSd',
    #                  'BOSSd', 'AC', 'ACA', 'AI', 'AIR', 'ALO', 'ALU', 'BN',
    #                  'BNP', 'CA', 'CAP', 'CS', 'DG', 'EDF', 'EL', 'EN', 'FP',
    #                  'FR', 'GLE', 'ENGI', 'KER', 'LG', 'LR', 'MC', 'ML', 'MTS',
    #                  'OR', 'ORA', 'PUB', 'RI', 'RNO', 'SAF', 'SAN', 'SGO',
    #                  'SOLB', 'SU', 'TEC', 'UG', 'UL', 'VIE', 'VIV', 'A2A',
    #                  'AGL', 'ATL', 'AZM', 'BMPS', 'BP', 'BPE', 'BZU', 'CNHI',
    #                  'CPR', 'EGPW', 'ENEL', 'ENI', 'EXO', 'FCA', 'LDO', 'GASI',
    #                  'ISP', 'LUX', 'MDBI', 'MED', 'MONC', 'MS', 'PC', 'PMI',
    #                  'PRY', 'SFER', 'SPM', 'SRG', 'STM', 'STS', 'TEN', 'TLIT',
    #                  'TOD', 'TRN', 'UBI', 'UCG', 'US', 'WDF', 'YNAP', 'ABE',
    #                  'ABG-P', 'ACS', 'AMS', 'ANA', 'BBVA', 'BKIA', 'BKT', 'BME',
    #                  'CABK', 'DIAm', 'ELE', 'ENG', 'FCC', 'FER', 'SGRE', 'NTGY',
    #                  'GRF', 'IBE', 'IDR', 'ITX', 'JAZ', 'MAP', 'OHL', 'POP',
    #                  'REE', 'REP', 'SAB', 'SCYR', 'TL5', 'TRE', 'OGZDL', 'KMGL',
    #                  'KCELL', 'LKODL', 'MGNTL', 'MFONL', 'MNODL', 'NVTKL',
    #                  'PHORL', 'ROSNL', 'SBERL', 'SVSTL', 'SGGDL', 'ATADL',
    #                  'URKAL', 'QIHU', 'JMEI', 'CYOU', 'QUNR', 'NTES', 'JD',
    #                  'CTRP', 'ATHM', 'JOBS', 'GSOL', 'MOMO', 'TTWO', 'NESN',
    #                  'NOVN', 'ROG', 'CSGN', 'ABB', 'ADEN', 'ATLN', 'BAER',
    #                  'CFR', 'GEBN', 'GIVN', 'LHN', 'SCMN', 'SGSN', 'SLHN',
    #                  'SREN', 'SYNN', 'UBSG', 'UHR', 'ZURN', 'ACN', 'AAL',
    #                  'ABBV', 'BIIB', 'CBS', 'CELG', 'CMCSA', 'COF', 'COST',
    #                  'CVS', 'EMC', 'FOX', 'GILD', 'GPRO', 'GS', 'HON', 'K',
    #                  'CPRI', 'LOW', 'M', 'MCK', 'MDT', 'MET', 'BKNG', 'PM',
    #                  'PSX', 'PYPL', 'QCOM', 'RDS.B', 'S', 'SHAK', 'TGT', 'TTM',
    #                  'UNP', 'UPS', 'WBA', 'WFC', 'WMB', 'VLO', 'ABC', 'FNMA',
    #                  'KR', 'GM', 'ESRX', 'MPC', 'CAH', 'ANTM', 'FMC', 'AIG',
    #                  'DOW', 'AET', 'COP', 'ET', 'HUM', 'EPD', 'SYY', 'IM',
    #                  'JCI', 'PAGP', 'INT', 'BBY', 'DAL', 'HCA', 'ANDV', 'UAL',
    #                  'TSN', 'DE', 'ALL', 'CI', 'MDLZ', 'INTL', 'HAL', 'SHLD',
    #                  'GD', 'TJX', 'TECD', 'AVT', 'EXC', 'IP', 'QRTEA', 'DUK',
    #                  'RAD', 'BHI', 'EMR', 'NOC', 'NOV', 'RTN', 'TWC', 'ARW',
    #                  'AFL', 'SPLS', 'ABT', 'CYH', 'FLR', 'FCX', 'USB', 'NUE',
    #                  'KMB', 'HES', 'CHK', 'XRX', 'MAN', 'DHR', 'WHR', 'PBF',
    #                  'HFC', 'LLY', 'DVN', 'PGR', 'CMI', 'IEP', 'KSS', 'PCAR',
    #                  'HIG', 'LUV', 'APC', 'SO', 'SVU', 'GT', 'EOG', 'CTL',
    #                  'MO', 'THC', 'GIS', 'CAG', 'LEA', 'X', 'PAG', 'AES',
    #                  'GLP', 'TMO', 'PCG', 'NEE', 'AEP', 'BAX', 'CNC', 'BK',
    #                  'JBL', 'PNC', 'KMI', 'ODP', 'BMY', 'NRG', 'MON', 'PPG',
    #                  'GPC', 'OMC', 'ITW', 'MUSA', 'WNR', 'FE', 'ARMK', 'DISH',
    #                  'L', 'ECL', 'WFM', 'CB', 'HNT', 'WM', 'APA', 'TXT', 'SNX',
    #                  'VIAB', 'LNC', 'JWN', 'CHRW', 'EIX', 'YUM', 'PH', 'DVA',
    #                  'KMX', 'TXN', 'WCG', 'MMC', 'ED', 'OKE', 'JEC', 'CSX',
    #                  'ETR', 'D', 'JEF', 'AMP', 'VFC', 'PX', 'JCP', 'ADP', 'LLL',
    #                  'CDW', 'XEL', 'NSC', 'PPL', 'RRD', 'HUN', 'BBBY', 'SWK',
    #                  'LB', 'SHW', 'BLK', 'VOYA', 'ROST', 'SRE', 'EL', 'RGA',
    #                  'PEG', 'CAM', 'NAV', 'CST', 'STT', 'UNM', 'HLT', 'PFG',
    #                  'RS', 'APD', 'AIZ', 'HSIC', 'CTSH', 'MGM', 'GWW', 'GPI',
    #                  'BBT', 'AAP', 'ALLY', 'AGCO', 'GLW', 'NGL', 'SYK', 'MOH',
    #                  'PCP', 'DFS', 'GNW', 'EMN', 'DF', 'AZO', 'OMI', 'HRL',
    #                  'GME', 'CNP', 'FNF', 'SAH', 'HDS', 'CHTR', 'CCK', 'AMAT',
    #                  'CBRE', 'AVP', 'RSG', 'UHS', 'DRI', 'STLD', 'STI', 'CZR',
    #                  'TRGP', 'DLTR', 'NWSA', 'BLL', 'MAS', 'BEN', 'RAI', 'BDX',
    #                  'BRCM', 'CPB', 'ACM', 'VC', 'DK', 'DOV', 'BWA', 'JAH',
    #                  'UGI', 'MUR', 'PVH', 'CORE', 'CPN', 'DHI', 'WY', 'KKR',
    #                  'FTI', 'SPTN', 'WCC', 'PWR', 'MHK', 'LEN', 'TA', 'SEE',
    #                  'ES', 'CCE', 'ASH', 'IPG', 'BX', 'DGX', 'NEM', 'ORLY',
    #                  'CASY', 'CMS', 'FL', 'WRB', 'CMC', 'A', 'HII', 'LYV',
    #                  'DKS', 'OSK', 'CE', 'SPR', 'UNFI', 'BTU', 'OI', 'DDS',
    #                  'LVLT', 'LKQ', 'SYMC', 'BPL', 'R', 'ROK', 'DAN', 'NCR',
    #                  'EXPD', 'AKS', 'FITB', 'SEB', 'NI', 'CVC', 'AXE', 'EME',
    #                  'FIS', 'BKS', 'KBR', 'AVY', 'NTAP', 'DISCA', 'SANM',
    #                  'JBHT', 'SCHW', 'AEE', 'MAT', 'LH', 'HOT', 'BGC', 'AMRI',
    #                  'SPB', 'MRC', 'SE', 'ABG', 'PKG', 'WIN', 'PHM', 'JBLU',
    #                  'NWL', 'CLMT', 'EXPE', 'AFG', 'URI', 'INGR', 'NAVI', 'STJ',
    #                  'SJM', 'CLX', 'UFS', 'KELYA', 'ORI', 'AMD', 'BAH', 'IQV',
    #                  'WYNN', 'JLL', 'RF', 'LAD', 'CRM', 'ALK', 'HST', 'HAR',
    #                  'APH', 'RLGY', 'ESND', 'HBI', 'KND', 'ARRS', 'NSIT',
    #                  'LPNT', 'PXD', 'WYND', 'OC', 'Y', 'SPGI', 'BIG', 'NTI',
    #                  'MKL', 'LDOS', 'COL', 'SRLP', 'YRCW', 'THG', 'FISV',
    #                  'ABM', 'SON', 'HRS', 'TDS', 'WEC', 'LINE', 'RJF', 'BERY',
    #                  'SCG', 'CINF', 'ATO', 'POM', 'FLS', 'SPG', 'QUAD', 'BURL',
    #                  'BMS', 'TPR', 'CLR', 'ASNA', 'Z', 'OA', 'FTR', 'SPW',
    #                  'CF', 'MIK', 'MTB', 'RUSHB', 'GMCR', 'SPN', 'WSM', 'RHI',
    #                  'FAF', 'MDU', 'JNPR', 'AJG', 'CFX', 'CLF', 'MTZ', 'LRCX',
    #                  'AXLL', 'ICE', 'CTAS', 'COTY', 'ANDE', 'VAL', 'NTRS',
    #                  'INTU', 'TPC', 'PII', 'RACE', 'NVR', 'FWONA', 'ENR',
    #                  'BLMN', 'WLK', 'H', 'MJN', 'EVHC', 'FBHS', 'RPM', 'VWR',
    #                  'LPLA', 'KEY', 'KNX', 'AGN', 'HAS', 'RFP', 'TIF', 'MKC',
    #                  'GPK', 'GEF', 'ATI', 'BEAV', 'TAP', 'CNO', 'AE', 'CMG',
    #                  'AMT', 'AFSI', 'BC', 'PDCO', 'SWN', 'AME', 'TROW', 'TMK',
    #                  'DAR', 'LEG', 'WSO', 'CEQP', 'XYL', 'SLGN', 'TOL', 'MTW',
    #                  'ARGO', 'ARG', 'GNC', 'MAR', 'SQ', 'MTCH', 'SEDG', 'FIT',
    #                  'NTDOY', 'REGN', 'SGMO', 'NTLA', 'EDIT', 'MS', 'CLLS',
    #                  'HRMS', 'SNAP', 'AAL', 'ABF', 'ADM', 'ADN', 'AGK', 'AHT',
    #                  'AMFW', 'ANTO', 'ARM', 'AV', 'AZN', 'BAl', 'BAB', 'BARC',
    #                  'BATS', 'BG', 'BLND', 'BHP', 'BNZL', 'BP', 'BRBY', 'SKY',
    #                  'BT', 'CCH', 'CCL', 'CNA', 'CPG', 'CPI', 'CRH', 'DGE',
    #                  'EXPN', 'EZJ', 'FRES', 'GFS', 'GKN', 'GLEN', 'GSK', 'HL',
    #                  'HMSO', 'HSBA', 'IAG', 'IHG', 'IMI', 'IMB', 'ITRK', 'ITV',
    #                  'JMAT', 'KGF', 'LAND', 'LGEN', 'LLOY', 'LSE', 'MGGT',
    #                  'MKS', 'MNDI', 'MRO', 'MRW', 'NG', 'NXT', 'OML', 'PFC',
    #                  'PRU', 'PSN', 'PSON', 'RB', 'RBS', 'RDSA', 'REL', 'REX',
    #                  'RIO', 'RMG', 'RR', 'RRS', 'RSA', 'RSL', 'SAB', 'SBRY',
    #                  'SDR', 'SGE', 'SHP', 'SLA', 'SMIN', 'SN', 'SPD', 'SSE',
    #                  'STAN', 'SVT', 'TATE', 'TLW', 'TPK', 'TSCO', 'TT', 'ULVR',
    #                  'UU', 'VOD', 'WEIR', 'WMH', 'FERG', 'WPP', 'WTB', 'DMGT',
    #                  'III', 'BPM', 'HZD', 'BDEV.L', 'CRDA.L', 'DCC.L', 'DLG.L',
    #                  'INF.L', 'MCRO.L', 'PPB.L', 'RTO.L', 'SGRO.L', 'SKG.L',
    #                  'SMT.L', 'TUI.L', 'TW.L', 'QLT.L', 'MDC.L', 'AML.L',
    #                  'FEVR.L', 'ASC.L', 'RDSB.L', 'RYA.L', 'AVST.L', 'VVO.L',
    #                  'SMSN.L', 'TLN.L', 'AKA', 'AKSO', 'BWLPG', 'DNB', 'DNO',
    #                  'FOE', 'GJF', 'GOGL', 'MHG', 'NHY', 'NAS', 'OTELLO', 'ORK',
    #                  'PGS', 'PRS', 'STB', 'SUBC', 'TEL', 'TGS', 'YAR', 'ERIC-A',
    #                  'TELIA', 'NDA_SE', 'VOLV-A', 'SEB-A', 'SAND', 'HM-B',
    #                  'SHB-A', 'SKF-A', 'SWED-A', 'ASSA-B', 'ATCO-A', 'BOL',
    #                  'SCA-B', 'ALFA', 'ELUX-B', 'SECU-B', 'SKA-B', 'INVE-A',
    #                  'LUPE', 'TREL-B', 'SWMA', 'HEXA-B', 'KINV-A', 'ICA',
    #                  'INDU-A', 'LUND-B', 'NOVO-B', 'DANSKE', 'ORSTED', 'VWS',
    #                  'NZYM-B', 'PNDORA', 'TRYG', 'DSV', 'WDH', 'ISS', 'LUN',
    #                  'GEN', 'COLO-B', 'CARL-B', 'CHR', 'OSSR', 'NDA1V',
    #                  'SSABAH', 'STEAV', 'FORTUM', 'UPM', 'NESTE', 'KNEBV',
    #                  'SAMPO', 'ELISA', 'NRE1V', 'WRT1V', 'ORNAV', '0939.HK']

    # Removed non-US time stocks: GFRD.L, 1810.HK (XI), RDSB.L, SAN.PA, CDI.PA,
    # UBI.PA, AIR.PA, ENC.MC, VOLV-A.ST, 2222.SR, (SAOC), HSBA.L, NESN.SW,
    # PAH3.DE, BOSS.DE, KPN.AS, VPK.AS, RYA.L
    stockSymbols = [# Technology
                    "ATVI", "ADBE", "ADSK", "AMAT", "AMD", "ANSS", "CDNS",
                    "CSCO", "DBX", "DELL", "DLG.DE", "EA", "GOOG", "IBM",
                    "INTC", "LOGI", "MSFT", "NOK", "NVDA", "NXPI", "SNAP",
                    "PINS", "PYPL", "QCOM", "SOXX", "SPOT", "TWTR", "TXN",
                    "WDC", "ZM", "DOCU", "SYNA", "NET",
                    "CRM", "FSLR", "SHOP", "HPE", "HPQ", "ASML", "AVGO", "INTU",
                    # Entertainment
                    "DIS", "NFLX",
                    # Services
                    "0700.HK", "AAL", "AMZN", "BABA", "DAL",
                    "DISH", "EBAY", "ETSY", "FB", "FSLY", "FVRR", "JMIA",
                    "LYFT", "MELI", "PTON", "ROKU", "SQ", "TTD", "UAL", "UBER",
                    "WORK", "AMWL", "BYND", "MCD", "SBUX", "CHTR",
                    "OKTA", "LYV", "MAR", "MNST", "OSTK", "SAVE", "SIX", "URBN",
                    "W",
                    # Consumer Goods
                    "AAPL", "ADS.DE", "ALXN", "AML.L", "APRN", "COST",
                    "NIO", "NIU", "NKE", "PEP", "PG",
                    "PM", "SHAK", "SNE", "TPR", "TSLA"
                    "TDOC", "PDD", "U", "MRNA", "CHWY", "WBA", "RACE",
                    "CL", "AEO", "BILI", "CHGG",
                    "IRBT", "RCL", "UA",
                    # Financial
                    "BAC", "BBD", "CIT", "ITUB", "JPM", "O", "SLV", "V",
                    "BRK-B", "MA", "BLK", "VTI", "AGNC", "BPY", "DB",
                    "MAIN", "OHI", "SPG", "STOR",
                    # Basic Materials
                    "GOLD", "NEM", "SLB", "XOM", "APD", "RDS-B",
                    "RIO",
                    # Industrial Goods
                    "BA", "LMT", "NTDOY", "RTX", "SIE.DE", "UPS", "NTLA",
                    "ENPH", "DHI", "ROK", "CAT", "UNP",
                    # Healthcare
                    "GILD", "PFE", "MDT", "ILMN",
                    # Utilities
                    "NEE", "OAOFY", #ATADL.L
                    # Coglomerates
                    "MMM"
                    ]

    # stockSymbols = ["NFLX", "DELL"]  # for testing

    def downloadData(data_3d, stockSymbols, current, i):
        # Get history data
        data_3d[i] = yf1.download(stockSymbols[i], period="3d")

        # Get current price
        ticker_yahoo = yf1.Ticker(stockSymbols[i])
        data = ticker_yahoo.history()
        current_ = (data.tail(1)['Close'].iloc[0])
        # print(f"Stock {stockSymbols[i]}: current {current_}")

        current[i] = current_

    # dummies
    data_3d = []
    current = []

    with Manager() as manager:
        # stockSymbols = [# Technology
        #                 "ATVI", "ADBE", "ADSK", "ANSS", "AMD", "CDNS", "CSCO"]
        a = manager.list(range(len(stockSymbols)))  # <-- can be shared between processes.
        b = manager.list(range(len(stockSymbols)))
        processes = [None] * len(stockSymbols)
        for i in range(len(stockSymbols)):
            p = Process(target=downloadData, args=(a, stockSymbols,
                                                   b, i))  # Passing the lists
            p.start()

            # Join processes at certain point, otherwise the RAM gets eaten
            # for huge lists
            step = 100
            if i % step == 0 and i != 0:
                p.join()
            elif i == len(stockSymbols) - 1:
                print(len(stockSymbols))
                print(i)
                p.join()

        # print(a[0]["Open"][-1])
        p.close()
        data_3d = list(a)
        current = list(b)

    return stockSymbols, current, data_3d

def filterStocks():

    import numpy as np
    import yfinance as yf1
    from multiprocessing import Process, Manager

    # Filter stocks
    stockSymbols, current, data_3d = ExtractStocksData()

    def filterData(stockSymbols, current, data_3d, filteredSymbols,
                   filteredPrevCloses, filteredOpens, filteredCurrents,
                   filteredOpenVsPrevClosePerc, filteredOpenVsCurrentPerc,
                   filteredOpenVsCurrentDiff, i):

        print("Stock: ", stockSymbols[i])
        # print("Open: ", data_3d[i]["Open"][-1])
        # print("Close: ", data_3d[i]["Close"][-2])

        # Check if the data is available
        if len(data_3d[i]["Close"]) > 2:
            premarket_up_percent = \
                (data_3d[i]["Open"][-1] / data_3d[i]["Close"][-2] - 1) * 100
            current_percent = (current[i] / data_3d[i]["Close"][-2] - 1) * 100
        else:
            premarket_up_percent = 0.0  # Dummy
            current_percent = 0.0  # Dummy

        # print("premarket_up_percent: ", premarket_up_percent)
        # print("current_percent: ", current_percent)

        # SMA - SIMPLE MOVING AVERAGE
        pandas_table = yf1.download(stockSymbols[i], period="180m", interval="1m")
        # Calculating the n-window simple moving average
        sma = pandas_table.rolling(window=50).mean()
        # print(f"Current: {current[i]}; SMA: {sma['Open'][-1]}")

        if (current_percent > premarket_up_percent) and \
           ((current_percent - premarket_up_percent) > 1.5) and \
           (sma["Open"][-1] < current[i]):
            filteredSymbols.append(stockSymbols[i])
            filteredPrevCloses.append(np.round(data_3d[i]["Close"][-2], 2))
            filteredOpens.append(np.round(data_3d[i]["Open"][-1], 2))
            filteredCurrents.append(np.round(current[i], 2))
            filteredOpenVsPrevClosePerc.append(
                np.round((filteredOpens[-1] / filteredPrevCloses[-1] - 1) * 100, 2))
            filteredOpenVsCurrentPerc.append(
                np.round((filteredCurrents[-1] / filteredPrevCloses[-1] - 1) * 100, 2))
            filteredOpenVsCurrentDiff.append(
                np.round(filteredOpenVsCurrentPerc[-1] - filteredOpenVsPrevClosePerc[-1], 2))

    # Dummies
    filteredSymbols = []
    filteredPrevCloses = []
    filteredOpens = []
    filteredCurrents = []
    filteredOpenVsPrevClosePerc = []
    filteredOpenVsCurrentPerc = []
    filteredOpenVsCurrentDiff = []

    with Manager() as manager:
        a = manager.list(range(len(stockSymbols)))  # <-- can be shared between processes.
        b = manager.list(range(len(stockSymbols)))
        c = manager.list(range(len(stockSymbols)))
        d = manager.list(range(len(stockSymbols)))
        e = manager.list(range(len(stockSymbols)))
        f = manager.list(range(len(stockSymbols)))
        g = manager.list(range(len(stockSymbols)))
        processes = []
        for i in range(len(stockSymbols)):
            p = Process(target=filterData, args=(stockSymbols, current,
                                                 data_3d, a, b, c, d, e, f, g,
                                                 i))  # Passing the lists
            p.start()

            # Join processes at certain point, otherwise the RAM gets eaten
            # for huge lists
            step = 100
            if i % step == 0 and i != 0:
                p.join()
            elif i == len(stockSymbols) - 1:
                print(len(stockSymbols))
                print(i)
                p.join()

        # print(a[0]["Open"][-1])
        p.close()
        filteredSymbols = list(a)
        filteredPrevCloses = list(b)
        filteredOpens = list(c)
        filteredCurrents = list(d)
        filteredOpenVsPrevClosePerc = list(e)
        filteredOpenVsCurrentPerc = list(f)
        filteredOpenVsCurrentDiff = list(g)

    # Remove obsolete integers from list
    filteredSymbols = [x for x in filteredSymbols if not isinstance(x, int)]
    filteredPrevCloses = [x for x in filteredPrevCloses if not isinstance(x, int)]
    filteredOpens = [x for x in filteredOpens if not isinstance(x, int)]
    filteredCurrents = [x for x in filteredCurrents if not isinstance(x, int)]
    filteredOpenVsPrevClosePerc = [x for x in filteredOpenVsPrevClosePerc if not isinstance(x, int)]
    filteredOpenVsCurrentPerc = [x for x in filteredOpenVsCurrentPerc if not isinstance(x, int)]
    filteredOpenVsCurrentDiff = [x for x in filteredOpenVsCurrentDiff if not isinstance(x, int)]

    zipped = zip(filteredSymbols, filteredPrevCloses, filteredOpens,
                 filteredCurrents, filteredOpenVsPrevClosePerc,
                 filteredOpenVsCurrentPerc, filteredOpenVsCurrentDiff)

    return zipped

if __name__ == "__main__":
    # test
    # stockSymbols, current, data_3d = ExtractStocksData()
    # print(stockSymbols)
    zipped = filterStocks()
    # print(zipped)