import sys, time, json, threading, queue, requests, io, base64, picamera, logging, datetime, configparser
import numpy as np
import RPi.GPIO as GPIO
from PIL import Image

try:
        config = configparser.ConfigParser()
        config.read('config.ini')
except:
        raise Exception("config.ini file missing.")

class DispenserClient:
    def __init__(self, ):

        self.route = config.get('SERVER', 'Route')
        self.init_logger()
        self.init_camera()
        self.init_ir_sensor()

        # temporary data structure for storage
        self.payload_queue =  queue.Queue() # If length no supply for queue, it's dynamics
        self.logger.info('dispenser client payload queue initialized')

        # http thread instantiate
        unit = config.get('PROPERTY', 'Unit')
        self.http = http_thread("http thread", self.node_id, type, unit, self.payload_queue)
        self.http.daemon = True
        self.http.start()
        self.logger.info('dispenser client http thread initialized')

    def init_logger(self, ):
        ## Retrive parameters from configuration file
        debug_level = config.get('DEBUG', 'LogLevel')
        property_type = config.get('PROPERTY', 'Type')

        ## Logger
        if debug_level == 'Info':
            level = logging.INFO
        else:
            level = logging.DEBUG

        self.logger = logging.getLogger(property_type)
        self.logger.setLevel(level)
        ch = logging.StreamHandler()
        #ch = logging.FileHandler('dispenser_client.log')
        ch.setLevel(level)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        ch.setFormatter(formatter)
        self.logger.addHandler(ch)

    def init_camera(self, ):
        ## Retrive parameters from configuration file
        resolution = config.get('CAMERA', 'Resolution')
        node_id = config.get('PROPERTY', 'Id')
        shape = config.get('CAMERA', 'Shape')
        width = config.getint('CAMERA', 'Width')
        height = config.getint('CAMERA', 'Height')
        channel = config.getint('CAMERA', 'Channel')
        rotation = config.getint('CAMERA', 'Rotation')

        ## Camera
        self.camera = picamera.PiCamera()
        self.camera.rotation = rotation
        self.camera.resolution = resolution
        self.node_id = node_id
        self.shape = shape
        size = (width, height, channel)
        self.image = np.empty(size, dtype=np.uint8)
        self.camera.start_preview(fullscreen=False, window = (100,20,0,0))
        self.logger.info('dispenser client camera initialized, start_preview executed')

    def init_ir_sensor(self, ):
        GPIO.setwarnings(False)
        GPIO.setmode(GPIO.BCM)
        GPIO.setup(4, GPIO.IN) #motion sensor
        self.logger.info('dispenser client GPIO check')

    def capture(self):
        self.camera.capture(self.image, 'rgb')
        image_temp = self.image.astype(np.float64)
        image_64 = base64.b64encode(image_temp).decode('ascii')
        payload = {'NodeID': self.node_id, 'Timestamp': time.time() ,'Image': image_64, 'Shape': self.shape}
        headers = {'Content_Type': 'application/json', 'Accept': 'text/plain'}
        self.payload_queue.put((payload, headers, self.route)) # dispenser thread

    def update_route(self, new_route):
        self.route = new_route

    def update_node_id(self, new_node_id):
        self.node_id = new_node_id

    def update_unit(self, new_unit):
        self.unit = new_unit

class http_thread(threading.Thread):
    def __init__(self, name, node_id, type, unit, payload_queue):
        ###########################################################################################
        # Logger
        level = self.log_level(config.get('DEBUG', 'LogLevel'))
        self.logger = logging.getLogger('http thread')
        self.logger.setLevel(level)
        ch = logging.StreamHandler()
        #ch = logging.FileHandler('dispenser_client.log')
        ch.setLevel(level)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        ch.setFormatter(formatter)
        self.logger.addHandler(ch)
        ###########################################################################################

        # Initialize thread
        threading.Thread.__init__(self)
        self.name = name
        self.node_id = node_id
        self.type = type
        self.unit = unit
        self.payload_queue = payload_queue
        self.storage_queue = queue.Queue()

        # second_http thread instantiate
        self.second_http = second_http_thread("http thread", node_id, type, unit, self.storage_queue)
        self.second_http.daemon = True
        self.second_http.start()
        self.logger.info('dispenser client second http thread initialized')

    def log_level(self, level):
        if level == 'Info':
            return logging.INFO
        else:
            return logging.DEBUG

    def run(self):
        while 1:
            if self.payload_queue.qsize():
                payload, headers, route = self.payload_queue.get()
                try:
                    result = requests.post(route, json=payload, headers=headers)
                    code = result.status_code
                    self.logger.debug(result.json())
                    if code == 200:
                        self.logger.info('HTTP request received.')
                    else:
                        self.storage_queue.put((payload, headers, route))
                except:
                    self.logger.info("Failed to establish connection with server. Try again in 5s.")
                    self.storage_queue.put((payload, headers, route))

class second_http_thread(threading.Thread):
    def __init__(self, name, node_id, type, unit, storage_queue):
        ###########################################################################################
        # Logger
        level = self.log_level(config.get('DEBUG', 'LogLevel'))
        self.logger = logging.getLogger('second http thread')
        self.logger.setLevel(level)
        ch = logging.FileHandler('dispenser_client.log')
        ch.setLevel(level)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        ch.setFormatter(formatter)
        self.logger.addHandler(ch)
        ###########################################################################################

        # Initialize thread
        threading.Thread.__init__(self)
        self.name = name
        self.node_id = node_id
        self.type = type
        self.unit = unit
        self.storage_queue = storage_queue

    def log_level(self, level):
        if level == 'Info':
            return logging.INFO
        else:
            return logging.DEBUG

    def run(self):
        while 1:
            if self.storage_queue.qsize():
                time.sleep(5) # wait for 5 seconds before second attempt
                payload, headers, route = self.storage_queue.get()
                try:
                    result = requests.post(route, json=payload, headers=headers)
                    code = result.status_code
                    self.logger.debug(result.json())
                    if code == 200:
                        self.logger.info("second attempt successful.")
                    else:
                        self.logger.info("second attempt failed with code: " + str(code))
                        self.storage_queue.put((payload, headers, route))
                except:
                    self.storage_queue.put((payload, headers, route))
                    self.logger.info("Failed to establish connection with server again. Try again in 30s.")
                    time.sleep(25)

if __name__ == "__main__":

    # Initialization
    client = DispenserClient()

    #Main loop
    while 1:
        try:
            if GPIO.input(4):
                time.sleep(0.25)
            else:
                cur_time = time.time()
                respond = client.capture()
                time.sleep(2)
        except KeyboardInterrupt:
            sys.exit()


class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

    def __iter__(self):
        p = self
        while p:
            yield p.val
            p = p.next

def addTwoNumbers(l1, l2):
    p1 = l1
    p2 = l2
    flag = False
    head = ListNode(-1)
    pr = head
    while p1 and p2:
        pr.next = ListNode((p1.val + p2.val + int(flag)) % 10)
        flag = p1.val + p2.val + int(flag) > 9
        pr = pr.next
        p1 = p1.next
        p2 = p2.next

    if p1:
        pr.next = p1
    elif p2:
        pr.next = p2

    while pr.next:
        value = (pr.next.val + int(flag)) % 10
        flag = (pr.next.val + int(flag)) > 9
        pr.next.val = value
        pr = pr.next

    if flag:
        pr.next = ListNode(1)
    return head.next

if __name__ == '__main__':
    n1 = ListNode(0)
    n2 = ListNode(0)
    assert addTwoNumbers(n1, n2).val == 0

    n1 = ListNode(5)
    n2 = ListNode(6)
    assert addTwoNumbers(n1, n2).val == 1
    assert addTwoNumbers(n1, n2).next.val == 1

    n1 = ListNode(9)
    n2 = ListNode(9)
    n2.next = ListNode(9)
    n2.next.next = ListNode(9)
    assert list(addTwoNumbers(n1, n2)) == [8, 0, 0, 1]
    assert list(addTwoNumbers(n2, n1)) == [8, 0, 0, 1]

top-interview-questions/Medium/2-Linked List/1-Add Two Numbers.py


# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Fabian Pedregosa <fabian.pedregosa@inria.fr>
#
# License: BSD 3 clause

from math import log

import numpy as np
import pytest

from sklearn import datasets
from sklearn.linear_model import ARDRegression, BayesianRidge, Ridge
from sklearn.utils import check_random_state
from sklearn.utils._testing import (
    assert_almost_equal,
    assert_array_almost_equal,
    assert_array_less,
)
from sklearn.utils.extmath import fast_logdet

diabetes = datasets.load_diabetes()

def test_bayesian_ridge_scores():
    """Check scores attribute shape"""
    X, y = diabetes.data, diabetes.target

    clf = BayesianRidge(compute_score=True)
    clf.fit(X, y)

    assert clf.scores_.shape == (clf.n_iter_ + 1,)

def test_bayesian_ridge_score_values():
    """Check value of score on toy example.

    Compute log marginal likelihood with equation (36) in Sparse Bayesian
    Learning and the Relevance Vector Machine (Tipping, 2001):

    - 0.5 * (log |Id/alpha + X.X^T/lambda| +
             y^T.(Id/alpha + X.X^T/lambda).y + n * log(2 * pi))
    + lambda_1 * log(lambda) - lambda_2 * lambda
    + alpha_1 * log(alpha) - alpha_2 * alpha

    and check equality with the score computed during training.
    """

    X, y = diabetes.data, diabetes.target
    n_samples = X.shape[0]
    # check with initial values of alpha and lambda (see code for the values)
    eps = np.finfo(np.float64).eps
    alpha_ = 1.0 / (np.var(y) + eps)
    lambda_ = 1.0

    # value of the parameters of the Gamma hyperpriors
    alpha_1 = 0.1
    alpha_2 = 0.1
    lambda_1 = 0.1
    lambda_2 = 0.1

    # compute score using formula of docstring
    score = lambda_1 * log(lambda_) - lambda_2 * lambda_
    score += alpha_1 * log(alpha_) - alpha_2 * alpha_
    M = 1.0 / alpha_ * np.eye(n_samples) + 1.0 / lambda_ * np.dot(X, X.T)
    M_inv_dot_y = np.linalg.solve(M, y)
    score += -0.5 * (
        fast_logdet(M) + np.dot(y.T, M_inv_dot_y) + n_samples * log(2 * np.pi)
    )

    # compute score with BayesianRidge
    clf = BayesianRidge(
        alpha_1=alpha_1,
        alpha_2=alpha_2,
        lambda_1=lambda_1,
        lambda_2=lambda_2,
        max_iter=1,
        fit_intercept=False,
        compute_score=True,
    )
    clf.fit(X, y)

    assert_almost_equal(clf.scores_[0], score, decimal=9)

def test_bayesian_ridge_parameter():
    # Test correctness of lambda_ and alpha_ parameters (GitHub issue #8224)
    X = np.array([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
    y = np.array([1, 2, 3, 2, 0, 4, 5]).T

    # A Ridge regression model using an alpha value equal to the ratio of
    # lambda_ and alpha_ from the Bayesian Ridge model must be identical
    br_model = BayesianRidge(compute_score=True).fit(X, y)
    rr_model = Ridge(alpha=br_model.lambda_ / br_model.alpha_).fit(X, y)
    assert_array_almost_equal(rr_model.coef_, br_model.coef_)
    assert_almost_equal(rr_model.intercept_, br_model.intercept_)

def test_bayesian_sample_weights():
    # Test correctness of the sample_weights method
    X = np.array([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
    y = np.array([1, 2, 3, 2, 0, 4, 5]).T
    w = np.array([4, 3, 3, 1, 1, 2, 3]).T

    # A Ridge regression model using an alpha value equal to the ratio of
    # lambda_ and alpha_ from the Bayesian Ridge model must be identical
    br_model = BayesianRidge(compute_score=True).fit(X, y, sample_weight=w)
    rr_model = Ridge(alpha=br_model.lambda_ / br_model.alpha_).fit(
        X, y, sample_weight=w
    )
    assert_array_almost_equal(rr_model.coef_, br_model.coef_)
    assert_almost_equal(rr_model.intercept_, br_model.intercept_)

def test_toy_bayesian_ridge_object():
    # Test BayesianRidge on toy
    X = np.array([[1], [2], [6], [8], [10]])
    Y = np.array([1, 2, 6, 8, 10])
    clf = BayesianRidge(compute_score=True)
    clf.fit(X, Y)

    # Check that the model could approximately learn the identity function
    test = [[1], [3], [4]]
    assert_array_almost_equal(clf.predict(test), [1, 3, 4], 2)

def test_bayesian_initial_params():
    # Test BayesianRidge with initial values (alpha_init, lambda_init)
    X = np.vander(np.linspace(0, 4, 5), 4)
    y = np.array([0.0, 1.0, 0.0, -1.0, 0.0])  # y = (x^3 - 6x^2 + 8x) / 3

    # In this case, starting from the default initial values will increase
    # the bias of the fitted curve. So, lambda_init should be small.
    reg = BayesianRidge(alpha_init=1.0, lambda_init=1e-3)
    # Check the R2 score nearly equals to one.
    r2 = reg.fit(X, y).score(X, y)
    assert_almost_equal(r2, 1.0)

def test_prediction_bayesian_ridge_ard_with_constant_input():
    # Test BayesianRidge and ARDRegression predictions for edge case of
    # constant target vectors
    n_samples = 4
    n_features = 5
    random_state = check_random_state(42)
    constant_value = random_state.rand()
    X = random_state.random_sample((n_samples, n_features))
    y = np.full(n_samples, constant_value, dtype=np.array(constant_value).dtype)
    expected = np.full(n_samples, constant_value, dtype=np.array(constant_value).dtype)

    for clf in [BayesianRidge(), ARDRegression()]:
        y_pred = clf.fit(X, y).predict(X)
        assert_array_almost_equal(y_pred, expected)

def test_std_bayesian_ridge_ard_with_constant_input():
    # Test BayesianRidge and ARDRegression standard dev. for edge case of
    # constant target vector
    # The standard dev. should be relatively small (< 0.01 is tested here)
    n_samples = 10
    n_features = 5
    random_state = check_random_state(42)
    constant_value = random_state.rand()
    X = random_state.random_sample((n_samples, n_features))
    y = np.full(n_samples, constant_value, dtype=np.array(constant_value).dtype)
    expected_upper_boundary = 0.01

    for clf in [BayesianRidge(), ARDRegression()]:
        _, y_std = clf.fit(X, y).predict(X, return_std=True)
        assert_array_less(y_std, expected_upper_boundary)

def test_update_of_sigma_in_ard():
    # Checks that `sigma_` is updated correctly after the last iteration
    # of the ARDRegression algorithm. See issue #10128.
    X = np.array([[1, 0], [0, 0]])
    y = np.array([0, 0])
    clf = ARDRegression(max_iter=1)
    clf.fit(X, y)
    # With the inputs above, ARDRegression prunes both of the two coefficients
    # in the first iteration. Hence, the expected shape of `sigma_` is (0, 0).
    assert clf.sigma_.shape == (0, 0)
    # Ensure that no error is thrown at prediction stage
    clf.predict(X, return_std=True)

def test_toy_ard_object():
    # Test BayesianRegression ARD classifier
    X = np.array([[1], [2], [3]])
    Y = np.array([1, 2, 3])
    clf = ARDRegression(compute_score=True)
    clf.fit(X, Y)

    # Check that the model could approximately learn the identity function
    test = [[1], [3], [4]]
    assert_array_almost_equal(clf.predict(test), [1, 3, 4], 2)

@pytest.mark.parametrize("n_samples, n_features", ((10, 100), (100, 10)))
def test_ard_accuracy_on_easy_problem(global_random_seed, n_samples, n_features):
    # Check that ARD converges with reasonable accuracy on an easy problem
    # (Github issue #14055)
    X = np.random.RandomState(global_random_seed).normal(size=(250, 3))
    y = X[:, 1]

    regressor = ARDRegression()
    regressor.fit(X, y)

    abs_coef_error = np.abs(1 - regressor.coef_[1])
    assert abs_coef_error < 1e-10

def test_return_std():
    # Test return_std option for both Bayesian regressors
    def f(X):
        return np.dot(X, w) + b

    def f_noise(X, noise_mult):
        return f(X) + np.random.randn(X.shape[0]) * noise_mult

    d = 5
    n_train = 50
    n_test = 10

    w = np.array([1.0, 0.0, 1.0, -1.0, 0.0])
    b = 1.0

    X = np.random.random((n_train, d))
    X_test = np.random.random((n_test, d))

    for decimal, noise_mult in enumerate([1, 0.1, 0.01]):
        y = f_noise(X, noise_mult)

        m1 = BayesianRidge()
        m1.fit(X, y)
        y_mean1, y_std1 = m1.predict(X_test, return_std=True)
        assert_array_almost_equal(y_std1, noise_mult, decimal=decimal)

        m2 = ARDRegression()
        m2.fit(X, y)
        y_mean2, y_std2 = m2.predict(X_test, return_std=True)
        assert_array_almost_equal(y_std2, noise_mult, decimal=decimal)

def test_update_sigma(global_random_seed):
    # make sure the two update_sigma() helpers are equivalent. The woodbury
    # formula is used when n_samples < n_features, and the other one is used
    # otherwise.

    rng = np.random.RandomState(global_random_seed)

    # set n_samples == n_features to avoid instability issues when inverting
    # the matrices. Using the woodbury formula would be unstable when
    # n_samples > n_features
    n_samples = n_features = 10
    X = rng.randn(n_samples, n_features)
    alpha = 1
    lmbda = np.arange(1, n_features + 1)
    keep_lambda = np.array([True] * n_features)

    reg = ARDRegression()

    sigma = reg._update_sigma(X, alpha, lmbda, keep_lambda)
    sigma_woodbury = reg._update_sigma_woodbury(X, alpha, lmbda, keep_lambda)

    np.testing.assert_allclose(sigma, sigma_woodbury)

@pytest.mark.parametrize("dtype", [np.float32, np.float64])
@pytest.mark.parametrize("Estimator", [BayesianRidge, ARDRegression])
def test_dtype_match(dtype, Estimator):
    # Test that np.float32 input data is not cast to np.float64 when possible
    X = np.array([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]], dtype=dtype)
    y = np.array([1, 2, 3, 2, 0, 4, 5]).T

    model = Estimator()
    # check type consistency
    model.fit(X, y)
    attributes = ["coef_", "sigma_"]
    for attribute in attributes:
        assert getattr(model, attribute).dtype == X.dtype

    y_mean, y_std = model.predict(X, return_std=True)
    assert y_mean.dtype == X.dtype
    assert y_std.dtype == X.dtype

@pytest.mark.parametrize("Estimator", [BayesianRidge, ARDRegression])
def test_dtype_correctness(Estimator):
    X = np.array([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
    y = np.array([1, 2, 3, 2, 0, 4, 5]).T
    model = Estimator()
    coef_32 = model.fit(X.astype(np.float32), y).coef_
    coef_64 = model.fit(X.astype(np.float64), y).coef_
    np.testing.assert_allclose(coef_32, coef_64, rtol=1e-4)

# TODO(1.5) remove
@pytest.mark.parametrize("Estimator", [BayesianRidge, ARDRegression])
def test_bayesian_ridge_ard_n_iter_deprecated(Estimator):
    """Check the deprecation warning of `n_iter`."""
    depr_msg = (
        "'n_iter' was renamed to 'max_iter' in version 1.3 and will be removed in 1.5"
    )
    X, y = diabetes.data, diabetes.target
    model = Estimator(n_iter=5)

    with pytest.warns(FutureWarning, match=depr_msg):
        model.fit(X, y)

# TODO(1.5) remove
@pytest.mark.parametrize("Estimator", [BayesianRidge, ARDRegression])
def test_bayesian_ridge_ard_max_iter_and_n_iter_both_set(Estimator):
    """Check that a ValueError is raised when both `max_iter` and `n_iter` are set."""
    err_msg = (
        "Both `n_iter` and `max_iter` attributes were set. Attribute"
        " `n_iter` was deprecated in version 1.3 and will be removed in"
        " 1.5. To avoid this error, only set the `max_iter` attribute."
    )
    X, y = diabetes.data, diabetes.target
    model = Estimator(n_iter=5, max_iter=5)

    with pytest.raises(ValueError, match=err_msg):
        model.fit(X, y)


from dash import html, dcc, Input, Output, State
from styles.styles import *

def sliders(label, description, min, max, step, marks, value, id_name):
    br = html.Br()
    label = html.Label(label)
    desc = html.P(description, className="description")
    slider = dcc.Slider(id=id_name, min=min, max=max, step=step, marks={i: str(i) for i in marks}, value=value)
    return [br, label, desc, slider]

low_marks = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
generations = [1, 2, 3, 4, 5]
high_marks = [20, 30, 40, 50, 60]

top_p_slider = sliders("Top P", top_p_description, 0.1, 1.0, 0.1, low_marks, 0.8, "top-p")
temperature_slider = sliders("Temperature", temperature_description, 0.1, 1.0, 0.1, low_marks, 1.0, "temp")
output_length = sliders("Poem Length", output_len_description, 20, 60, 10, high_marks, 30, "poem-length")
options_number = sliders("Number of poems", options_num_description, 1, 5, 1, generations, 1, "n-poems")

sliders_elements = top_p_slider + temperature_slider + output_length + options_number

header = html.Div(style=header_style, children=[
    html.H1(children='AiLan-P03m', style=header_text_style)
    ])

subheader = html.Div(children=[
    html.H2('Generador de poemas basado en Transformers')], style=div_header_child_style)

body = html.Div([
    html.Div(children=sliders_elements, style=inputs_div_style, id="inputs-left"),
    html.Div(children=[
        html.Label('Fragmento de input'),
        html.P(className="description", children="Insertar el fragmento de poesía a partir del cual al modelo hará la generación"),
        dcc.Textarea(style=text_area, id="input-text",
                     value="Esto es un ejemplo\ndel input que debes dar\nsustituye estas líneas\npor tu poema")
    ], style=inputs_div_style, id="inputs-right")
], style=body_style)

button = html.Div(style=under_div_style, children=[
    html.Button("Generar", id="submit-button", n_clicks=0),
])

footer = dcc.Loading(children=html.Div(id="my-poems", style=footer_style), type="graph", fullscreen=False)


"""
    Написать программу, которая принимает номер телефона в любом формате:
    +38 (050) 12-34-567 или 099 123 -45 67 или 80501234567 или 888 050 123 4567
    а выводит в формате: 380501234567.
    Если цифр в номере недостаточно, чтобы описать номер в нужном формате -
        попросить пользователя повторить ввод.
"""

phone = input('Enter phone number: ')
number = ''.join(char for char in phone if char.isdigit())
if number.rfind('8', 0, 5) >= 1:
    number = '3' + number[number.rfind('8', 0, 5) :]
    if len(number) == 12:
        print(number)
    else:
        print("Недостатньо цифр в номері, повторіть")
elif number[:1] == '8':
    number = '3' + number
    if len(number) == 12:
        print(number)
    else:
        print("Недостатньо цифр в номері, повторіть")
# elif number[:2] == '38':
#     if len(number) == 12:
#         print(number)
#     else:
#         print("Недостатньо цифр в номері, повторіть")
else:
    number = '38' + number
    if len(number) == 12:
        print(number)
    else:
        print("Недостатньо цифр в номері, повторіть")


n = int(input())
cs = list(map(int, input().split()))
ma = (0, 0, 0)
for i in range(1, 10):
    ma = max(ma, ((n // cs[i - 1], i, cs[i - 1])))
res = 0
mo = (0, 0)
for i in range(1, 10):
    if ma[2] + n % ma[2] >= cs[i - 1]:
        mo = max(mo, (i, cs[i - 1]))
n -= mo[1]
res += mo[0]
while n >= ma[2]:
    res = res * 10 + ma[1]
    n -= ma[2]
print(res)


import time

ordered_list = []
for number in range(10):
    ordered_list.append(number)
print(ordered_list)

user_value = input("Now enter your number: ")
time.sleep(1)
print("Now we're going to see if your number belongs to the list...")
time.sleep(1)
for number2 in ordered_list:
    if number2 == int(user_value):
        print("Yes, it's a match!")


#!/usr/local/bin/python3

# counter exercise, return sum of my_list
# my_list = [1,2,3,4,5,6,7,8,9,10]

def counter_checker1(num):
    # Solution 1
    try:
        my_list = sum([x for x in range(1, int(num), 1)])
        return my_list
    except ValueError as err:
        return err

def counter_checker2(num):
    # Solution 2
    total_sum = 0
    for num in range(1, num, 1):
        total_sum += num
    return total_sum

if __name__ == '__main__':
    solution1 = counter_checker1('')
    print(f'Solution 1 Sum is: {solution1}', end='\n')  # Answer = 55
    solution2 = counter_checker2(11)
    print(f'Solution 2 Sum is: {solution2}', end='\n')  # Answer = 55
    print("All done!")


# coding=utf-8
import numpy as np
from feature import Counter, Feature
import sqlite3

db = sqlite3.connect('ptt.db')
cur = db.execute('select * from articles')
f = Feature()
a = 0
for i in cur:
    a += 1
    try:
        f.features(i[5])
    except:
        print(i[5])
    print(f.size())

f.store()


import csv
import os
import typing
from collections import OrderedDict
from datetime import datetime
from pathlib import PurePosixPath

from sqlalchemy import or_
from sqlalchemy.orm import Session

from urllib.parse import urlparse

from backend.common.entities.dataset_asset import DatasetAsset
from backend.common.entities.entity import Entity
from backend.common.entities.geneset import Geneset
from backend.common.corpora_orm import (
    DbDataset,
    DbDatasetArtifact,
    DbDatasetProcessingStatus,
    UploadStatus,
    ProcessingStatus,
    DbGenesetDatasetLink,
    generate_id,
    DatasetArtifactFileType,
    ConversionStatus,
)
from backend.common.utils.corpora_constants import CorporaConstants
from backend.common.utils.db_helpers import clone
from backend.common.utils.ontology_mappings.ontology_map_loader import ontology_mappings
from backend.common.utils.s3_buckets import buckets

class Dataset(Entity):
    table = DbDataset

    def __init__(self, db_object: DbDataset):
        super().__init__(db_object)

    @classmethod
    def create(
        cls,
        session: Session,
        revision: int = 0,
        name: str = "",
        artifacts: list = None,
        processing_status: dict = None,
        **kwargs,
    ) -> "Dataset":
        """
        Creates a new dataset and related objects and store in the database. IDs are generated for all new table
        entries.
        """
        dataset = DbDataset(
            revision=revision,
            name=name,
            **kwargs,
        )
        if artifacts:
            dataset.artifacts = [DbDatasetArtifact(dataset_id=dataset.id, **art) for art in artifacts]
        processing_status = processing_status if processing_status else {}
        dataset.processing_status = DbDatasetProcessingStatus(dataset_id=dataset.id, **processing_status)
        session.add(dataset)
        session.commit()

        return cls(dataset)

    def update(self, artifacts: list = None, processing_status: dict = None, commit=True, **kwargs) -> None:
        """
        Update an existing dataset to match provided the parameters. The specified column are replaced.
        :param artifacts: Artifacts to create and connect to the dataset. If present, the existing attached entries will
         be removed and replaced with new entries. If an empty list is provided, all dataset artifacts will be deleted.
        :param processing_status: A Processing status entity to create and connect to the dataset. If present, the
         existing attached entries will be removed and replaced with new entries. If an empty dictionary is provided the
         processing_status will be deleted.
        :param kwargs: Any other fields in the dataset that will be replaced.
        """
        if any([i is not None for i in [artifacts, processing_status]]):
            if artifacts is not None:
                for af in self.artifacts:
                    self.session.delete(af)
                new_objs = [DbDatasetArtifact(dataset_id=self.id, **art) for art in artifacts]
                self.session.add_all(new_objs)
            if processing_status is not None:
                if self.processing_status:
                    self.session.delete(self.processing_status)
                if processing_status:
                    new_obj = DbDatasetProcessingStatus(dataset_id=self.id, **processing_status)
                    self.session.add(new_obj)
        super().update(commit=commit, **kwargs)

    @classmethod
    def get(
        cls, session: Session, dataset_id=None, include_tombstones=False, collection_id=None
    ) -> typing.Optional["Dataset"]:
        filters = []
        if not include_tombstones:
            filters.append(cls.table.tombstone != True)  # noqa
        if collection_id:
            filters.append(cls.table.collection_id == collection_id)
        if dataset_id:
            filters.append(cls.table.id == dataset_id)
        result = session.query(cls.table).filter(*filters).one_or_none()
        dataset = cls(result) if result else None
        return dataset

    @classmethod
    def get_by_explorer_url(cls, session: Session, explorer_url):
        """
        Return the most recently created dataset with the given explorer_url or None
        """

        def _get_by_explorer_url_query(url):
            filters = [cls.table.explorer_url == url]
            dataset = session.query(cls.table).filter(*filters).order_by(cls.table.created_at.desc()).limit(1).all()
            return cls(dataset[0]) if dataset else None

        # If dataset cannot be found, look for the url with (or without) the final slash
        dataset = _get_by_explorer_url_query(explorer_url)
        if not dataset:
            if explorer_url[-1] == "/":
                dataset = _get_by_explorer_url_query(explorer_url[:-1])
            else:
                dataset = _get_by_explorer_url_query(f"{explorer_url}/")

        return dataset

    @classmethod
    def list(
        cls, session: Session, collection_id: str = None, include_tombstones: bool = False
    ) -> typing.List["Dataset"]:
        """
        Retrieves a list of Datasets from the database

        :param session: an open database session
        :param include_tombstones: True of to include tombstoned datasets
        :return: A list of Datasets
        """
        filters = []

        if not include_tombstones:
            filters.append(or_(cls.table.tombstone == False, cls.table.tombstone == None))  # noqa
        if collection_id:
            filters.append(cls.table.collection_id == collection_id)

        if filters:
            return [cls(obj) for obj in session.query(cls.table).filter(*filters).all()]
        else:
            return [cls(obj) for obj in session.query(cls.table)]

    def get_asset(self, asset_id) -> typing.Union[DatasetAsset, None]:
        """
        Retrieve the asset if it exists in the dataset.
        :param asset_id: uuid of the asset to find
        :return: If the asset is found it is returned, else None is returned.
        """
        asset = [asset for asset in self.artifacts if asset.id == asset_id]
        return None if not asset else DatasetAsset(asset[0])

    def get_assets(self):
        """
        Retrieve all the assets for the dataset
        """
        assets = [
            asset for asset in self.artifacts if asset.filename != CorporaConstants.ORIGINAL_H5AD_ARTIFACT_FILENAME
        ]
        return assets

    @staticmethod
    def transform_sex_for_schema_2_0_0(dataset):
        schema_version = dataset.get("schema_version")
        if "sex" in dataset and (schema_version is None or schema_version < "2.0.0"):
            dataset["sex"] = [{"label": s, "sex_ontology_term_id": "unknown"} for s in dataset["sex"]]

    @staticmethod
    def transform_organism_for_schema_2_0_0(dataset):
        schema_version = dataset.get("schema_version")
        # If organism is an object (version 1.1.0), wrap it into an array to be 2.0.0 compliant
        if "organism" in dataset and (schema_version is None or schema_version < "2.0.0"):
            dataset["organism"] = [dataset["organism"]]

    @staticmethod
    def enrich_development_stage_with_ancestors(dataset):
        Dataset._enrich_with_ancestors(
            dataset, "development_stage", ontology_mappings.development_stage_ontology_mapping
        )

    @staticmethod
    def enrich_tissue_with_ancestors(dataset):
        """
        Tag dataset with ancestors for all tissues in the given dataset, if any.
        """
        Dataset._enrich_with_ancestors(dataset, "tissue", ontology_mappings.tissue_ontology_mapping)

    @staticmethod
    def enrich_cell_type_with_ancestors(dataset):
        """
        Tag dataset with ancestors for all cell types in the given dataset, if any.
        """
        Dataset._enrich_with_ancestors(dataset, "cell_type", ontology_mappings.cell_type_ontology_mapping)

    def _enrich_with_ancestors(dataset, key, ontology_mapping):
        """
        Tag dataset with ancestors for all values of the given key, if any.
        """
        if key not in dataset:
            return

        terms = [e["ontology_term_id"] for e in dataset[key]]

        if not terms:
            return

        ancestors = [ontology_mapping.get(term) for term in terms]
        flattened_ancestors = [item for sublist in ancestors if sublist for item in sublist]
        unique_ancestors = list(OrderedDict.fromkeys(flattened_ancestors))
        if unique_ancestors:
            dataset[f"{key}_ancestors"] = unique_ancestors

    def _create_new_explorer_url(self, new_id: str) -> str:
        if self.explorer_url is None:
            return None
        original_url = urlparse(self.explorer_url)
        original_path = PurePosixPath(original_url.path)
        new_path = str(original_path.parent / f"{new_id}.cxg/")
        new_url = original_url._replace(path=new_path).geturl()
        # Note: the final slash is mandatory, otherwise the explorer won't load this link
        return f"{new_url}/"

    def create_revision(self, revision_collection_id: str) -> "Dataset":
        """
        Generate a dataset revision from a dataset in a public collection
        :param revision_collection_id: specify the collection revision to which this dataset revision belongs
        :return: dataset revision.

        """
        revision_dataset_id = generate_id()
        revision_explorer_url = self._create_new_explorer_url(revision_dataset_id)
        revision_dataset = clone(
            self.db_object,
            id=revision_dataset_id,
            collection_id=revision_collection_id,
            original_id=self.id,
            explorer_url=revision_explorer_url,
        )
        self.session.add(revision_dataset)
        for artifact in self.artifacts:
            self.session.add(clone(artifact, dataset_id=revision_dataset.id))
        if self.processing_status:
            self.session.add(clone(self.processing_status, dataset_id=revision_dataset.id))
        self.session.commit()
        return Dataset(revision_dataset)

    def tombstone_dataset_and_delete_child_objects(self):
        self.update(tombstone=True, artifacts=[], processing_status={})
        self.session.query(DbGenesetDatasetLink).filter(DbGenesetDatasetLink.dataset_id == self.id).delete(
            synchronize_session="evaluate"
        )

    def asset_deletion(self):
        for artifact in self.artifacts:
            asset = DatasetAsset.get(self.session, artifact.id)
            asset.queue_s3_asset_for_deletion()
            asset.delete(commit=False)

    @staticmethod
    def new_processing_status() -> dict:
        return {
            "upload_status": UploadStatus.WAITING,
            "upload_progress": 0,
            "processing_status": ProcessingStatus.PENDING,
        }

    def copy_csv_to_s3(self, csv_file: str) -> str:
        object_name = get_cxg_bucket_path(self.explorer_url)
        s3_file = f"{object_name}-genesets.csv"
        buckets.explorer_bucket.upload_file(csv_file, s3_file)
        return s3_file

    def generate_tidy_csv_for_all_linked_genesets(self, csv_file_path: str) -> str:
        csv_file = os.path.join(csv_file_path, "geneset.csv")
        fieldnames = ["GENE_SET_NAME", "GENE_SET_DESCRIPTION", "GENE_SYMBOL", "GENE_DESCRIPTION"]
        genesets = []
        max_additional_params = 0
        for geneset in self.genesets:
            geneset_entity = Geneset(geneset)
            gene_rows, gene_max = geneset_entity.convert_geneset_to_gene_dicts()
            if gene_max > max_additional_params:
                max_additional_params = gene_max
            genesets += gene_rows

        for i in range(1, max_additional_params + 1):
            fieldnames.append(f"PROVENANCE{i}")
            fieldnames.append(f"PROVENANCE{i}_DESCRIPTION")

        with open(csv_file, "w") as f:
            writer = csv.DictWriter(f, fieldnames=fieldnames)
            writer.writeheader()
            for gene in genesets:
                writer.writerow(gene)
        return csv_file

    def reprocess(self):
        if not self.published:
            self.asset_deletion()
        self.update(
            name="",
            organism=None,
            tissue=None,
            assay=None,
            disease=None,
            sex=None,
            self_reported_ethnicity=None,
            development_stage=None,
            cell_type=None,
            published=False,
            revision=self.revision + 1,
            explorer_url=None,
            artifacts=[],
        )

    def get_most_recent_artifact(self, filetype=DatasetArtifactFileType.CXG):
        filters = [DbDatasetArtifact.dataset_id == self.id, DbDatasetArtifact.filetype == filetype]
        artifact = (
            self.session.query(DbDatasetArtifact)
            .filter(*filters)
            .order_by(DbDatasetArtifact.created_at.desc())
            .limit(1)
            .all()
        )
        return DatasetAsset(artifact[0]) if artifact else None

    def publish_new(self, now: datetime):
        """
        Publish a new dataset with the published_at datetime populated
        with the provided datetime.
        :param now: Datetime to populate dataset's published_at.
        """
        collection_id = self.collection.revision_of if self.collection.revision_of else self.collection.id
        self.update(collection_id=collection_id, published=True, published_at=now, commit=False)

    def publish_revision(self, revision: "Dataset", now: datetime) -> bool:
        """
        Publish a revision of a dataset if the dataset under revision is
        different from the existing dataset.
        :return: True if revision differs from existing dataset, else False.
        """
        if revision.tombstone or revision.revision > self.revision:
            # If the revision is different from the original
            self.asset_deletion()
            if revision.revision > self.revision:
                # connect revised artifacts with published dataset
                for artifact in revision.artifacts:
                    if (
                        artifact.filetype == DatasetArtifactFileType.RDS
                        and revision.processing_status.rds_status == ConversionStatus.SKIPPED
                    ):
                        # Delete old .rds (Seurat) dataset artifact clones if rds_status for dataset revision is SKIPPED
                        DatasetAsset(artifact).delete(commit=False)
                    else:
                        artifact.dataset_id = self.id

            elif revision.tombstone:
                # tombstone
                revision.tombstone_dataset_and_delete_child_objects()

            updates = revision.to_dict(
                remove_attr=[
                    "updated_at",
                    "created_at",
                    "id",
                    "collection_id",
                    "original_id",
                    "published",
                    "revised_at",
                    "explorer_url",
                ],
                remove_relationships=True,
            )

            if revision.tombstone is not False:
                self.update(commit=False, **updates)
            else:
                # There was an update to a dataset, so update revised_at
                self.update(commit=False, **updates, revised_at=now)

            return True
        return False

def get_cxg_bucket_path(explorer_url: str) -> str:
    """Parses the S3 cellxgene bucket object prefix for all resources related to this dataset from the explorer_url"""
    object_name = urlparse(explorer_url).path.split("/", 2)[2]
    if object_name.endswith("/"):
        object_name = object_name[:-1]
    base, _ = os.path.splitext(object_name)
    return base


from django.db import models

class Category(models.Model):
    parent_category = models.ForeignKey(
        "Category", verbose_name='Родительская категория',
        related_name='children', null=True, blank=True,
        on_delete=models.CASCADE
    )
    name = models.CharField(verbose_name='Название', max_length=200)

    class Meta:
        verbose_name = 'Категория'
        verbose_name_plural = 'Категории'

    def __str__(self):
        return '{}'.format(self.name)

class Product(models.Model):
    article = models.CharField(verbose_name='Код', max_length=15)
    name = models.CharField(verbose_name='Наименование', max_length=200)
    price = models.DecimalField(verbose_name='Цена', max_digits=100, decimal_places=2)
    special_price = models.DecimalField(verbose_name='Цена специального предложения', max_digits=100, decimal_places=2)
    count = models.IntegerField(verbose_name='Количество')
    measurement = models.CharField(verbose_name='Ед. измерения', max_length=5)
    united_purchases = models.BooleanField(verbose_name='Совместные покупки', default=False)
    image = models.ImageField(verbose_name='Изображение', upload_to='images', null=True, blank=True)
    show_main = models.BooleanField(verbose_name='Отображать на главной', default=False)
    description = models.TextField(verbose_name='Описание')
    category = models.ForeignKey(
        Category, verbose_name='Категория',
        related_name='products', on_delete=models.CASCADE
    )

    class Meta:
        verbose_name = 'Продукт'
        verbose_name_plural = 'Продукты'

    def __str__(self):
        return '{}'.format(self.name)

class Properties(models.Model):
    key = models.CharField(verbose_name='Название свойства', max_length=100)
    value = models.CharField(verbose_name='Значение свойства', max_length=100)
    product = models.ForeignKey(
        Product, verbose_name='Продукт', related_name='options',
        on_delete=models.CASCADE
    )

    class Meta:
        verbose_name = 'Опиция'
        verbose_name_plural = 'Опции'

    def __str__(self):
        return '{} - {}:{}'.format(self.product, self.key, self.value)


import torch
import opt_einsum as oe

device = "cuda"
big_core = torch.randn(
    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, dtype=torch.float64, device=device
)
batches_of_small_cores = [
    torch.randn(512, 25, 25, 2, dtype=torch.float64, device=device) for _ in range(16)
]

equation = "αβγi,αβγj,αβγk,αβγl,αβγm,αβγn,αβγo,αβγp,αβγq,αβγr,αβγs,αβγt,αβγu,αβγv,αβγw,αβγx,ijklmnopqrstuvwxω->αβγω"
print(oe.contract_path(equation, *batches_of_small_cores, big_core, optimize="auto"))
result = oe.contract(equation, *batches_of_small_cores, big_core, optimize="auto")


import httplib
import re
import vkontakte
import sys
sys.path.append('../../uglyweb/')
from uglyweb import settings

vk = vkontakte.API(settings.VK_ID, settings.VK_SECRET)

class ProfileNotFound(Exception):
    def __init__(self, name):
        self.name = name
        super(ProfileNotFound, self).__init__()

    def __repr__(self):
        return 'Profile with data: %s not found!' % self.name

def get_profile(name):
    """Get profile from VK api"""
    try:
        return vk.getProfiles(uids=name,
            fields='uid, first_name, last_name, photo, screen_name'
        )[0]
    except IndexError:
        raise ProfileNotFound(name)

def check_url(addr, name):
    """Check url exist"""
    conn = httplib.HTTPConnection(addr)
    conn.request('GET', name)
    response = conn.getresponse()
    return response.status == 200

def get_name_from_url(url):
    """Get fb or vk name from url"""
    vk = re.search(r'([^v]*)(vk\.com|vkontakte\.ru)\/([^\/]*)(.*)', url)
    if vk:
        vk_name = vk.group(3)
        if vk_name and check_url('vk.com', vk_name):
            return vk_name
    raise ProfileNotFound(url)


from prettytable import PrettyTable
import numpy as np
def init():
    Q = np.array([[5, 2, 8, 5], [4, 5, 9, 4], [4, 1, 7, 7], [4, 6, 9, 7], [1, 3, 4, 3], [0, 2, 3, 2], [1, 2, 4, 2],
                  [4, 3, 1, 0], [0, 0, 0, 0]])
    table_q = PrettyTable()
    table_q.field_names = ["x[i]", "z1", "z2", "z3", "z4"]
    for i in range(8):
        table_q.add_row([i+1,Q[i][0], Q[i][1], Q[i][2], Q[i][3]])
    beta=[]
    arr_trans=Q.transpose()
    beta = []
    for i in range(4):
        beta.append(arr_trans[i].max())
    table_q.add_row(["b",beta[0], beta[1], beta[2], beta[3]])
    print("Q:")
    print(table_q)
    print("R:")
    Q[8][0] = beta[0]
    Q[8][1] = beta[1]
    Q[8][2] = beta[2]
    Q[8][3] = beta[3]
    return Q

def R_matrix(Q):
    R=np.empty((8, 4))
    table_r = PrettyTable()
    table_r.field_names = ["x[i]", "z1", "z2", "z3", "z4"]
    for i in range(8):
        for j in range(4):
            R[i][j] = int(Q[8][j] - Q[i][j])
        table_r.add_row([i+1,R[i][0], R[i][1], R[i][2], R[i][3]])
    print(table_r)
    return R
def vald(Q):
    a=np.empty((8))
    table_vald = PrettyTable()
    table_vald.field_names = ["i", "a[i]"]
    for i in range(8):
        a[i]=(Q[i].min())
        table_vald.add_row([i+1, a[i]])
    print("1. Критерий Вальда: ")
    print(table_vald)
    max = a.max()
    number = []
    for i in range(8):
        if a[i]==max:
            number.append(i)
    for i in range(len(number)):
        print("Оптимальное решение по критерию Вальда: "+str(number[i]+1)+"e, значение x* = "+ str(a[number[i]]))
    return number

def maximum(Q):
    a=np.empty((8))
    table_max = PrettyTable()
    table_max.field_names = ["i", "a[i]"]
    for i in range(8):
        a[i]=(Q[i].max())
        table_max.add_row([i+1, a[i]])
    print("2. Критерий максимума: ")
    print(table_max)
    max = a.max()
    number = []
    for i in range(8):
        if a[i] == max:
            number.append(i)
    for i in range(len(number)):
        print("Оптимальное решение по критерию максимума: "+str(number[i]+1)+"e, значение x* = "+ str(a[number[i]]))
    return number

def savige(R):
    a=np.empty((8))
    table_sav = PrettyTable()
    table_sav.field_names = ["i", "a[i]"]
    for i in range(8):
        a[i]=(R[i].max())
        table_sav.add_row([i+1, a[i]])
    print("3. Критерий Сэвиджа: ")
    print(table_sav)
    sav = a.min()
    number = []
    for i in range(8):
        if a[i] == sav:
            number.append(i)
    for i in range(len(number)):
        print("Оптимальное решение по критерию Сэвиджа: "+str(number[i]+1)+"e, значение x* = "+ str(a[number[i]]))
    return number

def hurvic(Q, alpha):
    a=np.empty((8))
    table_hurv = PrettyTable()
    table_hurv.field_names = ["i", "a[i]"]
    for i in range(8):
        a[i]=round(alpha*(Q[i].min())+(1-alpha)*(Q[i].max())*10)/10
        table_hurv.add_row([i+1, a[i]])
    print("4. Критерий Гурвица: ")
    print(table_hurv)
    sav = a.max()
    number = []
    for i in range(8):
        if a[i] == sav:
            number.append(i)
    for i in range(len(number)):
        print("Оптимальное решение по критерию Гурвица: "+str((number[i])+1)+"e, значение x* = "+ str(a[number[i]]))
    return number

def laplas(Q):
    a = np.empty((8))
    table_sav = PrettyTable()
    table_sav.field_names = ["i", "a[i]"]
    for i in range(8):
        q_sum=0
        for j in range(4):
            q_sum+=Q[i][j]
        a[i] = 1/8 * q_sum
        table_sav.add_row([i + 1, a[i]])
    print("5. Критерий Лапласа: ")
    print(table_sav)
    sav = a.max()
    number = []
    for i in range(8):
        if a[i] == sav:
            number.append(i)
    for i in range(len(number)):
        print("Оптимальное решение по критерию Лапласа  : " + str(number[i] + 1) + "e, значение x* = " + str(a[number[i]]))
    return number

def bayes(Q, p):
    a = np.empty((8))
    table_sav = PrettyTable()
    table_sav.field_names = ["i", "a[i]"]
    for i in range(8):
        q_sum = -1
        for j in range(4):
            q_sum += p[j]*Q[i][j]
        a[i] = round(q_sum*10)/10
        table_sav.add_row([i + 1, a[i]])
    print("6. Критерий Байеса: ")
    print(table_sav)
    sav = a.max()
    number = []
    for i in range(8):
        if a[i] == sav:
            number.append(i)
    for i in range(len(number)):
        print("Оптимальное решение по критерию Байеса  : " + str(number[i] + 1) + "e, значение x* = " + str(a[number[i]]))
    return number

def matrix(vald_list,max_list, sav_list, hurv_list, laplas_list, bayes_list):
    all_lists =np.zeros((8, 6))
    for i in range(len(vald_list)):
        all_lists[vald_list[i]][0] += 1
    for i in range(len(max_list)):
        all_lists[max_list[i]][1] += 1
    for i in range(len(sav_list)):
        all_lists[sav_list[i]][2] += 1
    for i in range(len(hurv_list)):
        all_lists[hurv_list[i]][3] += 1
    for i in range(len(laplas_list)):
        all_lists[laplas_list[i]][4] += 1
    for i in range(len(bayes_list)):
        all_lists[bayes_list[i]][5] += 1
    print("Матрица выбора:")
    matrix_table = PrettyTable()
    matrix_table.field_names = ["проект", "Вальда", "Максимума", "Сэвиджа", "Гурвица", "Лапласа", "Байеса", "Сумма"]
    sum = []
    for i in range(8):
        sum.append(0)
        for j in range(6):
            sum[i]+=all_lists[i][j]
        matrix_table.add_row(["x"+str(i+1), all_lists[i][0], all_lists[i][1], all_lists[i][2], all_lists[i][3], all_lists[i][4],
                              all_lists[i][5], sum[i]])
    print(matrix_table)

def main():
    Q = init()
    R = R_matrix(Q)
    alpha = 0.6
    p = [0.1, 0.4, 0.4, 0.1]
    vald_list = []
    max_list = []
    sav_list = []
    hurv_list = []
    laplas_list = []
    bayes_list = []
    vald_list = vald(Q)
    max_list = maximum(Q)
    sav_list = savige(R)
    hurv_list = hurvic(Q, alpha)
    laplas_list = laplas(Q)
    bayes_list = bayes(Q, p)
    matrix(vald_list, max_list, sav_list, hurv_list, laplas_list, bayes_list)
if __name__=="__main__":
    main()


import sys

from mq_http_sdk.mq_exception import MQExceptionBase
from mq_http_sdk.mq_producer import *
from mq_http_sdk.mq_client import *
import time

# 初始化client。
mq_client = MQClient(
    # 设置HTTP协议客户端接入点，进入消息队列RocketMQ版控制台实例详情页面的接入点区域查看。
     "${HTTP_ENDPOINT}",
    # AccessKey ID阿里云身份验证，在阿里云RAM控制台创建。
     "${ACCESS_KEY}",
    # AccessKey Secret阿里云身份验证，在阿里云RAM控制台创建。
    "${SECRET_KEY}"
    )
# 消息所属的Topic，在消息队列RocketMQ版控制台创建。
topic_name = "${TOPIC}"
# Topic所属的实例ID，在消息队列RocketMQ版控制台创建。
# 若实例有命名空间，则实例ID必须传入；若实例无命名空间，则实例ID传入空字符串。实例的命名空间可以在消息队列RocketMQ版控制台的实例详情页面查看。
instance_id = "${INSTANCE_ID}"

producer = mq_client.get_producer(instance_id, topic_name)

# 循环发送2条消息。
msg_count = 2
print("%sPublish Message To %s\nTopicName:%s\nMessageCount:%s\n" % (10 * "=", 10 * "=", topic_name, msg_count))

try:
    for i in range(msg_count):
            msg = TopicMessage(
                    # 消息内容。
                    "I am test message %s.hello" % i,
                    # 消息标签。
                    "tag %s" % i
                        )
            # 设置消息的自定义属性。
            msg.put_property("a", i)
            # 设置消息的Key。
            msg.set_message_key("MessageKey")
            re_msg = producer.publish_message(msg)
            print("Publish Message Succeed. MessageID:%s, BodyMD5:%s" % (re_msg.message_id, re_msg.message_body_md5))

except MQExceptionBase as e:
    if e.type == "TopicNotExist":
         print("Topic not exist, please create it.")
         sys.exit(1)
    print("Publish Message Fail. Exception:%s" % e)


# -*- coding: utf-8 -*-

from matplotlib import pyplot as plt
import numpy as np
import cv2
from skimage.morphology import medial_axis
from skimage.morphology import skeletonize
from scipy import ndimage
import pymorph as m
from shapely.geometry import Point
from util import disegna as dsg
from scipy import ndimage

def flip_vertici(vertici_porte, altezza):
	for v in vertici_porte:
		v[1] = altezza - v[1]
	return vertici_porte

def isolato(immagine,riga,col):
	for r in xrange(max(0,riga-1),min(immagine.shape[0],riga+2)):
		for c in xrange(max(0,col-1),min(immagine.shape[1],col+2)):
			if (immagine[r][c]==True) and not (r==riga and c==col):
				return False
	return True

def get_distance_transforme(im_bw):
	'''
	restituisce la distance transforme
	'''
	distanceMap = ndimage.distance_transform_edt(im_bw)
	return distanceMap

def medial_points(thresh):
	#leggo l'immagine e la trasformo in 0 o 255

	(th, im_bw) = cv2.threshold(thresh, 240, 255, cv2.THRESH_BINARY_INV)
	#plt.imshow(cv2.cvtColor(im_bw,cv2.COLOR_GRAY2RGB))
	#plt.show()

	#points = punti dei muri
	points = []
	for y in xrange(0,im_bw.shape[1]):
		for x in xrange(im_bw.shape[0]):
			if im_bw[x, y] == (0):
				points.append([y,x])

	points = np.asarray(points)

	#distance transform
# 	distanceMap = ndimage.distance_transform_edt(im_bw)
# 	plt.title('8.distance map')
# 	plt.imshow(distanceMap)
# 	plt.show()
	#disegno distance transforme

	#distanceMap = dsg.disegna_distance_transform(im_bw, filepath = filepath)#plot
	distanceMap = get_distance_transforme(im_bw)

	#medial axis sulla distance transform
	skel, distance = medial_axis(distanceMap, return_distance=True)

	#elimino parte delle imprecisioni della skeletonization
	#b3 = m.thin(skel, m.endpoints('homotopic'), 15)
	b3 = m.thin(skel, m.endpoints('homotopic'), 500)

	'''
	#------
	import copy
	b4 = copy.copy(b3)
	#ottengo solo critical points
	#critical_points = []
	for riga in xrange(b4.shape[0]):
		for col in xrange(b4.shape[1]):
			if b4[riga][col]==True:
				#controllo se in un intorno di quel punto c'è una frontiera
				#creo intorno del pixel
				start_x = riga-17 #15
				end_x = riga+17
				start_y = col-17
				end_y = col+17
				count = 0
				for x in xrange(start_x,end_x):
					for y in xrange(start_y, end_y):
						if im_bw[x, y] <= (200): #se almeno una componente rgb di un pixel nell'introno del pixel cha fa parte del medial axis e' un pixel della frontiera
							count+=1
				if count <= 5:
					b4[riga][col]=False
				# else:
# 					#è un critical_points
# 					critical_points.append([riga, col])

	critical_points = []
	#salvo in una lista i critical points che fanno parte del medial axis
	for riga in xrange(b4.shape[0]):
		for col in xrange(b4.shape[1]):
			if (b4[riga][col]==True):
				critical_points.append((col,riga))
	critical_points = np.asarray(critical_points)
	flip_vertici(critical_points,b4.shape[0]-1)
	#------
	'''

	#elimino i punti isolati del medial axis, sono rumore
	for riga in xrange(b3.shape[0]):
		for col in xrange(b3.shape[1]):
			if b3[riga][col]==True and isolato(b3,riga,col):
				b3[riga][col]=False

	#disegno medial axis
# 	plt.title('9.medial axis')
# 	plt.plot(points[:,0],points[:,1],'.')
# 	plt.imshow(b3,cmap='Greys')
# 	plt.show()
	#disegno medial axis
	#dsg.disegna_medial_axis(points, b3, filepath = filepath)#quiiiiiiiii

	punti_medial = []
	#salvo in una lista i punti che fanno parte del medial axis
	for riga in xrange(b3.shape[0]):
		for col in xrange(b3.shape[1]):
			if (b3[riga][col]==True):
				punti_medial.append((col,riga))
	punti_medial = np.asarray(punti_medial)
	flip_vertici(punti_medial,b3.shape[0]-1)

	return punti_medial, distanceMap, points, b3, distance

def trovaCollegamenti(stanze,stanze_collegate,punti_medial):
	'''
Date le stanze e i punti del medial axis, trova i punti che confinano con più di una stanza, e restituisce la lista di coppie di stanze collegate e la lista di coordinate dei punti/porte.
	'''
	porte_colleganti = []
	for p in punti_medial:
		a = 0
		temp = []
		for index,s in enumerate(stanze):
			if (s.boundary).distance(Point(p[0],p[1])) < 1: #se il bordo della stanza dista meno di 1 dal punto.
				a+=1
				if not (s in temp): #se la stanza non è già stata aggiunta la aggiungo.
					temp.append(index)
		temp.sort()
		#se la coppia non è già in stanze_porte, la aggiungo.
		if not (temp in stanze_collegate) and (len(temp)==2):
			stanze_collegate.append(temp)
			porte_colleganti.append(Point(p[0],p[1]))
		if (len(temp)>2):
			for index,a in enumerate(temp):
				for b in temp[index+1:]:
					tmp = []
					tmp.extend((a,b))
					if not (tmp in stanze_collegate):
						stanze_collegate.append(tmp)
						porte_colleganti.append(Point(p[0],p[1]))

	#print(stanze_collegate)
	return stanze_collegate, porte_colleganti

def critical_points(metricMap):
	'''
	ottengo i critical points dal medial axis
	'''

	if cv2.__version__[0] == '3' :
		flag = cv2.IMREAD_GRAYSCALE
	elif cv2.__version__[0] == '2' :
		flag = cv2.CV_LOAD_IMAGE_GRAYSCALE
	else :
		raise EnvironmentError('Opencv Version Error. You should have OpenCv 2.* or 3.*')

	im_gray = cv2.imread(metricMap, flag)
	#leggo l'immagine e la trasformo in 0 o 255
	(thresh, im_bw) = cv2.threshold(im_gray, 240, 255, cv2.THRESH_BINARY_INV)

	(th, im_bw) = cv2.threshold(im_bw, 240, 255, cv2.THRESH_BINARY_INV)

	#points = punti dei muri
	points = []
	for y in xrange(0,im_bw.shape[1]):
		for x in xrange(im_bw.shape[0]):
			if im_bw[x, y] == (0):
				points.append([y,x])

	points = np.asarray(points)

	#distance map
	distanceMap = get_distance_transforme(im_bw)

	#medial axis sulla distance transform
	skel, distance = medial_axis(distanceMap, return_distance=True)

	#elimino parte delle imprecisioni della skeletonization
	#b3 = m.thin(skel, m.endpoints('homotopic'), 15)
	b3 = m.thin(skel, m.endpoints('homotopic'), 500)

	#---
	import copy
	b4 = copy.copy(b3)
	#ottengo solo critical points
	#critical_points = []
	for riga in xrange(b4.shape[0]):
		for col in xrange(b4.shape[1]):
			if b4[riga][col]==True:
				#controllo se in un intorno di quel punto c'è una frontiera
				#creo intorno del pixel
				start_x = riga-17 #15
				end_x = riga+17
				start_y = col-17
				end_y = col+17
				count = 0
				for x in xrange(start_x,end_x):
					for y in xrange(start_y, end_y):
						if im_bw[x, y] <= (200): #se almeno una componente rgb di un pixel nell'introno del pixel cha fa parte del medial axis e' un pixel della frontiera
							count+=1
				if count <= 5:
					b4[riga][col]=False
				# else:
# 					#è un critical_points
# 					critical_points.append([riga, col])

	critical_points = []
	#salvo in una lista i critical points che fanno parte del medial axis
	for riga in xrange(b4.shape[0]):
		for col in xrange(b4.shape[1]):
			if (b4[riga][col]==True):
				critical_points.append((col,riga))
	critical_points = np.asarray(critical_points)
	flip_vertici(critical_points,b4.shape[0]-1)
	#------

	return distanceMap,points, b3, b4, critical_points


import logging

from bs4 import BeautifulSoup
from rdflib import Graph, OWL, SKOS
import requests as rq

from consts import AGROVOC_PREFIX, BASE_AGROVOC_URL, LOCAL_GRAPH_PREFIX as PREFIX
from utils import normalize_resource

def extend_with_agrovoc(g, concept):
    """
    Recives a concept/word and if found returns agrovoc's broader and narrower concepts
    """

    normalized_concept = normalize_resource(concept)

    # Prevent further requests if it's already processed
    if (PREFIX[normalized_concept], OWL.sameAs, None) in g:
        return

    concept_URI = search_agrovoc_concept(concept)
    if not concept_URI:
        return

    logging.info(f'Concept "{concept}" found in agrovoc, extracting data...')
    graph = create_remote_graph(concept_URI)

    # Remove the URL prefix from the ID, wich has a length of 47 chars
    # https://agrovoc.fao.org/browse/agrovoc/en/page/<ID>
    concept_ID = concept_URI[47:]
    g.add(
        (
            PREFIX[normalized_concept],
            OWL.sameAs,
            AGROVOC_PREFIX[concept_ID],
        )
    )

    agrovoc_data = get_related_triplets(graph, concept_ID)
    for triplet in agrovoc_data:
        g.add(triplet)

def search_agrovoc_concept(concept):
    response = rq.get(f"{BASE_AGROVOC_URL}agrovoc/en/search?clang=en&q={concept}")
    response.raise_for_status()

    soup = BeautifulSoup(response.text, "html.parser")
    anchors = soup.findAll("a", class_="conceptlabel")
    if anchors:
        URI = BASE_AGROVOC_URL + anchors[0].attrs["href"]
        return URI
    else:
        return None

def create_remote_graph(URI):
    response = rq.get(URI)
    response.raise_for_status()

    soup = BeautifulSoup(response.text, "html.parser")
    turtle_anchors = soup.select("span.versal.concept-download-links > a:nth-child(1)")
    link = turtle_anchors[0].attrs["href"]
    g = Graph()
    g.parse(BASE_AGROVOC_URL + link)

    return g

def get_related_triplets(g, subject_id):
    broader_concepts = g.triples((AGROVOC_PREFIX[subject_id], SKOS.narrower, None))
    narrower_concepts = g.triples((AGROVOC_PREFIX[subject_id], SKOS.broader, None))
    return list(broader_concepts) + list(narrower_concepts)


import unittest
import sys
from math import floor
from datetime import datetime
#Used to import StockAnalyzer from parent folder
sys.path.append("..")
from Stock_Analyzer import StockAnalyzer

class TestStockAnalyzer(unittest.TestCase):
    def setUp(self):
        self.sa = StockAnalyzer()

    def test_get_date_to_string(self):
        r=self.sa.get_date_to_string(datetime(1994, 5, 20))
        e="May 20 1994"
        self.assertEqual(r,e)

    def test_get_string_to_date_object(self):
        #class datetime.datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None)¶
        r=self.sa.get_string_to_date_object("May 20 1994")
        e=datetime(1994, 5, 20)
        self.assertEqual(r,e)

    def test_get_sorted_dates_array(self):
        reverse=False
        e=[]
        e.append(self.sa.get_string_to_date_object('Dec  11 2014'))
        e.append(self.sa.get_string_to_date_object('May 20 2014'))
        e.append(self.sa.get_string_to_date_object('May 21 2013'))
        e.append(self.sa.get_string_to_date_object('Apr 22 2013'))
        r = self.sa.get_sorted_dates_array(['May 21 2013', 'Dec 11 2014', 'Apr 22 2013', 'May 20 2014'], reverse)

        #Testing that they are in sorted order
        for i in range(1,len(r)):
            cur_date=r[i]
            prev_date=r[i-1]
            if reverse==True:
                self.assertTrue( cur_date<=prev_date )
            else:
                self.assertTrue( cur_date>=prev_date )

        #Testing that they consist from input dates
        for date in e:
            r.remove(date)
        self.assertTrue(len(r)==0)

    def test_calculate_cashflow_growth(self):
        #CF_growth= (CF_current - CF_previous)/CF_current

        ci={'May 20 2014': 10, 'Dec 11 2014': 50, 'May 11 2015' :60}
        r=self.sa._calculate_cashflow_growth(ci)
        print(r)
        self.assertEqual(r['May 11 2015'], .1667)
        self.assertEqual(r['Dec 11 2014'], .8)

        #Check that nothing else is calculated as we only have 2 periods
        #to report
        self.assertTrue(len(r)==2)

    def tearDown(self):
        pass


from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert

import exceptions
import models
from database.models import AccountCookies
from repositories.base import Repository

__all__ = ('AccountCookiesRepository',)

class AccountCookiesRepository(Repository):

    def get_by_account_name(self, account_name: str) -> models.AuthCookies:
        statement = select(AccountCookies).where(AccountCookies.account_name == account_name)
        with self._session_factory() as session:
            account_cookies = session.scalar(statement)
        if account_cookies is None:
            raise exceptions.DoesNotExistInDatabase('Cookies not found')
        return models.AuthCookies(account_name=account_name, cookies=account_cookies.cookies)

    def update(self, *, account_name: str, cookies: dict[str, str]):
        statement = (
            insert(AccountCookies)
            .values(account_name=account_name, cookies=cookies)
            .on_conflict_do_update(
                index_elements=('account_name',),
                set_={'cookies': cookies},
                where=(AccountCookies.account_name == account_name),
            )
        )
        with self._session_factory() as session:
            with session.begin():
                session.execute(statement)


import requests
import bs4
from datetime import datetime
import re

def get_wiki(language):
    date = datetime.now().strftime("%Y%m%d000000")

    link = f'https://{language}.wikipedia.org/wiki/Special:FeedItem/featured/{date}/{language}'
    res = requests.get(link)

    wiki = bs4.BeautifulSoup(res.text, "lxml")

    if language == "de":
        elems = wiki.select('.hauptseite-box-content')
        text = elems[0].getText()
    elif language == "fr":
        header = wiki.find_all('h2')[1]
        text = ""
        for tag in wiki.find("h1").next_siblings:
            if tag.name == "ul":
                break
            elif tag.name == "p":
                text += tag.getText()
        elems = wiki.select('p')
        text = elems[4]
    else:
        elems = wiki.select('p')
        text = elems[0]

    imgs = wiki.select('img')
    img = imgs[0]
    img['src'] = 'https:'+img['src']

    for a in text.findAll('a'):
        a['href'] = 'https://en.wikipedia.org'+a['href']

    return str(img), str(text)

def get_wiki_for_twitter():
    date = datetime.now().strftime("%Y%m%d000000")

    link = f'https://en.wikipedia.org/wiki/Special:FeedItem/featured/{date}/en'
    res = requests.get(link)

    wiki = bs4.BeautifulSoup(res.text, "lxml")

    elems = wiki.select('p')

    link = 'https://en.wikipedia.org' + \
        elems[0].select('a', href=True)[0]['href']

    imgs = wiki.select('img')
    img_url = 'https:'+imgs[0]['src']
    img_url = re.sub('.jpg/.*?px', '.jpg/1200px', img_url, flags=re.DOTALL)

    text = elems[0].getText()

    return str(text), link, img_url


import random
import numpy as np

def train_valid_split(record_list1, record_list2, record_list3, split_percent=0.2):
    # Check paired data
    assert len(record_list1) == len(record_list2)
    assert len(record_list2) == len(record_list3)
    assert len(record_list1) == len(record_list3)

    #
    data_len = len(record_list1)
    valid_num = int(data_len * split_percent)

    # Index setting
    valid_index = np.random.choice(data_len, valid_num, replace=False)
    train_index = list(set(range(data_len)) - set(valid_index))

    # Split
    train_record_list1 = [record_list1[i] for i in train_index]
    train_record_list2 = [record_list2[i] for i in train_index]
    train_record_list3 = [record_list3[i] for i in train_index]
    valid_record_list1 = [record_list1[i] for i in valid_index]
    valid_record_list2 = [record_list2[i] for i in valid_index]
    valid_record_list3 = [record_list3[i] for i in valid_index]

    # Dictionary setting
    split_record1 = {'train': train_record_list1,
                     'valid': valid_record_list1}

    split_record2 = {'train': train_record_list2,
                     'valid': valid_record_list2}

    split_record3 = {'train': train_record_list3,
                     'valid': valid_record_list3}

    return split_record1, split_record2, split_record3


from __future__ import print_function, absolute_import
import os
import glob
import re
import sys
import urllib
import tarfile
import zipfile
import os.path as osp
from scipy.io import loadmat
import numpy as np
import h5py
from scipy.misc import imsave
import scipy.io as sio

from manage_data.get_density_map import create_density_map
from manage_data.utils import mkdir_if_missing, copy_to_directory
from manage_data.data_augmentation import augment

"""class that has a train and test unit"""
class train_test_unit(object):
    train_dir_img = ""
    train_dir_den = ""
    test_dir_img = ""
    test_dir_den = ""
    metadata = dict()
    def __init__(self, _train_dir_img, _train_dir_den, _test_dir_img, _test_dir_den, kwargs):
        self.train_dir_img = _train_dir_img
        self.train_dir_den = _train_dir_den
        self.test_dir_img = _test_dir_img
        self.test_dir_den = _test_dir_den
        self._check_before_run()
        self.metadata = kwargs

    def _check_before_run(self):
        """Check if all files are available before going deeper"""
        if not osp.exists(self.train_dir_img):
            raise RuntimeError("'{}' is not available".format(self.train_dir_img))
        if not osp.exists(self.train_dir_den):
            raise RuntimeError("'{}' is not available".format(self.train_dir_den))
        if not osp.exists(self.test_dir_img):
            raise RuntimeError("'{}' is not available".format(self.test_dir_img))
        if not osp.exists(self.test_dir_den):
            raise RuntimeError("'{}' is not available".format(self.test_dir_den))

    def to_string(self):
        return "_".join([ str(key) + "_" + str(value) for key, value in sorted(self.metadata.items()) if key != 'name'])

"""Dataset classes"""

"""Crowd counting dataset"""

class UCF_CC_50(object):
    root = 'data/ucf_cc_50/'
    ori_dir = osp.join(root, 'UCF_CC_50')
    ori_dir_lab = osp.join(ori_dir, 'labels')
    ori_dir_img = osp.join(ori_dir, 'images')

    ori_dir_den = osp.join(ori_dir, 'density_maps') #.npy files of density maps matrices
    augmented_dir = ""
    train_test_set = []
    signature_args = ['people_thr', 'gt_mode']
    metadata = dict()
    train_test_size = 5

    def __init__(self, force_create_den_maps = False, force_augmentation = False, **kwargs):
        self._check_before_run()
        self.metadata = kwargs
        self._create_original_density_maps(force_create_den_maps)
        self._create_train_test(force_augmentation, kwargs)

    def _create_original_density_maps(self, force_create_den_maps):
        if not osp.exists(self.ori_dir_den):
            os.makedirs(self.ori_dir_den)
        elif not force_create_den_maps:
            return
        create_density_map(self.ori_dir_img, self.ori_dir_lab, self.ori_dir_den, mode = self.metadata['gt_mode'])

    def _check_before_run(self):
        """Check if all files are available before going deeper"""
        if not osp.exists(self.root):
            raise RuntimeError("'{}' is not available".format(self.root))
        if not osp.exists(self.ori_dir):
            raise RuntimeError("'{}' is not available".format(self.ori_dir))
        if not osp.exists(self.ori_dir_img):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_img))
        if not osp.exists(self.ori_dir_lab):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_lab))

    def signature(self):
        return "_".join(["{}_{}".format(sign_elem, self.metadata[sign_elem]) for sign_elem in self.signature_args])

    def _create_train_test(self, force_augmentation, kwargs):
        slide_window_params = {'displace' : kwargs['displace'], 'size_x' : kwargs['size_x'], 'size_y' : kwargs['size_y'], 'people_thr' : kwargs['people_thr']}
        noise_params = {'augment_noise' : kwargs['augment_noise']}
        light_params = {'augment_light' : kwargs['augment_light'], 'bright' : kwargs['bright'], 'contrast' : kwargs['contrast']}

        file_names = os.listdir(self.ori_dir_img)
        file_names.sort()
        img_names = []
        img_ids = []
        for file_name in file_names:
            file_extention = file_name.split('.')[-1]
            file_id = file_name[:len(file_name) - len(file_extention)]
            if file_extention != 'png' and file_extention != 'jpg':
                continue
            img_names.append(file_name)
            img_ids.append(file_id)
        if len(img_names) != 50:
            raise RuntimeError("UCF_CC_50 dataset expects 50 images, {} found".format(len(img_names)))

        self.augmented_dir = osp.join(self.root, self.signature())
        augment_data = False
        if osp.exists(self.augmented_dir):
            print("'{}' already exists".format(self.augmented_dir))
            if force_augmentation:
                augment_data = True
                print("augmenting data anyway")
            else:
                augment_data = False
                print("will not augmenting data")
        else:
            augment_data = True
            os.makedirs(self.augmented_dir)

        #using 5 fold cross validation protocol
        for fold in range(5):
            fold_dir = osp.join(self.augmented_dir, 'fold{}'.format(fold + 1))
            aug_train_dir_img = osp.join(fold_dir, 'train_img')
            aug_train_dir_den = osp.join(fold_dir, 'train_den')
            aug_train_dir_lab = osp.join(fold_dir, 'train_lab')
            fold_test_dir_img = osp.join(fold_dir, 'test_img')
            fold_test_dir_den = osp.join(fold_dir, 'test_den')
            fold_test_dir_lab = osp.join(fold_dir, 'test_lab')

            mkdir_if_missing(aug_train_dir_img)
            mkdir_if_missing(aug_train_dir_den)
            mkdir_if_missing(aug_train_dir_lab)
            mkdir_if_missing(fold_test_dir_img)
            mkdir_if_missing(fold_test_dir_den)
            mkdir_if_missing(fold_test_dir_lab)

            kwargs['name'] = 'ucf-fold{}'.format(fold + 1)
            train_test = train_test_unit(aug_train_dir_img, aug_train_dir_den, fold_test_dir_img, fold_test_dir_den, kwargs.copy())
            self.train_test_set.append(train_test)

            if augment_data:
                test_img = img_names[fold * 10: (fold + 1) * 10]
                test_ids = img_ids[fold * 10: (fold + 1) * 10]
                test_den_paths = [osp.join(self.ori_dir_den, img_id + 'npy') for img_id in test_ids]
                test_lab_paths = [osp.join(self.ori_dir_lab, img_id + 'json') for img_id in test_ids]
                test_img_paths = [osp.join(self.ori_dir_img, img) for img in test_img]

                train_img = sorted(list(set(img_names) - set(test_img)))
                train_ids = sorted(list(set(img_ids) - set(test_ids)))
                train_den_paths = [osp.join(self.ori_dir_den, img_id + 'npy') for img_id in train_ids]
                train_lab_paths = [osp.join(self.ori_dir_lab, img_id + 'json') for img_id in train_ids]
                train_img_paths = [osp.join(self.ori_dir_img, img) for img in train_img]

                #augment train data
                print("Augmenting {}".format(kwargs['name']))
                augment(train_img_paths, train_lab_paths, train_den_paths, aug_train_dir_img, aug_train_dir_lab, aug_train_dir_den, slide_window_params, noise_params, light_params)
                copy_to_directory(test_den_paths, fold_test_dir_den)
                copy_to_directory(test_lab_paths, fold_test_dir_lab)
                copy_to_directory(test_img_paths, fold_test_dir_img)

class ShanghaiTech(object):
    root = 'data/ShanghaiTech/'
    ori_dir_partA = osp.join(root, 'part_A')
    ori_dir_partA_train = osp.join(ori_dir_partA, 'train_data')
    ori_dir_partA_train_mat = osp.join(ori_dir_partA_train, 'ground-truth')
    ori_dir_partA_train_img = osp.join(ori_dir_partA_train, 'images')
    ori_dir_partA_test = osp.join(ori_dir_partA, 'test_data')
    ori_dir_partA_test_mat = osp.join(ori_dir_partA_test, 'ground-truth')
    ori_dir_partA_test_img = osp.join(ori_dir_partA_test, 'images')

    ori_dir_partB = osp.join(root, 'part_B')
    ori_dir_partB_train = osp.join(ori_dir_partB, 'train_data')
    ori_dir_partB_train_mat = osp.join(ori_dir_partB_train, 'ground-truth')
    ori_dir_partB_train_img = osp.join(ori_dir_partB_train, 'images')
    ori_dir_partB_test = osp.join(ori_dir_partB, 'test_data')
    ori_dir_partB_test_mat = osp.join(ori_dir_partB_test, 'ground-truth')
    ori_dir_partB_test_img = osp.join(ori_dir_partB_test, 'images')

    #to be computed
    ori_dir_partA_train_lab = osp.join(ori_dir_partA_train, 'labels')
    ori_dir_partA_train_den = osp.join(ori_dir_partA_train, 'density_maps')
    ori_dir_partA_test_lab = osp.join(ori_dir_partA_test, 'labels')
    ori_dir_partA_test_den = osp.join(ori_dir_partA_test, 'density_maps')

    ori_dir_partB_train_lab = osp.join(ori_dir_partB_train, 'labels')
    ori_dir_partB_train_den = osp.join(ori_dir_partB_train, 'density_maps')
    ori_dir_partB_test_lab = osp.join(ori_dir_partB_test, 'labels')
    ori_dir_partB_test_den = osp.join(ori_dir_partB_test, 'density_maps')

    augmented_dir_partA = ""
    augmented_dir_partB = ""
    train_test_set = []
    signature_args = ['people_thr', 'gt_mode']
    metadata = dict()
    train_test_size = 2

    def __init__(self, force_create_den_maps = False, force_augmentation = False, **kwargs):
        self._check_before_run()
        self.metadata = kwargs
        self._create_labels()
        self._create_original_density_maps(force_create_den_maps)
        self._create_train_test(force_augmentation, kwargs)

    def _create_original_density_maps(self, force_create_den_maps):
        all_density_dirs_exist = osp.exists(self.ori_dir_partA_train_den)
        all_density_dirs_exist = all_density_dirs_exist and osp.exists(self.ori_dir_partA_test_den)
        all_density_dirs_exist = all_density_dirs_exist and osp.exists(self.ori_dir_partB_train_den)
        all_density_dirs_exist = all_density_dirs_exist and osp.exists(self.ori_dir_partB_test_den)
        if not all_density_dirs_exist:
            mkdir_if_missing(self.ori_dir_partA_train_den)
            mkdir_if_missing(self.ori_dir_partA_test_den)
            mkdir_if_missing(self.ori_dir_partB_train_den)
            mkdir_if_missing(self.ori_dir_partB_test_den)
        elif not force_create_den_maps:
            return
        create_density_map(self.ori_dir_partA_train_img, self.ori_dir_partA_train_lab, self.ori_dir_partA_train_den, mode = self.metadata['gt_mode'])
        create_density_map(self.ori_dir_partA_test_img, self.ori_dir_partA_test_lab, self.ori_dir_partA_test_den, mode = self.metadata['gt_mode'])
        create_density_map(self.ori_dir_partB_train_img, self.ori_dir_partB_train_lab, self.ori_dir_partB_train_den, mode = self.metadata['gt_mode'])
        create_density_map(self.ori_dir_partB_test_img, self.ori_dir_partB_test_lab, self.ori_dir_partB_test_den, mode = self.metadata['gt_mode'])

    def _check_before_run(self):
        """Check if all files are available before going deeper"""
        if not osp.exists(self.root):
            raise RuntimeError("'{}' is not available".format(self.root))
        if not osp.exists(self.ori_dir_partA):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partA))
        if not osp.exists(self.ori_dir_partB):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partB))
        if not osp.exists(self.ori_dir_partA_train):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partA_train))
        if not osp.exists(self.ori_dir_partB_train):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partB_train))
        if not osp.exists(self.ori_dir_partA_test):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partA_test))
        if not osp.exists(self.ori_dir_partB_test):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partB_test))
        if not osp.exists(self.ori_dir_partA_train_img):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partA_train_img))
        if not osp.exists(self.ori_dir_partA_train_mat):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partA_train_mat))
        if not osp.exists(self.ori_dir_partB_train_img):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partB_train_img))
        if not osp.exists(self.ori_dir_partB_train_mat):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partB_train_mat))
        if not osp.exists(self.ori_dir_partA_test_img):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partA_test_img))
        if not osp.exists(self.ori_dir_partA_test_mat):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partA_test_mat))
        if not osp.exists(self.ori_dir_partB_test_img):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partB_test_img))
        if not osp.exists(self.ori_dir_partB_test_mat):
            raise RuntimeError("'{}' is not available".format(self.ori_dir_partB_test_mat))

    def _json_to_string(self, array):
        """
        converts json to string specifically for shanghai tech dataset
        """
        if len(array)==0:
            return '[]'
        line = '['
        for i in range(len(array)):
            line += '{\"x\":'+str(array[i][0])+',\"y\":'+str(array[i][1])+'},'
        return line[0:len(line)-1]+']'

    def _convert_mat_to_json(self, in_dir, out_dir):
        """
        converts every .mat file in in_dir to a .json equivalent in out_dir
        """
        print("converting mat to json from {}".format(in_dir))
        file_names = os.listdir(in_dir)
        for mat_file in file_names:
            mat_file_path = osp.join(in_dir, mat_file)
            file_extention = mat_file.split('.')[-1]
            file_id = mat_file[3:len(mat_file) - len(file_extention)]
            json_file_path = osp.join(out_dir, file_id + 'json')
            labels = sio.loadmat(mat_file_path)
            labels = labels['image_info'][0][0][0][0][0]
            labels = str(self._json_to_string(labels))
            with open(json_file_path, 'w') as outfile:
                outfile.write(labels)

    def _create_labels(self):
        mkdir_if_missing(self.ori_dir_partA_train_lab)
        mkdir_if_missing(self.ori_dir_partA_test_lab)
        mkdir_if_missing(self.ori_dir_partB_train_lab)
        mkdir_if_missing(self.ori_dir_partB_test_lab)
        #check if number os files is equal
        if len(os.listdir(self.ori_dir_partA_train_mat)) != len(os.listdir(self.ori_dir_partA_train_lab)):
            self._convert_mat_to_json(self.ori_dir_partA_train_mat, self.ori_dir_partA_train_lab)
        if len(os.listdir(self.ori_dir_partA_test_mat)) != len(os.listdir(self.ori_dir_partA_test_lab)):
            self._convert_mat_to_json(self.ori_dir_partA_test_mat, self.ori_dir_partA_test_lab)
        if len(os.listdir(self.ori_dir_partB_train_mat)) != len(os.listdir(self.ori_dir_partB_train_lab)):
            self._convert_mat_to_json(self.ori_dir_partB_train_mat, self.ori_dir_partB_train_lab)
        if len(os.listdir(self.ori_dir_partB_test_mat)) != len(os.listdir(self.ori_dir_partB_test_lab)):
           self._convert_mat_to_json(self.ori_dir_partB_test_mat, self.ori_dir_partB_test_lab)

    def signature(self):
        return "_".join(["{}_{}".format(sign_elem, self.metadata[sign_elem]) for sign_elem in self.signature_args])

    def _create_train_test(self, force_augmentation, kwargs):
        slide_window_params = {'displace' : kwargs['displace'], 'size_x' : kwargs['size_x'], 'size_y' : kwargs['size_y'], 'people_thr' : kwargs['people_thr']}
        noise_params = {'augment_noise' : kwargs['augment_noise']}
        light_params = {'augment_light' : kwargs['augment_light'], 'bright' : kwargs['bright'], 'contrast' : kwargs['contrast']}

        #shanghaiTech part A
        self.augmented_dir_partA = osp.join(self.ori_dir_partA, self.signature())
        augment_data_A = False
        if osp.exists(self.augmented_dir_partA):
            print("'{}' already exists".format(self.augmented_dir_partA))
            if force_augmentation:
                augment_data_A = True
                print("augmenting data anyway")
            else:
                augment_data_A = False
                print("will not augmenting data")
        else:
            augment_data_A = True
            os.makedirs(self.augmented_dir_partA)

        aug_dir_partA_img = osp.join(self.augmented_dir_partA, "train_img")
        aug_dir_partA_den = osp.join(self.augmented_dir_partA, "train_den")
        aug_dir_partA_lab = osp.join(self.augmented_dir_partA, "train_lab")
        mkdir_if_missing(aug_dir_partA_img)
        mkdir_if_missing(aug_dir_partA_den)
        mkdir_if_missing(aug_dir_partA_lab)

        kwargs['name'] = 'shanghai-partA'
        part_A_train_test = train_test_unit(aug_dir_partA_img, aug_dir_partA_den, self.ori_dir_partA_test_img, self.ori_dir_partA_test_den, kwargs.copy())
        self.train_test_set.append(part_A_train_test)

        if augment_data_A:
            ori_img_paths = [osp.join(self.ori_dir_partA_train_img, file_name) for file_name in sorted(os.listdir(self.ori_dir_partA_train_img))]
            ori_lab_paths = [osp.join(self.ori_dir_partA_train_lab, file_name) for file_name in sorted(os.listdir(self.ori_dir_partA_train_lab))]
            ori_den_paths = [osp.join(self.ori_dir_partA_train_den, file_name) for file_name in sorted(os.listdir(self.ori_dir_partA_train_den))]
            augment(ori_img_paths, ori_lab_paths, ori_den_paths, aug_dir_partA_img, aug_dir_partA_lab, aug_dir_partA_den, slide_window_params, noise_params, light_params)

        #shanghaiTech part B
        self.augmented_dir_partB = osp.join(self.ori_dir_partB, self.signature())
        augment_data_B = False
        if osp.exists(self.augmented_dir_partB):
            print("'{}' already exists".format(self.augmented_dir_partB))
            if force_augmentation:
                augment_data_B = True
                print("augmenting data anyway")
            else:
                augment_data_B = False
                print("will not augmenting data")
        else:
            augment_data_B = True
            os.makedirs(self.augmented_dir_partB)

        aug_dir_partB_img = osp.join(self.augmented_dir_partB, "train_img")
        aug_dir_partB_den = osp.join(self.augmented_dir_partB, "train_den")
        aug_dir_partB_lab = osp.join(self.augmented_dir_partB, "train_lab")
        mkdir_if_missing(aug_dir_partB_img)
        mkdir_if_missing(aug_dir_partB_den)
        mkdir_if_missing(aug_dir_partB_lab)

        kwargs['name'] = 'shanghai-partB'
        part_B_train_test = train_test_unit(aug_dir_partB_img, aug_dir_partB_den, self.ori_dir_partB_test_img, self.ori_dir_partB_test_den, kwargs.copy())
        self.train_test_set.append(part_B_train_test)

        if augment_data_B:
            ori_img_paths = [osp.join(self.ori_dir_partB_train_img, file_name) for file_name in sorted(os.listdir(self.ori_dir_partB_train_img))]
            ori_lab_paths = [osp.join(self.ori_dir_partB_train_lab, file_name) for file_name in sorted(os.listdir(self.ori_dir_partB_train_lab))]
            ori_den_paths = [osp.join(self.ori_dir_partB_train_den, file_name) for file_name in sorted(os.listdir(self.ori_dir_partB_train_den))]
            augment(ori_img_paths, ori_lab_paths, ori_den_paths, aug_dir_partB_img, aug_dir_partB_lab, aug_dir_partB_den, slide_window_params, noise_params, light_params)

"""Create dataset"""

__factory = {
    'ucf-cc-50': UCF_CC_50,
    'shanghai-tech': ShanghaiTech
}

def get_names():
    return __factory.keys()

def init_dataset(name, force_create_den_maps = False, force_augmentation = False, **kwargs):
    if name not in __factory.keys():
        raise KeyError("Unknown dataset: {}".format(name))
    return __factory[name](force_create_den_maps, force_augmentation, **kwargs)

if __name__ == '__main__':
    pass


from tabulate import tabulate
import os
import numpy as np
import matplotlib.pyplot as plt
from statistics import stdev

class StatisticsCalculator:

	def __init__(self,ToutLearner,ToutProposer,drty):
		self.TimeOutLearner = ToutLearner
		self.TimeOutProposer = ToutProposer
		self.Table = []
		self.fileDirectory = drty
		self.avgCpuTimes = []
		self.avgElapsedTimes = []
		self.stdCpuTimes = []
		self.stdElapsedTimes = []
		self.varyingParameterValues = []
		self.varyingParameter = None

	def createDirectory(self):
		if not os.path.exists(self.fileDirectory):
			os.makedirs(self.fileDirectory)

	def addTableHeader(self,varyingParameterName , param2Name , param3Name):
		tableHeader = ["Timeout","Preemtion","TimeOutLearner","TimeOutProposer",varyingParameterName,param2Name,param3Name,"Safety","Liveness","AvgElpTime","SDElpTime","ElpTimeRange","AvgCpuTime","SDCpuTime","CpuTimeRange"]
		self.Table.append(tableHeader)
		self.varyingParameter = varyingParameterName

	def plot(self,title,parameter1Name,parameter1Value,parameter2Name,parameter2Value):
		plt.figure()
		plt.title(title, size = 14)
		plt.xlabel(parameter1Name, size = 14)
		plt.ylabel(parameter2Name, size = 14)
		plt.plot(parameter1Value,parameter2Value,'-r',label='Label')
		plt.legend(loc='upper left')
		self.createDirectory()
		filename = './' + self.fileDirectory +'/' + title +'.png'
		plt.savefig(filename)
		plt.close()

	def addEntry(self,varyingParameter,cpuTimeList,elapsedTimeList,isTimeOut,isPreemtionEnabled,param2Value,param3Value,isSafe,isLive):
		tableRow = []
		tableRow.append(isTimeOut)
		tableRow.append(isPreemtionEnabled)
		tableRow.append(self.TimeOutLearner)
		tableRow.append(self.TimeOutProposer)
		tableRow.append(round(varyingParameter,5))
		tableRow.append(round(param2Value,5))
		tableRow.append(round(param3Value,5))
		tableRow.append(isSafe)
		tableRow.append(isLive)
		tableRow.append(round(sum(elapsedTimeList) / len(elapsedTimeList),5))
		tableRow.append(round(stdev(elapsedTimeList),5))
		tableRow.append(round(max(elapsedTimeList) - min(elapsedTimeList),5))
		tableRow.append(round(sum(cpuTimeList) / len(cpuTimeList),5))
		tableRow.append(round(stdev(cpuTimeList),5))
		tableRow.append(round(max(cpuTimeList) - min(cpuTimeList),5))
		self.Table.append(tableRow)

		self.avgElapsedTimes.append(sum(elapsedTimeList) / len(elapsedTimeList))
		self.avgCpuTimes.append(sum(cpuTimeList) / len(cpuTimeList))
		self.stdElapsedTimes.append(stdev(elapsedTimeList))
		self.stdCpuTimes.append(stdev(cpuTimeList))
		self.varyingParameterValues.append(varyingParameter)

	def report(self):
		print(tabulate(self.Table))
		self.plot("Average Cpu Time vs " + self.varyingParameter ,self.varyingParameter,self.varyingParameterValues,"Average Cpu Time" , self.avgCpuTimes  )
		self.plot("Average Elapsed Time vs " + self.varyingParameter ,self.varyingParameter,self.varyingParameterValues,"Average Elapsed Time" , self.avgElapsedTimes  )
		self.plot("Standard Deviation - Elapsed Time vs " + self.varyingParameter ,self.varyingParameter,self.varyingParameterValues,"Standard Deviation - Elapsed Time" , self.stdElapsedTimes  )
		self.plot("Standard Deviation - Cpu Time vs " + self.varyingParameter ,self.varyingParameter,self.varyingParameterValues,"Standard Deviation - CPU Time" , self.stdCpuTimes  )
		filename = './' + self.fileDirectory +'/' + self.varyingParameter +'Analysis.txt'
		f = open(filename,'w')
		f.write(tabulate(self.Table))
		f.flush()
		f.close()


"""
  Tests for Verb endpoints
"""
# disabling pylint rules that conflict with pytest fixtures
# pylint: disable=unused-argument,redefined-outer-name,unused-import
import os
import json
import pytest
from copy import deepcopy
from fastapi import FastAPI
from fastapi.testclient import TestClient
from routes.verb import router
from testing.test_config import API_URL, TESTING_FOLDER_PATH
from schemas.schema_examples import BASIC_VERB_MODEL_EXAMPLE
from common.models import Verb
from common.testing.firestore_emulator import (firestore_emulator,
                                               clean_firestore)
from common.utils.http_exceptions import add_exception_handlers

app = FastAPI()
add_exception_handlers(app)
app.include_router(router, prefix="/learning-record-service/api/v1")

client_with_emulator = TestClient(app)

os.environ["FIRESTORE_EMULATOR_HOST"] = "localhost:8080"
os.environ["GOOGLE_CLOUD_PROJECT"] = "fake-project"

def test_post_and_get_verb(clean_firestore):
  verb_dict = deepcopy(BASIC_VERB_MODEL_EXAMPLE)

  url = f"{API_URL}/verb"
  resp = client_with_emulator.post(url, json=verb_dict)
  json_response = resp.json()
  verb_uuid = json_response["data"]["uuid"]

  assert resp.status_code == 200, "Status is not 200"
  assert json_response.get("success") is True, "Success not true"
  assert json_response.get(
      "message"
  ) == "Successfully created the verb", "Expected response not same"
  assert json_response.get("data").get("name") == verb_dict.get(
      "name"), "Expected response not same"

  url = f"{API_URL}/verb/{verb_uuid}"
  resp = client_with_emulator.get(url)
  json_response_get_req = resp.json()

  assert resp.status_code == 200, "Status code not 200"
  assert json_response_get_req.get("data") == json_response.get(
      "data"), "Expected response not same"

def test_get_verb_negative(clean_firestore):
  invalid_verb_uuid = "random_id"
  url = f"{API_URL}/verb/{invalid_verb_uuid}"
  response = {
      "success": False,
      "message": "Verb with uuid random_id not found",
      "data": None
  }
  resp = client_with_emulator.get(url)
  json_response = resp.json()

  assert resp.status_code == 404, "Status 404"
  assert json_response == response, "Expected response not same"

def test_delete_verb_positive(clean_firestore):
  verb_dict = deepcopy(BASIC_VERB_MODEL_EXAMPLE)

  url = f"{API_URL}/verb"
  resp = client_with_emulator.post(url, json=verb_dict)
  json_response = resp.json()
  verb_uuid = json_response["data"]["uuid"]

  url = f"{API_URL}/verb/{verb_uuid}"
  resp = client_with_emulator.delete(url)
  json_response_delete_req = resp.json()

  expected_data = {"success": True, "message": "Successfully deleted the verb"}
  assert resp.status_code == 200, "Status code not 200"
  assert json_response_delete_req == expected_data, "Expected response not same"

def test_delete_verb_negative(clean_firestore):
  invalid_verb_uuid = "random_id"
  url = f"{API_URL}/verb/{invalid_verb_uuid}"
  response = {
      "success": False,
      "message": "Verb with uuid random_id not found",
      "data": None
  }
  resp = client_with_emulator.delete(url)
  json_response = resp.json()

  assert resp.status_code == 404, "Status 404"
  assert json_response == response, "Expected response not same"

def test_update_verb_positive(clean_firestore):
  verb_dict = deepcopy(BASIC_VERB_MODEL_EXAMPLE)
  url = f"{API_URL}/verb"

  resp = client_with_emulator.post(url, json=verb_dict)
  json_response = resp.json()

  updated_data = json_response["data"]
  updated_data["name"] = "some random name"

  uuid = updated_data.get("uuid")
  url = f"{API_URL}/verb/{uuid}"
  resp = client_with_emulator.put(url, json=updated_data)
  json_response_update_req = resp.json()

  assert json_response_update_req.get("success") is True, "Success not true"
  assert json_response_update_req.get(
      "message"
  ) == "Successfully updated the verb", "Expected response not same"
  assert json_response_update_req.get("data").get(
      "name") == "some random name", "Expected response not same"

def test_update_verb_negative(clean_firestore):
  req_json = deepcopy(BASIC_VERB_MODEL_EXAMPLE)
  uuid = req_json["uuid"] = "random_id"
  response = {
      "success": False,
      "message": "Verb with uuid random_id not found",
      "data": None
  }

  url = f"{API_URL}/verb/{uuid}"
  resp = client_with_emulator.put(url, json=req_json)
  json_response = resp.json()

  assert resp.status_code == 404, "Status 404"
  assert json_response == response, "Expected response not same"

def test_import_verbs(clean_firestore):
  url = f"{API_URL}/verb/import/json"
  file_path = os.path.join(TESTING_FOLDER_PATH, "verbs.json")
  with open(file_path, "rb") as file:
    resp = client_with_emulator.post(url, files={"json_file": file})

  json_response = resp.json()

  assert resp.status_code == 200, "Success not 200"
  assert isinstance(json_response.get("data"), list), "Response is not a list"
  assert len(json_response.get("data")) > 0, "Empty list returned"

def test_get_verbs(clean_firestore):
  verb_dict = deepcopy(BASIC_VERB_MODEL_EXAMPLE)
  verb_dict["name"] = "Communicate"
  verb = Verb.from_dict(verb_dict)
  verb.uuid = ""
  verb.save()
  verb.uuid = verb.id
  verb.update()
  params = {"skip": 0, "limit": "30"}

  url = f"{API_URL}/verbs"
  resp = client_with_emulator.get(url, params=params)
  json_response = resp.json()
  assert resp.status_code == 200, "Status 200"
  saved_names = [i.get("name") for i in json_response.get(
    "data")["records"]]
  assert verb_dict["name"] in saved_names, "all data not retrieved"

def test_get_verbs_negative(clean_firestore):
  verb_dict = deepcopy(BASIC_VERB_MODEL_EXAMPLE)
  verb = Verb.from_dict(verb_dict)
  verb.uuid = ""
  verb.save()
  verb.uuid = verb.id
  verb.update()
  params = {"skip": "-1", "limit": "30"}

  url = f"{API_URL}/verbs"
  resp = client_with_emulator.get(url, params=params)
  json_response = resp.json()
  assert resp.status_code == 422, "Status not 422"
  assert json_response.get(
    "message"
  ) == "Validation Failed", \
    "unknown response received"


from src.utils.dictionary import get_values, get_nested_value
from src.parsers.answers_details_parser import AnswerDetailsParser

class DetailsParser:
    STATIC_VARIABLES = [
        "recipient_id",
        "collector_id",
        "date_created",
        "date_modified",
        "ip_address",
        "email_address",
        "first_name",
        "last_name",
    ]

    @classmethod
    def parse_question(cls, question: dict) -> dict:
        """Get question details"""
        question_id, family, subtype, text = get_values(question, "id", "family", "subtype", "headings/0/heading")

        return {
            "question_id": question_id,
            "family": family,
            "subtype": subtype,
            "text": text,
            "answers": AnswerDetailsParser.parse(question, family, subtype),
        }

    @classmethod
    def parse_questions(cls, questions: list) -> list:
        """Get questions details"""
        return [cls.parse_question(question) for question in questions]

    @classmethod
    def parse_survey(cls, survey: dict) -> dict:
        """Get survey details"""
        static_details = {}

        for variable in cls.STATIC_VARIABLES:
            static_details[variable] = survey.get(variable, None)

        survey_id, pages, title = get_values(survey, "id", "pages", "title")

        questions = []

        for page in pages:
            questions.extend(page["questions"])

        return {
            "survey_id": survey_id,
            "title": title,
            **static_details,
            "questions": cls.parse_questions(questions),
        }