from datetime import datetime
import heapq
import itertools
import math
import operator
import string
import sys
import typing as tp
from abc import abstractmethod, ABC
from math import radians
from math import log, sin, cos, radians, acos

from dateutil import parser as dateutil_parser

TRow = tp.Dict[str, tp.Any]
TRowsIterable = tp.Iterable[TRow]
TRowsGenerator = tp.Generator[TRow, None, None]

class Operation(ABC):
    @abstractmethod
    def __call__(self, rows: TRowsIterable, *args: tp.Any, **kwargs: tp.Any) -> TRowsGenerator:
        pass

# Operations

class Mapper(ABC):
    """Base class for mappers"""

    @abstractmethod
    def __call__(self, row: TRow) -> TRowsGenerator:
        """
        :param row: one table row
        """
        pass

class Map(Operation):
    """Map operation"""

    def __init__(self, mapper: Mapper) -> None:
        self.mapper = mapper

    def __call__(self, rows: TRowsIterable, *args: tp.Any, **kwargs: tp.Any) -> TRowsGenerator:
        """
        :param rows: iterator over TRow
        :return: Generator of mapped TRow
        """
        for row in rows:
            yield from self.mapper(row)

class Reducer(ABC):
    """Base class for reducers"""

    @abstractmethod
    def __call__(self, group_key: tp.Tuple[str, ...], rows: TRowsIterable) -> TRowsGenerator:
        """
        :param rows: table rows
        """
        pass

class Reduce(Operation):
    """Reduce operation"""

    def __init__(self, reducer: Reducer, keys: tp.Tuple[str, ...]) -> None:
        self.reducer = reducer
        self.keys = keys

    def __call__(self, rows: TRowsIterable, *args: tp.Any, **kwargs: tp.Any) -> TRowsGenerator:
        """
        :param rows: iterator over TRow, sorted by self.keys
        :return: reduced groups of TRow
        """

        def get_value(row: TRow) -> tp.Optional[tp.Iterable[str]]:
            """get values in TRow pointed by key"""
            return [row[key] for key in self.keys]

        for _, g in itertools.groupby(rows, get_value):
            yield from self.reducer(self.keys, g)

class Joiner(ABC):
    """Base class for joiners"""

    def __init__(self, suffix_a: str = "_1", suffix_b: str = "_2") -> None:
        self._a_suffix = suffix_a
        self._b_suffix = suffix_b

    @abstractmethod
    def __call__(
            self, keys: tp.Sequence[str], rows_a: TRowsIterable, rows_b: TRowsIterable
    ) -> TRowsGenerator:
        """
        :param keys: join keys
        :param rows_a: left table rows
        :param rows_b: right table rows
        """
        pass

    def _inner_join(
            self, keys: tp.Sequence[str], rows_a: TRowsIterable, rows_b: TRowsIterable
    ) -> TRowsIterable:
        rows_b_cp = list(rows_b)
        suffix_a, suffix_b = None, None

        for row_a in rows_a:
            for row_b in rows_b_cp:

                if suffix_a is None:
                    suffix_a = {k for k in row_a.keys() if k in row_b and k not in keys}

                if suffix_b is None:
                    suffix_b = {k for k in row_b.keys() if k in row_a and k not in keys}

                res = {
                          k + self._a_suffix if k in suffix_a else k: row_a[k]
                          for k in row_a.keys()
                      } | {
                          k + self._b_suffix if k in suffix_b else k: row_b[k]
                          for k in row_b.keys()
                      }

                yield res

class Join(Operation):
    def __init__(self, joiner: Joiner, keys: tp.Sequence[str]):
        self.keys = keys
        self.joiner = joiner

    def __call__(
            self, rows: TRowsIterable, *args: tp.Any, **kwargs: tp.Any
    ) -> TRowsGenerator:
        def grouper(row):  # type: ignore
            return tuple(row[key] for key in self.keys)

        def take_next_from_iterable(iterable_obj: tp.Iterator[tp.Any]) -> tp.Any:
            try:
                return next(iterable_obj)
            except StopIteration:
                return None, None

        rows_a, rows_b = rows, args[0]

        groups_a = itertools.groupby(rows_a, key=grouper)
        groups_b = itertools.groupby(rows_b, key=grouper)

        need_take_a = True
        need_take_b = True

        key_a, group_a = None, None
        key_b, group_b = None, None

        while True:
            if need_take_a:
                key_a, group_a = take_next_from_iterable(groups_a)
            if need_take_b:
                key_b, group_b = take_next_from_iterable(groups_b)

            if group_b is None and group_a is None:
                break

            if key_a == key_b:
                need_take_a = True
                need_take_b = True

                yield from self.joiner(self.keys, group_a or [], group_b or [])
            # elif key_a is None or (key_b is not None and key_b < key_a):
            elif key_a is not None and (key_b is None or key_b >= key_a):
                need_take_a = True
                need_take_b = False
                yield from self.joiner(self.keys, group_a or [], [])
            else:
                need_take_a = False
                need_take_b = True
                yield from self.joiner(self.keys, [], group_b or [])

# Dummy operators

class DummyMapper(Mapper):
    """Yield exactly the row passed"""

    def __call__(self, row: TRow) -> TRowsGenerator:
        yield row

class FirstReducer(Reducer):
    """Yield only first row from passed ones"""

    def __call__(self, group_key: tp.Tuple[str, ...], rows: TRowsIterable) -> TRowsGenerator:
        first_row: tp.Union[TRow, None] = None
        for row in rows:
            if first_row is None:
                first_row = row
        assert first_row is not None
        yield first_row

# Mappers

class FilterPunctuation(Mapper):
    """Left only non-punctuation symbols"""

    def __init__(self, column: str):
        """
        :param column: name of column to process
        """
        self.column = column

    def __call__(self, row: TRow) -> TRowsGenerator:
        row[self.column] = row[self.column].translate(str.maketrans('', '', string.punctuation))
        yield row

class LowerCase(Mapper):
    """Replace column value with value in lower case"""

    def __init__(self, column: str):
        """
        :param column: name of column to process
        """
        self.column = column

    def __call__(self, row: TRow) -> TRowsGenerator:
        row[self.column] = row[self.column].lower()
        yield row

class Split(Mapper):
    """Split row on multiple rows by separator"""

    def __init__(self, column: str, separator: tp.Optional[str] = None) -> None:
        """
        :param column: name of column to split
        :param separator: string to separate by
        """
        self.column = column
        self.separator = separator

    def __call__(self, row: TRow) -> TRowsGenerator:
        for part in row[self.column].split(self.separator):
            initial_row = row.copy()
            initial_row[self.column] = part
            yield initial_row

class Product(Mapper):
    """Calculates product of multiple columns"""

    def __init__(self, columns: tp.Sequence[str], result_column: str = 'product') -> None:
        """
        :param columns: column names to product
        :param result_column: column name to save product in
        """
        self.columns = columns
        self.result_column = result_column

    def __call__(self, row: TRow) -> TRowsGenerator:
        res = 1
        for col in self.columns:
            res *= row[col]
        row[self.result_column] = res
        yield row

class Filter(Mapper):
    """Remove records that don't satisfy some condition"""

    def __init__(self, condition: tp.Callable[[TRow], bool]) -> None:
        """
        :param condition: if condition is not true - remove record
        """
        self.condition = condition

    def __call__(self, row: TRow) -> TRowsGenerator:
        if self.condition(row):
            yield row

class Project(Mapper):
    """Leave only mentioned columns"""

    def __init__(self, columns: tp.Sequence[str]) -> None:
        """
        :param columns: names of columns
        """
        self.columns = columns

    def __call__(self, row: TRow) -> TRowsGenerator:
        yield {key: row[key] for key in self.columns}

class Devide_mapper(Mapper):
    """Divide value from one column to number"""

    def __init__(self, numerator: str, denominator: str, result_column: str) -> None:
        self.numerator = numerator
        self.denominator = denominator
        self.result_column = result_column

    def __call__(self, row: TRow) -> TRowsGenerator:
        new_row = row.copy()
        new_row[self.result_column] = row[self.numerator] / row[self.denominator]
        yield new_row

class Idf(Mapper):
    """Count idf metrics based on number of docs and \
            number of docs containing the word"""

    def __init__(self, doc_count: str, num_word_entries: str, text_column: str, result_column: str) -> None:
        """
        :param doc_count: name of doc_count column
        :param num_word_entries: name of word entries number column
        :param text_column: name of column with word
        :param result_colum: name of column for idf
        """
        self.doc_count = doc_count
        self.num_word_entries = num_word_entries
        self.text_column = text_column
        self.result_column = result_column

    def __call__(self, row: TRow) -> TRowsGenerator:
        total_doc = row[self.doc_count]
        entries_count = row[self.num_word_entries]
        word = row[self.text_column]

        result = dict()
        result[self.text_column] = word
        result[self.result_column] = math.log(total_doc / entries_count)
        yield result

class Pmi(Mapper):
    """Count pmi metrics based on frequency of word \
            in docs and total"""

    def __init__(self, doc_freq: str, total_freq: str, result_column: str) -> None:
        """
        :param doc_freq: name of doc frequency column
        :param total_freq: name of total frequency column
        :param result_colum: name of column for pmi
        """
        self.doc_freq = doc_freq
        self.total_freq = total_freq
        self.result_column = result_column

    def __call__(self, row: TRow) -> TRowsGenerator:
        doc_freq = row[self.doc_freq]
        total_freq = row[self.total_freq]
        row[self.result_column] = math.log(doc_freq / total_freq)
        yield row

class Len_mapper(Mapper):
    """Calc len among two points"""

    def __init__(self, start: str, stop: str, len: str) -> None:
        self.start = start
        self.stop = stop
        self.len = len

    def __call__(self, row: TRow) -> TRowsGenerator:
        row_new = row.copy()
        start_ = row[self.start]
        stop_ = row[self.stop]
        d_ = 6371. * acos(
            sin(radians(start_[1])) * sin(radians(stop_[1])) + cos(radians(start_[1])) * cos(radians(stop_[1]))
            * (cos(radians(stop_[0]) - radians(start_[0]))))
        row_new[self.len] = d_
        yield row_new

class Week_Day_Hour(Mapper):
    """Day of the week and hour from Date"""

    def __init__(self, date_col: str, day_of_week: str, hour_col: str):
        self.date_col = date_col
        self.day_of_week = day_of_week
        self.hour_col = hour_col

    def __call__(self, row: TRow) -> TRowsGenerator:
        row_new = row.copy()
        if "." in row[self.date_col]:
            row_new[self.day_of_week] = datetime.strptime(row[self.date_col], "%Y%m%dT%H%M%S.%f").strftime("%A")[:3]
            row_new[self.hour_col] = datetime.strptime(row[self.date_col], "%Y%m%dT%H%M%S.%f").hour
        else:
            row_new[self.day_of_week] = datetime.strptime(row[self.date_col], "%Y%m%dT%H%M%S").strftime("%A")[:3]
            row_new[self.hour_col] = datetime.strptime(row[self.date_col], "%Y%m%dT%H%M%S").hour

        yield row_new

class Time_diff(Mapper):
    """Calc time-delta between two dates"""

    def __init__(self, start_date: str, end_date: str, time_delta: str):
        self.start_date = start_date
        self.end_date = end_date
        self.time_delta = time_delta

    def __call__(self, row: TRow) -> TRowsGenerator:
        if "." in row[self.start_date]:
            time1 = datetime.strptime(row[self.start_date], "%Y%m%dT%H%M%S.%f")
        else:
            time1 = datetime.strptime(row[self.start_date], "%Y%m%dT%H%M%S")
        if "." in row[self.end_date]:
            time2 = datetime.strptime(row[self.end_date], "%Y%m%dT%H%M%S.%f")
        else:
            time2 = datetime.strptime(row[self.end_date], "%Y%m%dT%H%M%S")

        row_new = row.copy()
        d_ = (time2 - time1).total_seconds() / 3600
        row_new[self.time_delta] = d_
        yield row_new

class ProcessLength(Mapper):
    """Get edge length"""

    def __init__(self, start_coord_column: str, end_coord_column: str, length_column: str) -> None:
        """
        :param start_coord_column: name of start column
        :param end_coord_column: name of end column
        :param length_column: name of column for length
        """
        self.start = start_coord_column
        self.end = end_coord_column
        self.length = length_column

    def __call__(self, row: TRow) -> TRowsGenerator:
        l1, f1 = row[self.start]
        l2, f2 = row[self.end]
        l1 = radians(l1)
        l2 = radians(l2)
        f1 = radians(f1)
        f2 = radians(f2)
        row[self.length] = 6371 * 2 * math.asin(math.sqrt(math.sin(f2 / 2 - f1 / 2) * math.sin(f2 / 2 - f1 / 2) +
                                                          math.cos(f1) * math.cos(f2) * math.sin(l2 / 2 - l1 / 2) *
                                                          math.sin(l2 / 2 - l1 / 2)))
        yield row

class ProcessTime(Mapper):
    """Get edge length"""

    def __init__(self, enter_time_column: str, leave_time_column: str, time_column: str, weekday_column: str,
                 hour_column: str) -> None:
        """
        :param enter_time_column: name of enter time column
        :param leave_time_column: name of leave time column
        :param time_column: name of column for time
        :param weekday_column: name of column for week day
        :param hour_column: name of column for hour
        """
        self.enter = enter_time_column
        self.leave = leave_time_column
        self.time = time_column
        self.day = weekday_column
        self.hour = hour_column

    def __call__(self, row: TRow) -> TRowsGenerator:
        date1 = dateutil_parser.parse(row[self.enter])
        date2 = dateutil_parser.parse(row[self.leave])

        row[self.day] = date1.strftime('%a')
        row[self.hour] = date1.hour
        row[self.time] = (date2 - date1).total_seconds()
        yield row

class ProcessSpeed(Mapper):
    """Get speed based on length and time"""

    def __init__(self, length_column: str, time_column: str, speed_column: str) -> None:
        """
        :param length_column: column for total length
        :param time_column: column for total time
        :param speed_column: column for redult speed
        """
        self.length = length_column
        self.time = time_column
        self.speed = speed_column

    def __call__(self, row: TRow) -> TRowsGenerator:
        row[self.speed] = row[self.length] / row[self.time] * 3600
        yield row

# Reducers

class TopN(Reducer):
    """Calculate top N by value"""

    def __init__(self, column: str, n: int) -> None:
        """
        :param column: column name to get top by
        :param n: number of top values to extract
        """
        self.column_max = column
        self.n = n

    def __call__(self, group_key: tp.Tuple[str, ...], rows: TRowsIterable) -> TRowsGenerator:
        for row in heapq.nlargest(self.n, rows, key=operator.itemgetter(self.column_max)):
            yield row

class TermFrequency(Reducer):
    """Calculate frequency of values in column"""

    def __init__(self, words_column: str, result_column: str = 'tf') -> None:
        """
        :param words_column: name for column with words
        :param result_column: name for result column
        """
        self.words_column = words_column
        self.result_column = result_column

    def __call__(self, group_key: tp.Tuple[str, ...], rows: TRowsIterable) -> TRowsGenerator:
        counts: tp.Dict[tp.Any, int] = dict()
        length = 0
        row_sample = None
        for row in rows:
            if row_sample is None:
                row_sample = row
            word = row[self.words_column]
            if word not in counts:
                counts[word] = 1
            else:
                counts[word] += 1
            length += 1

        assert row_sample is not None
        for word, count in counts.items():
            result: TRow = dict()
            for key in group_key:
                result[key] = row_sample[key]
            result[self.words_column] = word
            result[self.result_column] = count / length
            yield result

class Count(Reducer):
    """Count rows passed and yield single row as a result"""

    def __init__(self, column: str) -> None:
        """
        :param column: name of column to count
        """
        self.column = column

    def __call__(self, group_key: tp.Tuple[str, ...], rows: TRowsIterable) -> TRowsGenerator:
        result: TRow = dict()
        row_sample = None
        length = 0
        for row in rows:
            if row_sample is None:
                row_sample = row
            length += 1
        assert row_sample is not None
        for key in group_key:
            result[key] = row_sample[key]
        result[self.column] = length
        yield result

class SafeCount(Reducer):
    """Count rows passed and yield multiple row as a result"""

    def __init__(self, column: str) -> None:
        """
        :param column: name of column to count
        """
        self.column = column

    def __call__(self, group_key: tp.Tuple[str, ...], rows: TRowsIterable) -> TRowsGenerator:
        result = dict()
        row_sample = None
        length = 0
        for row in rows:
            if row_sample is None:
                row_sample = row
            length += 1

        assert row_sample is not None
        for key in group_key:
            result[key] = row_sample[key]
        result[self.column] = length
        for i in range(length):
            yield result

class Sum(Reducer):
    """
    Sum values aggregated by key
    Example for key=('a',) and column='b'
        {'a': 1, 'b': 2, 'c': 4}
        {'a': 1, 'b': 3, 'c': 5}
        =>
        {'a': 1, 'b': 5}
    """

    def __init__(self, column: str) -> None:
        """
        :param column: name for sum column
        """
        self.column = column

    def __call__(self, group_key: tp.Tuple[str, ...], rows: TRowsIterable) -> TRowsGenerator:
        sum_ = 0
        keys: TRow = {}
        print('lolo', self.column, file=sys.stderr)

        for r in rows:
            sum_ += r[self.column]
            if not keys:
                keys = {k: r[k] for k in group_key}

        yield keys | {self.column: sum_}

class MultipleSum(Reducer):
    """Sum values in columns passed and yield single row as a result"""

    def __init__(self, columns: tp.Iterable[str]) -> None:
        """
        :param column: name of columns to sum
        """
        self.columns = columns

    def __call__(self, group_key: tp.Tuple[str, ...], rows: TRowsIterable) -> TRowsGenerator:
        result = dict()
        row_sample = None
        s = dict()

        for col in self.columns:
            s[col] = 0

        for row in rows:
            if row_sample is None:
                row_sample = row
            for col in self.columns:
                s[col] += row[col]

        assert row_sample is not None
        for key in group_key:
            result[key] = row_sample[key]
        for col in self.columns:
            result[col] = s[col]
        yield result

class CalcMean(Reducer):
    """Calc mean of a group"""

    def __init__(self, column: str, mean_val: str) -> None:
        self.column = column
        self.mean_val = mean_val
        self.sum = 0

    def __call__(self, group_key: tp.Tuple[str, ...], rows: TRowsIterable) -> TRowsGenerator:
        self.sum = 0
        num_of_rows = 0
        for row in rows:
            num_of_rows += 1
            self.sum += row[self.column]

        row_new = {key: row[key] for key in row if key in group_key}
        value = self.sum / num_of_rows
        row_new[self.mean_val] = value
        yield row_new

# Joiners

class InnerJoiner(Joiner):
    """Join with inner strategy"""

    def __call__(
            self, keys: tp.Sequence[str], rows_a: TRowsIterable, rows_b: TRowsIterable
    ) -> TRowsGenerator:
        if rows_a or rows_b:
            yield from self._inner_join(keys, rows_a, rows_b)
        else:
            return

class OuterJoiner(Joiner):
    """Join with outer strategy"""

    def __call__(
            self, keys: tp.Sequence[str], rows_a: TRowsIterable, rows_b: TRowsIterable
    ) -> TRowsGenerator:
        rows_b_cp = list(rows_b)
        if not rows_a:
            yield from rows_b_cp
        elif not rows_b_cp:
            yield from rows_a
        else:
            yield from self._inner_join(keys, rows_a, rows_b_cp)

class LeftJoiner(Joiner):
    """Join with left strategy"""

    def __call__(
            self, keys: tp.Sequence[str], rows_a: TRowsIterable, rows_b: TRowsIterable
    ) -> TRowsGenerator:
        rows_b_cp = list(rows_b)
        if not rows_a:
            return
        if not rows_b_cp:
            yield from rows_a
        else:
            yield from self._inner_join(keys, rows_a, rows_b_cp)

class RightJoiner(Joiner):
    """Join with right strategy"""

    def __call__(
            self, keys: tp.Sequence[str], rows_a: TRowsIterable, rows_b: TRowsIterable
    ) -> TRowsGenerator:
        if not rows_b:
            return
        if not rows_a:
            yield from rows_b
        else:
            yield from self._inner_join(keys, rows_a, rows_b)


from django.shortcuts import render, redirect
from django.utils import translation
from ratelimit.exceptions import Ratelimited

from base import settings

def page_not_found(request, exception, **kwargs):
    response = render(request, 'status_page/page_not_found.html', {})
    response.status_code = 404
    return response

def method_not_allowed(request, **kwargs):
    response = render(request, 'status_page/method_not_allowed.html', {})
    response.status_code = 405
    return response

def access_denied(request, exception, **kwargs):
    if isinstance(exception, Ratelimited):
        response = render(request, 'status_page/rate_limited.html', {'exception': exception})
        response.status_code = 429
    else:
        response = render(request, 'status_page/access_denied.html', {'exception': exception})
        response.status_code = 403
    return response

def server_error(request, **kwargs):
    response = render(request, 'status_page/server_error.html', {})
    response.status_code = 500
    return response

def noscript(request):
    return render(request, 'status_page/noscript.html', {})

def common_context_processor(request):
    if hasattr(settings, 'ENVIRONMENT'):
        env = settings.ENVIRONMENT
    else:
        env = 'LOCAL'
    context = {
        'installed_apps': settings.INSTALLED_APPS,
        'environment': env,
    }
    return context

def edit_language(request, lang):
    translation.activate(lang)
    response = redirect(request.META['HTTP_REFERER'])
    response.set_cookie(settings.LANGUAGE_COOKIE_NAME, translation.get_language())
    return response


import warnings
warnings.filterwarnings("ignore")
from torch.utils import data
from torch import nn, optim
from vis_tools import *
from datasets import *
from models import *
import argparse, os
import itertools
import torch
import time
import pdb

# Training Configurations
# (You may put your needed configuration here. Please feel free to add more or use argparse. )
img_dir = './edges2shoes/train/'
test_img_dir = './edges2shoes/val/'
img_shape = (3, 128, 128) # Please use this image dimension faster training purpose
num_epochs = 20
batch_size = 16
lr_rate = 0.0002  	  # Adam optimizer learning rate
betas = (0.5, 0.999)  # Adam optimizer beta 1, beta 2
lambda_pixel = 15       # Loss weights for L1 image loss
lambda_latent = 50      # Loss weights for L1 latent regression
lambda_kl = 0.01          # Loss weights for kl divergence
latent_dim = 8         # latent dimension for the encoded images from domain B
gpu_id = 0

# Normalize image tensor
def norm(image):
	return (image/255.0-0.5)*2.0

# Denormalize image tensor
def denorm(tensor):
	return ((tensor+1.0)/2.0)*255.0

# Reparameterization helper function
# (You may need this helper function here or inside models.py, depending on your encoder implementation)
def reparameterization(mean, log_var):
    ################################
    # Please fill in std, eps and z:
    std = torch.exp(0.5 * log_var)
    eps = torch.normal(0,1,size=std.shape,device=std.device, requires_grad=False)
    z = mean + (std * eps)
    ################################

    return z

def init_weights(m):
  if isinstance(m, nn.Conv2d):
    nn.init.xavier_uniform(m.weight)
  elif isinstance(m, nn.BatchNorm2d):
    nn.init.normal_(m.weight.data, 0.0, 0.02)
    nn.init.constant_(m.bias.data, 0.01)

def set_requires_grad(nets, requires_grad=False):
  for net in nets:
    if net is not None:
      for param in net.parameters():
        param.requires_grad = requires_grad

# Random seeds (optional)
torch.manual_seed(1); np.random.seed(1)
# set device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# Initialize networks
generator = Generator(latent_dim, img_shape).to(device)
encoder = Encoder(latent_dim).to(device)
D_VAE = Discriminator().to(device)
D_CLR = Discriminator().to(device)

# Initialize Weights for networks
generator.apply(init_weights)
D_VAE.apply(init_weights)
D_CLR.apply(init_weights)

# Define optimizers for networks
optimizer_E = torch.optim.Adam(encoder.parameters(), lr=lr_rate/40, betas=betas)
optimizer_G = torch.optim.Adam(generator.parameters(), lr=lr_rate*4, betas=betas)
optimizer_D_VAE = torch.optim.Adam(D_VAE.parameters(), lr=lr_rate/500, betas=betas)
optimizer_D_CLR = torch.optim.Adam(D_CLR.parameters(), lr=lr_rate/500, betas=betas)

# Define DataLoader
dataset = Edge2Shoe(img_dir)
loader = data.DataLoader(dataset, batch_size=batch_size, num_workers=2)
test_img_dir = './edges2shoes/val/'
test_dataset = Edge2Shoe(test_img_dir)
test_loader = data.DataLoader(test_dataset, batch_size=1)

# Losses for networks
L1_VAE_Loss = nn.L1Loss(reduction='sum')
L1_CRL_Loss = nn.L1Loss(reduction='sum')
Dis_loss = nn.BCELoss(reduction='sum')
def KLD_VAE_Loss(mu, log_var):
  std = torch.exp(log_var)
  KLD = torch.sum(std + torch.square(mu) - log_var - 1) / 2
  return KLD

# For adversarial loss (optional to use)
valid = 1; fake = 0
# Fixed input to test the performance of GAN
fixed_noise = torch.randn(36, latent_dim, device=device)
for test_idx, test_data in enumerate(test_loader):
  test_edge_tensor, test_rgb_tensor = test_data
  test_edge_tensor = test_edge_tensor.repeat(36,1,1,1)
  test_edge_tensor = norm(test_edge_tensor).to(device)
  break

GE_losses = []
D_VAE_losses = []
D_CLR_losses = []
L1_VAE_losses = []
G_VAE_losses = []
G_CLR_losses = []
KL_VAE_losses = []
L1_CLR_losses = []
img_list = []
len_dataset = len(loader)

for e in range(num_epochs):
    for idx, datat in enumerate(loader):
      edge_tensor, rgb_tensor = datat
      len_batch = edge_tensor.shape[0]
      len_data = len_batch // 2
      # Send the data to device
      edge_tensor, rgb_tensor = norm(edge_tensor).to(device), norm(rgb_tensor).to(device)
      # Split the data for vae and clr
      if (len_data == 0):
        real_A_vae = edge_tensor
        real_A_clr = edge_tensor
        real_B_vae = rgb_tensor
        real_B_clr = rgb_tensor
        Valid_label_vae = torch.full((1,1,1,1), valid, dtype=torch.float, device=device, requires_grad=False)
        Fake_label_vae = torch.full((1,1,1,1), fake, dtype=torch.float, device=device, requires_grad=False)
        Valid_label_clr = torch.full((1,1,1,1), valid, dtype=torch.float, device=device, requires_grad=False)
        Fake_label_clr = torch.full((1,1,1,1), fake, dtype=torch.float, device=device, requires_grad=False)
      else:
        real_A_vae = edge_tensor[0:len_data,:,:,:]
        real_A_clr = edge_tensor[len_data:,:,:,:]
        real_B_vae = rgb_tensor[0:len_data,:,:,:]
        real_B_clr = rgb_tensor[len_data:,:,:,:]
        Valid_label_vae = torch.full((len_data,1,1,1), valid, dtype=torch.float, device=device, requires_grad=False)
        Fake_label_vae = torch.full((len_data,1,1,1), fake, dtype=torch.float, device=device, requires_grad=False)
        Valid_label_clr = torch.full(((len_batch - len_data),1,1,1), valid, dtype=torch.float, device=device, requires_grad=False)
        Fake_label_clr = torch.full(((len_batch - len_data),1,1,1), fake, dtype=torch.float, device=device, requires_grad=False)

      #-------------------------------
      #  Train Generator and Encoder
      #------------------------------

      # Pass B through Encoder for VAE
      encoder_output_mean, encoder_output_logvar = encoder(real_B_vae)
      z = reparameterization(encoder_output_mean, encoder_output_logvar)
      # z = z.detach()
      # Get a random normal tensor for CLR
      z_clr = torch.randn(len_batch - len_data, latent_dim, device=device, requires_grad=False)
      # Generate fake B for VAE with A and z
      fake_B_vae = generator(real_A_vae, z)
      # Generate fake B for CLR with A and z_clr
      fake_B_clr = generator(real_A_clr, z_clr)
      # Pass fake B through Encoder
      fake_mu, fake_logvar = encoder(fake_B_clr)

      # Assign all the descriminator to not calculate gradients
      set_requires_grad([D_VAE, D_CLR], False)
      # Clear Optimizers
      optimizer_E.zero_grad()
      optimizer_G.zero_grad()
      # Calculate all the losses
      loss_L1_vae = L1_VAE_Loss(fake_B_vae, real_B_vae) * lambda_pixel
      loss_G_vae = Dis_loss(D_VAE(fake_B_vae), Valid_label_vae) * 2
      loss_G_clr = Dis_loss(D_CLR(fake_B_clr), Valid_label_clr) * 2
      loss_kl_vae = KLD_VAE_Loss(encoder_output_mean, encoder_output_logvar) * lambda_kl

      # Calculate the total loss
      total_loss = loss_G_vae + loss_G_clr + loss_kl_vae + loss_L1_vae
      # Backtrack it
      total_loss.backward(retain_graph=True)

      # Assign the encoder to not calculate gradients
      set_requires_grad([encoder], False)
      # Calculate loss for latent dim in CLR
      loss_L1_clr = L1_CRL_Loss(fake_mu, z_clr) * lambda_latent
      loss_L1_clr.backward()
      # Assign the encoder to again calculate gradients
      set_requires_grad([encoder], True)

      # Take a step for Optimizers
      optimizer_E.step()
      optimizer_G.step()

      # if (e % 3 == 0):
      #----------------------------------
      #  Train Discriminator (cVAE-GAN)
      #----------------------------------

      # Assign all the descriminator to not calculate gradients
      set_requires_grad([D_VAE, D_CLR], True)
      optimizer_D_VAE.zero_grad()
      # Calculate output for generated B (with detach from generator) and Real B
      pred_fake = D_VAE(fake_B_vae.detach())
      pred_real = D_VAE(real_B_vae)
      # Calculate Loss
      loss_D_vae_fake = Dis_loss(pred_fake, Fake_label_vae + 0.2)
      loss_D_vae_real = Dis_loss(pred_real, Valid_label_vae - 0.2)
      # Sum the losses and do backward
      loss_D_vae = (loss_D_vae_fake + loss_D_vae_real) / 2
      loss_D_vae.backward()
      # Take a step for Optimizer
      optimizer_D_VAE.step()

      #---------------------------------
      #  Train Discriminator (cLR-GAN)
      #---------------------------------

      optimizer_D_CLR.zero_grad()
      # Calculate output for generated B (with detach from generator) and Real B
      pred_fake_clr = D_CLR(fake_B_clr.detach())
      pred_real_clr = D_CLR(real_B_clr)
      # Calculate Loss
      loss_D_clr_fake = Dis_loss(pred_fake_clr, Fake_label_clr + 0.2)
      loss_D_clr_real = Dis_loss(pred_real_clr, Valid_label_clr - 0.2)
      # Sum the losses and do backward
      loss_D_clr = (loss_D_clr_fake + loss_D_clr_real) / 2
      loss_D_clr.backward()
      # Take a step for Optimizer
      optimizer_D_CLR.step()
      # else:
      #   loss_D_vae = torch.tensor(0)
      #   loss_D_clr = torch.tensor(0)

      # Save information
      if idx % 150 == 0:
        print("[",e,"/",num_epochs,"][",idx,"/",len_dataset,"]\tLoss_D_clr: ",loss_D_clr.item(),"\t",loss_D_vae.item(), "\tLoss_GE: ",loss_L1_clr.item(), "\t", loss_L1_vae.item(), "\t", loss_kl_vae.item(), "\t", loss_G_vae.item(), "\t", loss_G_clr.item())
        GE_losses.append(total_loss.item())
        D_VAE_losses.append(loss_D_vae.item())
        D_CLR_losses.append(loss_D_clr.item())
        L1_VAE_losses.append(loss_L1_vae.item())
        G_VAE_losses.append(loss_G_vae.item())
        G_CLR_losses.append(loss_G_clr.item())
        KL_VAE_losses.append(loss_kl_vae.item())
        L1_CLR_losses.append(loss_L1_clr.item())

      if ((e % 1 == 0) or (e == num_epochs-1)) and (idx == 0):
        with torch.no_grad():
          gen_output = generator(test_edge_tensor, fixed_noise)
        grid_gen_output = vutils.make_grid(denorm(gen_output), nrow=6, padding=2, normalize=True)
        img_list.append(grid_gen_output)
        # plt.imshow(np.transpose(grid_gen_output.cpu().detach().numpy(), (1, 2, 0)))
        # plt.title("Gan's output at epoch: " + str(e))
        # plt.show()

checkpoint = {'netG': generator.state_dict(),
              'netE': encoder.state_dict(),
              'netD1': D_VAE.state_dict(),
              'netD2': D_CLR.state_dict()}
torch.save(checkpoint, './bicyclegan8_{}.pt'.format(num_epochs))

# Display Discriminator losses
plt.title("Discriminator_VAE and Discriminator_CLR Loss During Training")
plt.plot(D_VAE_losses,label="D_vae")
plt.plot(D_CLR_losses,label="D_clr")
plt.xlabel("iterations")
plt.ylabel("Loss")
plt.legend()
plt.savefig("gan_D_loss.png")
# Display Total Generator-Encoder Losses
plt.title("Generator-Encoder Loss During Training")
plt.plot(GE_losses,label="GE")
plt.xlabel("iterations")
plt.ylabel("Loss")
plt.legend()
plt.savefig("gan_total_loss_loss.png")
# Display L1 VAE Loss
plt.title("L1 VAE Loss During Training")
plt.plot(L1_VAE_losses,label="L1 VAE")
plt.xlabel("iterations")
plt.ylabel("Loss")
plt.legend()
plt.savefig("gan_L1_vae_loss.png")
# Display L1 CLR Loss
plt.title("L1 CLR Loss During Training")
plt.plot(L1_CLR_losses,label="L1 CLR")
plt.xlabel("iterations")
plt.ylabel("Loss")
plt.legend()
plt.savefig("gan_L1_clr_loss.png")
# Display KL VAE Loss
plt.title("KL VAE Loss During Training")
plt.plot(KL_VAE_losses,label="KL")
plt.xlabel("iterations")
plt.ylabel("Loss")
plt.legend()
plt.savefig("gan_kl_vae_loss.png")
# Display G VAE Loss
plt.title("G VAE Loss During Training")
plt.plot(G_VAE_losses,label="G VAE")
plt.xlabel("iterations")
plt.ylabel("Loss")
plt.legend()
plt.savefig("gan_g_vae_loss.png")
# Display G CLR Loss
plt.title("G CLR Loss During Training")
plt.plot(G_CLR_losses,label="G CLR")
plt.xlabel("iterations")
plt.ylabel("Loss")
plt.legend()
plt.savefig("gan_g_clr_loss.png")

for i,image_item in enumerate(img_list):
  plt.imshow(np.transpose(image_item.cpu().detach().numpy(), (1, 2, 0)))
  plt.title("Gan's output at epoch: " + str(i+1) + ".png")
  plt.savefig("gan_output_at_epoch_" + str(i+1) + ".png")


import argparse
import h5py
import numpy as np
import os

def main(args):
    # Folder containing the experiment parameters
    params_folder = args.params_folder
    # Experiment tag
    tag = args.tag
    if tag is None:
        tag = os.path.basename(params_folder).split('_')[0]
    # Name for group to store experiment results
    group_name = args.group
    # Folder containing the batch scripts
    batch_folder = args.batch_folder
    # Folder containing the outputs
    output_folder = args.output_folder
    # Path to the script
    script_path = args.script_path

    # Get all the experiments
    paths = [os.path.join(params_folder, f)
             for f in sorted(os.listdir(params_folder))]
    n_jobs = len(paths)

    # NERSC parameters
    run_nersc = not args.run_local
    n_nodes = args.n_nodes
    if run_nersc:
        n_tasks = args.n_tasks
        n_nodes_per_job = int(n_tasks / 32)
        n_batch_scripts = args.n_batch_scripts
        n_simultaneous = args.n_simultaneous
        time = args.time
        qos = args.qos
        n_jobs_per_batch = int(np.ceil(n_jobs / n_batch_scripts))

        # Header of batch script
        def batch_header(out):
            return (
                "#!/bin/bash\n"
                f"#SBATCH -N {n_nodes}\n"
                "#SBATCH -C haswell\n"
                f"#SBATCH -q {qos}\n"
                "#SBATCH -J nb\n"
                f"#SBATCH --output={output_folder}/{out}_out.o\n"
                f"#SBATCH --error={output_folder}/{out}_error.o\n"
                "#SBATCH --mail-user=pratik.sachdeva@berkeley.edu\n"
                "#SBATCH --mail-type=ALL\n"
                f"#SBATCH -t {time}\n"
                f"#SBATCH --image=docker:pssachdeva/neuro:latest\n\n"
                "export OMP_NUM_THREADS=1\n\n")
    else:
        n_simultaneous = n_jobs
        n_jobs_per_batch = n_jobs

    # Create hyperparameters for CV fitting
    n_coupling_lambdas = args.n_coupling_lambdas
    coupling_lambdas = np.logspace(args.coupling_lambda_lower,
                                   args.coupling_lambda_upper,
                                   num=n_coupling_lambdas)
    n_tuning_lambdas = args.n_tuning_lambdas
    tuning_lambdas = np.logspace(args.tuning_lambda_lower,
                                 args.tuning_lambda_upper,
                                 num=n_tuning_lambdas)
    Ks = np.arange(args.max_K) + 1

    batch_counter = 0
    for job, path in enumerate(paths):
        # Load the model configuration
        with h5py.File(path, 'a') as exp:
            group = exp.create_group(group_name)
            # Training hyperparameters
            group['Ks'] = Ks
            group['coupling_lambdas'] = coupling_lambdas
            group['tuning_lambdas'] = tuning_lambdas
            # Training settings
            group.attrs['model'] = args.model
            group.attrs['criterion'] = args.criterion
            group.attrs['cv'] = args.cv
            group.attrs['fine_sweep_frac'] = args.fine_sweep_frac
            group.attrs['n_coupling_lambdas_fine'] = args.n_coupling_lambdas_fine
            group.attrs['n_tuning_lambdas_fine'] = args.n_tuning_lambdas_fine
            group.attrs['solver'] = args.solver
            group.attrs['initialization'] = args.initialization
            group.attrs['max_iter'] = args.max_iter
            group.attrs['tol'] = args.tol
            group.attrs['Psi_transform'] = args.Psi_transform

        # If we have reset the job counter, create a new batch script
        job_idx = job % n_jobs_per_batch

        if job_idx == 0:
            if run_nersc:
                batch_script = batch_header(f"{tag}_{batch_counter}")
                batch_path = f"{batch_folder}/{tag}_{batch_counter}.sh"
                batch_counter += 1
            else:
                batch_script = ""
                batch_path = f"{batch_folder}/{tag}.sh"

        # Create run command
        command = f"python -u {script_path} --file_path={path} --group={group_name}"
        if run_nersc:
            command = (
                f"srun -N {n_nodes_per_job} -n {n_tasks} "
                "-c $OMP_NUM_THREADS shifter {command} &")

            if (job + 1) % n_simultaneous == 0:
                command += "wait"
        else:
            command = f"mpirun -n {n_nodes} " + command
        command += "\n"
        batch_script += command

        # If we're at the last counter, write file
        if job_idx + 1 == n_jobs_per_batch:
            if run_nersc and ((job + 1) % n_simultaneous != 0):
                batch_script += "wait"

            with open(batch_path, 'w') as batch:
                batch.write(batch_script)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Run CV solver on triangular model.')
    # Required arguments
    parser.add_argument('--params_folder', type=str)
    parser.add_argument('--tag', type=str)
    parser.add_argument('--group', type=str)
    parser.add_argument('--batch_folder', type=str)
    parser.add_argument('--output_folder', type=str)
    parser.add_argument('--script_path', type=str)
    # NERSC arguments
    parser.add_argument('--run_local', action='store_true')
    parser.add_argument('--n_nodes', type=int, default=64)
    parser.add_argument('--n_tasks', type=int, default=32)
    parser.add_argument('--n_batch_scripts', type=int, default=100)
    parser.add_argument('--n_simultaneous', type=int, default=50)
    parser.add_argument('--time', type=str, default='00:30:00')
    parser.add_argument('--qos', type=str, default='debug')
    # Solver hyperparameters
    parser.add_argument('--model', type=str, default='tm')
    parser.add_argument('--n_coupling_lambdas', type=int, default=25)
    parser.add_argument('--n_coupling_lambdas_fine', type=int, default=10)
    parser.add_argument('--coupling_lambda_lower', type=float, default=-7)
    parser.add_argument('--coupling_lambda_upper', type=float, default=-2)
    parser.add_argument('--n_tuning_lambdas', type=int, default=25)
    parser.add_argument('--n_tuning_lambdas_fine', type=int, default=10)
    parser.add_argument('--tuning_lambda_lower', type=float, default=-7)
    parser.add_argument('--tuning_lambda_upper', type=float, default=-2)
    parser.add_argument('--fine_sweep_frac', type=float, default=0.1)
    parser.add_argument('--max_K', type=int, default=1)
    parser.add_argument('--cv', type=int, default=3)
    parser.add_argument('--criterion', type=str, default='bic')
    # Fitter arguments
    parser.add_argument('--solver', default='ow_lbfgs')
    parser.add_argument('--initialization', default='fits')
    parser.add_argument('--max_iter', type=int, default=500)
    parser.add_argument('--tol', type=float, default=1e-8)
    parser.add_argument('--Psi_transform', type=str, default='softplus')
    args = parser.parse_args()

    main(args)


# -*- coding: utf-8 -*-
# @File    : client.py
# @Author  : Hua Guo
# @Time    : 2021/9/14 上午5:19
# @Disc    :
import requests
import json
import cv2

from minimal_api_demo.config import host, port

# basic config
base_request_url = 'http://' + host + ":" + str(port) + '/'
norm_inference_url = base_request_url+'normal_inference/inference'
request_header = {'Content-Type': 'application/json'}

# get method
result_get = requests.get(norm_inference_url)
res = result_get.json()
print(f'Result: {res}')

# post
request_data = {
          "name": "Xiaoye",
          "age": 13,
          "from": [
            "Friend1",
            "FriendB",
            "Friend C"
          ]
        }
# convert request_data to json data
json_data = json.dumps(request_data)
# send requests
res = requests.post(headers=request_header, data=json_data, url=norm_inference_url)
# convert from json to normal data
result = json.loads(res.content)
print(f"Normal inference result: {result}")

# # image post
#
# frame = cv2.imread(image_path)
# image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# img_resized = cv2.resize(frame, (IMAGE_W, IMAGE_H))
#
# # 得到request反馈
# boxes, scores, labels = get_inference(img_resized)
# # 得到结果都是相对于 img_resized
# # boxes: 框位置：[top_left_x, top_left_y, bottom_right_x, bottom_right_y]
# # score: probability
# # labels: card name
#
#
# # 画框，保存图片
# draw_save_image(image, (boxes, scores, labels), "output.png")
# _, img_encoded = cv2.imencode('.jpg', img_resized)
# content_type = 'image/jpeg'
# headers = {'content-type': content_type}
# r = requests.post(
#     'http://' + client_host + ':' + str(client_port) + '/inference',
#     data=img_encoded.tostring(), headers=headers
# )


from recombee_api_client.api_client import RecombeeClient, Region
from recombee_api_client.api_requests import AddItemProperty, SetItemValues, AddPurchase
from recombee_api_client.api_requests import RecommendItemsToItem, SearchItems, Batch, ResetDatabase, RecommendUsersToUser,\
RecommendItemsToUser
from pyspark.sql import *
from pyspark.sql.functions import *

client = RecombeeClient('vega-corp-prod', \
        'ra2rKUlqZwUq5MV8Icse0jOMhFLDTILrE2UTZ4JwP6Kiy9qgN0eJIrB1nmWGmyEC', \
         region=Region.AP_SE)

spark = SparkSession.builder.getOrCreate()

sqlUserRead = """
select user_id, collect_set(cast(content_id as string)) as content_id from waka.waka_pd_fact_reader
where  data_date_key > 20220631 and data_date_key <= 20220731
group by user_id
order by user_id
"""

scenario = "Items-to-User" #Items-to-User
logic = "recombee:default" #"recombee:default"
dfAllUserid = spark.sql(sqlUserRead).select(col("user_id")).orderBy(col("user_id"))
listAllUserId = dfAllUserid.collect()
dictUserRecommend = {} # Dict chua' userId va` list item id cua? cac' item dc recommend cho user do'
listUserIdNotExist = [] # List chua userId k ton` tai. trong Train Set
print("step 1")
for row in listAllUserId:

    numOfReturnRecom = 4
    try:
        returnQuery = client.send(
            RecommendItemsToUser(scenario=scenario, user_id=str(row["user_id"]),
                                 count=4, logic=logic)
            )
    except:
        listUserIdNotExist.append(str(row["user_id"]))
        print(f'the user id {row["user_id"]} doesnt exist')
        continue
    #{'recommId': 'c0f4f5d50ba6cda16e25a57cd3104401', 'recomms': [{'id': '21058'}, {'id': '38045'}, {'id': '37976'},
    listItemId = returnQuery["recomms"]
    #[{'id': '21058'}, {'id': '38045'}, {'id': '37976'}, {'id': '37736'}, {'id': '37535'}]
    listItem = map(lambda x: x["id"], listItemId)
    # [24671, 36986, ...]
    dictUserRecommend.update({str(row["user_id"]): list(listItem)})
    # {1: [24671, 36986,...], 2: [23245, ...], ...}
    print(row["user_id"])
print("step 2")
with open('userRecommendRecom.txt', 'w+') as f:
    for key, value in dictUserRecommend.items():
        f.write('%s:%s\n' % (key, value))

# Lay' ds cac' userid va` itemid ma` ho. read trong thang' 7
dfUserRead = spark.sql(sqlUserRead)
listUserRead = dfUserRead.collect()
# row[userid: 1, content_id: [24671, 36986,...]]
dictUserRead = {} # Dict chua' cac' userid va` list itemid ma` ho. read trong thang' 7
for row in listUserRead:
    dictUserRead.update({str(row["user_id"]): row["content_id"]})
    # {1: [24671, 36986, ...], 2: [23245, ...], ...}
print("step 3")
with open('userReadRecom.txt', 'w+') as f:
    for key, value in dictUserRead.items():
        f.write('%s:%s\n' % (key, value))

# __Recommendations__	__Precision @k's__	        __AP@3__
# [0, 0, 1]	            [0, 0, 1/3]	            (1/3)(1/3) = 0.11
# [0, 1, 1]	            [0, 1/2, 2/3]	        (1/3)[(1/2) + (2/3)] = 0.38
# [1, 1, 1]	            [1/1, 2/2, 3/3]	        (1/3)[(1) + (2/2) + (3/3)] = 1
def averagePrecisionAtK(list1, list2):
    # Average Precision At 4
    averagePrecisionAtK = 0
    j = 1
    for i in range(4):
        if list1[i] in list2:
            averagePrecisionAtK += (1/4)*(j/(i+1))
            #print(list1[i], '-', str(j), '-', str(i+1), '-', str((1/4)*(j/(i+1))))
            j += 1
    return averagePrecisionAtK

# Tuong tu. nhu tren, nhung k quan tam den' thu' tu.
# __Recommendations__	__Precision
# [0, 0, 1]	            1/3
# [0, 1, 1]	            2/3
# [1, 1, 1]	            3/3
def Precision(list1, list2):
    return len(list(set(list1) & set(list2)))/4

dictResult = {} # Dict tong? hop. kq de? debug
totalAveragePrecision = 0 # Total AveragePrecision at 4 of all users
totalPrecision = 0 # Total Precision at 4 of all users
for userid in dictUserRead.keys():
    if userid in listUserIdNotExist:
        print(f'the user id {row["user_id"]} doesnt exist (calculate MAP part)')
        continue
    averagePrecision = averagePrecisionAtK(dictUserRecommend[userid], dictUserRead[userid])
    totalAveragePrecision += averagePrecision

    precision = Precision(dictUserRecommend[userid], dictUserRead[userid])
    totalPrecision += precision

    dictResult.update({str(userid): {"Recommend": dictUserRecommend[userid],
    "Read": dictUserRead[userid], "averagePrecision": averagePrecision, "precision": precision}})
    #{6913688:{'Recommend': ['37064', '1365', '1344', '36920'], 'Purchase':
    # ['37064', '1365', '1344', '36479'], 'precision': 0.75}

with open('debugRecom.txt', 'w+') as f:
    # Sort lai. theo precision cho de~ debug
    for key, value in sorted(dictResult.items(), key=lambda x_y: x_y[1]["precision"],reverse=True):
        f.write('%s:%s\n' % (key, value))

# Mean Average Precision and Mean Precision
MAP = totalAveragePrecision / (len(dictUserRead)-len(listUserIdNotExist)) * 100
MP = totalPrecision / (len(dictUserRead)-len(listUserIdNotExist)) * 100
with open('resultRecom.txt', 'w+') as f:
    f.write(str(scenario) + ' - ' + str(logic))
    f.write("Mean Average Precision: "+ str(MAP)+"%" + "\n" + "Mean Precision: "+ str(MP)+"%")

spark.stop()
#"recombee:default" 1 Month
# Mean Average Precision: 8.37%
# Mean Precision: 9.95%
# Scenario: Items to User, ur - default
#"recombee:personal"

#"recombee:default" 6 Month
# Mean Average Precision: 8.38
# Mean Precision: 9.95
# Scenario: Items to User, ur - default

# UserARow x R
# 0 1 2 3 4
#[4 8 4 7 4] #User A

# [0, 0, 1]	            [0, 0, 1/3]	            (1/3)(1/3) = 0.11


import dill

import pandas as pd
import json

def main():

    with open('model/sber_pipe.pkl', 'rb') as file:
        model = dill.load(file)

    with open('model/data/json_files/data_753170_277.json') as fin:
        form = json.load(fin)
        df = pd.DataFrame.from_dict([form])
        y = model.predict(df)
        print(f'{y[0]}')

if __name__ == '__main__':
    main()


import customtkinter as tk
from base_frame import BaseFrame

class AddNewTrain(BaseFrame):

    def create_widgets(self, controller):
        self.add_route_main_window_label_info = tk.CTkLabel(self, text="Добавление нового поезда", font=("Arial Bold", 20))
        self.add_route_main_window_label_info.pack(padx=100, pady=5)

        self.train_name_label = tk.CTkLabel(self, text="Название поезда", font=("Arial Bold", 15))
        self.train_name_label.pack()

        self.train_name = tk.CTkEntry(self, width=150)
        self.train_name.pack(pady=5)

        self.coupe_wagon_label = tk.CTkLabel(self, text="Количество вагонов купе: 4", font=("Arial Bold", 15))
        self.coupe_wagon_label.pack()

        self.coupe_wagon = tk.CTkSlider(self, from_=3, to=5, command=self.slider_event_coupe)
        self.coupe_wagon.configure(number_of_steps=2)
        self.coupe_wagon.pack(pady=5)

        self.placcart_wagon_label = tk.CTkLabel(self, text="Количество вагонов плацкарт: 6", font=("Arial Bold", 15))
        self.placcart_wagon_label.pack()

        self.placcart_wagon = tk.CTkSlider(self, from_=5, to=7, command=self.slider_event_placcart)
        self.placcart_wagon.configure(number_of_steps=2)
        self.placcart_wagon.pack(pady=5)

        self.add_train_button = tk.CTkButton(self, text="Добавить поезд",
                                                                           fg_color="#FF7F50",
                                                                           font=("Arial Bold", 15),
                                                                       command=lambda: controller.add_new_train())
        self.add_train_button.pack(padx=10, pady=10)

        self.exit_submit = tk.CTkButton(self, text="Назад",
                                                                 fg_color="#FF6347",
                                                                 font=("Arial Bold", 15),
                                                                 command=lambda: controller.show_trains())

        self.exit_submit.pack(padx=10, pady=10)

        self.pack()

    def slider_event_coupe(self, value):
        self.coupe_wagon_label.configure(text = "Количество вагонов купе: " + str(round(value)))

    def slider_event_placcart(self, value):
        self.placcart_wagon_label.configure(text = "Количество вагонов плацкарт: " + str(round(value)))


# https://github.com/shenweichen/DeepCTR-Torch

import tensorflow as tf
import tensorflow.keras.layers as layers

class FM(tf.keras.layers.Layer):
    """Factorization Machine models pairwise (order-2) feature interactions
     without linear term and bias.
      Input shape
        - 3D tensor with shape: ``(batch_size,field_size,embedding_size)``.
      Output shape
        - 2D tensor with shape: ``(batch_size, 1)``.
      References
        - [Factorization Machines](https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf)
    """

    def __init__(self):
        super(FM, self).__init__()

    def call(self, input):
        square_of_sum = tf.math.pow(tf.math.reduce_sum(input, 1, keepdims=True), 2)
        sum_of_square = tf.math.reduce_sum(input * input, 1, keepdims=True)
        cross_term = square_of_sum - sum_of_square
        cross_term = 0.5 * tf.math.reduce_sum(cross_term, axis=2, keepdims=False)

        return tf.squeeze(cross_term, -1)

class DNN(tf.keras.layers.Layer):
    """The Multi Layer Percetron
      Input shape
        - nD tensor with shape: ``(batch_size, ..., input_dim)``. The most common situation would be a 2D input with shape ``(batch_size, input_dim)``.
      Output shape
        - nD tensor with shape: ``(batch_size, ..., hidden_size[-1])``. For instance, for a 2D input with shape ``(batch_size, input_dim)``, the output would have shape ``(batch_size, hidden_size[-1])``.
      Arguments
        - **inputs_dim**: input feature dimension.
        - **hidden_units**:list of positive integer, the layer number and units in each layer.
        - **activation**: Activation function to use.
        - **dropout_rate**: float in [0,1). Fraction of the units to dropout.
        - **use_bn**: bool. Whether use BatchNormalization before activation or not.
    """

    def __init__(self, inputs_dim, hidden_units, activation=tf.nn.relu, dropout_rate=0, use_bn=False):
        super(DNN, self).__init__()
        self.activation = activation
        self.dropout_rate = dropout_rate
        self.dropout = layers.Dropout(dropout_rate)
        self.use_bn = use_bn
        if len(hidden_units) == 0:
            raise ValueError("hidden_units is empty!!")

        self.linears = [layers.Dense(hidden_units[i]) for i in range(len(hidden_units))]

        if self.use_bn:
            self.bn = [nn.BatchNorm1d(hidden_units[i]) for i in range(len(hidden_units))]

    def call(self, input):
        for i in range(len(self.linears)):

            fc = self.linears[i](input)
            if self.use_bn:
                fc = self.bn[i](fc)
            fc = self.activation(fc)
            fc = self.dropout(fc)
            input = fc

        return input

class CIN(tf.keras.layers.Layer):
    """Compressed Interaction Network used in xDeepFM.
      Input shape
        - 3D tensor with shape: ``(batch_size,field_size,embedding_size)``.
      Output shape
        - 2D tensor with shape: ``(batch_size, featuremap_num)`` ``featuremap_num =  sum(self.layer_size[:-1]) // 2 + self.layer_size[-1]`` if ``split_half=True``,else  ``sum(layer_size)`` .
      Arguments
        - **filed_size** : Positive integer, number of feature groups.
        - **layer_size** : list of int.Feature maps in each layer.
        - **activation** : activation function used on feature maps.
        - **split_half** : bool.if set to False, half of the feature maps in each hidden will connect to output unit.
      References
        - [Lian J, Zhou X, Zhang F, et al. xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems[J]. arXiv preprint arXiv:1803.05170, 2018.] (https://arxiv.org/pdf/1803.05170.pdf)
    """

    def __init__(self, field_size, layer_size=(128, 128), activation=tf.nn.relu, split_half=True, l2_reg=1e-5):
        super(CIN, self).__init__()

        self.layer_size = layer_size
        self.activation = activation
        self.split_half = split_half
        self.conv1ds = []

        for i, size in enumerate(layer_size):
            self.conv1ds.append(layers.Conv1D(size, 1, data_format='channels_first'))

    def call(self, input):
        batch_size, field_size, emb_size = input.shape
        hidden_nn_layers = [input]
        final_result = []

        for i, size in enumerate(self.layer_size):
            x = tf.einsum('bhd,bmd->bhmd', hidden_nn_layers[i], hidden_nn_layers[0])
            x = tf.reshape(x, (batch_size, -1, emb_size))
            x = self.conv1ds[i](x)
            if self.activation:
                x = self.activation(x)

            if self.split_half:
                if i != len(self.layer_size) - 1:
                    next_hidden, direct_connect = tf.split(x, 2 * [size // 2], 1)
                else:
                    # last layer
                    direct_connect = x
                    next_hidden = 0
            else:
                direct_connect = x
                next_hidden = x

            final_result.append(direct_connect)
            hidden_nn_layers.append(next_hidden)

        result = tf.reduce_sum(tf.concat(final_result, axis=1), -1)

        return result


import errno, os, re, sys

from ansible.module_utils.basic import AnsibleModule

PY2 = sys.version_info[0] < 3
if PY2:
    import ConfigParser as configparser, io
else:
    import configparser

DOCUMENTATION = r"""
---
module: kernel_facts
author: "David Fischer (@davidfischer-ch)"
short_description: Gather facts about the GNU/Linux Kernel configuration.
options:
  release:
    required: false
    default: null
    description:
      - Release to scan, defaults to active Kernel.
"""

EXAMPLES = r"""
- kernel_facts:
"""

CONFIG_PREFIX = re.compile(r'^config_')
DRIVER_IN_KERNEL = 'y'
DRIVER_HAS_MODULE = 'm'

def main():
    module = AnsibleModule(
        argument_spec=dict(
            release=dict(default=None)
        ),
        supports_check_mode=True
    )
    module.exit_json(
        changed=False,
        ansible_facts=dict(
            kernel_config=get_kernel_config()
        )
    )

def get_kernel_config(release=None):
    # Using index on os.uname() to be retro-compatible with old python releases
    try:
        with open('/boot/config-{0}'.format(release or os.uname()[2])) as f:
            config = configparser.ConfigParser()
            config_string = '[kernel]' + f.read()
            config.readfp(io.BytesIO(config_string)) if PY2 else config.read_string(config_string)
    except IOError as e:
        if e.errno == errno.ENOENT:
            return {}
        raise
    return {CONFIG_PREFIX.sub('', k): v for k, v in config.items('kernel')}

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
The following details the two commands used to reduce the L1 point cloud for
classification and surface reconstruction. By installing my lib pointutils you
will have access to these. In addition you will need cloud compare to visualise
results and save the final .ply file.

The pointcloud is also re-coloured using the
ortho-mosiac as the 'factory' colour illumination provided by the L1
is hopeless for machine learning.

There is much that could be done to a point cloud prior to doing anything
with it. There is no silver bullet here so consult the lit/libs etc.
These are just three of many potential steps depending on the application
(e.g. https://doc.cgal.org/latest/Point_set_processing_3/index.html).

So you would need:

- the original dense pointcloud

- an orthomosaic generated from the L1 or P1 camera

"""
from WBT.whitebox_tools import WhiteboxTools
from pointutils import utils as ut
wbt = WhiteboxTools()

#the files are called Betws as it was just up the road....
# 190 ma points - far too big much info redundant
incld = 'BetwsOrig.las'

# the size of the square area to assess neighbouring points
res = 0.5
outcld = incld[:-4]+'_culled'+str(res)+'.las'
# the filter flag culls the overlap points. You will have to adjust
# resolution to suit the data you have e.g. estimate the overlap visually or
# inspect results by trial/error

wbt.classify_overlap_points(incld, outcld, resolution=0.5, filter=True)
# 90 ma points as a result of the above

# thin using poisson as we wish to preserve surface texture
rad = 0.5
thinned = incld[:-4]+'cull_thin'+str(rad)+'.las'

# this may take a while.....
# now reduced to 3.5 ma - much better....
ut.pdal_thin(outcld, thinned, method='filters.sample', radius=rad)

# the path to the orthomosaic
inras = 'BetwsOrtho.tif'

final = thinned[:-4]+'_rgb.las'
ut.colour_cloud(thinned, inras, final, writer='las')
"""
 A bug with pdal prevents the correct translation of rgb vals to .ply format
 YOU WILL NEED cloud compare for this either open the las and save via GUI
 to ply or with the following function which calls the CC cmd line
"""
ut.las2ply_cc(final)


# import packages and set the precision
import numpy as np
import matplotlib.pyplot as plt
import cv2

np.set_printoptions(precision=5)

# computing the desired low-rank approximation by adding sufficient number of singular values
def compute_lower_rank_approx_via_SVD(data_matrix, desired_quality):
    u, s, v = np.linalg.svd(data_matrix)
    global current_approximant

    for i in range(1, len(s)+1):
        # compute low-rank approximation for different number of sigular values
        re_mat = np.dot(np.dot(u[:, :i], np.diag(s[:i])), v[:i, :])
        quality = np.linalg.norm(re_mat)/np.linalg.norm(data_matrix)
        # pick the proper number that fits the desired fidelity best
        if quality >= desired_quality:
            current_approximant = re_mat
            break

    return current_approximant

# this function divides the n x d data matrix into k-many "evenly divided, as closely as possible"
# blocks.
# for n*m matrix the splitting follows the rule as the code:
def compute_image_block(data_matrix, k):
    # get the size of given matrix, and compute the relation with k
    nrows = data_matrix.shape[0]
    ncols = data_matrix.shape[1]
    quo1 = int(nrows/k)
    rem1 = int(nrows%k)
    quo2 = int(ncols/k)
    rem2 = int(ncols%k)
    # initialize a tuple matrix
    image_block = np.zeros(shape=(k,k))
    image_block = np.array(list(zip(image_block.ravel(),image_block.ravel())),
                            dtype=('i4,i4')).reshape(image_block.shape)

    # insert values
    image_block[:rem1, :rem2] = tuple([quo1+1, quo2+1])
    image_block[:rem1, rem2:] = tuple([quo1+1, quo2])
    image_block[rem1:, :rem2] = tuple([quo1, quo2+1])
    image_block[rem1:, rem2:] = tuple([quo1, quo2])

    return image_block

# find the lower rank approximation for a given quality on each block of segmented data
# the "verbose" boolean variable is set to True if we want to print the shape of the segmented data
def get_approximation(data_matrix, k, quality, verbose):
    #first get the matrix blocks
    matrix_blocks = compute_image_block(data_matrix, k)

    #for every tuple, get the sub-matrix and store them all into a list
    comb_matrix = []
    for i in range(k):
        for j in range(k):
            dim = matrix_blocks[i, j]
            start_col  = sum([pair[1] for pair in matrix_blocks[0,:j]])
            start_row  = sum([pair[0] for pair in matrix_blocks[:i,0]])
            submat = np.matrix(data_matrix[int(start_row):int(start_row+dim[0]), int(start_col):int(start_col+dim[1])])
            comb_matrix.append(submat)

    # calculate the low-rank approximation of each sub-matrix and store
    blocked_data_matrix = []
    for i in range(k):
        for j in range(k):
            temp_mat = comb_matrix[i*k+j]
            if verbose:
                print('Shape of ({0}, {1}) block: {2}'.format(i+1, j+1, temp_mat.shape))
            approx = np.matrix(compute_lower_rank_approx_via_SVD(temp_mat, quality))
            blocked_data_matrix.append(approx)

    return reconstruct_data_from_image_block(blocked_data_matrix,k)

# this function takes the k x k image_block and reconstucts a single data_matrix from it
def reconstruct_data_from_image_block(image_block, k) :
    row_join = []
    for i in range(k):
        row_join.append(np.hstack(image_block[i*k:(i+1)*k]))

    data_matrix = np.vstack(row_join)

    return data_matrix

def convert_float64_to_uint8(A) :
    A = A/A.max()
    A = 255 * A
    return A.astype(np.uint8)

# this function "combines" the three color matrices (in the required order) to form a 3D
# array that can be rendered/viewed
def reconstruct_image_from_RGB_64bit_matrices(red, blue, green) :
    reconstructed_image = cv2.merge([convert_float64_to_uint8(blue),
                                     convert_float64_to_uint8(green),
                                     convert_float64_to_uint8(red)])
    return reconstructed_image

# verifying the block reconstruction procedure
A = np.random.random((10,10))
B = get_approximation(A, 4, 0.99999999999999, True)
C = get_approximation(A, 4, 0.9, False)
print(np.allclose(A,B))
print(np.allclose(A,C))

IMAGE = 'image.jpeg'
image = cv2.imread(IMAGE)
# change the colorspace to make the colors in the image look right
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

# split the image into blue-green-red components -- keep in mind that each of
# these entries are 8-bit ints (i.e. have a value between 0 and 255)
blue_image = image[:,:,0]
green_image = image[:,:,1]
red_image = image[:,:,2]

quality = 0.99

# let us try k = 2, 3, 4, 5 and see how the image segmentation works out
fig = plt.figure(figsize=(20, 30))
image_index = 1
axs = fig.add_subplot(5,1, image_index)
fig.tight_layout()
plt.imshow(image)
axs.set_title('Original')
image_index = image_index + 1

for k in range(2,6) :
    b = get_approximation(blue_image, k, 1 - ((1-quality)/k), False)
    g = get_approximation(green_image, k, 1 - ((1-quality)/k), False)
    r = get_approximation(red_image, k, 1 - ((1-quality)/k), False)
    axs = fig.add_subplot(5,1, image_index)
    fig.tight_layout()
    reconstructed_image = reconstruct_image_from_RGB_64bit_matrices(r, b, g)
    plt.imshow(reconstructed_image)
    axs.set_title('Quality = ' + str(round(quality,5)) + '; #Segments =' + str(k))
    image_index = image_index + 1

plt.show()


import meshio
import numpy as np
import scipy.special as special

def read_mesh(filename):

    mesh = meshio.read(filename)
    facets = mesh.points[mesh.cells_dict['triangle']]

    v0 = facets[:,0]
    v1 = facets[:,1]
    v2 = facets[:,2]

    return v0,v1,v2

class Mesh():
    def __init__(self, filename = []):

        if filename == []:
            self.v0 = np.array([])
            self.v1 = np.array([])
            self.v2 = np.array([])
        else:
            self.v0, self.v1, self.v2 = read_mesh(filename)

        self.v0 = self.v0.astype(np.double)
        self.v1 = self.v1.astype(np.double)
        self.v2 = self.v2.astype(np.double)

        self.facet_normals = np.array([])
        self.facet_area = np.array([])

        self.min = np.zeros(3)
        self.max = np.zeros(3)

        self.facet_COG=np.array([])
        self.COG= np.array([])

        self.nodes = np.array([],dtype = np.double)
        self.nodes_normal = np.array([])

        self.edges= np.array([], dtype = int)

        self.facets = np.array([], dtype = int)
        self.facet_edges = np.zeros((len(self.v0), 3), dtype = int)
        self.nodes_radius = np.ones([])

        self.vol_coords = np.array([])

def append(mesh_assembly, mesh_obj):

    if  mesh_assembly.v0.size == 0:
        mesh_assembly.v0 = np.copy(mesh_obj.v0)
        mesh_assembly.v1 = np.copy(mesh_obj.v1)
        mesh_assembly.v2 = np.copy(mesh_obj.v2)
        mesh_assembly.facet_normals = np.copy(mesh_obj.facet_normals)
        mesh_assembly.facet_area = np.copy(mesh_obj.facet_area)
        mesh_assembly.facet_COG = np.copy(mesh_obj.facet_COG)

    else:
        mesh_assembly.v0 = np.append(mesh_assembly.v0, mesh_obj.v0, axis=0)
        mesh_assembly.v1 = np.append(mesh_assembly.v1, mesh_obj.v1, axis=0)
        mesh_assembly.v2 = np.append(mesh_assembly.v2, mesh_obj.v2, axis=0)
        mesh_assembly.facet_normals = np.append(mesh_assembly.facet_normals, mesh_obj.facet_normals, axis=0)
        mesh_assembly.facet_area = np.append(mesh_assembly.facet_area, mesh_obj.facet_area, axis=0)
        mesh_assembly.facet_COG = np.append(mesh_assembly.facet_COG, mesh_obj.facet_COG, axis=0)

    return mesh_assembly

def compute_mesh(mesh, compute_radius = True):
    mesh.facet_area = compute_facet_area(mesh.v0, mesh.v1, mesh.v2)
    mesh.facet_COG = compute_facet_COG(mesh.v0, mesh.v1, mesh.v2)
    mesh.COG = compute_geometrical_COG(mesh.facet_COG, mesh.facet_area)
    mesh.facet_normals = compute_facet_normals(mesh.COG, mesh.facet_COG, mesh.v0, mesh.v1, mesh.v2)
    mesh.nodes, mesh.facets = map_facets_connectivity(mesh.v0, mesh.v1, mesh.v2)
    mesh.nodes_normal = compute_nodes_normals(len(mesh.nodes), mesh.facets ,mesh.facet_COG, mesh.v0,mesh.v1,mesh.v2)
    mesh.min, mesh.max = compute_min_max(mesh.nodes)
    mesh.edges, mesh.facet_edges = map_edges_connectivity(mesh.facets)

    if compute_radius: mesh.nodes_radius = compute_curvature(mesh.nodes, mesh.facets, mesh.nodes_normal, mesh.facet_normals, mesh.facet_area, mesh.v0,mesh.v1,mesh.v2)
    else: mesh.node_radius = np.ones((len(mesh.nodes)))

    mesh.surface_displacement = np.zeros((len(mesh.nodes),3))

    return mesh

def update_surface_displacement(mesh, surface_displacement_vector):
    mesh.nodes += surface_displacement_vector
    mesh.v0 = mesh.nodes[mesh.facets[:,0]]
    mesh.v1 = mesh.nodes[mesh.facets[:,1]]
    mesh.v2 = mesh.nodes[mesh.facets[:,2]]

    mesh.facet_area = compute_facet_area(mesh.v0, mesh.v1, mesh.v2)
    mesh.facet_COG = compute_facet_COG(mesh.v0, mesh.v1, mesh.v2)
    mesh.COG = compute_geometrical_COG(mesh.facet_COG, mesh.facet_area)
    mesh.facet_normals = compute_facet_normals(mesh.COG, mesh.facet_COG, mesh.v0, mesh.v1, mesh.v2)
    mesh.nodes_normal = compute_nodes_normals(len(mesh.nodes), mesh.facets ,mesh.facet_COG, mesh.v0,mesh.v1,mesh.v2)
    mesh.min, mesh.max = compute_min_max(mesh.nodes)

def update_volume_displacement(mesh, volume_displacement_vector):
    mesh.vol_coords += volume_displacement_vector

def compute_facet_area(v0,v1,v2):
    #Compute Area of the Mesh Facets

    v1_v0 = v1-v0
    v2_v0 = v2-v0
    v2_v1 = v2-v1

    a=np.sqrt(np.einsum('ij,ij->i',v1_v0,v1_v0))
    b=np.sqrt(np.einsum('ij,ij->i',v2_v0,v2_v0))
    c=np.sqrt(np.einsum('ij,ij->i',v2_v1,v2_v1))

    s = 0.5*(a+b+c)
    area = np.sqrt(s*(s-a)*(s-b)*(s-c))

    return area

def compute_facet_COG(v0,v1,v2):
    #Compute the center of mass of each vertex

    facet_COG = (v0+v1+v2)/3.0
    return facet_COG

def compute_geometrical_COG(facet_COG, facet_area):
    #Compute Geometrical center of the Mesh

    COG = np.sum(facet_COG*facet_area[:,None], axis = 0)/np.sum(facet_area)
    return COG

def compute_facet_normals(COG, facet_COG, v0,v1,v2):
    #Compute Facet Normals

    facet_normals = np.cross(v1 - v0, v2 - v0)
    norms = np.linalg.norm(facet_normals, axis = 1, ord=2)
    facet_normals /= norms[:,None]

    return facet_normals

def map_facets_connectivity(v0,v1,v2):
    #Map the Facets with respect to the Mesh Nodes

    nodes = np.hstack((v0,v1,v2))
    nodes.shape=(3*len(v0),3)
    unique , idx, inv =np.unique(nodes, axis = 0, return_index = True, return_inverse = True)
    nodes = unique

    facets=np.copy(inv)
    facets.shape = (len(v0),3)

    return nodes, facets

def map_edges_connectivity(facets):
    #Map the Edges with respect to the Mesh Nodes
    #Additionally, maps the Facets with respect to the Edges

    facet_edges = np.zeros((len(facets)*3), dtype = int)
    edges = np.array([], dtype = int)

    edge1 = facets[:,0:2]
    edge2 = facets[:,1:3]
    edge3 = np.stack((facets[:,2],facets[:,0]), axis = -1)

    edges = np.hstack((edge1,edge2,edge3))
    edges.shape=(len(facets)*3,2)

    edges_original = np.copy(edges)
    edges_sorted = np.copy(np.sort(edges)) #Sort Edges from the lower node ID to higher node ID for each edge

    unique, idx, inv = np.unique(edges_sorted, axis = 0, return_index = True, return_inverse = True)

    edges= unique
    num_edges=len(edges)

    # Create Matrix of Elems_connects wrt vector of Edges (The +1 is for Ease the use in GMSH
    facet_edges = np.copy(inv) + 1

    edges_inv = edges[inv]

    mask = (edges_inv[:,0] == edges_original[:,0]) * (edges_inv[:,1] == edges_original[:,1])
    facet_edges[~mask] *=-1
    facet_edges.shape=(len(facets),3)

    return edges, facet_edges

def compute_nodes_normals(num_nodes, facets ,facet_COG, v0,v1,v2):
    #Compute the normal at the Vertex

    nodes_normal = np.zeros((num_nodes,3))

    CG = facet_COG
    CE_01 = (v0+v1)/2.0
    CE_12 = (v1+v2)/2.0
    CE_20 = (v2+v0)/2.0

    cross_CE_01_v0 = np.cross(CE_01 - v0, CG - v0, axis = 1)
    cross_v0_CE_20 = np.cross(CG - v0, CE_20 - v0, axis = 1)
    cross_CE_12_v1 = np.cross(CE_12 - v1, CG - v1, axis = 1)
    cross_v1_CE_01 = np.cross(CG - v1, CE_01 - v1, axis = 1)
    cross_CE_20_v2 = np.cross(CE_20 - v2, CG - v2, axis = 1)
    cross_v2_CE_12 = np.cross(CG - v2, CE_12 - v2, axis = 1)

    np.add.at(nodes_normal, facets[:,0], 0.5* (cross_CE_01_v0 + cross_v0_CE_20))
    np.add.at(nodes_normal, facets[:,1], 0.5* (cross_CE_12_v1 + cross_v1_CE_01))
    np.add.at(nodes_normal, facets[:,2], 0.5* (cross_CE_20_v2 + cross_v2_CE_12))

    return nodes_normal

def compute_min_max(nodes):
    #Computes the minimum and maximum coordinates of the mesh

    _min = np.zeros((3))
    _max = np.zeros((3))

    _min=np.min(nodes, axis = 0)
    _max=np.max(nodes, axis = 0)

    return _min, _max

def compute_curvature(nodes, facets,nodes_normal, facets_normals, facets_area, v0, v1, v2):

    avType = 'e'
    Nsmooth = int(np.round(len(facets)/100, decimals = 0))
    M2M_RR = 10.0
    flatEdge = 0.9
    flatWeightFlag = 2

    free_vector = np.array([1,0,0])
    p1 = np.dot(facets_normals, free_vector)
    p1 = p1<0

    Theta = np.pi/2 - np.arccos(np.clip(np.sum(- free_vector * facets_normals[p1] , axis = 1), -1.0, 1.0))

    area = np.sum(facets_area[p1]*np.sin(Theta))

    Rmax = np.sqrt(area/np.pi)*2
    Rmin = Rmax/M2M_RR

    # Based on Fostrad module -> based on the work of Itzik Ben Shabat

    VertexNormals,Avertex,Acorner,up,vp, avEdge = calculate_vertex_normals(nodes, facets, facets_normals, facets_area, v0,v1,v2)
    VertexSFM=calculate_curvature(VertexNormals,Avertex,Acorner,up,vp,nodes,facets,facets_normals,v0,v1,v2)
    CurvUxVy,PrincipalDir1,PrincipalDir2=getPrincipalCurvatures(nodes,VertexSFM,up,vp)

    CurvUxVy = np.abs(CurvUxVy)

    radiiOnVerts = np.zeros((len(nodes)))

    SearchRadius=avEdge

    ParFlag = len(nodes)/50000.0

    with np.errstate(divide='ignore'):
        radiiOnVerts[:] = np.sqrt(1/(CurvUxVy[:,0]*CurvUxVy[:,1]))

    radiiOnVerts[(CurvUxVy[:,0] > 1/Rmax) * (CurvUxVy[:,1]< 1/Rmax)] = 1./CurvUxVy[(CurvUxVy[:,0] > 1/Rmax) * (CurvUxVy[:,1]< 1/Rmax),0]
    radiiOnVerts[(CurvUxVy[:,1] > 1/Rmax) * (CurvUxVy[:,0]< 1/Rmax)] = 1./CurvUxVy[(CurvUxVy[:,1] > 1/Rmax) * (CurvUxVy[:,0]< 1/Rmax),1]

    radiiOnVerts[radiiOnVerts > Rmax] = Rmax
    radiiOnVerts[radiiOnVerts < Rmin] = Rmin

    for i in range(len(nodes)):
        InnerPointsInd = searchableRadius(nodes[i],nodes,SearchRadius,Nsmooth)
        if len(InnerPointsInd) <= 1: continue
        radiiOnVerts[i]= sphVolSmoothing(InnerPointsInd, radiiOnVerts, avType, Nsmooth, Rmax, flatEdge, flatWeightFlag)

    node_radius = radiiOnVerts

    node_radius[node_radius <= 0] = Rmin

    return node_radius

def sphVolSmoothing(InnerPointsInd, propOnVerts, avType, Nsmooth, MaxRefRadius, flatEdge, flatWeightFlag):

    propOnVerts = propOnVerts[InnerPointsInd]
    MaxRefRadius*= 0.99

    Npoints = len(propOnVerts)
    if Npoints<Nsmooth+1: Nsmooth = Npoints-2

    propOnVerts=propOnVerts[-Nsmooth::]
    NpointsNonInf = np.sum(propOnVerts<MaxRefRadius)
    NpointsInf = Nsmooth+1-NpointsNonInf

    flatRatio = NpointsInf/Nsmooth
    propBaseline = np.mean(propOnVerts[propOnVerts<MaxRefRadius])

    if flatRatio >= flatEdge: flatFlag = 1
    else:
        flatFlag = 0
        if flatWeightFlag ==2:
            propOnVerts[propOnVerts>=MaxRefRadius] = propBaseline + (MaxRefRadius - propBaseline)*((1+special.erf(flatRatio*4-2))/2)

    if flatFlag == 1: smoothSphProp = MaxRefRadius
    else:
        if avType == 'e':
            smoothSphProp = exponential_moving_average(propOnVerts[propOnVerts<MaxRefRadius], Nsmooth-NpointsInf , Nsmooth/32.0)#Nsmooth/2.0)
            smoothSphProp = smoothSphProp[-1]

    return smoothSphProp
    pass

def searchableRadius(center, nodes, SearchRadius, Nsmooth):

    dist = nodes - center
    dist = np.sqrt(dist[:,0]**2 + dist[:,1]**2 + dist[:,2]**2)

    SearchPointsInd = np.where((dist <=  SearchRadius))[0] # selected searchable indexes on original V reference
    SearchPointsXYZ = nodes[SearchPointsInd] # selected searchable points

    if len(SearchPointsXYZ) <= Nsmooth:
        SearchPointsInd = np.where((dist <= SearchRadius*10))[0] # selected searchable indexes on original V reference
        SearchPointsXYZ = nodes[SearchPointsInd] # selected searchable points

    Dxyz = center - SearchPointsXYZ
    SearchPointsDist = np.sqrt(Dxyz[:,0]**2 + Dxyz[:,1]**2 + Dxyz[:,2]**2)

    ind = np.argsort(SearchPointsDist )[::-1]
    SearchPointsDist = SearchPointsDist[ind]
    SearchPointsInd = SearchPointsInd[ind]

    SearchPosInd = SearchPointsInd[SearchPointsDist<=SearchRadius*10]
    Nind = np.min((Nsmooth+3, len(SearchPosInd)-1))
    SearchPosInd = SearchPosInd[-Nind::]

    return SearchPosInd

def calculate_vertex_normals(nodes, facets, facets_normals, facets_area, v0,v1,v2):
    VertexNormals = np.zeros((len(nodes),3))
    up = np.zeros((len(nodes),3))
    vp = np.zeros((len(nodes),3))

    Acorner = np.zeros((len(facets),3))
    Avertex = np.zeros((len(nodes)))

    e0=v2-v1
    e1=v0-v2
    e2=v1-v0

    e0_norm=np.linalg.norm(e0 , ord = 2, axis = 1)
    e1_norm=np.linalg.norm(e1 , ord = 2, axis = 1)
    e2_norm=np.linalg.norm(e2 , ord = 2, axis = 1)

    #edge lengths
    de0=np.sqrt(e0[:,0]**2+e0[:,1]**2+e0[:,2]**2)
    de1=np.sqrt(e1[:,0]**2+e1[:,1]**2+e1[:,2]**2)
    de2=np.sqrt(e2[:,0]**2+e2[:,1]**2+e2[:,2]**2)

    De0 = np.mean(de0[de0>1e-6])
    De1 = np.mean(de1[de1>1e-6])
    De2 = np.mean(de2[de2>1e-6])

    avEdge = np.mean([De0,De1,De2])

    # Change to numpy.dot (should be faster)
    l0_2 = np.linalg.norm(e0,axis=1)*np.linalg.norm(e0,axis=1)
    l1_2 = np.linalg.norm(e1,axis=1)*np.linalg.norm(e1,axis=1)
    l2_2 = np.linalg.norm(e2,axis=1)*np.linalg.norm(e2,axis=1)
    ew = np.stack( (l0_2*(l1_2+l2_2-l0_2), l1_2*(l0_2+l2_2-l1_2), l2_2*(l0_2+l1_2-l2_2)), axis = -1)

    wfv0 = facets_area/(l1_2*l2_2)
    wfv1 = facets_area/(l0_2*l2_2)
    wfv2 = facets_area/(l1_2*l0_2)

    #Has to be like this to not find simultaneous acess to the same memory
    np.add.at(VertexNormals,facets[:,0], (wfv0[:,None]*facets_normals))
    np.add.at(VertexNormals,facets[:,1], (wfv1[:,None]*facets_normals))
    np.add.at(VertexNormals,facets[:,2], (wfv2[:,None]*facets_normals))

    ew_0 = (ew[:,0]<=0)
    ew_1 = (ew[:,1]<=0)
    ew_2 = (ew[:,2]<=0)
    ew_t = np.ones(len(ew_0), dtype = bool)*(~ew_0)*(~ew_1)*(~ew_2)

    if ew_0.any():
        Acorner[ew_0, 1] = -0.25*l2_2[ew_0]*facets_area[ew_0]/(np.sum(e0[ew_0]*e2[ew_0],axis=1))
        Acorner[ew_0, 2] = -0.25*l1_2[ew_0]*facets_area[ew_0]/(np.sum(e0[ew_0]*e1[ew_0],axis=1))
        Acorner[ew_0, 0] = facets_area[ew_0] - Acorner[ew_0, 1] - Acorner[ew_0, 2]

    if ew_1.any():
        Acorner[ew_1, 0] = -0.25*l2_2[ew_1]*facets_area[ew_1]/(np.sum(e1[ew_1]*e2[ew_1],axis=1))
        Acorner[ew_1, 2] = -0.25*l0_2[ew_1]*facets_area[ew_1]/(np.sum(e1[ew_1]*e0[ew_1],axis=1))
        Acorner[ew_1, 1] = facets_area[ew_1] - Acorner[ew_1, 0] - Acorner[ew_1, 2]

    if ew_2.any():
        Acorner[ew_2, 0] = -0.25*l1_2[ew_2]*facets_area[ew_2]/(np.sum(e2[ew_2]*e1[ew_2],axis=1))
        Acorner[ew_2, 1] = -0.25*l0_2[ew_2]*facets_area[ew_2]/(np.sum(e2[ew_2]*e0[ew_2],axis=1))
        Acorner[ew_2, 2] = facets_area[ew_2] - Acorner[ew_2, 0] - Acorner[ew_2, 1]

    if ew_t.any():
        ewscale=0.5*facets_area[ew_t]/(ew[ew_t,0]+ew[ew_t,1]+ew[ew_t,2])
        Acorner[ew_t, 0] = ewscale*(ew[ew_t,1]+ew[ew_t,2])
        Acorner[ew_t, 1] = ewscale*(ew[ew_t,0]+ew[ew_t,2])
        Acorner[ew_t, 2] = ewscale*(ew[ew_t,0]+ew[ew_t,1])

    np.add.at(Avertex,facets, Acorner)
    up[facets] = np.stack((e2/e2_norm[:,None],e0/e0_norm[:,None],e1/e1_norm[:,None]), axis = 1)

    VertexNormals=VertexNormals/np.linalg.norm(VertexNormals, axis=1, ord=2)[:,None]
    up = np.cross(up, VertexNormals, axis = 1)/np.linalg.norm(np.cross(up,VertexNormals, axis = 1), axis=1, ord = 2)[:,None]
    vp = np.cross(VertexNormals, up, axis = 1)

    return VertexNormals,Avertex,Acorner,up,vp, avEdge

def calculate_curvature(VertexNormals,Avertex,Acorner,up,vp,nodes,facets,facets_normals,v0,v1,v2):

    e0=v2-v1
    e1=v0-v2
    e2=v1-v0

    e0_norm=e0/np.linalg.norm(e0 , ord = 2, axis = 1)[:,None]
    #e1_norm=e1/numpy.linalg.norm(e1 , ord = 2, axis = 1)[:,None]
    #e2_norm=e2/numpy.linalg.norm(e2 , ord = 2, axis = 1)[:,None]

    B = np.cross(facets_normals,e0_norm,axis=1)
    B/= np.linalg.norm(B, axis = 1)[:,None]

    A = np.zeros((len(facets_normals),6,3))
    b = np.zeros((len(facets_normals),6))

    A[:,0,0] = np.sum(e0*e0_norm, axis = 1);         A[:,0,1] = np.sum(e0*B, axis = 1) ;              A[:,0,2] = 0;
    A[:,1,0] = 0;                                    A[:,1,1] = np.sum(e0*e0_norm, axis = 1);         A[:,1,2] = np.sum(e0*B, axis = 1) ;
    A[:,2,0] = np.sum(e1*e0_norm, axis = 1);         A[:,2,1] = np.sum(e1*B, axis = 1) ;              A[:,2,2] = 0;
    A[:,3,0] = 0;                                    A[:,3,1] = np.sum(e1*e0_norm, axis = 1) ;        A[:,3,2] = np.sum(e1*B, axis = 1) ;
    A[:,4,0] = np.sum(e2*e0_norm, axis = 1);         A[:,4,1] = np.sum(e2*B, axis = 1) ;              A[:,4,2] = 0;
    A[:,5,0] = 0;                                    A[:,5,1] = np.sum(e2*e0_norm, axis = 1) ;        A[:,5,2] = np.sum(e2*B, axis = 1) ;

    n0 = VertexNormals[facets[:,0]]
    n1 = VertexNormals[facets[:,1]]
    n2 = VertexNormals[facets[:,2]]

    b[:,0] = np.sum((n2-n1)*e0_norm , axis = 1)
    b[:,1] = np.sum((n2-n1)*B  , axis = 1)
    b[:,2] = np.sum((n0-n2)*e0_norm , axis = 1)
    b[:,3] = np.sum((n0-n2)*B  , axis = 1)
    b[:,4] = np.sum((n1-n0)*e0_norm , axis = 1)
    b[:,5] = np.sum((n1-n0)*B  , axis = 1)

    x=np.zeros((int(len(b)),3))

    #BOTTLENECK
    for i in range(len(b)):
        x[i]=np.linalg.lstsq(A[i],b[i], rcond=None)[0]

    wfp = Acorner[:,:]/Avertex[facets[:]]

    VertexSFM = np.zeros((len(nodes),4))

    new_ku,new_kuv,new_kv = ProjectCurvatureTensor(e0_norm,B,facets_normals,x[:,0],x[:,1],x[:,2],up[facets],vp[facets])

    horizontal_stack_0 = np.stack((new_ku[:,0],new_kuv[:,0],new_kuv[:,0],new_kv[:,0]), axis = -1)
    horizontal_stack_1 = np.stack((new_ku[:,1],new_kuv[:,1],new_kuv[:,1],new_kv[:,1]), axis = -1)
    horizontal_stack_2 = np.stack((new_ku[:,2],new_kuv[:,2],new_kuv[:,2],new_kv[:,2]), axis = -1)

    np.add.at(VertexSFM, facets[:,0], wfp[:,0][:,None]*horizontal_stack_0)
    np.add.at(VertexSFM, facets[:,1], wfp[:,1][:,None]*horizontal_stack_1)
    np.add.at(VertexSFM, facets[:,2], wfp[:,2][:,None]*horizontal_stack_2)

    VertexSFM.shape = (-1,2,2)

    return VertexSFM

def RotateCoordinateSystem_nd(up,vp,nf, axis = 2):
    r_new_u=up
    r_new_v=vp

    if axis == 2:
        _np=np.cross(up,vp, axis = -1)
        _np=_np/np.linalg.norm(_np, axis = -1)[:,:,None]

        arrays_temp = nf
        nf = np.repeat(nf[:,:,np.newaxis], repeats = 3, axis = -1)
        nf[:] = arrays_temp[:,None]

        ndot=np.sum(nf*_np, axis = -1)

        p = ndot<=-1
        r_new_u[p]=-r_new_u[p]
        r_new_v[p]=-r_new_v[p]

        p = ~p

        arrays_temp = ndot
        ndot = np.repeat(ndot[:,:,np.newaxis], repeats = 3, axis = -1)
        perp = nf-ndot*_np
        dperp=(_np+nf)/(1+ndot)

        aux_1 = np.sum(perp*r_new_u, axis = -1)
        aux_1 = np.repeat(aux_1[:,:,np.newaxis], repeats = 3, axis = -1)

        aux_2 = np.sum(perp*r_new_v, axis = -1)
        aux_2 = np.repeat(aux_2[:,:,np.newaxis], repeats = 3, axis = -1)

        r_new_u[p]=(r_new_u-dperp*aux_1)[p]
        r_new_v[p]=(r_new_v-dperp*aux_2)[p]

    if axis == 1:

        _np=np.cross(up,vp, axis = -1)
        _np=_np/np.linalg.norm(_np, axis = -1)[:,None]

        arrays_temp = nf
        nf[:] = arrays_temp[None]
        ndot=np.sum(nf*_np, axis = -1)

        p = ndot<=-1

        r_new_u[p]=-r_new_u[p]
        r_new_v[p]=-r_new_v[p]

        p = ~p

        arrays_temp = ndot
        perp = nf-ndot[:,None]*_np
        dperp=(_np+nf)/(1+ndot[:,None])

        aux_1 = np.sum(perp*r_new_u, axis = 1)
        aux_2 = np.sum(perp*r_new_v, axis = -1)

        r_new_u[p]=(r_new_u-dperp*aux_1[:,None])[p]
        r_new_v[p]=(r_new_v-dperp*aux_2[:,None])[p]

    return r_new_u,r_new_v

def RotateCoordinateSystem_1d(up,vp,nf):
    r_new_u=up;
    r_new_v=vp;
    _np=np.cross(up,vp);
    _np=_np/np.linalg.norm(_np);
    ndot=np.sum(nf*_np);

    if ndot<=-1:
        r_new_u=-r_new_u;
        r_new_v=-r_new_v;
        return r_new_u,r_new_v
    perp=nf-ndot*_np;
    dperp=(_np+nf)/(1+ndot);

    r_new_u=r_new_u-dperp*(np.sum(perp*r_new_u));
    r_new_v=r_new_v-dperp*(np.sum(perp*r_new_v));
    return r_new_u,r_new_v

def ProjectCurvatureTensor(uf,vf,nf,old_ku,old_kuv,old_kv,up,vp):
    r_new_u,r_new_v=RotateCoordinateSystem_nd(up,vp,nf)
    OldTensor=np.transpose(np.array([[old_ku, old_kuv],[old_kuv, old_kv]]),(2,1,0))

    arrays_temp = uf
    uf = np.repeat(uf[:,:,np.newaxis], repeats = 3, axis = -1)
    uf[:] = arrays_temp[:,None]

    arrays_temp = vf
    vf = np.repeat(vf[:,:,np.newaxis], repeats = 3, axis = -1)
    vf[:] = arrays_temp[:,None]

    u1=np.sum(r_new_u*uf, axis = -1)
    v1=np.sum(r_new_u*vf, axis = -1)
    u2=np.sum(r_new_v*uf, axis = -1)
    v2=np.sum(r_new_v*vf, axis = -1)

    A1= np.transpose(np.array([u1,v1]),(1,2,0))
    A2= np.transpose(np.array([u2,v2]),(1,2,0))

    diagonal = np.array([[True,False,False],[False,True,False],[False,False,True]])

    new_ku  = np.matmul(np.matmul(A1,OldTensor),A1.transpose((0, 2, 1)))[:, diagonal]
    new_kuv = np.matmul(np.matmul(A1,OldTensor),A2.transpose((0, 2, 1)))[:, diagonal]
    new_kv  = np.matmul(np.matmul(A2,OldTensor),A2.transpose((0, 2, 1)))[:, diagonal]

    return new_ku, new_kuv, new_kv

def getPrincipalCurvatures(nodes,VertexSFM,up,vp):
    PrincipalCurvatures=np.empty((len(nodes),2))
    PrincipalDir1=np.empty((len(nodes),3))
    PrincipalDir2=np.empty((len(nodes),3))

    c = np.ones((len(up)))
    s = np.zeros((len(up)))
    tt = np.zeros((len(up)))
    nf = np.cross(up,vp)
    r_old_u,r_old_v=RotateCoordinateSystem_nd(up,vp,nf, axis = 1)

    ku = VertexSFM[:,0,0]
    kuv = VertexSFM[:,0,1]
    kv = VertexSFM[:,1,1]

    mask = (kuv != 0)
    h = np.zeros((len(kuv)))
    h[mask] = 0.5 * (kv[mask]-ku[mask])/kuv[mask]

    mask2 = (h < 0)

    tt[mask*mask2] = 1/(h-np.sqrt(1+h*h))[mask*mask2]
    tt[mask*~mask2] = 1/(h+np.sqrt(1+h*h))[mask*~mask2]

    c[mask] = (1/np.sqrt(1+tt*tt))[mask]
    s[mask] = (tt*c)[mask]

    k1 = ku - tt * kuv
    k2 = kv + tt * kuv

    mask = (np.abs(k1) >= np.abs(k2))
    PrincipalDir1[mask]= (c[:,None]*r_old_u - s[:,None]*r_old_v)[mask]
    PrincipalDir1[~mask]= (c[:,None]*r_old_u + s[:,None]*r_old_v)[~mask]
    PrincipalDir2 = np.cross(nf, PrincipalDir1)

    PrincipalCurvatures[:,0][mask] = k1[mask]
    PrincipalCurvatures[:,1][mask] = k2[mask]

    PrincipalCurvatures[:,0][~mask] = k2[~mask]
    PrincipalCurvatures[:,1][~mask] = k1[~mask]

    return PrincipalCurvatures,PrincipalDir1,PrincipalDir2

def exponential_moving_average(signal, points, smoothing=2):
    """
    Calculate the N-point exponential moving average of a signal

    Inputs:
        signal: numpy array -   A sequence of price points in time
        points:      int    -   The size of the moving average
        smoothing: float    -   The smoothing factor

    Outputs:
        ma:     numpy array -   The moving average at each point in the signal
    """

    weight = smoothing / (points + 1)
    ema = np.zeros(len(signal))
    ema[0] = signal[0]

    for i in range(1, len(signal)):
        ema[i] = (signal[i] * weight) + (ema[i - 1] * (1 - weight))

    return ema

def remove_repeated_facets(assembly_mesh):

    facet_COG = np.around(assembly_mesh.facet_COG, decimals = 6)

    ___, idx, count = np.unique(facet_COG, axis = 0, return_index = True, return_counts = True)

    p = idx.argsort()
    idx=idx[p]
    count=count[p]

    i = 0

    assembly_mesh.facet_COG = assembly_mesh.facet_COG[idx]
    assembly_mesh.v0 =       assembly_mesh.v0[idx]
    assembly_mesh.v1 =       assembly_mesh.v1[idx]
    assembly_mesh.v2 =       assembly_mesh.v2[idx]
    assembly_mesh.facet_normals =  assembly_mesh.facet_normals[idx]
    assembly_mesh.facet_area =     assembly_mesh.facet_area[idx]

    #idx is the CFD Grid

    while i < len(idx):
        if count[i] > 1:
            idx = np.delete(idx,i)
            count = np.delete(count,i)
        else: i+=1

    return idx

def create_index(body_atr, obj_atr):

    cond_obj = np.char.add(np.char.add(obj_atr[:,0].astype(str),obj_atr[:,1].astype(str)),obj_atr[:,2].astype(str) )
    cond_body = np.char.add(np.char.add(body_atr[:,0].astype(str),body_atr[:,1].astype(str)), body_atr[:,2].astype(str))

    mask = np.in1d(cond_body,cond_obj)
    index = np.where(mask)[0]

    return index, mask

def compute_new_volume(assembly, old_nodes):

    assembly.mesh.vol_elements = np.array([], dtype = int)
    assembly.mesh.vol_density = np.array([])
    assembly.mesh.vol_tag = np.array([], dtype = int)

    for obj in assembly.objects:
        size = len(obj.mesh.vol_elements)
        assembly.mesh.vol_elements = np.append(assembly.mesh.vol_elements,obj.mesh.vol_elements)
        assembly.mesh.vol_density = np.append(assembly.mesh.vol_density, np.ones(size)*obj.material.density)
        assembly.mesh.vol_tag = np.append(assembly.mesh.vol_tag, np.ones(size)*obj.id)

    assembly.mesh.vol_elements.shape = (-1,4)
    old_elements = np.copy(assembly.mesh.vol_elements)

    #Creation of the new nodes list:
    nodes = old_nodes[assembly.mesh.vol_elements]
    nodes.shape = (-1,3)

    unique, index, idx, count = np.unique(nodes, axis = 0, return_index= True, return_inverse = True, return_counts = True)
    idx.shape = (-1,4)

    assembly.mesh.vol_coords = unique
    assembly.mesh.vol_elements = idx

    #Order elements according to surface mesh
    surf_nodes = np.copy(assembly.mesh.nodes)

    condition = np.append(unique, surf_nodes,axis = 0)
    unique, idx, count = np.unique(condition, axis = 0, return_inverse = True, return_counts = True)
    unique2 = np.copy(unique)

    #Switch the nodes:
    #Change the elements connectivity #TODO better way
    index_surf_1 = idx[len(unique):]
    index_surf_2 = range(len(surf_nodes))

    index_old = np.arange(len(unique))
    for i in range(len(surf_nodes)):

        temp = np.copy(unique[index_surf_2[i]])
        unique[index_surf_2[i]] = unique[index_surf_1[i]] # surf_nodes
        unique[index_surf_1[i]] = temp

        temp = np.copy(index_old[index_surf_2[i]])
        index_old[index_surf_2[i]] = index_old[index_surf_1[i]]
        index_old[index_surf_1[i]] = temp

        temp1 = (assembly.mesh.vol_elements == index_surf_1[i])
        temp2 = (assembly.mesh.vol_elements == index_surf_2[i])

        assembly.mesh.vol_elements[temp1] = index_surf_2[i]
        assembly.mesh.vol_elements[temp2] = index_surf_1[i]

    assembly.mesh.vol_coords = unique

    start = 0
    for obj in assembly.objects:
        obj.mesh.vol_elements = np.copy(assembly.mesh.vol_elements[start:start+len(obj.mesh.vol_elements)])
        start += len(obj.mesh.vol_elements)

    return index,old_elements,index_old


from entities import Entities

class OutsideLights(Entities):
    def initialize(self):
        super().initialize()
        for l in self.args['lights']:
            e = self.register_entity(l, l)
            e.listen(self.found, {'entity': e})

        self.listen_event(self.update, 'TOD_TOD')
        self.run_in(lambda *_: self.update(None, None, None), 2)

    def update(self, event, data, kwarg):
        self.state = self.get_app('timeofday').dark
        if self.get_app('timeofday').tod == 'night':
            self.state = False

        self.log(f"OUTSIDE LIGHTS - Turning lights {self.state}")
        for l in self.e:
            self.e[l].state = self.state
            self.e[l].push()

    def found(self, old, new, kwarg):
        entity = kwarg.get('entity', None)
        if not entity: return;

        if old == "unavailable":
            self.log(f"{entity.entity_id} showed up, setting to {self.state}")
            entity.state = self.state
            entity.push()

class DecorativeLights(OutsideLights):
    def update(self, event, data, kwarg):
        self.state = self.get_app('timeofday').dark
        self.log(f"DECORATIVE LIGHTS - Turning lights {self.state}")
        for l in self.e:
            self.e[l].state = self.state
            self.e[l].push()


import tensorflow as tf

time_steps = None
batch_size = None
num_feat = None
lstm_size = None

timeseries = tf.placeholder(tf.float32,[time_steps,batch_size,num_feat])

lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size)

state = lstm.zero_state(batch_size,dtype=tf.float32)

probabilities = []
loss = 0.0

for current_batch_of_input_data in timeseries:
    output, state = lstm(current_batch_of_input_data,state)
    logits = tf.matmul(output,softmax_w+softmax_b)
    probabilities.append(tf.nn.softmax(logits))
    loss += loss_function
# def model_fn():
#
#     layer1 = tf.keras.layers.LSTM()
#     layer2 =