"""
This example uses the match operator to match three records with different
sampling instances.
"""

import math
from typing import Dict

import rx
from matplotlib import pyplot
from rx import operators as rxop
from rx.subject import Subject

import rxbp
from rxbp.indexed.indexedflowable import IndexedFlowable

subject = Subject()

fdict = {
    't_main': rxbp.indexed.from_flowable(source=rxbp.from_rx(subject), base='t_main').share(),
    't_1': rxbp.indexed.range(0, 100, base='t_1').pipe(
        rxbp.op.map(lambda i: 10 * i/100 * 0.9 + 2.1),
    ).share(),
    't_2': rxbp.indexed.range(0, 100, base='t_2').pipe(
        rxbp.op.map(lambda i: 10 * i/100 * 1.05 + 0.9),
    ).share(),
}

def time_value_flowables(fdict: Dict[str, IndexedFlowable]):
    return {
        'v_main': fdict['t_main'].pipe(
            rxbp.op.map(lambda t: math.cos(t * 2 * math.pi - math.pi / 8)),
        ),
        'v_1': fdict['t_1'].pipe(
            rxbp.op.map(lambda t: math.cos(t * 2 * math.pi - math.pi / 8)),
        ),
        'v_2': fdict['t_2'].pipe(
            rxbp.op.map(lambda t: math.cos(t * 2 * math.pi - math.pi / 8)),
        ),
        **fdict,
    }

def interpolate_value(t_pair, v_pair, t):
    (t1, t2) = t_pair
    (v1, v2) = v_pair

    return v1 + (t - t1) / (t2 - t1) * (v2 - v1)

def sync_t1_to_tmain(fdict: Dict[str, IndexedFlowable]):
    t_1_base_main = fdict['t_1'].pipe(
        rxbp.op.pairwise(),
        rxbp.op.controlled_zip(
            right=fdict['t_main'],
            request_left=lambda l, r: l[1] <= r,
            request_right=lambda l, r: r < l[1],
            match_func=lambda l, r: l[0] <= r < l[1],
        ),
    )

    return {
        't_1_base_main': t_1_base_main,
        **fdict
    }

def sync_t2_to_tmain(fdict: Dict[str, IndexedFlowable]):
    t_2_base_main = fdict['t_2'].pipe(
        rxbp.op.pairwise(),
        rxbp.op.controlled_zip(
            right=fdict['t_main'],
            request_left=lambda l, r: l[1] <= r,
            request_right=lambda l, r: r < l[1],
            match_func=lambda l, r: l[0] <= r < l[1],
        ),
    )

    return {
        't_2_base_main': t_2_base_main,
        **fdict
    }

def sync_t1_and_t2(fdict: Dict[str, IndexedFlowable]):
    t_sync = fdict['t_1_base_main'].pipe(
        rxbp.indexed.op.match(
            fdict['t_2_base_main'],
        ),
    )

    return {
        't_sync': t_sync.pipe(rxbp.op.map(lambda t: t[0][1])),
        't_1_sync': t_sync.pipe(rxbp.op.map(lambda t: t[0])),
        't_2_sync': t_sync.pipe(rxbp.op.map(lambda t: t[1])),
        **fdict,
    }

def interpolate_v1(fdict: Dict[str, IndexedFlowable]):
    v_1_sync = fdict['v_1'].pipe(
        rxbp.op.pairwise(),
        rxbp.indexed.op.match(
            fdict['t_1_sync'],
        ),
        rxbp.op.map(lambda t: interpolate_value(t_pair=t[1][0], v_pair=t[0], t=t[1][1]))
    )

    return {
        'v_1_sync': v_1_sync,
        **fdict,
    }

def interpolate_v2(fdict: Dict[str, IndexedFlowable]):
    v_2_sync = fdict['v_2'].pipe(
        rxbp.op.pairwise(),
        rxbp.indexed.op.match(
            fdict['t_2_sync'],
        ),
        rxbp.op.map(lambda t: interpolate_value(t_pair=t[1][0], v_pair=t[0], t=t[1][1]))
    )

    return {
        'v_2_sync': v_2_sync,
        **fdict,
    }

def sel_flow(fdict: Dict[str, IndexedFlowable]):
    output = fdict['t_sync'].pipe(
        rxbp.op.zip(
            fdict['v_1_sync'],
            fdict['v_2_sync'],
        ),
    )

    return output

result = []

rxbp.multicast.return_value(fdict).pipe(
    rxbp.multicast.op.map(time_value_flowables),
    rxbp.multicast.op.map(sync_t1_to_tmain),
    rxbp.multicast.op.map(sync_t2_to_tmain),
    rxbp.multicast.op.map(sync_t1_and_t2),
    rxbp.multicast.op.map(interpolate_v1),
    rxbp.multicast.op.map(interpolate_v2),
    rxbp.multicast.op.map(sel_flow),
).to_flowable().subscribe(
    on_next=lambda v: result.append(v),
    on_completed=lambda: print('completed')
)

rx.range(0, 1000).pipe(
    rxop.map(lambda i: 10 * i/1000 * 1.01 + 1.4),
).subscribe(subject)

t, v1, v2 = list(zip(*result))

pyplot.plot(t, v1)
pyplot.plot(t, v2)
pyplot.show()


from common.CommonUtility import CommonUtility
from utils.CryptoDotComExchangeManager import CryptoDotComExchangeManager

def handleGetTrading(event):

    try:
        # Initialize CommonUtility Class
        common_utility = CommonUtility()
        # Create Exchange manager class
        cryptoExchange = CryptoDotComExchangeManager()

        # Get current BTC price
        cryptoExchange.getCrrentPriceForTicker("BTC", "USD")
    except Exception as e:
        print(f"handleGetTrading(): ${e}")
        return common_utility.buildResponse(500, "A server error occurred. Please try again later")


dx = [-1, 1, 0, 0]
dy = [0, 0, -1, 1]

# x, y: 기준점, max_depth: BFS 의 depth
# res: 함수의 반환값 -> 해당 범위 내의 최대 house 의 개수
def houses_in_k_area(x, y, max_depth):
    queue = [(x, y)]
    depth, res = 1, 0

    visited = [[0] * n_city for _ in range(n_city)]
    visited[x][y] = depth
    # 처음 위치(x, y) 에 집이 있으면, result 에 counting 한다.
    if city[x][y]:
        res += 1

    while queue:
        x, y = queue.pop(0)
        depth = visited[x][y]

        # 보안 범위 (max_depth = k) 만큼 탐색을 완료했으면, whlie 문(BFS 탐색) 을 종료한다.
        if depth == max_depth:
            break

        # 현재 기준점(x, y) 에서 4 방향의 nx, ny 를 확인하며
        # 방문하지 않은 곳으로 BFS 탐색을 최종적으로 depth 만큼 한다.
        for p in range(4):
            nx, ny = x + dx[p], y + dy[p]
            if 0 <= nx < n_city and 0 <= ny < n_city and not visited[nx][ny]:
                visited[nx][ny] = depth + 1
                queue.append((nx, ny))
                # 방문할 위치(nx, ny) 에 집이 있으면, result 에 counting 한다.
                if city[nx][ny]:
                    res += 1

    return res

n_tests = int(input())

for t in range(1, n_tests + 1):
    # n_city: 도시의 크기, income: 각 집마다 지불하는 비용
    # city: 도시의 정보
    n_city, income = map(int, input().split())
    city = [list(map(int, input().split())) for _ in range(n_city)]

    # k: 방범 운영 범위 상수 (k = 1 ~ )
    k, max_k = 1, n_city + (n_city // 2) + 2
    result = 0
    while True:
        # n_houses: 운영 범위(k) 가 고정되었을 때, 얼마나 많은 집을 지킬 수 있는가
        # (기준의 되는 위치를 바꾸며, 항상 최대값을 확인)
        n_houses = 0
        # expenses: 서비스 운영 비용, profit: 서비스를 제공하고 얻는 수익
        expenses, profit = k * k + (k - 1) * (k - 1), 0

        for i in range(n_city):
            for j in range(n_city):
                # 1. 매 방범 운영 범위(기준은 for 문을 돌며 바뀜)에 집이 얼마나 존재하는지 확인한다.
                n_houses = max(n_houses, houses_in_k_area(i, j, k))

        # 2. k 가 고정되었을 때, 최대로 얻을 수 있는 수익을 계산한다.
        # 손해가 없다면, result 값에 최대 집의 개수를 저장해놓는다.
        profit = income * n_houses - expenses
        if profit >= 0:
            result = n_houses

        # 만약 max_k 에 k 가 도달하면, while 문을 종료하며 결과를 출력한다.
        if k == max_k:
            print(f"#{t}", result)
            break
        else:
            k += 1

SWEA/Unrated/2117. ［모의 SW 역량테스트］ 홈 방범 서비스/［모의 SW 역량테스트］ 홈 방범 서비스.py

［모의 SW 역량테스트］ 홈 방범 서비스.py


import unittest
from unittest.mock import patch
from game import TicTacToe

class TestGame(unittest.TestCase):

    def setUp(self):
        self.game = TicTacToe()
        self.game.board[4] = "X"

    def test_available_moves(self):
        self.assertTrue(4 not in self.game.available_moves())

    def test_make_move(self):
        self.assertFalse(self.game.make_move(4, "X"))
        self.assertTrue(self.game.make_move(5, "X"))
        self.assertTrue(self.game.board[5] != " ")

    def test_winner(self):
        """
        у функції test_winner є три інші функції columns_check, rows_check, diagonal_check
        у кожній  функції створюється всі можливі варіанти заповнення дошки для виграшу (але тільки одна лінія, всі інші клітинки дошки пусті)
        :return:
        """

        def columns_check(letter):
            for columns in range(3):
                for rows in range(3):
                    self.game.board = [" " for i in range(9)]
                    row = [letter] * 3
                    self.game.board[rows * 3: (rows + 1) * 3] = row
                    # print(self.game.board)
                    self.assertTrue(self.game.winner(columns + rows * 3, letter))

        def rows_check(letter):
            for columns in range(3):
                for rows in range(3):
                    self.game.board = [" " for i in range(9)]
                    self.game.board[columns], self.game.board[columns + 3], self.game.board[columns + 6] = letter * 3
                    # print(self.game.board)
                    self.assertTrue(self.game.winner(rows * 3 + columns, letter))

        def diagonal_check(letter):
            a = [0, 4, 8]
            b = [2, 4, 6]
            for i in (a, b):
                for x in i:
                    self.game.board = [" " for i in range(9)]
                    for m in range(3):
                        self.game.board[i[m]] = letter
                    # print(self.game.board)
                    self.assertTrue(self.game.winner(x, letter))

        for l in ("X", "O"):
            columns_check(l)
            rows_check(l)
            diagonal_check(l)

if __name__ == '__main__':
    unittest.main()


class Solution:
    """
    O(m*n) time, O(m*n) space, only need dfs, set compare takes O(1)
    we do the dfs store all island relative position into set, then every time add frozenset into set to compare
    """
    def numDistinctIslands(self, grid: [[int]]) -> int:
        # this compare unique function is too slow, used set add set (frozenset)
        # def current_island_is_unique():
        #     for other_island in unique_islands:
        #         if len(other_island) != len(current_island):
        #             continue
        #         for cell_1, cell_2 in zip(current_island, other_island):
        #             if cell_1 != cell_2:
        #                 break
        #         else:
        #             return False
        #     return True

        # Do a DFS to find all cells in the current island.
        def dfs(row, col):
            if row < 0 or col < 0 or row >= len(grid) or col >= len(grid[0]):
                return
            if (row, col) in seen or grid[row][col] == 0:
                return
            seen.add((row, col))
            current_island.add((row - row_origin, col - col_origin))
            dfs(row + 1, col)
            dfs(row - 1, col)
            dfs(row, col + 1)
            dfs(row, col - 1)

        # Repeatedly start DFS's as long as there are islands remaining.
        seen = set()
        unique_islands = set()
        for row in range(len(grid)):
            for col in range(len(grid[0])):
                if grid[row][col] == 1:
                    current_island = set()
                    row_origin = row
                    col_origin = col
                    dfs(row, col)
                    if current_island:
                        unique_islands.add(frozenset(current_island))
        return len(unique_islands)


from django.shortcuts import render

# Create your views here.

def home(response):
    degree = 0
    if response.method == "POST":
        answers = response.POST.get('answers')
        result = response.POST.get('result')
        if answers == "django":
            degree += 50
        if result == "3":
            degree += 50
    return render(response, "main/home.html", {"degree": degree})


"""
Based on http://www.logarithmic.net/pfh/blog/01164790008
"""
import deprecation
import heapq

from ..dists import Dist
from .._version import __version__

class _Node:
    def __init__(self):
        self.lower_bounds = []
        self.upper_bounds = []
        self.children = []
        self.vantage = None

    def minimum_distance(self, distances):
        minimum = 0.0
        for i in range(len(distances)):
            if distances[i] < self.lower_bounds[i]:
                minimum = max(minimum, self.lower_bounds[i] - distances[i])
            elif distances[i] > self.upper_bounds[i]:
                minimum = max(minimum, distances[i] - self.upper_bounds[i])
        return minimum

    def help_find(self, item, distances, heap, distance):
        d = distance(self.vantage, item)
        new_distances = distances + (d,)

        heapq.heappush(heap, (d, 0, self.vantage))

        for child in self.children:
            heapq.heappush(heap, (child.minimum_distance(new_distances), 1, child, new_distances))

    def __lt__(self, other):
        return len(self.children) < len(other.children)

    def __gt__(self, other):
        return len(self.children) > len(other.children)

class VP_tree(object):
    @deprecation.deprecated(deprecated_in="0.6", removed_in="1.0", current_version=__version__,
                            details="Use jvptree.VPTree instead")
    def __init__(self, distance, max_children=2):
        """ items        : list of items to make tree out of
            distance     : function that returns the distance between two items
            max_children : maximum number of children for each node

            Using larger max_children will reduce the time needed to construct the tree,
            but may make queries less efficient.
        """
        self.distance = distance
        self.max_children = max_children

    @deprecation.deprecated(deprecated_in="0.6", removed_in="1.0", current_version=__version__,
                            details="Use jvptree.VPTree instead")
    def fit(self, points) -> None:
        self.root = self._make_node([(item, ()) for item in points])

    def _make_node(self, items):
        if not items:
            return None

        node = _Node()
        for i in range(len(items[0][1])):
            distance_list = [item[1][i] for item in items]
            node.lower_bounds.append(min(distance_list))
            node.upper_bounds.append(max(distance_list))

        item = items.pop(0)
        node.vantage = item[0]

        if not items:
            return node

        items = self._fill_items(node, items)

        distances = {}
        for item in items:
            distances[item[1][-1]] = True
        distance_list = list(distances.keys())
        distance_list.sort()
        n_children = self._get_n_children(distance_list)
        split_points = [-1]
        for i in range(n_children):
            split_points.append(distance_list[(i + 1) * (len(distance_list) - 1) // n_children])

        for i in range(n_children):
            child_items = [item for item in items if split_points[i] < item[1][-1] <= split_points[i + 1]]
            child = self._make_node(child_items)
            if child:
                node.children.append(child)

        return node

    def _fill_items(self, node, items):
        return [(item[0], item[1] + (self.distance(node.vantage, item[0]),)) for item in items]

    def _get_n_children(self, distance_list):
        return min(self.max_children, len(distance_list))

    @deprecation.deprecated(deprecated_in="0.6", removed_in="1.0", current_version=__version__,
                            details="Use jvptree.VPTree instead")
    def find(self, item):
        """ Return iterator yielding items in tree in order of distance from supplied item.
        """
        if not self.root:
            return

        heap = [(0, 1, self.root, ())]

        while heap:
            top = heapq.heappop(heap)
            if top[1]:
                top[2].help_find(item, top[3], heap, self.distance)
            else:
                yield top[2], top[0]

class CustomVPTree(VP_tree):
    @deprecation.deprecated(deprecated_in="0.6", removed_in="1.0", current_version=__version__,
                            details="Use jvptree.VPTree instead")
    def __init__(self, dist: Dist):
        super().__init__(dist.dist)
        self.dist_batch_fn = dist.batch_dist
        self.max_children = 2
        self.leaf_size = 5

    @deprecation.deprecated(deprecated_in="0.6", removed_in="1.0", current_version=__version__,
                            details="Use jvptree.VPTree instead")
    def set_dist(self, dist: Dist):
        self.distance = dist.dist
        self.dist_batch_fn = dist.batch_dist

    def __getstate__(self):
        """Return state values to be pickled."""
        return self.root, self.max_children, self.leaf_size

    def __setstate__(self, state):
        """Restore state from the unpickled state values."""
        self.root, self.max_children, self.leaf_size = state

    def _fill_items(self, node, items):
        dists = self.dist_batch_fn([(node.vantage, item[0]) for item in items])
        return [(item[0], item[1] + (dists[i],)) for i, item in enumerate(items)]

    def _get_n_children(self, distance_list):
        if len(distance_list) > self.leaf_size:
            return self.max_children
        else:
            return len(distance_list)


from Products.CMFCore.utils import UniqueObject
from OFS.SimpleItem import SimpleItem
from AccessControl import ClassSecurityInfo
from Products.CMFCore.utils import getToolByName
from Globals import InitializeClass
import Missing
from lxml import etree

try:
    from zope.app.component.hooks import getSite
except ImportError:
    from zope.component.hooks import getSite

class SolrCatalogTool(UniqueObject, SimpleItem):

    id = 'solr_catalog'
    meta_type = 'Solr Catalog Tool'

    security = ClassSecurityInfo()

    @property
    def hello(self):
        return "hello"

    def getProperty(self, property_name, default=''):
        site = getSite()

        ptool = getToolByName(site, 'portal_properties')
        props = ptool.get("agsci_solr_properties")

        if props:
            return props.getProperty(property_name, default)
        else:
            return default

    def getSolrURL(self):
        protocol = self.getProperty('solr_protocol', 'http')
        host = self.getProperty('solr_host', '127.0.0.1')
        port = self.getProperty('solr_port', '8080')
        base_url = self.getProperty('solr_base_url', '/solr')

        return "%s://%s:%s%s" % (protocol, host, port, base_url)

    @property
    def siteId(self):
        site = getSite()
        return site.getId()

    def indexObject(self, context):
        metadata = self.getMetaDataFor(context)
        indexdata = self.getIndexDataFor(context)
        return False

    @property
    def portal_catalog(self):
        return getToolByName(getSite(), 'portal_catalog')

    def getMetadataFor(self, context):
        m = self.portal_catalog.getMetadataForUID("/".join(context.getPhysicalPath()))
        for k in m.keys():
            if m[k] == Missing.Value:
                m[k] = ''
        return m

    def getIndexDataFor(self, context):
        return self.portal_catalog.getIndexDataForUID("/".join(context.getPhysicalPath()))

    def getCatalogIndexes(self):
        return self.portal_catalog.indexes()

    def getCatalogMetadata(self):
        return self.portal_catalog.schema()

    def exportSchemaXML(self):
        # <schema name="example" version="1.5">
        schema = etree.Element("schema", name="example", version="1.5")

        # Create unique key (UID)
        unique_key = etree.Element('uniqueKey')
        unique_key.text = 'UID'
        schema.append(unique_key)

        # Get indexes and metadata
        indexes = self.getCatalogIndexes()
        metadata = self.getCatalogMetadata()

        # Add a <field> tag for all (most) indexed fields
        fields = list(set(indexes).union(set(metadata)))

        for f in fields:
            p = self.getIndexProperties(f, indexes=indexes, metadata=metadata)

            if not p.get('skip'):
                el = etree.Element("field",
                                    name=p['name'], type=p['type'],
                                    indexed=repr(p['indexed']).lower(), stored=repr(p['stored']).lower(),
                                    multiValued=repr(p['multiValued']).lower(),
                                    required=repr(p['required']).lower()
                                    )
                schema.append(el)

        # Return XML
        return etree.tostring(schema, pretty_print=True, xml_declaration=True, encoding='UTF-8')

    def getIndexProperties(self, index, indexes=[], metadata=[]):

        required = ['getId', 'UID', 'id']

        p = {
                'skip' : False,
                'type' : 'string',
                'name' : index,
                'indexed' : False,
                'stored' : False,
                'multiValued' : False,
                'required' : False,
        }

        i_obj = self.portal_catalog.Indexes.get(index)

        index_config = {
                        'DateIndex' : ('date', False),
                        'FieldIndex' : ('string', False),
                        'ZCTextIndex' : ('text', False),
                        'BooleanIndex' : ('boolean', False),
                        'KeywordIndex' : ('string', True),
                        'UUIDIndex' : ('string', True),
        }

        if index in ('Tags', ):
            import pdb; pdb.set_trace()

        if i_obj:
            (index_type, multiValued) = index_config.get(i_obj.meta_type, (None,False))

            if not index_type:
                p['skip'] = True
            else:
                p['type'] = index_type
                p['multiValued'] = multiValued
                if index in indexes:
                    p['indexed'] = True
                if index in metadata:
                    p['stored'] = True
                if index in required:
                    p['required'] = True
                if hasattr(i_obj, 'indexed_attrs') and i_obj.indexed_attrs:
                    p['name'] = i_obj.indexed_attrs[0]

        return p

InitializeClass(SolrCatalogTool)


#linear regression

import tensorflow as tf
tf.set_random_seed(777)

w=tf.Variable(tf.random_normal([1])) #정규분포 난수 1개 -변수
b=tf.Variable(tf.random_normal([1])) #정규분포 난수 1개 -변수
x= tf.placeholder(tf.float32, shape=[None])
y= tf.placeholder(tf.float32, shape=[None])

hf = x*w+b

cost=tf.reduce_mean(tf.square(hf-y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
train=optimizer.minimize(cost)
##########▲그래프 정의

sess=tf.Session()
sess.run(tf.global_variables_initializer()) # 초기화

for step in range(2001):
    sess.run(train,feed_dict={x:[1,2,3],y:[1,2,3]}) #그래프 실행
    if step%20==0:
        print(step,sess.run(cost,feed_dict={x:[1,2,3],y:[1,2,3]}))
print("---------모델완성-----------")
# print(step,sess.run(cost,feed_dict={x:[1,2,3],y:[1,2,3]}))
# print(step,sess.run(w))
# print(step,sess.run(b))
cv,wv,bv=sess.run([cost,w,b],feed_dict={x:[1,2,3],y:[1,2,3]})
print("비용:",cv, "가중치:",wv, "편향:",bv)

#예측해보기
#x=2.5 ,x=5, x=1.5, 3.5???
print("2.5일때 예상되는 값: ",sess.run(hf,feed_dict={x:[2.5]}))
print("5일때 예상되는 값: ",sess.run(hf,feed_dict={x:[5]}))
print("1.5일때와 3.5일때 예상되는 값: ",sess.run(hf,feed_dict={x:[1.5,3.5]}))


import os
import sys
from datetime import datetime

import dateutil.relativedelta as relativedelta
import gspread as gd
import numpy as np
import pandas as pd
import pandas_ta as pta
import requests
import yfinance as yf

gc = gd.service_account(filename='../options-349716-50a9f6e13067.json')
worksheet = gc.open("work_table").worksheet('Активы с Call опционами')

d_len = worksheet.get('D2:D60')
q_len = worksheet.get('Q2:Q60')
a_tick = worksheet.get('A2:A60')

d = []
q = []
a = []
test = []
for i in range(len(d_len)):
    if (len(d_len[i])) == 1:
        d.append(d_len[i])
        q.append(q_len[i])
        q[i] = int(str(q[i])[2:-2])
        print(f"{d[i]}   {q[i]}")
    else:
        break

for i in range(len(a_tick)):
    a.append(a_tick[i])
    a[i] = str(a[i])[2:-2]
    if a[i] == 'Ticker':
        tmp = i
        break

print(a)

guru_tickers = []
for i in range(len(d)):
    if q[i] == 0:
        guru_tickers.append(str(d[i])[2:-2])

print(d)
print(len(d))
print(guru_tickers)
print(len(guru_tickers))

print(guru_tickers)

#guru_tickers = ['XPAR:UBI', 'HKSE:00941', 'HKSE:09988', 'NYSE:EBS', 'NAS:AEIS', 'NAS:VRTX', 'FRA:BAYN', 'NAS:CHKP', 'ARCA:GDX', 'ARCA:SLV', 'NYSE:TME', 'HKSE:01099', 'NAS:INTC', 'NYSE:EPD', 'ARCA:AMLP', 'NYSE:RIO', 'NYSE:BHP', 'NAS:STNE', 'NYSE:CI', 'NYSE:EVN', 'HKSE:00700', 'NAS:MKTX', 'NAS:GBOX', 'NAS:TER', 'NYSE:TDOC', 'MIL:DIA']

tickers_list = []
gf_list = []
TOTAL_DF = pd.DataFrame()

for tick in guru_tickers:
    print(tick)
    try:
        data_json = requests.get(
            f'https://api.gurufocus.com/public/user/34b27ff5b013ecb09d2eeafdf8724472:683d6c833f9571582151f19efe2278a8/stock/{tick}/summary').json()
        print(data_json)
        gf = data_json['summary']['company_data']['p2gf_value']

        tickers_list.append(tick)
        gf_list.append(float(gf))

    except Exception as e:
        tickers_list.append(tick)
        gf_list.append(0)
        exc_type, exc_obj, exc_tb = sys.exc_info()
        fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        print(exc_type, fname, exc_tb.tb_lineno)
        pass

TOTAL_DF['Ticker'] = tickers_list
TOTAL_DF['Gf_value'] = gf_list

print('First')
print(TOTAL_DF)

listus = TOTAL_DF['Ticker'].tolist()
yahoo_list = []
j = 0
for num in range(len(listus)):
    print(j)
    print(listus[num])
    if "TSX:" in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.TO')
    elif 'TSXV:' in listus[num]:
         yahoo_list.append(listus[num].split(':')[1]+'.V')
    elif 'XSWX:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.SW')
    elif 'IST:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.IS')
    elif 'XKLS:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.KL')
    elif 'LSE:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.L')
    elif 'SGX:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.SI')
    elif 'HKSE:0' in listus[num]:
        yahoo_list.append(listus[num].replace('HKSE:0', '')+'.HK')
    elif 'ASX:' in listus[num]:
            yahoo_list.append(listus[num].split(':')[1]+'.AX')
    elif 'JSE:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.JO')
    elif 'XTAE:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.TA')
    elif 'TSE:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.T')
    elif 'MIC:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.ME')
        # elif 'MEX:' in listus[num]:
        #     yahoo_list.append(listus[num].split(':')[1]+'.MX')
    elif 'XMAD:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.MC')
    elif 'FRA:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.F')
    elif 'XAMS:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.AS')
    elif 'XBRU:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.BR')
    elif 'XPAR:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.PA')
    elif 'MIL:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1]+'.MI')
    elif 'NAS:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1])
    elif 'NYSE:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1])
    elif 'AMEX:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1])
    elif 'ARCA:' in listus[num]:
        yahoo_list.append(listus[num].split(':')[1])
    else:
        yahoo_list.append(listus[num])
    print(yahoo_list[-1])
    j += 1

std_list = []
std_30_list = []
rsi = []
i = 0
print(listus)
print(len(listus))
print(yahoo_list)
print(len(yahoo_list))

for tick in yahoo_list:
    try:
        today = datetime.today()
        start_vol = (datetime.today() - relativedelta.relativedelta(years=1))
        end_vol = today

        ohlc = pd.DataFrame()
        ohlc[tick] = yf.download(tick, start_vol, end_vol)['Close']

        log_df = pd.DataFrame()

        log_df['returns'] = np.log(ohlc[tick] / ohlc[tick].shift(-1))
        daily_std = np.std(log_df['returns'])
        std = (daily_std * 252 ** 0.5)

        days_30 = ohlc[tick][-10:]
        returns_30 = np.log(days_30 / days_30.shift(-1))
        daily_std_30 = np.std(returns_30)
        std_30 = (daily_std_30 * 252 ** 0.5)

        std_list.append(std)
        std_30_list.append(std_30)

        rsi_pr = pta.rsi(ohlc[tick])
        print(tick)
        print(rsi_pr[-1])
        i += 1
        rsi.append(rsi_pr[-1])

        tickers_list.append(tick)
    except:
        print(f'Ticker {tick} dont works today!')

print(rsi)
print(f"Total cycles {i}")
TOTAL_DF['RSI'] = rsi
TOTAL_DF['std'] = std_list
TOTAL_DF['std_30'] = std_30_list
TOTAL_DF['dif_vol'] = TOTAL_DF['std_30'] - TOTAL_DF['std']

TOTAL_DF = TOTAL_DF.sort_values('Gf_value', ascending=False).reset_index(drop=True)
TOTAL_DF['Gf_value_Score'] = TOTAL_DF.index.tolist()

TOTAL_DF = TOTAL_DF.sort_values('RSI',ascending=False).reset_index(drop=True)
TOTAL_DF['RSI_Score'] = TOTAL_DF.index.tolist()

TOTAL_DF = TOTAL_DF.sort_values('dif_vol',ascending=False).reset_index(drop=True)
TOTAL_DF['Vol_Score'] = TOTAL_DF.index.tolist()

TOTAL_DF['Total_score'] = TOTAL_DF['Gf_value_Score'] + TOTAL_DF['RSI_Score'] + TOTAL_DF['Vol_Score']
TOTAL_DF = TOTAL_DF.sort_values('Total_score').reset_index(drop=True)

print('Last')
print(TOTAL_DF)

print(TOTAL_DF['Ticker'][:10].tolist())

worksheet.update(f'A{tmp+3}', TOTAL_DF.values.tolist())

at = worksheet.get(f'A{tmp+3}:A{tmp+12}')

res = []
for i in range(len(at)):
    res.append(str(at[i]))
    res[i] = res[i][2:-2]

print(res)


#!/usr/bin/env python3
import sys
import os
import time
import argparse
import requests
from datetime import datetime
from urllib.parse import urlparse
from termcolor import colored

def append_http(url, port=None):
    if "https://" in url:
        url = f"{url}"
    elif "http://" not in url:
        o = urlparse(f"http://{url}")
        if o.port:
            url = f"http://{url}"
        else:
            url = f"http://{url}:{port}"
    return url

def get_loopchain_state(ipaddr="localhost", port=os.environ.get('RPC_PORT', 9000)):
    url = append_http(ipaddr, port) + "/admin/chain"
    return_result = {}
    try:
        session = requests.Session()
        r = session.get(url, verify=False, timeout=10)
        return_result = r.json()[0]
        return_result['prev_time'] = time.time()
        return_result['url'] = url
    except Exception as e:
        print(f"Error while connecting server... {url}: {str(e)}")
        return_result = {
            "block_height": 0
        }
    return return_result

def disable_ssl_warnings():
    from urllib3.exceptions import InsecureRequestWarning
    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

def print_status(count, block_diff, sync_speed, finish_block_height, finish_time):
    status = f"({count})   BlockDiff [{block_diff}]    SyncSpeed [{sync_speed:.0f}]    FinishHeight [{finish_block_height:.0f}]    FinishTime [{finish_time}]"
    if sync_speed == 0:
        return colored(status, 'red')
    return colored(status, 'green')

if __name__ == '__main__':
    disable_ssl_warnings()
    parser = argparse.ArgumentParser(prog='checker')
    parser.add_argument('url', nargs='?', default="localhost:9000")
    parser.add_argument('-n', '--network', type=str, help='parent network type (mainnet|berlin|sejong|lisbon)', default="mainnet")
    parser.add_argument('-v', '--verbose', action='count', help='verbose mode. view level', default=0)
    parser.add_argument('-c', '--count', type=int, help='check count(sec).', default=0)

    args = parser.parse_args()

    network_info = {
        "mainnet": "http://52.196.159.184:9000",
        "berlin": "https://berlin.net.solidwallet.io",
        "sejong": "https://sejong.net.solidwallet.io",
        "lisbon": "https://lisbon.net.solidwallet.io",
        "havah": "https://ctz.havah.io",
        "vega": "https://ctz.vega.havah.io",
    }

    if os.environ.get('SERVICE', None):
        args.network = os.environ['SERVICE']
    elif os.environ.get('NETWORK_ENV', None):
        args.network = os.environ['NETWORK_ENV']

    if args.network in network_info.keys():
        parent_network_url = network_info.get(args.network)
    else:
        parent_network_url = args.network

    block_diff = 0
    block_diff2 = 0
    network_block_height = 0
    first_block_height = 0
    second_block_height = 0
    finish_block_height = 0
    prev_time = 0
    sync_speed = 0
    height = 0
    count = 0

    print(f"Target peer: {args.url}")
    if args.verbose > 0:
        print(f"Get parent status from {parent_network_url}")

    first_block_height, first_prevtime = get_loopchain_state(args.url)["height"], get_loopchain_state(args.url)["prev_time"]

    try:
        while True:
            count += 1
            second_block_height, second_prevtime = get_loopchain_state(args.url)["height"], get_loopchain_state(args.url)["prev_time"]
            network_block_height, prevtime = get_loopchain_state(parent_network_url)["height"], get_loopchain_state(parent_network_url)["prev_time"]

            block_diff = network_block_height - second_block_height
            block_diff2 = second_block_height - first_block_height
            sync_speed = block_diff2 / (second_prevtime - first_prevtime) if first_prevtime != 0 and second_prevtime != first_prevtime else 0
            finish_block_height = (block_diff / sync_speed / 2 + network_block_height) if sync_speed != 0 else float('inf')
            finish_time = datetime.fromtimestamp(int(((finish_block_height - second_block_height) / sync_speed) + time.time())) if sync_speed != 0 and finish_block_height != float('inf') else 'N/A'

            status = print_status(count, block_diff, sync_speed, finish_block_height, finish_time)
            sys.stdout.write('\r' + status)
            sys.stdout.flush()

            if count == args.count:
                break

            time.sleep(1)
    except KeyboardInterrupt:
        print("\nProgram interrupted. Exiting...")
        sys.exit(0)


def doi_dong(A,B):
	l=len(A)
	for i in range (l):
		A[i],B[i]=B[i],A[i]

def chuyenhesove1(A,j):
	l=len(A)
	chia=A[j]
	if(chia!=0):
		for i in range(l):
			A[i]=A[i]/chia

def congvoimatrankhac(A,B,n):
	l=len(A)
	for i in range (l):
		B[i]+=A[i]*n
def Gauss_elimination(A):
	m=len(A)
	n=len(A[0])

	for i in range(m):
		dieukien=0
		for j in range(i,m):
			if(A[j][i]!=0):
				dieukien=1
				doi_dong(A[i],A[j])
				break
		if(dieukien==0):
			continue
		chuyenhesove1(A[i],i)
		for j in range(i+1,m):
			bientam=-(A[j][i]/A[i][i])
			congvoimatrankhac(A[i],A[j],bientam)
	print("Ma tran bac thang: ")
	for i in range(len(A)):
		print(A[i])
def back_substitution(A):
	m=len(A)
	n=len(A[0])
	for i in range(m):
		if A[i][i]==0:
			if(A[i][n-1]!=0):
				print("VN")
				return
			elif ((m-n+1) <=0):
				print("VSN")
				return
	x=[0]
	for i in range (n-2,-1,-1): #Chay tu n-1 den 0 (so phan tu nghiem)
		k=0
		b=A[i][n-1]
		for j in range(i+1,n-1):
			b-=(A[i][j]*x[k])

			k=k+1
		x.insert(0,b)
	x.pop()
	print("Nghiem:")
	for i in range(len(x)):
		print("x",i,"=",x[i])

A=[[1,-2,3,-3],[2,2,0,0],[0,-3,4,1],[1,0,1,-1]]
Gauss_elimination(A)
back_substitution(A)


############
# APPROACH #
############

#The data given behaves as an absorbing Markov chain.

#To find the stable distribution of an absorbing Markov chain (in other
# words, the most likely final distribution of the ore between the terminal states),
# you need to first write up the transition matrix (making sure it is in standard form).
#Then, you calculate the limiting distribution.
#This can be plugged back into a matrix of the same dimensions as the
# transition matrix and multiplied by a matrix of the initial state distributions.
#This will then yield the stable distribution of the Markov chain.
#https://en.wikipedia.org/wiki/Absorbing_Markov_chain
#https://www.youtube.com/watch?v=u89Sd514EDI

#An important distinction mentioned in the instructions is that a path
# from the current state to a terminal state will always exist.
#This is a necessary condition for the limiting distribution of the
# absorbing chain to exist.

#I am working in the standard form below:
# I | 0
#-------
# R | Q

#Based off the following video: https://www.youtube.com/watch?v=qhnFHnLkrfA

########
# NOTE #
########

#If running this script from the commandline, the input list of lists
# must be put in quotation marks

###########
# IMPORTS #
###########

import sys, ast
import numpy as np
from numpy.linalg import inv

##########################
# LOAD AND FORMAT MATRIX #
##########################

def Load_matx(m):
    """Take matrix input from commandline and format for analysis"""
    #Convert counts to conversion rates
    m=[[num/sum(row) for num in row] if sum(row)>0 else row for row in m]
    #Convert list of list to matrix
    M=np.matrix(m,dtype=float)

    #Find index of unreachable states (will have column sum equal to 0)
    unreach=M.sum(axis=0)
    elimIdx=[x for x in np.where(unreach==0)[1].tolist() if x!=0]
    #Remove unreachable states for the time being
    #(For the analysis to work, all rows must sum to 1 and having an unreachable
    # state will disrupt this)
    M=np.delete(M, elimIdx, axis=1)
    M=np.delete(M, elimIdx, axis=0)

    return([M,elimIdx])

def Stand_form(M):
    """Convert matrix of transition probabilities to standard form of
    absorbant Markov chain"""
    #Make template for standard form matrix
    Mstand=np.matrix(np.zeros(M.shape))

    #Fill in the I (identity) section of the standard form matrix by
    # identifying absorbant/terminal states
    absorb=M.sum(axis=1)
    identIdx=[x for x in np.where(absorb==0)[0].tolist() if x!=0]
    for pos,val in enumerate(identIdx):
        Mstand[pos,pos]=1

    #Fill in the R and Q sections of the standard form matrix
    saveIdx=np.where(M.any(axis=1))[0].tolist()
    saveRows=M[saveIdx]
    #New order of axes
    ord_new=identIdx+saveIdx
    #Put new section in with data reflecting new axes order
    Mstand[len(identIdx):]=saveRows[:,ord_new]
    return([Mstand,identIdx])
    #I did consider just returning the R and Q components at this stage
    # instead of the full standard matrix but thought it might be useful
    # to have the standard form accessible at some point.

###################################
# CALCULATE LIMITING DISTRIBUTION #
###################################

def Lim_dist(Mstand,identIdx):
    """Take a standard form of an absorbant Markov chain matrix and
    return the limting distribution matrix"""
    #Get R and Q sections
    R=Mstand[len(identIdx):,:len(identIdx)]
    Q=Mstand[len(identIdx):,len(identIdx):]

    #Calculate the fundamental matrix (F=(I-Q)^-1)
    F=inv(np.identity(len(Q))-Q)

    #Calculate new section that will replace R
    newSec=F.dot(R)

    #Assemble fundamental matrix
    LDmatx=np.matrix(np.zeros(Mstand.shape))
    #Add in identity section
    for pos,val in enumerate(identIdx):
        LDmatx[pos,pos]=1
    #Add in product of F and R
    LDmatx[len(identIdx):,:len(identIdx)]=newSec

    return(LDmatx)

#######################################
# CALCULATE STABLE/FINAL DISTRIBUTION #
#######################################

def Fin_dist(LDmatx,elimIdx,identIdx):
    """Create an initial state matrix (reflecting all ore in state 0
    to calculate the final distribution between terminal states.
    Format answer to reflect question requirements"""
    #Create matrix for initial state with 100% of the ore being in
    # state 0 (wherever that is located on the new axes)
    initState=np.matrix(np.zeros(len(LDmatx)))
    initState[0,len(identIdx)]=1

    #Solve dot product of initial and limiting distribution matrices
    endDist=initState.dot(LDmatx).tolist()[0]
    #Put states back in order
    endDist_ord=endDist[len(identIdx):]+endDist[:len(identIdx)]
    #Add in any unreachable states
    if elimIdx:
        for i in elimIdx:
            endDist_ord.insert(i,0.)

    #Get rid of non-terminal states to give answer
    ans=[]
    for pos,val in enumerate(endDist_ord):
        if val!=0 or pos in elimIdx:
            ans.append(val)

    return(ans)

##################
# FULL EXECUTION #
##################

def main(listOfLists):
    """Runs all functions to return the final distributions from the
    input list of lists"""
    M1, eIdx=Load_matx(listOfLists)
    M2, iIdx=Stand_form(M1)
    M3=Lim_dist(M2,iIdx)
    M4=Fin_dist(M3,eIdx,iIdx)

    print("The final predicted proportion of ore in each terminal state is:")
    print(M4)

    return(M4)

###############################
# TAKE INPUT FROM COMMANDLINE #
###############################

if __name__ == "__main__":
    #Run the input if given
    if len(sys.argv)>1:
        main(ast.literal_eval(sys.argv[1]))

    #If no input given, run the test cases
    else:
        print("Distribution for test case 1:")
        main([[0, 2, 1, 0, 0], [0, 0, 0, 3, 4], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]])

        print("Distribution for test case 2:")
        main([[0, 1, 0, 0, 0, 1], [4, 0, 0, 3, 2, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0]])

################
# RUN EXAMPLES #
################

#This section contains the two test cases and code to run through them
# step by step, examining the matrix created after each step.
#The final 2 lines will run the full pipeline on each sample.

#~ testr=[[0, 2, 1, 0, 0], [0, 0, 0, 3, 4], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]
#~ testr=[[0, 1, 0, 0, 0, 1], [4, 0, 0, 3, 2, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0]]

#~ M1, eIdx=Load_matx(testr)
#~ M1
#~ M2, iIdx=Stand_form(M1)
#~ M2
#~ M3=Lim_dist(M2,iIdx)
#~ M3
#~ M4=Fin_dist(M3,eIdx,iIdx)
#~ M4


import os
import requests
from pprint import pprint

class DataManager:
    def __init__(self):
        self.Sheet_get_endpoint=os.environ.get("SHEET_GET")
        self.Sheet_post_endpoint=os.environ.get("SHEET_POST")
        self.Sheet_put_endpoint=os.environ.get("SHEET_PUT")
        self.header={
            "Authorization":os.environ.get("SHEET_AUTHOURIZATION")
        }

    def acquire_sheet_data(self):
        response=requests.get(url=self.Sheet_get_endpoint,headers=self.header)
        response.raise_for_status()
        result=response.json()
        return result['prices']

    def update_data(self,updated_sheet_data):
        for item in updated_sheet_data:
            updated_sheet_put_endpoint=f"{self.Sheet_put_endpoint}/{item['id']}"

            parameter={
                "price":item
            }
            response=requests.put(url=updated_sheet_put_endpoint,json=parameter,headers=self.header)

            response.raise_for_status()


import sys

M, N = map(int, sys.stdin.readline().split())
C = list(map(int, sys.stdin.readline().split()))
result = 0

#무식하게 다 뒤져 본다

for i in range(M):
    for j in range(i+1, M):
        for k in range(j+1, M):
            if C[i]+C[j]+C[k] > N :
                continue
            else:
                result = max(result, C[i]+C[j]+C[k]) #둘중 큰값
print(result)


import sys
sys.stdin = open("input.txt", "r") # 제거
input = sys.stdin.readline

n = int(input())
arr = [0] + [list(map(int, input().split())) for _ in range(n)]
result = 0
def dfs(s, exp):
    global result
    t = s + arr[s][0]
    if t > n + 1:
        result = max(result, exp)
        return
    if t == n + 1:
        result = max(result, exp + arr[s][1])
        return
    for i in range(t, n + 1):
        dfs(i, exp + arr[s][1])
for i in range(1, n + 1):
    dfs(i, 0)
print(result)


import keras
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense
from keras.preprocessing.image import ImageDataGenerator

# Define the paths to your image repositories
train_path = 'files/train'
validation_path = 'files/validation'
test_path = 'files/test'

# Define the image size and number of classes
img_size = (512, 512)
img_shape = (*img_size, 3)
num_classes = 2

# Create an ImageDataGenerator for data loading and preprocessing
datagen = ImageDataGenerator(rescale=1./255)

# Load and preprocess the training data
train_generator = datagen.flow_from_directory(
    train_path,
    target_size=img_size,
    batch_size=32,
    class_mode='categorical',
    shuffle=True
)

# Load and preprocess the validation data
validation_generator = datagen.flow_from_directory(
    validation_path,
    target_size=img_size,
    batch_size=32,
    class_mode='categorical',
    shuffle=True
)

# Load and preprocess the testing data
test_generator = datagen.flow_from_directory(
    test_path,
    target_size=img_size,
    batch_size=32,
    class_mode='categorical',
    shuffle=True
)

# Define the model
model = Sequential()
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=img_shape))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

# Flatten the output from previous layers
model.add(Flatten())

# Add a fully connected layer
model.add(Dense(128, activation='relu'))

# Add the output layer with softmax activation for classification
model.add(Dense(num_classes, activation='softmax'))

# Compile the model
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy', keras.metrics.Recall()])

# Train the mode
model.fit(train_generator, epochs=1, validation_data=validation_generator)

# Evaluate the model
print(model.evaluate(test_generator))
loss, accuracy, sensitivity = model.evaluate(test_generator)
print('Test Loss:', loss)
print('Test Accuracy:', accuracy)
print('Test Sensivity:', sensitivity)


import os
from datetime import datetime

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from matplotlib.pyplot import figure
from sklearn.manifold import TSNE
from sklearn.neighbors import KNeighborsClassifier

from src.experiments import Adult
from src.utils.normalizer import Normalizer

sns.set()
sns.set_context("paper")

def create_meshgrid(points, h=0.1):
    """
    Create the mesh grid to plot in.

    :param h: The mesh size
    :param points: The dataset

    :return: The mesh grid to plot in
    """
    x_min, x_max = points[:, 0].min() - 0.1, points[:, 0].max() + 0.1
    y_min, y_max = points[:, 1].min() - 0.1, points[:, 1].max() + 0.1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))

    return xx, yy

def plot_decision_surface(running_instance, query_idx=None, y_pred=None, soft=True, title=""):
    """
    Plots the decision surface of the model together with the data points.

    :param running_instance: An object containing the information about the current experiment and arguments
    :param query_idx: The index of chosen least confident example
    :param y_pred: The predictions of the model
    :param soft: Whether to plot  kernel-like boundary
    :param title: The title of the plot

    """
    experiment = running_instance.experiment
    known_idx = running_instance.known_idx
    train_idx = running_instance.train_idx

    if experiment.X.shape[1] > 2:
        plot_decision_surface_tsne(running_instance, query_idx, y_pred, title)
    else:
        X_known, y_known = experiment.X[known_idx], experiment.y[known_idx]
        X_train, y_train = experiment.X[train_idx], experiment.y[train_idx]
        X_known_norm, X_train_norm = Normalizer(experiment.normalizer).normalize_known_train(X_known, X_train)
        X_known_train_norm = np.concatenate((X_known_norm, X_train_norm), axis=0)
        model = experiment.model

        figure(num=None, figsize=(10, 8), facecolor='w', edgecolor='k')
        # create a mesh to plot in
        h = 0.05  # step size in the mesh
        xx, yy = create_meshgrid(X_known_train_norm, h)

        if soft:
            if hasattr(model.sklearn_model, "decision_function"):
                Z = model.decision_function(np.c_[xx.ravel(), yy.ravel()])
            else:
                Z = model.predict(np.c_[xx.ravel(), yy.ravel()])
        else:
            Z = model.predict(np.c_[xx.ravel(), yy.ravel()])

        # Put the result into a color plot
        Z = Z.reshape(xx.shape)
        plt.contourf(xx, yy, Z, cmap=plt.cm.RdBu_r, alpha=0.8)

        if y_pred is not None:
            wrong_points = X_train_norm[np.where(y_pred != y_train)]
            plt.scatter(X_train_norm[:, 0], X_train_norm[:, 1], c=y_pred, cmap=plt.cm.RdBu_r, s=45)
            plt.scatter(wrong_points[:, 0], wrong_points[:, 1], s=120, cmap=plt.cm.RdBu_r, facecolors='none',
                        edgecolors='green', linewidths=2)
        else:
            plt.scatter(X_train_norm[:, 0], X_train_norm[:, 1], c=y_train, cmap=plt.cm.RdBu_r, s=45)
            plt.scatter(X_known_norm[:, 0], X_known_norm[:, 1], c=y_known, cmap=plt.cm.RdBu_r, s=45,
                    edgecolors="yellow", linewidths=2)

        if query_idx is not None:
            least_conf = experiment.X[query_idx]
            idx_array = np.where((X_train[:, 0] == least_conf[0]) & (X_train[:, 1] == least_conf[1]))[0]
            if len(idx_array):
                idx_in_train = idx_array[0]
                least_conf_norm = X_train_norm[idx_in_train]
                plt.scatter(least_conf_norm[0], least_conf_norm[1], marker='x', s=400, linewidths=5, color='yellow')

        plt.xlim(X_known_train_norm[:, 0].min() - 0.1, X_known_train_norm[:, 0].max() + 0.1)
        plt.ylim(X_known_train_norm[:, 1].min() - 0.1, X_known_train_norm[:, 1].max() + 0.1)

        save_plot(plt, experiment.path, title, title, False)

def plot_decision_surface_tsne(running_instance, query_idx, y_pred, title):
    exp = running_instance.experiment
    X_known, y_known, X_train, y_train, _, _ = running_instance.get_all_data()
    X_known_norm, X_train_norm = Normalizer(exp.normalizer).normalize_known_train(X_known, X_train)

    X_known_train_norm = np.concatenate((X_known_norm, X_train_norm), axis=0)

    X_initial = Normalizer(exp.normalizer).normalize(exp.X)
    X_embedded = get_tsne_embedding(X_initial)
    X_known_embedded = X_embedded[running_instance.known_idx]
    X_train_embedded = X_embedded[running_instance.train_idx]
    X_known_train_embedded = np.concatenate((X_known_embedded, X_train_embedded), axis=0)

    y_predicted = running_instance.predict(X_known_train_norm)

    h = 10
    x_min, x_max = X_known_train_embedded[:, 0].min() - 1, X_known_train_embedded[:, 0].max() + 1
    y_min, y_max = X_known_train_embedded[:, 1].min() - 1, X_known_train_embedded[:, 1].max() + 1
    xx, yy = np.meshgrid(np.linspace(x_min, x_max, h), np.linspace(y_min, y_max, h))
    # approximate Voronoi tesselation on resolution x resolution grid using 1-NN
    background_model = KNeighborsClassifier(n_neighbors=1).fit(X_known_train_embedded, y_predicted)
    voronoiBackground = background_model.predict(np.c_[xx.ravel(), yy.ravel()])
    voronoiBackground = voronoiBackground.reshape((h, h))

    plt.figure(num=None, figsize=(10, 8), facecolor='w', edgecolor='k')
    plt.contourf(xx, yy, voronoiBackground, cmap=plt.cm.RdBu_r, alpha=0.8)

    if y_pred is not None:
        plt.scatter(X_train_embedded[:, 0], X_train_embedded[:, 1], c=y_pred, cmap=plt.cm.RdBu_r, s=45)
    else:
        plt.scatter(X_train_embedded[:, 0], X_train_embedded[:, 1], c=y_train, cmap=plt.cm.RdBu_r, s=45)
        plt.scatter(X_known_embedded[:, 0], X_known_embedded[:, 1], c=y_known, cmap=plt.cm.RdBu_r, s=45,
                    edgecolors="yellow", linewidths=2)

    if query_idx is not None:
        least_conf = running_instance.get_from_indexes(exp.X, query_idx)
        if isinstance(exp, Adult):
            X_train = X_train.to_numpy()
        idx_array = np.where((X_train[:, 0] == least_conf[0]) & (X_train[:, 1] == least_conf[1]))[0]
        if len(idx_array):
            idx_in_train = idx_array[0]
            least_conf_norm = running_instance.get_from_indexes(X_train_embedded, idx_in_train)
            plt.scatter(least_conf_norm[0], least_conf_norm[1], marker='x', s=400, linewidths=5, color='yellow')

    save_plot(plt, exp.path, title, title, False)

def plot_kmedoids(points, kmedoids_instance, other_points, path):
    # Plot the decision boundary
    plt.figure(num=None, figsize=(10, 8), facecolor='w', edgecolor='k')
    plt.xlim(points[:, 0].min() - 0.1, points[:, 0].max() + 0.1)
    plt.ylim(points[:, 1].min() - 0.1, points[:, 1].max() + 0.1)
    xx, yy = create_meshgrid(points)

    # Obtain labels for each point in mesh.
    Z = kmedoids_instance.predict((np.c_[xx.ravel(), yy.ravel()]))
    Z = Z.reshape(xx.shape)

    plt.imshow(Z, interpolation='nearest',
               extent=(xx.min(), xx.max(), yy.min(), yy.max()),
               cmap=plt.cm.Paired,
               aspect='auto', origin='lower')

    if other_points is not None:
        plt.scatter(other_points.iloc[:, 0], other_points.iloc[:, 1], c=other_points.predictions, marker='o', s=30)
    else:
        plt.scatter(points[:, 0], points[:, 1], c=points[:, 2], marker='o', s=30)

    # Plot the centroids
    centroids_idx = kmedoids_instance.get_medoids()
    centroids = points[centroids_idx]
    plt.scatter(centroids[:, 0], centroids[:, 1],
                marker='x', s=169, linewidths=3,
                color='red', zorder=10)

    save_plot(plt, path, "K-Medoids", "K-Medoids")

def plot_rules(clf, X, y, title, path, rules_f1):

    if X.shape[1] > 2:
        plot_rules_tsne(X, y, title, path)
    else:
        figure(num=None, figsize=(10, 8), facecolor='w', edgecolor='k')
        xx, yy = create_meshgrid(X, 0.005)
        if hasattr(clf, "decision_function"):
            Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
        else:
            Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]
        Z = Z.reshape(xx.shape)
        plt.contourf(xx, yy, Z, cmap=plt.cm.RdBu_r, alpha=0.8)
        plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.RdBu_r, s=45)

        save_plot(plt, path, " rules " + str(title), "F1 rules wrt svm: {}".format(rules_f1), False)

def plot_rules_tsne(X, y, title, path):

    X_embedded = get_tsne_embedding(X)
    h = 10
    x_min, x_max = X_embedded[:, 0].min() - 1, X_embedded[:, 0].max() + 1
    y_min, y_max = X_embedded[:, 1].min() - 1, X_embedded[:, 1].max() + 1
    xx, yy = np.meshgrid(np.linspace(x_min, x_max, h), np.linspace(y_min, y_max, h))
    # approximate Voronoi tesselation on resolution x resolution grid using 1-NN
    background_model = KNeighborsClassifier(n_neighbors=1).fit(X_embedded, y)
    voronoiBackground = background_model.predict(np.c_[xx.ravel(), yy.ravel()])
    voronoiBackground = voronoiBackground.reshape((h, h))

    plt.figure(num=None, figsize=(10, 8), facecolor='w', edgecolor='k')
    plt.contourf(xx, yy, voronoiBackground, cmap=plt.cm.RdBu_r, alpha=0.8)
    plt.scatter(X_embedded[:, 0], X_embedded[:, 1], c=y, cmap=plt.cm.RdBu_r, s=45)

    save_plot(plt, path, title, " rules", False)

def plot_initial_points(known_idx, test_idx, experiment):

    X_initial = Normalizer(experiment.normalizer).normalize(experiment.X)
    if experiment.X.shape[1] > 2:
        X_initial = get_tsne_embedding(X_initial)

    plot_points(X_initial, experiment.y, "Initial points", experiment.path)
    plot_points(X_initial[known_idx], experiment.y[known_idx], "Known points", experiment.path)
    plot_points(X_initial[test_idx], experiment.y[test_idx], "Test points", experiment.path)

def plot_points(X, y, title="", path=None):
    figure(num=None, figsize=(10, 8), facecolor='w', edgecolor='k')
    plt.scatter(X[:, 0], X[:, 1], c=y, marker='o', s=45, cmap=plt.cm.coolwarm)

    save_plot(plt, path, title, title)

def get_tsne_embedding(X):
    return TSNE(n_components=2, n_iter=300, random_state=0).fit_transform(X)

def save_plot(plt, path, img_name, plot_title=None, use_grid=True, use_date=True):
    img_name = "{}.pdf".format(img_name)
    plt.grid(use_grid)
    if plot_title is not None:
        plt.title(plot_title)
    if path:
        try:
            if use_date:
                img_name = "{}-{}.pdf".format(datetime.now().strftime('%Y-%m-%d_%H-%M-%S.%f'), img_name)
            plt.savefig(os.path.join(path, img_name), bbox_inches='tight', format='pdf')
        except ValueError:
            print("Something went wrong while saving image: ", img_name)
    else:
        plt.show()
    plt.close()


import matplotlib
import pdb
#matplotlib.use('Agg')

import matplotlib.pyplot as plt

def main():
    pass

def getVariables(lineOfText):
    variables = []
    variable = str()
    whiteCount = 0
    #print(lineOfText)
    isVariable = True
    insideParenthasis = False
    for i in lineOfText:
        #print (i)
        if i == ' ' and not insideParenthasis:
            if whiteCount == 0:
                variables.append(variable)
                variable = str('')
            isVariable = False
            whiteCount += 1
            if whiteCount == 2:
                isVariable = True
        elif i == '(':
            insideParenthasis = True
        elif i == ')':
            insideParenthasis = False
            whiteCount -= 1
        if isVariable == True and i != ' ' and i != ')' and not insideParenthasis:
            whiteCount = 0
            variable = str(variable) + str(i)
            #print(variable)
    return variables

def  convertToNum(num):
    power10 = len(num) - 1
    total = 0
    for i in num:
        total += i * (10 ** power10)
        power10 -= 1
    return total

def  getNums(lineOfText):
    nums = []
    number = []
    num = []
    lineOfText.seek(0)
    for i, line in enumerate(lineOfText):
        if i > 1:
            #print(line[0])
            if line[0] == "S":
                number = []
                for i in line:
                    if i.isdigit():
                        num.append(int(i))
                    #print(num)
                    if i == ',' or i == ']' and num:
                            number.append(convertToNum(num))
                            num = []
            if number:
                nums.append(number)
                number = []
        #print(nums)
    return nums

def getVariInitalNums(readFile):
    readFile.seek(0)
    varNames = []
    varVals = []
    for i, line in enumerate(readFile):
        if i == 1:
            varNames = getVariables(line)
            #print(varNames)
    #print("THIS IS A TEST STATMENT")
    varVals = getNums(readFile)
    #print(varVals)
    return(varNames, varVals)

def collate(data):
    dataVals = data[1]
    reOrderedData = []
    d = []
    print(dataVals)
    print(len(dataVals[0]))
    print(len(dataVals))

    i = 0
    j = 0
    print(len(dataVals))
    while j < len(dataVals[i]):
        while i < len(dataVals):
            d.append(dataVals[i][j])
            i += 1
        reOrderedData.append(d)
        d = []
        i = 0
        j += 1
    print(reOrderedData)
    return reOrderedData

def setLineMarker(lMarker):
    lm = 'nil'
    if (lMarker == 0):
        lm = '.'
    elif (lMarker == 1):
        lm = ','
    elif (lMarker == 2):
        lm = 'o'
    elif (lMarker == 3):
        lm = 'v'
    elif (lMarker == 4):
        lm = '^'
    elif (lMarker == 5):
        lm = '<'
    elif (lMarker == 6):
        lm = '>'
    elif (lMarker == 7):
        lm = '1'
    elif (lMarker == 8):
        lm = '2'
    elif (lMarker == 9):
        lm = '3'
    elif (lMarker == 10):
        lm = '4'
    elif (lMarker == 11):
        lm = 's'
    elif (lMarker == 12):
        lm = 'p'
    elif (lMarker == 13):
        lm = '*'
    elif (lMarker == 14):
        lm = 'h'
    elif (lMarker == 15):
        lm = 'H'
    elif (lMarker == 16):
        lm = '+'
    elif (lMarker == 17):
        lm = 'x'
    elif (lMarker == 18):
        lm = 'D'
    elif (lMarker == 19):
        lm = 'd'
    elif (lMarker == 20):
        lm = '|'
    elif (lMarker == 21):
        lm = '_'
    #print(lMarker)
    #print(lm)
    #pdb.set_trace()
    return lm

def setLineType(lType):
    lt = "nil"
    if (lType == 0):
        lt = '-'
    elif (lType == 1):
        lt = '--'
    elif (lType == 2):
        lt = '-.'
    return lt

def setLineColor(lColor):
    lc = 'nil'
    if (lColor == 0):
        lc = 'b'
    elif (lColor == 1):
        lc = 'g'
    elif (lColor == 2):
        lc = 'r'
    elif (lColor == 3):
        lc = 'c'
    elif (lColor == 4):
        lc = 'm'
    elif (lColor == 5):
        lc = 'y'
    elif (lColor == 6):
        lc = 'k'
    return lc

def  incLineMarker(lm):
    lm += 1
    if lm > 21:
        lm = 0
    return lm

def  incLineType(lt):
    lt += 1
    if lt > 2:
        lt = 0
    return lt

def  incLineColor(lc):
    lc += 1
    if lc > 6:
        lc = 0
    return lc

if __name__ == '__main__':
    main()
#  Comment the following lines to test ALEAE-STATS-GRABBER by itself
import fileinput
for line in fileinput.input():
    fileName = line
fileName = fileName.replace("\n", "")
# END COMMENTED SECTION FOR TESTING

#fileName = "demo" # USED FOR TESTING ALEAE-STATS-GRABBER BY ITSELF

directory = "output/"
DirectoryFile = directory + fileName

outPutFile = open(DirectoryFile + ".outpt", mode='r')
for line in outPutFile:
    #print(line)
    i = 2
data = getVariInitalNums(outPutFile)
#print(data)
#pdb.set_trace()
reData = collate(data)
#print(reData)
#pdb.set_trace()

i = 0  # counter to cycle through the data list
lineMarker = 0  # for cycling through matplotlib line markers
lineType = 0  # for cycling through matplotlib line types
lineColor = 0  # for cycling through matplotlib line colors

while i < len(reData):
    #print(('data', len(data[0])))
    #print(('reData', len(reData)))
    #print(('i', i))
    #print((len(reData)))
    #print((len(data[0][i])))

    lMarker = setLineMarker(lineMarker)
    lType = setLineType(lineType)
    lColor = setLineColor(lineColor)

    #  multi line parenthasis: '(plt.plot...)'
    (plt.plot(
        reData[i],
        marker=lMarker,
        linestyle=lType,
        color=lColor,
        label=data[0][i],
        linewidth=2
        ))

    i += 1
    lineMarker = incLineMarker(lineMarker)
    lineType = incLineType(lineType)
    lineColor = incLineColor(lineColor)

i = 0
print((data[0]))
print(data[len(data)-1][len(data[len(data)-1])-1])
while i < (len(data[0])-1):
    print(data[0][i])
    print(data[len(data)-1][len(data[len(data)-1])-1][i])
    i += 1

plt.legend()
fig = plt.gcf()
plt.show()
fig.savefig(DirectoryFile)
i = 0


def chartSwitcher(chart):
    """
    Choose which billboard chart to search, when given argument in command line

    chartName taken from billboard.com urls
    """

    if chart == "hot100":
        chartName = "hot-100"

    elif chart == "pop":
        chartName = "pop-songs"

    elif chart == "latin":
        chartName = "latin-songs"

    elif chart == "hiphop":
        chartName = "r-b-hip-hop-songs"

    elif chart == "edm":
        chartName = "dance-electronic-songs"

    elif chart == "alternative":
        chartName = "alternative-songs"

    elif chart == "rock":
        chartName = "rock-songs"

    elif chart == "country":
        chartName = "country-songs"

    else:
        print(
            "No genre specified. Try again with [hot100/pop/rock/latin/hiphop/alternative/edm/country].")

    return chartName


from dateutil.parser import parse
from ods.kktix_ticket_orders.udfs import kktix_api, klaviyo_loader

def main(**context):
    """
    Extract user info from kktix api and load to mailer
    """
    schedule_interval = context["dag"].schedule_interval
    # If we change the schedule_interval, we need to update the logic in condition_filter_callback
    assert schedule_interval == "0 * * * *"  # nosec
    ts_datetime_obj = parse(context["ts"])
    year = ts_datetime_obj.year
    timestamp = ts_datetime_obj.timestamp()
    event_raw_data_array = kktix_api._extract(year=year, timestamp=timestamp,)
    # load name and email to mailer before data has been hashed
    klaviyo_loader.load(event_raw_data_array)
    print(f"Batch load {len(event_raw_data_array)} data to downstream task")


import typer

def main(delete: bool = typer.Option(False, help="Delete found files"), extension: str = typer.Argument("txt", help="File extension")):
    """
    Display files find by extension
    """

    print(delete)

    typer.echo(f"Recherche de fichiers avec l'extension { extension }.")

    ext = typer.prompt("Quelle extension souhaitez-vous rechercher ?")
    print(ext)
    if delete:
        do_delete = typer.confirm("Voulez-vous vraiment supprimer les fichiers trouvés ?")
        # do_delete = typer.confirm("Voulez-vous vraiment supprimer les fichiers trouvés ?", abort=True)
        if not do_delete:
            typer.echo("Annulation de l'opération.")
            raise typer.Abort()
        typer.echo("Suppression des fichiers trouvés.")

typer.run(main)


import json

data = '''
[
    { "id" : "001",
    "x" : "2",
    "name" : "Chuck"
    },
    { "id" : "009",
    "x" : "7",
    "name" : "Chuck"
    }
]'''

# json.loads() processes dictionaries in java format
# using len() will count total dictionaries
info = json.loads(data)
print('User count:' , len(info))

for item in info:
    print('Name:' , item['name'])
    print('ID:' , item['id'])
    print('Attribute:' , item['x'])


import socket
import gevent

from gevent import monkey

monkey.patch_all(thread=False)  # 如果使用vscode启动调试器，则必须加上thread=False,否则报错
# monkey.patch_all()

hosts = ['www.baidu.com', 'www.taobao.com',
         'www.jd.com', 'www.bilibili.com']
jobs = [gevent.spawn(socket.gethostbyname, host) for host in hosts]
gevent.joinall(jobs, timeout=5)
for job in zip(hosts, jobs):
    print(job[0], '->', job[1].value)


import csv
import multiprocessing
import sys
import warnings
import output_change

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from deap import base
from deap import creator
from deap import gp
from sklearn.cluster import KMeans
from sklearn.metrics import adjusted_rand_score
from sklearn.metrics import pairwise_distances
from sklearn.metrics import silhouette_score

import ea_simple_elitism
import multi_tree as mt
import vector_tree as vt
from ParallelToolbox import ParallelToolbox
from read_data import read_data
from selection import *

from get_features import FeatureMaker
import random
import copy
from scipy.spatial import distance
from clustering import cluster

POP_SIZE = 1024
NGEN = 1
CXPB = 0.8
MUTPB = 0.2
ELITISM = 10
MAX_HEIGHT = 8
SEED = 100
REP = mt  # individual representation {mt (multi-tree) or vt (vector-tree)}
N_TREES = 1
DATA_DIR = '.'  # "/home/schofifinn/PycharmProjects/SSResearch/data"
METRIC = 'intra'
MAXIMISE = METRIC != 'intra'

def make_feature_file(fm, input_data_name):
    feature_file = open(input_data_name + ".data", 'w')
    feature_file.write("classLast,{0},{1},space\n".format(fm.term_num(), fm.class_num()))
    for test_case_info in fm.case_info_list:
        str_vector = map(str, test_case_info.get_vector())
        feature_file.write("{0} {1}\n".format(" ".join(str_vector), test_case_info.get_classification()))
    feature_file.close()

def connectedness(cluster):
    print(cluster)

def evaluate(individual, toolbox, data, k, metric, maximise, distance_vector=None, labels_true=None):
    """
    Evaluates an individuals fitness. The fitness is the clustering performance on the data
    using the metric.

    :param individual: the individual to be evaluated
    :param toolbox: the evolutionary toolbox
    :param data: the data to be used to evaluate the individual
    :param k: the number of clusters for k-means
    :param metric: the metric to be used to evaluate clustering performance
    :param distance_vector: a pre-computed distance vector, required for silhouette-pre metric
    :param labels_true: the ground truth cluster labels, required for ari metric
    :return: the fitness of the individual
    """
    X = REP.process_data(individual, toolbox, data, MAX_HEIGHT)
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        # print("Kmenas SEED",SEED)
        kmeans = KMeans(n_clusters=k, random_state=SEED).fit(X)
    labels = kmeans.labels_
    # print(labels)
    # print("\n")
    # print("\n")
    # print("\n")
    # individuals that find single cluster are unfit
    nlabels = len(set(labels))
    if nlabels == 1:
        return [np.inf] if maximise else [-1]
    # uses precomputed distances of original data to avoid trending towards single dimension to
    # minimise silhouette.
    if metric == 'silhouette_pre':
        if distance_vector is None:
            raise Exception("Must provide distance vector for silhouette-pre metric.")
        return [silhouette_score(distance_vector, labels, metric='precomputed')]
    elif metric == 'silhouette':
        return [silhouette_score(X, labels, metric='euclidean')]
    elif metric == 'ari':
        if labels_true is None:
            raise Exception("Must provide ground truth labels for ARI")
        return [adjusted_rand_score(labels_true, labels)]
    elif metric == 'intra':
        sum = 0.
        for l in np.unique(labels):
            pts = data[labels == l]
            avg = pts.mean(axis=0)
            diff = np.linalg.norm(pts - avg)
            sum += diff
        return [sum]
    elif metric == 'connectedness':
        sum = 0.
        for l in np.unique(labels):
            idxs = labels == l
            dists = distance_vector[idxs][:, idxs]
            # not a single point in a cluster by itself
            if dists.shape[0] > 1:
                dists = np.divide(1., dists, out=np.zeros_like(dists), where=dists != 0)
                dists = np.minimum(dists, 10)
                np.fill_diagonal(dists, 0)
                sum += dists[dists > 0].sum() / dists.shape[0]  # .mean()
        return [sum / nlabels]
    else:
        raise Exception("invalid metric: {}".format(metric))

def write_ind_to_file(ind, run_num, results, datafile, data, toolbox):
    """
    Writes the attributes of an individual to file.

    :param run_num: the number of the current run
    :param ind: the individual
    :param results: a dictionary of results, titles to values
    """
    line_list = []
    # add constructed features to lines
    if REP is mt:
        for cf in [str(tree) for tree in ind]:
            line_list.append(cf + "\n")
    elif REP is not vt:
        raise Exception("Invalid representation")
    line_list.append("\n")
    fl = open("%d_ind.txt" % run_num, 'w')
    if REP is mt:
        fl.writelines(line_list)
    else:
        for i in ind:
            fl.write(str(i))
            fl.write("\n")
    fl.close()
    the_rep = 0
    if REP is vt:
        the_rep = 1
    output_change.change_output(str(run_num) + "_ind.txt", N_TREES, datafile + ".data", run_num, the_rep)

def init_toolbox(toolbox, pset):
    """
    Initialises the toolbox with operators.
    :param toolbox: the toolbox to initialise
    :param pset: primitive set for evolution
    """
    REP.init_toolbox(toolbox, pset, N_TREES)
    toolbox.register("select", selElitistAndTournament, tournsize=7, elitism=ELITISM)

def init_stats():
    """
    Initialises a MultiStatistics object to capture data.
    :return: the MultiStatistics object
    """
    fitness_stats = tools.Statistics(lambda ind: ind.fitness.values)
    stats = tools.MultiStatistics(fitness=fitness_stats)
    stats.register("avg", np.mean)
    stats.register("std", np.std)
    stats.register("min", np.min)
    stats.register("max", np.max)
    return stats

def final_evaluation(best, data, labels, num_classes, toolbox, print_output=True):
    """
    Performs a final performance evaluation on an individual.

    :param best: the individual to evaluate
    :param data: the dataset associated with the individual
    :param labels: the ground-truth labels of the dataset
    :param num_classes: the number of classes of the dataset
    :param toolbox: the evolutionary toolbox
    :param print_output: whether or not to print output
    :return: a dictionary of results, titles to values
    """
    return

rd = {}

def eval_wrapper(*args, **kwargs):
    return evaluate(*args, **kwargs, toolbox=rd['toolbox'], data=rd['data'], k=rd['num_classes'], metric=rd['metric'],
                    maximise=rd['maximise'], distance_vector=rd['distance_vector'])

# copies data over from parent process
def init_data(rundata):
    global rd
    rd = rundata

def maingptree(datafile, run_num):
    print("maing tree NGEN", NGEN)
    print("maing tree SEED", SEED)
    random.seed(SEED)
    all_data = read_data("%s/%s.data" % (DATA_DIR, datafile))
    rd['data'] = all_data["data"]
    rd['labels'] = all_data["labels"]
    rd['num_classes'] = len(set(rd['labels']))
    print("%d classes found." % rd['num_classes'])
    rd['distance_vector'] = pairwise_distances(rd['data'])
    rd['num_instances'] = rd['data'].shape[0]
    rd['num_features'] = rd['data'].shape[1]
    rd['metric'] = METRIC
    rd['maximise'] = MAXIMISE
    pset = gp.PrimitiveSet("MAIN", rd['num_features'], prefix="f")
    pset.context["array"] = np.array
    REP.init_primitives(pset)
    weights = (1.0,) if MAXIMISE else (-1.,)
    creator.create("FitnessMax", base.Fitness, weights=weights)
    # set up toolbox
    toolbox = ParallelToolbox()  # base.Toolbox()
    init_toolbox(toolbox, pset)
    toolbox.register("evaluate", eval_wrapper)
    rd['toolbox'] = toolbox
    pop = toolbox.population(n=POP_SIZE)
    hof = tools.HallOfFame(1)
    if REP is vt:
        hof = tools.HallOfFame(N_TREES)
    stats = init_stats()
    with multiprocessing.Pool(initializer=init_data, initargs=[rd]) as pool:
        toolbox.map = pool.map
        pop, logbook = ea_simple_elitism.eaSimple(pop, toolbox, CXPB, MUTPB, ELITISM, NGEN, stats,
                                                  halloffame=hof, verbose=True)
    if REP is not vt:
        best = hof[0]
    else:
        best = []
        for i in range(N_TREES):
            if i < len(hof):
                best.append(hof[i])
    res = final_evaluation(best, rd['data'], rd['labels'], rd['num_classes'], toolbox)
    write_ind_to_file(best, run_num, res, datafile, rd['data'], toolbox)
    return pop, stats, hof

if __name__ == "__main__":
    feature_maker = FeatureMaker()
    make_feature_file(feature_maker, "features")
    print("Getting feature information done")

    while True:
        vector_name = input(
            "Please input which type of representation you will use, vt(vetor tree) or mt(multi tree): ")
        if vector_name == "vt":
            REP = vt
            break
        elif vector_name == "mt":
            REP = mt
            break

    NGEN = int(input("Please input the length of generations: "))
    N_TREES = int(input("Please input the number of features: "))
    SEED = int(input("Please input seed number: "))
    maingptree("features", SEED)
    print("GP process done")

    k = int(input("Please input the number of clusters, k: "))
    cluster("result_" + str(SEED) + ".txt", k, method='random', num_iter=10)


import re
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader
from torch.autograd import Function
from tqdm import tqdm

device = torch.device('cuda:3' if torch.cuda.is_available() else 'cpu')
batch_size = 32

# Define GRL for common feature extraction
class GradientReversalFunction(Function):
    @staticmethod
    def forward(ctx, x):
        ctx.lambda_ = 1
        return x.clone()

    @staticmethod
    def backward(ctx, grads):
        lambda_ = 1
        lambda_ = grads.new_tensor(lambda_)
        dx = -lambda_ * grads
        return dx, None

# Define model
class SER(nn.Module):
    def __init__(self):
        super(SER, self).__init__()
        # Num of CNN filter, CNN filter size 5x100
        self.filters_num = 100
        self.kernel_size = 5
        # Word embedding dimension
        self.word_dim = 100
        # Loss for siamese encoder
        self.dist = nn.MSELoss()

        self.s_user_feature_extractor = nn.Sequential(
            nn.Conv2d(1, self.filters_num, (self.kernel_size, self.word_dim)),
            nn.BatchNorm2d(self.filters_num),
            nn.Sigmoid(),
            nn.MaxPool2d((496, 1)),
            nn.Dropout(),
        )

        self.s_item_feature_extractor = nn.Sequential(
            nn.Conv2d(1, self.filters_num, (self.kernel_size, self.word_dim)),
            nn.BatchNorm2d(self.filters_num),
            nn.Sigmoid(),
            nn.MaxPool2d((496, 1)),
            nn.Dropout(),
        )

        self.t_user_feature_extractor = nn.Sequential(
            nn.Conv2d(1, self.filters_num, (self.kernel_size, self.word_dim)),
            nn.BatchNorm2d(self.filters_num),
            nn.Sigmoid(),
            nn.MaxPool2d((496, 1)),
            nn.Dropout(),
        )

        self.t_item_feature_extractor = nn.Sequential(
            nn.Conv2d(1, self.filters_num, (self.kernel_size, self.word_dim)),
            nn.BatchNorm2d(self.filters_num),
            nn.Sigmoid(),
            nn.MaxPool2d((496, 1)),
            nn.Dropout(),
        )

        self.c_user_feature_extractor = nn.Sequential(
            nn.Conv2d(1, self.filters_num, (self.kernel_size, self.word_dim)),
            nn.BatchNorm2d(self.filters_num),
            nn.Sigmoid(),
            nn.MaxPool2d((496, 1)),
            nn.Dropout(),
        )

        self.c_item_feature_extractor = nn.Sequential(
            nn.Conv2d(1, self.filters_num, (self.kernel_size, self.word_dim)),
            nn.BatchNorm2d(self.filters_num),
            nn.Sigmoid(),
            nn.MaxPool2d((496, 1)),
            nn.Dropout(),
        )

        self.discriminator = nn.Sequential(
            nn.Linear(200, 64),
            nn.Sigmoid(),
            nn.Linear(64, 1),
        )

        self.s_encoder = nn.Sequential(
            nn.Linear(200, 200)
        )

        self.s_classifier = nn.Sequential(
            nn.Linear(200, 32),
            nn.Sigmoid(),
            nn.Linear(32, 1),
        )

        self.t_encoder = nn.Sequential(
            nn.Linear(200, 200)
        )

        self.t_classifier = nn.Sequential(
            nn.Linear(200, 32),
            nn.Sigmoid(),
            nn.Linear(32, 1),
        )

        self.reset_para()

    def reset_para(self):
        for cnn in [self.s_user_feature_extractor[0], self.s_item_feature_extractor[0]]:
            nn.init.xavier_normal_(cnn.weight)
            nn.init.constant_(cnn.bias, 0.1)

        for cnn in [self.t_user_feature_extractor[0], self.t_item_feature_extractor[0]]:
            nn.init.xavier_normal_(cnn.weight)
            nn.init.constant_(cnn.bias, 0.1)

        for cnn in [self.c_user_feature_extractor[0], self.c_item_feature_extractor[0]]:
            nn.init.xavier_normal_(cnn.weight)
            nn.init.constant_(cnn.bias, 0.1)

        for fc in [self.s_classifier[0]]:
            nn.init.uniform_(fc.weight, -0.1, 0.1)
            nn.init.constant_(fc.bias, 0.1)

        for fc in [self.t_classifier[0]]:
            nn.init.uniform_(fc.weight, -0.1, 0.1)
            nn.init.constant_(fc.bias, 0.1)

    def forward(self, user, item, ans, label):
        # Source individual review FE
        s_u_ans_fea = self.s_user_feature_extractor(ans).squeeze(2).squeeze(2)
        c_u_ans_fea = self.c_user_feature_extractor(ans).squeeze(2).squeeze(2)
        s_u_ans_fea = (s_u_ans_fea + c_u_ans_fea) / 2

        s_i_ans_fea = self.s_item_feature_extractor(ans).squeeze(2).squeeze(2)
        c_i_ans_fea = self.c_item_feature_extractor(ans).squeeze(2).squeeze(2)
        s_i_ans_fea = (s_i_ans_fea + c_i_ans_fea) / 2

        s_ans_fea = torch.cat((s_u_ans_fea, s_i_ans_fea), 1).squeeze(1)

        # Label of source individual review
        s_cls_out = self.s_classifier(s_ans_fea)

        # Output is [Source | Target] --> Masking target output for loss calculation
        masking = torch.cat([torch.ones(batch_size), torch.zeros(batch_size)]).view(
            batch_size * 2, -1).to(device)
        s_ans_out, s_label = torch.mul(
            s_cls_out, masking), torch.mul(label, masking)

        # Source aggregated reviews FE
        s_u_fea = self.s_user_feature_extractor(user).squeeze(2).squeeze(2)
        s_i_fea = self.s_item_feature_extractor(item).squeeze(2).squeeze(2)

        s_c_u_fea = self.c_user_feature_extractor(user).squeeze(2).squeeze(2)
        s_c_i_fea = self.c_item_feature_extractor(item).squeeze(2).squeeze(2)

        s_u_fea = (s_u_fea + s_c_u_fea) / 2
        s_i_fea = (s_i_fea + s_c_i_fea) / 2

        s_fea = torch.cat((s_u_fea, s_i_fea), 1).squeeze(1)

        # Passing through encoder for aggregated review embedding
        s_fea = self.s_encoder(s_fea)

        s_cls_out = self.s_classifier(s_fea)
        s_out = torch.mul(s_cls_out, masking)

        # Distance between individual review & aggregated review
        s_dist = self.dist(torch.mul(s_ans_fea, masking),
                           torch.mul(s_fea, masking))

        # Same for target domain
        t_u_ans_fea = self.t_user_feature_extractor(ans).squeeze(2).squeeze(2)
        c_u_ans_fea = self.c_user_feature_extractor(ans).squeeze(2).squeeze(2)
        t_u_ans_fea = (t_u_ans_fea + c_u_ans_fea) / 2

        t_i_ans_fea = self.t_item_feature_extractor(ans).squeeze(2).squeeze(2)
        c_i_ans_fea = self.c_item_feature_extractor(ans).squeeze(2).squeeze(2)
        t_i_ans_fea = (t_i_ans_fea + c_i_ans_fea) / 2

        t_ans_fea = torch.cat((t_u_ans_fea, t_i_ans_fea), 1).squeeze(1)

        t_cls_out = self.t_classifier(t_ans_fea)

        masking = torch.cat([torch.zeros(batch_size), torch.ones(batch_size)]).view(
            batch_size * 2, -1).to(device)
        t_ans_out, t_label = torch.mul(
            t_cls_out, masking), torch.mul(label, masking)

        # Target classification loss
        t_u_fea = self.t_user_feature_extractor(user).squeeze(2).squeeze(2)
        t_i_fea = self.t_item_feature_extractor(item).squeeze(2).squeeze(2)

        t_c_u_fea = self.c_user_feature_extractor(user).squeeze(2).squeeze(2)
        t_c_i_fea = self.c_item_feature_extractor(item).squeeze(2).squeeze(2)

        t_u_fea = (t_u_fea + t_c_u_fea) / 2
        t_i_fea = (t_i_fea + t_c_i_fea) / 2

        t_fea = torch.cat((t_u_fea, t_i_fea), 1).squeeze(1)

        t_fea = self.t_encoder(t_fea)

        t_cls_out = self.t_classifier(t_fea)
        t_out = torch.mul(t_cls_out, masking)

        t_dist = self.dist(torch.mul(t_ans_fea, masking),
                           torch.mul(t_fea, masking))

        # Discriminator label
        s_domain_specific = torch.zeros(batch_size).to(device)
        t_domain_specific = torch.ones(batch_size).to(device)

        # Common source discriminator loss
        s_c_d_fea = torch.cat((s_c_u_fea, s_c_i_fea), 1)
        s_c_d_fea = GradientReversalFunction.apply(s_c_d_fea)
        s_c_d_fea = self.discriminator(s_c_d_fea).squeeze(1)[0:batch_size]
        s_c_domain_loss = F.binary_cross_entropy_with_logits(
            s_c_d_fea, s_domain_specific)

        # Common target discriminator loss
        t_c_d_fea = torch.cat((t_c_u_fea, t_c_i_fea), 1)
        t_c_d_fea = GradientReversalFunction.apply(t_c_d_fea)
        t_c_d_fea = self.discriminator(t_c_d_fea).squeeze(1)[
            batch_size:batch_size * 2]
        t_c_domain_loss = F.binary_cross_entropy_with_logits(
            t_c_d_fea, t_domain_specific)

        domain_common_loss = (s_c_domain_loss + t_c_domain_loss) / 2

        # Source specific discriminator loss
        s_d_fea = torch.cat((s_u_fea, s_i_fea), 1)
        s_d_fea = self.discriminator(s_d_fea).squeeze(1)[0:batch_size]

        # Target specific discriminator loss
        t_d_fea = torch.cat((t_u_fea, t_i_fea), 1)
        t_d_fea = self.discriminator(t_d_fea).squeeze(1)[
            batch_size:batch_size * 2]

        s_domain_specific = torch.zeros(batch_size).to(device)
        s_domain_loss = F.binary_cross_entropy_with_logits(
            s_d_fea, s_domain_specific)
        t_domain_specific = torch.ones(batch_size).to(device)
        t_domain_loss = F.binary_cross_entropy_with_logits(
            t_d_fea, t_domain_specific)
        domain_specific_loss = (s_domain_loss + t_domain_loss) / 2

        return s_ans_out, s_out, s_label, s_dist, t_ans_out, t_out, t_label, t_dist, domain_common_loss, domain_specific_loss

# Clean strings for reviews
def clean_str(string):
    string = re.sub(r"[^A-Za-z0-9]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    string = re.sub(r"\s{2,}", " ", string)
    string = re.sub(r"sssss ", " ", string)

    return string.strip().lower()

# Review embedding layer
def pre_processing(s_data, s_dict, t_data, t_dict, w_embed, valid_idx):
    # Return embedded vector [user, item, rev_ans, rat]
    u_embed, i_embed, ans_embed, label = [], [], [], []
    limit = 500

    for idx in range(batch_size):
        u, i, rat = s_data[0][idx], s_data[1][idx], s_data[2][idx]

        u_rev, i_rev, ans_rev = [], [], []

        reviews = s_dict[u]
        for review in reviews:
            if review[0] != i:
                review = review[1].split(' ')
                for rev in review:
                    try:
                        rev = clean_str(rev)
                        rev = w_embed[rev]
                        u_rev.append(rev)
                        if len(u_rev) > limit:
                            break
                    except KeyError:
                        continue

        reviews = s_dict[i]
        for review in reviews:
            if review[0] != u:
                review = review[1].split(' ')
                for rev in review:
                    try:
                        rev = clean_str(rev)
                        rev = w_embed[rev]
                        i_rev.append(rev)
                        if len(i_rev) > limit:
                            break
                    except KeyError:
                        continue

        reviews = s_dict[u]
        for review in reviews:
            if review[0] == i:
                review = review[1].split(' ')
                for rev in review:
                    try:
                        rev = clean_str(rev)
                        rev = w_embed[rev]
                        ans_rev.append(rev)
                        if len(ans_rev) > limit:
                            break
                    except KeyError:
                        continue

        if len(u_rev) > limit:
            u_rev = u_rev[0:limit]
        else:
            lis = [0.0] * 100
            pend = limit - len(u_rev)
            for p in range(pend):
                u_rev.append(lis)

        if len(i_rev) > limit:
            i_rev = i_rev[0:limit]
        else:
            lis = [0.0] * 100
            pend = limit - len(i_rev)
            for p in range(pend):
                i_rev.append(lis)

        if len(ans_rev) > limit:
            ans_rev = ans_rev[0:limit]
        else:
            lis = [0.0] * 100
            pend = limit - len(ans_rev)
            for p in range(pend):
                ans_rev.append(lis)

        u_embed.append(u_rev)
        i_embed.append(i_rev)
        ans_embed.append(ans_rev)
        label.append([rat])

    if valid_idx:
        u_embed = torch.tensor(u_embed, requires_grad=True).view(
            batch_size, 1, 500, 100).to(device)
        i_embed = torch.tensor(i_embed, requires_grad=True).view(
            batch_size, 1, 500, 100).to(device)
        ans_embed = torch.tensor(ans_embed, requires_grad=True).view(
            batch_size, 1, 500, 100).to(device)
        label = torch.FloatTensor(label).to(device)

        return u_embed, i_embed, ans_embed, label

    for idx in range(batch_size):
        u, i, rat = t_data[0][idx], t_data[1][idx], t_data[2][idx]

        u_rev, i_rev, ans_rev = [], [], []

        reviews = t_dict[u]
        for review in reviews:
            if review[0] != i:
                review = review[1].split(' ')
                for rev in review:
                    try:
                        rev = clean_str(rev)
                        rev = w_embed[rev]
                        u_rev.append(rev)
                        if len(u_rev) > limit:
                            break
                    except KeyError:
                        continue

        reviews = t_dict[i]
        for review in reviews:
            if review[0] != u:
                review = review[1].split(' ')
                for rev in review:
                    try:
                        rev = clean_str(rev)
                        rev = w_embed[rev]
                        i_rev.append(rev)
                        if len(i_rev) > limit:
                            break
                    except KeyError:
                        continue

        reviews = t_dict[u]
        for review in reviews:
            if review[0] == i:
                review = review[1].split(' ')
                for rev in review:
                    try:
                        rev = clean_str(rev)
                        rev = w_embed[rev]
                        ans_rev.append(rev)
                        if len(ans_rev) > limit:
                            break
                    except KeyError:
                        continue

        if len(u_rev) > limit:
            u_rev = u_rev[0:limit]
        else:
            lis = [0.0] * 100
            pend = limit - len(u_rev)
            for p in range(pend):
                u_rev.append(lis)

        if len(i_rev) > limit:
            i_rev = i_rev[0:limit]
        else:
            lis = [0.0] * 100
            pend = limit - len(i_rev)
            for p in range(pend):
                i_rev.append(lis)

        if len(ans_rev) > limit:
            ans_rev = ans_rev[0:limit]
        else:
            lis = [0.0] * 100
            pend = limit - len(ans_rev)
            for p in range(pend):
                ans_rev.append(lis)

        u_embed.append(u_rev)
        i_embed.append(i_rev)
        ans_embed.append(ans_rev)
        label.append([rat])

    u_embed = torch.tensor(u_embed, requires_grad=True).view(
        batch_size * 2, 1, 500, 100).to(device)
    i_embed = torch.tensor(i_embed, requires_grad=True).view(
        batch_size * 2, 1, 500, 100).to(device)
    ans_embed = torch.tensor(ans_embed, requires_grad=True).view(
        batch_size * 2, 1, 500, 100).to(device)
    label = torch.FloatTensor(label).to(device)

    return u_embed, i_embed, ans_embed, label

# Validation & Inference
def valid(v_data, t_data, t_dict, w_embed, save, write_file):
    model = SER()
    model.load_state_dict(torch.load(save, map_location=device))
    model.to(device)
    model.eval()

    criterion = nn.MSELoss()

    t_user_feature_extractor = model.t_user_feature_extractor
    t_item_feature_extractor = model.t_item_feature_extractor
    t_encoder = model.t_encoder
    t_clf = model.t_classifier

    c_user_feature_extractor = model.c_user_feature_extractor
    c_item_feature_extractor = model.c_item_feature_extractor

    v_batch = DataLoader(v_data, batch_size=batch_size,
                         shuffle=True, num_workers=2)
    v_loss, idx = 0, 0

    for v_data in tqdm(v_batch, leave=False):
        if len(v_data[0]) != batch_size:
            continue
        u_embed, i_embed, ans_embed, label = pre_processing(
            v_data, t_dict, v_data, t_dict, w_embed, 1)

        with torch.no_grad():
            # Target rating encoder
            c_u_fea = c_user_feature_extractor(u_embed).squeeze(2).squeeze(2)
            c_i_fea = c_item_feature_extractor(i_embed).squeeze(2).squeeze(2)

            t_u_fea = t_user_feature_extractor(u_embed).squeeze(2).squeeze(2)
            t_i_fea = t_item_feature_extractor(i_embed).squeeze(2).squeeze(2)

            u_fea, i_fea = (c_u_fea + t_u_fea) / 2, (c_i_fea + t_i_fea) / 2

            t_fea = t_encoder(torch.cat((u_fea, i_fea), 1).squeeze(1))

            t_out = t_clf(t_fea)

            v_loss += criterion(t_out, label)
        idx += 1
    v_loss = v_loss / idx

    t_batch = DataLoader(t_data, batch_size=batch_size,
                         shuffle=True, num_workers=2)
    t_loss, idx = 0, 0

    for t_data in tqdm(t_batch, leave=False):
        if len(t_data[0]) != batch_size:
            continue
        u_embed, i_embed, ans_embed, label = pre_processing(
            t_data, t_dict, t_data, t_dict, w_embed, 1)

        with torch.no_grad():
            # Target rating encoder
            c_u_fea = c_user_feature_extractor(u_embed).squeeze(2).squeeze(2)
            c_i_fea = c_item_feature_extractor(i_embed).squeeze(2).squeeze(2)

            t_u_fea = t_user_feature_extractor(u_embed).squeeze(2).squeeze(2)
            t_i_fea = t_item_feature_extractor(i_embed).squeeze(2).squeeze(2)

            u_fea, i_fea = (c_u_fea + t_u_fea) / 2, (c_i_fea + t_i_fea) / 2

            t_fea = t_encoder(torch.cat((u_fea, i_fea), 1).squeeze(1))

            t_out = t_clf(t_fea)

            t_loss += criterion(t_out, label)
        idx += 1

    t_loss = t_loss / idx

    print('Loss: %.4f %.4f' % (v_loss, t_loss))

    w = open(write_file, 'a')
    w.write('%.6f %.6f\n' % (v_loss, t_loss))

# Training
def learning(s_data, s_dict, t_data, t_dict, w_embed, save, idx):
    # Model
    print('Start Training ... \n')
    enc_loss_ratio, domain_loss_ratio = 0.05, 0.1
    model = SER()
    # After 1 epoch, load trained parameters
    if idx == 1:
        model.load_state_dict(torch.load(save, map_location=device))
    model.to(device)
    model.train()

    criterion = nn.MSELoss()

    optim = torch.optim.Adam(model.parameters(), lr=1e-4)

    # Make batch
    batch_size = 32
    s_batch = DataLoader(s_data, batch_size=batch_size,
                         shuffle=True, num_workers=2)
    t_batch = DataLoader(t_data, batch_size=batch_size,
                         shuffle=True, num_workers=2)

    batch_data, zip_size = zip(s_batch, t_batch), min(
        len(s_batch), len(t_batch))

    for source_x, target_x in tqdm(batch_data, leave=False, total=zip_size):
        # Pre processing
        if len(source_x[0]) != batch_size or len(target_x[0]) != batch_size:
            continue

        # Get embedding of user and item reviews
        u_embed, i_embed, ans_embed, label = pre_processing(
            source_x, s_dict, target_x, t_dict, w_embed, 0)

        s_ans_out, s_out, s_label, s_dist, t_ans_out, t_out, t_label, t_dist, \
            c_domain_loss, domain_loss = model(
                u_embed, i_embed, ans_embed, label)

        # Loss
        s_ans_loss, s_loss = criterion(
            s_ans_out, s_label) * 2, criterion(s_out, s_label) * 2
        t_ans_loss, t_loss = criterion(
            t_ans_out, t_label) * 2, criterion(t_out, t_label) * 2

        # Train
        loss_func = (s_loss + t_loss + s_ans_loss + t_ans_loss) / 2 + \
                    (s_dist + t_dist) * enc_loss_ratio + \
            (c_domain_loss + domain_loss) * domain_loss_ratio

        optim.zero_grad()
        loss_func.backward()
        optim.step()

        torch.save(model.state_dict(), save)

        print('Prediction Loss / Encoder Loss / Domain Loss: %.2f %.2f %.2f %.2f %.2f %.2f' %
              (s_loss, t_loss, s_dist, t_dist, c_domain_loss, domain_loss))