import logging

import torch
import torch.nn
import torch.nn.functional as F
from torch.autograd import Variable

from utils.torch.rnn import BaseRNNCell, dynamic_rnn, dynamic_rnn_reversed

class Bottle(torch.nn.Module):
    def forward(self, input):
        if len(input.size()) <= 2:
            return super(Bottle, self).forward(input)
        size = input.size()[:2]
        out = super(Bottle, self).forward(input.view(size[0] * size[1], -1))
        return out.view(size[0], size[1], -1)

class Linear(Bottle, torch.nn.Linear):
    pass

class LSTMEncoder(torch.nn.Module):
    def __init__(self, input_size, hidden_size, bidirectional=False, return_sequence=False):
        super(LSTMEncoder, self).__init__()

        self.input_size = input_size
        self.hidden_size = hidden_size
        self.return_sequence = return_sequence
        self.bidirectional = bidirectional

        self.rnn = torch.nn.LSTM(input_size=input_size, hidden_size=hidden_size, bidirectional=bidirectional,
                                 batch_first=True)

    def zero_state(self, inputs):
        batch_size = inputs.size()[0]

        # The axes semantics are (num_layers, batch_size, hidden_dim)
        nb_layers = 1 if not self.bidirectional else 2
        state_shape = (nb_layers, batch_size, self.hidden_size)

        # will work on both GPU and CPU in contrast to just Variable(*state_shape)
        h0, c0 = (
            Variable(inputs.data.new(*state_shape).zero_()),
            Variable(inputs.data.new(*state_shape).zero_())
        )

        return h0, c0

    def forward(self, inputs, lengths=None):
        h0, c0 = self.zero_state(inputs)

        if lengths is not None:
            # sort by length
            lengths_sorted, inputs_sorted_idx = lengths.sort(descending=True)
            inputs_sorted = inputs[inputs_sorted_idx]

            # pack sequences
            packed = torch.nn.utils.rnn.pack_padded_sequence(inputs_sorted, list(lengths_sorted.data), batch_first=True)

            outputs, (h, c) = self.rnn(packed, (h0, c0))

            # unpack sequences
            outputs, _ = torch.nn.utils.rnn.pad_packed_sequence(outputs, batch_first=True)

            # un-sort
            _, inputs_unsorted_idx = inputs_sorted_idx.sort(descending=False)
            outputs = outputs[inputs_unsorted_idx]

            # concat in case of bidirectional, and just remove the first dim in case of unidirectional
            h = torch.cat([x for x in h], dim=-1)
            h = h[inputs_unsorted_idx]
        else:
            outputs, (h, c) = self.rnn(inputs, (h0, c0))

            # concat in case of bidirectional, and just remove the fisrt dim in case of unidirectional
            h = torch.cat(h, dim=-1)

        if self.return_sequence:
            return outputs
        else:
            return h

class InterSentenceAttention(torch.nn.Module):
    def forward(self, a, b):
        assert a.size()[2] == b.size()[2]
        assert a.size()[0] == b.size()[0]

        # performs batch mat mul
        attention = torch.matmul(
            a,
            b.transpose(1, 2),
        )

        return attention

def softmax(input, axis=1):
    """https://discuss.pytorch.org/t/why-softmax-function-cant-specify-the-dimension-to-operate/2637"""
    input_size = input.size()

    trans_input = input.transpose(axis, len(input_size) - 1)
    trans_size = trans_input.size()

    input_2d = trans_input.contiguous().view(-1, trans_size[-1])

    soft_max_2d = F.softmax(input_2d)

    soft_max_nd = soft_max_2d.view(*trans_size)
    return soft_max_nd.transpose(axis, len(input_size) - 1)

class InterSentenceInteraction(torch.nn.Module):
    def __init__(self):
        super(InterSentenceInteraction, self).__init__()

        self._attention = InterSentenceAttention()

    def forward(self, a, b, e=None):
        if e is None:
            e = self._attention(a, b)

        e_b = softmax(e, axis=2)
        e_a = softmax(e, axis=1)

        a_tilda = torch.matmul(e_b, b)
        b_tilda = torch.matmul(e_a.transpose(2, 1), a)

        return a_tilda, b_tilda

class KeyValueMemoryLookup(torch.nn.Module):
    def forward(self, sentence, memory_key, memory_value):
        squeeze = False
        if len(sentence.size()) == 2:
            sentence = sentence.unsqueeze(1)
            squeeze = True

        sentence_normalized = F.normalize(sentence, p=2, dim=-1)
        memory_key_normalized = F.normalize(memory_key, p=2, dim=-1)

        mem_attention = torch.matmul(sentence_normalized,
                                     memory_key_normalized.transpose(1, 2))  # (batch_size, max_len, mem_size)
        mem_attention = softmax(mem_attention, axis=2)

        mem_attended = mem_attention.unsqueeze(3) * memory_value.unsqueeze(1)  # (?, max_len, mem_size, emd_dim)
        mem_attended = mem_attended.sum(dim=2)  # (?, max_len, emd_dim)

        if squeeze:
            mem_attended = mem_attended.squeeze()

        return mem_attended

class InteractionEnhancement(torch.nn.Module):
    def __init__(self, extended=True):
        super(InteractionEnhancement, self).__init__()

        self.extended = extended

    def forward(self, *args):
        to_concat = []
        to_concat.extend(args)

        if self.extended:
            a0 = args[0]
            for a1 in args[1:]:
                to_concat.append(a0 - a1)
                to_concat.append(a0 * a1)

        m_a = torch.cat(to_concat, dim=-1)

        return m_a

class MaxAvgPool(torch.nn.Module):
    def forward(self, inputs, lengths=None):
        inputs_max, _ = inputs.max(dim=1, keepdim=False)

        if lengths is None:
            inputs_avg = inputs.mean(dim=1, keepdim=False)
        else:
            inputs_avg = inputs.sum(dim=1, keepdim=False) / lengths.float().unsqueeze(1)

        result = torch.cat([inputs_avg, inputs_max], dim=-1)

        return result

class DynamicMemoryCell(BaseRNNCell):
    def __init__(self, nb_slots=4, memory_size=300):
        super(DynamicMemoryCell, self).__init__()

        self.nb_slots = nb_slots
        self.memory_size = memory_size

        self.keys = torch.nn.ParameterList()
        for i in range(nb_slots):
            key = torch.nn.Parameter(torch.randn(memory_size), requires_grad=False)
            key.data.uniform_(-0.1, 0.1)
            self.keys.append(key)

        self.U = torch.nn.Parameter(torch.randn(memory_size, memory_size))
        self.V = torch.nn.Parameter(torch.randn(memory_size, memory_size))
        self.W = torch.nn.Parameter(torch.randn(memory_size, memory_size))
        torch.nn.init.xavier_normal(self.U)
        torch.nn.init.xavier_normal(self.V)
        torch.nn.init.xavier_normal(self.W)

        self.gate_bias = torch.nn.Parameter(torch.zeros(1))
        self.state_bias = torch.nn.Parameter(torch.zeros(1, memory_size))

        logging.info('Creating memory: [%s, %s]', self.nb_slots, self.memory_size)

    def zero_state(self, batch_size):
        states = tuple([
            Variable(self.keys[i].data).unsqueeze(0).expand(batch_size, self.memory_size)
            for i in range(self.nb_slots)
        ])
        return states

    def forward(self, inputs, states):
        # inputs have the shape of (?, mem_size)
        # states is a list which elements have the shape of (?, mem_size)

        new_states = []
        for j, state in enumerate(states):
            key = self.keys[j]

            g_state = torch.matmul(inputs.unsqueeze(1), state.unsqueeze(2)).squeeze()  # shape (?,)
            g_key = torch.matmul(inputs, key)  # shape (?,)
            g = F.sigmoid(g_state + g_key + self.gate_bias)  # shape (?,)

            # get h_tilda
            h_tilda_state = torch.matmul(state, self.U)  # shape (?, mem_size)
            h_tilda_key = torch.matmul(key.unsqueeze(0), self.V)  # shape (1, mem_size)
            h_tilda_inputs = torch.matmul(inputs, self.W)  # shape (?, mem_size)
            h_tilda = F.elu(h_tilda_state + h_tilda_key + h_tilda_inputs + self.state_bias)  # shape (?, mem_size)

            h = state + g.unsqueeze(1) * h_tilda  # shape (?, mem_size)

            # normalize
            h = F.normalize(h, p=2, dim=-1)

            new_states.append(h)

        new_states = tuple(new_states)
        return new_states

class DynamicMemory(torch.nn.Module):
    """Based on https://github.com/siddk/entity-network"""

    def __init__(self, nb_slots=4, memory_size=300, bidirectional=False, **kwargs):
        super(DynamicMemory, self).__init__()

        self.cell = DynamicMemoryCell(nb_slots, memory_size)
        self.bidirectional = bidirectional

    def forward(self, inputs, lengths):
        lengths_sorted, inputs_sorted_idx = lengths.sort(descending=True)
        inputs_sorted = inputs[inputs_sorted_idx]
        inputs_lenghts = list(lengths_sorted.data)

        hiddens, outputs = dynamic_rnn(self.cell, inputs_sorted, inputs_lenghts)
        hiddens = torch.stack(hiddens, dim=1)

        if self.bidirectional:
            hiddens_rev, outputs_rev = dynamic_rnn_reversed(self.cell, inputs_sorted, inputs_lenghts)
            hiddens_rev = torch.stack(hiddens_rev, dim=1)

            hiddens = torch.cat([hiddens, hiddens_rev], dim=-1)

        _, inputs_unsorted_idx = inputs_sorted_idx.sort(descending=False)
        hiddens = hiddens[inputs_unsorted_idx]

        return hiddens

Variable.__repr__ = lambda x: 'Variable {}'.format(tuple(x.size()))
torch.nn.Parameter.__repr__ = lambda x: 'Parameter {}'.format(tuple(x.size()))


import socket
import serial
import ipaddress
import ts_modbus.const as const

class _TSConnection:

    def __init__(self, config: dict):
        self.config = const.MB_DEFAULT_CONFIG
        if not self._check_config(config):
            config = const.MB_DEFAULT_CONFIG
        self.config = config

    # noinspection PyMethodMayBeStatic
    def _check_config(self, config: dict) -> bool:
        return False

    def send(self, data: bytes) -> bool:
        pass

    def response(self) -> bytes:
        pass

    def open_connection(self, timeout: int = 10) -> bool:
        pass

    def close_connection(self) -> bool:
        pass

class COMConnection(_TSConnection):
    def __init__(self, config: dict):
        super().__init__(config)
        self.serialPort = None
        self.COMPort = str(self.config.get('COM', ''))
        self.baudrate = int(self.config.get('baudrate', -1))
        self.parity = str(self.config.get('parity', serial.serialutil.PARITY_NONE))
        self.stopbits = int(self.config.get('stopbits', serial.serialutil.STOPBITS_ONE))
        self.bytesizes = int(self.config.get('bytesizes', serial.serialutil.EIGHTBITS))

    def open_connection(self, timeout: int = 10) -> bool:
        self.serialPort = serial.Serial(port=self.COMPort,
                                        baudrate=self.baudrate,
                                        parity=self.parity,
                                        stopbits=self.stopbits,
                                        bytesize=self.bytesizes,
                                        timeout=timeout)
        return False

    def close_connection(self) -> bool:
        if self.serialPort is not None:
            self.serialPort.close()
        del self.serialPort
        return True

    def send(self, data: bytes) -> bool:
        if self.serialPort is None:
            return False
        if len(data) <= 0:
            return False
        self.serialPort.write(data)
        return True

    def response(self) -> bytes:
        if self.serialPort is None:
            return b''
        try:
            recv = self.serialPort.read(254)
            return recv
        except Exception:
            return b''

    def _check_config(self, config: dict) -> bool:
        if not isinstance(config, dict):
            return False
        COMPort = str(config.get('COM', ''))
        if len(COMPort) <= 0:
            return False
        baudrate = int(config.get('baudrate', -1))
        if baudrate not in serial.SerialBase.BAUDRATES:
            return False
        parity = str(config.get('parity', serial.PARITY_NONE))
        if parity not in serial.SerialBase.PARITIES:
            return False
        stopbits = int(config.get('stopbits', serial.STOPBITS_ONE))
        if stopbits not in serial.SerialBase.STOPBITS:
            return False
        bytesizes = int(config.get('bytesizes', serial.EIGHTBITS))
        if bytesizes not in serial.SerialBase.BYTESIZES:
            return False

        return True

class TCPConnection(_TSConnection):
    def __init__(self, config: dict):
        super().__init__(config)
        self.sock: socket.socket = None

    def open_connection(self, timeout: int = 10) -> bool:
        ip = self.config.get('ip')
        port = self.config.get('port')
        try:
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            self.sock.setblocking(timeout > 0)
            self.sock.settimeout(timeout)
            self.sock.connect((ip, port))
            return True
        except ConnectionRefusedError:
            return False
        except Exception:
            return False

    def close_connection(self) -> bool:
        self.sock.close()
        del self.sock
        return True

    def send(self, data: bytes) -> bool:
        if self.sock is None:
            return False
        if len(data) <= 0:
            return False
        try:
            self.sock.send(data)
            return True
        except Exception:
            return False

    def response(self) -> bytes:
        if self.sock is None:
            return b''
        try:
            recv = self.sock.recv(2048)
            return recv
        except Exception:
            return b''

    def _check_config(self, config: dict) -> bool:
        if not isinstance(config, dict):
            return False
        ipaddress.ip_address(config.get('ip', ''))
        port = int(config.get('port', -1))
        if port > 65535 or port <= 0:
            return False
        return True


from django.shortcuts import render
from django.core.mail import send_mail
from .models import input
import json  # import json to load json data to python dictionary
import urllib.request  # urllib.request to make a request to api

def index(request):
    if request.method == 'POST':
        city = request.POST['city']
        email = request.POST['email']
        name = request.POST['name']
        ins = input(name=name, email=email, city=city)
        ins.save()

        # source contain JSON data from API

        source = urllib.request.urlopen(
            'http://api.openweathermap.org/data/2.5/weather?q='
            + city + '&units=metric&appid=c1fa08aadf5b95cc7eb1721e78661798').read()

        # converting JSON data to a dictionary
        list_of_data = json.loads(source)

        # data for variable list_of_data
        data = {
            "name": name,
            "temp": (list_of_data['main']['temp']),
            "mood": 'ðŸ˜„',
        }
        if (data["temp"]) < 22:
            data["mood"] = 'ðŸ¥¶'
        elif (data["temp"]) > 26:
            data["mood"] = 'ðŸ¥µ'
        data["temp"] = str(data["temp"]) + ' Â°C'

        send_mail(
            "Hi " + name + ", interested in our services", # subject
            "The temperature of " + city + " is " + data["temp"] + " " + data["mood"],# message
            'abhijitprasadband@gmail.com', # from email
            [email], # To email
        )

    else:
        data = {}
    return render(request, "django_weather/home.html", data)


from matplotlib import pylab
from sklearn.decomposition import PCA
from gensim.models import Word2Vec
from sklearn.manifold import TSNE

## Function to reduce 200-dimension word2vec model down to 2D for visualizing word similarities
def plot(words):
    embeddings = [model[w] for w in words]

    pca = PCA(n_components = 2)
    two_d_embeddings = pca.fit_transform(embeddings)

    pylab.figure(figsize=(5,5))
    for i, label in enumerate(words):
        x, y = two_d_embeddings[i,:]
        pylab.scatter(x,y)
        pylab.annotate(label, xy=(x,y), xytext=(28, 2),
            textcoords = 'offset points', ha='right', va='bottom')
    pylab.savefig('testplot19.png')

model = Word2Vec.load('bigram_model_updated')

#plot('cake pizza salad cat horse dog car motorcycle train'.split())


import sys
from collections import Counter

def yesQs(l):
    yeses = 0
    for group in l:
        people = group.split("\n")
        answers = Counter()
        for person in people:
            answers.update(set(person))

        for k, v in answers.most_common():
            if v < len(people):
                break
            else:
                yeses += 1
    return yeses

my_list = ["fm\nif"]
l = [w for w in sys.stdin.read().split("\n\n")]
print(len(l))
print(yesQs(l))


import os
import sys
import shutil

def get_next_cache_dir():
    work_dir = os.getcwd()
    try:
        # change working directory to avoid import from local folder
        # during installation process
        os.chdir(os.path.dirname(sys.executable))
        import comtypes.client
        return comtypes.client._code_cache._find_gen_dir()
    except ImportError:
        return None
    finally:
        os.chdir(work_dir)

def _remove(directory):
    shutil.rmtree(directory)
    print('Removed directory "%s"' % directory)

def remove_directory(directory, silent):
    if directory:
        if silent:
            _remove(directory)
        else:
            try:
                confirm = raw_input('Remove comtypes cache directories? (y/n): ')
            except NameError:
                confirm = input('Remove comtypes cache directories? (y/n): ')
            if confirm.lower() == 'y':
                _remove(directory)
            else:
                print('Directory "%s" NOT removed' % directory)
                return False
    return True

if len(sys.argv) > 1 and "-y" in sys.argv[1:]:
    silent = True
else:
    silent = False

# First iteration may get folder with restricted rights.
# Second iteration always gets temp cache folder (writable for all).
directory = get_next_cache_dir()
removed = remove_directory(directory, silent)

if removed:
    directory = get_next_cache_dir()

    # do not request the second confirmation
    # if the first folder was already removed
    remove_directory(directory, silent=removed)


from PIL import Image, ImageGrab
import pytesseract
import cv2
import numpy as np

SHOP_COORDS = (480, 1040, 1480, 1070)
GOLD_COORDS = (868, 885, 910, 910)
STREAK_COORDS = (984, 876, 1025, 908)
LOSE_COLOR = (15, 59, 70)
STAGE_COORDS = (770, 12, 860, 33)

class Tesseract:
    # C:\Users\maelb\Documents\GitHub\pytft\tesseract\tesseract.exe
    # C:\Users\Romain\Documents\GitHub\pytft\tesseract\tesseract.exe
    pytesseract.pytesseract.tesseract_cmd = r"C:\Users\maelb\Documents\GitHub\pytft\tesseract\tesseract.exe"

    @staticmethod
    def to_gray(d):
        image = np.asarray(d)
        gry = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

        return Image.fromarray(cv2.cvtColor(gry, cv2.COLOR_BGR2RGB))

    @staticmethod
    def clean(d):
        image = np.asarray(d)

        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        blur = cv2.GaussianBlur(gray, (3, 3), 0)
        thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]

        # Morph open to remove noise and invert image
        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
        opening = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel, iterations=1)
        invert = 255 - opening

        return Image.fromarray(cv2.cvtColor(invert, cv2.COLOR_BGR2RGB))

    @staticmethod
    def image_to_string(data):
        result = pytesseract.image_to_string(data, config = r"--oem 3 --psm 7 -c tessedit_char_whitelist=0123456789-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
        return result

    @staticmethod
    def get_shop():
        im = ImageGrab.grab(bbox=SHOP_COORDS)
        im = Tesseract.to_gray(im)
        im.save("gray.png")
        unit_length = im.size[0] / 5
        units = []
        for x in range(0, 5):
            box = (x * unit_length, 0, (x + 1) * unit_length - 50, im.size[1])
            #print(box)
            new = im.crop(box)
            text = Tesseract.image_to_string(new).replace("\n","")
            units.append(text)
        return units

    @staticmethod
    def gold_balance():
        im = ImageGrab.grab(bbox=GOLD_COORDS)
        im = Tesseract.to_gray(im)
        gold_count = Tesseract.image_to_string(im)
        return gold_count

    @staticmethod
    def streak_value():
        im = ImageGrab.grab(bbox=STREAK_COORDS)
        im.save("streak.png")
        color = im.getpixel((2, 23))
        side = (-1 if color[2] > 17 else 1)  # check if blue channel of rgb is higher 17, if so its lose streak
        try:
            val = int(Tesseract.image_to_string(im))
        except ValueError:
            val = 0
        return val*side

    @staticmethod
    def stage_value():
        im = ImageGrab.grab(bbox=STAGE_COORDS)
        im = Tesseract.to_gray(im)
        im.save("stage.png")
        stage = Tesseract.image_to_string(im)
        return stage

u = Tesseract.get_shop()
print(u)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf

from tensorflow_ranking.python import utils

class UtilsTest(tf.test.TestCase):

  def setUp(self):
    super(UtilsTest, self).setUp()
    tf.compat.v1.reset_default_graph()

  def test_is_label_valid(self):
    with tf.Graph().as_default():
      labels = [[1.0, 0.0, -1.0]]
      labels_validity = [[True, True, False]]
      with tf.compat.v1.Session() as sess:
        is_valid = sess.run(utils.is_label_valid(labels))
        self.assertAllEqual(is_valid, labels_validity)

  def test_sort_by_scores(self):
    with tf.Graph().as_default():
      scores = [[1., 3., 2.], [1., 2., 3.]]
      positions = [[1, 2, 3], [4, 5, 6]]
      names = [['a', 'b', 'c'], ['d', 'e', 'f']]
      with tf.compat.v1.Session() as sess:
        sorted_positions, sorted_names = sess.run(
            utils.sort_by_scores(scores, [positions, names]))
        self.assertAllEqual(sorted_positions, [[2, 3, 1], [6, 5, 4]])
        self.assertAllEqual(sorted_names,
                            [[b'b', b'c', b'a'], [b'f', b'e', b'd']])

        sorted_positions, sorted_names = sess.run(
            utils.sort_by_scores(scores, [positions, names], topn=2))
        self.assertAllEqual(sorted_positions, [[2, 3], [6, 5]])
        self.assertAllEqual(sorted_names, [[b'b', b'c'], [b'f', b'e']])

        sorted_positions, sorted_names = sess.run(
            utils.sort_by_scores([scores[0]], [[positions[0]], [names[0]]]))
        self.assertAllEqual(sorted_positions, [[2, 3, 1]])
        self.assertAllEqual(sorted_names, [[b'b', b'c', b'a']])

  def test_sort_by_scores_for_3d_features(self):
    with tf.Graph().as_default():
      scores = [[1., 3., 2.], [1., 2., 3.]]
      example_feature = [[[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]],
                         [[10., 20., 30.], [40., 50., 60.], [70., 80., 90.]]]
      with tf.compat.v1.Session() as sess:
        sorted_example_feature = sess.run(
            utils.sort_by_scores(scores, [example_feature])[0])
        self.assertAllEqual(
            sorted_example_feature,
            [[[4., 5., 6.], [7., 8., 9.], [1., 2., 3.]],
             [[70., 80., 90.], [40., 50., 60.], [10., 20., 30.]]])

        sorted_example_feature = sess.run(
            utils.sort_by_scores(scores, [example_feature], topn=2)[0])
        self.assertAllEqual(
            sorted_example_feature,
            [[[4., 5., 6.], [7., 8., 9.]], [[70., 80., 90.], [40., 50., 60.]]])

        sorted_example_feature = sess.run(
            utils.sort_by_scores([scores[0]], [[example_feature[0]]])[0])
        self.assertAllEqual(sorted_example_feature,
                            [[[4., 5., 6.], [7., 8., 9.], [1., 2., 3.]]])

  def test_sort_by_scores_shuffle_ties(self):
    with tf.Graph().as_default():
      tf.compat.v1.set_random_seed(589)
      scores = [[2., 1., 1.]]
      names = [['a', 'b', 'c']]
      with tf.compat.v1.Session() as sess:
        sorted_names = sess.run(
            utils.sort_by_scores(scores, [names], shuffle_ties=False))[0]
        self.assertAllEqual(sorted_names, [[b'a', b'b', b'c']])
        sorted_names = sess.run(
            utils.sort_by_scores(scores, [names], shuffle_ties=True, seed=2))[0]
        self.assertAllEqual(sorted_names, [[b'a', b'c', b'b']])

  def test_sorted_ranks(self):
    with tf.Graph().as_default():
      scores = [[1., 3., 2.]]
      with tf.compat.v1.Session() as sess:
        ranks = sess.run(utils.sorted_ranks(scores, seed=1))
        self.assertAllEqual(ranks, [[3, 1, 2]])

      tf.compat.v1.set_random_seed(3)
      scores = [[1., 2., 1.]]
      with tf.compat.v1.Session() as sess:
        ranks = sess.run(utils.sorted_ranks(scores, shuffle_ties=False, seed=1))
        self.assertAllEqual(ranks, [[2, 1, 3]])
        ranks = sess.run(utils.sorted_ranks(scores, shuffle_ties=True, seed=1))
        self.assertAllEqual(ranks, [[3, 1, 2]])

  def test_organize_valid_indices(self):
    with tf.Graph().as_default():
      tf.compat.v1.set_random_seed(4)
      labels = [[1.0, 0.0, -1.0], [-1.0, 1.0, 2.0]]
      is_valid = utils.is_label_valid(labels)
      shuffled_indices = utils.shuffle_valid_indices(is_valid, seed=2)
      organized_indices = utils.organize_valid_indices(is_valid, shuffle=False)
      with tf.compat.v1.Session() as sess:
        shuffled_indices = sess.run(shuffled_indices)
        self.assertAllEqual(
            shuffled_indices,
            [[[0, 1], [0, 0], [0, 2]], [[1, 1], [1, 2], [1, 0]]])
        organized_indices = sess.run(organized_indices)
        self.assertAllEqual(
            organized_indices,
            [[[0, 0], [0, 1], [0, 2]], [[1, 1], [1, 2], [1, 0]]])

  def test_reshape_first_ndims_dense_tensor(self):
    with tf.Graph().as_default():
      # Batch size = 2, list size = 5, embedding size = 10.
      tensor = tf.reshape(tf.range(100), shape=(2, 5, 10))
      target_tensor = tf.reshape(tf.range(100), shape=(10, 10))
      reshaped_tensor = utils.reshape_first_ndims(tensor, 2, [10])
      self.assertAllEqual(reshaped_tensor.get_shape().as_list(), [10, 10])

      with tf.compat.v1.Session() as sess:
        reshaped, target = sess.run([reshaped_tensor, target_tensor])
        self.assertAllEqual(reshaped, target)

  def test_reshape_first_ndims_sparse_tensor(self):
    with tf.Graph().as_default():
      # Batch size = 2, list size = 3, embedding size = 3.
      # Tensor:
      # [[[1, 0, 0], [0, 2, 0], [0, 0, 3]], [[4, 0, 0], [0, 5, 0], [0, 0, 6]]].
      # Reshaped :
      # [[[1, 0, 0], [0, 2, 0], [0, 0, 3], [4, 0, 0], [0, 5, 0], [0, 0, 6]]].
      sparse_tensor = tf.SparseTensor(
          indices=[[0, 0, 0], [0, 1, 1], [0, 2, 2], [1, 0, 0], [1, 1, 1],
                   [1, 2, 2]],
          values=[1, 2, 3, 4, 5, 6],
          dense_shape=[2, 3, 3])
      target = tf.SparseTensor(
          indices=[[0, 0], [1, 1], [2, 2], [3, 0], [4, 1], [5, 2]],
          values=[1, 2, 3, 4, 5, 6],
          dense_shape=[6, 3])
      reshaped = utils.reshape_first_ndims(sparse_tensor, 2, [6])
      with tf.compat.v1.Session() as sess:
        reshaped_array, target_array = sess.run([reshaped, target])
        self.assertAllEqual(reshaped_array.indices, target_array.indices)
        self.assertAllEqual(reshaped_array.values, target_array.values)
        self.assertAllEqual(reshaped_array.dense_shape,
                            target_array.dense_shape)

  def test_reshape_to_2d(self):
    with tf.Graph().as_default():
      tensor_3d = tf.constant([[[1], [2], [3]], [[4], [5], [6]]])
      tensor_3d_reshaped = utils.reshape_to_2d(tensor_3d)
      tensor_1d = tf.constant([1, 2, 3])
      tensor_1d_reshaped = utils.reshape_to_2d(tensor_1d)
      with tf.compat.v1.Session() as sess:
        self.assertAllEqual(
            sess.run(tensor_3d_reshaped), [[1, 2, 3], [4, 5, 6]])
        self.assertAllEqual(sess.run(tensor_1d_reshaped), [[1], [2], [3]])

  def test_circular_indices(self):
    with tf.Graph().as_default():
      with tf.compat.v1.Session() as sess:
        # All valid.
        indices, mask = sess.run(utils._circular_indices(3, [3]))
        self.assertAllEqual(indices, [[0, 1, 2]])
        self.assertAllEqual(mask, [[True, True, True]])
        # One invalid.
        indices, mask = sess.run(utils._circular_indices(3, [2]))
        self.assertAllEqual(indices, [[0, 1, 0]])
        self.assertAllEqual(mask, [[True, True, False]])
        # All invalid.
        indices, mask = sess.run(utils._circular_indices(3, [0]))
        self.assertAllEqual(indices, [[0, 0, 0]])
        self.assertAllEqual(mask, [[False, False, False]])
        # batch_size = 2
        indices, mask = sess.run(utils._circular_indices(3, [3, 2]))
        self.assertAllEqual(indices, [[0, 1, 2], [0, 1, 0]])
        self.assertAllEqual(mask, [[True, True, True], [True, True, False]])

  def test_padded_nd_indices(self):
    with tf.Graph().as_default():
      tf.compat.v1.set_random_seed(1)
      # batch_size, list_size = 2, 3.
      is_valid = [[True, True, True], [True, True, False]]
      # Disable shuffling.
      indices, mask = utils.padded_nd_indices(is_valid, shuffle=False)

      with tf.compat.v1.Session() as sess:
        indices, mask = sess.run([indices, mask])
        # shape = [2, 3, 2] = [batch_size, list_size, 2].
        self.assertAllEqual(
            indices,
            [  # batch_size = 2.
                [  # list_size = 3.
                    [0, 0], [0, 1], [0, 2]
                ],
                [  # list_size = 3.
                    [1, 0], [1, 1], [1, 0]
                ]
            ])
        # shape = [2, 3] = [batch_size, list_size]
        self.assertAllEqual(mask, [[True, True, True], [True, True, False]])

      # Enable shuffling.
      indices, mask = utils.padded_nd_indices(
          is_valid, shuffle=True, seed=87124)

      with tf.compat.v1.Session() as sess:
        indices, mask = sess.run([indices, mask])
        # shape = [2, 3, 2] = [batch_size, list_size, 2].
        self.assertAllEqual(
            indices,
            [  # batch_size = 2.
                [  # list_size = 3.
                    [0, 0], [0, 1], [0, 2]
                ],
                [  # list_size = 3
                    [1, 1], [1, 0], [1, 1]
                ]
            ])
        # shape = [2, 3] = [batch_size, list_size]
        self.assertAllEqual(mask, [[True, True, True], [True, True, False]])

  def test_de_noise(self):
    with tf.Graph().as_default():
      counts = [[1, 2, 3], [1, 2, 3]]
      noise = [[3, 3, 4], [3, 2, 1]]
      with tf.compat.v1.Session() as sess:
        # Larger noise ratio -> the results are more sharp.
        de_noised = sess.run(utils.de_noise(counts, noise, ratio=0.9))
        self.assertAllClose(de_noised, [[0., 0.22, 0.78], [0., 0., 1.]])
        # Smaller noise ratio -> the results are close to counts.
        de_noised = sess.run(utils.de_noise(counts, noise, ratio=0.1))
        self.assertAllClose(de_noised, [
            [0.151852, 0.337037, 0.511111],
            [0.12963, 0.333333, 0.537037],
        ])

  def test_de_noise_exception(self):
    with tf.Graph().as_default():
      with self.assertRaises(ValueError):
        utils.de_noise([[1, 2, 3]], [[1, 2, 3]], ratio=1.1)
      with self.assertRaises(ValueError):
        utils.de_noise([[1, 2, 3]], [[1, 2, 3]], ratio=-0.1)
      with self.assertRaises(tf.errors.InvalidArgumentError):
        utils.de_noise([[-1, 2, 3]], [[1, 2, 3]])
      with self.assertRaises(tf.errors.InvalidArgumentError):
        utils.de_noise([[1, 2, 3]], [[0, 2, 3]])

if __name__ == '__main__':
  tf.compat.v1.enable_v2_behavior()
  tf.test.main()


# -*- coding: utf-8 -*-
# @Time    : 2019/2/14 15:29
# @Author  : Ming
import json
import logging
import re
import time

import scrapy

from dou_ban.items import DoubanItem

from scrapy import Selector, Request

class DoubanSpider(scrapy.Spider):
    name = "douban"
    allowed_domains = ["douban.com"]
    start_urls = ['https://movie.douban.com/subject/30145962/']
    # start_urls = ['https://movie.douban.com/j/new_search_subjects?sort=U&range=0,10&tags=%E7%94%B5%E5%BD%B1&start=0&year_range=2018,2018']
    base_url_prefix = 'https://movie.douban.com/j/new_search_subjects?sort=U&range=0,10&tags=%E7%94%B5%E5%BD%B1&start='
    base_url_suffix = '&year_range=2018,2018'

    test = 0

    def parse(self, response):
        urls = []
        start = re.search(r'start=(\d*)', response.url).group(1)
        rs = json.loads(response.text)
        data = rs.get('data')
        if len(data) != 0:
            # and self.test < 1
            self.test += 1
            try:
                yield Request(self.base_url_prefix + str(int(start) + 20) + self.base_url_suffix, callback=self.parse)
            except Exception as e:
                logging.error(e)
                raise e
        for temp in data:
            urls.append(temp.get('url'))
        for url in urls:
            yield Request(url, callback=self.parse_item)
            time.sleep(0.25)

    def parse_item(self, response):
        item = DoubanItem()
        selector = Selector(response)
        one_movie = selector.xpath('//div[@id="content"]')
        text = one_movie.xpath('string(.//div[@id="info"])')  # 提取标签内的文本

        # item['score'] = one_movie.xpath('.//strong[@class="ll rating_num"]/text()').extract()
        # 从script中获取其余的值
        json_data = json.loads(selector.xpath('//script[@type="application/ld+json"]/text()').extract()[0].replace('\n', ''), strict=False)
        item['url'] = str(response.url)
        item['name'] = json_data.get('name')
        also_known = text.re(r'又名:(.*)\n')
        item['also_known'] = '' if len(also_known) == 0 else also_known[0]
        item['movie_info'] = json_data.get('description')
        score = json_data.get('aggregateRating').get('ratingValue')
        if score == '':
            item['score'] = 0.0
        else:
            item['score'] = float(score)

        screenwriter = text.re(r'编剧:(.*)\n')
        director = text.re(r'导演:(.*)\n')
        genre = text.re(r'类型:(.*)\n')
        runtime = text.re(r'片长:(.*)\n')
        country = text.re(r'地区:(.*)\n')
        stars = text.re(r'主演:(.*)\n')
        language = text.re(r'语言:(.*)\n')

        item['director'] = '' if len(director) == 0 else director[0]
        item['screenwriter'] = '' if len(screenwriter) == 0 else screenwriter[0]
        # item['stars'] = self.json_util(json_data.get('actor'))
        item['stars'] = '' if len(stars) == 0 else stars[0]
        # item['genre'] = ''.join([e for e in one_movie.xpath('.//span[@property="v:genre"]/text()').extract()])

        item['genre'] = '' if len(genre) == 0 else genre[0]
        item['country'] = '' if len(country) == 0 else country[0]
        item['language'] = '' if len(language) == 0 else language[0]
        item['release_date'] = json_data.get('datePublished')
        item['runtime'] = '' if len(runtime) == 0 else runtime[0]
        # print(json.loads(json_data[0]).get('director'))
        # print(len(json_data))
        yield(item)

    # 提取json中的name，并拼接字符串返回
    def json_util(self, json_data):
        result = ''
        for temp in json_data:
            result = result + temp.get('name') + '/'
        return result.rstrip('/')


# -*- coding: UTF-8 -*-
# Given two strings s and t, return true if t is an anagram of s, and false
# otherwise.
#
#  An Anagram is a word or phrase formed by rearranging the letters of a
# different word or phrase, typically using all the original letters exactly once.
#
#
#  Example 1:
#  Input: s = "anagram", t = "nagaram"
# Output: true
#
#  Example 2:
#  Input: s = "rat", t = "car"
# Output: false
#
#
#  Constraints:
#
#
#  1 <= s.length, t.length <= 5 * 10â�´
#  s and t consist of lowercase English letters.
#
#
#
#  Follow up: What if the inputs contain Unicode characters? How would you
# adapt your solution to such a case?
#
#  Related Topics Hash Table String Sorting ðŸ‘� 7680 ðŸ‘Ž 248

# leetcode submit region begin(Prohibit modification and deletion)
class Solution(object):
    def isAnagram(self, s, t):
        """
        :type s: str
        :type t: str
        :rtype: bool
        """
        from collections import Counter
        a_count = Counter(s)
        b_count = Counter(t)
        return a_count == b_count

# leetcode submit region end(Prohibit modification and deletion)

s = Solution()
print(s.isAnagram("hello", "lloeh"))


import random
counter = 0
result = [12, 1, 45, 76, 50, 23]

for number in result:
    if 1 <= number <= 49:
        continue
    else:
        print(f"Liczba {number} jest poza zakresem. Dokonuję zmiany.")
        idx = result.index(number)
        result[idx] = random.randint(1, 50)

print(result)
print(counter)


"""20220610更

Revision ID: 1fcf18bd7d15
Revises:
Create Date: 2022-06-10 21:58:06.777007

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '1fcf18bd7d15'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('LaitGood_Newsdata',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('image', sa.String(length=80), nullable=False),
    sa.Column('title', sa.String(length=80), nullable=False),
    sa.Column('slogan', sa.String(length=80), nullable=True),
    sa.Column('content', sa.String(length=80), nullable=True),
    sa.Column('newsdate', sa.DateTime(), nullable=False),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('LaitGood_UserRegister',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('username', sa.String(length=80), nullable=False),
    sa.Column('email', sa.String(length=80), nullable=False),
    sa.Column('password_hash', sa.String(length=50), nullable=False),
    sa.Column('agreecheck', sa.Boolean(), nullable=True),
    sa.Column('registered_on', sa.DateTime(), nullable=False),
    sa.Column('confirm', sa.Boolean(), nullable=True),
    sa.Column('confirmed_on', sa.DateTime(), nullable=True),
    sa.Column('roles', sa.Boolean(), nullable=True),
    sa.Column('birthday', sa.DateTime(), nullable=True),
    sa.Column('gender', sa.String(length=5), nullable=True),
    sa.Column('cellphone', sa.Integer(), nullable=True),
    sa.Column('phone', sa.Integer(), nullable=True),
    sa.Column('address', sa.String(length=80), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('email')
    )
    op.create_table('LaitGood_commends_data',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('image', sa.String(length=80), nullable=False),
    sa.Column('title', sa.String(length=80), nullable=False),
    sa.Column('slogan', sa.String(length=80), nullable=True),
    sa.Column('url', sa.String(length=80), nullable=False),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('posts',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('posts')
    op.drop_table('LaitGood_commends_data')
    op.drop_table('LaitGood_UserRegister')
    op.drop_table('LaitGood_Newsdata')
    # ### end Alembic commands ###


"""ARQUIVO COM DADOS GLOBAIS PARA O SISTEMA"""

import threading

global telaSelecionada, guindasteSelecionado
telaSelecionada = 'POR01001'
guindasteSelecionado = ''

"""EVENTO PARA GERENCIAR INPUT DE DADOS"""
eventos = threading.Event()

"""VARIAVEIS GLOBAIS DE FLUXO DE DADOS COM INTERFACE"""
global dataInput, dataOutput
dataInput = None
"""interface for data flow."""
dataInput2 = None
"""interface for data flow."""
dataInput3 = None
"""interface for data flow."""
dataOutput = None
"""interface for data flow."""

"""DADOS DE TELEMETRIA"""
distanceTool = 0
"""Distancia da ferramenta com algo abaixo mais proximo, em centimetros"""
toolPosition = 0
"""Posicao da ferramenta em centimetros."""
towerPosition = 0
"""Posicao da torre em graus."""
electromagnet = 0
"""Indica se o eletroima esta ligado ou nao."""


import os
import re
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import scipy.stats as stats

filename_pattern = re.compile(r"output-(\d+)-(n|W)\.txt")

def extract_time(content):
    match = re.search(r"Tempo de execução: ([\d.]+) segundos", content)
    return float(match.group(1)) if match else None

def read_times(directory, file_ending):
    times = {}
    for filename in os.listdir(directory):
        if filename.endswith(f"-{file_ending}.txt"):
            path = os.path.join(directory, filename)
            with open(path, 'r') as file:
                content = file.read()
                time = extract_time(content)
                size = filename_pattern.search(filename).group(1)
                times[int(size)] = time
    return times

def plot_data(times_dict, approach, file_ending):
    os.makedirs('data/plots', exist_ok=True)
    if times_dict:
        df = pd.DataFrame({
            'Tamanho': sorted(times_dict.keys()),
            'Tempo': [times_dict[k] for k in sorted(times_dict.keys())]
        })

        plt.figure(figsize=(10, 5))
        plt.plot('Tamanho', 'Tempo', data=df, marker='', color='blue', linewidth=2)
        plt.xlabel('Tamanho da Instância')
        plt.ylabel('Tempo de Execução (segundos)')
        plt.title(f'Tempo de Execução - {approach} - {file_ending}')
        plt.legend([f'{approach} - {file_ending}'])

        filename = f"grafico_{approach.replace('/', '_')}_{file_ending}.png"
        plt.savefig(os.path.join('data/plots', filename))
        plt.close()
    else:
        print(f"Não há dados para plotar para {approach.split('/')[-1]} com arquivos terminando em '{file_ending}'.")

def calculate_confidence_interval(times):
    mean = np.mean(times)
    std_err = stats.sem(times)
    h = std_err * stats.t.ppf((1 + 0.95) / 2, len(times) - 1)
    return mean, mean - h, mean + h

def write_parity_tests(times_dict, approach, file_ending):
    os.makedirs('data/tests', exist_ok=True)
    with open(os.path.join('data/tests', 'parity_tests.txt'), 'a') as file:
        file.write(f"Testes de paridade para {approach} com arquivos terminando em '{file_ending}':\n")
        for size, time in times_dict.items():
            file.write(f"Tamanho: {size}, Tempo: {time:.4f} segundos\n")
        if times_dict:
            times = list(times_dict.values())
            mean, lower, upper = calculate_confidence_interval(times)
            file.write(f"Intervalo de Confiança de 95%: Média = {mean:.4f}, Limite Inferior = {lower:.4f}, "
                       f"Limite Superior = {upper:.4f}\n")
        file.write("\n")

approaches = ['backtracking', 'branch_and_bound', 'dynamic']
for approach in approaches:
    approach_path = f'data/outputs/{approach}'
    for file_ending in ['n', 'W']:
        times = read_times(approach_path, file_ending)
        write_parity_tests(times, approach, file_ending)
        plot_data(times, approach, file_ending)


# en-US,en;q=0.8,pt;q=0.6

def parse_AcceptLanguage(acceptLanguage):
  languages = acceptLanguage.split(",")
  locale_q_pairs = []

  for language in languages:
    if language.split(";")[0] == language:
      # no q => q = 1
      locale_q_pairs.append((language.strip(), "1"))
    else:
      locale = language.split(";")[0].strip()
      q = language.split(";")[1].split("=")[1]
      locale_q_pairs.append((locale, q))
  return locale_q_pairs

def simple_langs(acceptLanguage):
    'parse the Accept-Language header'
    langs = parse_AcceptLanguage(acceptLanguage)
    langs = [l[0][0:2] for l in langs]
    return list(set(langs))


#!/usr/bin/env python3
import pickle
from copy import deepcopy
from random import randint, uniform
import cv2
import matplotlib.pyplot as plt
import numpy as np
from models2 import Line
from scipy.optimize import least_squares

def main():
    # -----------------------------------------------------
    # Initialization
    # -----------------------------------------------------

    file = open('pts.pkl', 'rb')
    pts = pickle.load(file)
    file.close()
    print('pts = ' + str(pts))

    plt.figure()
    plt.xlim(-10, 10)
    plt.ylim(-10, 10)
    plt.grid()
    plt.xlabel("X")
    plt.ylabel("Y")
    print('Created a figure')

    plt.plot(pts['xs'], pts['ys'], 'sk', linewidth=2, markersize=6)

    # Define the model
    # y = m * x + b
    line = Line(pts)

    # -----------------------------------------------------
    # Execution
    # -----------------------------------------------------

    # Set new values
    line.randomize_Params()

    result = least_squares(line.objectiveFunction, [line.m, line.b], verbose=2)

    line.draw()
    plt.show()

    # -----------------------------------------------------
    # Termination
    # -----------------------------------------------------

if __name__ == "__main__":
    main()


from b_lambda_layer_common.util.list_bytes_batcher import ListBytesBatcher

def test_FUNC_iterate_batches_WITH_dummy_list_EXPECT_successful_split() -> None:
    """
    Checks do ListBatcher correctly splits a given list into desired size batches.

    :return: No return.
    """

    items = [item for item in range(1000*1000)]
    batch_size = 256*1024
    # Split list of items into batches not exceeding batch size.
    pool = ListBytesBatcher(items=items, batch_size=batch_size)

    # Check size of each batch in pool.
    for batch in pool.iterate_batches():
        assert ListBytesBatcher.size(batch) <= batch_size

    # Convert batches back to list of items.
    items_flatten = [item for batch in pool.iterate_batches() for item in batch]

    # Compare initial list of items with flatten one.
    assert items == items_flatten, items_flatten


import numpy as np
import sklearn
from sklearn.svm import SVC
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.decomposition import PCA
from scipy.stats import pearsonr
import statistics
import pickle
import sys
from tkinter import *
from tkinter.ttk import *

#!/usr/bin/env python

from tkinter import *

from tkinter.ttk import *

window = Tk()

window.title("Principal Indicators of Divorce")
window.geometry('600x250')
window.configure(background = "gray")

var1 = StringVar()
var2 = StringVar()
var3 = StringVar()

selected = IntVar()

rad1_1 = Radiobutton(window, text='1', value=0, variable=var1)
rad2_1 = Radiobutton(window, text='2', value=1, variable=var1)
rad3_1 = Radiobutton(window, text='3', value=2, variable=var1)
rad4_1 = Radiobutton(window, text='4', value=3, variable=var1)
rad5_1 = Radiobutton(window, text='5', value=4, variable=var1)

rad1_2 = Radiobutton(window, text='1', value=0, variable=var2)
rad2_2 = Radiobutton(window, text='2', value=1, variable=var2)
rad3_2 = Radiobutton(window, text='3', value=2, variable=var2)
rad4_2 = Radiobutton(window, text='4', value=3, variable=var2)
rad5_2 = Radiobutton(window, text='5', value=4, variable=var2)

rad1_3 = Radiobutton(window, text='1', value=0, variable=var3)
rad2_3 = Radiobutton(window, text='2', value=1, variable=var3)
rad3_3 = Radiobutton(window, text='3', value=2, variable=var3)
rad4_3 = Radiobutton(window, text='4', value=3, variable=var3)
rad5_3 = Radiobutton(window, text='5', value=4, variable=var3)

def predict(q35, q36, q40):

    clf = pickle.load(open("svmweb.p", 'rb'))
    response = [[q35, q36, q40]]

    result = clf.predict(response)

    return(result)

def clicked():
    print(int(var1.get()), int(var2.get()), int(var3.get()))
    divorce = predict(int(var1.get()), int(var2.get()), int(var3.get()))
    if divorce:
        div = Label(window, text="   You're getting a divorce  ").grid(column=0, row=8)
    else:
        div = Label(window, text="You're not getting a divorce").grid(column=0, row=8)

btn_1 = Button(window, text="Click Me", command=clicked)
a_1 = Label(window ,text = "35. I can insult my spouse when we argue.").grid(row = 0,column = 0)
rad1_1.grid(column=1, row=0)
rad2_1.grid(column=2, row=0)
rad3_1.grid(column=3, row=0)
rad4_1.grid(column=4, row=0)
rad5_1.grid(column=5, row=0)

btn_2 = Button(window, text="Click Me", command=clicked)
a_2 = Label(window ,text = "36. I can humiliate my spouse when we argue.").grid(row = 2,column = 0)
rad1_2.grid(column=1, row=2)
rad2_2.grid(column=2, row=2)
rad3_2.grid(column=3, row=2)
rad4_2.grid(column=4, row=2)
rad5_2.grid(column=5, row=2)

btn_3 = Button(window, text="Click Me", command=clicked)
a_3 = Label(window ,text = "40. I start to argue with my spouse before I know what's going on.").grid(row = 4,column = 0)
rad1_3.grid(column=1, row=4)
rad2_3.grid(column=2, row=4)
rad3_3.grid(column=3, row=4)
rad4_3.grid(column=4, row=4)
rad5_3.grid(column=5, row=4)
btn_3.grid(column=0, row=6)

window.mainloop()

#
# print("On a scale of 1 to 5, to what degree do you agree with the following statements?\n")
# print("I can insult my spouse during our discussions")
# q35 = float(input("Response: "))
# while q35 not in [1, 2, 3, 4, 5]:
#     q35 = float(input("Please enter an integer between 1 and 5: "))
#
# print("I can be humiliating during discussions with my spouse")
# q36 = float(input("Response: "))
# while q36 not in [1, 2, 3, 4, 5]:
#     q36 = float(input("Please enter an integer between 1 and 5: "))
#
# print("I am just starting a discussion with my spouse before I know what's going on")
# q40 = float(input("Response: "))
# while q40 not in [1, 2, 3, 4, 5]:
#     q40 = float(input("Please enter an integer between 1 and 5: "))
#
# result = predict(q35-1, q36-1, q40-1)
#
# if result == 1:
#     print("You will get divorced!")
# else:
#     print("You will not get divorced!")


import os
import random
import re
import sys
import subprocess

from scapy.all import Ether, IPv6, UDP, \
                      IPv6ExtHdrHopByHop, IPv6ExtHdrDestOpt, \
                      IPv6ExtHdrFragment, IPv6ExtHdrRouting, \
                      ICMPv6ParamProblem, ICMPv6TimeExceeded, \
                      sendp, srp1, AsyncSniffer
from testrunner import run, check_unittests

EXT_HDR_NH = {
    IPv6ExtHdrHopByHop: 0,
    IPv6ExtHdrRouting: 43,
    IPv6ExtHdrFragment: 44,
    # IPSec headers currently unsupported by scapy
    IPv6ExtHdrDestOpt: 60,
    # Mobility header currently unsupported by scapy
  }

class Sniffer(object):
    def __init__(self, iface, *args, **kwargs):
        self.iface = iface
        self.sniffer = None
        self.stop_filter = None

    def start_sniff(self, stop_filter):
        assert self.sniffer is None
        self.stop_filter = stop_filter
        self.sniffer = AsyncSniffer(
            iface=self.iface,
            stop_filter=stop_filter,
        )
        self.sniffer.start()

    def wait_for_sniff_results(self, timeout=5):
        assert self.sniffer is not None
        self.sniffer.join(timeout=timeout)
        sniffer = self.sniffer
        self.sniffer = None
        if sniffer.results is None:
            return []
        return [p for p in sniffer.results
                # filter out packets only belonging to stop_filter if
                # it existed
                if sniffer.kwargs.get("stop_filter") is None or
                sniffer.kwargs["stop_filter"](p)]

sniffer = None

def check_and_search_output(cmd, pattern, res_group, *args, **kwargs):
    output = subprocess.check_output(cmd, *args, **kwargs).decode("utf-8")
    for line in output.splitlines():
        m = re.search(pattern, line)
        if m is not None:
            return m.group(res_group)
    return None

def get_bridge(tap):
    res = check_and_search_output(
            ["bridge", "link"],
            r"{}.+master\s+(?P<master>[^\s]+)".format(tap),
            "master"
        )
    return tap if res is None else res

def get_host_lladdr(tap):
    res = check_and_search_output(
            ["ip", "addr", "show", "dev", tap, "scope", "link"],
            r"inet6\s+(?P<lladdr>[0-9A-Fa-f:]+)/\d+",
            "lladdr"
        )
    if res is None:
        raise AssertionError(
                "Can't find host link-local address on interface {}".format(tap)
            )
    else:
        return res

def get_host_hwaddr(tap):
    res = check_and_search_output(
            ["ip", "addr", "show", "dev", tap, "scope", "link"],
            r"link[^\s]+\s+(?P<hwaddr>[0-9A-Fa-f:]+)",
            "hwaddr"
        )
    if res is None:
        raise AssertionError(
                "Can't find host hardware address on interface {}".format(tap)
            )
    else:
        return res

def pktbuf_empty(child):
    child.sendline("pktbuf")
    child.expect(r"packet buffer: first byte: (?P<first_byte>0x[0-9a-fA-F]+), "
                 r"last byte: 0x[0-9a-fA-F]+ \(size: (?P<size>\d+)\)")
    first_byte = child.match.group("first_byte")
    size = child.match.group("size")
    child.expect(
            r"~ unused: {} \(next: (\(nil\)|0), size: {}\) ~".format(
                first_byte, size))

def register_protnum(child):
    child.sendline("ip reg")
    child.expect("Registered to protocol number 59")

def unregister(child):
    child.sendline("ip unreg")
    child.expect(r"Unregistered from protocol number \d")

def get_first_interface(child):
    child.sendline("ifconfig")
    child.expect(r"Iface\s+(\d+)\s")
    return int(child.match.group(1))

def add_ipv6_address(child, iface, ipv6_addr):
    child.sendline("ifconfig {} add {}".format(iface, ipv6_addr))
    child.expect(r"success: added [a-f0-9:]+/\d+ to interface \d+")

def del_ipv6_address(child, iface, ipv6_addr):
    child.sendline("ifconfig {} del {}".format(iface, ipv6_addr))
    child.expect(r"success: removed [a-f0-9:]+ to interface \d+")

def add_neighbor(child, iface, ipv6_addr, hw_addr):
    child.sendline("nib neigh add {} {} {}".format(iface, ipv6_addr, hw_addr))
    child.sendline("nib neigh")
    child.expect(r"{} dev #{} lladdr {}".format(ipv6_addr.lower(), iface,
                                                hw_addr.upper()))

def del_neighbor(child, iface, ipv6_addr):
    child.sendline("nib neigh del {} {}".format(iface, ipv6_addr))

def test_wrong_type(child, iface, hw_dst, ll_dst, ll_src):
    p = srp1(Ether(dst=hw_dst) / IPv6(dst=ll_dst, src=ll_src) /
             IPv6ExtHdrRouting(type=255, segleft=1, addresses=["abcd::1"]),
             iface=iface, timeout=1, verbose=0)
    assert(p is not None)
    assert(ICMPv6ParamProblem in p)
    assert(p[ICMPv6ParamProblem].code == 0)     # erroneous header field encountered
    assert(p[ICMPv6ParamProblem].ptr == 42)     # routing header type field
    pktbuf_empty(child)

def test_inconsistent_header(child, iface, hw_dst, ll_dst, ll_src):
    # send routing header with no (0) addresses but segleft set to a value
    # larger than 0
    p = srp1(Ether(dst=hw_dst) / IPv6(dst=ll_dst, src=ll_src) /
             IPv6ExtHdrRouting(type=3, segleft=18, addresses=[]),
             iface=iface, timeout=1, verbose=0)
    assert(p is not None)
    assert(ICMPv6ParamProblem in p)
    assert(p[ICMPv6ParamProblem].code == 0)     # erroneous header field encountered
    assert(p[ICMPv6ParamProblem].ptr == 41)     # len field
    pktbuf_empty(child)

def test_multicast_dst(child, iface, hw_dst, ll_dst, ll_src):
    # sniffing for ICMPv6 parameter problem message
    sniffer.start_sniff(lambda p: p.haslayer(ICMPv6ParamProblem) or
                        (p.haslayer(UDP) and (p[IPv6].dst != "ff02::1")))
    # send routing header with multicast destination
    sendp(Ether(dst=hw_dst) / IPv6(dst="ff02::1", src=ll_src) /
          IPv6ExtHdrRouting(type=3, segleft=1, addresses=["abcd::1"]) /
          UDP(dport=2606), iface=iface, verbose=0)
    ps = sniffer.wait_for_sniff_results()
    p = [p for p in ps if (ICMPv6ParamProblem in p) or
                          ((UDP in p) and (p[UDP].dport == 2606) and
                           (p[IPv6].dst != "ff02::1"))]
    # packet should be discarded silently:
    # see https://tools.ietf.org/html/rfc6554#section-4.2
    assert(len(p) == 0)
    pktbuf_empty(child)

def test_multicast_addr(child, iface, hw_dst, ll_dst, ll_src):
    # sniffing for ICMPv6 parameter problem message
    sniffer.start_sniff(lambda p: p.haslayer(ICMPv6ParamProblem) or
                        (p.haslayer(UDP) and (p[IPv6].dst != ll_dst)))
    # Send routing header with multicast address in its destinations
    sendp(Ether(dst=hw_dst) / IPv6(dst=ll_dst, src=ll_src) /
          IPv6ExtHdrRouting(type=3, segleft=1, addresses=["abcd::1"]) /
          UDP(dport=2606), iface=iface, verbose=0)
    ps = sniffer.wait_for_sniff_results()
    p = [p for p in ps if (ICMPv6ParamProblem in p) or
                          ((UDP in p) and (p[UDP].dport == 2606) and
                           (p[IPv6].dst != ll_dst))]
    # packet should be discarded silently:
    # see https://tools.ietf.org/html/rfc6554#section-4.2
    assert(len(p) == 0)
    pktbuf_empty(child)

def test_multiple_addrs_of_mine_uncomp(child, iface, hw_dst, ll_dst, ll_src):
    dummy = "affe::1"
    # add dummy IPv6 address
    dst_iface = get_first_interface(child)
    add_ipv6_address(child, dst_iface, dummy)
    p = srp1(Ether(dst=hw_dst) / IPv6(dst=ll_dst, src=ll_src) /
             IPv6ExtHdrRouting(type=3, segleft=3, addresses=[ll_dst, ll_src,
                                                             dummy]),
             iface=iface, timeout=1, verbose=0)
    assert(p is not None)
    assert(ICMPv6ParamProblem in p)
    assert(p[ICMPv6ParamProblem].code == 0)             # erroneous header field encountered
    assert(p[ICMPv6ParamProblem].ptr == 40+8+(2 * 16))  # dummy in routing header
    pktbuf_empty(child)
    del_ipv6_address(child, dst_iface, dummy)

def test_forward_uncomp(child, iface, hw_dst, ll_dst, ll_src):
    dummy = "affe::1"
    hl = random.randint(2, 255)
    # sniffing for packets to dummy
    sniffer.start_sniff(lambda p: p[Ether].src == hw_dst)
    # add dummy IPv6 address
    dst_iface = get_first_interface(child)
    hw_src = get_host_hwaddr(iface)
    add_neighbor(child, dst_iface, dummy, hw_src)
    sendp(Ether(dst=hw_dst) / IPv6(dst=ll_dst, src=ll_src, hlim=hl) /
          IPv6ExtHdrRouting(type=3, segleft=1, addresses=[dummy]),
          iface=iface, verbose=0)
    ps = sniffer.wait_for_sniff_results()
    p = [p for p in ps if p[Ether].src == hw_dst]
    assert(len(p) > 0)
    p = p[0]
    assert(IPv6 in p)
    assert(IPv6ExtHdrRouting in p)
    assert(p[IPv6].src == ll_src)
    assert(p[IPv6].dst == dummy)
    assert(p[IPv6].hlim == (hl - 1))
    assert(p[IPv6ExtHdrRouting].type == 3)
    assert(p[IPv6ExtHdrRouting].segleft == 0)
    pktbuf_empty(child)
    del_neighbor(child, dst_iface, dummy)

def test_forward_uncomp_not_first_ext_hdr(child, iface, hw_dst, ll_dst, ll_src):
    dummy = "affe::1"
    hl = random.randint(2, 255)
    # sniffing for packets to dummy
    sniffer.start_sniff(lambda p: p[Ether].src == hw_dst)
    # add dummy IPv6 address
    dst_iface = get_first_interface(child)
    hw_src = get_host_hwaddr(iface)
    add_neighbor(child, dst_iface, dummy, hw_src)
    sendp(Ether(dst=hw_dst) / IPv6(dst=ll_dst, src=ll_src, hlim=hl) /
          IPv6ExtHdrHopByHop() /
          IPv6ExtHdrRouting(type=3, segleft=1, addresses=[dummy]),
          iface=iface, verbose=0)
    ps = sniffer.wait_for_sniff_results()
    p = [p for p in ps if p[Ether].src == hw_dst]
    assert(len(p) > 0)
    p = p[0]
    assert(IPv6 in p)
    assert(IPv6ExtHdrRouting in p)
    assert(p[IPv6].src == ll_src)
    assert(p[IPv6].dst == dummy)
    assert(p[IPv6].hlim == (hl - 1))
    assert(p[IPv6ExtHdrRouting].type == 3)
    assert(p[IPv6ExtHdrRouting].segleft == 0)
    pktbuf_empty(child)
    del_neighbor(child, dst_iface, dummy)

def test_seq_left_0(child, iface, hw_dst, ll_dst, ll_src):
    register_protnum(child)
    sendp(Ether(dst=hw_dst) / IPv6(dst=ll_dst, src=ll_src) /
          IPv6ExtHdrRouting(type=3, segleft=0), iface=iface, verbose=0)
    # we are the target, so the packet should be dumped
    # empty snip
    child.expect(r"~~ SNIP  0 - size:\s+0 byte, type: NETTYPE_UNDEF \(\d+\)")
    ipv6_payload_len = 0
    # parsed routing header
    child.expect(r"~~ SNIP  1 - size:\s+(\d+) byte, type: NETTYPE_\w+ \(\d+\)")
    ipv6_payload_len += int(child.match.group(1))
    # NH = 59 (IPV6_NONXT), len = 0x00, routing type = 3, segments left = 0
    child.expect(r"00000000  3B  00  03  00  00  00  00  00")
    # IPv6 header
    child.expect(r"~~ SNIP  2 - size:\s+40 byte, type: NETTYPE_IPV6 \(\d+\)")
    child.expect_exact(r"length: {}  next header: {}".format(
            ipv6_payload_len, EXT_HDR_NH[IPv6ExtHdrRouting]
        ))
    child.expect_exact(r"destination address: {}".format(ll_dst))
    pktbuf_empty(child)
    unregister(child)

def test_time_exc(child, iface, hw_dst, ll_dst, ll_src):
    dummy = "affe::1"
    p = srp1(Ether(dst=hw_dst) / IPv6(dst=ll_dst, hlim=1, src=ll_src) /
             IPv6ExtHdrRouting(type=3, segleft=1, addresses=[dummy]),
             iface=iface, timeout=1, verbose=0)
    assert(p is not None)
    assert(ICMPv6TimeExceeded in p)
    assert(p[ICMPv6TimeExceeded].code == 0)
    pktbuf_empty(child)

def testfunc(child):
    global sniffer
    tap = get_bridge(os.environ["TAP"])
    child.sendline("unittests")
    # wait for and check result of unittests
    print("." * check_unittests(child), end="", flush=True)
    lladdr_src = get_host_lladdr(tap)
    child.sendline("ifconfig")
    child.expect(r"HWaddr: (?P<hwaddr>[A-Fa-f:0-9]+)\s")
    hwaddr_dst = child.match.group("hwaddr").lower()
    child.expect(r"(?P<lladdr>fe80::[A-Fa-f:0-9]+)\s")
    lladdr_dst = child.match.group("lladdr").lower()
    sniffer = Sniffer(tap)

    def run(func):
        if child.logfile == sys.stdout:
            func(child, tap, hwaddr_dst, lladdr_dst, lladdr_src)
        else:
            try:
                func(child, tap, hwaddr_dst, lladdr_dst, lladdr_src)
                print(".", end="", flush=True)
            except Exception as e:
                print("FAILED")
                raise e

    run(test_wrong_type)
    run(test_inconsistent_header)
    run(test_multicast_dst)
    run(test_multicast_addr)
    run(test_multiple_addrs_of_mine_uncomp)
    run(test_forward_uncomp)
    run(test_forward_uncomp_not_first_ext_hdr)
    # compressed tests hard to implement with scapy and also covered in
    # unittests
    run(test_seq_left_0)
    run(test_time_exc)
    print("SUCCESS")

if __name__ == "__main__":
    if os.geteuid() != 0:
        print("\x1b[1;31mThis test requires root privileges.\n"
              "It's constructing and sending Ethernet frames.\x1b[0m\n",
              file=sys.stderr)
        sys.exit(1)
    sys.exit(run(testfunc, timeout=1, echo=False))


import ujson as json
import torch
# from pytorch_transformers import *
import logging
import argparse
from torch.nn.utils.rnn import pad_sequence
import torch.nn.functional as F
import os
import math

import collections
import random
from tqdm import tqdm
# from pytorch_transformers import *
from transformers import *

import pandas as pd
import numpy as np
from sklearn.utils import shuffle

# tools
# from nltk.tokenize import sent_tokenize
# from nltk.tokenize import word_tokenize
# from textblob import TextBlob
# from arglex.Classifier import Classifier

# load pretrained embedding
from big_issue_embedding import big_issue_embedding
from user_aspect_embedding import user_attritbute_embedding

# metrics
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score

# argument lexicon
# arglex = Classifier()

# big issues
BIG_ISSUES = ['Abortion', 'Affirmative Action', 'Animal Rights', 'Barack Obama', 'Border Fence', 'Capitalism', 'Civil Unions', 'Death Penalty', 'Drug Legalization', 'Electoral College', 'Environmental Protection', 'Estate Tax', 'European Union', 'Euthanasia', 'Federal Reserve', 'Flat Tax', 'Free Trade', 'Gay Marriage', 'Global Warming Exists', 'Globalization', 'Gold Standard', 'Gun Rights', 'Homeschooling', 'Internet Censorship', 'Iran-Iraq War', 'Labor Union', 'Legalized Prostitution', 'Medicaid & Medicare', 'Medical Marijuana', 'Military Intervention', 'Minimum Wage', 'National Health Care', 'National Retail Sales Tax', 'Occupy Movement', 'Progressive Tax', 'Racial Profiling', 'Redistribution', 'Smoking Ban', 'Social Programs', 'Social Security', 'Socialism', 'Stimulus Spending', 'Term Limits', 'Torture', 'United Nations', 'War in Afghanistan', 'War on Terror', 'Welfare']
USEFUL_CATS = ['political_ideology', 'education', 'ethnicity', 'interested', 'gender' , 'religious_ideology']
TARGET_LABEL = ["Pro","Con"]

class BertEncoder(BertModel):
    def __init__(self, config):
        super(BertModel, self).__init__(config)
        self.lm = BertModel(config)

    def forward(self, sents):
        # forwarding the sents and use the average embedding as the results

        representation  = self.lm(sents) #.unsqueeze(0)) # num_sent * sent_len * emb
        # print(representation[0].size)
        sent_representation = torch.mean(representation[0], dim=1) # num_sent * emb
        # print(sent_representation.size)
        overall_representation = torch.mean(sent_representation, dim=0) # 1 *  emb

        return overall_representation

class TopicModel(torch.nn.Module):

    def __init__(self, config):
        super(TopicModel, self).__init__()

        self.bert = BertEncoder.from_pretrained(config)
        # self.lstm = LSTMEncoder(300,300)

        self.issue_specific = torch.nn.Linear(10, 10)

        self.text_specific1 = torch.nn.Linear(768*2, 128)
        self.text_specific2 = torch.nn.Linear(128, 128)
        self.text_specific3 = torch.nn.Linear(128, 128)

        self.ling_specific = torch.nn.Linear(12, 10) # linguistic + topical
        self.cat_specific = torch.nn.Linear(len(USEFUL_CATS) * 10 + len(BIG_ISSUES)*10 + 2, 64)
        # 6 * 10 + opinion + similarity

        self.embedding_size = 300
        self.hidden_dim = 200

        self.dropout = torch.nn.Dropout(0.5)
        self.classification = torch.nn.Linear(128+64+10+10, self.hidden_dim) # remaining
        self.third_last_layer = torch.nn.Linear(self.hidden_dim,self.hidden_dim)
        self.second_last_layer = torch.nn.Linear(self.hidden_dim,self.hidden_dim)
        self.last_layer = torch.nn.Linear(self.hidden_dim,2)

    def init_hidden(self):
        return (torch.zeros(1, 1, self.hidden_dim),
                torch.zeros(1, 1, self.hidden_dim))

    def forward(self, cat_vec, ling_vec, tokenized_sents1, tokenized_sents2, issue_emb):
        cat_rep = self.cat_specific(cat_vec)
        ling_rep = self.ling_specific(ling_vec)
        issue_rep = self.issue_specific(issue_emb)

        text_rep1 = self.bert(tokenized_sents1)
        text_rep2 = self.bert(tokenized_sents2)
        text_rep = torch.cat([text_rep1, text_rep2])
        text_rep = self.text_specific1(text_rep)
        text_rep = self.text_specific2(self.text_specific3(text_rep))

        cls_rep = torch.cat([cat_rep, ling_rep, text_rep, issue_rep])
        # three layers NN for the classification module
        cls_rep = self.classification(cls_rep)
        cls_rep = self.second_last_layer(cls_rep)
        pred = self.last_layer(cls_rep)

        # predictions = torch.cat(task_results)

        return pred

class LSTMEncoder(torch.nn.Module):
    def __init__(self, embedding_size, hidden_size):
        super(LSTM, self).__init__()
        self.embedding_size = embedding_size
        self.hidden_dim = hidden_size
        self.lstm = torch.nn.LSTM(self.embedding_size, self.hidden_dim, bidirectional=True)

    def init_hidden(self):
        return (torch.zeros(1, 1, self.hidden_dim),
                torch.zeros(1, 1, self.hidden_dim))

    def forward(self, sents):
        representation, _ = self.lstm(sents)
        sent_representation = torch.mean(sent1_representation, dim=1)
        overall_representation = torch.mean(sent_representation, dim=0) # 1 *  emb

        return overall_representation

class OneHot(TopicModel):
    def __init__(self, config):
        super().__init__(config)

        self.cat_specific = torch.nn.Linear(119+240+2, 64) # remaining

class RemoveDebater(TopicModel):
    def __init__(self, config):
        super().__init__(config)

        self.classification = torch.nn.Linear(128+10+10, self.hidden_dim) # remaining

    def forward(self, cat_vec, ling_vec, tokenized_sents1, tokenized_sents2, issue_emb):
        # cat_rep = self.cat_specific(cat_vec)
        ling_rep = self.ling_specific(ling_vec)
        issue_rep = self.issue_specific(issue_emb)

        text_rep1 = self.bert(tokenized_sents1)
        text_rep2 = self.bert(tokenized_sents2)
        text_rep = torch.cat([text_rep1, text_rep2])
        text_rep = self.text_specific1(text_rep)
        text_rep = self.text_specific2(self.text_specific3(text_rep))

        cls_rep = torch.cat([ling_rep, text_rep, issue_rep])
        # three layers NN for the classification module
        cls_rep = self.classification(cls_rep)
        cls_rep = self.second_last_layer(cls_rep)
        pred = self.last_layer(cls_rep)

        return pred

class RemoveTopic(TopicModel):
    def __init__(self, config):
        super().__init__(config)

        self.text_specific1 = torch.nn.Linear(768, 128)
        self.classification = torch.nn.Linear(128+64+10, self.hidden_dim) # remaining

    def forward(self, cat_vec, ling_vec, tokenized_sents1, tokenized_sents2, issue_emb):
        cat_rep = self.cat_specific(cat_vec)
        ling_rep = self.ling_specific(ling_vec)
        # issue_rep = self.issue_specific(issue_emb)

        text_rep1 = self.bert(tokenized_sents1)

        text_rep = self.text_specific1(text_rep1)
        text_rep = self.text_specific2(self.text_specific3(text_rep))

        cls_rep = torch.cat([cat_rep, ling_rep, text_rep])
        # three layers NN for the classification module
        cls_rep = self.classification(cls_rep)
        cls_rep = self.second_last_layer(cls_rep)
        pred = self.last_layer(cls_rep)

        return pred

class RemoveArgument(TopicModel):
    def __init__(self, config):
        super().__init__(config)

        self.classification = torch.nn.Linear(64+10, self.hidden_dim) # remaining

    def forward(self, cat_vec, ling_vec, tokenized_sents1, tokenized_sents2, issue_emb):
        cat_rep = self.cat_specific(cat_vec)
        # ling_rep = self.ling_specific(ling_vec)
        issue_rep = self.issue_specific(issue_emb)

        cls_rep = torch.cat([cat_rep, issue_rep])
        # three layers NN for the classification module
        cls_rep = self.classification(cls_rep)
        cls_rep = self.second_last_layer(cls_rep)
        pred = self.last_layer(cls_rep)

        return pred

class SBERT(torch.nn.Module):
    def __init__(self, config):
        super(SBERT, self).__init__()

        self.bert = BertEncoder.from_pretrained(config)

        self.issue_specific = torch.nn.Linear(10, 10)

        self.text_specific1 = torch.nn.Linear(768*2, 128)
        self.text_specific2 = torch.nn.Linear(128, 128)
        self.text_specific3 = torch.nn.Linear(128, 128)

        self.ling_specific = torch.nn.Linear(12, 10) # linguistic + topical
        self.cat_specific = torch.nn.Linear(len(USEFUL_CATS) * 10 + len(BIG_ISSUES)*10 + 2, 64)
        # 6 * 10 + opinion + similarity

        self.embedding_size = 300
        self.hidden_dim = 200

        self.dropout = torch.nn.Dropout(0.5)
        self.classification = torch.nn.Linear(128+64+10+10, self.hidden_dim) # remaining
        self.third_last_layer = torch.nn.Linear(self.hidden_dim,self.hidden_dim)
        self.second_last_layer = torch.nn.Linear(self.hidden_dim,self.hidden_dim)
        self.last_layer = torch.nn.Linear(128,2)

    def init_hidden(self):
        return (torch.zeros(1, 1, self.hidden_dim),
                torch.zeros(1, 1, self.hidden_dim))

    def forward(self, cat_vec, ling_vec, tokenized_sents1, tokenized_sents2, issue_emb):

        text_rep1 = self.bert(tokenized_sents1)
        text_rep2 = self.bert(tokenized_sents2)
        text_rep = torch.cat([text_rep1, text_rep2])
        text_rep = self.text_specific1(text_rep)
        text_rep = self.text_specific2(self.text_specific3(text_rep))

        # cls_rep = torch.cat([cat_rep, ling_rep, text_rep, issue_rep])
        # # three layers NN for the classification module
        # cls_rep = self.classification(cls_rep)
        # cls_rep = self.third_last_layer(self.second_last_layer(cls_rep))

        pred = self.last_layer(text_rep)

        # predictions = torch.cat(task_results)

        return pred

class DataLoader:
    def __init__(self, data_path, args):

        self.args = args
        self.split = self.initial_loading(data_path)

        print("Successfully load the debate data")

        self.tokenizer = BertTokenizer.from_pretrained(args.model)
        # self.word_embeddings = self.load_embedding_dict('glove.txt')

        # Load pretrained
        self.issue_emb_dic, self.att_emb_dic = self.load_pretrained_embedding_dict()

        print("Successfully load all the embeddings")

        print("Start processing the data.")
        self.tensor_split = self.tensorize_examples(self.split)

        print("Successfully processed the data.")

    def initial_loading(self, data_path):
        with open(data_path, "r",encoding="UTF-8") as f:
            split = json.load(f)

        return split

    def load_pretrained_embedding_dict(self):

        with open("./big_issue_embedding.json","r",encoding="UTF-8") as f:
            issue_emb_dic = json.load(f)

        with open("./user_attritbute_embedding.json","r",encoding="UTF-8") as f:
            att_emb_dic = json.load(f)

        return issue_emb_dic, att_emb_dic

    def load_embedding_dict(self, path):
        print("Loading word embeddings from {}...".format(path))
        default_embedding = numpy.zeros(300)
        embedding_dict = collections.defaultdict(lambda: default_embedding)
        if len(path) > 0:
            vocab_size = None
            with open(path, 'r', encoding='utf-8') as f:
                for line in f.readlines():
                    word_end = line.find(" ")
                    word = line[:word_end]
                    embedding = numpy.fromstring(line[word_end + 1:], numpy.float32, sep=" ")
                    assert len(embedding) == 300
                    embedding_dict[word] = embedding
            if vocab_size is not None:
                assert vocab_size == len(embedding_dict)
            print("Done loading word embeddings.")
        return embedding_dict

    def tensorize_examples(self, split):
        tensor_split = {}
        tensor_split["train"] = []
        tensor_split["dev"] = []
        tensor_split["test"] = []

        for split_key, _ in split.items():
            for item in tqdm(split[split_key]):

                # categorical attributes
                if self.args.baseline == "full":
                    cat_emb = item["cat_emb"]
                    op_emb = item["op_emb"]
                    cat_sims = []
                    cat_sims.append(item["sims"][1])
                    cat_sims.append(item["sims"][3])

                elif self.args.baseline == "one_hot":

                    # for one hot
                    cat_emb = item["cat_one_hot"] # 119
                    op_emb = item["op_one_hot"]   # 240
                    cat_sims = [] # 2
                    cat_sims.append(item["sims"][0])
                    cat_sims.append(item["sims"][2])

                # linguistic vectors
                linguistic_vector = item["ling_features"]

                # topical features
                topic_vector = item["topic_features"]

                # issue embedding
                issue_emb = item["issue_emb"]

                # tokenized sentences for BERT: 1 - arguments; 2 - title
                tokenized_sents1 = []
                for sent_num, sent in enumerate(item["args"]):

                    if sent_num >= (args.max_sent_num - 1):
                        # limit the number of senteneces
                        break

                    token_sent = self.tokenizer.tokenize('[CLS] ' + sent)
                    if len(token_sent) > (self.args.max_len-1):
                        token_sent = token_sent[:self.args.max_len-1]
                    # elif len(tokenized_sent) < (self.args.max_len-1):
                    #     while len(tokenized_sent) < (self.args.max_len-1):
                    #         tokenized_sent.append(0)
                    token_sent.append(" [SEP]")
                    tokenized_sent = self.tokenizer.convert_tokens_to_ids(token_sent)

                    while len(tokenized_sent) < (self.args.max_len):
                        tokenized_sent.append(0)
                    tokenized_sents1.append(tokenized_sent)

                tokenized_sents2 = []
                tokenized_sent = self.tokenizer.tokenize('[CLS] ' + item["title"])
                if len(token_sent) > (self.args.max_len-1):
                    token_sent = token_sent[:self.args.max_len-1]

                token_sent.append(" [SEP]")
                tokenized_sents2.append(self.tokenizer.convert_tokens_to_ids(token_sent))

                # label
                label = item["label"] # 0/1

                # name of the main issue
                # cat_vec (cat+op+sim), ling_vec(ling+topic), tokenized_sents1, tokenized_sents2, issue_emb
                cat_vec = [] # 361 if onehot
                cat_vec.extend(cat_emb)
                cat_vec.extend(op_emb)
                cat_vec.extend(cat_sims)

                ling_vec = []
                ling_vec.extend(linguistic_vector)
                ling_vec.extend(topic_vector)

                tensor_split[split_key].append(
                {   'cat_vec': torch.tensor(cat_vec).to(device),
                    'ling_vec':torch.tensor(ling_vec).to(device),
                    'tokenized_sents1': torch.tensor(tokenized_sents1).to(device),
                    'tokenized_sents2': torch.tensor(tokenized_sents1).to(device),
                    'label': torch.tensor(label).to(device),
                    'issue_emb': torch.tensor(issue_emb).to(device)
                    })

        return tensor_split

def train(model, data, args, loss_func, optimizer):
    all_loss = 0
    print('training:')
    random.shuffle(data)
    model.train()
    selected_data = data

    # tmp_example
        # {   'cat_vec': torch.tensor(cat_vec).to(device),
        #     'ling_vec':torch.tensor(ling_vector).to(device),
        #     'tokenized_sents1': torch.tensor(tokenized_sents1).to(device),
        #     'tokenized_sents2': torch.tensor(tokenized_sents1).to(device),
        #     'label': torch.tensor(label).to(device),
        #     'issue_emb': torch.tensor(issue_emb).to(device)
        #     })

    for tmp in tqdm(selected_data):
        # cat_vec, ling_vec, tokenized_sents1, tokenized_sents2, issue_emb)
        prediction = model(cat_vec=tmp['cat_vec'].to(device), ling_vec=tmp['ling_vec'].to(device), tokenized_sents1=tmp['tokenized_sents1'].to(device), tokenized_sents2=tmp['tokenized_sents2'].to(device), issue_emb=tmp['issue_emb'].to(device))

        loss = loss_func(prediction.view(1, -1), tmp['label'].unsqueeze(0))

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        all_loss += loss.item()

    print('current loss:', all_loss / len(data))

def test(model, data, args, loss_func, optimizer):
    correct_count = 0

    pred_seq = []
    label_seq = []
    ans_seq = []
    # print('Testing:')
    model.eval()
    for tmp in tqdm(data):
        prediction = model(cat_vec=tmp['cat_vec'].to(device), ling_vec=tmp['ling_vec'].to(device), tokenized_sents1=tmp['tokenized_sents1'].to(device), tokenized_sents2=tmp['tokenized_sents2'].to(device), issue_emb=tmp['issue_emb'].to(device))

        if tmp['label'].item() == 1:
            # current example is positive
            label_seq.append(1)

            if prediction.data[1] >= prediction.data[0]:
                pred_seq.append(1)
                correct_count += 1
                ans_seq.append(1)
            else:
                pred_seq.append(0)
                ans_seq.append(0)
        else:
            # current example is negative
            label_seq.append(0)
            if prediction.data[1] <= prediction.data[0]:
                pred_seq.append(0)
                correct_count += 1
                ans_seq.append(1)
            else:
                pred_seq.append(1)
                ans_seq.append(0)
    # print('current accuracy:', correct_count, '/', len(data), correct_count / len(data))

    acc = accuracy_score(label_seq, pred_seq)
    p = precision_score(label_seq, pred_seq, average='macro')
    r = recall_score(label_seq, pred_seq, average='macro')
    f1 = f1_score(label_seq, pred_seq, average='macro')

    return acc, p, r, f1, label_seq, pred_seq

def train_test_each(args, tensor_split):

    all_devs = []
    print("Start training...")

    # correct_count[issue] = []
    tr_data, val_data, test_data = tensor_split["train"], tensor_split["dev"], tensor_split["test"]

    print("Loading the " + args.baseline + " model...")
    logging.disable(logging.CRITICAL)
    # initialize model
    if args.baseline == "full":
        current_model = TopicModel(args.model)
    elif args.baseline == "sbert":
        current_model = SBERT(args.model)
    elif args.baseline == "no_debater":
        current_model = RemoveDebater(args.model)
    elif args.baseline == "no_argument":
        current_model = RemoveArgument(args.model)
    elif args.baseline == "no_topic":
        current_model = RemoveTopic(args.model)
    elif args.baseline == "one_hot":
        current_model = OneHot(args.model)

    current_model.to(device)

    # continue training
    if args.continue_train:
        try:
            current_model.load_state_dict(torch.load('./best_models/' + args.model + "_" + args.baseline + '.pth'))
        except:
            a = "do nothing"

    optimizer = torch.optim.SGD(current_model.parameters(), lr=args.lr)
    loss_func = torch.nn.CrossEntropyLoss()
    logging.disable(logging.NOTSET)

    best_dev_performance = 0
    for i in range(args.epochs):
        print("========================= epoch " + str(i) + " ==========================")
        train(current_model, tr_data, args, loss_func, optimizer) # save a few checkpoints and pick the best
        acc, p, r, f1, label_seq, pred_seq = test(current_model, val_data, args, loss_func, optimizer)
        print(" Dev Acc, P, R, F1: {}, {}, {}, {}".format(acc, p, r, f1))
        all_devs.append([acc, p, r, f1, label_seq, pred_seq])
        if f1 > best_dev_performance:
            print("New Best F1!!! " + str(f1))
            best_dev_performance = f1
            torch.save(current_model.state_dict(), './best_models_continue/' + args.model + "_" + args.baseline + '.pth')

    current_model.load_state_dict(torch.load('./best_models_continue/' + args.model + "_" + args.baseline + '.pth'))

    acc, p, r, f1, label_seq, pred_seq = test(current_model, test_data, args, loss_func, optimizer)
    all_devs.append([acc, p, r, f1, label_seq, pred_seq])

    with open('./best_models_continue/' + args.model + "_" + args.baseline + '.json', "w", encoding="UTF-8") as f:
        json.dump(all_devs, f)

    print(" Test Acc, P, R, F1: {}, {}, {}, {}".format(acc, p, r, f1))
    return [acc, p, r, f1, label_seq, pred_seq]

parser = argparse.ArgumentParser()

## Required parameters
parser.add_argument("--gpu", default='0', type=str, required=False,
                    help="choose which gpu to use")
parser.add_argument("--seed", default=2020, type=int, required=False,
                    help="default random seed")
parser.add_argument("--model", default='bert-base-uncased', type=str, required=False,
                    help="choose the model to test")
parser.add_argument("--lr", default=0.0005, type=float, required=False,
                    help="initial learning rate")
parser.add_argument("--epochs", default=5, type=int, required=False,
                    help="number of training epochs")
parser.add_argument("--lrdecay", default=0.8, type=float, required=False,
                    help="learning rate decay every 5 epochs")
parser.add_argument("--max_len", default=128, type=int, required=False,
                    help="number of words")
parser.add_argument("--baseline", default="full", type=str, required=False,
                    help="the baseline to test") # full, sbert, no_debater, no_argument, no_topic, one_hot
parser.add_argument("--max_sent_num", default=5, type=int, required=False,
                    help="max number of sentences to process")
parser.add_argument("--continue_train", default=False, type=bool, required=False,
                    help="train with the previous best model")

args = parser.parse_args()

logging.basicConfig(level=logging.INFO)
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print('current device:', device)
n_gpu = torch.cuda.device_count()
print('number of gpu:', n_gpu)
torch.cuda.get_device_name(0)

# set your data path here
if args.baseline == "one_hot":
    data_class = DataLoader('./split.json', args)
    # torch.save(data_class, './data_file.pt')

else:
    print("Loading Cached Weights...")
    data_class = torch.load('./data_file.pt')

# testing
# [acc, p, r, f1, label_seq, pred_seq]
answers = train_test_each(args, data_class.tensor_split)

# io
print('end')