import torch
import torch.nn as nn
from torch.nn import Parameter
import torch.jit as jit
import warnings
from collections import namedtuple
from typing import List, Tuple
from torch import Tensor
import numbers
import warnings
warnings.filterwarnings("ignore", message=r"'layers' was found in ScriptModule constants,  but it is a non-constant submodule. Consider removing it.")

def script_lstm(input_size, hidden_size, num_layers, dropout=False, bidirectional=False):
    '''Returns a ScriptModule that mimics a PyTorch native LSTM.'''

    if bidirectional:
        stack_type = StackedLSTM2
        layer_type = BidirLSTMLayer
        dirs = 2
    elif dropout:
        stack_type = StackedLSTMWithDropout
        layer_type = LSTMLayer
        dirs = 1
    else:
        stack_type = StackedLSTM
        layer_type = LSTMLayer
        dirs = 1

    return stack_type(num_layers, layer_type,
                      first_layer_args=[LSTMCell, input_size, hidden_size],
                      other_layer_args=[LSTMCell, hidden_size * dirs,
                                        hidden_size])

def script_lnlstm(input_size, hidden_size, num_layers, bias=True,
                  batch_first=False, dropout=False, bidirectional=False,
                  decompose_layernorm=False):
    '''Returns a ScriptModule that mimics a PyTorch native LSTM.'''

    # The following are not implemented.
    assert bias
    assert not batch_first
    assert not dropout

    if bidirectional:
        stack_type = StackedLSTM2
        layer_type = BidirLSTMLayer
        dirs = 2
    else:
        stack_type = StackedLSTM
        layer_type = LSTMLayer
        dirs = 1

    return stack_type(num_layers, layer_type,
                      first_layer_args=[LayerNormLSTMCell, input_size, hidden_size,
                                        decompose_layernorm],
                      other_layer_args=[LayerNormLSTMCell, hidden_size * dirs,
                                        hidden_size, decompose_layernorm])

LSTMState = namedtuple('LSTMState', ['hx', 'cx'])

def reverse(lst: List[Tensor]) -> List[Tensor]:
    return lst[::-1]

class LSTMCell(nn.Module):
    def __init__(self, input_size, hidden_size):
        super(LSTMCell, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.weight_ih = Parameter(torch.randn(4 * hidden_size, input_size))
        self.weight_hh = Parameter(torch.randn(4 * hidden_size, hidden_size))
        self.bias_ih = Parameter(torch.randn(4 * hidden_size))
        self.bias_hh = Parameter(torch.randn(4 * hidden_size))

    def forward(self, input: Tensor, state: Tuple[Tensor, Tensor]) -> Tuple[Tensor, Tuple[Tensor, Tensor]]:
        hx, cx = state
        gates = (torch.mm(input, self.weight_ih.t()) + self.bias_ih +
                 torch.mm(hx, self.weight_hh.t()) + self.bias_hh)
        ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)

        ingate = torch.sigmoid(ingate)
        forgetgate = torch.sigmoid(forgetgate)
        cellgate = torch.tanh(cellgate)
        outgate = torch.sigmoid(outgate)

        cy = (forgetgate * cx) + (ingate * cellgate)
        hy = outgate * torch.tanh(cy)

        return hy, (hy, cy)

class LayerNorm(nn.Module):
    def __init__(self, normalized_shape):
        super(LayerNorm, self).__init__()
        if isinstance(normalized_shape, numbers.Integral):
            normalized_shape = (normalized_shape,)
        normalized_shape = torch.Size(normalized_shape)

        # XXX: This is true for our LSTM / NLP use case and helps simplify code
        assert len(normalized_shape) == 1

        self.weight = Parameter(torch.ones(normalized_shape))
        self.bias = Parameter(torch.zeros(normalized_shape))
        self.normalized_shape = normalized_shape

    def compute_layernorm_stats(self, input):
        mu = input.mean(-1, keepdim=True)
        sigma = input.std(-1, keepdim=True, unbiased=False)
        return mu, sigma

    def forward(self, input):
        mu, sigma = self.compute_layernorm_stats(input)
        return (input - mu) / sigma * self.weight + self.bias

class LayerNormLSTMCell(nn.Module):
    def __init__(self, input_size, hidden_size, decompose_layernorm=False):
        super(LayerNormLSTMCell, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.weight_ih = Parameter(torch.randn(4 * hidden_size, input_size))
        self.weight_hh = Parameter(torch.randn(4 * hidden_size, hidden_size))
        # The layernorms provide learnable biases

        if decompose_layernorm:
            ln = LayerNorm
        else:
            ln = nn.LayerNorm

        self.layernorm_i = ln(4 * hidden_size)
        self.layernorm_h = ln(4 * hidden_size)
        self.layernorm_c = ln(hidden_size)

    def forward(self, input: Tensor, state: Tuple[Tensor, Tensor]) -> Tuple[Tensor, Tuple[Tensor, Tensor]]:
        hx, cx = state
        igates = self.layernorm_i(torch.mm(input, self.weight_ih.t()))
        hgates = self.layernorm_h(torch.mm(hx, self.weight_hh.t()))
        gates = igates + hgates
        ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)

        ingate = torch.sigmoid(ingate)
        forgetgate = torch.sigmoid(forgetgate)
        cellgate = torch.tanh(cellgate)
        outgate = torch.sigmoid(outgate)

        cy = self.layernorm_c((forgetgate * cx) + (ingate * cellgate))
        hy = outgate * torch.tanh(cy)

        return hy, (hy, cy)

class LSTMLayer(nn.Module):
    def __init__(self, cell, *cell_args):
        super(LSTMLayer, self).__init__()
        self.cell = cell(*cell_args)

    def forward(self, input: Tensor, state: Tuple[Tensor, Tensor]) -> Tuple[Tensor, Tuple[Tensor, Tensor]]:
        inputs = input.unbind(0)
        outputs = torch.jit.annotate(List[Tensor], [])
        for i in range(len(inputs)):
            out, state = self.cell(inputs[i], state)
            outputs += [out]
        return torch.stack(outputs), state

class ReverseLSTMLayer(nn.Module):
    def __init__(self, cell, *cell_args):
        super(ReverseLSTMLayer, self).__init__()
        self.cell = cell(*cell_args)

    def forward(self, input: Tensor, state: Tuple[Tensor, Tensor]) -> Tuple[Tensor, Tuple[Tensor, Tensor]]:
        inputs = reverse(input.unbind(0))
        outputs = jit.annotate(List[Tensor], [])
        for i in range(len(inputs)):
            out, state = self.cell(inputs[i], state)
            outputs += [out]
        return torch.stack(reverse(outputs)), state

class BidirLSTMLayer(nn.Module):
    __constants__ = ['directions']

    def __init__(self, cell, *cell_args):
        super(BidirLSTMLayer, self).__init__()
        self.directions = nn.ModuleList([
            LSTMLayer(cell, *cell_args),
            ReverseLSTMLayer(cell, *cell_args),
        ])

    def forward(self, input: Tensor, states: List[Tuple[Tensor, Tensor]]) -> Tuple[Tensor, List[Tuple[Tensor, Tensor]]]:
        # List[LSTMState]: [forward LSTMState, backward LSTMState]
        outputs = jit.annotate(List[Tensor], [])
        output_states = jit.annotate(List[Tuple[Tensor, Tensor]], [])
        # XXX: enumerate https://github.com/pytorch/pytorch/issues/14471
        i = 0
        for direction in self.directions:
            state = states[i]
            out, out_state = direction(input, state)
            outputs += [out]
            output_states += [out_state]
            i += 1
        return torch.cat(outputs, -1), output_states

def init_stacked_lstm(num_layers, layer, first_layer_args, other_layer_args):
    layers = [layer(*first_layer_args)] + [layer(*other_layer_args)
                                           for _ in range(num_layers - 1)]
    return nn.ModuleList(layers)

class StackedLSTM(nn.Module):
    __constants__ = ['layers']  # Necessary for iterating through self.layers

    def __init__(self, num_layers, layer, first_layer_args, other_layer_args):
        super(StackedLSTM, self).__init__()
        self.layers = init_stacked_lstm(num_layers, layer, first_layer_args,
                                        other_layer_args)

    def forward(self, input: Tensor, states: List[Tuple[Tensor, Tensor]]) -> Tuple[Tensor, List[Tuple[Tensor, Tensor]]]:
        # List[LSTMState]: One state per layer
        output_states = jit.annotate(List[Tuple[Tensor, Tensor]], [])
        output = input
        # XXX: enumerate https://github.com/pytorch/pytorch/issues/14471
        i = 0
        for rnn_layer in self.layers:
            state = states[i]
            output, out_state = rnn_layer(output, state)
            output_states += [out_state]
            i += 1
        return output, output_states

# Differs from StackedLSTM in that its forward method takes
# List[List[Tuple[Tensor,Tensor]]]. It would be nice to subclass StackedLSTM
# except we don't support overriding script methods.
# https://github.com/pytorch/pytorch/issues/10733
class StackedLSTM2(nn.Module):
    __constants__ = ['layers']  # Necessary for iterating through self.layers

    def __init__(self, num_layers, layer, first_layer_args, other_layer_args):
        super(StackedLSTM2, self).__init__()
        self.layers = init_stacked_lstm(num_layers, layer, first_layer_args,
                                        other_layer_args)

    def forward(self, input: Tensor, states: List[List[Tuple[Tensor, Tensor]]]) -> Tuple[Tensor, List[List[Tuple[Tensor, Tensor]]]]:
        # List[List[LSTMState]]: The outer list is for layers,
        #                        inner list is for directions.
        output_states = jit.annotate(List[List[Tuple[Tensor, Tensor]]], [])
        output = input
        # XXX: enumerate https://github.com/pytorch/pytorch/issues/14471
        i = 0
        for rnn_layer in self.layers:
            state = states[i]
            output, out_state = rnn_layer(output, state)
            output_states += [out_state]
            i += 1
        return output, output_states

class StackedLSTMWithDropout(nn.Module):
    # Necessary for iterating through self.layers and dropout support
    __constants__ = ['layers', 'num_layers']

    def __init__(self, num_layers, layer, first_layer_args, other_layer_args):
        super(StackedLSTMWithDropout, self).__init__()
        self.layers = init_stacked_lstm(num_layers, layer, first_layer_args,
                                        other_layer_args)
        # Introduces a Dropout layer on the outputs of each LSTM layer except
        # the last layer, with dropout probability = 0.4.
        self.num_layers = num_layers

        if (num_layers == 1):
            warnings.warn("dropout lstm adds dropout layers after all but last "
                          "recurrent layer, it expects num_layers greater than "
                          "1, but got num_layers = 1")

        self.dropout_layer = nn.Dropout(0.4)

    def forward(self, input: Tensor, states: List[Tuple[Tensor, Tensor]]) -> Tuple[Tensor, List[Tuple[Tensor, Tensor]]]:
        # List[LSTMState]: One state per layer
        output_states = jit.annotate(List[Tuple[Tensor, Tensor]], [])
        output = input
        # XXX: enumerate https://github.com/pytorch/pytorch/issues/14471
        i = 0
        for rnn_layer in self.layers:
            state = states[i]
            output, out_state = rnn_layer(output, state)
            # Apply the dropout layer except the last layer
            if i < self.num_layers - 1:
                output = self.dropout_layer(output)
            output_states += [out_state]
            i += 1
        return output, output_states

def test_script_stacked_lnlstm(seq_len, batch, input_size, hidden_size,
                               num_layers):
    inp = torch.randn(seq_len, batch, input_size)
    states = [LSTMState(torch.randn(batch, hidden_size),
                        torch.randn(batch, hidden_size))
              for _ in range(num_layers)]
    rnn = script_lnlstm(input_size, hidden_size, num_layers)

    # just a smoke test
    out, out_state = rnn(inp, states)


# write a function to return the duplicates of a list
# 02.01.2021

test_list = [1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 9]
# expected output is [4, 7, 9]

# solution:
# empty list to record the duplicate numbers
# iterate through the list as a set
# if the number appears more than once
# append to extras

def return_dupes(test_list):
    dupes = []
    for x in set(test_list):
        if test_list.count(x) > 1:
            dupes.append(x)
    return print(dupes)


import random
lang = ['p','y','t','h','o','n']
random.shuffle(lang)

new_lang=''.join(lang)
print(new_lang)

char = random.choice(lang)
print(char)

#########
import math
x=3.14
print(math.floor(x))
print(math.ceil(x))

#########
import calculator
x = calculator.calc(3,4)
print(x)

y = calculator.differ(1,2)
print(y)


class DrPose(object):

    def __init__(self, x, y, z=0.0, yaw=0.0, pitch=0.0, roll=0.0):
        # Member
        self.x = x
        self.y = y
        self.z = z
        self.yaw = yaw
        self.pitch = pitch
        self.roll = roll


#编写人：张文君，编写时间5月28号
#对数据进行预处理，pca提取99的特征值，画出散点图。分析数据空间分布的大概

import numpy as np
import scipy as sp
import matplotlib
import matplotlib.pyplot as plt
import sklearn
import IPython
import platform
from sklearn.decomposition import PCA
import pandas as pd
import numpy as np

dataset = pd.read_csv('S1018bf.csv')
dataset = dataset.dropna()
#提取特征和类别
X= dataset.ix[:,'Spectral.Index':'variability_index']
y= dataset.ix[:,'Optical.Class']

y_label=y.copy()

#主成分分析
size_dimention=2
pca=PCA(n_components=size_dimention)     #加载PCA算法，设置降维后主成分数目为
pca.fit(X)
print('方差占比是'+str(pca.explained_variance_ratio_))   #显示特征数值的方差占比，要达到99%以上
reduced_x=pca.fit_transform(X)#对样本进行降维
x_data=reduced_x

#归一化处理和中心化处理
X -= np.mean(X, axis = 0)
X /= np.std(X, axis = 0)

#把类别变成数字
for i in range(1018):
    if y_label[i].strip()=='bll':
        y_label[i]='0'
    else:
        y_label[i]='1'

#看看点的分布图
y_label=np.array(y_label)
x_data=np.array(x_data)
for i in range(1018):
    if y_label[i]=='0':
        plt.scatter(x_data[i,0],x_data[i,1],c="r",label="bell")
    else:
        plt.scatter(x_data[i,0],x_data[i,1],c="b",label="fsrq")

plt.show()


# 1. Write a Python function called add_daily_temp that is given a (possibly empty) dictionary meant to hold the average daily temperature for each day of the week, a temperature value, and the day of the week for the recorded temperature. The function should then add the temperature to the dictionary only if does not already contain a temperature for that day. The function should return the resulting dictionary, whether it is updated or not.

def add_daily_temp(day, temp):
    daily_temp = {'monday': 21}
    if day in daily_temp:
        print(f'already in record, {day}: {daily_temp.get(day)}')
    else:
        daily_temp[day] = temp
    return daily_temp

# one = add_daily_temp('Friday', 23)
# print(one)

# 2. Write a Python function named moderate_days that is given a dictionary containing the average daily temperature for each of the days of a week and returns a list of the days in which the average was between 70 and 79 degrees.

def moderate_days():
    D = {'monday': 121, 'tuesday': 98, 'wednesday':90, 'thursday':65, 'friday':91, 'sturday':80, 'sunday': 72}
    days = [x for x in D if  D.get(x) >= 70 and D.get(x) <= 90]
    return days

# one = moderate_days()
# print(one)

# 3. Write a Python function named get_daily_temps that prompts the user for the average temperature for each day of the week and returns a dictionary containing the information the user entered.

def get_daily_temps():
    dict_week = dict()
    days_of_week = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']

    for x in days_of_week:
        temp = int(input(f'Please enter the temp for {x}: '))
        dict_week[x] = temp
    return dict_week

# one = get_daily_temps()
# print(one)

# 4. Write a Python function named get_weekend_average_temp that is passed a dictionary of daily temperatures and returns the average temperature over the weekend for the weekly temperatures given.

not_weekend = {'monday': 17, 'tuesday': 17, 'wednesday': 14, 'thursday': 13, 'friday': 14}
def get_weekend_average_temp(days):
    total = 0
    for x in days:
        total += days.get(x)
    average = total / len(days)
    return f'the average temp for the weekend is : {round(average, 2)}'

# one = get_weekend_average_temp(not_weekend)
# print(one)

# 5. Write a Python function named add_vegetable that is passed a (possible empty) set of vegetable names and raises a ValueError exception if the given vegetable is already in the set, otherwise, add the vegetable and return a new set.

def add_vegetable(veg):
    set_veg = {'ornage', 'banana', 'kiwi'}
    if veg in set_veg:
        raise Exception (f'{veg} is in the set already!')
    else:
        set_veg.add(veg)
    return set_veg

# one = add_vegetable('grapes')
# print(one)

# 6. Write a Python function named num_vowels_consonants that is passed a string containing letters, each of which may be in either upper or lower case and returns how many vowels and consonants the string contains.

def num_vowels_consonants(something):
    num_vowels = 0
    num_consonants = 0
    vowels = ['a', 'e', 'i', 'o', 'u']

    for x in something.lower():
        if x in vowels:
            num_vowels += 1
        else:
            num_consonants += 1
    return (f'there is {num_vowels} vowels and {num_consonants} consonants in {something}')

# one = num_vowels_consonants('mAnsoud MANsouri')
# print(one)


import datetime
from model import Config

def check_time(config: Config, time: datetime.datetime, check_period: datetime.timedelta):
    time_array = config.schedules.split(',')
    runs = []
    for i in time_array:
        hour, minutes = map(int, i.split(':'))
        time_for_check = datetime.datetime(
            year=time.year, month=time.month,
            day=time.day, hour=hour, minute=minutes)
        runs.append(time_for_check)

    for i in runs:
        if i <= time < i+check_period:
            return True
    return False


"""Data Module"""
from __future__ import division
from collections import defaultdict
import numpy as np
#from time import time
#import random

class DataModule():
    """Data Module Class"""
    def __init__(self, conf, filename):
        self.conf = conf
        self.data_dict = {}
        self.terminal_flag = 1
        self.filename = filename
        self.index = 0

    def Preparemodelsupplement(self):
        """Prepare data"""
        data_dict = {}
        self.Arrangepositivedata()
        self.Generateconsumeditemssparsematrix()
        data_dict['CONSUMED_ITEMS_INDICES_INPUT'] = self.consumed_items_indices_list
        data_dict['CONSUMED_ITEMS_VALUES_INPUT'] = self.consumed_items_values_list

        self.Generateconsumeditemssparsematrixforitemuser()
        data_dict['ITEM_CUSTOMER_INDICES_INPUT'] = self.item_customer_indices_list
        data_dict['ITEM_CUSTOMER_VALUES_INPUT'] = self.item_customer_values_list

        self.Readsocialneighbors()
        self.Generatesocialneighborssparsematrix()
        data_dict['SOCIAL_NEIGHBORS_INDICES_INPUT'] = self.social_neighbors_indices_list
        data_dict['SOCIAL_NEIGHBORS_VALUES_INPUT'] = self.social_neighbors_values_list
        return data_dict

    def Initializefeaturetrainforuser(self):
        """Initialize train feature for user"""
        self.Readuserfeature()
        self.Arrangepositivefeatureforuser()

    def Initializefeaturetrainforitem(self):
        """Initialize train feature for item"""
        self.Readitemfeature()
        self.Arrangepositivefeatureforitem()

    def Initializerankingtrain(self):
        """Initialize training set"""
        self.Readdata()
        self.Arrangepositivedata()
        self.Arrangepositivedataforitemuser()  # add
        self.Generatetrainnegative()

    def Initializerankingvt(self):
        """Initialize validation set"""
        self.Readdata()
        self.Arrangepositivedata()
        self.Arrangepositivedataforitemuser()  # add
        self.Generatetrainnegative()

    def Initalizerankingeva(self):
        """Initialize test set"""
        self.Readdata()
        self.Getevapositivebatch()
        self.Generateevanegative()

    def Linkedmap(self):
        """Data mapping"""
        self.data_dict['USER_LIST'] = self.user_list
        self.data_dict['ITEM_LIST'] = self.item_list
        self.data_dict['LABEL_LIST'] = self.labels_list
        self.data_dict['CONSUMED_ITEMS_INDICES_INPUT'] = self.consumed_items_indices_list
        self.data_dict['CONSUMED_ITEMS_VALUES_INPUT'] = self.consumed_items_values_list
        self.data_dict['ITEM_CUSTOMER_INDICES_INPUT'] = self.item_customer_indices_list
        self.data_dict['ITEM_CUSTOMER_VALUES_INPUT'] = self.item_customer_values_list
        self.data_dict['SOCIAL_NEIGHBORS_INDICES_INPUT'] = self.social_neighbors_indices_list
        self.data_dict['SOCIAL_NEIGHBORS_VALUES_INPUT'] = self.social_neighbors_values_list

    def Linkedrankingevamap(self):
        """s1 data mapping"""
        self.data_dict['EVA_USER_LIST'] = self.eva_user_list
        self.data_dict['EVA_ITEM_LIST'] = self.eva_item_list

    def Linkedrankingevamaps4(self):
        """s4 data mapping"""
        self.data_dict['s4_EVA_USER_LIST'] = self.s4_eva_user_list
        self.data_dict['s4_EVA_ITEM_LIST'] = self.s4_eva_item_list

###########################################  Ranking ############################################

    def Readuserfeature(self):
        """Read user feature data"""
        f = open(self.filename)
        total_user_feature_list = set()
        user_feature_hash_data = defaultdict(int)
        for _, line in enumerate(f):
            arr = line.split(" ")
            user_feature_hash_data[(int(arr[0]), int(arr[1]))] = 1
            total_user_feature_list.add(int(arr[0]))
        self.user_feature_hash_data = user_feature_hash_data
        self.total_user_feature_list = list(total_user_feature_list)

    def Readuserfeatures4(self):
        """Read user feature data s4"""
        f = open(self.filename)
        total_user_feature_list = set()
        user_feature_hash_data = defaultdict(int)
        for _, line in enumerate(f):
            arr = line.split(" ")
            user_feature_hash_data[(int(arr[0]), int(arr[1]))] = 1
            total_user_feature_list.add(int(arr[0]))
        self.s4_user_feature_hash_data = user_feature_hash_data
        self.s4_total_user_feature_list = list(total_user_feature_list)

    def Readitemfeature(self):
        """Read item feature data"""
        f = open(self.filename)
        total_item_feature_list = set()
        item_feature_hash_data = defaultdict(int)
        for _, line in enumerate(f):
            arr = line.split(" ")
            item_feature_hash_data[(int(arr[0]), int(arr[1]))] = 1
            total_item_feature_list.add(int(arr[0]))
        self.total_item_feature_list = list(total_item_feature_list)
        self.item_feature_hash_data = item_feature_hash_data

    def Readitemfeatures4(self):
        """Read item feature data s4"""
        f = open(self.filename)
        total_item_feature_list = set()
        item_feature_hash_data = defaultdict(int)
        for _, line in enumerate(f):
            arr = line.split(" ")
            item_feature_hash_data[(int(arr[0]), int(arr[1]))] = 1
            total_item_feature_list.add(int(arr[0]))
        self.s4_total_item_feature_list = list(total_item_feature_list)
        self.s4_item_feature_hash_data = item_feature_hash_data

    def Readdata(self):
        """Read data"""
        f = open(self.filename)
        total_user_list = set()
        hash_data = defaultdict(int)  # hash_data = 0
        for i, line in enumerate(f):
            if i == 0:
                continue
            arr = line.split(" ")
            hash_data[(int(arr[0]), int(arr[1]))] = 1
            total_user_list.add(int(arr[0]))
        self.total_user_list = list(total_user_list)
        self.hash_data = hash_data

    def Arrangepositivedata(self):
        """Arrange positive data"""
        positive_data = defaultdict(set)
        user_item_num_dict = defaultdict(set)
        total_data = set()
        hash_data = self.hash_data
        for (u, i) in hash_data:
            total_data.add((u, i))
            positive_data[u].add(i)

        for u in range(self.conf.num_users):
            user_item_num_dict[u] = len(positive_data[u])+1
        self.positive_data = positive_data
        self.user_item_num_dict = user_item_num_dict
        self.total_data = len(total_data)

    def Arrangepositivefeatureforuser(self):
        """Arrange positive feature for user"""
        positive_user_feature = defaultdict(set)
        user_feature_num_dict = defaultdict(set)
        total_user_feature = set()
        user_feature_hash_data = self.user_feature_hash_data
        for (u, i) in user_feature_hash_data:
            total_user_feature.add((u, i))
            positive_user_feature[u].add(i)

        for u in positive_user_feature:
            user_feature_num_dict[u] = len(positive_user_feature[u])
        self.positive_user_feature = positive_user_feature
        self.total_user_feature = len(total_user_feature)
        self.user_feature_num_dict = user_feature_num_dict

    def Arrangepositivefeatureforitem(self):
        """Arrange positive feature for item"""
        positive_item_feature = defaultdict(set)
        item_feature_num_dict = defaultdict(set)
        total_item_feature = set()
        item_feature_hash_data = self.item_feature_hash_data
        for (u, i) in item_feature_hash_data:
            total_item_feature.add((u, i))
            positive_item_feature[u].add(i)

        for u in positive_item_feature:
            item_feature_num_dict[u] = len(positive_item_feature[u])
        self.positive_item_feature = positive_item_feature
        self.total_item_feature = len(total_item_feature)
        self.item_feature_num_dict = item_feature_num_dict

    def Arrangepositivedataforitemuser(self):
        """Arrange positive feature for item user"""
        positive_data_for_item_user = defaultdict(set)
        item_user_num_dict = defaultdict(set)

        total_data_for_item_user = set()
        hash_data_for_item_user = self.hash_data
        for (u, i) in hash_data_for_item_user:
            total_data_for_item_user.add((i, u))
            positive_data_for_item_user[i].add(u)

        for i in range(self.conf.num_items):
            item_user_num_dict[i] = len(positive_data_for_item_user[i]) + 1

        self.item_user_num_dict = item_user_num_dict
        self.positive_data_for_item_user = positive_data_for_item_user
        self.total_data_for_item_user = len(total_data_for_item_user)

    def Generatetrainnegative(self):
        """Generate training negative samples"""
        num_items = self.conf.num_items
        num_negatives = self.conf.num_negatives
        negative_data = defaultdict(set)
        total_data = set()
        hash_data = self.hash_data
        for (u, i) in hash_data:
            total_data.add((u, i))
            for _ in range(num_negatives):
                j = np.random.randint(num_items)
                while (u, j) in hash_data:
                    j = np.random.randint(num_items)
                negative_data[u].add(j)
                total_data.add((u, j))
        self.negative_data = negative_data
        self.terminal_flag = 1

    def Getvtrankingonebatch(self):
        """Get batch data for training or testing"""
        positive_data = self.positive_data
        negative_data = self.negative_data
        total_user_list = self.total_user_list
        user_list = []
        item_list = []
        labels_list = []
        for u in total_user_list:
            user_list.extend([u] * len(positive_data[u]))
            item_list.extend(positive_data[u])
            labels_list.extend([1] * len(positive_data[u]))
            user_list.extend([u] * len(negative_data[u]))
            item_list.extend(negative_data[u])
            labels_list.extend([0] * len(negative_data[u]))

        self.user_list = np.reshape(user_list, [-1, 1])
        self.item_list = np.reshape(item_list, [-1, 1])
        self.labels_list = np.reshape(labels_list, [-1, 1])

    def Gettrainrankingbatch(self):
        """Get batch data for training or testing"""
        positive_data = self.positive_data
        negative_data = self.negative_data
        total_user_list = self.total_user_list
        index = self.index
        batch_size = self.conf.training_batch_size

        user_list, item_list, labels_list = [], [], []

        if index + batch_size < len(total_user_list):
            target_user_list = total_user_list[index:index+batch_size]
            self.index = index + batch_size
        else:
            target_user_list = total_user_list[index:len(total_user_list)]
            self.index = 0
            self.terminal_flag = 0

        for u in target_user_list:
            user_list.extend([u] * len(positive_data[u]))
            item_list.extend(list(positive_data[u]))
            labels_list.extend([1] * len(positive_data[u]))
            user_list.extend([u] * len(negative_data[u]))
            item_list.extend(list(negative_data[u]))
            labels_list.extend([0] * len(negative_data[u]))

        self.user_list = np.reshape(user_list, [-1, 1])
        self.item_list = np.reshape(item_list, [-1, 1])
        self.labels_list = np.reshape(labels_list, [-1, 1])

    def Getevapositivebatch(self):
        """Get test positive batch data"""
        hash_data = self.hash_data
        user_list = []
        item_list = []
        index_dict = defaultdict(list)
        index = 0
        for (u, i) in hash_data:
            user_list.append(u)
            item_list.append(i)
            index_dict[u].append(index)
            index = index + 1
        self.eva_user_list = np.reshape(user_list, [-1, 1])
        self.eva_item_list = np.reshape(item_list, [-1, 1])
        self.eva_index_dict = index_dict

    def Generateevanegative(self):
        """Generate test negative samples"""
        hash_data = self.hash_data
        total_user_list = self.total_user_list
        num_evaluate = self.conf.num_evaluate
        num_items = self.conf.num_items
        eva_negative_data = defaultdict(list)
        for u in total_user_list:
            for _ in range(num_evaluate):
                j = np.random.randint(num_items)
                while (u, j) in hash_data:
                    j = np.random.randint(num_items)
                eva_negative_data[u].append(j)
        self.eva_negative_data = eva_negative_data

    def Getevarankingbatch(self):
        """Get Eva batch data"""
        batch_size = self.conf.evaluate_batch_size
        num_evaluate = self.conf.num_evaluate
        eva_negative_data = self.eva_negative_data
        total_user_list = self.total_user_list
        index = self.index
        terminal_flag = 1
        total_users = len(total_user_list)
        user_list = []
        item_list = []
        if index + batch_size < total_users:
            batch_user_list = total_user_list[index:index+batch_size]
            self.index = index + batch_size
        else:
            terminal_flag = 0
            batch_user_list = total_user_list[index:total_users]
            self.index = 0
        for u in batch_user_list:
            user_list.extend([u]*num_evaluate)
            item_list.extend(eva_negative_data[u])
        self.eva_user_list = np.reshape(user_list, [-1, 1])
        self.eva_item_list = np.reshape(item_list, [-1, 1])
        return batch_user_list, terminal_flag

    #Supplement for Sparse Computation
    def Readsocialneighbors(self, friends_flag=1):
        """Read social neighbors"""
        social_neighbors = defaultdict(set)
        social_neighbors_num_dict = defaultdict(int)

        links_file = open(self.conf.links_filename)
        for _, line in enumerate(links_file):
            tmp = line.split(' ')
            u1, u2 = int(tmp[0]), int(tmp[1])
            social_neighbors[u1].add(u2)
            if friends_flag == 1:
                social_neighbors[u2].add(u1)

        for u in range(self.conf.num_users):
            social_neighbors_num_dict[u] = len(social_neighbors[u])+1

        self.social_neighbors_num_dict = social_neighbors_num_dict
        self.social_neighbors = social_neighbors

    def Generatesocialneighborssparsematrix(self):
        """Generate Social Neighbors Sparse Matrix Indices and Values"""
        social_neighbors = self.social_neighbors
        social_neighbors_num_dict = self.social_neighbors_num_dict  # weight avg

        social_neighbors_indices_list = []
        social_neighbors_values_list = []
        social_neighbors_values_weight_avg_list = []
        social_neighbor_num_list = []
        social_neighbors_dict = defaultdict(list)

        user_user_num_for_sparsity_dict = defaultdict(set)
        user_user_sparsity_dict = {}

        user_user_sparsity_dict['0-4'] = []
        user_user_sparsity_dict['4-8'] = []
        user_user_sparsity_dict['8-16'] = []
        user_user_sparsity_dict['16-32'] = []
        user_user_sparsity_dict['32-64'] = []
        user_user_sparsity_dict['64-'] = []

        for u in range(self.conf.num_users):
            user_user_num_for_sparsity_dict[u] = len(social_neighbors[u])

        for u in social_neighbors:
            social_neighbors_dict[u] = sorted(social_neighbors[u])

        user_list = sorted(list(social_neighbors.keys()))

        for user in range(self.conf.num_users):
            if user in social_neighbors_dict:
                social_neighbor_num_list.append(
                    len(social_neighbors_dict[user]))
            else:
                social_neighbor_num_list.append(1)

        for user in user_list:
            for friend in social_neighbors_dict[user]:
                social_neighbors_indices_list.append([user, friend])
                social_neighbors_values_list.append(
                    1.0/len(social_neighbors_dict[user]))
                social_neighbors_values_weight_avg_list.append(
                    1.0/(np.sqrt(social_neighbors_num_dict[user])* \
                    np.sqrt(social_neighbors_num_dict[friend])))  # weight avg

        for u in range(self.conf.num_users):
            cur_user_neighbors_num = user_user_num_for_sparsity_dict[u]
            if (cur_user_neighbors_num >= 0) & (cur_user_neighbors_num < 4):
                user_user_sparsity_dict['0-4'].append(u)
            elif (cur_user_neighbors_num >= 4) & (cur_user_neighbors_num < 8):
                user_user_sparsity_dict['4-8'].append(u)
            elif (cur_user_neighbors_num >= 8) & (cur_user_neighbors_num < 16):
                user_user_sparsity_dict['8-16'].append(u)
            elif (cur_user_neighbors_num >= 16) & (cur_user_neighbors_num < 32):
                user_user_sparsity_dict['16-32'].append(u)
            elif (cur_user_neighbors_num >= 32) & (cur_user_neighbors_num < 64):
                user_user_sparsity_dict['32-64'].append(u)
            elif cur_user_neighbors_num >= 64:
                user_user_sparsity_dict['64-'].append(u)

        self.user_user_sparsity_dict = user_user_sparsity_dict

        self.social_neighbors_indices_list = np.array(
            social_neighbors_indices_list).astype(np.int64)
        self.social_neighbors_values_list = np.array(
            social_neighbors_values_list).astype(np.float32)
        self.social_neighbors_values_weight_avg_list = np.array(
            social_neighbors_values_weight_avg_list).astype(np.float32)   # weight avg
        self.social_neighbor_num_list = np.array(
            social_neighbor_num_list).astype(np.int64)

    def Generateconsumeditemssparsematrix(self):
        """Generate Consumed Items Sparse Matrix Indices and Values"""
        positive_data = self.positive_data
        consumed_items_indices_list = []
        consumed_items_values_list = []
        consumed_item_num_list = []
        consumed_items_dict = defaultdict(list)
        user_item_num_for_sparsity_dict = defaultdict(set)
        user_item_sparsity_dict = {}
        user_item_sparsity_dict['0-4'] = []
        user_item_sparsity_dict['4-8'] = []
        user_item_sparsity_dict['8-16'] = []
        user_item_sparsity_dict['16-32'] = []
        user_item_sparsity_dict['32-64'] = []
        user_item_sparsity_dict['64-'] = []

        for u in positive_data:
            consumed_items_dict[u] = sorted(positive_data[u])

        user_list = sorted(list(positive_data.keys()))

        for u in range(self.conf.num_users):
            user_item_num_for_sparsity_dict[u] = len(positive_data[u])

        for user in range(self.conf.num_users):
            if user in consumed_items_dict:
                consumed_item_num_list.append(len(consumed_items_dict[user]))
            else:
                consumed_item_num_list.append(1)

        for u in user_list:
            for i in consumed_items_dict[u]:
                consumed_items_indices_list.append([u, i])
                consumed_items_values_list.append(
                    1.0/len(consumed_items_dict[u]))

        for u in range(self.conf.num_users):
            cur_user_consumed_item_num = user_item_num_for_sparsity_dict[u]
            if (cur_user_consumed_item_num >= 0) & (cur_user_consumed_item_num < 4):
                user_item_sparsity_dict['0-4'].append(u)
            elif (cur_user_consumed_item_num >= 4) & (cur_user_consumed_item_num < 8):
                user_item_sparsity_dict['4-8'].append(u)
            elif (cur_user_consumed_item_num >= 8) & (cur_user_consumed_item_num < 16):
                user_item_sparsity_dict['8-16'].append(u)
            elif (cur_user_consumed_item_num >= 16) & (cur_user_consumed_item_num < 32):
                user_item_sparsity_dict['16-32'].append(u)
            elif (cur_user_consumed_item_num >= 32) & (cur_user_consumed_item_num < 64):
                user_item_sparsity_dict['32-64'].append(u)
            elif cur_user_consumed_item_num >= 64:
                user_item_sparsity_dict['64-'].append(u)

        self.user_item_sparsity_dict = user_item_sparsity_dict
        self.consumed_items_indices_list = np.array(
            consumed_items_indices_list).astype(np.int64)
        self.consumed_items_values_list = np.array(
            consumed_items_values_list).astype(np.float32)
        self.consumed_item_num_list = np.array(
            consumed_item_num_list).astype(np.int64)

    def Generateconsumeditemssparsematrixforitemuser(self):
        """Generate Consumed Items Sparse Matrix Indices and Values for item user"""
        positive_data_for_item_user = self.positive_data_for_item_user
        item_customer_indices_list = []
        item_customer_values_list = []
        item_customer_num_list = []
        item_customer_dict = defaultdict(list)

        for i in positive_data_for_item_user:
            item_customer_dict[i] = sorted(positive_data_for_item_user[i])
        item_list = sorted(list(positive_data_for_item_user.keys()))

        for item in range(self.conf.num_items):
            if item in item_customer_dict:
                item_customer_num_list.append(len(item_customer_dict[item]))
            else:
                item_customer_num_list.append(1)

        for i in item_list:
            for u in item_customer_dict[i]:
                item_customer_indices_list.append([i, u])
                item_customer_values_list.append(
                    1.0/len(item_customer_dict[i]))

        self.item_customer_indices_list = np.array(
            item_customer_indices_list).astype(np.int64)
        self.item_customer_values_list = np.array(
            item_customer_values_list).astype(np.float32)
        self.item_customer_num_list = np.array(
            item_customer_num_list).astype(np.int64)

    def Generatefeaturesparsematrixforuser(self):
        """Generate feature sparse matrix for user"""
        positive_feature_for_user = self.positive_user_feature
        user_feature_indices_list = []
        user_feature_values_list = []
        #user_feature_values_weight_avg_list = []
        user_feature_num_list = []
        user_feature_dict = defaultdict(list)
        #user_feature_num_dict = self.user_feature_num_dict  # weight avg
        for i in positive_feature_for_user:
            user_feature_dict[i] = sorted(positive_feature_for_user[i])
        user_list = sorted(list(positive_feature_for_user.keys()))

        for u in user_list:
            for f in user_feature_dict[u]:
                user_feature_indices_list.append([u, f])
                user_feature_values_list.append(1.0/len(user_feature_dict[u]))

        self.user_feature_indices_list = np.array(
            user_feature_indices_list).astype(np.int64)
        self.cur_user_feature_user_num = len(user_feature_dict)
        self.user_feature_values_list = np.array(
            user_feature_values_list).astype(np.float32)
        self.user_feature_num_list = np.array(
            user_feature_num_list).astype(np.int64)

    def Generatefeaturesparsematrixforitem(self):
        """Generate feature sparse matrix for item"""
        positive_feature_for_item = self.positive_item_feature
        item_feature_indices_list = []
        item_feature_values_list = []
        #item_feature_values_weight_avg_list = []
        item_feature_num_list = []
        item_feature_dict = defaultdict(list)
        #item_feature_num_dict = self.item_feature_num_dict  # weight avg

        for i in positive_feature_for_item:
            item_feature_dict[i] = sorted(positive_feature_for_item[i])
        item_list = sorted(list(positive_feature_for_item.keys()))

        for v in item_list:
            for f in item_feature_dict[v]:
                item_feature_indices_list.append([v, f])
                item_feature_values_list.append(1.0/len(item_feature_dict[v]))

        self.item_feature_indices_list = np.array(
            item_feature_indices_list).astype(np.int64)
        self.cur_item_feature_item_num = len(item_feature_dict)
        self.item_feature_values_list = np.array(
            item_feature_values_list).astype(np.float32)
        self.item_feature_num_list = np.array(
            item_feature_num_list).astype(np.int64)


def syn_resolve_references(json_schema):

    """
    Function: syn_resolve_references

    Purpose: Resolves $ref statements referring to schemas registered in
             Synapse.

    Arguments: JSON schema to be de-referenced

    Returns: A de-referenced JSON schema in dictionary form

    """

    import jsonschema

    # The full URL is defined in the Synapse docs.
    syn_url = "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered"

    ref_resolver = jsonschema.RefResolver.from_schema(json_schema)

    for schema_key in json_schema["properties"]:
        if "$ref" in json_schema["properties"][schema_key]:
            ref_url = f"{syn_url}/{json_schema['properties'][schema_key]['$ref']}"
            deref_object = ref_resolver.resolve(ref_url)
            json_schema["properties"][schema_key] = deref_object[1]

    return(json_schema)


import os
from tkinter import Tk, Label, Button
import tkinter.font as font
import tkinter.simpledialog as simpledialog
from PIL import Image, ImageTk
from tkinter import messagebox as mbox
import pyttsx3
import cv2
import face_recognition
from numpy import info
from sklearn import svm
import datetime

def test_svm():

    # Training the SVC classifier

    # The training data would be all the face encodings from all the known images and the labels are their names
    encodings = []
    names = []

    engine = pyttsx3.init()
    voices = engine.getProperty('voices')
    engine.setProperty('voice', voices[1].id)
    engine.setProperty('rate', 150)

    train_dir = os.listdir('train/')

    for person in train_dir:
        pix = os.listdir("train/" + person)

        # Loop through each training image for the current person
        for person_img in pix:
            # Get the face encodings for the face in each image file
            face = face_recognition.load_image_file(
                "train/" + person + "/" + person_img)
            face_bounding_boxes = face_recognition.face_locations(face)

            # If training image contains exactly one face
            if len(face_bounding_boxes) == 1:
                face_enc = face_recognition.face_encodings(face)[0]
                # Add face encoding for current image with corresponding label (name) to the training data
                encodings.append(face_enc)
                names.append(person)
            else:

                print(person + "/" + person_img +" was skipped and can't be used for training")
                os.remove("train/" + person + "/" + person_img)

    # Create and train the SVC classifier
    clf = svm.SVC(gamma='scale')
    clf.fit(encodings, names)

    # menghitung acurasi
    accuracy = clf.score(encodings, names)
    print("Accuracy:", accuracy)

    # Load the test image with unknown faces into a numpy array
    test_image = face_recognition.load_image_file('test/test.jpg')

    # Find all the faces in the test image using the default HOG-based model
    face_locations = face_recognition.face_locations(test_image)
    num = len(face_locations)
    print("Number of faces detected: ", num)

    list_names = []
    if num == 0:
        text = "detection face is failed cause nothing face in the picture.  please give me a picture, with the face"
        engine.say(text)
        engine.runAndWait()
        print("Face Not Found")
    else :

        print("Found:")
        for i in range(num):
            test_image_enc = face_recognition.face_encodings(test_image)[i]
            name = clf.predict([test_image_enc])

        if name[0] in names:
            text = "detection is successfuly. i detected a face, and i recognized him, and he was named" + name[0]
            engine.say(text)
            engine.runAndWait()
            print(name)
            list_names.append(*name)
        else:
            print("Undefined Faces")

    # Predict all the faces in the test image using the trained classifier

    display_name(list_names)
    engine.runAndWait()

def test_img_capture():

    window.destroy()

    img_counter = 0

    if(True):
        cam = cv2.VideoCapture(0)
        cv2.namedWindow("Face Recognition")
        while True:
            ret, frame = cam.read()
            if not ret:
                print("failed to grab frame")
                break

            frame = cv2.flip(frame, 1)
            cv2.imshow("Face Recognition", frame)

            k = cv2.waitKey(1)
            if k % 256 == 32:
                # SPACE pressed
                img_name = "test.jpg"
                cv2.imwrite(os.path.join('test', img_name), frame)
                print("{} written!".format(img_name))
                print("Closing now")
                img_counter += 1
                break
    cam.release()
    cv2.destroyAllWindows()
    test_svm()

def train_img_capture():

    cam = cv2.VideoCapture(0)

    cv2.namedWindow("Face Training")
    img_counter = 0

    file_name = ''
    file_name= simpledialog.askstring(title="Face Recognition",prompt="What's your Name?:")
    window = Tk()
    window.withdraw()

    os.mkdir("train/"+file_name)

    while True:
        ret, frame = cam.read()
        if not ret:
            print("failed to grab frame")
            break

        frame = cv2.flip(frame, 1)
        cv2.imshow("Face Training", frame)

        k = cv2.waitKey(1)
        if k%256 == 27:
            # ESC pressed
            print("Escape hit, closing...")
            break
        elif k%256 == 32:
            # SPACE pressed
            img_name = file_name+"_{}.jpg".format(img_counter)
            cv2.imwrite(os.path.join("train/"+file_name, img_name), frame)
            print("{} written!".format(img_name))
            img_counter += 1

    cam.release()

    cv2.destroyAllWindows()
    window.destroy()

def display_name(list_name):
    window=Tk()
    label = Label(window, text="Faces Recognized")
    listbox = Listbox(window, width=50)
    label.pack()
    listbox.pack(fill=BOTH, expand=1)  # adds listbox to window
    for row in list_name:
        listbox.insert(END, row)   # one line for loop
    window.mainloop()

engine = pyttsx3.init()
voices = engine.getProperty('voices')
engine.setProperty('voice', voices[1].id)
engine.setProperty('rate', 180)

waktu = datetime.datetime.now()
jam = waktu.hour

if jam >= 1 and jam <= 9:
    hour = "Morning"
elif jam > 9 and jam <= 15:
    hour = "Afternoon"
elif jam > 15 and jam <= 22:
    hour = "Evening"
else :
    hour = "night"

text = "hello sir, good" +  hour +  ", how are you today?. welcome to the face detection program. display screen now"
engine.say(text)
engine.runAndWait()

window=Tk()
window.config(width=300, height=300,padx=20,pady=50)
label = Label(
    window, text='WELCOME TO FACE RECOGNITION. SELECT AN OPTION BELOW:\n',font=font.Font(size=16))
label.pack()
button = Button(window, text="Train",command=train_img_capture,width=20,bg="red",fg="white",pady=10)
button['font']=font.Font(size=16)
button.pack()
label = Label(window, text='\n')
label.pack()
button = Button(window, text="Test", command=test_img_capture,width=20,bg="#0052cc",fg="white",pady=10)
button['font']=font.Font(size=16)
button.pack()
label=Label(window,text="\nInstructions\n1).In Train Mode, enter your name and then press SPACEBAR to capture images. Hit ESC when done.\n2).In Test Mode, press SPACEBAR to capture image and display detected face",font=font.Font(size=14))
label.pack()
window.mainloop()

text = "thanks very much,  for using this program. don't forget to try again. see you."
engine.say(text)
engine.runAndWait()

Face-Recognition-Using-SVM/Face Recognition using SVM.py


#%%
import os
from selenium import webdriver
from bs4 import BeautifulSoup
import re
import argparse
import time
import csv
import pandas as pd
from pprint import pprint

#%%
#game_sites = 'https://www.metacritic.com/browse/' + genres + scores_by + '/year/all/filtered'

if not os.path.exists('./metacritic_review'):
    os.makedirs('./metacritic_review')

#%%
def hasXpath(xpath):
    try:
        driver.find_element_by_xpath(xpath)
        return True
    except:
        return False

#%%

def metacritic_review(genre_name):
    os.chdir('/Users/hyunjaecho/Documents/GitHub/Team-Amazing')
    driver=webdriver.Chrome(executable_path=r"/Users/HyunjaeCho/Documents/Fall2020/DataMining/chromedriver")
    #genre_type = re.findall(r'\w+', str(genre_name))
    genre_type = str(genre_name).replace("['", "")
    genre_type = str(genre_type).replace("']", "")
    genre_type = str(genre_type).replace(",", "")
    genre_type = str(genre_type).replace("('", "")
    genre_type = str(genre_type).replace("')", "")
    site = 'https://www.metacritic.com/browse/games/genre/metascore/' + str(genre_type) + '/all?view=detailed'
    driver.get(site)
    driver.implicitly_wait(10)
    print(site)
    html = driver.page_source
    bsobj = BeautifulSoup(html, 'html.parser')
    last_page = bsobj.find('li', {'class': re.compile('page last_page.*')}).text
    last_page = re.findall('\d+', str(last_page))

    titles = []
    platforms = []
    scores = []
    genres = []
    total_df = pd.DataFrame([])
    for page in range(0, int(last_page[0])):
        os.chdir('/Users/hyunjaecho/Documents/GitHub/Team-Amazing')
        driver=webdriver.Chrome(executable_path=r"/Users/HyunjaeCho/Documents/Fall2020/DataMining/chromedriver")
        sites = site + '&page=' + str(page)
        driver.get(sites)
        print(sites)
        driver.implicitly_wait(5)
        html = driver.page_source
        bsobj = BeautifulSoup(html, 'html.parser')
        title = [i.text for i in bsobj.find_all('a', {'class': 'title'})]
        platform = [i.text for i in bsobj.find_all('div', {'class': 'platform'})]
        platform = platform[1:]
        score = [i.text for i in bsobj.find_all('div', {'class': re.compile('metascore_w large game.*')})]
        score = score[::2] #since it extracts two scores at the same time
        #genre_type = re.findall(r'\w+', str(genre_name))
        #genre_type = str(genre_type).replace("['", "")
        #genre_type = str(genre_type).replace("']", "")
        genre = [genre_type for i in range(len(score))]
        titles += title
        platforms += platform
        scores += score
        genres += genre
        data = {'platform': platforms, 'genre': genres, 'title': titles, 'score': scores}
    total_df = total_df.append(pd.DataFrame(data))
    total_df['platform'] = total_df['platform'].str.extract(r"( .*)")
    total_df['platform'] = total_df['platform'].str.replace('\n', '')
    #total_df['genre'] = total_df['genre'].str.extract(r"(\w+)")
    total_df.to_csv('metacritic_review_' + str(genre_type) + '.csv', encoding = 'utf-8')

if __name__ == '__main__':
    genres = ['adventure', 'fighting', 'first-person', 'flight', 'party', 'platformer', 'puzzle',
    'racing', 'real-time', 'role-playing', 'simulation', 'sports', 'strategy', 'third-person',
    'turn-based', 'wargame', 'wrestling']
    for genre_name in zip(genres):
        metacritic_review(genre_name)

#%%


import threading
import urllib3
import urllib.parse
import time

# config
url = 'https://www.cloudflare.com/ips-v6'
num_threads = 60
ip_list = 'valid_ip.txt'
output = 'cf_checked.txt'
timeout = 150/1000
# ----------

host = urllib.parse.urlsplit(url).hostname

ip_list = open(ip_list, 'r').read().splitlines()

total_line = len(ip_list)

output = open(output, 'w')

count = 0
def check_ip(ip):
    global count
    print(f'{format(count/total_line*100,".2f")}% checking: {ip}')

    try:
        pool = urllib3.HTTPSConnectionPool(ip, server_hostname=host,retries=False)
        response = pool.urlopen('GET',url, assert_same_host=False, timeout=timeout)

        if response.status == 200:
            return ip
        return None
    except:
        return None

def thread_function():
    while(True):
        global count
        line_count = count
        count += 1
        if(line_count < total_line):
            ip = ip_list[line_count]
            valid_ip = check_ip(ip)
            if valid_ip:
                global output
                output.write(f'{valid_ip}\n')
                output.flush()
        else:
            break

start = time.time()
threads = []
for i in range(num_threads):
    thread = threading.Thread(target=thread_function, args=())
    threads.append(thread)
    thread.start()

for thread in threads:
    thread.join()

print(f'time: {time.time() - start}s')
print('Done')


sum = count = 0
while True:
    num = eval(input())
    if num == -1:
        if count == 0:
            print(0.0)
        else:
            print(sum / count)
        break
    if num > 0:
        sum += num # sum = sum + num
        count += 1


from ibapi.client import EClient
import math
from queue import Queue
from threading import Thread
import time

class IBWriter(Thread):
    MAX_REQS_PER_SECOND = 30

    """
    Sends messages to TWS in a dedicated thread to avoid blocking the asyncio thread
    Also enforces message throttling
    """
    def __init__(self, client: EClient):
        super().__init__()
        self.queue = Queue()
        self._client = client
        self._last_msg_time_sec = math.floor(time.time())
        self._enable_message_throttling = True
        self._requests_in_last_sec = 0

    def enforce_msg_rate(self):
        if not self._enable_message_throttling:
            return

        cur_time_sec = math.floor(time.time())
        if (cur_time_sec == self._last_msg_time_sec):
            if self._requests_in_last_sec > IBWriter.MAX_REQS_PER_SECOND:
                time.sleep(1)
                self.enforce_msg_rate()
                return
            else:
                self._requests_in_last_sec += 1
        else:
            self._last_msg_time_sec = cur_time_sec
            self._requests_in_last_sec = 1

    def run(self):
        while self._client.isConnected():
            req = self.queue.get()
            self.enforce_msg_rate()
            req()


########################################################################################################################
# This script is to be used with realtime sticknet obs
# The purpose of this script is to create reflectivity and velocity sticknet station plots,
# and reflectivity and thetaV and thetaE objective analysis plots
#
# This script should be run every 10 MINUTES during a project
#
# Initially developed by A. Hill, Overhauled by J. McDonald 2021
########################################################################################################################

import matplotlib
matplotlib.use('Agg')

import cartopy.crs as ccrs
import cartopy.feature as cfeature
import cartopy
from metpy.plots import USCOUNTIES
from metpy.plots import  StationPlot
from metpy.interpolate import interpolate_to_grid

import matplotlib.pyplot as plt
import matplotlib.cm as cm
from mpl_toolkits.axes_grid1 import make_axes_locatable
import matplotlib.axes as maxes
import matplotlib.patheffects as path_effects
import cmocean

import pandas as pd
from netCDF4 import num2date
import numpy as np
import pyart;
import glob
import pytz
import datetime as dt
from collections import OrderedDict

# IMPORTANT
from info import TZ, savedir, probe_locs

import SNmods as snmods
from functions import calc_dewpoint,calc_thetae,calc_thetav,C_to_F,calc_mslp,convert_wind,parse_currtime
from functions_plotting import plot_logo, scale_bar, drop_nans_var, homeyer_rainbow_ramp
from functions_radar import get_radar_from_aws,_nearestDate, get_radar_ID

# get refl colormap
homeyer_rainbow_ramp()

# read in cartopy information
crs = ccrs.PlateCarree()
# Get data to plot state and province boundaries
states_provinces = cfeature.NaturalEarthFeature(
        category='cultural',
        name='admin_1_states_provinces_lakes',
        scale='10m',
        facecolor='none')

def add_radar(ax, radar, field):
    '''
    returns information for title
    '''
    if field == 'reflectivity':
        vmin = 0
        vmax = 70
        sweep = 0
        cbar_label = 'Reflectivity (dBZ)'
        cmap = 'pyart_HomeyerRainbow_ramp'

    elif field == 'velocity':
        vmin  = -45
        vmax  = 45
        sweep = 1
        cbar_label = 'Radial Velocity (m/s)'
        cmap = cmocean.cm.balance

    display = pyart.graph.RadarMapDisplay(radar)
    lat_0 = display.loc[0]
    lon_0 = display.loc[1]

    ### Grab indeces and times from radar object to make nice title
    index_at_start = radar.sweep_start_ray_index['data'][0]
    time_at_start_of_radar = num2date(radar.time['data'][index_at_start],
                                      radar.time['units'])
    index_at_start = radar.sweep_start_ray_index['data'][0]
    ts = num2date(radar.time['data'][index_at_start], radar.time['units'])

    rdate = dt.datetime(ts.year, ts.month, ts.day, ts.hour, ts.minute, ts.second)
    timezone = pytz.timezone(TZ) #if this errors, replace TZ with "US/Central" or "US/Eastern", etc
    local_time = timezone.fromutc(rdate)
    fancy_date_string = local_time.strftime('%A, %B %d at %I:%M %p %Z')
    fancy_date_string_utc = time_at_start_of_radar.strftime('%Y-%m-%d %H:%M UTC')

    display.plot_ppi_map(field, sweep, cmap=cmap, ax=ax, vmin=vmin, vmax=vmax,
                         colorbar_flag=False, title_flag=False,
                        lat_lines=[0], lon_lines=[0])

    ### gives stylistic control over colorbar
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="3%", pad=0.15, axes_class=maxes.Axes)
    norm = plt.Normalize(vmin=vmin, vmax=vmax)
    cbar = plt.colorbar(plt.cm.ScalarMappable(norm=norm, cmap=cmap), pad=0.02, cax=cax, orientation='vertical')
    cbar.set_label(label=cbar_label, size=16, weight='bold', labelpad=12)
    cbar.ax.tick_params(labelsize=15)

    ### Mark the radar
    ax.scatter(lon_0, lat_0, marker='o',s=30, c='r', ec='k', label=radar.metadata['instrument_name'])

    return fancy_date_string, fancy_date_string_utc, radar.metadata['instrument_name']

def radar_and_sticknet(lats, lons, data_df, mslp, dew_plot, IDs, u, v, obtime, radar, product='reflectivity'):

    ### find extent of plot
    # exact center of plot
    clat = np.amin(lats)+ abs(np.amax(lats) - np.amin(lats))/2
    clon = np.amin(lons) + abs(np.amin(lons) - np.amax(lons))/2

    # NOTE: change these hardcoded values if you want to change relative domain size
    # larger (smaller) numbers = larger (smaller) domain
    dlat = 0.78 * abs(np.amax(lats) - np.amin(lats))
    dlon = 0.78 * abs(np.amin(lons) - np.amax(lons))

    if dlon < .1:
        dlon = .1
    if dlat < .1:
        dlat = .1

    # find corners using the center and the buffers
    north_lat, south_lat = clat+dlat, clat-dlat
    west_lon, east_lon = clon-dlon, clon+dlon

    ### initialize figure
    fig = plt.figure(figsize = [10,10])
    ax = fig.add_subplot(1,1,1, projection=crs)
    ax.set_extent([west_lon, east_lon, north_lat,south_lat], crs )
    ax.add_feature(states_provinces, edgecolor='k', alpha=0.25, linewidth=1)
    ax.add_feature(USCOUNTIES.with_scale('20m'), alpha=0.4, linewidth=0.2)

    #### RADAR
    rtime, rtime_utc, radarname = add_radar(ax, radar, product)

    ##### STICKNET
    ax.plot(lons,lats,marker='s',color='k',markersize=5, linewidth=0)

    pe= [path_effects.Stroke(linewidth=3, foreground='white'), path_effects.Normal()]

    # use metpy to plot T, Td, MSLP (coded), and the 4-letter identifiers
    stationplot = StationPlot(ax, lons, lats,clip_on=True, transform=crs, fontsize=9, weight='bold')

    stationplot.plot_parameter((-1.5,1), data_df['T'].values, color='#b30000',
                               formatter='0.1f', path_effects=pe)
    stationplot.plot_parameter((-1.5,-1), dew_plot, color='darkgreen',
                               formatter='0.1f', path_effects=pe)
    stationplot.plot_parameter((1.5,1), mslp, formatter=lambda v: format(10 * v, '.0f')[-3:],
                               path_effects=pe)
    stationplot.plot_text((1.5, -.9), IDs, fontsize=8, weight='bold', path_effects=pe)

    # Add wind barbs, dropping bad vals, cuz WS can == -999
    ax.barbs(lons, lats, u, v, length=7.5,sizes={'emptybarb':.18}, lw=1.2,zorder=20)

    ### title
    ax.set_title('{0} {1}\n{2} ({3})\nStickNet Observations at {4}'.format(radarname,
                                product.capitalize(), rtime, rtime_utc, obtime.strftime('%D %H:%M UTC')),
                 fontsize=16, y=1.01)

    ### Plot the TTU logo, have it update location based on shape of base map
    plt.draw()
    plot_logo(fig, ax,alpha=0.7)

    ### Scale Bar
    center = np.average([ax.get_position().x0, ax.get_position().x1])
    scale_len = np.ceil((dlon*10)/3.5)*10
    if scale_len < 10: scale_len = 10
    scale_bar(fig, ax, length=int(scale_len), location=(center, ax.get_position().y0),fontsize=10)

    ### legend
    ax.legend(loc=1, fontsize=11,handletextpad=0.01,borderpad=0.3)

    # save figure
    nametime = parse_currtime()[1] # note that this may be different from last time on plot
                                 # THATS OKAY... it will make data drops more apparent!
    plt.savefig('{0}{1}_station_{2}.png'.format(savedir,product,nametime),dpi=300,bbox_inches = 'tight')
    plt.close()

def radar_and_oban(lats, lons, variable, name, savename, obtime, radar, line_color='C0',dK=1, product='reflectivity'):

    ### find extent of plot
    # exact center of plot
    clat = np.amin(lats)+ abs(np.amax(lats) - np.amin(lats))/2
    clon = np.amin(lons) + abs(np.amin(lons) - np.amax(lons))/2

    # NOTE: change these hardcoded values if you want to change relative domain size
    # larger (smaller) numbers = larger (smaller) domain
    dlat = 0.78 * abs(np.amax(lats) - np.amin(lats))
    dlon = 0.78 * abs(np.amin(lons) - np.amax(lons))

    if dlon < .1:
        dlon = .1
    if dlat < .1:
        dlat = .1

    # find corners using the center and the buffers
    north_lat, south_lat = clat+dlat, clat-dlat
    west_lon, east_lon = clon-dlon, clon+dlon

    ### initialize figure
    fig = plt.figure(figsize = [10,10])
    ax = fig.add_subplot(1,1,1, projection=crs)
    ax.set_extent([west_lon, east_lon, north_lat,south_lat], crs )
    ax.add_feature(states_provinces, edgecolor='k', alpha=0.25, linewidth=1)
    ax.add_feature(USCOUNTIES.with_scale('20m'), alpha=0.4, linewidth=0.2)

    #### RADAR
    rtime, rtime_utc, radarname = add_radar(ax, radar, product)

    ### OBAN
    ### interpolate data. Hres is 0.05 degrees
    varn, latsn, lonsn, FLAG = drop_nans_var(variable, lats, lons)
    gx,gy,img1 = interpolate_to_grid(np.asarray(lonsn),np.asarray(latsn),np.asarray(varn),\
                         interp_type='natural_neighbor',hres=.05)
    levels = np.arange(int(np.floor(np.amin(variable))),int(np.ceil(np.amax(variable)))+1,dK)
    ax.contour(gx, gy, img1, colors='w', levels=levels, linewidths=3)
    c=ax.contour(gx, gy, img1, colors=line_color, levels=levels, linewidths=1.5)

    # labels
    cl = ax.clabel(c, fontsize=10,inline=1, inline_spacing=2,fmt='%i',
              rightside_up=True, use_clabeltext=True)
    for t in cl:
        t.set_weight('bold')
        t.set_path_effects([path_effects.Stroke(linewidth=3, foreground='white'), path_effects.Normal()])

    # add box around the grid for reference
    x0, x1 = np.min(gx), np.max(gx)
    y0, y1 = np.min(gy), np.max(gy)

    ax.plot([x0,x0,x1,x1,x0],[y0,y1, y1, y0,y0], color='k', alpha=0.2, linewidth=3, linestyle='--')

    ### title
    ax.set_title('{0} {1}\n{2} ({3})\nStickNet {4} Analysis at {5}'.format(radarname,
                                product.capitalize(), rtime, rtime_utc, name, obtime.strftime('%D %H:%M UTC')),
                 fontsize=16, y=1.01)

    ### Plot the TTU logo, have it update location based on shape of base map
    plt.draw()
    plot_logo(fig, ax, alpha=0.7)

    ### Scale Bar
    center = np.average([ax.get_position().x0, ax.get_position().x1])
    scale_len = np.ceil((dlon*10)/3.5)*10
    if scale_len < 10: scale_len = 10
    scale_bar(fig, ax, length=int(scale_len), location=(center, ax.get_position().y0),fontsize=10)

    ### legend
    ax.legend(loc=1, fontsize=11,handletextpad=0.01,borderpad=0.3)

    ## warn if variable data is missing!
    if FLAG != 0:
        plt.draw()
        ax.text(center,0.98,  f'{FLAG} StickNets missing, plot may be inaccurate',
                fontsize=10,color='r',ha='center', va='center', transform=ax.transAxes)

    # save figure
    nametime = parse_currtime()[1] # note that this may be different from last time on plot
                                 # THATS OKAY... it will make data drops more apparent!
    plt.savefig('{0}{1}_{2}_{3}.png'.format(savedir,product,savename,nametime),dpi=300,bbox_inches = 'tight')
    plt.close()

################################
# pull in data

analysis_time = dt.datetime(2017,4,30,19,9) #dt.datetime.utcnow()
probes = list(probe_locs.keys())

# Grab the data from files in filedir we want to plot
data_df = snmods.get_sticknet_data(analysis_time,analysis_time,dataset='latest',probes=probes)

if not data_df.empty: # only works if you have data

    # pull sticknet locations and time
    lats =       data_df['Lat'].values
    lons =       data_df['Lon'].values
    IDs =        [ids[1:] for ids in data_df.index.values]
    obtime = data_df['date'][0]

    # calculate variables
    dewp_c = calc_dewpoint(data_df['T'].values,data_df['RH'].values)
    thetae = calc_thetae(data_df['T'].values,dewp_c, data_df['P'].values)
    thetav = calc_thetav(data_df['T'].values,dewp_c, data_df['P'].values)
    mslp = calc_mslp(data_df['T'].values, data_df['P'].values, data_df['Elevation'].values)
    u,v = convert_wind(data_df['WS'].values,data_df['WD'].values)

    # find appropriate radar, and pull in data
    clat = np.amin(lats)+ abs(np.amax(lats) - np.amin(lats))/2
    clon = np.amin(lons) + abs(np.amin(lons) - np.amax(lons))/2
    station, city = get_radar_ID(clat, clon)
    print('Radar: {}, from {}'.format(station, city))

    radar_namelist, radar_list = get_radar_from_aws(station, analysis_time, analysis_time)
    radar = radar_list[0]

    # radar products and station plots - this will be of Lubbock if no data is available
    radar_and_sticknet(lats, lons, data_df, mslp, dewp_c, IDs, u, v, obtime, radar, product='reflectivity')
    radar_and_sticknet(lats, lons, data_df, mslp, dewp_c, IDs, u, v, obtime, radar, product='velocity')

    # reflectivity and oban plots
    if len(data_df) >= 4: # at least 4 pts needed for interpolation
        radar_and_oban(lats, lons, thetav, r'${\theta_v}$','TV', obtime, radar, line_color='#002775', dK=1)
        radar_and_oban(lats, lons, thetae,r'${\theta_e}$', 'TE', obtime, radar, line_color='#014522', dK=1)

else:
    print('No data at {}'.format(analysis_time.strftime('%D %H:%M UTC')))


r"""
==============================================
Scaling the regularization parameter for SVCs
==============================================

The following example illustrates the effect of scaling the regularization
parameter when using :ref:`svm` for :ref:`classification <svm_classification>`.
For SVC classification, we are interested in a risk minimization for the
equation:

.. math::

    C \sum_{i=1, n} \mathcal{L} (f(x_i), y_i) + \Omega (w)

where

    - :math:`C` is used to set the amount of regularization
    - :math:`\mathcal{L}` is a `loss` function of our samples
      and our model parameters.
    - :math:`\Omega` is a `penalty` function of our model parameters

If we consider the loss function to be the individual error per sample, then the
data-fit term, or the sum of the error for each sample, increases as we add more
samples. The penalization term, however, does not increase.

When using, for example, :ref:`cross validation <cross_validation>`, to set the
amount of regularization with `C`, there would be a different amount of samples
between the main problem and the smaller problems within the folds of the cross
validation.

Since the loss function dependens on the amount of samples, the latter
influences the selected value of `C`. The question that arises is "How do we
optimally adjust C to account for the different amount of training samples?"
"""

# Author: Andreas Mueller <amueller@ais.uni-bonn.de>
#         Jaques Grobler <jaques.grobler@inria.fr>
# License: BSD 3 clause

# %%
# Data generation
# ---------------
#
# In this example we investigate the effect of reparametrizing the regularization
# parameter `C` to account for the number of samples when using either L1 or L2
# penalty. For such purpose we create a synthetic dataset with a large number of
# features, out of which only a few are informative. We therefore expect the
# regularization to shrink the coefficients towards zero (L2 penalty) or exactly
# zero (L1 penalty).

from sklearn.datasets import make_classification

n_samples, n_features = 100, 300
X, y = make_classification(
    n_samples=n_samples, n_features=n_features, n_informative=5, random_state=1
)

# %%
# L1-penalty case
# ---------------
# In the L1 case, theory says that provided a strong regularization, the
# estimator cannot predict as well as a model knowing the true distribution
# (even in the limit where the sample size grows to infinity) as it may set some
# weights of otherwise predictive features to zero, which induces a bias. It does
# say, however, that it is possible to find the right set of non-zero parameters
# as well as their signs by tuning `C`.
#
# We define a linear SVC with the L1 penalty.

from sklearn.svm import LinearSVC

model_l1 = LinearSVC(penalty="l1", loss="squared_hinge", dual=False, tol=1e-3)

# %%
# We compute the mean test score for different values of `C` via
# cross-validation.

import numpy as np
import pandas as pd

from sklearn.model_selection import ShuffleSplit, validation_curve

Cs = np.logspace(-2.3, -1.3, 10)
train_sizes = np.linspace(0.3, 0.7, 3)
labels = [f"fraction: {train_size}" for train_size in train_sizes]
shuffle_params = {
    "test_size": 0.3,
    "n_splits": 150,
    "random_state": 1,
}

results = {"C": Cs}
for label, train_size in zip(labels, train_sizes):
    cv = ShuffleSplit(train_size=train_size, **shuffle_params)
    train_scores, test_scores = validation_curve(
        model_l1,
        X,
        y,
        param_name="C",
        param_range=Cs,
        cv=cv,
        n_jobs=2,
    )
    results[label] = test_scores.mean(axis=1)
results = pd.DataFrame(results)

# %%
import matplotlib.pyplot as plt

fig, axes = plt.subplots(nrows=1, ncols=2, sharey=True, figsize=(12, 6))

# plot results without scaling C
results.plot(x="C", ax=axes[0], logx=True)
axes[0].set_ylabel("CV score")
axes[0].set_title("No scaling")

for label in labels:
    best_C = results.loc[results[label].idxmax(), "C"]
    axes[0].axvline(x=best_C, linestyle="--", color="grey", alpha=0.7)

# plot results by scaling C
for train_size_idx, label in enumerate(labels):
    train_size = train_sizes[train_size_idx]
    results_scaled = results[[label]].assign(
        C_scaled=Cs * float(n_samples * np.sqrt(train_size))
    )
    results_scaled.plot(x="C_scaled", ax=axes[1], logx=True, label=label)
    best_C_scaled = results_scaled["C_scaled"].loc[results[label].idxmax()]
    axes[1].axvline(x=best_C_scaled, linestyle="--", color="grey", alpha=0.7)

axes[1].set_title("Scaling C by sqrt(1 / n_samples)")

_ = fig.suptitle("Effect of scaling C with L1 penalty")

# %%
# In the region of small `C` (strong regularization) all the coefficients
# learned by the models are zero, leading to severe underfitting. Indeed, the
# accuracy in this region is at the chance level.
#
# Using the default scale results in a somewhat stable optimal value of `C`,
# whereas the transition out of the underfitting region depends on the number of
# training samples. The reparametrization leads to even more stable results.
#
# See e.g. theorem 3 of :arxiv:`On the prediction performance of the Lasso
# <1402.1700>` or :arxiv:`Simultaneous analysis of Lasso and Dantzig selector
# <0801.1095>` where the regularization parameter is always assumed to be
# proportional to 1 / sqrt(n_samples).
#
# L2-penalty case
# ---------------
# We can do a similar experiment with the L2 penalty. In this case, the
# theory says that in order to achieve prediction consistency, the penalty
# parameter should be kept constant as the number of samples grow.

model_l2 = LinearSVC(penalty="l2", loss="squared_hinge", dual=True)
Cs = np.logspace(-8, 4, 11)

labels = [f"fraction: {train_size}" for train_size in train_sizes]
results = {"C": Cs}
for label, train_size in zip(labels, train_sizes):
    cv = ShuffleSplit(train_size=train_size, **shuffle_params)
    train_scores, test_scores = validation_curve(
        model_l2,
        X,
        y,
        param_name="C",
        param_range=Cs,
        cv=cv,
        n_jobs=2,
    )
    results[label] = test_scores.mean(axis=1)
results = pd.DataFrame(results)

# %%
import matplotlib.pyplot as plt

fig, axes = plt.subplots(nrows=1, ncols=2, sharey=True, figsize=(12, 6))

# plot results without scaling C
results.plot(x="C", ax=axes[0], logx=True)
axes[0].set_ylabel("CV score")
axes[0].set_title("No scaling")

for label in labels:
    best_C = results.loc[results[label].idxmax(), "C"]
    axes[0].axvline(x=best_C, linestyle="--", color="grey", alpha=0.8)

# plot results by scaling C
for train_size_idx, label in enumerate(labels):
    results_scaled = results[[label]].assign(
        C_scaled=Cs * float(n_samples * np.sqrt(train_sizes[train_size_idx]))
    )
    results_scaled.plot(x="C_scaled", ax=axes[1], logx=True, label=label)
    best_C_scaled = results_scaled["C_scaled"].loc[results[label].idxmax()]
    axes[1].axvline(x=best_C_scaled, linestyle="--", color="grey", alpha=0.8)
axes[1].set_title("Scaling C by sqrt(1 / n_samples)")

fig.suptitle("Effect of scaling C with L2 penalty")
plt.show()

# %%
# For the L2 penalty case, the reparametrization seems to have a smaller impact
# on the stability of the optimal value for the regularization. The transition
# out of the overfitting region occurs in a more spread range and the accuracy
# does not seem to be degraded up to chance level.
#
# Try increasing the value to `n_splits=1_000` for better results in the L2
# case, which is not shown here due to the limitations on the documentation
# builder.


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import logging
from timeit import default_timer as timer

import compas
from compas.geometry import Transformation
from compas.robots import Configuration

if compas.RHINO:
    import clr
    import rhinoscriptsyntax as rs
    import Rhino.Geometry as rg

    from compas_rhino.geometry import RhinoMesh
    from compas_rhino.geometry.transformations import xform_from_transformation

__all__ = [
    "PathVisualizer",
]

LOG = logging.getLogger("compas_fab.ghpython.path_planning")

def vrep_pose_from_plane(plane):
    """Creates a vrep-compatible transformation matrix from a Rhino/Grasshopper
    plane.

    This function might need rework as the source of the 90-deg Y rotation
    need is not entirely clear to me (related to the RFL model mismatch).
    """
    translation_matrix = rs.XformTranslation(((plane[0][0]), (plane[0][1]), plane[0][2]))
    plane_start = rs.PlaneFromFrame(rs.AddPoint(0, 0, 0), rs.AddPoint(1, 0, 0), rs.AddPoint(0, 1, 0))
    plane_end = rs.PlaneFromFrame(
        rs.AddPoint(0, 0, 0),
        rs.AddPoint(plane[1][0], (plane[1][1]), plane[1][2]),
        rs.AddPoint(plane[2][0], plane[2][1], plane[2][2]),
    )
    rotation_matrix = rs.XformRotation1(plane_start, plane_end)
    matrix = rs.XformMultiply(translation_matrix, rotation_matrix)
    return [
        matrix.M00,
        matrix.M01,
        matrix.M02,
        matrix.M03,
        matrix.M10,
        matrix.M11,
        matrix.M12,
        matrix.M13,
        matrix.M20,
        matrix.M21,
        matrix.M22,
        matrix.M23,
    ]

def _transform_to_origin(mesh, xform):
    inverse = clr.StrongBox[rg.Transform]()
    if not xform.TryGetInverse(inverse):
        raise ValueError("Unable to get inverse of matrix")

    mesh.Transform(inverse.Value)

    return mesh

def _create_rhino_mesh(vertices, faces):
    vertices = [vertices[i : i + 3] for i in range(0, len(vertices), 3)]
    faces = [faces[i : i + 3] for i in range(0, len(faces), 3)]
    mesh = rg.Mesh()
    for a, b, c in vertices:
        mesh.Vertices.Add(a, b, c)
    for face in faces:
        mesh.Faces.AddFace(face[0], face[1], face[2])

    return mesh

def _to_xform(m):
    m = m if len(m) == 16 else m + [0, 0, 0, 1]
    return xform_from_transformation(Transformation.from_list(m))

# TODO: This module needs to be heavily refactored
# Currently only compatible with V-REP client and most helper functions
# are not superseded by compas features
class PathVisualizer(object):
    """Handles the generation of meshes to visualize a full path plan
    in Rhino/Grasshopper.
    """

    def __init__(self, simulator, robot, building_member=None, building_member_pickup_config=None):
        self.simulator = simulator
        self.robot = robot
        self.building_member = building_member
        self.building_member_pickup_config = building_member_pickup_config

    def get_frame_meshes(self, path, frame, ctx):
        """Retrieves all meshes required to render a specific frame of a path plan.

        Args:
            path (:obj:`list` of :class:`Configuration`): Represents a collision-free
                path to a goal pose. It is the output of the path planning generated
                by a :class:`Simulator` object.
            frame (:obj:`int`): Frame number to retrieve.
            ctx (:obj:`dict`): A dictionary to keep context. Within Grasshopper, this is
                normally the ``sc.sticky`` object.

        Returns:
            list: list of Rhino meshes that can be used to visualize the selected frame.
        """
        first_start = timer() if self.debug else None
        shape_handles = self.simulator.get_all_visible_handles()
        if self.debug:
            LOG.debug("Execution time: get_all_visible_handles=%.2f", timer() - first_start)

        if "scene_meshes" not in ctx:
            ctx["scene_meshes"] = self._get_scene_meshes(shape_handles)

        frame_config = path[frame]

        start = timer() if self.debug else None
        self.simulator.set_robot_config(self.robot, frame_config)
        if self.debug:
            LOG.debug("Execution time: set_robot_config=%.2f", timer() - start)

        start = timer() if self.debug else None
        meshes = []
        mesh_matrices = self.simulator.get_object_matrices(shape_handles)
        for handle, mesh_matrix in mesh_matrices.iteritems():
            mesh = ctx["scene_meshes"][handle].DuplicateShallow()
            mesh.Transform(_to_xform(mesh_matrix))
            meshes.append(mesh)

        if self.building_member:
            gripping_config = self.building_member_pickup_config if self.building_member_pickup_config else path[0]
            info = self._get_building_member_info(gripping_config)
            mesh = info["mesh"].DuplicateShallow()
            parent_transform = _to_xform(mesh_matrices[info["parent_handle"]])
            relative_transform = info["relative_transform"]

            mesh.Transform(rg.Transform.Multiply(parent_transform, relative_transform))
            meshes.append(mesh)

        if self.debug:
            LOG.debug("Execution time: get all transformed meshes=%.2f", timer() - start)
            LOG.debug("Execution time: total=%.2f", timer() - first_start)

        return meshes

    @property
    def debug(self):
        """Indicates whether the path visualizer is in debug mode or not."""
        return self.simulator.debug

    def _get_scene_meshes(self, shape_handles):
        start = timer() if self.debug else None
        shape_geometry = []
        for handle in shape_handles:
            _, faces, vertices, _, _ = self.simulator.run_child_script("getShapeMesh", [handle], [], [])
            shape_geometry.append((vertices, faces))

        scene_meshes = {}
        _, _, mesh_matrices, _, _ = self.simulator.run_child_script("getShapeMatrices", shape_handles, [], [])
        for i in range(0, len(mesh_matrices), 12):
            handle = shape_handles[i // 12]
            vertices, faces = shape_geometry[i // 12]
            transform = _to_xform(mesh_matrices[i : i + 12])
            mesh = _transform_to_origin(_create_rhino_mesh(vertices, faces), transform)
            scene_meshes[handle] = mesh

        if self.debug:
            LOG.debug("Execution time: create RFL meshes at origin=%.2f", timer() - start)

        return scene_meshes

    def _get_building_member_info(self, gripping_config):
        start = timer() if self.debug else None
        self.simulator.set_robot_config(self.robot, gripping_config)
        mesh = RhinoMesh.from_guid(self.building_member).to_compas()
        handle = self.simulator.add_attached_collision_mesh(mesh, options={"robot_name": self.robot.name})
        matrix = self.simulator.get_object_matrices([handle])[handle]

        parent_handle = self.simulator.get_object_handle("customGripper" + self.robot.name)
        _, _, mesh_matrix, _, _ = self.simulator.run_child_script(
            "getShapeMatrixRelative", [handle, parent_handle], [], []
        )

        relative_transform = _to_xform(mesh_matrix)

        transform = _to_xform(matrix)
        mesh_at_origin = _transform_to_origin(rs.coercemesh(self.building_member), transform)

        if self.debug:
            LOG.debug("Execution time: building member=%.2f", timer() - start)

        return {"mesh": mesh_at_origin, "parent_handle": parent_handle, "relative_transform": relative_transform}

class InputParameterParser(object):
    """Simplifies some tasks related to parsing Grasshopper input parameters
    for path planning."""

    def compact_list(self, list):
        """Compacts a list filtering all `None` values."""
        return filter(None, list)

    def get_pose(self, plane_or_pose):
        """Gets a pose that is compatible with V-REP from a string containing
        comma-separated floats or from a Rhino/Grasshopper plane.

        Args:
            plane_or_pose (comma-separated :obj:`str` of a Rhino :obj:`Plane`):

        Returns:
            pose: :class:`.Pose` instance representing a transformation matrix.
        """
        if not plane_or_pose:
            return None

        try:
            return Pose.from_list(vrep_pose_from_plane(plane_or_pose))
        except (TypeError, IndexError):
            return Pose.from_list(map(float, plane_or_pose.split(","))) if plane_or_pose else None

    def get_config_or_pose(self, config_values_or_plane):
        """Parses multiple input data types and returns a configuration or a pose
        represented as a transformation matrix.

        Args:
            config_values_or_plane (comma-separated :obj:`string`, or :class:`Configuration` instance,
                or a Rhino :obj:`Plane`).
        """
        if not config_values_or_plane:
            return None

        try:
            return Pose.from_list(vrep_pose_from_plane(config_values_or_plane))
        except (TypeError, IndexError):
            try:
                if config_values_or_plane.external_axes and config_values_or_plane.joint_values:
                    return config_values_or_plane
            except AttributeError:
                values = map(float, config_values_or_plane.split(","))
                return Configuration.from_degrees_list(values)


# coding: utf-8

# In[61]:

import pandas as pd
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
import numpy as np
import os
df=pd.read_excel(r'C:\Users\jyothi\Downloads\Sample - Superstore.xlsx')
df=df[['Order Date','Sales']].sort_values('Order Date',ascending=False)
df= df.rename(columns={'Order Date': 'od', 'Sales': 'sales'})
df.index=df.od
df=df[(df.od>'2018-01-01')]
df=df.drop('od',axis=1)
df
f=plt.figure(figsize=(20,5))
plt.plot(df)
plt.xlabel('Dates')
plt.ylabel('sales')
plt.title('sales by date')

# In[62]:

from pandas.plotting import autocorrelation_plot
autocorrelation_plot(df)

# In[67]:

from statsmodels.tsa.arima_model import ARIMA
model=ARIMA(df,order=(5,1,0))
model_fit = model.fit(disp=0)
print(model_fit.summary())
# plot residual errors
residuals =pd.DataFrame(model_fit.resid)
residuals.plot()

residuals.plot(kind='kde')
print(residuals.describe())

# In[69]:

X = df.values
size = int(len(X) * 0.66)
train, test = X[0:size], X[size:len(X)]
history = [x for x in train]
predictions = list()
for t in range(len(test)):
	model = ARIMA(history, order=(10,1,0))
	model_fit = model.fit(disp=0)
	output = model_fit.forecast()
	yhat = output[0]
	predictions.append(yhat)
	obs = test[t]
	history.append(obs)
	print('predicted=%f, expected=%f' % (yhat, obs))
error = mean_squared_error(test, predictions)
print('Test MSE: %.3f' % error)
# plot
plt.plot(test)
plt.plot(predictions, color='red')


"""! @brief Run pipeline stage, which will run the project's pipeline on the retrieved data."""
import sys
import pandas as pd
from typing import Any
from json import loads as json_loads
from pyspark.sql.types import *
from typing import Iterator

from dsframework.base.batch.stage_base import ZIDS_Stage

##
# @file
# @brief Stage main class, implements ZIDS_Stage base class.
class RunPipelineStage(ZIDS_Stage):
    """! Stage class

    Implement a stage that will later be converted to an executable job in a specific workflow.
    """

    def __init__(self, stage_config):
        """! The Stage class (RunPipelineStage) initializer.
        Base class will load basic configuration parameters, additional fields should be added here.

            Args:
                stage_config : Configuration dictionary, loaded from configuration file.
        """

        ##
        # @hidecallgraph @hidecallergraph
        super().__init__(stage_config)
        self.udf_schema = StructType([StructField("id", IntegerType()),
                                      StructField("url", StringType()),
                                      StructField("res", StringType())])

    def get_stage_name(self):
        """! Get the stage name

            Returns:
                A string, containing the stage's name
        """
        return self.__class__.__name__

    @staticmethod
    def map_in_pandas(partition: Iterator[pd.DataFrame]):
        """! Executes the project main pipeline

                Args:
                    partition: Iterator[pd.DataFrame], we get an iterator to a Pandas dataframe, on which
                    we can perform actions, such as running the project pipeline and writing the output
        """
        def test():
            from pipeline.pipeline import PrivacyclassifierPipeline
            p = PrivacyclassifierPipeline()
            pipeline_output = p.execute()
            return pipeline_output

        for pdf in partition:
            pdf_output = test()
            print(f'output: {pdf_output}')
            pdf.at[0, 'res'] = pdf_output.out_text
            yield pdf[pdf.id == 2]

    def main(self, **kwargs: Any):
        """! Executes the main functionality of the stage.

            Args:
                **kwargs : Whatever is needed for the stage to run properly.
        """
        spark_session = self.load_spark()
        df = spark_session.createDataFrame([(1, 'http://aaa', "I"),
                                            (2, 'http://bbb', "am"),
                                            (3, 'http://ccc', "testing")], ("id", "url", "res"))
        df.show(3)
        out_df = df.mapInPandas(func=self.map_in_pandas, schema=self.udf_schema)
        out_df.show(1)

if __name__ == "__main__":
    """! Executes the stage by instantiating it and calling the main function.
    Set up argument condition according to the usage of the written stage

        Args:
            System argument 1 - Configuration file
            System argument 2 - Start date
    """
    if sys.argv and len(sys.argv) > 1:
        config = json_loads(sys.argv[1])
        stage = RunPipelineStage(config)
        try:
            start_date = sys.argv[2]
            end_date = sys.argv[3]
            params = sys.argv[4]
            stage.update_stage_params(start_date, end_date, params)
            stage.main()
        except Exception as e:
            print(f" Stage failed with error: {e}")
    else:
        print(f"Stage configuration not provided, Can't run stage")


from random import (randint,
randrange, random)

def factorial(number: int = 5) -> int:
    """Get number factorial
    input:
      number - int
    output:
      mul - int
    """
    mul = 1 # default value
    for i in range(1, number + 1):
        mul *= i
    return mul

print(factorial(7))

x = randint(1,5)
print(x)


from typing import Dict, List, Optional
import glob
import json
import logging
import sys
import time

import numpy as np
import pandas as pd

from pyhailing import RidehailEnv

def get_eval_files(the_dir: str) -> List[str]:
    """Provides a list of all evaluation files in the directory."""

    if the_dir is None:
        the_dir = "./"

    if the_dir[-1] != '/':
        the_dir = the_dir + "/"

    return glob.glob(the_dir + "pyhailing_eval_results*.json")

def load_result_file(eval_file) -> Optional[Dict]:
    """Loads the results data from a submitted file.

    If the file cannot be read, returns None.

    """

    try:
        with open(eval_file, 'r') as f:
            result = json.load(f)
            return result

    except Exception:
        return None

def get_instance_category(entry) -> Optional[str]:
    """Determines the instance category for which the entry was submitted.

    If it does not match the config of any instance category, returns None.

    """

    instance_categories = RidehailEnv.DIMACS_CONFIGS.ALL_CONFIGS

    entry_config = entry["config"]
    keys_to_check = list(entry_config.keys())

    try:
        keys_to_check.remove("nickname")
    except:
        return None

    for category_name, category_config in instance_categories.items():
        if all((entry_config[key] == category_config[key] for key in keys_to_check)):
            return category_name

    return None

def get_rwd_upperbound(env) -> float:
    """Provides an upperbound on the reward for the env's current episode."""

    # TODO
    ub = np.inf

    return ub

def check_vehicle_sequences(assignments, env) -> None:
    """Checks that assignments are plausible."""

    # TODO

    return True

def get_entry_performance(entry):
    """Checks the performance of an entry."""

    eps_results = entry["episodes"]

    # Make sure the proper number of episodes were performed
    expected_num_eps = RidehailEnv.DIMACS_NUM_EVAL_EPISODES

    if len(eps_results) != expected_num_eps:
        logging.warning(
            f"Invalid entry. Expected results for {len(eps_results)} episodes but got results for {expected_num_eps}.")
        return None

    # Create an environment to check the solution
    config = entry["config"]
    config["for_evaluation"] = False
    env = RidehailEnv(entry["config"])

    # For each episode...
    for i, eps_result in enumerate(eps_results):

        env.reset()

        # Make sure the claimed reward does not exceed a computable UB
        rwd_upperbound = get_rwd_upperbound(env)

        if eps_result['final_reward'] > rwd_upperbound:
            logging.warning("Invalid reward achieved.")
            return None

        # Get the sequence for each vehicle
        # get the sequence of reqs for each vehicle. for each, make sure it is time feasible (with an easy check)
        seqs_feasible = check_vehicle_sequences(eps_result['assignments'], env)
        if not seqs_feasible:
            logging.warning("Invalid vehicle-to-request assignments.")
            return None

    # All good. Get the mean episode reward
    mean_reward = (
        sum((eps_result["final_reward"] for eps_result in eps_results))
        / expected_num_eps
    )

    return mean_reward

def get_competition_results(eval_files:List[str]) -> pd.DataFrame:

    # Initialize the results of the competition
    competition_results = []

    for eval_file in eval_files:

        # Initialize a results record for this entry
        entry_results = {
            "filename": eval_file,
            "instance_category": None,
            "mean_reward": None,
        }

        # Load the file
        entry = load_result_file(eval_file)

        if entry is None:
            logging.warning(f"Entry file could not be read: {eval_file}")
            competition_results.append(entry_results)
            continue

        # Determine the entry's instance category
        instance_category = get_instance_category(entry)

        if instance_category is None:
            logging.warning(
                f"Entry file's config does not match the config of any instance category: {eval_file}"
            )
            competition_results.append(entry_results)
            continue

        entry_results["instance_category"] = instance_category

        # Get the entry's mean episode reward
        mean_reward = get_entry_performance(entry)

        # Done.
        competition_results.append(entry_results)

    # Combine all results into df and return it
    return pd.DataFrame(competition_results)

if __name__ == "__main__":

    now = time.strftime("%Y%m%d%H%M%S")

    args = sys.argv

    if len(args) != 2:
        logging.warning("Grabbing eval files from the current directory.")
        eval_files_dir = "./"

    else:
        eval_files_dir = args[1]

    eval_files = get_eval_files(eval_files_dir)

    eval_results = get_competition_results(eval_files)

    eval_results.to_csv(f"competition_results_{now}.csv", index=None)


import requests
from bs4 import BeautifulSoup
from urllib.request import unquote
import re

url  = 'https://www.mca.gov.in/MinistryV2/monthlyinformationbulletin.html'
agents = {"User-Agent":'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'}

response = requests.get(url,headers=agents)
content = BeautifulSoup(response.text, 'lxml')

all_urls = content.find_all('a')

pdf_urls = []

for url in all_urls:
    try:
        if 'pdf' in url['href']:
            if re.search('MIB',url['href']) and re.search('202[0-1]',url['href']):
                pdf_url = ''
                if 'https' not in url['href']:
                    pdf_url = 'https://www.mca.gov.in/' + url['href']
                else:
                    pdf_url = url['href']

                pdf_response = requests.get(pdf_url,headers=agents)
                filename = unquote(pdf_response.url).split('/')[-1].replace(' ','_')
                print(filename)
                with open('./pdf/'+filename, 'wb') as f:
                    f.write(pdf_response.content)

    except Exception as e:
        print("Error: ", e)


import json

import requests

# Search by language
path = "https://datalab.nlpedia.ai/api/fetch_data_for_sdk_language"
data = {"language": "fr"}
r = requests.post(path, json=data)
list_ = json.loads(r.content)
print(len(list_))
print(list_[0])

# Search by task
path = "https://datalab.nlpedia.ai/api/fetch_data_for_sdk_task"
data = {
    "task": "sentiment-classification",
}
r = requests.post(path, json=data)
list_ = json.loads(r.content)
print(len(list_))
print(list_[0])

# Get all languages, tasks and task_category
path = "https://datalab.nlpedia.ai/api/fetch_meta_data_for_sdk_task_language"
r = requests.get(path)
dic_ = json.loads(r.content)
print(dic_["languages"])  # list of all languages
print(dic_["tasks"])  # list of all tasks
print(dic_["task_categories"])  # list of all task_category