import os
import torch
from matplotlib import pyplot as plt
from tqdm import tqdm

def weights_init(net, init_type='normal', init_gain=0.01, path=None):
    """权重初始化"""

    def init_func(m):
        # 卷积层权重
        if m.__class__.__name__ == 'Conv2d':
            if init_type == 'normal':
                torch.nn.init.normal_(m.weight.data, 0.0, init_gain)
            elif init_type == 'xavier':
                torch.nn.init.xavier_normal_(m.weight.data, gain=init_gain)
        # 归一层
        elif m.__class__.__name__ == 'BatchNorm2d':
            torch.nn.init.normal_(m.weight.data, 1.0, 0.02)
            torch.nn.init.constant_(m.bias.data, 0.0)

    if path and os.path.exists(path):
        print(f'initialize network with path')
        net.load_state_dict(torch.load(path))
    else:
        print(f'initialize network with {init_type} type')
        net.apply(init_func)

def update_log(root, loc_loss, conf_loss):
    with open(root, 'r', encoding='utf-8') as f:
        loc_loss_list = [float(ll) for ll in f.readline().split()]
        conf_loss_list = [float(cl) for cl in f.readline().split()]

    loc_loss_list.append(loc_loss)
    conf_loss_list.append(conf_loss)

    with open(root, 'w', encoding='utf-8') as f:
        f.writelines(' '.join([f'{ll:.3f}' for ll in loc_loss_list]) + '\n')
        f.writelines(' '.join([f'{cl:.3f}' for cl in conf_loss_list]))

def train_epoch(net, train_iter, loss, updater, device, mode='train'):
    loc_loss = 0
    conf_loss = 0
    count = 0
    for X, y in tqdm(train_iter):
        if mode == 'train':
            net.train()
        else:
            net.eval()
        X, y = X.to(device), y.to(device)
        output = net(X)
        loss_l, loss_c = loss(output, y)
        all_loss = loss_l + loss_c

        # 反向传播
        if mode == 'train':
            updater.zero_grad()
            all_loss.backward()
            updater.step()

        loc_loss += loss_l.item()
        conf_loss += loss_c.item()
        count += 1

    return loc_loss / count, conf_loss / count

def train(net, train_iter, num_epoch, loss, updater, updater_scheduler, device, net_param_path, log_path,
          train_threshold=None):
    net.to(device)

    for epoch in range(num_epoch):
        # train
        train_loc_loss, train_conf_loss = train_epoch(net, train_iter, loss, updater, device, mode='train')
        print(f'{epoch + 1}, {train_loc_loss:.3f}, {train_conf_loss:.3f}')
        updater_scheduler.step(train_loc_loss + train_conf_loss)

        # eval
        # val_loc_loss, val_conf_loss = train_epoch(net, train_iter, updater, device, mode='eval')
        # print(f'{epoch + 1}, {val_loc_loss:.3f}, {val_conf_loss:.3f}')

        if train_threshold is None:
            torch.save(net.state_dict(), net_param_path)
            update_log(log_path, train_loc_loss, train_conf_loss)
            print('save model param and log')
            train_threshold = train_conf_loss
        elif train_conf_loss < train_threshold:
            torch.save(net.state_dict(), net_param_path)
            update_log(log_path, train_loc_loss, train_conf_loss)
            print('save model param and log')
            train_threshold = train_conf_loss

def set_figsize(figsize=(3.5, 2.5)):
    """设置matplotlib的图表大小。"""
    plt.rcParams['figure.figsize'] = figsize

def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend):
    """设置matplotlib的轴。"""
    axes.set_xlabel(xlabel)
    axes.set_ylabel(ylabel)
    axes.set_xscale(xscale)
    axes.set_yscale(yscale)
    axes.set_xlim(xlim)
    axes.set_ylim(ylim)
    if legend:
        axes.legend(legend)
    axes.grid()

def plot(X, Y=None, xlabel=None, ylabel=None, legend=None, xlim=None, ylim=None, xscale='linear', yscale='linear',
         fmts=('-', 'm--', 'g-.', 'r:'), figsize=(3.5, 2.5), axes=None, tpye=0):
    """绘制数据点。"""
    if legend is None:
        legend = []

    # 设置画布大小
    set_figsize(figsize)
    # 移动坐标轴
    axes = axes if axes else plt.gca()

    # 如果 `X` 有一个轴，输出True
    def has_one_axis(X):
        return hasattr(X, "ndim") and X.ndim == 1 or isinstance(X, list) and not hasattr(X[0], "__len__")

    if has_one_axis(X):
        X = [X]
    if Y is None:
        X, Y = [[]] * len(X), X
    elif has_one_axis(Y):
        Y = [Y]
    if len(X) != len(Y):
        X = X * len(Y)
    # 清除当前活动轴
    axes.cla()
    for x, y, fmt in zip(X, Y, fmts):
        if len(x):
            axes.plot(x, y, fmt)
        else:
            axes.plot(y, fmt)
    set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend)
    if not tpye:
        plt.show()

def plot_log(size=300):
    with open(f'../logs/m_mask_ssd{size}.log', 'r', encoding='utf-8') as f:
        loc_loss_list = [float(ll) for ll in f.readline().split()]
        conf_loss_list = [float(cl) for cl in f.readline().split()]
    num_epoch = len(loc_loss_list)
    print(num_epoch)
    plot(X=[range(1, num_epoch + 1)], Y=[loc_loss_list, conf_loss_list], xlabel='epoch', ylabel='loss',
         xlim=[1, num_epoch], legend=['loc_loss', 'conf_loss'], figsize=(6, 4))

if __name__ == '__main__':
    plot_log(300)
    # plot_log(512)


class Solution:
    def mergeAlternately(self, word1: str, word2: str) -> str:
        i = j = 0
        n, m = len(word1), len(word2)
        result = []
        while i < n or j < m:
            if i < n:
                result.append(word1[i])
            if j < m:
                result.append(word2[j])
            i += 1
            j += 1

        return "".join(result)


class Node:
    def __init__(self, data):
        self.data = data
        self.next = None

class SinglyLinkedList:
    def __init__(self):
        self.head = None
        self.tail = None

    def addNode(self, data):
        newNode = Node(data)

        if (self.head == None):
            self.head = newNode
            self.tail = newNode
        else:
            self.tail.next = newNode
            self.tail = newNode

    def deleteNode(self, position):
        cur = self.head
        if not self.head:
            break
        elif position == 0:
            self.head = self.head.next
        else:
            for i in range(position-1):
                cur = cur.next

            if cur.next == self.tail:
                self.tail = cur

            cur.next = cur.next.next

    def printLinkedList(self):
        arr = []
        cur = self.head
        while cur:
            arr.append(cur.data)
            cur = cur.next

        return arr

llist = SinglyLinkedList()
llist.addNode(1)
llist.addNode(2)
llist.addNode(3)
llist.addNode(4)
llist.addNode(5)
llist.deleteNode(2)
print(llist.printLinkedList())


import basf2
from reconstruction import add_reconstruction
import sys

# Set the global log level
basf2.set_log_level(basf2.LogLevel.INFO)

input = basf2.register_module('RootInput')
input.param('inputFileName', sys.argv[1])

gearbox = basf2.register_module('Gearbox')

eklm_alignment_along_strips = basf2.register_module('EKLMAlignmentAlongStripsCollector')

# Create the main path and add the modules
main = basf2.create_path()
main.add_module(input)
main.add_module("HistoManager", histoFileName=sys.argv[2])
main.add_module(gearbox)
add_reconstruction(main)
main.add_module(eklm_alignment_along_strips)

# generate events
basf2.process(main)

# show call statistics
print(basf2.statistics)


import pycuda.driver as cuda
import pycuda.autoinit
from pycuda.compiler import SourceModule
import numpy
import matplotlib.pyplot as plt
import time
import math

###x = numpy.linspace(0, 2*numpy.pi, 10)
#Prepared data
x = numpy.linspace(-10, 10, 10000000)
y = numpy.arctan(x)

#######################################
############## CUDA ###################
#######################################
x = x.astype(numpy.float32)
y = y.astype(numpy.float32)

#Create interval for interpolation
##interval = numpy.asarray([0, 2*numpy.pi])
interval = numpy.asarray([-10, 10])
min_el = numpy.min(interval)
max_el = numpy.max(interval)

#To transfer data to allocate memory on the device
x_gpu = cuda.mem_alloc(x.nbytes)
y_gpu = cuda.mem_alloc(y.nbytes)

#To transfer the data to the GPU
cuda.memcpy_htod(x_gpu, x)
cuda.memcpy_htod(y_gpu, y)

#Kernel module on C
mod = SourceModule("""
  #include <stdio.h>
  __global__ void linear_interpolation(float *x, float *y, const float min_el, const float max_el)
  {
    int idx = threadIdx.x + blockDim.x*blockIdx.x;
    int jdy = threadIdx.y + blockDim.y*blockIdx.y;
    x[idx] = y[idx]-x[idx]/(max_el-min_el)*1+x[idx];
  }
  """)

#To get the kernel function
func = mod.get_function("linear_interpolation")
start_timer_1 = time.time()
func(x_gpu, y_gpu, numpy.float32(min_el), numpy.float32(max_el), block=(1000,1,1))
x_int = numpy.empty_like(x)
cuda.memcpy_dtoh(x_int, x_gpu)

stop_timer_1 = time.time()
timer_1 = stop_timer_1 - start_timer_1
print('CUDA: ', timer_1, 'sec.')

########################################
######## NUMPY INTERPOLATION ###########
########################################

xvals = numpy.linspace(0, 2*numpy.pi, 10000000)
start_timer_2 = time.time()
yinterp = numpy.interp(xvals,x,y)
stop_timer_2 = time.time()
timer_2 = stop_timer_2 - start_timer_2
print('NUMPY: ', timer_2, 'sec.')

########################################
####### BASE PYTHON INTERPOLATION ######
########################################

start_timer_3 = time.time()
x_list = numpy.array(x).tolist()
y_list = []
interval_list = [-10, 10]
x_iterp = []
for i in x_list:
    y_element = math.atan(i)
    y_list.append(y_element)

for j in range(len(x_list)):
    x_int_elem = (y_list[j] - x_list[j])/(interval_list[1] - interval_list[0])*1 + x_list[j]
    x_iterp.append(x_int_elem)
stop_timer_3 = time.time()
timer_3 = stop_timer_3 - start_timer_3
print('PYTHON BASE: ', timer_3, 'sec.')

#######################################
############### END ###################
#######################################

#x_l = [1, 2, 3]
#x_label = ['CUDA', 'NUMPY', 'BASE PYTHON']
#y_label = [timer_1, timer_2, timer_3]
#plt.axis([1, 3, 0., 1])
#plt.grid(True)
#plt.bar(x_l,y_label, align='center')
#plt.ylabel('Time, sec.')
#plt.title('Comparison CUDA vs Python and Numpy')
#plt.xticks(x_l, x_label)
#plt.show()


import os
from openai import OpenAI
from record import listen
#-------------------------------
#  1.录音到文件
#-------------------------------

# init API
client = OpenAI()

# 开始录音
filename = listen()

# 读录音文件
audio_file = open(filename,"rb")
#-------------------------------
#  2.OpenAI - 语音转文本
#-------------------------------
# transcript 保存转换的文本
response = client.audio.transcriptions.create(
    model="whisper-1",
    file=audio_file
)
qs = response.text
print(f'Question:{qs}')

#-------------------------------
#  3.OpenAI - 创建对话
#-------------------------------
completion = client.chat.completions.create(
    model="gpt-3.5-turbo",
    messages=[
        {
            "role":"user",
            "content":qs
        }
    ],
    # 最大返回的token数
    max_tokens=300,
    # 模型发散想象力, 0~2,float,写做类要数值高,逻辑类数值低
    temperature=0.8,
)
# 获取AI模型返回的消息
result = completion.choices[0].message.content
# total_tokens 消费token的总数
print(f'AI[tokens({completion.usage.total_tokens})]:{result}')

# 可调用OpenAI文本转语音API(会有成本)
# response = client.audio.speech.create(
#   model="tts-1",
#   voice="alloy",
#   input=result
# )
# # 默认mp3格式
# response.stream_to_file(os.path.dirname(__file__)+"/response.mp3")

#-------------------------------
#  4.MacOS - AI回答的文本转语音输出
#-------------------------------
# 调用系统输出语音
os.system(f'say "{result}"')
print("Done.")


"""
    st_innocent/run.py
"""

import os
import sys
import uvicorn
import argparse
import threading
import subprocess
from server.config import settings
from server.logger import CustomLogger, logger as server_logger

def client_dev():
    """
    Start the client application in dev mode.
    """

    client_logger = CustomLogger.make_logger("client")
    client_logger.info("Starting next.js client.")
    with subprocess.Popen(
        "cd client && yarn run dev",
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        encoding="utf-8",
        shell=True,
        env=os.environ,
    ) as proc:
        while proc.poll() is None:
            out = proc.stdout.readline()
            err = proc.stderr.readline()
            out = out.replace("\n", "")
            err = err.replace("\n", "")
            sys.stdout.flush()
            sys.stderr.flush()
            if len(err) > 0:
                client_logger.error(err)
            elif len(out) > 0:
                client_logger.info(out)

def admin_dev():
    """
    Start the admin application in dev mode.
    """
    admin_logger = CustomLogger.make_logger("admin")
    admin_logger.info("Starting admin app.")
    with subprocess.Popen(
        "cd admin && yarn run start",
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        encoding="utf-8",
        shell=True,
        env=os.environ,
    ) as proc:
        while proc.poll() is None:
            out = proc.stdout.readline()
            err = proc.stderr.readline()
            out = out.replace("\n", "")
            err = err.replace("\n", "")
            sys.stdout.flush()
            sys.stderr.flush()
            if len(err) > 0:
                admin_logger.error(err)
            elif len(out) > 0:
                admin_logger.info(out)

def server_dev():
    """
    Start the server in dev mode.
    """
    server_logger.info("Starting fastapi/uvicorn server.")
    uvicorn.run(
        "server.main:app",
        host=settings.host,
        port=int(settings.server_port),
        log_level="debug" if settings.mode == "dev" else "info",
        reload=True,
    )

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        prog="St. Innocent's Webapp",
        description="FastAPI server / Next.js client / CRA Admin",
        epilog="run.py: command line utilities",
    )
    parser.add_argument("-b", "--build", action="store_true", help="production build")
    parser.add_argument("-s", "--server", action="store_true", help="only uvicorn")
    parser.add_argument("-a", "--admin", action="store_true", help="only cra admin")
    parser.add_argument("-d", "--dev", action="store_true", help="development run")
    args = parser.parse_args()

    if args.dev:
        admin = threading.Thread(target=admin_dev, args=())
        client = threading.Thread(target=client_dev, args=())
        client.start()
        admin.start()
        server_dev()
    if args.server:
        server_dev()
    if args.admin:
        admin_dev()
    else:
        server_logger.info(f"{args} not supported.")


from django.urls import path
from . import views
from leave.views import LeaveEntrySearchView

app_name = 'payroll'

component_patterns = [
    path('master/salary-components/', views.PayrollComponentList.as_view(), name='component_list'),
    path('master/salary-components/create/', views.PayrollComponentCreate.as_view(), name='component_create'),
    path('master/salary-components/<pk>/update/', views.PayrollComponentUpdate.as_view(), name='component_update'),
    path('master/salary-components/<pk>/delete/', views.PayrollComponentDelete.as_view(), name='component_delete'),
]

salary_group_patterns = [
    path('master/salary-group/', views.SalaryGroupList.as_view(), name='salary_group_list'),
    path('master/salary-group/create/', views.SalaryGroupCreate.as_view(), name='salary_group_create'),
    path('master/salary-group/<pk>/update/', views.SalaryGroupUpdate.as_view(), name='salary_group_update'),
    path('master/salary-group/<pk>/delete/', views.SalaryGroupDelete.as_view(), name='salary_group_delete'),
]

salary_group_settings_patterns = [
    path('master/salary-group/<pk>/settings', views.SalaryGroupSettingsList.as_view(),
         name='salary_group_settings_list'),
    path('master/salary-group/<pk>/settings/<pk2>/update', views.SalaryGroupSettingsUpdate.as_view(),
         name='salary_group_settings_update'),
]

deduction_component_patterns = [
    path('master/deduction-components/', views.DeductionComponentList.as_view(), name='deduction_component_list'),
    path('master/deduction-components/create/', views.DeductionComponentCreate.as_view(),
         name='deduction_component_create'),
    path('master/deduction-components/<pk>/update/', views.DeductionComponentUpdate.as_view(),
         name='deduction_component_update'),
    path('master/deduction-components/<pk>/delete/', views.DeductionComponentDelete.as_view(),
         name='deduction_component_delete'),
]

deduction_component_setting_patterns = [
    path('master/deduction-components/<pk>/settings/', views.DeductionComponentSettingRedirect.as_view(),
         name='deduction_component_settings'),
    path('master/deduction-components/<pk>/settings/absent/', views.DeductionAbsentSetting.as_view(),
         name='deduction_absent_settings'),
    path('master/deduction-components/<pk>/settings/late/', views.DeductionLateSetting.as_view(),
         name='deduction_late_settings'),
    path('master/deduction-components/<pk>/settings/early-out/', views.DeductionEarlyOutSetting.as_view(),
         name='deduction_early_out_settings'),
    path('master/deduction-components/<pk>/settings/under-work/', views.DeductionUnderWorkSetting.as_view(),
         name='deduction_under_work_settings'),
]

late_slab_patterns = [
    path('master/deduction-components/<pk>/settings/late/slab/create/', views.LateSlabCreate.as_view(),
         name='late_slab_create'),
    path('master/deduction-components/<pk>/settings/late/slab/<pk2>/update/', views.LateSlabUpdate.as_view(),
         name='late_slab_update'),
    path('master/deduction-components/<pk>/settings/late/slab/<pk2>/delete/', views.LateSlabDelete.as_view(),
         name='late_slab_delete'),
]

early_out_slab_patterns = [
    path('master/deduction-components/<pk>/settings/early-out/slab/create/', views.EarlyOutSlabCreate.as_view(),
         name='early_out_slab_create'),
    path('master/deduction-components/<pk>/settings/early-out/slab/<pk2>/update/', views.EarlyOutSlabUpdate.as_view(),
         name='early_out_slab_update'),
    path('master/deduction-components/<pk>/settings/early-out/slab/<pk2>/delete/', views.EarlyOutSlabDelete.as_view(),
         name='early_out_slab_delete'),
]

under_work_slab_patterns = [
    path('master/deduction-components/<pk>/settings/under-work/slab/create/', views.UnderWorkSlabCreate.as_view(),
         name='under_work_slab_create'),
    path('master/deduction-components/<pk>/settings/under-work/slab/<pk2>/update/', views.UnderWorkSlabUpdate.as_view(),
         name='under_work_slab_update'),
    path('master/deduction-components/<pk>/settings/under-work/slab/<pk2>/delete/', views.UnderWorkSlabDelete.as_view(),
         name='under_work_slab_delete'),
]

deduction_group_patterns = [
    path('master/deduction-groups/', views.DeductionGroupList.as_view(), name='deduction_group_list'),
    path('master/deduction-groups/create/', views.DeductionGroupCreate.as_view(),
         name='deduction_group_create'),
    path('master/deduction-groups/<pk>/update/', views.DeductionGroupUpdate.as_view(),
         name='deduction_group_update'),
    path('master/deduction-groups/<pk>/delete/', views.DeductionGroupDelete.as_view(),
         name='deduction_group_delete'),
]

bonus_component_patterns = [
    path('master/bonus-components/', views.BonusComponentList.as_view(), name='bonus_component_list'),
    path('master/bonus-components/create/', views.BonusComponentCreate.as_view(), name='bonus_component_create'),
    path('master/bonus-components/<pk>/update/', views.BonusComponentUpdate.as_view(), name='bonus_component_update'),
    path('master/bonus-components/<pk>/delete/', views.BonusComponentDelete.as_view(), name='bonus_component_delete'),
]

bonus_group_patterns = [
    path('master/bonus-groups/', views.BonusGroupList.as_view(), name='bonus_group_list'),
    path('master/bonus-groups/create/', views.BonusGroupCreate.as_view(), name='bonus_group_create'),
    path('master/bonus-groups/<pk>/update/', views.BonusGroupUpdate.as_view(), name='bonus_group_update'),
    path('master/bonus-groups/<pk>/delete/', views.BonusGroupDelete.as_view(), name='bonus_group_delete'),
]

bonus_component_settings_patterns = [
    path('master/bonus-components/<pk>/settings/', views.BonusComponentSettingsUpdate.as_view(),
         name='bonus_component_settings'),
]

# Updated

grade_patterns = [
    path('grade/', views.GradeListView.as_view(), name='paygrade_list'),
    path('grade/create/', views.GradeCreateView.as_view(), name='paygrade_create'),
    path('grade/<pk>/update/', views.GradeUpdateView.as_view(), name='paygrade_update'),
    path('grade/<pk>/delete/', views.GradeDeleteView.as_view(), name='paygrade_delete'),
]

# pay_scale_patterns = [
#     path('pay-scale/', views.PayScaleListView.as_view(), name='pay_scale_list'),
#     path('pay-scale/create/', views.PayScaleCreateView.as_view(), name='pay_scale_create'),
#     path('pay-scale/<pk>/update/', views.PayScaleUpdateView.as_view(), name='pay_scale_update'),
#     path('pay-scale/<pk>/delete/', views.PayScaleDeleteView.as_view(), name='pay_scale_delete'),
# ]

# process patterns

payroll_process_patterns = [
    path('process/salary/', views.SalaryProcessGenerate.as_view(), name='salary_process_employee_selection'),
]

payroll_adjustment_patterns = [
    path('process/salary/adjustment/', views.SalaryAdjustmentGenerate.as_view(), name='salary_adjustment'),
    path('process/salary/adjustment/leave_entry_search/', LeaveEntrySearchView.as_view(), name='entry_search_employee'),
    path('process/salary/adjustment/<pk>/', views.SalaryAdjustmentList.as_view(), name='employee_salary_adjust_list'),
    path('process/salary/confirmation/', views.SalaryConfirmationGenerate.as_view(), name='salary_confirmation'),
    path('process/salary/confirmation/leave_entry_search/', LeaveEntrySearchView.as_view(), name='entry_search_employee'),
    path('process/salary/disbursement/', views.SalaryDisbursementGenerate.as_view(), name='salary_disbursement'),
    path('process/salary/disbursement/leave_entry_search/', LeaveEntrySearchView.as_view(), name='entry_search_employee'),
    path('process/salary/disbursement/<pk>/details/', views.SalaryDisbursementDetails.as_view(), name='salary_disbursement_details'),
    path('process/salary/disbursement/<pk>/edit/', views.SalaryDisbursementEdit.as_view(), name='salary_disbursement_edit'),
    path('process/salary/disbursement/group/<pk>/', views.SalaryDisbursementGroup.as_view(), name='salary_disbursement_all'),
    path('process/salary/disbursement/<pk>/salary_breakdown/', views.SalaryBreakdownlList.as_view(), name='salary_breakdown_details'),
]

payroll_report_patterns = [
    path('report/payslip/', views.PayslipReportView.as_view(), name='report_payslip'),
    path('report/payslip/leave_entry_search/', LeaveEntrySearchView.as_view(), name='leave_entry_search'),
    path('report/pay-sheet/', views.PaySheetReportView.as_view(), name='report_pay_sheet'),
    path('report/pay-sheet/leave_entry_search/', LeaveEntrySearchView.as_view(), name='leave_entry_search'),
    path('report/pay-suggestions/', views.PaySuggestionsReportView.as_view(), name='report_pay_suggestions'),
    path('report/pay-suggestions/leave_entry_search/', LeaveEntrySearchView.as_view(), name='leave_entry_search'),
]

urlpatterns = component_patterns + salary_group_patterns + salary_group_settings_patterns + \
              deduction_component_patterns + deduction_group_patterns + bonus_component_patterns +\
              bonus_group_patterns + bonus_component_settings_patterns + deduction_component_setting_patterns +\
              late_slab_patterns + early_out_slab_patterns + under_work_slab_patterns + grade_patterns + \
              payroll_process_patterns + payroll_adjustment_patterns + payroll_report_patterns


x=lambda a:a+10
print("Sum = ",x(20))

print()

x= lambda a,b:a+b
print("Sum is ",x(20,30))

print()

def table(n):
    x = lambda a:a*n
    return x
n=int(input("Enter the number"))
b=table(n)

for i in range(1,11):
    print(n,"x",i,"=",b(i))

#finding factorial of a number

print()

def factorial(n):
    x = lambda a:n*i
    return x
n=int(input("Enter the number"))
b=factorial(n)
f=1
for i in range(1,n+1):
    print(b(i))


from django.db import models
from bs4 import BeautifulSoup
import requests

def do_if_sub(table, sub_tag):
	for part in table:
		leg = part.find_all(sub_tag)
		if len(leg) > 1:
			array = []
			for splinter in leg:
				if sub_tag == 'a':
					array.append(splinter['href'])
				if sub_tag != 'a':
					try:
						array.append(splinter.text)
					except:
						array.append(splinter)
			return array
		if len(leg) == 1:
			try:
				return leg.text
			except:
				return leg

def do_no_sub(table, main_tag):
	if len(table) > 1:
		array = []
		for item in table:
			if main_tag == 'a':
				array.append(item['href'])
			if main_tag != 'a':
				try:
					array.append(item.text)
				except:
					array.append(item)
		return array
	if len(table) == 1:
		try:
			return table.text
		except:
			return table

def page(hyper, main_tag=None, sub_tag=None, class_name=None, id_name=None):
	req_headers = {
			'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
			'accept-encoding': 'gzip, deflate, br',
			'accept-language': 'en-US,en;q=0.8',
			'upgrade-insecure-requests': '1',
			'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
		}
	source = requests.get(hyper,headers=req_headers).text
	soup = BeautifulSoup(source, 'lxml')
	if class_name == '' and id_name == '':
		table = soup.find_all(main_tag)
		if sub_tag == '':
			return do_no_sub(table, main_tag)
		if sub_tag != '':
			return do_if_sub(table, sub_tag)

	if class_name and id_name == '':
		table = soup.find_all(main_tag, class_=class_name)
		if sub_tag == '':
			return do_no_sub(table, main_tag)
		if sub_tag != '':
			return do_if_sub(table, sub_tag)

	if class_name == '' and id_name:
		table = soup.find(main_tag, id=id_name)
		if sub_tag == 'a':
			array = []
			leg = table.find_all(sub_tag)
			for splinter in leg:
				array.append(splinter['href'])
			return array
		if sub_tag != 'a':
			return [table.text]

class URL(models.Model):
	main_tag = models.CharField(max_length=7, default='div')
	sub_tag = models.CharField(max_length=7, default='a')
	class_name = models.CharField(max_length=100, default='')
	id_name = models.CharField(max_length=100, default='')
	url = models.URLField(max_length=200)
	get_date = models.DateTimeField('date scrapped')
	def __str__(self):
		return self.url

	def get_data(self):
		return page(self.url, self.main_tag, self.sub_tag, self.class_name, self.id_name)


# import sys
# sys.stdin = open("input.txt", 'rt', encoding='UTF8')

for i in range(10):
    N = int(input())
    find = input()
    S = input()

    count = S.count(find)
    print(f"#{i+1} {count}")


# Copyright (c) 2001-2020 Seagate Technology LLC and/or its Affiliates
#
# This program is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <https://www.gnu.org/licenses/>. For any questions
# about this software or licensing, please email opensource@seagate.com or
# cortx-questions@seagate.com.

"""
 ****************************************************************************
  Description:       Message Handler for requesting node data
 ****************************************************************************
"""

import json
import time
import os

from framework.base.internal_msgQ import InternalMsgQ
from framework.base.module_thread import ScheduledModuleThread
from framework.base.sspl_constants import enabled_products, DATA_PATH
from framework.utils.conf_utils import (GLOBAL_CONF, SSPL_CONF, Conf,
                                        NODE_ID_KEY)
from framework.utils.service_logging import logger
from framework.utils.severity_reader import SeverityReader
from json_msgs.messages.sensors.cpu_data import CPUdataMsg
from json_msgs.messages.sensors.disk_space_alert import DiskSpaceAlertMsg
from json_msgs.messages.sensors.host_update import HostUpdateMsg
from json_msgs.messages.sensors.if_data import IFdataMsg
from json_msgs.messages.sensors.local_mount_data import LocalMountDataMsg
from json_msgs.messages.sensors.node_hw_data import NodeIPMIDataMsg
from json_msgs.messages.sensors.raid_data import RAIDdataMsg
from json_msgs.messages.sensors.raid_integrity_msg import RAIDIntegrityMsg
from framework.messaging.egress_processor import EgressProcessor
from framework.utils.store_factory import file_store
from framework.utils.utility import Utility

# Override default store
store = file_store

class NodeDataMsgHandler(ScheduledModuleThread, InternalMsgQ):
    """Message Handler for generic node requests and generating
        host update messages on a regular interval"""

    MODULE_NAME = "NodeDataMsgHandler"
    PRIORITY    = 2

    # Section and keys in configuration file
    NODEDATAMSGHANDLER = MODULE_NAME.upper()
    TRANSMIT_INTERVAL = 'transmit_interval'
    HIGH_CPU_USAGE_WAIT_THRESHOLD = 'high_cpu_usage_wait_threshold'
    HIGH_MEMORY_USAGE_WAIT_THRESHOLD = 'high_memory_usage_wait_threshold'
    UNITS = 'units'
    DISK_USAGE_THRESHOLD = 'disk_usage_threshold'
    DEFAULT_DISK_USAGE_THRESHOLD = 80
    CPU_USAGE_THRESHOLD = 'cpu_usage_threshold'
    DEFAULT_CPU_USAGE_THRESHOLD = 80
    HOST_MEMORY_USAGE_THRESHOLD = 'host_memory_usage_threshold'
    DEFAULT_HOST_MEMORY_USAGE_THRESHOLD = 80

    SYSTEM_INFORMATION = "SYSTEM_INFORMATION"

    IPMI_RESOURCE_TYPE_PSU = "node:hw:psu"
    IPMI_RESOURCE_TYPE_FAN = "node:hw:fan"
    IPMI_RESOURCE_TYPE_DISK = "node:hw:disk"
    IPMI_RESOURCE_TYPE_TEMPERATURE = "node:sensor:temperature"
    IPMI_RESOURCE_TYPE_VOLTAGE = "node:sensor:voltage"
    IPMI_RESOURCE_TYPE_CURRENT = "node:sensor:current"
    NW_RESOURCE_TYPE = "node:interface:nw"
    NW_CABLE_RESOURCE_TYPE = "node:interface:nw:cable"
    high_usage = {
        'cpu' : False,
        'disk' : False,
        'memory' : False
    }
    usage_time_map = {
        'cpu' : -1,
        'memory' : -1,
        'disk' : -1
    }
    fault_resolved_iterations = {
        'cpu': 0,
        'memory': 0,
        'disk': 0
    }
    prev_nw_if_status = {}
    prev_cable_status = {}
    FAULT = "fault"
    FAULT_RESOLVED = "fault_resolved"

    INTERFACE_FAULT_DETECTED = False

    CACHE_DIR_NAME = "server"

    # Dependency list
    DEPENDENCIES = {
                    "plugins": ["EgressProcessor"],
                    "rpms": []
    }

    @staticmethod
    def name():
        """ @return: name of the module."""
        return NodeDataMsgHandler.MODULE_NAME

    def __init__(self):
        super(NodeDataMsgHandler, self).__init__(self.MODULE_NAME,
                                                  self.PRIORITY)
        # Flag to indicate suspension of module
        self._suspended = False

    @staticmethod
    def dependencies():
        """Returns a list of plugins and RPMs this module requires
           to function.
        """
        return NodeDataMsgHandler.DEPENDENCIES

    def initialize(self, conf_reader, msgQlist, product):
        """initialize configuration reader and internal msg queues"""
        # Initialize ScheduledMonitorThread
        super(NodeDataMsgHandler, self).initialize(conf_reader)

        # Initialize internal message queues for this module
        super(NodeDataMsgHandler, self).initialize_msgQ(msgQlist)

        self._transmit_interval = int(Conf.get(SSPL_CONF, f"{self.NODEDATAMSGHANDLER}>{self.TRANSMIT_INTERVAL}",
                                               60))
        self._high_cpu_usage_wait_threshold = int(Conf.get(SSPL_CONF,
                                                f"{self.NODEDATAMSGHANDLER}>{self.HIGH_CPU_USAGE_WAIT_THRESHOLD}",60))
        self._high_memory_usage_wait_threshold = int(Conf.get(SSPL_CONF,
                                                f"{self.NODEDATAMSGHANDLER}>{self.HIGH_MEMORY_USAGE_WAIT_THRESHOLD}",60))
        self._units = Conf.get(SSPL_CONF, f"{self.NODEDATAMSGHANDLER}>{self.UNITS}",
                                                "MB")
        self._disk_usage_threshold = Conf.get(SSPL_CONF, f"{self.NODEDATAMSGHANDLER}>{self.DISK_USAGE_THRESHOLD}",
                                                self.DEFAULT_DISK_USAGE_THRESHOLD)

        self._cpu_usage_threshold = Conf.get(SSPL_CONF, f"{self.NODEDATAMSGHANDLER}>{self.CPU_USAGE_THRESHOLD}",
                                                self.DEFAULT_CPU_USAGE_THRESHOLD)

        self._host_memory_usage_threshold = Conf.get(SSPL_CONF, f"{self.NODEDATAMSGHANDLER}>{self.HOST_MEMORY_USAGE_THRESHOLD}",
                                                self.DEFAULT_HOST_MEMORY_USAGE_THRESHOLD)

        self.node_id = Conf.get(GLOBAL_CONF, NODE_ID_KEY, "SN01")

        self.bmcNwStatus = None
        self.severity_reader = SeverityReader()
        self._node_sensor    = None
        self._login_actuator = None
        self.disk_sensor_data = None
        self.host_sensor_data = None
        self.if_sensor_data = None
        self.cpu_sensor_data = None
        self.raid_sensor_data = None
        self.sensor_type = None
        self._epoch_time = str(int(time.time()))
        self._raid_drives = []
        self._raid_device = "N/A"
        self.os_sensor_type = {
            "disk_space" : self.disk_sensor_data,
            "memory_usage" : self.host_sensor_data,
            "nw"   : self.if_sensor_data,
            "cpu_usage"  : self.cpu_sensor_data,
            "raid_data" : self.raid_sensor_data
        }

        # UUID used in json msgs
        self._uuid = None

        # Dict of drives by device name from systemd
        self._drive_by_device_name = {}

        # Dict of drive path by-ids by serial number from systemd
        self._drive_byid_by_serial_number = {}

        self._import_products(product)
        self.cache_dir_path = os.path.join(DATA_PATH, self.CACHE_DIR_NAME)

        self.persistent_data = {
            'cpu' : {},
            'disk' : {},
            'memory' : {},
            'nw' : {}
        }
        # Persistent Cache for High CPU usage
        self.init_from_persistent_cache('cpu', 'CPU_USAGE_DATA')
        # Persistent Cache for High Disk Usage
        self.init_from_persistent_cache('disk', 'DISK_USAGE_DATA')
        # Persistent Cache for High Memory Usage
        self.init_from_persistent_cache('memory', 'MEMORY_USAGE_DATA')
        # Persistent Cache for Nework sensor
        self.init_from_persistent_cache('nw', 'NW_SENSOR_DATA')

    def init_from_persistent_cache(self, resource, data_path):
        PER_DATA_PATH = os.path.join(self.cache_dir_path,
                            f'{data_path}_{self.node_id}')

        if os.path.isfile(PER_DATA_PATH):
            self.persistent_data[resource] = store.get(PER_DATA_PATH)
        if self.persistent_data[resource]:
            if resource == 'nw':
                self.prev_nw_if_status = \
                    self.persistent_data[resource].get('prev_nw_if_status', {})
                self.prev_cable_status = \
                    self.persistent_data[resource].get('prev_cable_status', {})
            elif self.persistent_data[resource]\
                [f'high_{resource}_usage'].lower() == "true":
                self.high_usage[resource] = True
            else:
                self.high_usage[resource] = False
        else:
            self.persist_state_data(resource, data_path)

    def persist_state_data(self, resource, data_path):
        PER_DATA_PATH = os.path.join(self.cache_dir_path,
                            f'{data_path}_{self.node_id}')
        if resource == 'nw':
            self.persistent_data[resource] = {
                'prev_nw_if_status': self.prev_nw_if_status,
                'prev_cable_status': self.prev_cable_status
            }
        else:
            self.persistent_data[resource] = {
                f'high_{resource}_usage': str(self.high_usage[resource]),
                f'{resource}_usage_time_map':
                    str(self.usage_time_map[resource]),
                f'{resource}_fault_resolved_iterations':
                    str(self.fault_resolved_iterations[resource])
            }
        store.put(self.persistent_data[resource], PER_DATA_PATH)

    def read_persistent_data(self, data_path):
        """Read resource data from persistent cache."""
        persistent_data = {}
        PER_DATA_PATH = os.path.join(self.cache_dir_path,
                            f'{data_path}_{self.node_id}')
        if os.path.isfile(PER_DATA_PATH):
            persistent_data = store.get(PER_DATA_PATH)
        return persistent_data

    def _import_products(self, product):
        """Import classes based on which product is being used"""
        if product.lower() in [x.lower() for x in enabled_products]:
            from zope.component import queryUtility
            self._queryUtility = queryUtility

    def run(self):
        """Run the module periodically on its own thread."""
        self._log_debug("Start accepting requests")

        # Do not proceed if module is suspended
        if self._suspended == True:
            self._scheduler.enter(1, self._priority, self.run, ())
            return

        # self._set_debug(True)
        # self._set_debug_persist(True)

        try:
            # Query the Zope GlobalSiteManager for an object implementing INodeData
            if self._node_sensor is None:
                from sensors.INode_data import INodeData
                self._node_sensor = self._queryUtility(INodeData)()
                self._log_debug("_node_sensor name: %s" % self._node_sensor.name())

            # Delay for the desired interval if it's greater than zero
            if self._transmit_interval > 0:
                logger.debug("self._transmit_interval:{}".format(self._transmit_interval))
                timer = self._transmit_interval
                while timer > 0:
                    # See if the message queue contains an entry and process
                    jsonMsg, _ = self._read_my_msgQ_noWait()
                    if jsonMsg is not None:
                        self._process_msg(jsonMsg)

                    time.sleep(1)
                    timer -= 1

                # Generate the JSON messages with data from the node and transmit on regular interval
                self._generate_host_update()
                self._generate_cpu_data()
                self._generate_if_data()
                self._generate_disk_space_alert()

            # If the timer is zero then block for incoming requests notifying to transmit data
            else:
                # Block on message queue until it contains an entry
                jsonMsg, _ = self._read_my_msgQ()
                if jsonMsg is not None:
                    self._process_msg(jsonMsg)

                # Keep processing until the message queue is empty
                while not self._is_my_msgQ_empty():
                    jsonMsg, _ = self._read_my_msgQ()
                    if jsonMsg is not None:
                        self._process_msg(jsonMsg)

        except Exception as ae:
            # Log it and restart the whole process when a failure occurs
            logger.exception("NodeDataMsgHandler restarting: %s" % ae)

        self._scheduler.enter(1, self._priority, self.run, ())
        self._log_debug("Finished processing successfully")

    def _process_msg(self, jsonMsg):
        """Parses the incoming message and generate the desired data message"""
        self._log_debug("_process_msg, jsonMsg: %s" % jsonMsg)

        if isinstance(jsonMsg, dict) is False:
            jsonMsg = json.loads(jsonMsg)

        # Parse out the uuid so that it can be sent back in response message
        self._uuid = None
        if jsonMsg.get("sspl_ll_msg_header") is not None and \
           jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None:
            self._uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid")
            self._log_debug("_processMsg, uuid: %s" % self._uuid)

        if jsonMsg.get("sensor_request_type") is not None and \
           jsonMsg.get("sensor_request_type").get("node_data") is not None and \
           jsonMsg.get("sensor_request_type").get("node_data").get("sensor_type") is not None:
            self.sensor_type = jsonMsg.get("sensor_request_type").get("node_data").get("sensor_type").split(":")[2]
            self._log_debug("_processMsg, sensor_type: %s" % self.sensor_type)

            if self.sensor_type == "memory_usage":
                self._generate_host_update()
                sensor_message_type = self.os_sensor_type.get(self.sensor_type, "")
                if sensor_message_type:
                    self._write_internal_msgQ(EgressProcessor.name(),
                                          sensor_message_type)
                else:
                    self._log_debug(f"NodeDataMsgHandler, _process_msg, \
                        No past data found for {self.sensor_type} sensor type")

            elif self.sensor_type == "cpu_usage":
                self._generate_cpu_data()
                sensor_message_type = self.os_sensor_type.get(self.sensor_type, "")
                if sensor_message_type:
                    self._write_internal_msgQ(EgressProcessor.name(),
                                          sensor_message_type)
                else:
                    self._log_debug(f"NodeDataMsgHandler, _process_msg, \
                        No past data found for {self.sensor_type} sensor type")

            elif self.sensor_type == "nw":
                self._generate_if_data()
                sensor_message_type = self.os_sensor_type.get(self.sensor_type, "")
                if sensor_message_type:
                    self._write_internal_msgQ(EgressProcessor.name(),
                                          sensor_message_type)
                else:
                    self._log_debug(f"NodeDataMsgHandler, _process_msg, \
                        No past data found for {self.sensor_type} sensor type")

            elif self.sensor_type == "disk_space":
                self._generate_disk_space_alert()
                sensor_message_type = self.os_sensor_type.get(self.sensor_type, "")
                if sensor_message_type:
                    self._write_internal_msgQ(EgressProcessor.name(),
                                          sensor_message_type)
                else:
                    self._log_debug(f"NodeDataMsgHandler, _process_msg, \
                        No past data found for {self.sensor_type} sensor type")

            elif self.sensor_type == "raid_data":
                self._generate_raid_data(jsonMsg)
                sensor_message_type = self.os_sensor_type.get(self.sensor_type, "")
                if sensor_message_type:
                    self._write_internal_msgQ(EgressProcessor.name(),
                                            sensor_message_type)
                else:
                    self._log_debug("NodeDataMsgHandler, _process_msg " +
                            f"No past data found for {self.sensor_type} sensor type")

            elif self.sensor_type == "raid_integrity":
                self._generate_raid_integrity_data(jsonMsg)
                sensor_message_type = self.os_sensor_type.get(self.sensor_type, "")
                if sensor_message_type:
                    self._write_internal_msgQ(EgressProcessor.name(),
                                            sensor_message_type)
                else:
                    self._log_debug("NodeDataMsgHandler, _process_msg " +
                            f"No past data found for {self.sensor_type} sensor type")

        # Update mapping of device names to serial numbers for global use
        elif jsonMsg.get("sensor_response_type") is not None:
            if jsonMsg.get("sensor_response_type") == "devicename_serialnumber":
                self._update_devicename_sn_dict(jsonMsg)
        elif jsonMsg.get("sensor_request_type") is not None and \
            jsonMsg.get("sensor_request_type").get("node_data") is not None and \
            jsonMsg.get("sensor_request_type").get("node_data").get("info") is not None and \
            jsonMsg.get("sensor_request_type").get("node_data").get("info").get("resource_type") is not None:
                self._generate_node_fru_data(jsonMsg)

        # ... handle other node sensor message types

    def _update_devicename_sn_dict(self, jsonMsg):
        """Update the dict of device names to serial numbers"""
        drive_byid = jsonMsg.get("drive_byid")
        device_name = jsonMsg.get("device_name")
        serial_number = jsonMsg.get("serial_number")

        self._drive_by_device_name[device_name] = serial_number
        self._drive_byid_by_serial_number[serial_number] = drive_byid

        self._log_debug("NodeDataMsgHandler, device_name: %s, serial_number: %s, drive_byid: %s" %
                        (device_name, serial_number, drive_byid))

    def _generate_host_update(self):
        """Create & transmit a host update message as defined
            by the sensor response json schema"""

        current_time = Utility.get_current_time()

        # Notify the node sensor to update its data required for the host_update message
        successful = self._node_sensor.read_data("host_update", self._get_debug(), self._units)
        if not successful:
            logger.error("NodeDataMsgHandler, _generate_host_update was NOT successful.")

        self._host_memory_usage_threshold = str(self._host_memory_usage_threshold)
        try:
            if self._host_memory_usage_threshold.isdigit():
                self._host_memory_usage_threshold = int(self._host_memory_usage_threshold)
            else:
                self._host_memory_usage_threshold = float(self._host_memory_usage_threshold)
        except ValueError:
            logger.warn("Host Memory Alert, Invalid host_memory_usage_threshold value are entered in config.")
            # Assigning default value to _memory_usage_threshold
            self._host_memory_usage_threshold = self.DEFAULT_HOST_MEMORY_USAGE_THRESHOLD

        memory_persistent_data = self.read_persistent_data('MEMORY_USAGE_DATA')
        if memory_persistent_data.get('memory_usage_time_map') is not None:
            previous_check_time = int(memory_persistent_data['memory_usage_time_map'])
        else:
            previous_check_time = int(-1)
        if memory_persistent_data\
                .get('memory_fault_resolved_iterations') is not None:
            fault_resolved_iters = int(
                memory_persistent_data['memory_fault_resolved_iterations'])
        else:
            fault_resolved_iters = 0
        try:
            iteration_limit = int(
                self._high_memory_usage_wait_threshold/self._transmit_interval)
        except ZeroDivisionError:
            iteration_limit = 0
        self.usage_time_map['memory'] = current_time

        if self._node_sensor.total_memory["percent"] >= self._host_memory_usage_threshold \
           and not self.high_usage['memory']:
            if previous_check_time == -1:
                previous_check_time = current_time
                self.persist_state_data('memory', 'MEMORY_USAGE_DATA')

            if self.usage_time_map['memory'] - previous_check_time >= self._high_memory_usage_wait_threshold:
                self.high_usage['memory'] = True
                self.fault_resolved_iterations['memory'] = 0
                # Create the memory data message and hand it over
                # to the egress processor to transmit
                fault_event = "Host memory usage has increased to {}%,"\
                    "beyond the configured threshold of {}% "\
                    "for more than {} seconds.".format(
                        self._node_sensor.total_memory["percent"],
                        self._host_memory_usage_threshold,
                        self._high_memory_usage_wait_threshold
                    )

                logger.warn(fault_event)

                logged_in_users = []
                # Create the host update message and hand it over to the egress processor to transmit
                hostUpdateMsg = HostUpdateMsg(self._node_sensor.host_id,
                                        self._epoch_time,
                                        self._node_sensor.boot_time,
                                        self._node_sensor.up_time,
                                        self._node_sensor.uname, self._units,
                                        self._node_sensor.total_memory,
                                        self._node_sensor.logged_in_users,
                                        self._node_sensor.process_count,
                                        self._node_sensor.running_process_count,
                                        self.FAULT,
                                        fault_event
                                        )
                # Add in uuid if it was present in the json request
                if self._uuid is not None:
                    hostUpdateMsg.set_uuid(self._uuid)
                jsonMsg = hostUpdateMsg.getJson()
                # Transmit it to message processor
                self.host_sensor_data = jsonMsg
                self.os_sensor_type["memory_usage"] = self.host_sensor_data
                self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
                self.persist_state_data('memory', 'MEMORY_USAGE_DATA')

        if self._node_sensor.total_memory["percent"] < self._host_memory_usage_threshold:
            if not self.high_usage['memory']:
                self.persist_state_data('memory', 'MEMORY_USAGE_DATA')
            else:
                if fault_resolved_iters < iteration_limit:
                    fault_resolved_iters += 1
                    self.fault_resolved_iterations['memory'] = fault_resolved_iters
                    self.persist_state_data('memory', 'MEMORY_USAGE_DATA')
                elif fault_resolved_iters >= iteration_limit:
                    # Create the memory data message and hand it over
                    # to the egress processor to transmit
                    fault_resolved_event = "Host memory usage has decreased to {}%, "\
                        "lower than the configured threshold of {}%.".format(
                            self._node_sensor.total_memory["percent"],
                            self._host_memory_usage_threshold
                        )
                    logger.info(fault_resolved_event)
                    logged_in_users = []

                    # Create the host update message and hand it over to the egress processor to transmit
                    hostUpdateMsg = HostUpdateMsg(self._node_sensor.host_id,
                                            self._epoch_time,
                                            self._node_sensor.boot_time,
                                            self._node_sensor.up_time,
                                            self._node_sensor.uname, self._units,
                                            self._node_sensor.total_memory,
                                            self._node_sensor.logged_in_users,
                                            self._node_sensor.process_count,
                                            self._node_sensor.running_process_count,
                                            self.FAULT_RESOLVED,
                                            fault_resolved_event
                                            )

                    # Add in uuid if it was present in the json request
                    if self._uuid is not None:
                        hostUpdateMsg.set_uuid(self._uuid)
                    jsonMsg = hostUpdateMsg.getJson()
                    # Transmit it to message processor
                    self.host_sensor_data = jsonMsg
                    self.os_sensor_type["memory_usage"] = self.host_sensor_data
                    self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
                    self.high_usage['memory'] = False
                    self.usage_time_map['memory'] = int(-1)
                    self.fault_resolved_iterations['memory'] = 0
                    self.persist_state_data('memory', 'MEMORY_USAGE_DATA')

    def _generate_local_mount_data(self):
        """Create & transmit a local_mount_data message as defined
            by the sensor response json schema"""

        # Notify the node sensor to update its data required for the local_mount_data message
        successful = self._node_sensor.read_data("local_mount_data", self._get_debug(), self._units)
        if not successful:
            logger.error("NodeDataMsgHandler, _generate_local_mount_data was NOT successful.")

        # Create the local mount data message and hand it over to the egress processor to transmit
        localMountDataMsg = LocalMountDataMsg(self._node_sensor.host_id,
                                self._epoch_time,
                                self._node_sensor.free_space,
                                self._node_sensor.free_inodes,
                                self._node_sensor.free_swap,
                                self._node_sensor.total_space,
                                self._node_sensor.total_swap,
                                self._units)

        # Add in uuid if it was present in the json request
        if self._uuid is not None:
            localMountDataMsg.set_uuid(self._uuid)
        jsonMsg = localMountDataMsg.getJson()

        # Transmit it to message processor
        self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)

    def _generate_cpu_data(self):
        """Create & transmit a cpu_data message as defined
            by the sensor response json schema"""

        current_time = Utility.get_current_time()

        # Notify the node sensor to update its data required for the cpu_data message
        successful = self._node_sensor.read_data("cpu_data", self._get_debug())
        if not successful:
            logger.error("NodeDataMsgHandler, _generate_cpu_data was NOT successful.")

        self._cpu_usage_threshold = str(self._cpu_usage_threshold)
        try:
            if self._cpu_usage_threshold.isdigit():
                self._cpu_usage_threshold = int(self._cpu_usage_threshold)
            else:
                self._cpu_usage_threshold = float(self._cpu_usage_threshold)
        except ValueError:
            logger.warn("CPU Usage Alert, Invalid host_memory_usage_threshold value are entered in config.")
            # Assigning default value to _cpu_usage_threshold
            self._cpu_usage_threshold = self.DEFAULT_CPU_USAGE_THRESHOLD

        cpu_persistent_data = self.read_persistent_data('CPU_USAGE_DATA')
        if cpu_persistent_data.get('cpu_usage_time_map') is not None:
            previous_check_time = int(cpu_persistent_data['cpu_usage_time_map'])
        else:
            previous_check_time = int(-1)
        if cpu_persistent_data.get('cpu_fault_resolved_iterations') is not None:
            fault_resolved_iters = int(
                cpu_persistent_data['cpu_fault_resolved_iterations'])
        else:
            fault_resolved_iters = 0
        try:
            iteration_limit = int(
                self._high_cpu_usage_wait_threshold/self._transmit_interval)
        except ZeroDivisionError:
            iteration_limit = 0
        self.usage_time_map['cpu'] = current_time

        if self._node_sensor.cpu_usage >= self._cpu_usage_threshold \
           and not self.high_usage['cpu']:
            if previous_check_time == -1:
                previous_check_time = current_time
                self.persist_state_data('cpu', 'CPU_USAGE_DATA')

            if self.usage_time_map['cpu'] - previous_check_time >= self._high_cpu_usage_wait_threshold:

                self.high_usage['cpu'] = True
                self.fault_resolved_iterations['cpu'] = 0
                # Create the cpu usage data message and hand it over
                # to the egress processor to transmit
                fault_event = "CPU usage has increased to {}%, "\
                    "beyond the configured threshold of {}% "\
                    "for more than {} seconds.".format(
                        self._node_sensor.cpu_usage,
                        self._cpu_usage_threshold,
                        self._high_cpu_usage_wait_threshold
                    )
                logger.warn(fault_event)

                # Create the cpu usage update message and hand it over to the egress processor to transmit
                cpuDataMsg = CPUdataMsg(self._node_sensor.host_id,
                                    self._epoch_time,
                                    self._node_sensor.csps,
                                    self._node_sensor.idle_time,
                                    self._node_sensor.interrupt_time,
                                    self._node_sensor.iowait_time,
                                    self._node_sensor.nice_time,
                                    self._node_sensor.softirq_time,
                                    self._node_sensor.steal_time,
                                    self._node_sensor.system_time,
                                    self._node_sensor.user_time,
                                    self._node_sensor.cpu_core_data,
                                    self._node_sensor.cpu_usage,
                                    self.FAULT,
                                    fault_event
                                )

                # Add in uuid if it was present in the json request
                if self._uuid is not None:
                    cpuDataMsg.set_uuid(self._uuid)
                jsonMsg = cpuDataMsg.getJson()
                self.cpu_sensor_data = jsonMsg
                self.os_sensor_type["cpu_usage"] = self.cpu_sensor_data

                # Transmit it to message processor
                self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
                # Store the state to Persistent Cache.
                self.persist_state_data('cpu', 'CPU_USAGE_DATA')

        if self._node_sensor.cpu_usage < self._cpu_usage_threshold:
            if not self.high_usage['cpu']:
                self.persist_state_data('cpu', 'CPU_USAGE_DATA')
            else:
                if fault_resolved_iters < iteration_limit:
                    fault_resolved_iters += 1
                    self.fault_resolved_iterations['cpu'] = fault_resolved_iters
                    self.persist_state_data('cpu', 'CPU_USAGE_DATA')
                elif fault_resolved_iters >= iteration_limit:

                    # Create the cpu usage data message and hand it over
                    # to the egress processor to transmit
                    fault_resolved_event = "CPU usage has decreased to {}%, "\
                        "lower than the configured threshold of {}%.".format(
                            self._node_sensor.cpu_usage,
                            self._cpu_usage_threshold
                        )
                    logger.info(fault_resolved_event)

                    # Create the cpu usage update message and hand it over to the egress processor to transmit
                    cpuDataMsg = CPUdataMsg(self._node_sensor.host_id,
                                        self._epoch_time,
                                        self._node_sensor.csps,
                                        self._node_sensor.idle_time,
                                        self._node_sensor.interrupt_time,
                                        self._node_sensor.iowait_time,
                                        self._node_sensor.nice_time,
                                        self._node_sensor.softirq_time,
                                        self._node_sensor.steal_time,
                                        self._node_sensor.system_time,
                                        self._node_sensor.user_time,
                                        self._node_sensor.cpu_core_data,
                                        self._node_sensor.cpu_usage,
                                        self.FAULT_RESOLVED,
                                        fault_resolved_event
                                    )

                    # Add in uuid if it was present in the json request
                    if self._uuid is not None:
                        cpuDataMsg.set_uuid(self._uuid)
                    jsonMsg = cpuDataMsg.getJson()
                    self.cpu_sensor_data = jsonMsg
                    self.os_sensor_type["cpu_usage"] = self.cpu_sensor_data

                    # Transmit it to message processor
                    self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
                    self.high_usage['cpu'] = False
                    self.usage_time_map['cpu'] = int(-1)
                    self.fault_resolved_iterations['cpu'] = 0
                    # Store the state to Persistent Cache.
                    self.persist_state_data('cpu', 'CPU_USAGE_DATA')

    def _send_ifdata_json_msg(self, resource_id, resource_type, state, event=""):
        """A resuable method for transmitting IFDataMsg to RMQ and IEM logging"""
        ifDataMsg = IFdataMsg(self._node_sensor.host_id,
                                self._node_sensor.local_time,
                                self._node_sensor.if_data,
                                resource_id,
                                resource_type,
                                state,
                                self.severity_reader.map_severity(state), event)
        # Add in uuid if it was present in the json request
        if self._uuid is not None:
            ifDataMsg.set_uuid(self._uuid)
        jsonMsg = ifDataMsg.getJson()
        self.if_sensor_data = jsonMsg
        self.os_sensor_type["nw"] = self.if_sensor_data

        # Transmit it to message processor
        self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
        self.persist_state_data('nw', 'NW_SENSOR_DATA')

    def _generate_if_data(self):
        """Create & transmit a network interface data message as defined
            by the sensor response json schema"""
        # Notify the node sensor to update its data required for the if_data message
        successful = self._node_sensor.read_data("if_data", self._get_debug())
        if not successful:
            logger.error("NodeDataMsgHandler, _generate_if_data was NOT successful.")
        interfaces = self._node_sensor.if_data

        cable_alert_desc = "Newtork Cable for interface {} is {}."
        nw_alert_desc = "Network interface {} is {}."

        for interface in interfaces:
            if_name = interface["ifId"]
            nw_if_status = interface["nwStatus"].upper()
            # if nw_fault
            if nw_if_status in ["DOWN", "UNKNOWN"]:
                nw_cable_status = interface["nwCableConnStatus"].upper()
                # if nw_cable_disconneted
                if nw_cable_status == "DISCONNECTED":
                    if self.prev_cable_status.get(if_name) != nw_cable_status:
                        # raise nw_cable_fault_alert
                        self.prev_cable_status[if_name] = nw_cable_status
                        self._send_ifdata_json_msg(
                            if_name, self.NW_CABLE_RESOURCE_TYPE, self.FAULT,
                            cable_alert_desc.format(if_name, nw_cable_status))
                # elif nw_cable_connected
                elif nw_cable_status == "CONNECTED":
                    if self.prev_cable_status.get(if_name) == "DISCONNECTED":
                        # raise nw_cable_fault_resolved_alert
                        self.prev_cable_status[if_name] = nw_cable_status
                        self._send_ifdata_json_msg(
                            if_name, self.NW_CABLE_RESOURCE_TYPE,
                            self.FAULT_RESOLVED,
                            cable_alert_desc.format(if_name, nw_cable_status))
                    if self.prev_nw_if_status.get(if_name) != "DOWN":
                        # raise nw_fault_alert
                        self.prev_nw_if_status[if_name] = "DOWN"
                        self._send_ifdata_json_msg(
                            if_name, self.NW_RESOURCE_TYPE, self.FAULT,
                            nw_alert_desc.format(if_name, "DOWN"))
                # nw_cable_state is unknown and nw_is_faulty
                else:
                    if self.prev_nw_if_status.get(if_name) != "DOWN":
                        # raise nw_fault_alert
                        self.prev_nw_if_status[if_name] = "DOWN"
                        self._send_ifdata_json_msg(
                            if_name, self.NW_RESOURCE_TYPE, self.FAULT,
                            nw_alert_desc.format(if_name, "DOWN"))
            else: # no nw_fault
                if self.prev_nw_if_status.get(if_name) == "DOWN":
                    # raise nw_fault_resolved_alert
                    self.prev_nw_if_status[if_name] = "UP"
                    self._send_ifdata_json_msg(
                        if_name, self.NW_RESOURCE_TYPE, self.FAULT_RESOLVED,
                        nw_alert_desc.format(if_name, "UP"))
                if self.prev_cable_status.get(if_name) == "DISCONNECTED":
                    # raise nw_cable_fault_resolved_alert
                    self.prev_cable_status[if_name] = "CONNECTED"
                    self._send_ifdata_json_msg(
                        if_name, self.NW_CABLE_RESOURCE_TYPE,
                        self.FAULT_RESOLVED,
                        cable_alert_desc.format(if_name, "CONNECTED"))

    def _generate_disk_space_alert(self):
        """Create & transmit a disk_space_alert message as defined
            by the sensor response json schema"""

        # Notify the node sensor to update its data required for the disk_space_data message
        successful = self._node_sensor.read_data("disk_space_alert", self._get_debug(), self._units)
        if not successful:
            logger.error("NodeDataMsgHandler, _generate_disk_space_alert was NOT successful.")
            return

        # Changing disk_usage_threshold type according to what value type entered in config file
        self._disk_usage_threshold = str(self._disk_usage_threshold)
        try:
            if self._disk_usage_threshold.isdigit():
                self._disk_usage_threshold = int(self._disk_usage_threshold)
            else:
                self._disk_usage_threshold = float(self._disk_usage_threshold)
        except ValueError:
            logger.warn("Disk Space Alert, Invalid disk_usage_threshold value are entered in config.")
            # Assigning default value to _disk_usage_threshold
            self._disk_usage_threshold = self.DEFAULT_DISK_USAGE_THRESHOLD

        if self._node_sensor.disk_used_percentage >= self._disk_usage_threshold \
           and not self.high_usage['disk']:

            self.high_usage['disk'] = True
            # Create the disk space data message and hand it over
            # to the egress processor to transmit
            fault_event = "Disk usage has increased to {}%, "\
                "beyond the configured threshold of {}%.".format(
                    self._node_sensor.disk_used_percentage,
                    self._disk_usage_threshold
                )
            logger.warn(fault_event)
            diskSpaceAlertMsg = DiskSpaceAlertMsg(self._node_sensor.host_id,
                                    self._epoch_time,
                                    self._node_sensor.total_space,
                                    self._node_sensor.free_space,
                                    self._node_sensor.disk_used_percentage,
                                    self._units,
                                    self.FAULT,fault_event)

            # Add in uuid if it was present in the json request
            if self._uuid is not None:
                diskSpaceAlertMsg.set_uuid(self._uuid)
            jsonMsg = diskSpaceAlertMsg.getJson()
            self.disk_sensor_data = jsonMsg
            self.os_sensor_type["disk_space"] = self.disk_sensor_data

            # Transmit it to message processor
            self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
            # Save the new state in Persistent Cache.
            self.persist_state_data('disk', 'DISK_USAGE_DATA')

        if self._node_sensor.disk_used_percentage <= self._disk_usage_threshold \
           and self.high_usage['disk']:

            # Create the disk space data message and hand it over
            # to the egress processor to transmit
            fault_resolved_event = "Disk usage has decreased to {}%, "\
                "lower than the configured threshold of {}%.".format(
                    self._node_sensor.disk_used_percentage,
                    self._disk_usage_threshold
                )
            logger.info(fault_resolved_event)
            diskSpaceAlertMsg = DiskSpaceAlertMsg(self._node_sensor.host_id,
                                    self._epoch_time,
                                    self._node_sensor.total_space,
                                    self._node_sensor.free_space,
                                    self._node_sensor.disk_used_percentage,
                                    self._units,
                                    self.FAULT_RESOLVED,
                                    fault_resolved_event
                                    )

            # Add in uuid if it was present in the json request
            if self._uuid is not None:
                diskSpaceAlertMsg.set_uuid(self._uuid)
            jsonMsg = diskSpaceAlertMsg.getJson()
            self.disk_sensor_data = jsonMsg
            self.os_sensor_type["disk_space"] = self.disk_sensor_data

            # Transmit it to message processor
            self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
            self.high_usage['disk'] = False
            # Save the new state in Persistent Cache.
            self.persist_state_data('disk', 'DISK_USAGE_DATA')

    def _generate_raid_data(self, jsonMsg):
        """Create & transmit a RAID status data message as defined
            by the sensor response json schema"""
        successful = self._node_sensor.read_data("raid", self._get_debug(), self._units)
        if not successful:
            logger.error("NodeDataMsgHandler, updating RAID information was NOT successful.")
            return

        # See if status is in the msg; ie it's an internal msg from the RAID sensor
        if jsonMsg.get("sensor_request_type").get("node_data").get("status") is not None:
            sensor_request = jsonMsg.get("sensor_request_type").get("node_data")
            host_name = sensor_request.get("host_id")
            alert_type = sensor_request.get("alert_type")
            alert_id = sensor_request.get("alert_id")
            severity = sensor_request.get("severity")
            info = sensor_request.get("info")
            specific_info = sensor_request.get("specific_info")
            self._raid_device = jsonMsg.get("sensor_request_type").get("node_data").get("specific_info").get("device")
            self._raid_drives = list(jsonMsg.get("sensor_request_type").get("node_data").get("specific_info").get("drives"))
            raidDataMsg = RAIDdataMsg(host_name, alert_type, alert_id, severity, info, specific_info)
            # Add in uuid if it was present in the json request
            if self._uuid is not None:
                raidDataMsg.set_uuid(self._uuid)
            jsonMsg = raidDataMsg.getJson()
            self.raid_sensor_data = jsonMsg
            self.os_sensor_type["raid_data"] = self.raid_sensor_data

            # Loop thru each index of drives containing only paths and fill in with s/n
            for drive in self._raid_drives:
                self._log_debug("drive: %s" % str(drive))

                if drive.get("identity") is not None:
                    path = drive.get("identity").get("path")
                    self._log_debug("path: %s" % str(path))

                    # Lookup the serial number from the path
                    serial_number = str(self._drive_by_device_name.get(path))
                    self._log_debug("serial_number: %s" % str(serial_number))
                    if serial_number != "None":
                        drive["identity"]["serialNumber"] = serial_number

                    # Change device path to path-byid
                    drive_byid = str(self._drive_byid_by_serial_number.get(serial_number))
                    if drive_byid != "None":
                        drive["identity"]["path"] = drive_byid

            self._log_debug("_generate_raid_data, host_id: %s, device: %s, drives: %s" %
                    (self._node_sensor.host_id, self._raid_device, str(self._raid_drives)))

    def _generate_raid_integrity_data(self, jsonMsg):
        """Create & transmit a Validate RAID result data message as defined
            by the sensor response json schema"""
        logger.debug("NodeDataMsgHandler, Validating RAID information")

        # See if status is in the msg; ie it's an internal msg from the RAID sensor
        if jsonMsg.get("sensor_request_type").get("node_data").get("status") is not None:
            sensor_request = jsonMsg.get("sensor_request_type").get("node_data")
            host_name = sensor_request.get("host_id")
            alert_type = sensor_request.get("alert_type")
            alert_id = sensor_request.get("alert_id")
            severity = sensor_request.get("severity")
            info = sensor_request.get("info")
            specific_info = sensor_request.get("specific_info")
            self._alert = jsonMsg.get("sensor_request_type").get("node_data").get("specific_info").get("error")
            RAIDintegrityMsg = RAIDIntegrityMsg(host_name, alert_type, alert_id, severity, info, specific_info)
            # Add in uuid if it was present in the json request
            if self._uuid is not None:
                RAIDintegrityMsg.set_uuid(self._uuid)
            jsonMsg = RAIDintegrityMsg.getJson()
            self.raid_integrity_data = jsonMsg
            self.os_sensor_type["raid_integrity"] = self.raid_integrity_data

            self._log_debug("_generate_raid_integrity_data, host_id: %s" %
                    (self._node_sensor.host_id))

    def _generate_node_fru_data(self, jsonMsg):
        """Create & transmit a FRU IPMI data message as defined
            by the sensor response json schema"""

        if self._node_sensor.host_id is None:
            successful = self._node_sensor.read_data("None", self._get_debug(), self._units)
            if not successful:
                logger.error("NodeDataMsgHandler, updating host information was NOT successful.")

        if jsonMsg.get("sensor_request_type").get("node_data") is not None:
            self._fru_info = jsonMsg.get("sensor_request_type").get("node_data")
            node_ipmi_data_msg = NodeIPMIDataMsg(self._fru_info)

        if self._uuid is not None:
            node_ipmi_data_msg.set_uuid(self._uuid)
        jsonMsg = node_ipmi_data_msg.getJson()
        self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)

    def suspend(self):
        """Suspends the module thread. It should be non-blocking"""
        super(NodeDataMsgHandler, self).suspend()
        self._suspended = True

    def resume(self):
        """Resumes the module thread. It should be non-blocking"""
        super(NodeDataMsgHandler, self).resume()
        self._suspended = False

    def shutdown(self):
        """Clean up scheduler queue and gracefully shutdown thread"""
        super(NodeDataMsgHandler, self).shutdown()


import urllib.request
import sys
import os
PATH = os.path.dirname(os.path.abspath(__file__))
NSUBS = int(30) # hard coded into VICE

def download(verbose = True):
	r"""
	Downloads the h277 supplementary data from VICE's source tree on GitHub
	"""
	if not os.path.exists("%s/h277" % (PATH)): os.mkdir("%s/h277" % (PATH))
	for sub in range(NSUBS):
		url = "https://raw.githubusercontent.com/giganano/VICE/v1.3.x/vice/"
		url += "toolkit/hydrodisk/data/h277/sub%d.dat" % (sub)
		urllib.request.urlretrieve(url, "%s/h277/sub%d.dat" % (PATH, sub))
		if verbose: sys.stdout.write("\rDownloading subsample: %d of %d" % (
			sub + 1, NSUBS))
	if verbose: sys.stdout.write("\n")

def _h277_exists():
	r"""
	Determines if the h277 supplementary data has been downloaded.
	"""
	status = True
	for sub in range(NSUBS):
		status &= os.path.exists("%s/h277/sub%d.dat" % (PATH, sub))
		if not status: break
	return status

def _h277_remove():
	r"""
	Removes the h277 supplementary data.
	"""
	try:
		for sub in range(NSUBS):
			filename = "%s/h277/sub%d.dat" % (PATH, sub)
			if os.path.exists(filename): os.remove(filename)
		os.rmdir("%s/h277" % (PATH))
	except (FileNotFoundError, OSError):
		raise FileNotFoundError("Supplementary data not found.")


# -*- coding: UTF-8 -*-

import argparse
import os
import time
import pickle
import json
import logging
import math
import copy
import torch
import sys
import dgl
from random import randint, choice

import scipy.sparse
import pandas as pd
import numpy as np

from utils import utils
from torch.nn.utils.rnn import pad_sequence
from torch import nn

class MetaReader(object):
    @staticmethod
    def parse_data_args(parser):
        parser.add_argument('--path', type=str, default='../data/',
                            help='Input data dir.')
        parser.add_argument('--suffix', type=str, default='MeLON',
                            help='Input data dir of MeLON.')
        parser.add_argument('--dataset', type=str, default='Grocery_and_Gourmet_Food',
                            help='Choose a dataset.')
        parser.add_argument('--sep', type=str, default='\t',
                            help='Sep of csv file.')
        parser.add_argument('--train_ratio', type=float, default=0.8,
                            help='Ratio of the train dataset')
        parser.add_argument('--duplicate', type=int, default=1,
                            help='Coalesce duplicate elements in adjacency matrix')
        parser.add_argument('--fname', type=str, default='freq',
                            help='Freq (> 20 records) or whole')
        return parser

    def __init__(self, args):
        self.sep = args.sep
        self.prefix = args.path
        self.suffix = args.suffix
        self.dataset = args.dataset
        self.train_ratio = args.train_ratio
        self.batch_size = args.batch_size
        self.fname = args.fname
        self.random_seed = args.random_seed

        t0 = time.time()
        self._read_data()
        print(self.data_df['user_id'].max(),len(self.data_df['user_id'].unique()))
        print(self.data_df['item_id'].max(),len(self.data_df['item_id'].unique()))

        logging.info('Counting dataset statistics...')
        self.n_users, self.n_items = self.data_df['user_id'].max()+1, self.data_df['item_id'].max()+1
        self.dataset_size = len(self.data_df)
        self.n_batches = math.ceil(self.dataset_size/self.batch_size)
        logging.info('"# user": {}, "# item": {}, "# entry": {}'.format(self.n_users-1, self.n_items-1, self.dataset_size))

        path = os.path.join(self.prefix, self.dataset, self.suffix, self.fname)
        if not os.path.exists(path):
            os.mkdir(path)
        del path

        logging.info('Saving data into mini-batch pickle files')
        self.user_list = self.data_df['user_id'].to_numpy()

        self._save_user_clicked_set()
        self._save_hist_set()
        self._save_mini_batch()
        self._save_last_batch()
        self._save_heterograph()

        del self.data_df

        logging.info('Done! [{:<.2f} s]'.format(time.time() - t0) + os.linesep)

    def _read_data(self):
        logging.info('Reading data from \"{}\", dataset = \"{}\", suffix = \"{}\", fname = \"{}\" '.format(self.prefix, self.dataset, self.suffix, self.fname))
        df = pd.read_csv(os.path.join(self.prefix, self.dataset, self.suffix, self.fname +'.csv'), sep=self.sep)  # Let the main runner decide the ratio of train/test
        self.data_df = df.loc[:, ['user_id', 'item_id']]#.values.astype(np.int64) # (number of items, 2)

    def _save_user_clicked_set(self):
        user_clicked_set_path = os.path.join(self.prefix, self.dataset, self.suffix, self.fname, 'user_clicked_set.txt')
        logging.info('Load user_clicked_set')

        try:
            self.user_clicked_set = pickle.load(open(user_clicked_set_path, 'rb'))
            logging.info("Successfully loaded saved user_clicked_set")
        except FileNotFoundError as e:
            logging.info('File not found, create user_clicked_set')
            self.user_clicked_set = self.data_df.groupby(['user_id'])['item_id'].unique().to_dict()
            pickle.dump(self.user_clicked_set, open(user_clicked_set_path, 'wb'))
            logging.info('Saved user_clicked_set')

    def _save_hist_set(self):
        '''Prepare previous interactions of users and items during offline meta-model training.
        '''
        item_hist_set_path = os.path.join(self.prefix, self.dataset, self.suffix, self.fname, 'item_hist_set.txt')
        user_hist_set_path = os.path.join(self.prefix, self.dataset, self.suffix, self.fname, 'user_hist_set.txt')
        logging.info('Load hist_set')

        try:
            self.item_hist_set = pickle.load(open(item_hist_set_path, 'rb'))
            self.user_hist_set = pickle.load(open(user_hist_set_path, 'rb'))
            logging.info("Successfully loaded saved hist_set")
        except FileNotFoundError as e:
            logging.info('File not found, create hist_set')
            train_df = self.data_df.loc[:int(self.n_batches*self.train_ratio)*self.batch_size,:]
            self.item_hist_set = train_df.groupby(['item_id'])['user_id'].unique().to_dict()
            self.user_hist_set = train_df.groupby(['user_id'])['item_id'].unique().to_dict()
            pickle.dump(self.item_hist_set, open(item_hist_set_path, 'wb'))
            pickle.dump(self.user_hist_set, open(user_hist_set_path, 'wb'))
            logging.info('Saved hist_set')

    def _save_mini_batch(self):
        self.mini_batch_path = os.path.join(self.prefix, self.dataset, self.suffix, self.fname, 'mini_batch')
        if not os.path.exists(self.mini_batch_path):
            os.mkdir(self.mini_batch_path)

        for batch_idx in range(self.n_batches):
            ui_batch = torch.from_numpy(self.data_df[batch_idx*self.batch_size:(batch_idx+1)*self.batch_size].values.astype(np.int64)) # (batch_size, 2)
            torch.save(ui_batch, open(os.path.join(self.mini_batch_path, str(batch_idx)+'.pt'), 'wb'))

    def _save_last_batch(self):
        '''Save previous interaction of user and item in current mini-batch for online testing.
        Given user-item interaction (u, i) in current mini-batch, Prepare previous interaction of user and item

        The resulting user-item interactions (torch.LongTensor size of [batch_size*2, 2]) are saved in last_batch_path

            # different user, same item: (u', i), (u', -i)
            # for positive: get user, and pick another used item
            # for negative: get user, and pick random negative item

            # same user, different item: (u, i'), (u, -i')
            # for positive: get user, and pick another used item
            # for negative: get user, and pick random negative item

        Exception handling:
            If a user has no previous interactions, then the current item is selected.
            If an item has no previous interactions, then the current user is selected.
            -> Randomly sample previous user/item
        Prepare (torch.LongTensor size of [batch_size*2, 2])
        '''

        self.last_batch_path = os.path.join(self.prefix, self.dataset, self.suffix, self.fname, 'last_batch')
        if not os.path.exists(self.last_batch_path):
            os.mkdir(self.last_batch_path)

        test_start_batch_idx = int(self.n_batches*self.train_ratio)
        train_df = self.data_df.loc[:test_start_batch_idx*self.batch_size,:]

        ui_last = {u:None for u in range(self.n_users)}
        ui_last_ = train_df.groupby(['user_id'])['item_id'].unique().apply(lambda x: x[-1]).to_dict()
        ui_last.update(ui_last_)

        iu_last = {i:None for i in range(self.n_items)}
        iu_last_ = train_df.groupby(['item_id'])['user_id'].unique().apply(lambda x: x[-1]).to_dict()
        iu_last.update(iu_last_)

        for batch_idx in range(test_start_batch_idx, self.n_batches):
            user_id, item_id = torch.from_numpy(self.data_df[batch_idx*self.batch_size:(batch_idx+1)*self.batch_size].values.astype(np.int64)).T # (batch_size, 2)

            # same user, different item: (u, i'), (u, -i')
            # for positive: get user, and pick user's last item
            # for negative: get user, and pick random negative item
            pos_items_u = torch.zeros(size=(len(user_id), 1), dtype=torch.int64)
            neg_items_u = torch.zeros(size=(len(user_id), 1), dtype=torch.int64)
            for idx, user in enumerate(user_id):
                user_clicked_set = copy.deepcopy(self.user_clicked_set[user.item()])
                last_item = ui_last[user.item()]
                pos_items_u[idx] = last_item if last_item is not None else item_id[idx]
                neg_items_u[idx] = self._randint_w_exclude(user_clicked_set)
            items_u = torch.cat((pos_items_u, neg_items_u), axis=-1)

            # different user, same item: (u', i), (u', -i)
            # for positive: get item's last user, and pick item
            # for negative: get item's last user, and pick random negative item
            user_id_ = torch.zeros_like(user_id)
            for idx, item in enumerate(item_id): # pick u'
                last_user = iu_last[item.item()]
                user_id_[idx] = last_user if last_user is not None else user_id[idx]

            neg_items_u_ = torch.zeros(size=(len(user_id), 1), dtype=torch.int64)
            for idx, user in enumerate(user_id_):
                user_clicked_set = copy.deepcopy(self.user_clicked_set[user.item()])
                neg_items_u_[idx] = self._randint_w_exclude(user_clicked_set)
            items_u_ = torch.cat((item_id.reshape(-1, 1), neg_items_u_), axis=-1)

            # Update last interaction history
            for user, item in zip(user_id, item_id):
                ui_last[user.item()] = item.item()
                iu_last[item.item()] = user.item()

            # Save last interaction mini-batch
            user_id = torch.cat((user_id, user_id_), axis=0)
            item_id = torch.cat((items_u, items_u_), axis=0)

            last_batch = {'user_id': user_id, # [batch_size*2,]
                          'item_id': item_id} # [batch_size*2, 2]

            torch.save(last_batch, open(os.path.join(self.last_batch_path, str(batch_idx)+'.pt'), 'wb'))

    def _randint_w_exclude(self, clicked_set):
        randItem = randint(1, self.n_items-1)
        return self._randint_w_exclude(clicked_set) if randItem in clicked_set else randItem

    def _save_heterograph(self):
        self.graph_path = os.path.join(self.prefix, self.dataset, self.suffix, self.fname, 'heterograph_train.bin')
        logging.info('Load heterograph_train')
        try:
            g_list, _ = dgl.load_graphs(self.graph_path,[0])
            self.g = g_list[0]
            logging.info("Successfully loaded heterograph_train")
        except Exception as e:
            logging.info('File not found, create heterograph_train')
            u, i = self.data_df[:int(self.n_batches*self.train_ratio)*self.batch_size].values.T
            self.g = dgl.heterograph({('user', 'u_to_i', 'item'): (u, i),
                                      ('item', 'i_to_u', 'user'): (i, u)},
                                      num_nodes_dict={'user':self.n_users, 'item':self.n_items})
            dgl.save_graphs(self.graph_path, self.g)
            logging.info('Successfully saved heterograph_train')

    # Below are deprecated methods
    def _save_adj_batch(self):
        """Generate user/item history indices/counts (torch.LongTensor) for each batch and save them.
        Process:

            Preparation
            1. Construct offline user-item interaction matrix (scipy.sparse.csr_matrix)

            For each offline batch
            1. Index user/item for each batch and their value
            2. Pad_sequence
            3. Save them

            For each online batch
            1. Change offline matrix to LIL (scipy.sparse.lil_matrix)
            2. Increment user/item interaction
            3. Index user/item for each batch
            4. Construct batch matrix (torch.sparse.FloatTensor)
            5. Save them

        """

        # Preparation
        if self.adj is None:
            self._prepare_adj()

        # Offline
        self.adj_batch_path = os.path.join(self.prefix, self.dataset, self.suffix, self.fname, 'adj_batch')
        if not os.path.exists(self.adj_batch_path):
            os.mkdir(self.adj_batch_path)

        for batch_idx in range(int(self.n_batches*self.train_ratio)):
            users, items = self.adj_node_idx[batch_idx*self.batch_size:(batch_idx+1)*self.batch_size].T

            self._save_hist_seq(users, batch_idx, 'u')
            self._save_hist_seq(items, batch_idx, 'i', transpose=True)

        # Online
        for batch_idx in range(int(self.n_batches*self.train_ratio), self.n_batches):
            # Add new interactions
            self.adj = self.adj.tolil()
            for user, item in self.adj_node_idx[batch_idx*self.batch_size:(batch_idx+1)*self.batch_size]:
                self.adj[user, item] += 1
            self.adj = self.adj.tocsr()

            users, items = self.adj_node_idx[batch_idx*self.batch_size:(batch_idx+1)*self.batch_size].T

            self._save_hist_seq(users, batch_idx, 'u')
            self._save_hist_seq(items, batch_idx, 'i', transpose=True)

    def _save_hist_seq(self, batch, batch_idx, msg, transpose=False):
        indices = []
        data = []
        adj = self.adj.T if transpose else self.adj
        for i in batch:
            indices.append(torch.from_numpy(adj.getrow(i).indices))
            data.append(torch.from_numpy(adj.getrow(i).data))

        indices = pad_sequence(indices, batch_first=True, padding_value=0)
        data = pad_sequence(data, batch_first=True, padding_value=0)

        torch.save(indices, os.path.join(self.adj_batch_path, msg+'_idx'+str(batch_idx)+'.pt'))
        torch.save(data, os.path.join(self.adj_batch_path, msg+'_val'+str(batch_idx)+'.pt'))

    def update_adj(self, batch_idx):
        """Add current batch to adjacency matrix for online update"""
        if self.cur_idx == batch_idx:
            logging.info('Adjacency matrix not updated')
            return

        new_indices = torch.cat((self.adj._indices(), self.adj_node_idx[batch_idx:batch_idx+batch_size].T), axis=1)
        new_vals = torch.cat((self.adj._values(), self.adj_edge_val[batch_idx:batch_idx+batch_size]))
        self.adj = torch.sparse_coo_tensor(new_indices, new_vals)
        self.coalesce()

    def init_adj(self):
        self.cur_idx = int(self.n_batches*self.train_ratio)*self.batch_size # number of offline items
        self.adj = scipy.sparse.csr_matrix((self.adj_edge_val[:self.cur_idx],
                                            self.adj_node_idx[:self.cur_idx].T),
                                            shape=(self.n_users, self.n_items))
        logging.info('Adjacency matrix created with type: %s and shape: %s' %(str(type(self.adj)), str(self.adj.shape)))

    def _prepare_adj(self):
        logging.info('Preparing adjacency matrix')
        #self.adj_node_idx = torch.from_numpy(self.data_df.loc[:, ['user_id', 'item_id']].values).to(dtype=torch.int32) # (number of items, 2)
        #self.adj_edge_val = torch.ones(len(self.adj_node_idx), dtype=torch.int16) # (number_of_items)
        self.adj_node_idx = self.data_df.loc[:, ['user_id', 'item_id']].values.astype(np.int64) # (number of items, 2)
        self.adj_edge_val = np.ones(len(self.adj_node_idx), dtype=np.int16) # (number_of_items)

        self.init_adj()

    # Deprecated
    def _coalesce(self):
        """Coalesce duplicate edges into single (one-valued) edge.
        Require adj. matrix (torch.sparse_coo_tensor)
        """
        self.adj = self.adj.coalesce()
        assert self.adj.is_coalesced()

        if self.duplicate == 0: # No duplicate edge: edge value is always one
            self.adj = torch.sparse_coo_tensor(self.adj._indices(), self.adj._values().fill_(1)).coalesce()

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser = MetaReader.parse_data_args(parser)
    args, extras = parser.parse_known_args()

    args.path = '../../data/'
    corpus = MetaReader(args)

    corpus_path = os.path.join(args.path, args.dataset, 'mesia', 'Corpus.pkl')
    logging.info('Save corpus to {}'.format(corpus_path))
    pickle.dump(corpus, open(corpus_path, 'wb'))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed May 15 11:09:46 2019

@author: couto
"""

import random
import math

def leer_numero(ini, fin, mensaje):

    introducido = int(input("{}".format(mensaje)))

    while introducido < ini:
        introducido = int(input("{}".format(mensaje)))
    else:
        return introducido

def generador():
    numero = leer_numero(1, 20, "¿Cuantos números quieres generar? [1-20]:")
    modo = leer_numero(1, 3, "¿Cómo quieres redondear los números? [1]Al alza [2]A la baja [3]Normal: ")

    randomNumbers = []
    for i in range(numero):
        floatNumber = random.uniform(0,100)

        if modo == 1:
            finalNumber = math.ceil(floatNumber)
        elif modo == 2:
            finalNumber = math.floor(floatNumber)
        else:
            finalNumber = round(floatNumber)

        print("Original number: {}, Rounded number {}".format(floatNumber, finalNumber))
        randomNumbers.append(finalNumber)

    return randomNumbers

print(generador())


import sys

def f_gold ( low , high ) :
    """
 Return the number of gold numbers in a given range.

 Parameters
 -----
 low : int
 low of the range
 high : int
 high of the range

 Returns
 -----
 f_gold : int
 number of gold numbers in the range
 """
    fact , x = 1 , 1
    while fact < low :
        fact , x = fact * x , x + 1
    res = 0
    while fact <= high :
        res += 1
        fact , x = fact * x , x + 1
    return res

if __name__ == '__main__':
    param = [
    (57,79,),
    (57,21,),
    (31,37,),
    (62,87,),
    (49,98,),
    (82,76,),
    (31,45,),
    (5,52,),
    (76,43,),
    (55,6,)
        ]
    n_success = 0
    print("class,mutant,test_index,result")
    for i, parameters_set in enumerate(param):
        try:
            print("COUNT_FACTORIAL_NUMBERS_IN_A_GIVEN_RANGE," + sys.argv[1] + "," + str(i) + "," + str(f_gold(*parameters_set)))
        except:
            print("COUNT_FACTORIAL_NUMBERS_IN_A_GIVEN_RANGE," + sys.argv[1] + "," + str(i) + ",EXCEPTION")


#!/usr/bin/env python3

"""
Script de prueba para generar la distrubución de alfa-Rossi a partir de la
simulación con PTRAC en modo lista sin tener en cuenta las coincidencias
accidentales.
"""

import numpy as np
import sys

sys.path.append('/home/pablo/CinePy/')
from modules.alfa_rossi_procesamiento import alfa_rossi_procesamiento
from modules.io_modules import read_PTRAC_CAP_bin
from modules.alfa_rossi_escritura import escribe_datos_promedio
from modules.simulacion_modules import lee_nps_entrada, \
                                       separa_en_historias_sin_accidentales

if __name__ == '__main__':

    archivo_entrada = 'in_sdef'
    archivo_n = 'in_sdef_n.p'

    # -------------------------------------------------------------------------
    # Pre-procesamiento
    # -------------------------------------------------------------------------
    # Se lee la cantidad de eventos de fuente del archivo de entrada de MCNP
    nps = lee_nps_entrada(archivo_entrada)
    # Se leen los datos en binario
    print('Leyendo arhvo de neutrones....')
    datos_n, header_n = read_PTRAC_CAP_bin(archivo_n)
    # Se separan por historias
    print('Separando datos por hisotrias....')
    times_out = separa_en_historias_sin_accidentales(datos_n)

    # -------------------------------------------------------------------------
    # Procesamiento
    # -------------------------------------------------------------------------
    # Duración de cada bin (discretización del tau) [s]
    dt_s = 2e-4
    # Máximo bin analizado (mayor tau) [s]
    dtmax_s = 2e-2
    # Nombre de la carpeta donde se guardarán los resultados
    nombre_carpeta = 'resultados_arossi'
    tb = 1
    # Procesamiento de los datos
    dat_pro = alfa_rossi_procesamiento(times_out, dt_s, dtmax_s, tb, 'all')

    # En este caso la cantidad de historias lo determina la simulación
    Nhist = np.shape(dat_pro[0])[0]
    # Se escribe sólo el promedio
    # Si se usan más detectores, se debe crear un nombre para cada detector
    archivo_out = [archivo_n]
    escribe_datos_promedio(dat_pro, archivo_out, Nhist, dt_s, dtmax_s, tb,
                           nombre_carpeta)


#!/usr/bin/env python
"""
This program takes a file containing fragments and another containing MSRE sites and
writes to a third file a line for each MSRE site found within each fragment. It is assumed
that both input files are sorted and that there is no possible overlap of cut sites with
recognition sequences.

The output file looks like::

    proximal_NspI,distal_NspI,MSRE_site
"""

import os
import sys
import csv

sys.path.append("%s/lib/python" % os.environ['LAB_HOME'])
import util.cl

def int_file_generator(f):
    for line in f:
        yield int(line.strip())

def find_cuts_in_fragments(frag_file, msre_file, output_file):
    with open(frag_file) as frags, open(msre_file) as msre, open(output_file, 'w') as out:
        frag_reader = csv.reader(frags)
        msre_reader = int_file_generator(msre)

        cur_site = None
        for frag in frag_reader:
            start, end = map(int, frag)

            while cur_site < start:
                try:
                    cur_site = msre_reader.next()
                except:
                    return

            if start < cur_site < end:
                out.write("%i,%i,%i\n" % (start,end,cur_site))

if __name__ == '__main__':
    def add_args(parser):
        parser.add_argument("frag_file", metavar="FILE", type="readable_file")
        parser.add_argument("msre_file", metavar="FILE", type="readable_file")
        parser.add_argument("output_file", metavar="FILE", type="writeable_file")

    ns = util.cl.parse(add_args)
    find_cuts_in_fragments(ns.frag_file, ns.msre_file, ns.output_file)