import threading
from scapy.all import *
import csv
import time
import sys
from os import system, name
from threading import Lock
import p4runtime_sh.shell as sh # open source code

ingress_port = 0  # length 2 bytes mask 0000 1111 1111 000
timestamp_sec = 6  # length 6 bytes
timestamp_ns = 12  # length 4 bytes
mirror_reason = 39  # ingress or egress mirror
psn_idx = 20  # 2 bytes
latency = 31  # one byte
lsb_ing_occup = 23  # three bytes
lsb_eg_occup = 27  # three bytes
eg_port = 18
graph_interval = 10 # seconds
stop_threads = False

IS_USER = 5
USER_TOS = 104

#################### Globals ##################

list_lock = Lock()
known_lock = Lock()
user_lock = Lock()
congestion_lock = Lock()
info_lock = Lock()
shell_lock = Lock()
packets = []  # Global list containing all of the packets
known_tuples = []
full_info = []
still_congested = [[False, None], True]
user_found = [False]
standard_frequency = 10
connection_set = [False]

# ======================================#
'''
This is the P4Runtime shell-related setup. We are using p4runtime-shell
open-souce code from the git found at https://github.com/p4lang/p4runtime-shell
and installed on the lab's Collector.
the setup & teardown should only be done once!
'''
if not connection_set[0]:
    shell_lock.acquire()
    try:
        sh.setup(
            device_id=0,
            grpc_addr='132.68.36.62:50051',
            election_id=(0, 1),
            config=sh.FwdPipeConfig('/tmp/OurMirror.p4info', '/tmp/OurMirror.bin')
        )
        te = sh.TableEntry('control_out_port.ipv4_check_checksum')(action='DoMirror')
        connection_set[0] = True
        print("connection acomplished")
    finally:
        shell_lock.release()

# ======================================#

def clear():
    # for windows
    if name == 'nt':
        _ = system('cls')

        # for mac and linux(here, os.name is 'posix')
    else:
        _ = system('clear')

# ======================================#
class ErspanPacket:
    def __init__(self, reason, psn, flow_five, time_sec, time_nanosec, ingress_port, ingress_occupancy, egress_port,
                 egress_occupancy, latency):
        self.reason = reason
        self.psn = psn
        self.flow_five = flow_five
        self.time_sec = time_sec
        self.time_nanosec = time_nanosec
        self.ingress_port = ingress_port
        self.ingress_occupancy = ingress_occupancy
        self.egress_port = egress_port
        self.egress_occupancy = egress_occupancy
        self.latency = latency

    def is_egress(packet):
        if packet.reason == "EG":
            return True
        else:
            return False

    def is_ingress(packet):
        if packet.reason == "IN":
            return True
        else:
            return False

# ======================================#
def should_stop_threads(_ignore):
    if stop_threads is True:
        return True
    else:
        return False

def sniff_packets():
    sniff(iface="eth0", prn=packet_handler, stop_filter=should_stop_threads)

# ======================================#

def get_five_tuple(receivedPacket):
    user_traffic = False
    # receivedPacket=rawSocket.recv(2048)
    # print("****************************")
    # IP_header
    inner_pkt_IP = receivedPacket[14:34]
    src_0 = str(int(hex(inner_pkt_IP[12]), 16))
    src_1 = str(int(hex(inner_pkt_IP[13]), 16))
    src_2 = str(int(hex(inner_pkt_IP[14]), 16))
    src_3 = str(int(hex(inner_pkt_IP[15]), 16))
    inner_IP_source = src_0 + '.' + src_1 + '.' + src_2 + '.' + src_3
    # print(inner_IP_source)
    dst0 = str(int(hex(inner_pkt_IP[16]), 16))
    dst1 = str(int(hex(inner_pkt_IP[17]), 16))
    dst2 = str(int(hex(inner_pkt_IP[18]), 16))
    dst3 = str(int(hex(inner_pkt_IP[19]), 16))
    inner_IP_dest = dst0 + '.' + dst1 + '.' + dst2 + '.' + dst3
    # print(inner_IP_dest)
    protocol = str(int(hex(inner_pkt_IP[9]), 16))
    tos = int(hex(inner_pkt_IP[1]), 16)
    if tos == 104:
        # print("Got ya! TOS is 104, which means DSCP is 26")
        user_traffic = True
    # print(protocol)

    # TCP Header...
    pkt_gre = receivedPacket[34:54]
    src_0 = (int(hex(pkt_gre[0]), 16))
    src_1 = (int(hex(pkt_gre[1]), 16))
    port_source = str(int(hex((src_0 << 8) | src_1), 16))
    dest_0 = (int(hex(pkt_gre[0]), 16))
    dest_1 = (int(hex(pkt_gre[1]), 16))
    port_dest = str(int(hex((dest_0 << 8) | dest_1), 16))
    # print("Source Port: " + port_source)
    # print("Destination Port: " + port_dest)
    # print("****************************")
    return (inner_IP_source, inner_IP_dest, protocol, port_source, port_dest, user_traffic, tos)

# ======================================#

# TODO - check what is protocol 47, what should ours be
def packet_handler(pkt):
    if IP not in pkt:
        return
    if pkt[IP].proto != 47:
        return

    pkt_gre = list(pkt.load)

    if pkt_gre[mirror_reason] == 1:
        pkt_mirror = "IN"
    elif pkt_gre[mirror_reason] == 2:
        pkt_mirror = "EG"
    else:
        return

    flow_five_tuple = get_five_tuple(pkt_gre[50:])
    # Calculate the PSN
    psn_msb = int(hex(pkt_gre[psn_idx]), 16)
    psn_lsb = int(hex(pkt_gre[psn_idx + 1]), 16)
    psn = int(hex((psn_msb << 8) | psn_lsb), 16)
    # print("psn : " , psn)

    # Calculate the ingress_port
    ing_port_msb = int(hex(pkt_gre[ingress_port]), 16)
    ing_port_lsb = int(hex(pkt_gre[ingress_port + 1]), 16)
    ing_port = int(hex((((ing_port_msb << 8) | ing_port_lsb) << 4) >> 8), 16)
    # print("ingress port : " , ing_port)

    # Calculate the egress_port
    egress_port_msb = int(hex(pkt_gre[eg_port]), 16)
    egress_port_lsb = int(hex(pkt_gre[eg_port + 1]), 16)
    egress_port = int(hex((((egress_port_msb << 8) | egress_port_lsb) << 4) >> 8), 16)
    # print("egress port: ",egress_port)

    # Calculate ingress buff occupancy
    buff_1 = int(hex(pkt_gre[lsb_ing_occup]), 16)
    buff_2 = int(hex(pkt_gre[lsb_ing_occup + 1]), 16)
    buff_3 = int(hex(pkt_gre[lsb_ing_occup + 2]), 16)
    buff_ing_occupancy = int(hex((((buff_1 << 8) | buff_2) << 8) | buff_3), 16)
    # print('ingress_buff_occupancy :',buff_ing_occupancy)

    # Calculate egress buffer_occupancy
    buff_1 = int(hex(pkt_gre[lsb_eg_occup]), 16)
    buff_2 = int(hex(pkt_gre[lsb_eg_occup + 1]), 16)
    buff_3 = int(hex(pkt_gre[lsb_eg_occup + 2]), 16)
    buff_eg_occupancy = int(hex((((buff_1 << 8) | buff_2) << 8) | buff_3), 16)
    # print('egress_buff_occupancy :',buff_eg_occupancy)

    # Calculate latency
    buff_1 = int(hex(pkt_gre[latency]), 16)
    buff_2 = int(hex(pkt_gre[latency + 1]), 16)
    buff_3 = int(hex(pkt_gre[latency + 2]), 16)
    latency_ns = int(hex((((buff_1 << 8) | buff_2) << 8) | buff_3), 16)
    # print("latency = " , latency_ns)

    # TODO - check the latency process, what type of latency is it

    # Calculate timestamp_sec
    ts_sec_msb = int(hex(pkt_gre[timestamp_sec]), 16)
    for i in range(1, 6):
        ts_sec_lsb = int(hex(pkt_gre[i + timestamp_sec]), 16)
        res_sec = int(hex((ts_sec_msb << 8) | ts_sec_lsb), 16)
        ts_sec_msb = res_sec
    # Calculate timestamp_ns
    ts_msb = int(hex(pkt_gre[timestamp_ns]), 16)
    for i in range(1, 4):
        ts_lsb = int(hex(pkt_gre[i + timestamp_ns]), 16)
        res_ns = int(hex((ts_msb << 8) | ts_lsb), 16)
        ts_msb = res_ns
    # print('timestamp (sec) = ',res_ns)

    if flow_five_tuple[2] == 17:
        protocol = "TCP"
    else:
        protocol = "UDP"
    # (inner_IP_source, inner_IP_dest, protocol, port_source, port_dest, user_traffic)
    if flow_five_tuple[IS_USER] and flow_five_tuple not in known_tuples:
        info_lock.acquire()
        try:
            full_info.append([flow_five_tuple, ing_port, egress_port, protocol])
        finally:
            info_lock.release()
        known_lock.acquire()
        try:
            known_tuples.append(flow_five_tuple)
        finally:
            known_lock.release()
        if not user_found[0]:
            user_lock.acquire()
            try:
                user_found[0] = True
            finally:
                user_lock.release()
    elif flow_five_tuple not in known_tuples:
        info_lock.acquire()
        try:
            full_info.append([flow_five_tuple, ing_port, egress_port, protocol])
        finally:
            info_lock.release()
        known_lock.acquire()
        try:
            known_tuples.append(flow_five_tuple)
        finally:
            known_lock.release()

    # Add packet to list
    list_lock.acquire()
    try:
        # print("APPENDING")
        packets.append(
            ErspanPacket(pkt_mirror, psn, flow_five_tuple, res_sec, res_ns, ing_port, buff_ing_occupancy, egress_port,
                         buff_eg_occupancy, latency_ns))
    finally:
        list_lock.release()

    if buff_eg_occupancy > 3000 and not still_congested[0][0]:
        congestion_lock.acquire()
        try:
            #print("\n\tcongestions detected on port ", egress_port)
            still_congested[0] = [True, egress_port]
            if still_congested[1]:
                print("\n\tcongestions detected on port ", egress_port)
                print("\tcongestion diagnosed due to the following information:")
                print("\t - buffer occupancy on port is: ", buff_eg_occupancy)
                print("\t - latency (ns): ", latency_ns)
                print("\twhich exceeded normal behavior.")
                orig_std = sys.stdout
                logs_file = open("logs.txt","a")
                sys.stdout = logs_file
                logs_file.write("Congestion detected at time: ")
                logs_file.write(str(time.asctime( time.localtime(time.time()) )) + "\n\n")
                logs_file.write("congestions detected on port " + str(egress_port))
                logs_file.write("\ncongestion diagnosed due to the following information:")
                logs_file.write("\n\t - buffer occupancy on port is: " + str(buff_eg_occupancy))
                logs_file.write("\n\t - latency (ns): " + str(latency_ns))
                logs_file.write("\nwhich exceeded normal behavior.\n\n")
                still_congested[1] = False
                sys.stdout = orig_std
        finally:
            congestion_lock.release()
    elif still_congested[0][0] and buff_eg_occupancy < 3000:
        congestion_lock.acquire()
        try:
            still_congested[0] = [False, None]
        finally:
            congestion_lock.release()

# ======================================#

def extractFlowInformation():
    info_lock.acquire()
    i = 0
    clear()
    for tup in full_info:
        five_tup = tup[0]
        print("Flow informtion number: ", i)
        print("\t Ingress port: ", tup[1])
        print("\t Egress port: ", tup[2])
        print("\t Source IP: ", five_tup[0])
        print("\t Dest IP: ", five_tup[1])
        print("\t L4 protocol: ", tup[3])
        print("\t Source L4 Port: ", five_tup[3])
        print("\t Dest L4 Port: ", five_tup[4])
        print("\t TOS value: ", five_tup[6])
        print()
        i+=1
    info_lock.release()

def showUser():
    user_lock.acquire()
    if user_found[0]:
        info_lock.acquire()
        clear()
        for tup in full_info:
            five_tup = tup[0]
            if five_tup[6] == USER_TOS:
                print("***************************************")
                print("User's flow informtion:")
                print("\t Ingress port: ", tup[1])
                print("\t Egress port: ", tup[2])
                print("\t Source IP: ", five_tup[0])
                print("\t Dest IP: ", five_tup[1])
                print("\t L4 protocol: ", tup[3])
                print("\t Source L4 Port: ", five_tup[3])
                print("\t Dest L4 Port: ", five_tup[4])
                print("\t TOS value: ", five_tup[6])
                print("***************************************")
        info_lock.release()
    else:
        print("User's flow was not found")
    user_lock.release()

def stopShell():
    shell_lock.acquire()
    try:
        te = sh.TableEntry('control_out_port.ipv4_check_checksum')(action='DoMirror')
        sh.TableEntry('control_out_port.ipv4_check_checksum').read(lambda t: t.delete())
        sh.teardown()
        print("disconnected")
    finally:
        shell_lock.release()

def analyzeCongestion():
    congestion_lock.acquire()
    if still_congested[0][0]:
        print("Congestion found on port", still_congested[0][1])
        print("Setting up P4Runtime entry to match on port")
        shell_lock.acquire()
        try:
            te = sh.TableEntry('control_out_port.ipv4_check_checksum')(action='DoMirror')
            te.match['standard_metadata.egress_port'] = hex(still_congested[0][1])
            te.match['headers.ip.ipv4.hdr_checksum'] = '0x7f00&&&0x7f00'
            te.action['analyzer_port'] = '0x04'
            te.priority = 1
            te.insert()
        finally:
            shell_lock.release()
    else:
        print("No Congestion found")
    congestion_lock.release()

def maintain_list(clean_freq, clean_interval):
    while stop_threads == False:
        time.sleep(clean_freq)
        if len(packets) > 0:
            newest_ts = list(packets)[-1].time_sec
            list_lock.acquire()
            try:
                for pkt in packets:
                    if newest_ts - pkt.time_sec > clean_interval:
                        packets.remove(pkt)
            finally:
                list_lock.release()


letterPoints = {'a':1, 'b':3, 'c':3, 'd':2, 'e':1, 'f':4, 'g':2, 'h':4, 'i':1,'j':8, 'k':5, 'l':1, 'm':3, 'n':1, 'o':1, 'p':3, 'q':10, 'r':1,'s':1, 't':1, 'u':1, 'v':4, 'w':4, 'x':8, 'y':4, 'z':10}

def createWordList(filename):
    file = open(filename, 'r')
    s = file.read()
    file.close()
    L = s.split('\n')
    return L[:-1]

def canWeMakeIt(myWord, myLetters):
    listY = []
    x = list(myWord)
    y = list(myLetters)
    for block in range(len(myLetters)):
        if y[block] in x:
            x.remove(y[block])
    if len(x) == 0:
        return True
    else:
        return False

def getWordPoints(myWord, letterPoints):
    accumulator = 0
    for ch in myWord:
        accumulator += letterPoints[ch]
    return accumulator

def scrabbleWords(myLetters):
    wordList = createWordList('wordlist.txt')
    myWords = []
    for i in range(len(wordList)):
        if canWeMakeIt(wordList[i], myLetters) == True:
            myWords.append(wordList[i])
    pointWordList = []
    for q in myWords:
        pointValue = getWordPoints(q, letterPoints)
        pointWordList.append((pointValue, q))
    sortWordList = sorted(pointWordList, reverse=True)

    fmt = '{:' + str(len(myLetters)+4) + '} {} \n'
    s = ''
    for x,y in sortWordList:
        s += fmt.format(y,x)

    openfile = open('%s.txt'%myLetters, 'w')
    openfile2 = openfile.write(s)
    openfile.close()


from visualization_msgs.msg import MarkerArray
from visualization_msgs.msg import Marker
from geometry_msgs.msg import Point
from eufs_msgs.msg import ConeArray
import rospy

def callback(msg):
    frame = msg.header.frame_id
    stamp = msg.header.stamp
    markerArray = MarkerArray()
    i = 0
    for cone in msg.orange_cones:
        marker = Marker()
        marker.header.frame_id = frame
        marker.header.stamp = stamp
        marker.type = Marker.MESH_RESOURCE
        marker.ns = ''
        marker.id = i
        marker.action = 0
        cone.z = 0.0
        marker.pose.position = cone
        marker.pose.orientation.x = 0.0
        marker.pose.orientation.y = 0.0
        marker.pose.orientation.z = 0.0
        marker.pose.orientation.w = 1
        marker.scale.x = 1.5
        marker.scale.y = 1.5
        marker.scale.z = 1.5
        marker.color.r = 1.0
        marker.color.g = 0.270999997854
        marker.color.b = 0.0
        marker.color.a = 1.0
        marker.lifetime.secs = 0
        marker.lifetime.nsecs = 200000000
        marker.frame_locked = False
        marker.mesh_resource = "package://eufs_description/meshes/cone_big.dae"
        marker.mesh_use_embedded_materials = False
        print(marker)
        markerArray.markers.append(marker)
        i = i + 1
    pub.publish(markerArray)

rospy.init_node('cone_visual')
pub = rospy.Publisher("/visualCone", MarkerArray, queue_size=10)
subLS = rospy.Subscriber('/conoPercepcion/local', ConeArray, callback)
rospy.spin()


#!/usr/bin/python3
import socket
import threading

def handle_clients(client):
    client.send("Confirmation Message...".encode("utf-8"))

def recieve_connections():
    while True:
        try:
            client,addr = server.accept()
            print(f"[+] {addr} is connected with server")
            thread = threading.Thread(target=handle_clients,args=(client,))
            thread.start()

        except Exception as e:
            print("Error:- ",e)

host = "127.0.0.1"
port = 9090
# host = str(input("IP address: "))
# port = int(input("Port: "))
server = socket.socket(socket.AF_INET,socket.SOCK_STREAM)

server.bind((host,port))
print(f"[+] Server is running on {host}:{port}")
server.listen()
recieve_connections()


import logging
import os
from django.utils.log import DEFAULT_LOGGING

from randevu.conf.boilerplate import BASE_DIR

# log_path = os.path.join(BASE_DIR, 'logs')
# isExist = os.path.exists(log_path)
# if not isExist:
#     os.makedirs(log_path)

logger = logging.getLogger(__name__)

LOG_LEVEL = "INFO"

logging.config.dictConfig({
    "version": 1,
    "disable_existing_loggers": False,
    "formatters": {
        "console": {
            "format": "%(asctime)s %(name)-12s %(levelname)-8s %(message)s"
        },
        # "file": {
        #     "format": "%(asctime)s %(name)-12s %(levelname)-8s %(message)s"
        # },
        "django.server": DEFAULT_LOGGING["formatters"]["django.server"]
    },
    "handlers": {
        "console": {
            "class": "logging.StreamHandler",
            "formatter": "console"
        },
        # "file": {
        #     "level": "INFO",
        #     "class": "logging.handlers.TimedRotatingFileHandler",
        #     "formatter": "file",
        #     "filename": os.path.join(BASE_DIR, "logs/randevu-admin.log"),
        #     'when': 'midnight',
        #     'backupCount': 10,
        # },
        "django.server": DEFAULT_LOGGING["handlers"]["django.server"]
    },
    "loggers": {
        "": {
            "level": "INFO",
            "handlers": [
                "console",
                # "file"
            ],
            "propagate": False
        },
        "apps": {
            "level": "INFO",
            "handlers": [
                "console"
            ],
            "propagate": False
        },
        "django.server": DEFAULT_LOGGING["loggers"]["django.server"]
    }
})


import logging
import numpy as np
from pathlib import Path

from batch import BatchInput
from environment import GoState, GoEnv, resign_action, pass_action
import feature
from game_record import GoGameRecord
from model_config import ModelConfig
from model_controller import KerasModelController
from player import Player

class PolicyPlayer(Player):
    def __init__(self, config: ModelConfig, player, record_path=''):
        super().__init__(config=config, player=player, record_path=record_path)
        assert config.weight_root, f'please supply config.weight_root'
        config.allow_weight_init = True
        config.learner_log_dir = ''
        config.batch_size_inference = 1
        config.model_name = 'policy_gradient'
        self.model_controller = KerasModelController.controller(config)
        self.boards, self.actions, self.estimated_value = [], [], []
        self.model_controller.__enter__()

    def __del__(self):
        if hasattr(self, 'model_controller') and self.model_controller is not None:
            self.model_controller.__exit__(None, None, None)

    def reset(self, board):
        self.boards, self.actions, self.estimated_value = [], [], []

    def end_game(self, reward):
        if self.record_path and len(self.boards):
            assert len(self.boards) == len(self.actions)
            assert len(self.estimated_value) == len(self.actions)
            record = GoGameRecord(config=self.config)
            record.player = self.player
            record.reward = reward
            record.moves = self.actions
            record.boards = self.boards
            record.values = self.estimated_value
            record.write_to_path(self.record_path)

    def next_action(self, state: GoState, prev_state: GoState, prev_action):
        board_size = self.config.board_size
        ply_index = len(self.boards)
        encoded = state.board.encode()
        board = GoGameRecord.encoded_board_to_array(self.config, encoded=encoded)
        self.boards.append(board)
        feature_vector = feature.array_to_feature(self.config,
                                                  boards=self.boards,
                                                  player=state.color,
                                                  ply_index=ply_index)
        batch_input = BatchInput()
        batch_input.batch_xs = feature_vector[np.newaxis, ...]
        batch_output = self.model_controller.infer(batch_input)
        # first batch of the first result
        probabilities = batch_output.result[0][0]
        # restrict these two actions
        probabilities = np.power(probabilities, 0.5)
        if len(self.boards) < self.config.pg_noise_threshold and self.config.pg_noise_alpha > 0.0:
            # add noise for early moves
            probabilities = 0.75 * probabilities + \
                            0.25 * np.random.dirichlet([self.config.pg_noise_alpha] * len(probabilities))
        probabilities[resign_action(board_size)] = 0.0
        probabilities[pass_action(board_size)] = 0.0
        probabilities /= np.sum(probabilities)
        if self.config.print_board > 1:
            print(batch_output.result[0][0])
            print(probabilities)
        action = GoEnv.sample_action(self.config, state, probabilities)
        self.actions.append(action)
        self.estimated_value.append(batch_output.result[1][0])
        if self.config.print_board > 1:
            GoEnv.render(state)
            print(f'{probabilities}, action={action}')
        return action


import pandas as pd
import numpy as np

gdp = pd.DataFrame.from_csv('course1_downloads/world_bank.csv')
gdp

# In[126]:

gdp  = gdp[4:] #cut header
gdp

# In[127]:

gdp.reset_index(inplace = True) #make country names editable by taking them out of index
gdp

# In[128]:

#"Korea, Rep.": "South Korea",
#"Iran, Islamic Rep.": "Iran",
#"Hong Kong SAR, China": "Hong Kong"
gdp.replace(to_replace = ['Korea, Rep.', 'Iran, Islamic Rep.','Hong Kong SAR, China'],
               value = ['South Korea','Iran','Hong Kong'], inplace = True)
gdp

# In[129]:

gdp.set_index(keys = 'Data Source', inplace = True)
print(gdp)

# In[130]:

GDP = gdp


from builtins import tuple

import requests
import pprint
import json
import time
from flask import Flask, render_template, request
import sqlite3

app = Flask(__name__)

def get_vac(search='ml', per_page=20):
    url = 'https://api.hh.ru/vacancies'
    # search = 'ml'
    # per_page = 20
    lst_vac_for_html = []
    lst = []
    con = sqlite3.connect("hh.sqlite")
    cursor = con.cursor()
    params = {'page': 1,
              'per_page': per_page,
              'text': search,
              'area': 113,
              'only_with_salary': True}
    response = requests.get(url, params=params)
    print(f'Статус ({search}): {response.status_code}')
    result = response.json()
    list_vac = result['items']
    # pprint.pprint(response.json())

    # print(result['items'][0]['url'])
    skills = {'Python': 0, 'Spark': 0, 'Hadoop': 0, 'Docker': 0, 'Linux': 0, 'Git': 0, 'SQL': 0, 'PyTorch': 0,
              'Keras': 0,
              'Sklearn': 0, 'Pandas': 0, 'Scipy': 0, 'Numpy': 0, 'XGBoost': 0, 'CatBoost': 0, 'LightGBM': 0,
              'Matplotlib': 0, 'REST': 0, 'FastApi': 0, 'JSON': 0, 'Django': 0, 'Flask': 0, 'Redis': 0, 'Cassandra': 0,
              'ClickHouse': 0}
    key_skills = []
    salary = 0
    for i in range(len(list_vac)):
        cursor.execute("insert or ignore into city (name) VALUES (?)", (list_vac[i]['area']['name'],))
        cursor.execute("select * from city where name=?", (list_vac[i]['area']['name'],))
        city_values = cursor.fetchone()
        # print(city_values)
        cursor.execute("insert or ignore into currency (name) VALUES (?)", (list_vac[i]['salary']['currency'],))
        cursor.execute("select * from currency where name=?", (list_vac[i]['salary']['currency'],))
        currency_values = cursor.fetchone()
        # print(currency_values)
        cursor.execute("insert into vacancy (name, city_id, salary_from, salary_to, currency_id) VALUES (?,?,?,?,?)",
                       (list_vac[i]['name'], city_values[0], list_vac[i]['salary']['from'],
                        list_vac[i]['salary']['to'], currency_values[0]))
        dct = {'name': list_vac[i]['name'], 'city': list_vac[i]['area']['name'], 'salary': list_vac[i]['salary']}
        lst_vac_for_html.append(dct)
        # print(list_vac[i]['url'])
        # vacancy_url = list_vac[i]['url']
        # if list_vac[i]['salary']['from'] is None:
        #     sal = list_vac[i]['salary']['to']
        # else:
        #     sal = list_vac[i]['salary']['from']
        # if list_vac[i]['salary']['currency'] == 'USD':
        #     salary += sal * 80
        # elif list_vac[i]['salary']['currency'] == 'EUR':
        #     salary += sal * 90
        # else:
        #     salary += sal
        # time.sleep(1)
        # response = requests.get(vacancy_url)
        # print(f'Статус {i + 1}: {response.status_code}')
        # result = response.json()
        # # print(result)
        # desc = result['description']
        # key_sc = result['key_skills']
        # for key in skills.keys():
        #     if key.lower() in desc.lower():
        #         skills[key] += 1
        # if key_sc is not None:
        #     for key in key_sc:
        #         if key['name'] not in key_skills:
        #             key_skills.append(key['name'])

    # pprint.pprint(skills)
    # lst_2 = []
    # for key, value in skills.items():
    #     lst_2.append({'name': key, 'count': value, 'persent': round(value / len(list_vac) * 100, 1)})
    # lst.append({'keywords': search, 'salary_mean': round(salary / len(list_vac)), 'count': len(list_vac),
    #             'requirements': lst_2})
    #
    # pprint.pprint(lst)
    # pprint.pprint(key_skills)
    # with open(f"{search}.json", "w") as f:
    #     json.dump(lst, f)
    cursor.execute("select * from vacancy")
    print(cursor.fetchall())
    con.commit()
    return lst_vac_for_html

@app.route('/')
def index():
    return render_template('index.html', name='ПАРСЕР HH')

@app.route('/services/', methods=['POST', 'GET'])
def services():
    if request.method == 'POST':
        lst = get_vac(request.form['search'], int(request.form['per_page']))
        return render_template('services.html', lst=lst, methods='POST')
    else:
        return render_template('services.html', methods='GET')

@app.route('/contact/')
def contact():
    return render_template('contact.html')

if __name__ == "__main__":
    app.run(debug=True)


from utility.Logger import Logger
from ..template import Scripts_default
from utility.DriverSetup import *
from time import sleep

class template_Inline(Scripts_default):

    def start(self):
        # -----------------------------------------------------------------------------#
        # sessions
        # -----------------------------------------------------------------------------#
        def get_webpage():
            self.log.session_start('get webpage', 1)
            driver.get(self.URL)
            driver.maximize_window()
            self.log.session_end()

        def fill_people():
            self.log.session_start('select people', 2)
            element = driver.wait_find_element(By.ID, "adult-picker")
            Select(element).select_by_index(int(INFO['PEOPLE']))
            self.log.session_end()

        def click_calender():
            self.log.session_start('click calender', 2)
            driver.find_element_then_click(By.ID, "date-picker")
            self.log.session_end()
            driver.send_ScrollDown()

        def click_date():
            self.log.session_start('click date', 2)
            response = driver.find_element_then_click(By.XPATH, "//div[@data-date='"+INFO["DATE"]+"']//span[1]")
            if not response:
                self.log.session_start('info DATE is not available', 1, '\n')
                return False
            else:
                self.log.session_end()
                return True

        def click_time():
            self.log.session_start('click time', 2)
            response = driver.find_element_then_click(By.XPATH, "//button[@data-cy='book-now-time-slot-box-"+INFO["TIME"]+"']")
            if not response:
                self.log.session_start('info TIME is not available', 1, '\n')
                return False
            else:
                self.log.session_end()
                return True

        def click_next_step():
            # fill-in user info
            driver.find_element_then_click(By.XPATH, "//span[text()='下一步，填寫聯絡資訊']")
            sleep(0.5)
            self.log.session_start('click dining purpose', 2)
            driver.find_element_then_click(By.XPATH, "//div[@value='朋友聚餐']")
            self.log.session_start('fill name', 2)
            driver.find_element(By.ID, "name").send_keys(INFO["NAME"])
            self.log.session_start('fill phone number', 2)
            driver.find_element(By.ID, "phone").send_keys(INFO["PHONE"])
            self.log.session_end()

        def final_check_box_and_send():
            # final
            self.log.title('final step',1)
            self.log.session_start('check box', 2)
            driver.find_element_then_click(By.XPATH, "(//label[@class='sc-bqiRlB fCuXci'])[2]")
            self.log.session_end()
            self.log.session_start('send reservation request', 2)
            driver.find_element_then_click(By.XPATH, "//button[@class='sc-ieecCq eZhyRr']")
            self.log.session_end()

        # -----------------------------------------------------------------------------#
        # execute sessions
        # -----------------------------------------------------------------------------#
        INFO = self.INFO
        driver = DriverSetup()

        get_webpage()

        fill_people()

        click_calender()

        try:
            click_date()
        except:
            driver.send_ScrollDown()
            driver.save_screenshot(self.__class__.__name__+self.now()+'click_date_error.png')
            self.log.title('screen shot saved.')
            self.not_available(driver)
            return False

        try:
            click_time()
        except:
            driver.send_ScrollDown()
            driver.save_screenshot(self.__class__.__name__+self.now()+'click_time_error.png')
            self.log.title('screen shot saved.')
            self.not_available(driver)
            return False

        try:
            click_next_step()
        except:
            driver.send_ScrollDown()
            driver.save_screenshot(self.__class__.__name__+self.now()+'click_nextstep_error.png')
            self.log.title('screen shot saved.')

        final_check_box_and_send()

        sleep(2)
        driver.save_screenshot(self.__class__.__name__+self.now()+'_final.png')
        driver.endDriver()

        return True

    # -----------------------------------------------------------------------------#
    # error response
    # -----------------------------------------------------------------------------#
    def not_available(self, driver:DriverSetup):
        print("current date ("+ self.INFO['DATE'] +"), time (" + self.INFO['TIME'] + ") or branch-store is not available.")
        print("session end.")
        driver.endDriver(wait=1)


import torch
from .extension import _assert_has_ops

Tensor = torch.Tensor

def lsq(x: Tensor, scale: Tensor, shift: Tensor,
        quant_min: int = 0,
        quant_max: int = 255,
        type_min: int = None,
        type_max: int = None,
        axis: int = 1,
        use_grad_scaling: bool = True,
        grad_scaler: float = 1.,
        is_affine: bool = True,
        is_perchannel: bool = False,
        eval_mode: bool = False,
        init_mode: bool = False) -> Tensor:
    '''
        Pytorch FakeQuantizer Observer with Learned Step Size Quantization(LSQ+) (https://arxiv.org/abs/2004.09576)

        Theory:
            The Pytorch quantization scheme is x_q = round(clamp(x/scale + zero_point,quant_min,quant_max);
                where zero_point is integer and have same type as x_q.

            In the article, however, used another approach: x_q = round(clamp((x - shift)/scale,quant_min,quant_max);
                shift - have float type here. We adapt such scheme via implicit conversion of shift to zero_point

            Following article, LSQ+ emulate the quantization and dequantization of input treating `scale` and `shift` as learnable parameters.

            Forward pass(quantize->dequantize):
               1) Emualate zero_point behaviour zero_point = clamp(-shift/scale, type_min, type_max); where type_min(type_max) is min(max) of quantized type (i.e int8/uint8)
               2) Quantized: x_q = round(clamp(x/scale + zero_point, quant_min, quant_max)
               3) Dequantized: x_r = x_q * scale + shift

            Backward pass:
                Consider, 1) zero_point = clamp(-shift/scale, type_min, type_max);
                          2) x_q = round(clamp(x/scale + zero_point, quant_min, quant_max));

                w.r.t input: 1,  if ((quant_min < x_q) && (x_q < quant_max))
                             0,  otherwise

                w.r.t scale: (x_r - x)/scale, if ((quant_min < x_q) && (x_q < quant_max))
                             quant_min - zero_point,    if x_q <= quant_min
                             quant_max - zero_point,    if x_q >= quant_max

                w.r.t shfit:  0,  if ((quant_min < x_q) && x_q < quant_max))
                              1,  otherwise

        Note! By default in 8-bit assymetric quantization will used: `quant_min` = 0, `quant_max` = 255

        By default the affine quantization is used (defined by `is_affine` flag)
        In case of symmetric(`is_affine`=False), the shift forcible treat as 0 both during backward pass

        Implementation natively support per_tensor(default) and per_channel schemes via `is_perchannel` flag.

        We support parameters initialization mode via `init_mode` flag.
            In original article the authors propose to init `scale` and `shift` for affine case by minimizing: ||x_r-x||F^2
            Instead of using it as additional loss, we incorporate this natively inside code,
            due to this function supposed to be work inside Pytorch Observer class.
        When `init_mode`=True only 'scale' and 'shift' will be updated treating ||x_r-x||F^2 as main loss function.

        Based on article we support scaling of derivatives on `scale` and `shift` on 1/sqrt(n_elements(x)*quant_max)
        (See https://arxiv.org/abs/1902.08153 for details)

        Note! type_min/type_max is stands for numerical limits of used quantized type.
              For Pytorch e.g. quint8 and qint8. They may corresponds to quant_min/quant_max,
              but is not always true, for example for unsigned 7-bit the quant_min == type_min, but quant_min < type_max.

        Arguments:
            x(torch.tensor) - input tensor
            scale(torch.tensor) - what the name said, 1D tensor(dim==1), even in per_tensor case.
            shift(torch.tensor) - what the name said, 1D tensor(dim==1), even in per_tensor case.
            quant_min(int) - lower bound of quantized range. Default: 0 (for assymetric quantization)
            quant_max(int) - upper bound of quantized range. Default: 255.
            type_min(int) - lower bound of quantized type. Default: None (= quant_min)
            type_max(int) - upper bound of quantized type. Default: None (= quant_max)
            axis(int) - dimension for per channel quantization scheme. Default: 1.
            use_grad_scaling(bool) - Apply scale to scale and zero_point gradient. Default: True.
            grad_scaler(float) - additional grad scaler, helps to compensate the learning rate. Default: 1.
            is_affine(bool) - Control assymmetric(true) or symmetric(false) quantization type. Default: true(assymmetric).
            is_perchannel(bool) - Control quantization scheme: per_channel(true) and per_tensor(false). Default: false(per_tensor)
            eval_mode(bool) - Force LSQ work like PyTorch FakeQuantizer(only fake quant) if enabled. Default: false.
            init_mode(bool) - Control parameters initialization(see description above). Default: false(not used)
    '''
    _assert_has_ops()
    if not is_affine:
        assert quant_min <= 0 <= quant_max, 'quantization range must be covered 0 in symmetric quantization'
    type_min = quant_min if type_min is None else type_min
    type_max = quant_max if type_max is None else type_max

    return torch.ops.torchlsq.lsq(x, scale, shift, quant_min, quant_max, type_min, type_max,
                                     axis, use_grad_scaling, grad_scaler, is_affine, is_perchannel,
                                     eval_mode, init_mode)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

##################################################################
#Polytech Sorbonne - année 2018/2019
#Réalisé par : Fatine Bentires Alj et Alexia Zounias-Sirabella
#Cours d'apprentissage statistique de Patrick Gallinari
##################################################################

from .Annexe import *
from .Lecture import *
import pandas
import numpy as np
import matplotlib.pyplot as plt

############################## Etape 1 : manipulation des données ##################################

class Manipulation_donnees:
	#calcule les valeurs moyennes pour les données quantitatives uniquement
	def moyenne(data):
		nombre_lignes=Annexe.nombre_de_lignes(data)
		somme = 0
		moy = [0,0,0,0,0,0,0,0,0,0]

		for lignes in data.values:
			for i in range(0,10): #pour les variables quantitatives
				moy[i]+=int(lignes[i])

		for i in range(0,10):
			moy[i]/=nombre_lignes #pour diviser par le nombre de lignes

		print("La moyenne des éléments quantitatifs obtenue à la main est : ", moy)

	#donne la probabilité d'appartenance à chaque classe de forêt
	def type_foret(data):
		classe = [0,0,0,0,0,0,0,0]
		nombre_lignes = Annexe.nombre_de_lignes(data)
		somme_proba = 0

		for arbre in data.values:
			if(int(arbre[54])==1): #si l'élément vaut 1 alors on le met dans classe 1
				classe[1]+=1
			elif(int(arbre[54])==2): #si l'élément vaut 2 alors on le met dans classe 2
				classe[2]+=1
			elif(int(arbre[54])==3): #si l'élément vaut 3 alors on le met dans classe 3
				classe[3]+=1
			elif(int(arbre[54])==4): #si l'élément vaut 4 alors on le met dans classe 4
				classe[4]+=1
			elif(int(arbre[54])==5): #si l'élément vaut 5 alors on le met dans classe 5
				classe[5]+=1
			elif(int(arbre[54])==6): #si l'élément vaut 6 alors on le met dans classe 6
				classe[6]+=1
			elif(int(arbre[54])==7): #si l'élément vaut 7 alors on le met dans classe 7
				classe[7]+=1
			else:
				print("Pas de classe. Problème.")

		proba = np.zeros(7)
		proba[0] = classe[1]/nombre_lignes
		proba[1] = classe[2]/nombre_lignes
		proba[2] = classe[3]/nombre_lignes
		proba[3] = classe[4]/nombre_lignes
		proba[4] = classe[5]/nombre_lignes
		proba[5] = classe[6]/nombre_lignes
		proba[6] = classe[7]/nombre_lignes
		somme_proba = np.sum(proba)

		plt.bar(range(7),proba)
		plt.xlabel('Type de couverture')
		plt.ylabel('Probabilité')
		plt.show()

		print("Nombre de lignes : ",nombre_lignes, "\nExample : nombre d'arbres appartenant à la classe 1 : ",classe[1],"\nExample : probabilité d'appartenir à la classe 1 : ",proba[0],"\nSomme des probas = 1 ? : ",somme_proba)


# -*- coding: utf-8 -*-

from openerp import models, fields

class account_invoice(models.Model):
    _name = "account.invoice"
    _inherit = 'account.invoice'
    _order = 'date_invoice desc'

    remote_id = fields.Integer(
        'Remote ID',
        copy=False,
    )
    departure = fields.Char(
        'Departure',
        size=50
    )
    aduana = fields.Char(
        'Aduana',
        size=50
    )
    bultos = fields.Integer(
        'Número de bultos'
    )
    net_weight = fields.Float(
        'Net Weight'
    )
    gross_weight = fields.Float(
        'Gross Weight'
    )
    origin = fields.Char(
        'Origin',
        size=50
    )
    samples = fields.Boolean(
        'Samples'
    )
    invoice_number = fields.Char(
        'Invoice Number',
        size=50
    )
    packing_id = fields.Many2one(
        'expresso.packing',
        'Packing'
    )

class account_invoice_line(models.Model):
    _name = "account.invoice.line"
    _inherit = 'account.invoice.line'

    remote_id = fields.Integer(
        'Remote ID',
        copy=False,
    )


import re
flat_pattern = r'([A-Za-z0-9\!\@\#\$\?]+)=([0-9]+)<<(.+)'

while True:
    inp = input()
    if inp == 'Last note':
        exit()
    flat_reg = re.match(flat_pattern, inp)
    if flat_reg:
        geohashcode_length = int(flat_reg.group(2))
        geohash_code = len(flat_reg.group(3))
        if geohash_code == geohashcode_length:
            name_mountain = flat_reg.group(1)
            appender = ''
            for i in name_mountain:
                if i.isalnum():
                    appender += i
            print(f'Coordinates found! {appender} -> {flat_reg.group(3)}')
        else:
            print('Nothing found!')
    else:
        print('Nothing found!')

Python Fundamentals - January 2020/Exams/Technology Final Exam - 14 April 2019 Group 1/01-Arriving-in-Kathmandu.py


#!/usr/bin/python

#coding: utf-8

#Usando docstrings

from decimal import Decimal

class Relationship:
    """Class que representa um relacionamento entre  DataTables

        Essa class tem todas as informações que identificam um relacionamento
        entre tableas. Em qual coluna ele existe, de onde vem e pra onde vai.
    """
    def __init__(self, name, _from, to, on):

        """Construtor

            Args:
                name: Nome
                from: Tabela de onde sai
                to: Tabela pra onde vai
                on: instancia de coliuna onde existe
        """
        self._name = name
        self._from = _from
        self._to = to
        self._on = on

class DataTable:
    """ Representa uma Tabela de dados.

    Essa class representa uma tabela de dados do portal
    de transparência. Deve ser capaz d validar linhas
    inseridas de acordo com as colunas que possui.
    As linhas ficam registradas dentro dela.

    Attrbutes:
       name: Nome da Tabela
       columns: [Lista de colunas]
       data: [Lista de dados]

       """
    def __init__(self, name):
        """Constructor
             Args:
                 name: Nome da Tabela"""
        self._name = name
        self._columns = []
        self._references = []
        self._referenced = []
        self.data = []

    def _get_name(self):
        print('Getter executado!')
        return self._name

    def _set_name(self, _name):
        print('Setter executado!')
        self._name = _name

    def _del_name(self):
        print('Delleter executado!')
        raise AttributeError("Não pode deletar esse atributo")

    name = property(_get_name, _set_name, _del_name)

    def add_column(self, name, kind, description=""):
        column = Column(name, kind, description=description)
        self._columns.append(column)
        return column

    def add_references(self, name, to ,on):
        """Cria uma referencia dessa tabela para uma outra tabela

            Args:
                name: nome da relação
                to: instancia da tabela apontada
                on: instancia coluna em que existe a relação

        """
        relationship = Relationship(name, self, to, on)
        self._references.append(relationship)

    def add_referenced(self, name, by, on):
        """Cria uma referencia para outra tabela que aponta para essa.

            Args:
                name: nome da relação
                by: instancia da tabela que aponta para essa
                on: instancia coluna em que existe a relacao
        """
        relationship = Relationship(name, by, self, on)
        self._referenced.append(relationship)

class Column:
    """Representa uma coluna em um DataTable

        Essa classe contém as informações de uma coluna
        e deve validar um dado de acordo com o tipo de
        dado configurado no construtor.

        Attributes:
            name = Nome da coluna
            kind = Tipo de dado(varchar, bigint, numeric)
            description = Descrição da coluna.
            """
    def __init__(self, name, kind, description=""):
        """Constructor
            Args:
                name = Nome da coluna
                kind = Tipo de dado(varchar, bigint, numeric)
                description = Descrição da coluna.
            """

        self._name = name
        self._kind = kind
        self._description = description
        self._is_pk = False

    def __str__(self):
        _str = "Col: {} : {} {}".format(self._name,
                                        self._kind,
                                        self._description)

    def _validate(cls, kind, data):
        if kind == 'bigint':
            if isinstance(data, int):
                return True
            return False
        elif kind == 'varchar':
            if isinstance(data, str):
                return True
            return False
        elif kind == 'numeric':
            try:
                val = Decimal(data)
            except:
                return False
            return True

    #validate = staticmethod(_validate)
    validate = classmethod(_validate)

class PrimaryKey(Column):
    def __init__(self, table, name, kind, description=""):
        super().__init__(name, kind, description=description)
        self._is_pk = True

    def __str__(self):
        _str = "Col: {} : {} {}".format(self._name,
                                        self._kind,
                                        self._description)

        return "{}  - {} ".format('PK', _str)


#coding=utf-8
from math import log
import operator
#计算香浓熵
def calcShannonEnt(dataSet):
    numEntries = len(dataSet) #求数据集的长度
    labelCounts = {}
    for featVec in dataSet:
        currentLabel = featVec[-1]#提取标签
        if currentLabel not in labelCounts.keys():labelCounts[currentLabel] = 0
        labelCounts[currentLabel] += 1
    shannonEnt = 0.0
    for key in labelCounts:
        prob = float(labelCounts[key]) / numEntries #计算每种结果出现的可能性
        shannonEnt -= prob * log(prob,2) #计算出的熵的结果，再进行累加
    return shannonEnt

#按照给定特征划分数据集
def splitDataSet(dataSet,axis,value):
    retDataSet = []
    for featVec in dataSet:
        if featVec[axis] == value:
            reduceFeatVec = featVec[ : axis]
            reduceFeatVec.extend(featVec[axis+1 : ]) #函数用于在列表末尾一次性追加另一个序列中的多个值（用新列表扩展原来的列表）
            #上面两步实际上是选择了这个属性的某个值筛选出来然后去除了axis这一列
            retDataSet.append(reduceFeatVec)
    return retDataSet

#选择最好的数据集划分方式
'''
数据要求
（1）数据必须是一种有列表元素组成的列表，而且所有的列表元素都要具有相同的数据长度
（2）数据的最后一列或者每个实例的最后一个元素是当前实例的类别标签
'''
def chooseBestFeatureToSplit(dataSet):
    numFeatures = len(dataSet[0]) - 1 #判断数据集有多少特征属性
    baseEntropy = calcShannonEnt(dataSet) #计算原始的香浓熵 用于划分数据集后比较
    bestInfoGain = 0.0
    bestFeature = -1
    for i in range(numFeatures):#用于遍历数据集中所有的属性
        #下面使用列表推导式创建新的列表
        featList = [example[i] for example in dataSet]#将每一个属性的值都提取出来 第i个特征所有可能的值
        uniqueList = set(featList)#去重 搞清楚属性的值
        newEntropy = 0.0
        for value in uniqueList:#使用该属性划分数据集 最后一个属性就是标签 就是原始香浓熵 相减为零
            subDataSet = splitDataSet(dataSet,i,value)#用该属性的一个值划分数据集
            prob = len(subDataSet) / float(len(dataSet))#发生的概率
            newEntropy += prob * calcShannonEnt(subDataSet)#发生的概率乘以数据集的香浓熵 全概率公式
        infoGain = baseEntropy - newEntropy#计算信息增益 该信息增益就是去掉的属性的香浓熵
        if (infoGain > bestInfoGain):#香浓熵的值越大说明包含的信息越多 应该选择其为最好的特征
            bestInfoGain =infoGain
            bestFeature = i
    return bestFeature

#基本工作已经做好了 现在需要递归迭代构造decision-making tree
#选取最好的划分特征 划分数据集（不一定是两个，可能有很多个）再使用划分算法划分数据集
#递归结束的条件是：程序遍历完所有划分数据集的属性，或者每个分支下的所有实例都具有相同的分类
#到最后一个节点还存在多种分类的话使用多数表决方法决定到该节点的样例的类别
def majorityCnt(classlist):
    classCount = {}#设置一个字典 键为存在的类别 值为键出现的次数
    for vote in classlist:
        if (vote not in classCount.key):classCount[vote] = 0
        classCount[vote] += 1
    sortedClassCount = sorted(classCount.iteritems(),key=operator.itemgetter(1),reverse = True)#从大到小排序
    return sortedClassCount[0][0] #[[key,value]...]返回的是其键的值 就是出现次数最多的类别

#创建树的代码
def creatTree(dataSet,labels):
    classList = [example[-1] for example in dataSet] #取出标签
    if (classList.count(classList[0]) == len(classList)):return classList[0] #类别完全相同就不用再划分了 返回所属的种类
    if (len(dataSet[0]) == 1):return majorityCnt(classList) #只剩分类，当使用完所有的属性还不能得到完全的结论时使用多数表决的算法
    bestFeat = chooseBestFeatureToSplit(dataSet) #如果既没有出现相同的划分 又没有出现所有属性用完的情况 说明递归还未结束 第一步找出最好的划分的依据
    bestFeatLabel = labels[bestFeat]
    mytree = {bestFeatLabel:{}}
    del(labels[bestFeat])#del删除的是变量而不是数据对象 删除列表中的那个元素
    featValues = [example[bestFeat] for example in dataSet]
    uniqueVals = set(featValues)
    for value in uniqueVals:
        subLabels = labels[:]
        mytree[bestFeatLabel][value] = creatTree(splitDataSet(dataSet,bestFeat,value),subLabels)
    return mytree

def creatDataSet():
    dataset = [
        [1, 1, 'yes'],
        [1, 1, 'yes'],
        [1, 0, 'no'],
        [0, 1, 'no'],
        [0, 1, 'no']
    ]
    labels = ['no surfacing','flippers']
    return dataset,labels


import os
import numpy as np
import pandas as pd
import warnings
import logs
import clf_model
import annotate
#from splitLabelledData import splitDataSet
# lables are converted as
# {0: 'incorporates', 1: 'independent', 2: 'relates to'}
path = "../ExtractedData/"
fields = ['summary1','summary2','dependency','id1', 'id2','label']
#fullFile = path+"Solr_Processed_SplitAndTaggedData.csv"
fullFile = path+"Solr_Processed_SplitAndTaggedData_posNeg_5types.csv"
depType =  {'incorporates':0, 'independent':1, 'relates to':2}
#print(depType['independent'])
#input("hit enter")
label = 'label'
req1 = 'summary1'
req2 = 'summary2'
req1Id = 'id1'
req2Id = 'id2'
annStatus = 'AnnotationStatus'

projectName = "Solr"

def learnTargetLabel(args):
    '''
    Active Learning iterative process
    1. Prepare Data
    2. Create Classifier
    3. Evaluate Classifier
    4. Select Uncertain Samples and get them annotated by Oracle
    5. Update Data Set (Merge newly annotated samples to original dataset)
    6. Repeat steps 1-5 until stopping condition is reached.

    Parameters :
    args (dataframe) : Run-time arguments in a dataframe.

    Returns :
    df_rqmts (dataframe) : Updated / Final requirements dataset, included the prediction values at the last iteration of Active learning process.
    df_resultTracker (dataframe) : Results for tracking purpose

    '''
    #Read run time arguments
    idir = os.getcwd()+args.loc[0,'input']
    splitratio = float(args.loc[0,'testsize'])
    maxIterations = int(args.loc[0,'maxIterations'])
    resamplingTechnique = args.loc[0,'resampling']
    print(idir,splitratio,maxIterations,resamplingTechnique)
    #input("hit enter")

    logs.writeLog("Fetching data from the input directory.")
    #Read To be Annotated, Training, Test and Validation Sets generated after executing splitData.py
    try:
        df_rqmts = pd.read_csv(idir+fullFile) #this has training data with Annotated = 'M' and rest with Nothing ''
        print(df_rqmts[annStatus].value_counts())

    except FileNotFoundError as err:
        logs.writeLog ("File Not Found! Please provide correct path of the directory containing Training, Test, Validation, ToBeAnnotated and Manually Annotated DataSet.")
        print (err)
        exit()

    #Create a dataframe to track the results
    df_resultTracker = pd.DataFrame()
    iteration = 0
    df_rqmts=df_rqmts.sample(frac=1) #shuffulles
    df_rqmts[label] = df_rqmts[label].astype('int')

    df_training = df_rqmts[df_rqmts[annStatus]=='M']
    df_testing = df_rqmts[df_rqmts[annStatus]!='M']

    #these are two df's for local model(LM) training for first iteration
    df_LM_training = df_training
    df_LM_testing = df_testing

    while True:
        iteration+=1
        logs.writeLog("\n"+100*"-")
        logs.writeLog("\n\nIteration : "+str(iteration)+"\n")
        #####run it multiple times say 10 and accumulate average results
        V_f1=[]
        V_prec = []
        V_rcl=[]
        V_indPrec = []
        V_indRcl=[]
        V_indF1=[]
        V_ReqPre=[]
        V_ReqRcl=[]
        V_ReqF1=[]
        V_SimPrec=[]
        V_SimRcl=[]
        V_SimF1=[]
        V_confusioMatrix=[]

        LM_f1=[]
        LM_prec=[]
        LM_rcl=[]
        LM_indPrec=[]
        LM_indRcl=[]
        LM_indF1=[]
        LM_ReqPre=[]
        LM_ReqRcl=[]
        LM_ReqF1=[]
        LM_SimPrec=[]
        LM_SimRcl=[]
        LM_SimF1=[]
        LM_confusionMatrix = []
        for i in range(9):
            #-----------------------------------------AL model -------------------------------------#
            logs.writeLog("\nCreating Classifier...")
            #just pass the ones with AnnotationStatus = 'M'
            countVectorizer, tfidfTransformer, classifier, classifierTestScore = clf_model.createClassifier(args.loc[0,'classifier'],df_training,resamplingTechnique)
            logs.writeLog("\n\n5 fold Cross Validation Score : "+str(classifierTestScore))
            logs.writeLog ("\n\nValidating Classifier...")

            #pass the rest as testing data annStatus]!='M'
            classifierValidationScore,v_f1Score,v_precisionScore,v_recallScore,v_precisionArr,v_recallArr,v_fscoreArr,v_supportArr,v_confusionMatrix = clf_model.validateClassifier(countVectorizer,tfidfTransformer,classifier,df_testing)
            logs.writeLog("\n\nClassifier Validation Set Score : "+str(classifierValidationScore))

            #Update Analysis DataFrame (For tracking purpose)
            df_training[label] = df_training[label].astype('int')
            independentCount = len(df_training[df_training[label]==depType['independent']])
            requiresCount = len(df_training[df_training[label]==depType['relates to']])
            similarCount = len(df_training[df_training[label]==depType['incorporates']])
            #----------------------------------------------AL ends-----------------------------------------

            #-----------------------------------------LM starts-------------------------------------------
            LM_countVectorizer, LM_tfidfTransformer, LM_classifier, LM_classifierTestScore= clf_model.createClassifier(args.loc[0,'classifier'],df_LM_training,resamplingTechnique)

            #pass the rest as testing data annStatus]!='M'
            LM_classifierValidationScore,LM_f1Score,LM_precisionScore,LM_recallScore,LM_precisionArr,LM_recallArr,LM_fscoreArr,LM_supportArr,lm_confusionMatrix = clf_model.validateClassifier(LM_countVectorizer,LM_tfidfTransformer,LM_classifier,df_LM_testing)
            logs.writeLog("\n\nClassifier Validation Set Score for LM: "+str(LM_classifierValidationScore))

            #Update Analysis DataFrame (For tracking purpose)
            df_LM_training[label] = df_LM_training[label].astype('int')
            LM_independentCount = len(df_LM_training[df_LM_training[label]==depType['independent']])
            LM_requiresCount = len(df_LM_training[df_LM_training[label]==depType['relates to']])
            LM_similarCount = len(df_LM_training[df_LM_training[label]==depType['incorporates']])

            #-----------------------------------------LM Ends--------------------------------------------
            #store results to average later
            V_f1.append(v_f1Score)
            V_prec.append(v_precisionScore)
            V_rcl.append(v_recallScore)
            V_indPrec.append(v_precisionArr[0])
            V_indRcl.append(v_recallArr[0])
            V_indF1.append(v_fscoreArr[0])
            V_ReqPre.append(v_precisionArr[1])
            V_ReqRcl.append(v_recallArr[1])
            V_ReqF1.append(v_fscoreArr[1])
            V_SimPrec.append(v_precisionArr[2])
            V_SimRcl.append(v_recallArr[2])
            V_SimF1.append(v_fscoreArr[2])
            V_confusioMatrix.append(v_confusionMatrix)

            LM_f1.append(LM_f1Score)
            LM_prec.append(LM_precisionScore)
            LM_rcl.append(LM_recallScore)
            LM_indPrec.append(LM_precisionArr[0])
            LM_indRcl.append(LM_recallArr[0])
            LM_indF1.append(LM_fscoreArr[0])
            LM_ReqPre.append(LM_precisionArr[1])
            LM_ReqRcl.append(LM_recallArr[1])
            LM_ReqF1.append(LM_fscoreArr[1])
            LM_SimPrec.append(LM_precisionArr[2])
            LM_SimRcl.append(LM_recallArr[2])
            LM_SimF1.append(LM_fscoreArr[2])
            #

        tempList = (lm_confusionMatrix.tolist())
        LM_confusionMatrix.append(tempList)
        #print(tempList[-1],"\n", tempList)
        #input("hit enter")
        df_resultTracker = df_resultTracker.append({'Iteration':iteration,
                                                    'Total data':len(df_rqmts),
                                                    'TraiAKA_ManlyAntd':len(df_training),
                                                    'Testing':len(df_testing),
                                                    'CV':classifierTestScore,
                                                    '#Ind':independentCount,
                                                    '#Req':requiresCount,
                                                    '#Sim':similarCount,
                                                    'f1':"{:.2f}".format(np.average(V_f1)),
                                                    'prec':"{:.2f}".format(np.average(V_prec)),
                                                    'rcl':"{:.2f}".format(np.average(V_rcl)),
                                                    'indPrec': "{:.2f}".format(np.average(V_indPrec)),
                                                    'indRcl':"{:.2f}".format(np.average(V_indRcl)),
                                                    'indF1':"{:.2f}".format(np.average(V_indF1)),
                                                    'indSup':v_supportArr[0],
                                                    'ReqPre': "{:.2f}".format(np.average(V_ReqPre)),
                                                    'ReqRcl':"{:.2f}".format(np.average(V_ReqRcl)),
                                                    'ReqF1':"{:.2f}".format(np.average(V_ReqF1)),
                                                    'ReqSup':v_supportArr[1],
                                                    'SimPrec': "{:.2f}".format(np.average(V_SimPrec)),
                                                    'SimRcl':"{:.2f}".format(np.average(V_SimRcl)),
                                                    'SimF1':"{:.2f}".format(np.average(V_SimF1)),
                                                    'SimSup':v_supportArr[2],
                                                    'ConfusionM':v_confusionMatrix,

                                                    '#LM Training':len(df_LM_training),
                                                    '#LM Testing':len(df_LM_testing),
                                                    'LM CV':LM_classifierTestScore,
                                                    '#LM Ind':LM_independentCount,
                                                    '#LM Req':LM_requiresCount,
                                                    '#LM Sim':LM_similarCount,
                                                    'LM f1':"{:.2f}".format(np.average(LM_f1)),
                                                    'LM prec':"{:.2f}".format(np.average(LM_prec)),
                                                    'LM rcl':"{:.2f}".format(np.average(LM_rcl)),
                                                    'LM indPre': "{:.2f}".format(np.average(LM_indPrec)),
                                                    'LM indRcl':"{:.2f}".format(np.average(LM_indRcl)),
                                                    'LM indF1':"{:.2f}".format(np.average(LM_indF1)),
                                                    'LM indSup':LM_supportArr[0],
                                                    'LM ReqPre': "{:.2f}".format(np.average(LM_ReqPre)),
                                                    'LM ReqRcl':"{:.2f}".format(np.average(LM_ReqRcl)),
                                                    'LM ReqF1':"{:.2f}".format(np.average(LM_ReqF1)),
                                                    'LM ReqSup':LM_supportArr[1],
                                                    'LM SimPrec': "{:.2f}".format(np.average(LM_SimPrec)),
                                                    'LM SimRcl':"{:.2f}".format(np.average(LM_SimRcl)),
                                                    'LM SimF1':"{:.2f}".format(np.average(LM_SimF1)),
                                                    'LM SimSup':LM_supportArr[2],
                                                    'LM ConfusionM':LM_confusionMatrix[-1]

                                                    },ignore_index=True)

        logs.writeLog("\n\nAnalysis DataFrame : \n"+str(df_resultTracker))
        print("-----------Before-----------")
        print(len(df_rqmts),"=",len(df_training),"+", len(df_testing))
        logs.writeLog ("\n\nPredicting Labels....")
        df_predictionResults = clf_model.predictLabels(countVectorizer,tfidfTransformer,classifier,df_testing)   #operate on the df_rqmts only

        logs.writeLog("\n\nFinding Uncertain Samples and Annotating them.....")
        df_finalPredictions, df_remaining_Testing = annotate.analyzePredictions(args,df_predictionResults)
        logs.writeLog("\n\nMerging Newly Labelled Data Samples....")
        df_rqmts = pd.concat([df_training,df_finalPredictions],axis=0,ignore_index=True)
        df_rqmts = pd.concat([df_rqmts,df_remaining_Testing],axis=0,ignore_index=True)
        print("After")
        print(len(df_rqmts),"=", len(df_training),"+", len(df_remaining_Testing),"+", len(df_finalPredictions))
        print(df_rqmts[annStatus].value_counts())
        print("-"*100)
        #Remove unwanted columns
        df_rqmts = df_rqmts[[req1,req2,label,annStatus]]#df_rqmts[['req_2','req_1',label,annStatus]]#df_rqmts[['comboId','req1Id','req1','req_1','req2Id','req2','req_2',label,annStatus]]

        #input("hit enter to proceed")

        #if iteration >=maxIterations:
        #new stopping condition is if the validation set is more than equal to or less than 30% of training size
        #df_validation = df_rqmts[df_rqmts[annStatus]!='M']
        if int(len(df_testing)) <= int(0.3*(int(len(df_training)))) or iteration >=maxIterations:
            logs.writeLog("\n\nStopping Condition Reached... Exiting the program."+str(len(df_testing))+str(len(df_training)))
            break

        #----------------for next iteration-------------------------------------#
        df_rqmts = df_rqmts.sample(frac=1)
        df_training = df_rqmts[df_rqmts[annStatus]=='M']
        df_testing = df_rqmts[df_rqmts[annStatus]!='M']

        #add equal amount of randomly selected labels to LM models
        #for this extract the number from df_finalPredictions
        stats=df_finalPredictions[label].value_counts()

        print("-------------Before LM---------------")
        print(len(df_rqmts),len(df_LM_training), len(df_LM_testing))
        print(stats)
        for key in stats.keys():
            #fetch the sample of size values for each class randomly and add tp LM training set
            #and remove from testing set
            sampleCount = int(stats[key])
            print(type(sampleCount), sampleCount)
            #df_temp = df_LM_testing[df_LM_testing["Label"]==key].sample(sampleCount)
            df_temp= df_LM_testing[df_LM_testing[label]==key].sample(sampleCount)
            df_temp[annStatus] == 'M'
            df_LM_training = pd.concat([df_LM_training,df_temp],axis=0,ignore_index=True)
            #print(sampleCount, len(df_temp), len(df_LM_testing), len(df_LM_training))
            df_LM_testing = df_LM_testing[~df_LM_testing.isin(df_temp)].dropna(how='any', subset=[req1, req2]) #df_LM_testing[~df_LM_testing.isin(df_temp)].dropna(how='any', subset=['req1', 'req2'])
            df_LM_testing[annStatus] = ""
            #print(sampleCount, len(df_temp), len(df_LM_testing))

        print("-------------After LM---------------")
        print(len(df_rqmts),len(df_LM_training), len(df_LM_testing))
        #input ("Hit enter")
        #house keeping
        df_LM_training = df_LM_training.sample(frac=1)
        df_LM_training[label] = df_LM_training[label].astype('int')
        df_LM_testing[label] = df_LM_testing[label].astype('int')
        #---------------------------------------Next iteration ends ---------------- #

    return df_rqmts,df_resultTracker

def main():
    #Ignore Future warnings if any occur.
    warnings.simplefilter(action='ignore', category=FutureWarning)

    pd.set_option('display.max_columns', 500)   #To make sure all the columns are visible in the logs.
    pd.set_option('display.width', 1000)

    #initialize directory which contains all the data and which will contain logs and outputs
    currentFileDir = os.getcwd()

    #Reads run time arguments
    args = logs.getArguments(currentFileDir+"/ALParams.txt")
    comments = args.loc[0,'comments']
    #print(args)
    #code glueing to not disturb wht ikagarjot did
    #df_myargs = pd.DataFrame()
    #df_myargs['comments'] = args.loc[0,'comments']
    #df_myargs['input'] = args.loc[0,'input']
    #print(df_myargs)

    inputPath = str(args.loc[0,'input'])
    comment = str(args.loc[0,'comments'])

    args['input'] = inputPath #+ str(i) #"FirefoxBuildSystem"
    args['comments'] = projectName + comment
    #Creates Logs folder structure
    logFilePath,OFilePath = logs.createLogs(currentFileDir+"/Logs",args)

    df_rqmtComb,df_Analysis = learnTargetLabel(args)

    #Adds the Analysis DataFrame to Output File
    logs.addOutputToExcel(df_Analysis,"\nAnalysis of  Label Classification  : \n")

    #logs.updateResults(df_rqmtComb,args)   #Update Results in excel....

    logs.writeLog("\nOutput Analysis is available at : "+str(OFilePath))
    logs.writeLog("\nLogs are available at : "+str(logFilePath))

    print(args)
    #input("hit enter")

if __name__ == '__main__':
    main()


from typing import List

class Solution:
    def firstMissingPositive_1(self, nums: List[int]) -> int:
        # Good. Running time is n with only constant storage.
        # Rearrange the nums array. Swap the element that points positions
        # within the range of the array.
        for i in range(len(nums)):
            if 0 < nums[i] <= len(nums):
                # Important to have nums[i] != nums[nums[i] - 1] in while,
                # in case there will be infinite loop during swap.
                while 0 < nums[i] <= len(nums) and nums[i] != nums[nums[i] - 1]:
                    idx = nums[i] - 1# The index that nums[i] points to.
                    nums[i], nums[idx] = nums[idx], nums[i]
        print(nums)
        for i in range(1, len(nums) + 1):
            if i != nums[i-1]:
                return i
        return len(nums) + 1

    def firstMissingPositive_2(self, nums: List[int]) -> int:
        # We are looking range [1, n] inclusive from both ends
        # Exclude all numbers that are either non-positive, or larger than n
        n = len(nums)
        for i in range(n):
            if nums[i] <= 0 or nums[i] > n:
                nums[i] = n + 1
        for i in range(n):
            if abs(nums[i]) != n + 1:
                # Set the number that nums[i] (already in range [1, n]) points
                # to (abs(nums[i]) - 1) as negative.
                nums[abs(nums[i]) - 1] = -abs(nums[abs(nums[i]) - 1])
        for i in range(n):
            if nums[i] > 0:
                return i + 1
        return n + 1

    def firstMissingPositive_sort(nums: List[int]) -> int:#nlogn
        # Not qualified since running time is nlogn
        nums.sort()
        start = 1
        for n in nums:
            if start == n:
                start += 1
        return start

    def firstMissingPositive_set(nums: List[int]) -> int:#n + n storage
        # Not qualified since using n storage
        numSet = set(nums)
        start = 1
        while start in numSet:
            start += 1
        return start

if __name__ == '__main__':
    print(Solution.firstMissingPositive_set(nums=[1,2,0]))
    print(Solution.firstMissingPositive_set(nums=[3,4,-1,1]))
    print(Solution.firstMissingPositive_set(nums=[7,8,9,11,12]))


from flask import Flask, request,make_response
from flask_cors import CORS
import json
import traceback

app = Flask(__name__)
CORS(app)

@app.route("/", methods=['GET'])
def hello():
    return ({
        "service": "product"
    })

@app.route("/file", methods=['POST'])
def getFile():
    try:
        response = make_response(request.get_data())
        customHeader = {}
        for key, value in request.headers:
            customHeader[key] = request.headers.get(key)
        response.headers = customHeader
        response.headers['Access-Control-Allow-Origin'] = '*'
        return response
    except Exception as err:
        print(err)
        traceback.print_exc()
        return ({ 'err': str(err)})

@app.route("/lists", methods=['GET'])
def show_list():
    return ({
        "lists": ['Amazon', 'Flipkart']
    })

if __name__ == "__main__":
    app.run(debug=True, port=5000, host='0.0.0.0')


#!/usr/bin/env python3
"""
+================================+
| Script Name: CVE-List.py           |
| Author: Sandro Kocher          |
| Version: 1.0                   |
| Funktion: Compare Relases and CVE's                     |
| Bemerkung:                     |
+================================+
"""

import json
from natsort import natsorted

# realeases.txt mit "Read" Berechtigungen öffnen und in Variable speichern
releaseFile = open("./Data/releases.txt", "r")
"""
for line in releaseFile.readlines():
    print(line)
"""

#CVE File Laden und in Variable füllen
cvesFile = open("./Data/cves.json", "r")
cvesData = json.load(cvesFile)

#Dependencies File Laden und in Variable füllen
dependenciesFile = open("./Data/dependencies.json", "r")
dependenciesData = json.load(dependenciesFile)

# Vorbereiten Output file mit "Write" Berechtigungen
vulnerableFile = open("./Data/Output/vulnerable.txt", "w")

#test = {"key1": ["test1", "test2"], "Key2": "Blabla", "Key3": {"ukey1": ["utest1", "utest2"]}}

def main():
    """
{
  "ansible": [  ==> programm
    {
      "affects": [
        "1.17.2",
        "2.4.9",
        "2.0.7",
        "3.0.2",
        "1.5.7",
        "1.17.4",
        "1.20.5",
        "2.19.0",
        "1.5.4",
        "1.17.0",
        "2.2.7",
        "2.5.10",
        "1.4.3",
        "1.16.0"
      ],
      "cves": [
        "CVE-2014-4967"
      ],
      "advisory": "ansible 1.6.7 contains two security fixes:\r\n  * Strip lookup calls out of inventory variables and clean unsafe data\r\n    returned from lookup plugins (CVE-2014-4966)\r\n  * Make sure vars don't insert extra parameters into module args and prevent\r\n    duplicate params from superseding previous params (CVE-2014-4967)"
    },
    """
    cveList = {} #erstellt die Variabe cveList für die Speicherung der CVEs mit den entsprechenden Versionen {'Programm == Version': {'CVEs'}
    for programm in cvesData: #Nimmt den Key aus dem ersten Absatz
        #print(programm)
        for item in cvesData[programm]: #geht durch jedes Item (Key + Value) pro Key(von programm Oben)
            #print(item)
            for cves in item["cves"]: #geht durch alle Subkeys aus dem programm und sucht nach dem key "cves"
                #print(cves)
                for affectVersion in item["affects"]: #itreation durch alle "affects" Einträge pro Programm und listed diese unter den CVEs auf
                    #print(affectVersion)
                    affectString = '{}=={}'.format(programm, affectVersion) #Erstellt die Formatierung für
                    #print(affectString)
                    if (affectString not in cveList): #Wenn Programm==Version nicht in der Liste cveList sind
                        cveList[affectString] = set() #erstelle einen neuen Key mit Programm==Version
                    cveList[affectString].add(str(cves)) #Ergänze den String "cves" als Value unter dem Key Programm==Version
                    #print(cveList)

    for line in releaseFile.readlines(): #Liest jede Zeile aus dem Release File
        lineparts = line.split("\t") #Trennt die Strings beim Tabstop: ['Release 1.3.10', '["sqlalchemy==3.20.6", "flask==3.2.2", "urllib3==3.16.4", "certifi==3.18.10", "pyyaml==5.15.2", "pickle==4.17.19", "ansible==3.0.2"]']
        #print(lineparts)
        release = lineparts[0] #Setzt den Release aus dem ersten Teil
        swCsv = lineparts[1].replace('[', '').replace(']', '').replace(' ', '').replace('"', '').replace('\n', '') #Ersetzt die Zeichen '[',']',' ','"' und '\n' mit nichts (löscht diese). sqlalchemy==3.20.6,flask==3.2.2,urllib3==3.16.4,certifi==3.18.10,pyyaml==5.15.2,pickle==4.17.19,ansible==3.0.2
        #print(swCsv)
        swList = swCsv.split(',') #teilt den String in die einzelnen Versionen auf ['sqlalchemy==3.20.6', 'flask==3.2.2', 'urllib3==3.16.4', 'certifi==3.18.10', 'pyyaml==5.15.2', 'pickle==4.17.19', 'ansible==3.0.2']
        #print(swList)

        releasecve = set()
        for swItem in swList: #Für jeden Eintrag in der der Softwareliste ['sqlalchemy==3.20.6', 'flask==3.2.2', 'urllib3==3.16.4', 'certifi==3.18.10', 'pyyaml==5.15.2', 'pickle==4.17.19', 'ansible==3.0.2']
            if (swItem in cveList): #Wenn Eintrag in Softwareliste in der CVE-Liste auftaucht
                for Element in cveList[swItem]: #Dann für jedes Element in der CVE-Liste {'ansible==1.17.2': {'CVE-2014-4967', 'CVE-2016-3096'} mit dem Key aus der Softwareliste "ansible==1.17.2"
                    #print(Element)
                    releasecve.add(Element) #ergänze die Liste releasecve mit dem Element "CVE"

            for dependency in dependenciesData[swItem]: #Für jeden Software-Eintrag in der dependencies-Liste {'sqlalchemy==1.3.11': ['pycapnp-async==0.5.2', 'drf-jwt==1.15.1', 'pymisp==2.4.56'], usw.}
                #print(swItem)
                #print(dependency) #ansible==3.0.2 (swItem), errbot==1.3.2(dependency), ansible==3.0.2(swItem) katka-core==0.11.2(dependency) ansible==3.0.2(swItem) smeagol==0.1.1(dependency)
                if(dependency in cveList): #Wenn Eintrag in Dependency-Liste in der CVE-Liste auftaucht
                    for Eintrag in cveList[dependency]: #Dann für jedes Element in der CVE-Liste {'ansible==1.17.2': {'CVE-2014-4967', 'CVE-2016-3096'} mit dem Key aus der Softwareliste "ansible==1.17.2"
                        releasecve.add(Eintrag) #ergänze die Liste releasecve mit dem Eeintrag "CVE"

        # Immer noch in der Iteration der Releases Liste
        """
        vulnerableFile.write(release + "\t" + ', '.join((releasecve)) + "\n") #schreibe in das File "vulnerableFile" Release + Auflistung der CVEs
        print(release + "\t" + ', '.join((releasecve)))
        """

        vulnerableFile.write(release + "\t" + ', '.join(natsorted(releasecve)) + "\n")
        print(release + "\t" + ', '.join(natsorted(releasecve)))

if __name__ == '__main__':
    main()


import os
import sys
import pytest

from context import differ

class TestDiffer():
  base = "/tmp/differ/"

  @pytest.fixture(scope='function', autouse=True)
  def setup(self):
    """This function will be run before every test function in this class"""
    print("Running setup function")
    if os.path.exists(self.base):
      os.system("rm -rf {}".format(self.base))

  def test_utils_differ(self):
    """Test the basic functionality of differ.utils.Differ"""
    differ.utils.Differ(None, None)
    assert not differ.utils.Differ("FAIL", "FAIL")._valid
    obj = differ.utils.Differ(".", "FAIL")
    assert not obj._valid
    assert "NOT VALID" in repr(obj)
    assert not obj.extract_path(None, None)
    assert not obj.extract_path("Something", None)

    # Test with the same tar files
    obj = differ.utils.Differ(
      "tests/files/before.tgz",
      "tests/files/before.tgz",
      base=self.base)
    assert obj._valid
    assert "NOT VALID" not in repr(obj)
    obj.start()
    print(obj.path1_obj)
    assert not obj.changes.get_added()
    assert not obj.changes.get_removed()
    assert not obj.changes.get_removed()

    # Test with the different files
    before_items = [
      # TAR
      'before.tar',
      # TAR GZIP
      'before.tar.gz',
      'before.tgz',
      # TAR BZIP2
      'before.tar.bz2',
      # TAR LZMA
      'before.tar.xz',
      # ZIP
      'before.zip',
      # Directory
      'before',
    ]
    for f_name in before_items:
      obj = differ.utils.Differ(
        "tests/files/{}".format(f_name),
        "tests/files/after.tgz",
        base=self.base)
      assert obj._valid
      obj.start()
      assert len(obj.changes.get_added()) == 1
      assert len(obj.changes.get_removed()) == 1
      assert len(obj.changes.get_changed()) == 2
      assert os.path.exists(obj.summary_path)
      with open(obj.summary_path) as fp:
        summary = fp.read()
        assert "added 1" in summary
        assert "removed 1" in summary
        assert "changed 2" in summary

  def test_differ_stat_changes(self):
    """Test differ where there are mode changes"""
    obj = differ.utils.Differ(
      "tests/files/stat_changes/before",
      "tests/files/stat_changes/after",
      base=self.base)
    assert obj._valid
    obj.start()
    assert len(obj.changes.get_added()) == 0
    assert len(obj.changes.get_removed()) == 0
    assert len(obj.changes.get_changed()) == 1
    assert len(obj.changes.get_changed_stat()) == 2
    assert os.path.exists(obj.summary_path)
    with open(obj.summary_path) as fp:
      summary = fp.read()
      assert "added 0" in summary
      assert "removed 0" in summary
      assert "changed 1" in summary
      assert "changed stat 2" in summary

  def test_differ_fifo(self):
    """Test differ where there are fifo files"""

    # Create 2 directories
    before = os.path.join(self.base, "before_tmp")
    after = os.path.join(self.base, "after_tmp")
    os.makedirs(before)
    os.makedirs(after)

    os.system("mkfifo {}/test_file".format(before))
    os.system("touch {}/test_file".format(after))

    obj = differ.utils.Differ(
      before,
      after,
      base=self.base)
    assert obj._valid
    obj.start()
    assert len(obj.changes.get_added()) == 0
    assert len(obj.changes.get_removed()) == 0
    assert len(obj.changes.get_changed()) == 0
    assert len(obj.changes.get_changed_stat()) == 1
    assert os.path.exists(obj.summary_path)
    with open(obj.summary_path) as fp:
      summary = fp.read()
      assert "added 0" in summary
      assert "removed 0" in summary
      assert "changed 0" in summary
      assert "changed stat 1" in summary

    changed_path = obj.changes.get_changed_stat()[0]
    change = obj.changes.changes.get(changed_path)
    assert change.related
    with open(change.related[0], 'r') as fp:
      mode = fp.read()
      assert 'fifo => regular' in mode

  def test_plugin_iptables_strip(self):
    """Test the plugin iptables stripper"""
    obj = differ.utils.Differ(
      "tests/files/plugins/iptables/before.tgz",
      "tests/files/plugins/iptables/after.tgz",
      base=self.base)
    assert obj._valid
    obj.start()
    assert len(obj.changes.get_added()) == 0
    assert len(obj.changes.get_removed()) == 0
    assert len(obj.changes.get_changed()) == 1
    assert os.path.exists(obj.diff_dir)

    changed_path = os.path.join(
      obj.changed_dir,
      "my_iptables_mangle.diff")
    with open(changed_path, "r") as fp:
      changed_content = fp.read()
    assert "-DIFFER-TEST-3" in changed_content
    assert "+DIFFER-TEST-4" in changed_content


from rip.schema.base_field import FieldTypes
from rip.schema.string_field import StringField
from rip import url_constructor

class ResourceUriField(StringField):
    """This field is to be used to add a resource
    with respect to parent resource.

    URI of the resource in context is appended to that of parent.
    """
    def __init__(self, of_type=None, entity_attribute='id',
                 field_type=FieldTypes.READONLY,
                 required=False,
                 nullable=False):
        super(ResourceUriField, self).__init__(
            max_length=1024,
            required=required,
            field_type=field_type,
            nullable=nullable,
            entity_attribute=entity_attribute)
        self.of_type = of_type

    def _get_url_parts(self, entity_name, value, parent_bread_crumbs):
        all_bread_crumbs = parent_bread_crumbs[:]
        all_bread_crumbs.append([entity_name, value])
        return reduce(lambda url_parts,
                             resource_tuple: url_parts + [unicode(val)
                                                          for val
                                                          in resource_tuple],
                      all_bread_crumbs, [])

    def serialize(self, request, value):
        schema = self.of_type or self.schema_cls
        schema_name = schema._meta.schema_name

        return url_constructor.construct_url(
            api_name=request.context_params['api_name'],
            api_version=request.context_params['api_version'],
            schema_name=schema_name,
            entity_id=value,
            parents=request.context_params.get('api_breadcrumbs', [])
        )

    def clean(self, request, value):
        if value is None:
            return value
        id_index = -2 if value.endswith('/') else -1
        return value.split('/')[id_index]


import h5py
import numpy as np
import pandas as pd
from scipy.sparse.linalg import svds
from scipy.sparse import coo_matrix
from sklearn import linear_model
import os
import gc
#import ggplot as gg

def als(rating_matrix, prepare_path, prepare_name, transpose_prepare_name, init_user_matrix=None, init_item_matrix=None,\
        factor_num=200, method='l2', lambda_user=0.001, lambda_item=0.001, \
        iteration_num=100, user_loop_num=500, item_loop_num=500, rmse_loop_num=3, latent_file_path = "",
        latentFilePlace = "C:\\Users\\22560\\PycharmProjects\\lastFM\\hetrec2011-lastfm-2k\\latentFactor.hdf5",
        validationFilePlace = "C:\\Users\\22560\\PycharmProjects\\lastFM\\hetrec2011-lastfm-2k\\trainAfterReg.csv"
        ):
    # 如果p或者q为空值的话，分解原始矩阵作为初始值
    if((init_user_matrix==None)|(init_item_matrix==None)):
        init_user_matrix, init_item_matrix = decomposed_rating_matrix(rating_matrix, factor_num)
    with h5py.File(latent_file_path+'latentFactor.hdf5', 'w') as latent_factor_file:
        latent_factor_file.create_dataset('userLatentFactor', data=init_user_matrix)
        latent_factor_file.create_dataset('itemLatentFactor', data=init_item_matrix)
    with h5py.File(prepare_path, 'r') as prepare_y, \
            h5py.File(latent_file_path+'latentFactor.hdf5', 'r+') as latent_factor_file:
        y = prepare_y[prepare_name]
        transpose_y = prepare_y[transpose_prepare_name]
        user_latent_factor_matrix = latent_factor_file['userLatentFactor']
        item_latent_factor_matrix = latent_factor_file['itemLatentFactor']
        user_num, item_num = y.shape
        user_loop_interval = np.linspace(0, user_num, user_loop_num+1, dtype=np.int)
        item_loop_interval = np.linspace(0, item_num, item_loop_num+1, dtype=np.int)
        est_rmse = [5]
        est_rmse_on_vali = [5]
        # if(method=='l2'):
        #     for iter in range(iteration_num):
        #         for i, j in enumerate(user_loop_interval[:-1]):
        #             next_node = user_loop_interval[i+1]
        #             user_latent_factor_matrix[j:next_node, ] = ridge_reg(item_latent_factor_matrix[:], y[j:next_node, ].transpose(), lambdas=lambda_user)
        #         for i, j in enumerate(item_loop_interval[:-1]):
        #             next_node = item_loop_interval[i+1]
        #             item_latent_factor_matrix[j:next_node, ] = ridge_reg(user_latent_factor_matrix[:], y[:, j:next_node], lambdas=lambda_item)
        #         new_rmse = get_RMSE(rating_matrix, user_latent_factor_matrix[:], item_latent_factor_matrix[:], rmse_loop_num)
        #         est_rmse.append(new_rmse)
        #         print('this is the %s th iteration and the change of RMSE is %s:' %(iter, np.abs(new_rmse-est_rmse[iter])))
        if (method == 'l2'):
            for iter in range(iteration_num):
                for i, j in enumerate(user_loop_interval[:-1]):
                    next_node = user_loop_interval[i + 1]
                    user_latent_factor_matrix[j:next_node, ] = ridge_reg(item_latent_factor_matrix[:],
                                                                         y[j:next_node, ].transpose(),
                                                                         lambdas=lambda_user)
                for i, j in enumerate(item_loop_interval[:-1]):
                    next_node = item_loop_interval[i + 1]
                    item_latent_factor_matrix[j:next_node, ] = ridge_reg(user_latent_factor_matrix[:],
                                                                         transpose_y[j:next_node, ].transpose(),
                                                                         lambdas=lambda_item)
                # calculate the RMSE
                new_rmse = get_RMSE(rating_matrix, user_latent_factor_matrix[:], item_latent_factor_matrix[:],
                                    rmse_loop_num)
                est_rmse.append(new_rmse)
                print('this is the %s th iteration ,the RMSE is: %s and the change of RMSE is: %s' % (
                iter, new_rmse, (new_rmse - est_rmse[iter])/est_rmse[iter]))

                new_rmse_on_vali = validation(latentFilePlace,validationFilePlace )
                est_rmse_on_vali.append(new_rmse_on_vali)
                print('this is the %s th iteration ,the RMSE_on_validataion is: %s and the change of RMSE_on_vali is: %s' % (
                    iter, new_rmse_on_vali, (new_rmse_on_vali - est_rmse_on_vali[iter]) / est_rmse_on_vali[iter]))

        elif(method=='l1'):
            for iter in range(iteration_num):
                for i, j in enumerate(user_loop_interval[:-1]):
                    next_node = user_loop_interval[i + 1]
                    user_latent_factor_matrix[j:next_node, ] = lasso_reg(item_latent_factor_matrix[:],
                                                                         y[j:next_node, ].transpose(),
                                                                         lambdas=lambda_user)
                for i, j in enumerate(item_loop_interval[:-1]):
                    next_node = item_loop_interval[i + 1]
                    item_latent_factor_matrix[j:next_node, ] = lasso_reg(user_latent_factor_matrix[:],
                                                                         y[:, j:next_node], lambdas=lambda_item)
                # calculate the RMSE
                new_rmse = get_RMSE(rating_matrix, user_latent_factor_matrix[:], item_latent_factor_matrix[:],
                                    rmse_loop_num)
                est_rmse.append(new_rmse)
                print('this is the %s th iteration ,the RMSE is: %s and the change of RMSE is: %s' % (
                    iter, new_rmse, (new_rmse - est_rmse[iter]) / est_rmse[iter]))

                new_rmse_on_vali = validation(latentFilePlace, validationFilePlace)
                est_rmse_on_vali.append(new_rmse_on_vali)
                print(
                    'this is the %s th iteration ,the RMSE_on_validataion is: %s and the change of RMSE_on_vali is: %s' % (
                        iter, new_rmse_on_vali, (new_rmse_on_vali - est_rmse_on_vali[iter]) / est_rmse_on_vali[iter]))

        else:
            print('no this method')
            print (method)
            raise Exception('error occur !!')

        return est_rmse,est_rmse_on_vali

def decomposed_rating_matrix(sparse_matrix, k):
    u, s, vt = svds(sparse_matrix, k)
    return u.dot(np.diag(s)), vt.transpose()

def ridge_reg(X , y, lambdas):
    reg = linear_model.Ridge(alpha=lambdas, fit_intercept=False)
    reg.fit(X, y)
    return reg.coef_

def lasso_reg(X, y, lambdas):
    reg = linear_model.Lasso(alpha=lambdas, fit_intercept=False)
    reg.fit(X, y)
    return reg.coef_

#supposed p and q are small enough to fit the memory
def get_RMSE(rating_matrix, user_matrix, item_matrix, loop_num):
    count = len(rating_matrix.data)
    #m, n = rating_matrix.shape
    result = 0
    interval = np.linspace(0, count, num=loop_num+1, dtype=np.int)
    for i, j in enumerate(interval[:-1]):
        next_node = interval[i+1]
        part_row_index = rating_matrix.row[j:next_node]
        part_col_index = rating_matrix.col[j:next_node]
        result+= calcu_RMSE(rating_matrix.data[j:next_node], np.sum(np.multiply(user_matrix[part_row_index, ], item_matrix[part_col_index, ]), axis=1))
    return np.sqrt(result/count)

def calcu_RMSE(observed_rating, estimated_rating):
    return np.sum(np.square(observed_rating-estimated_rating))

# def graph(y):
#     data = pd.DataFrame({'iteration':list(range(len(y))), 'RMSE':y})
#     p = gg.ggplot(gg.aes(x='iteration', y='RMSE'), data=data)+gg.geom_point()+gg.geom_line()
#     return p

def validation(latentFilePlace,validationFilePlace ,
               userID = 'userID',itemID = 'artistID' ,targetID = 'weight'):
    # USE FOR TEST :userID = 'userID', itemID = 'artistID', targetID = 'weight'
    with h5py.File(latentFilePlace,'r') as h5f:
        #h5f = h5py.File(latentFilePlace)
        item = h5f['itemLatentFactor'].value
        user = h5f['userLatentFactor'].value
        evaluate = user.dot(item.transpose())
        validationset = pd.read_csv(validationFilePlace)
        lenOfValidation = validationset.shape[0]
        userlist  = validationset[userID]
        itemlist  = validationset[itemID]
        target = validationset[targetID]
        target_hat = evaluate[userlist,itemlist]
        eps =np.sqrt( ((target_hat - target)**2).sum() / lenOfValidation)
        return(eps)

def trainAndValidation(trainFilePlace, prepare_path,
                prepare_name, transpose_prepare_name, factor_num, method, iteration_num,
                 user_loop_num,item_loop_num, lambda_user, lambda_item,
                 latentFilePlace,validationFilePlace):

    trainAfterDealing = pd.read_csv(trainFilePlace)
    row = trainAfterDealing['userID'].get_values()
    col = trainAfterDealing['artistID'].get_values()
    data = trainAfterDealing['weight'].get_values()

    rating_matrix = coo_matrix((data, (row, col)), dtype=np.float)

    del  row, col, data
    gc.collect()

    rmse,rmse_val = als(rating_matrix=rating_matrix,  prepare_path= prepare_path,
                        prepare_name=prepare_name, transpose_prepare_name=transpose_prepare_name,
                        factor_num=factor_num, method=method, iteration_num=iteration_num,
                        user_loop_num=user_loop_num,item_loop_num=item_loop_num,
                        lambda_user=lambda_user, lambda_item=lambda_item,
                    latentFilePlace=latentFilePlace,validationFilePlace=validationFilePlace
                 )
    return (rmse,rmse_val)

if __name__=='__main__':
   pass