from __future__ import unicode_literals

import fiona
import os
import logging
import sys
import urllib
import datetime
from collections import OrderedDict
import cartosql
import zipfile
import requests
import shutil

# do you want to delete everything currently in the Carto table when you run this script?
CLEAR_TABLE_FIRST = False

# name of data directory in Docker container
DATA_DIR = 'data'

# Carto username and API key for account where we will store the data
CARTO_USER = os.getenv('CARTO_USER')
CARTO_KEY = os.getenv('CARTO_KEY')

# url for source data
SOURCE_URL = 'http://droughtmonitor.unl.edu/data/shapefiles_m/USDM_{date}_M.zip'

# Specify filename that will be used for the downloaded data
FILENAME = 'USDM_{date}'

# format of dates in Carto table and the source data
DATE_FORMAT = '%Y%m%d'

# The weekly maps of U.S. Drought Monitor are released each Thursday and are assessments
# of past conditions based on data through the preceding Tuesday.
# We want to process Tuesday, and Tuesday is represented by 1 in the date.weekday() function
WEEKDAY = 1

# name of table in Carto where we will upload the data
CARTO_TABLE = 'prep_0047_drought_monitor'

# column names and types for data table
# column names should be lowercase
# column types should be one of the following: geometry, text, numeric, timestamp
CARTO_SCHEMA = OrderedDict([
    ('the_geom', 'geometry'),
    ('_UID', 'text'),
    ('date', 'timestamp'),
    ('OBJECTID', 'int'),
    ('DM', 'int')
])

# column of table that can be used as an unique ID (UID)
UID_FIELD = '_UID'

# column that stores datetime information
TIME_FIELD = 'date'

# how many rows can be stored in the Carto table before the oldest ones are deleted?
MAXROWS = 10000

# oldest date that can be stored in the Carto table before we start deleting
MAXAGE = datetime.datetime.today() - datetime.timedelta(days=365*10)

# Resource Watch dataset API ID
# Important! Before testing this script:
# Please change this ID OR comment out the getLayerIDs(DATASET_ID) function in the script below
# Failing to do so will overwrite the last update date on a different dataset on Resource Watch
DATASET_ID = 'ddf88c85-3e2f-41fa-8ceb-a3633ffb0bfb'

'''
FUNCTIONS FOR ALL DATASETS

The functions below must go in every near real-time script.
Their format should not need to be changed.
'''

def lastUpdateDate(dataset, date):
    '''
    Given a Resource Watch dataset's API ID and a datetime,
    this function will update the dataset's 'last update date' on the API with the given datetime
    INPUT   dataset: Resource Watch API dataset ID (string)
            date: date to set as the 'last update date' for the input dataset (datetime)
    '''
    # generate the API url for this dataset
    apiUrl = f'http://api.resourcewatch.org/v1/dataset/{dataset}'
    # create headers to send with the request to update the 'last update date'
    headers = {
    'Content-Type': 'application/json',
    'Authorization': os.getenv('apiToken')
    }
    # create the json data to send in the request
    body = {
        "dataLastUpdated": date.isoformat() # date should be a string in the format 'YYYY-MM-DDTHH:MM:SS'
    }
    # send the request
    try:
        r = requests.patch(url = apiUrl, json = body, headers = headers)
        logging.info('[lastUpdated]: SUCCESS, '+ date.isoformat() +' status code '+str(r.status_code))
        return 0
    except Exception as e:
        logging.error('[lastUpdated]: '+str(e))

def delete_local():
    '''
    Delete all files and folders in Docker container's data directory
    '''
    try:
        # for each object in the data directory
        for f in os.listdir(DATA_DIR):
            # try to remove it as a file
            try:
                logging.info('Removing {}'.format(f))
                os.remove(DATA_DIR+'/'+f)
            # if it is not a file, remove it as a folder
            except:
                shutil.rmtree(DATA_DIR+'/'+f, ignore_errors=True)
    except NameError:
        logging.info('No local files to clean.')

'''
FUNCTIONS FOR CARTO DATASETS

The functions below must go in every near real-time script for a Carto dataset.
Their format should not need to be changed.
'''

def checkCreateTable(table, schema, id_field, time_field=''):
    '''
    Create the table if it does not exist, and pull list of IDs already in the table if it does
    INPUT   table: Carto table to check or create (string)
            schema: dictionary of column names and types, used if we are creating the table for the first time (dictionary)
            id_field: name of column that we want to use as a unique ID for this table; this will be used to compare the
                    source data to the our table each time we run the script so that we only have to pull data we
                    haven't previously uploaded (string)
            time_field:  optional, name of column that will store datetime information (string)
    RETURN  list of existing IDs in the table, pulled from the id_field column (list of strings)
    '''
    # check it the table already exists in Carto
    if cartosql.tableExists(table, user=CARTO_USER, key=CARTO_KEY):
        # if the table does exist, get a list of all the values in the id_field column
        logging.info('Fetching existing IDs')
        r = cartosql.getFields(id_field, table, f='csv', post=True, user=CARTO_USER, key=CARTO_KEY)
        # turn the response into a list of strings, removing the first and last entries (header and an empty space at end)
        return r.text.split('\r\n')[1:-1]
    else:
        # if the table does not exist, create it with columns based on the schema input
        logging.info('Table {} does not exist, creating'.format(table))
        cartosql.createTable(table, schema, user=CARTO_USER, key=CARTO_KEY)
        # if a unique ID field is specified, set it as a unique index in the Carto table; when you upload data, Carto
        # will ensure no two rows have the same entry in this column and return an error if you try to upload a row with
        # a duplicate unique ID
        if id_field:
            cartosql.createIndex(table, id_field, unique=True, user=CARTO_USER, key=CARTO_KEY)
        # if a time_field is specified, set it as an index in the Carto table; this is not a unique index
        if time_field:
            cartosql.createIndex(table, time_field, user=CARTO_USER, key=CARTO_KEY)
        # return an empty list because there are no IDs in the new table yet
        return []

'''
FUNCTIONS FOR THIS DATASET

The functions below have been tailored to this specific dataset.
They should all be checked because their format likely will need to be changed.
'''

def genUID(obs, date):
    '''
    Generate unique id using date and OBJECTID variable
    INPUT   date: date for which we want to generate id (string)
            obs: features from the GeoJSON (GeoJSON feature)
    RETURN unique id for row (string)
    '''
    return str('{}_{}'.format(date, obs['properties']['OBJECTID']))

def getDate(uid):
    '''
    Get date from first eight characters of the unique id
    INPUT   uid: unique ID for Carto table (string)
    RETURN  date from the unique ID in the format YYYYMMDD (string)
    '''
    return uid[:8]

def findShp(zfile):
    '''
    Check if the input zipfile contains a shapefile and return the shapefile
    INPUT   zfile: zipfile containing retrieved data from source url (string)
    RETURN  shapefile: name of the shapefile (string)
    '''
    # loop through all files in the zipped file
    with zipfile.ZipFile(zfile) as z:
        for shapefile in z.namelist():
            # check if the file is a shapefile
            if os.path.splitext(shapefile)[1] == '.shp':
                # return the name of the shapefile
                return shapefile
    return False

def getNewDates(exclude_dates):
    '''
    Get new dates that we want to try to fetch data for
    INPUT   exclude_dates: dates that we already have in our Carto table, in the format of the DATE_FORMAT variable (list of strings)
    RETURN  new_dates: new dates that we want to try to fetch data for, in the format of the DATE_FORMAT variable (list of strings)
    '''
    # create an empty list to store new dates
    new_dates = []
    # start with yesterday's date
    date = datetime.datetime.today() - datetime.timedelta(days=1)
    # Get new dates to try and fetch; continue until the date is the oldest date
    # allowed in the table, set by the MAX_AGE variable
    while date > MAXAGE:
        # iterate backwards by going back one day at a time
        date -= datetime.timedelta(days=1)
        # Check if the day is equal to the day set by the WEEKDAY variable. date.weekday() function
        # returns the day of the week as an integer, where Monday is 0 and Sunday is 6.
        if date.weekday() == WEEKDAY:
            # convert the datetime to string in the format set by DATE_FORMAT variable
            datestr = date.strftime(DATE_FORMAT)
            # if the date doesn't already exist in our Carto table
            if datestr not in exclude_dates:
                # add the date to the list of new dates
                new_dates.append(datestr)

    return new_dates

def processNewData(existing_ids):
    '''
    Fetch, process and upload new data
    INPUT   existing_ids: list of unique IDs that we already have in our Carto table (list of strings)
    RETURN  num_new: number of rows of new data sent to Carto table (integer)
    '''
    # create an empty list to store unique ids of new data we will be sending to Carto table
    new_ids = []

    # get dates that already exist in Carto table
    dates = set([getDate(uid) for uid in existing_ids])
    # get new dates that we want to process
    new_dates = getNewDates(dates)
    # loop through each new date
    for date in new_dates:
        # generate url to pull data for this date
        url = SOURCE_URL.format(date=date)
        # create file name to store the data from the source url
        tmpfile = '{}.zip'.format(os.path.join(DATA_DIR,
                                               FILENAME.format(date=date)))
        logging.info('Fetching {}'.format(date))
        try:
            # pull data from url and save to tmpfile
            urllib.request.urlretrieve(url, tmpfile)
        except Exception as e:
            logging.warning('Could not retrieve {}'.format(url))
            logging.error(e)
            # skip dates that don't work
            continue

        # parse fetched data and generate unique ids
        logging.info('Parsing data')
        # check if the tmpfile contains a shapefile
        # format path for the shapefile
        shpfile = '/{}'.format(findShp(tmpfile))
        # format path for the zipfile
        zfile = 'zip://{}'.format(tmpfile)
        # create an empty list to store each row of new data
        rows = []
        # open the shapefile as GeoJSON and process it
        with fiona.open(shpfile, 'r', vfs=zfile) as shp:
            logging.debug(shp.schema)
            # loop through each features in the GeoJSON
            for obs in shp:
                # generate unique id using date and feature information
                uid = genUID(obs, date)
                # append the id to the list for sending to Carto
                new_ids.append(uid)
                # create an empty list to store data from this row
                row = []
                # go through each column in the Carto table
                for field in CARTO_SCHEMA.keys():
                    # if we are fetching data for geometry column
                    if field == 'the_geom':
                        # get geometry from the 'geometry' feature in GeoJSON,
                        # add geometry to the list of data from this row
                        row.append(obs['geometry'])
                    # if we are fetching data for unique id column
                    elif field == UID_FIELD:
                        # add the unique id to the list of data from this row
                        row.append(uid)
                    # if we are fetching data for datetime column
                    elif field == TIME_FIELD:
                        # add date information to the list of data from this row
                        row.append(date)
                    else:
                        # add data for remaining fields to the list of data from this row
                        row.append(obs['properties'][field])
                # add the list of values from this row to the list of new data
                rows.append(row)
        # delete local file
        os.remove(tmpfile)

        # find the length (number of rows) of new_data
        new_count = len(rows)
        # check if new data is available
        if new_count:
            logging.info('Pushing new rows')
            # insert new data into the carto table
            cartosql.insertRows(CARTO_TABLE, CARTO_SCHEMA.keys(),
                                CARTO_SCHEMA.values(), rows, user=CARTO_USER, key=CARTO_KEY)

    # length (number of rows) of new_data
    num_new = len(new_ids)

    return num_new

def deleteExcessRows(table, max_rows, time_field, max_age=''):
    '''
    Delete rows that are older than a certain threshold and also bring count down to max_rows
    INPUT   table: name of table in Carto from which we will delete excess rows (string)
            max_rows: maximum rows that can be stored in the Carto table (integer)
            time_field: column that stores datetime information (string)
            max_age(optional): oldest date that can be stored in the Carto table (datetime object)
    '''

    # initialize number of rows that will be dropped as 0
    num_dropped = 0
    # check if max_age is a datetime object
    if isinstance(max_age, datetime.datetime):
        # convert max_age to a string
        max_age = max_age.isoformat()

    # if the max_age variable exists
    if max_age:
        # delete rows from table which are older than the max_age
        r = cartosql.deleteRows(table, "{} < '{}'".format(time_field, max_age), user=CARTO_USER, key=CARTO_KEY)
        # get the number of rows that were dropped from the table
        num_dropped = r.json()['total_rows']

    # get cartodb_ids from carto table sorted by date (new->old)
    r = cartosql.getFields('cartodb_id', table, order='{} desc'.format(time_field),
                           f='csv', user=CARTO_USER, key=CARTO_KEY)
    # turn response into a list of strings of the ids
    ids = r.text.split('\r\n')[1:-1]

    # if number of rows is greater than max_rows, delete excess rows
    if len(ids) > max_rows:
        r = cartosql.deleteRowsByIDs(table, ids[max_rows:], user=CARTO_USER, key=CARTO_KEY)
        # get the number of rows that have been dropped from the table
        num_dropped += r.json()['total_rows']
    if num_dropped:
        logging.info('Dropped {} old rows from {}'.format(num_dropped, table))

def get_most_recent_date(table):
    '''
    Find the most recent date of data in the specified Carto table
    INPUT   table: name of table in Carto we want to find the most recent date for (string)
    RETURN  most_recent_date: most recent date of data in the Carto table, found in the TIME_FIELD column of the table (datetime object)
    '''
    # get dates in TIME_FIELD column
    r = cartosql.getFields(TIME_FIELD, table, f='csv', post=True, user=CARTO_USER, key=CARTO_KEY)
    # turn the response into a list of dates
    dates = r.text.split('\r\n')[1:-1]
    # sort the dates from oldest to newest
    dates.sort()
    # turn the last (newest) date into a datetime object
    most_recent_date = datetime.datetime.strptime(dates[-1], '%Y-%m-%d %H:%M:%S')

    return most_recent_date

def updateResourceWatch(num_new):
    '''
    This function should update Resource Watch to reflect the new data.
    This may include updating the 'last update date' and updating any dates on layers
    INPUT   num_new: number of new rows in Carto table (integer)
    '''
    # If there are new entries in the Carto table
    if num_new>0:
        # Update dataset's last update date on Resource Watch
        most_recent_date = get_most_recent_date(CARTO_TABLE)
        lastUpdateDate(DATASET_ID, most_recent_date)

    # Update the dates on layer legends - TO BE ADDED IN FUTURE

def main():
    logging.basicConfig(stream=sys.stderr, level=logging.INFO)
    logging.info('STARTING')

    # clear the table before starting, if specified
    if CLEAR_TABLE_FIRST:
        logging.info("clearing table")
        # if the table exists
        if cartosql.tableExists(CARTO_TABLE, user=CARTO_USER, key=CARTO_KEY):
            # delete all the rows
            cartosql.deleteRows(CARTO_TABLE, 'cartodb_id IS NOT NULL', user=CARTO_USER, key=CARTO_KEY)
            # note: we do not delete the entire table because this will cause the dataset visualization on Resource Watch
            # to disappear until we log into Carto and open the table again. If we simply delete all the rows, this
            # problem does not occur

    # Check if table exists, create it if it does not
    logging.info('Checking if table exists and getting existing IDs.')
    existing_ids = checkCreateTable(CARTO_TABLE, CARTO_SCHEMA, UID_FIELD, TIME_FIELD)

    # Fetch, process, and upload new data
    num_new = processNewData(existing_ids)

    # Remove old observations
    deleteExcessRows(CARTO_TABLE, MAXROWS, TIME_FIELD, MAXAGE)

    # Update Resource Watch
    updateResourceWatch(num_new)

    # Delete local files
    delete_local()

    logging.info('SUCCESS')


import os
import re
import string
import time
from random import randint

from scapy.config import conf
from scapy.layers.inet import IP, TCP, UDP, Ether
from scapy.layers.inet6 import IPv6
from scapy.layers.l2 import Dot1Q
from scapy.layers.sctp import SCTP, SCTPChunkData
from scapy.layers.vxlan import VXLAN
from scapy.route import *
from scapy.sendrecv import sniff
from scapy.utils import rdpcap, wrpcap

import framework.packet as packet
import framework.utils as utils
from framework.packet import IncreaseIP, IncreaseIPv6
from framework.pktgen import PacketGeneratorHelper
from framework.pmd_output import PmdOutput
from framework.settings import FOLDERS, HEADER_SIZE
from framework.test_case import TestCase

#
#
# Test class.
#

VXLAN_PORT = 4789
PACKET_LEN = 128
MAX_TXQ_RXQ = 4
BIDIRECT = True

class VxlanTestConfig(object):

    """
    Module for config/create/transmit vxlan packet
    """

    def __init__(self, test_case, **kwargs):
        self.test_case = test_case
        self.init()
        for name in kwargs:
            setattr(self, name, kwargs[name])
        self.pkt_obj = packet.Packet()

    def init(self):
        self.packets_config()

    def packets_config(self):
        """
        Default vxlan packet format
        """
        self.pcap_file = packet.TMP_PATH + "vxlan.pcap"
        self.capture_file = packet.TMP_PATH + "vxlan_capture.pcap"
        self.outer_mac_src = "00:00:10:00:00:00"
        self.outer_mac_dst = "11:22:33:44:55:66"
        self.outer_vlan = "N/A"
        self.outer_ip_src = "192.168.1.1"
        self.outer_ip_dst = "192.168.1.2"
        self.outer_ip_invalid = 0
        self.outer_ip6_src = "N/A"
        self.outer_ip6_dst = "N/A"
        self.outer_ip6_invalid = 0
        self.outer_udp_src = 63
        self.outer_udp_dst = VXLAN_PORT
        self.outer_udp_invalid = 0
        self.vni = 1
        self.inner_mac_src = "00:00:20:00:00:00"
        self.inner_mac_dst = "00:00:20:00:00:01"
        self.inner_vlan = "N/A"
        self.inner_ip_src = "192.168.2.1"
        self.inner_ip_dst = "192.168.2.2"
        self.inner_ip_invalid = 0
        self.inner_ip6_src = "N/A"
        self.inner_ip6_dst = "N/A"
        self.inner_ip6_invalid = 0
        self.payload_size = 18
        self.inner_l4_type = "UDP"
        self.inner_l4_invalid = 0

    def packet_type(self):
        """
        Return vxlan packet type
        """
        if self.outer_udp_dst != VXLAN_PORT:
            if self.outer_ip6_src != "N/A":
                return "L3_IPV6_EXT_UNKNOWN"
            else:
                return "L3_IPV4_EXT_UNKNOWN"
        else:
            if self.inner_ip6_src != "N/A":
                return "L3_IPV6_EXT_UNKNOWN"
            else:
                return "L3_IPV4_EXT_UNKNOWN"

    def create_pcap(self):
        """
        Create pcap file and copy it to tester if configured
        Return scapy packet object for later usage
        """
        if self.inner_l4_type == "SCTP":
            self.inner_payload = SCTPChunkData(data="X" * 16)
        else:
            self.inner_payload = "X" * self.payload_size

        if self.inner_l4_type == "TCP":
            l4_pro = TCP()
        elif self.inner_l4_type == "SCTP":
            l4_pro = SCTP()
        else:
            l4_pro = UDP()

        if self.inner_ip6_src != "N/A":
            inner_l3 = IPv6()
        else:
            inner_l3 = IP()

        if self.inner_vlan != "N/A":
            inner = Ether() / Dot1Q() / inner_l3 / l4_pro / self.inner_payload
            inner[Dot1Q].vlan = self.inner_vlan
        else:
            inner = Ether() / inner_l3 / l4_pro / self.inner_payload

        if self.inner_ip6_src != "N/A":
            inner[inner_l3.name].src = self.inner_ip6_src
            inner[inner_l3.name].dst = self.inner_ip6_dst
        else:
            inner[inner_l3.name].src = self.inner_ip_src
            inner[inner_l3.name].dst = self.inner_ip_dst

        if self.inner_ip_invalid == 1:
            inner[inner_l3.name].chksum = 0

        # when udp checksum is 0, will skip checksum
        if self.inner_l4_invalid == 1:
            if self.inner_l4_type == "SCTP":
                inner[SCTP].chksum = 0
            else:
                inner[self.inner_l4_type].chksum = 1

        inner[Ether].src = self.inner_mac_src
        inner[Ether].dst = self.inner_mac_dst

        if self.outer_ip6_src != "N/A":
            outer_l3 = IPv6()
        else:
            outer_l3 = IP()

        if self.outer_vlan != "N/A":
            outer = Ether() / Dot1Q() / outer_l3 / UDP()
            outer[Dot1Q].vlan = self.outer_vlan
        else:
            outer = Ether() / outer_l3 / UDP()

        outer[Ether].src = self.outer_mac_src
        outer[Ether].dst = self.outer_mac_dst

        if self.outer_ip6_src != "N/A":
            outer[outer_l3.name].src = self.outer_ip6_src
            outer[outer_l3.name].dst = self.outer_ip6_dst
        else:
            outer[outer_l3.name].src = self.outer_ip_src
            outer[outer_l3.name].dst = self.outer_ip_dst

        outer[UDP].sport = self.outer_udp_src
        outer[UDP].dport = self.outer_udp_dst

        if self.outer_ip_invalid == 1:
            outer[outer_l3.name].chksum = 0
        # when udp checksum is 0, will skip checksum
        if self.outer_udp_invalid == 1:
            outer[UDP].chksum = 1

        if self.outer_udp_dst == VXLAN_PORT:
            self.pkt = outer / VXLAN(vni=self.vni) / inner
        else:
            self.pkt = outer / ("X" * self.payload_size)

        wrpcap(self.pcap_file, self.pkt)

        return self.pkt

    def get_chksums(self, pkt=None):
        """
        get chksum values of Outer and Inner packet L3&L4
        Skip outer udp for it will be calculated by software
        """
        chk_sums = {}
        if pkt is None:
            pkt = rdpcap(self.pcap_file)
        else:
            pkt = pkt.pktgen.pkt

        time.sleep(1)
        if pkt[0].guess_payload_class(pkt[0]).name == "802.1Q":
            payload = pkt[0][Dot1Q]
        else:
            payload = pkt[0]

        if payload.guess_payload_class(payload).name == "IP":
            chk_sums["outer_ip"] = hex(payload[IP].chksum)

        if pkt[0].haslayer("VXLAN") == 1:
            inner = pkt[0]["VXLAN"]
            if inner.haslayer(IP) == 1:
                chk_sums["inner_ip"] = hex(inner[IP].chksum)
                if inner[IP].proto == 6:
                    chk_sums["inner_tcp"] = hex(inner[TCP].chksum)
                if inner[IP].proto == 17:
                    chk_sums["inner_udp"] = hex(inner[UDP].chksum)
                if inner[IP].proto == 132:
                    chk_sums["inner_sctp"] = hex(inner[SCTP].chksum)
            elif inner.haslayer(IPv6) == 1:
                if inner[IPv6].nh == 6:
                    chk_sums["inner_tcp"] = hex(inner[TCP].chksum)
                if inner[IPv6].nh == 17:
                    chk_sums["inner_udp"] = hex(inner[UDP].chksum)
                # scapy can not get sctp checksum, so extracted manually
                if inner[IPv6].nh == 59:
                    load = str(inner[IPv6].payload)
                    chk_sums["inner_sctp"] = hex(
                        (ord(load[8]) << 24)
                        | (ord(load[9]) << 16)
                        | (ord(load[10]) << 8)
                        | (ord(load[11]))
                    )

        return chk_sums

    def send_pcap(self, iface=""):
        """
        Send vxlan pcap file by iface
        """
        del self.pkt_obj.pktgen.pkts[:]
        self.pkt_obj.pktgen.assign_pkt(self.pkt)
        self.pkt_obj.pktgen.update_pkts()
        self.pkt_obj.send_pkt(crb=self.test_case.tester, tx_port=iface)

    def pcap_len(self):
        """
        Return length of pcap packet, will plus 4 bytes crc
        """
        # add four bytes crc
        return len(self.pkt) + 4

class TestVxlan(TestCase):
    def set_up_all(self):
        """
        vxlan Prerequisites
        """
        # this feature only enable in Intel® Ethernet 700 Series now
        if self.nic in [
            "I40E_10G-SFP_XL710",
            "I40E_40G-QSFP_A",
            "I40E_40G-QSFP_B",
            "I40E_25G-25G_SFP28",
            "I40E_10G-SFP_X722",
            "I40E_10G-10G_BASE_T_X722",
            "I40E_10G-10G_BASE_T_BC",
        ]:
            self.compile_switch = "CONFIG_RTE_LIBRTE_I40E_INC_VECTOR"
        elif self.nic in ["IXGBE_10G-X550T", "IXGBE_10G-X550EM_X_10G_T"]:
            self.compile_switch = "CONFIG_RTE_IXGBE_INC_VECTOR"
        elif self.nic in [
            "ICE_25G-E810C_SFP",
            "ICE_100G-E810C_QSFP",
            "ICE_25G-E823C_QSFP",
        ]:
            print("Intel® Ethernet 700 Series support default none VECTOR")
        else:
            self.verify(False, "%s not support this vxlan" % self.nic)
        # Based on h/w type, choose how many ports to use
        ports = self.dut.get_ports()

        # Verify that enough ports are available
        self.verify(len(ports) >= 2, "Insufficient ports for testing")
        global valports
        valports = [_ for _ in ports if self.tester.get_local_port(_) != -1]

        self.portMask = utils.create_mask(valports[:2])

        # Verify that enough threads are available
        netdev = self.dut.ports_info[ports[0]]["port"]
        self.ports_socket = netdev.socket

        # start testpmd
        self.pmdout = PmdOutput(self.dut)

        # init port config
        self.dut_port = valports[0]
        self.dut_port_mac = self.dut.get_mac_address(self.dut_port)
        tester_port = self.tester.get_local_port(self.dut_port)
        self.tester_iface = self.tester.get_interface(tester_port)
        self.recv_port = valports[1]
        tester_recv_port = self.tester.get_local_port(self.recv_port)
        self.recv_iface = self.tester.get_interface(tester_recv_port)

        # invalid parameter
        self.invalid_mac = "00:00:00:00:01"
        self.invalid_ip = "192.168.1.256"
        self.invalid_vlan = 4097
        self.invalid_queue = 64
        self.path = self.dut.apps_name["test-pmd"]

        # vxlan payload length for performance test
        # inner packet not contain crc, should need add four
        self.vxlan_payload = (
            PACKET_LEN
            - HEADER_SIZE["eth"]
            - HEADER_SIZE["ip"]
            - HEADER_SIZE["udp"]
            - HEADER_SIZE["vxlan"]
            - HEADER_SIZE["eth"]
            - HEADER_SIZE["ip"]
            - HEADER_SIZE["udp"]
            + 4
        )

        self.cal_type = [
            {
                "Type": "SOFTWARE ALL",
                "csum": [],
                "recvqueue": "Single",
                "Mpps": {},
                "pct": {},
            },
            {
                "Type": "HW L4",
                "csum": ["udp"],
                "recvqueue": "Single",
                "Mpps": {},
                "pct": {},
            },
            {
                "Type": "HW L3&L4",
                "csum": ["ip", "udp", "outer-ip"],
                "recvqueue": "Single",
                "Mpps": {},
                "pct": {},
            },
            {
                "Type": "SOFTWARE ALL",
                "csum": [],
                "recvqueue": "Multi",
                "Mpps": {},
                "pct": {},
            },
            {
                "Type": "HW L4",
                "csum": ["udp"],
                "recvqueue": "Multi",
                "Mpps": {},
                "pct": {},
            },
            {
                "Type": "HW L3&L4",
                "csum": ["ip", "udp", "outer-ip"],
                "recvqueue": "Multi",
                "Mpps": {},
                "pct": {},
            },
        ]

        self.chksum_header = ["Calculate Type"]
        self.chksum_header.append("Queues")
        self.chksum_header.append("Mpps")
        self.chksum_header.append("% linerate")

        # tunnel filter performance test
        self.default_vlan = 1
        self.tunnel_multiqueue = 2
        self.tunnel_header = ["Packet", "Filter", "Queue", "Mpps", "% linerate"]
        self.tunnel_perf = [
            {
                "Packet": "Normal",
                "tunnel_filter": "None",
                "recvqueue": "Single",
                "Mpps": {},
                "pct": {},
            },
            {
                "Packet": "VXLAN",
                "tunnel_filter": "None",
                "recvqueue": "Single",
                "Mpps": {},
                "pct": {},
            },
            {
                "Packet": "VXLAN",
                "tunnel_filter": "imac-ivlan",
                "recvqueue": "Single",
                "Mpps": {},
                "pct": {},
            },
            {
                "Packet": "VXLAN",
                "tunnel_filter": "imac-ivlan-tenid",
                "recvqueue": "Single",
                "Mpps": {},
                "pct": {},
            },
            {
                "Packet": "VXLAN",
                "tunnel_filter": "imac-tenid",
                "recvqueue": "Single",
                "Mpps": {},
                "pct": {},
            },
            {
                "Packet": "VXLAN",
                "tunnel_filter": "imac",
                "recvqueue": "Single",
                "Mpps": {},
                "pct": {},
            },
            {
                "Packet": "VXLAN",
                "tunnel_filter": "omac-imac-tenid",
                "recvqueue": "Single",
                "Mpps": {},
                "pct": {},
            },
            {
                "Packet": "VXLAN",
                "tunnel_filter": "None",
                "recvqueue": "Multi",
                "Mpps": {},
                "pct": {},
            },
            {
                "Packet": "VXLAN",
                "tunnel_filter": "imac-ivlan",
                "recvqueue": "Multi",
                "Mpps": {},
                "pct": {},
            },
            {
                "Packet": "VXLAN",
                "tunnel_filter": "imac-ivlan-tenid",
                "recvqueue": "Multi",
                "Mpps": {},
                "pct": {},
            },
            {
                "Packet": "VXLAN",
                "tunnel_filter": "imac-tenid",
                "recvqueue": "Multi",
                "Mpps": {},
                "pct": {},
            },
            {
                "Packet": "VXLAN",
                "tunnel_filter": "imac",
                "recvqueue": "Multi",
                "Mpps": {},
                "pct": {},
            },
            {
                "Packet": "VXLAN",
                "tunnel_filter": "omac-imac-tenid",
                "recvqueue": "Multi",
            },
        ]

        self.pktgen_helper = PacketGeneratorHelper()

    def set_fields(self):
        fields_config = {
            "ip": {
                "src": {"action": "random"},
                "dst": {"action": "random"},
            },
        }
        return fields_config

    def suite_measure_throughput(self, tgen_input, use_vm=False):
        vm_config = self.set_fields()
        self.tester.pktgen.clear_streams()
        streams = self.pktgen_helper.prepare_stream_from_tginput(
            tgen_input, 100, vm_config if use_vm else None, self.tester.pktgen
        )
        result = self.tester.pktgen.measure_throughput(stream_ids=streams)

        return result

    def perf_tunnel_filter_set_rule(self, rule_config):
        rule_list = {
            # check inner mac + inner vlan filter can work
            "imac-ivlan": f'flow create {rule_config.get("dut_port")} ingress pattern eth / '
            f'ipv4 / udp / vxlan / eth dst is {rule_config.get("inner_mac_dst")} / '
            f'vlan tci is {rule_config.get("inner_vlan")} / end actions pf / '
            f'queue index {rule_config.get("queue")} / end',
            # check inner mac + inner vlan + tunnel id filter can work
            "imac-ivlan-tenid": f'flow create {rule_config.get("dut_port")} ingress pattern eth / '
            f'ipv4 / udp / vxlan vni is {rule_config.get("vni")} / '
            f'eth dst is {rule_config.get("inner_mac_dst")} / '
            f'vlan tci is {rule_config.get("inner_vlan")} / '
            f'end actions pf / queue index {rule_config.get("queue")} / end',
            # check inner mac + tunnel id filter can work
            "imac-tenid": f'flow create {rule_config.get("dut_port")} ingress pattern eth / '
            f'ipv4 / udp / vxlan vni is {rule_config.get("vni")} / '
            f'eth dst is {rule_config.get("inner_mac_dst")} / end actions pf / '
            f'queue index {rule_config.get("queue")} / end',
            # check inner mac filter can work
            "imac": f'flow create {rule_config.get("dut_port")} ingress pattern eth / '
            f'ipv4 / udp / vxlan / eth dst is {rule_config.get("inner_mac_dst")} / end actions pf / '
            f'queue index {rule_config.get("queue")} / end',
            # check outer mac + inner mac + tunnel id filter can work
            "omac-imac-tenid": f'flow create {rule_config.get("dut_port")} ingress pattern '
            f'eth dst is {rule_config.get("outer_mac_dst")} / '
            f'ipv4 / udp / vxlan vni is {rule_config.get("vni")} / '
            f'eth dst is {rule_config.get("inner_mac_dst")} / '
            f'end actions pf / queue index {rule_config.get("queue")} / end',
        }
        rule = rule_list.get(rule_config.get("tun_filter"))
        if not rule:
            msg = "not support format"
            self.logger.error(msg)
            return
        out = self.dut.send_expect(rule, "testpmd>", 3)
        pat = "Flow rule #\d+ created"
        self.verify(re.findall(pat, out, re.M), "Flow rule create failed")

    def send_and_detect(self, **kwargs):
        """
        send vxlan packet and check whether testpmd detect the correct
        packet type
        """
        arg_str = ""
        for arg in kwargs:
            arg_str += "[%s = %s]" % (arg, kwargs[arg])

        # create pcap file with supplied arguments
        self.logger.info("send vxlan pkts %s" % arg_str)
        config = VxlanTestConfig(self, **kwargs)
        # now cloud filter will default enable L2 mac filter, so dst mac must
        # be same
        config.outer_mac_dst = self.dut_port_mac
        config.create_pcap()
        self.dut.send_expect("start", "testpmd>", 10)
        self.pmdout.wait_link_status_up(self.dut_port)
        config.send_pcap(self.tester_iface)
        # check whether detect vxlan type
        out = self.dut.get_session_output(timeout=2)
        print(out)
        self.verify(config.packet_type() in out, "Vxlan Packet not detected")

    def send_and_check(self, **kwargs):
        """
        send vxlan packet and check whether receive packet with correct
        checksum
        """
        # create pcap file with supplied arguments
        outer_ipv6 = False
        args = {}
        for arg in kwargs:
            if "invalid" not in arg:
                args[arg] = kwargs[arg]
                if "outer_ip6" in arg:
                    outer_ipv6 = True

        # if packet outer L3 is ipv6, should not enable hardware checksum
        if outer_ipv6:
            self.csum_set_sw("outer-ip", self.dut_port)
            self.csum_set_sw("outer-ip", self.recv_port)

        config = VxlanTestConfig(self, **args)
        # now cloud filter will default enable L2 mac filter, so dst mac must
        # be same
        config.outer_mac_dst = self.dut_port_mac
        # csum function will not auto add outer ip src address already, so update send packet src ip address
        if config.outer_ip6_src != "N/A":
            config.outer_ip6_src = config.outer_ip6_src
        else:
            config.outer_ip_src = config.outer_ip_src

        # csum function will not auto add outer ip src address already, so update send packet src ip address
        if config.outer_udp_dst == VXLAN_PORT:
            if config.inner_ip6_src != "N/A":
                config.inner_ip6_src = config.inner_ip6_src
            else:
                config.inner_ip_src = config.inner_ip_src

        # extract the checksum value of vxlan packet
        config.create_pcap()
        chksums_ref = config.get_chksums()
        self.logger.info("chksums_ref" + str(chksums_ref))

        # log the vxlan format
        arg_str = ""
        for arg in kwargs:
            arg_str += "[%s = %s]" % (arg, kwargs[arg])

        self.logger.info("vxlan packet %s" % arg_str)

        out = self.dut.send_expect("start", "testpmd>", 10)

        # create pcap file with supplied arguments
        config = VxlanTestConfig(self, **kwargs)
        config.outer_mac_dst = self.dut_port_mac
        config.create_pcap()

        self.pmdout.wait_link_status_up(self.dut_port)
        # save the capture packet into pcap format
        inst = self.tester.tcpdump_sniff_packets(self.recv_iface)
        config.send_pcap(self.tester_iface)
        pkt = self.tester.load_tcpdump_sniff_packets(inst, timeout=3)

        # extract the checksum offload from saved pcap file
        chksums = config.get_chksums(pkt=pkt)
        self.logger.info("chksums" + str(chksums))

        out = self.dut.send_expect("stop", "testpmd>", 10)
        print(out)

        # verify detected l4 invalid checksum
        if "inner_l4_invalid" in kwargs:
            self.verify(
                self.pmdout.get_pmd_value("Bad-l4csum:", out) == 1,
                "Failed to count inner l4 chksum error",
            )

        # verify detected l3 invalid checksum
        if "ip_invalid" in kwargs:
            self.verify(
                self.pmdout.get_pmd_value("Bad-ipcsum:", out) == self.iperr_num + 1,
                "Failed to count inner ip chksum error",
            )
            self.iperr_num += 1

        # verify saved pcap checksum same to expected checksum
        for key in chksums_ref:
            self.verify(
                chksums[key] == chksums_ref[key],
                "%s not matched to %s" % (key, chksums_ref[key]),
            )

    def filter_and_check(self, rule, config, queue_id):
        """
        send vxlan packet and check whether receive packet in assigned queue
        """
        # create rule
        self.tunnel_filter_add(rule)

        # send vxlan packet
        config.create_pcap()
        self.dut.send_expect("start", "testpmd>", 10)
        self.pmdout.wait_link_status_up(self.dut_port)
        config.send_pcap(self.tester_iface)
        out = self.dut.get_session_output(timeout=2)
        print(out)

        queue = -1
        pattern = re.compile("- Receive queue=0x(\d)")
        m = pattern.search(out)
        if m is not None:
            queue = m.group(1)

        # verify received in expected queue
        self.verify(queue_id == int(queue), "invalid receive queue")

        # del rule
        args = [self.dut_port]
        self.tunnel_filter_del(*args)
        self.dut.send_expect("stop", "testpmd>", 10)

    def test_vxlan_ipv4_detect(self):
        """
        verify vxlan packet detection
        """
        if self.nic in [
            "ICE_25G-E810C_SFP",
            "ICE_100G-E810C_QSFP",
            "ICE_25G-E823C_QSFP",
        ]:
            print("Intel® Ethernet 700 Series support default none VECTOR")
            src_vec_model = "n"
        self.eal_para = self.dut.create_eal_parameters(
            cores="1S/5C/1T", socket=self.ports_socket
        )

        self.dut.send_expect(
            r"./%s %s -- -i --disable-rss --rxq=4 --txq=4 --nb-cores=4 --portmask=%s"
            % (self.path, self.eal_para, self.portMask),
            "testpmd>",
            30,
        )

        self.dut.send_expect("set fwd rxonly", "testpmd>", 10)
        self.dut.send_expect("set verbose 1", "testpmd>", 10)
        self.enable_vxlan(self.dut_port)
        self.enable_vxlan(self.recv_port)
        self.pmdout.wait_link_status_up(self.dut_port)
        # check normal packet
        self.send_and_detect(outer_udp_dst=1234)
        # check vxlan + UDP inner packet
        self.send_and_detect(inner_l4_type="UDP")
        # check vxlan + TCP inner packet
        self.send_and_detect(inner_l4_type="TCP")
        # check vxlan + SCTP inner packet
        self.send_and_detect(inner_l4_type="SCTP")
        # check vxlan + vlan inner packet
        self.send_and_detect(outer_vlan=1)
        # check vlan vxlan + vlan inner packet
        self.send_and_detect(outer_vlan=1, inner_vlan=1)

        out = self.dut.send_expect("stop", "testpmd>", 10)
        self.dut.send_expect("quit", "#", 10)

    def test_vxlan_ipv6_detect(self):
        """
        verify vxlan packet detection with ipv6 header
        """
        if self.nic in [
            "ICE_25G-E810C_SFP",
            "ICE_100G-E810C_QSFP",
            "ICE_25G-E823C_QSFP",
        ]:
            print("Intel® Ethernet 700 Series support default none VECTOR")
            src_vec_model = "n"

        self.eal_para = self.dut.create_eal_parameters(
            cores="1S/5C/1T", socket=self.ports_socket
        )

        self.dut.send_expect(
            r"./%s %s -- -i --disable-rss --rxq=4 --txq=4 --nb-cores=4 --portmask=%s"
            % (self.path, self.eal_para, self.portMask),
            "testpmd>",
            30,
        )

        self.dut.send_expect("set fwd rxonly", "testpmd>", 10)
        self.dut.send_expect("set verbose 1", "testpmd>", 10)
        self.enable_vxlan(self.dut_port)
        self.enable_vxlan(self.recv_port)
        self.pmdout.wait_link_status_up(self.dut_port)
        # check normal ipv6 packet
        self.send_and_detect(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            outer_udp_dst=1234,
        )
        # check ipv6 vxlan + UDP inner packet
        self.send_and_detect(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            inner_l4_type="UDP",
        )
        # check ipv6 vxlan + TCP inner packet
        self.send_and_detect(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            inner_l4_type="TCP",
        )
        # check ipv6 vxlan + SCTP inner packet
        self.send_and_detect(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            inner_l4_type="SCTP",
        )

        out = self.dut.send_expect("stop", "testpmd>", 10)
        self.dut.send_expect("quit", "#", 10)

    def test_vxlan_ipv4_checksum_offload(self):
        """
        verify vxlan packet checksum offload
        """
        # start testpmd with 2queue/1port

        self.eal_para = self.dut.create_eal_parameters(
            cores="1S/5C/1T", socket=self.ports_socket
        )

        self.dut.send_expect(
            r"./%s %s -- -i --portmask=%s --enable-rx-cksum"
            % (self.path, self.eal_para, self.portMask),
            "testpmd>",
            30,
        )
        self.iperr_num = 0

        # disable vlan filter
        self.dut.send_expect("vlan set filter off %d" % self.dut_port, "testpmd")
        # enable tx checksum offload
        self.dut.send_expect("set fwd csum", "testpmd>", 10)
        self.dut.send_expect("port stop all", "testpmd>")
        self.csum_set_type("ip", self.recv_port)
        self.csum_set_type("outer-ip", self.recv_port)
        self.csum_set_type("udp", self.recv_port)
        self.csum_set_type("tcp", self.recv_port)
        self.csum_set_type("sctp", self.recv_port)
        self.dut.send_expect("port start all", "testpmd>")
        self.dut.send_expect("csum parse-tunnel on %d" % self.recv_port, "testpmd>", 10)

        self.enable_vxlan(self.dut_port)
        self.enable_vxlan(self.recv_port)
        self.pmdout.wait_link_status_up(self.dut_port)
        # check normal packet + ip checksum invalid
        self.send_and_check(outer_ip_invalid=1, outer_udp_dst=1234)
        # check vxlan packet + inner ip checksum invalid
        self.send_and_check(inner_ip_invalid=1)
        # check vxlan packet + outer ip checksum invalid
        self.send_and_check(outer_ip_invalid=1)
        # check vxlan packet + outer ip + inner ip checksum invalid
        self.send_and_check(outer_ip_invalid=1, inner_ip_invalid=1)
        # check vxlan packet + inner udp checksum invalid
        self.send_and_check(inner_l4_invalid=1)
        # check vxlan packet + inner tcp checksum invalid
        self.send_and_check(inner_l4_invalid=1, inner_l4_type="TCP")
        # check vxlan packet + inner sctp checksum invalid
        self.send_and_check(inner_l4_invalid=1, inner_l4_type="SCTP")
        # check vlan vxlan packet + outer ip checksum invalid
        self.send_and_check(outer_vlan=1, outer_ip_invalid=1)
        # check vlan vxlan packet + inner ip checksum invalid
        self.send_and_check(outer_vlan=1, inner_ip_invalid=1)
        # check vlan vxlan packet + outer&inner ip checksum invalid
        self.send_and_check(outer_vlan=1, outer_ip_invalid=1, inner_ip_invalid=1)
        # check vlan vxlan packet + inner vlan + outer ip checksum invalid
        self.send_and_check(outer_vlan=1, inner_vlan=1, outer_ip_invalid=1)
        # check vlan vxlan packet + inner vlan + inner ip checksum invalid
        self.send_and_check(outer_vlan=1, inner_vlan=1, inner_ip_invalid=1)
        # check vlan vxlan packet + inner vlan + outer&inner ip checksum
        # invalid
        self.send_and_check(
            outer_vlan=1, inner_vlan=1, outer_ip_invalid=1, inner_ip_invalid=1
        )
        # check vlan vxlan packet + inner vlan + inner udp checksum invalid
        self.send_and_check(outer_vlan=1, inner_l4_invalid=1, inner_l4_type="UDP")
        # check vlan vxlan packet + inner vlan + inner tcp checksum invalid
        self.send_and_check(outer_vlan=1, inner_l4_invalid=1, inner_l4_type="TCP")
        # check vlan vxlan packet + inner vlan + inner sctp checksum invalid
        self.send_and_check(outer_vlan=1, inner_l4_invalid=1, inner_l4_type="SCTP")

        self.dut.send_expect("quit", "#", 10)

    def test_vxlan_ipv6_checksum_offload(self):
        """
        verify vxlan packet checksum offload with ipv6 header
        not support ipv6 + sctp
        """
        # start testpmd with 2queue/1port

        self.eal_para = self.dut.create_eal_parameters(
            cores="1S/5C/1T", socket=self.ports_socket
        )

        self.dut.send_expect(
            r"./%s %s -- -i --portmask=%s --enable-rx-cksum"
            % (self.path, self.eal_para, self.portMask),
            "testpmd>",
            30,
        )
        self.iperr_num = 0

        # disable vlan filter
        self.dut.send_expect("vlan set filter off %d" % self.dut_port, "testpmd")
        # enable tx checksum offload
        self.dut.send_expect("set fwd csum", "testpmd>", 10)
        self.csum_set_type("outer-ip", self.recv_port)
        self.csum_set_type("udp", self.recv_port)
        self.csum_set_type("outer-udp", self.recv_port)
        self.csum_set_type("tcp", self.recv_port)
        self.csum_set_type("sctp", self.recv_port)
        self.dut.send_expect("csum parse-tunnel on %d" % self.recv_port, "testpmd>", 10)

        self.enable_vxlan(self.dut_port)
        self.enable_vxlan(self.recv_port)
        # check normal ipv6 packet
        self.send_and_check(
            outer_ip6_src="FE80:0:0:0:0:0:0:0", outer_ip6_dst="FE80:0:0:0:0:0:0:1"
        )
        # check normal ipv6 packet + ip checksum invalid
        self.send_and_check(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            outer_udp_dst=1234,
        )
        # check ipv6 vxlan packet + inner ip checksum invalid
        self.send_and_check(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            inner_ip_invalid=1,
        )
        # check ipv6 vxlan packet + inner udp checksum invalid
        self.send_and_check(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            inner_l4_invalid=1,
            inner_l4_type="UDP",
        )
        # check ipv6 vxlan packet + inner udp checksum invalid
        self.send_and_check(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            inner_l4_invalid=1,
            inner_l4_type="UDP",
        )
        # check ipv6 vxlan packet + inner tcp checksum invalid
        self.send_and_check(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            inner_l4_invalid=1,
            inner_l4_type="TCP",
        )
        # check ipv6 vlan vxlan packet + inner udp checksum invalid
        self.send_and_check(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            inner_l4_invalid=1,
            inner_l4_type="UDP",
            outer_vlan=1,
        )
        # check ipv6 vlan vxlan packet + inner tcp checksum invalid
        self.send_and_check(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            inner_l4_invalid=1,
            inner_l4_type="TCP",
            outer_vlan=1,
        )
        # check ipv6 vlan vxlan packet + vlan + inner udp checksum invalid
        self.send_and_check(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            inner_l4_invalid=1,
            inner_l4_type="UDP",
            outer_vlan=1,
            inner_vlan=1,
        )
        # check ipv6 vlan vxlan packet + vlan + inner tcp checksum invalid
        self.send_and_check(
            outer_ip6_src="FE80:0:0:0:0:0:0:0",
            outer_ip6_dst="FE80:0:0:0:0:0:0:1",
            inner_l4_invalid=1,
            inner_l4_type="TCP",
            outer_vlan=1,
            inner_vlan=1,
        )

        self.dut.send_expect("quit", "#", 10)

    def test_tunnel_filter(self):
        """
        verify tunnel filter feature
        """
        self.eal_para = self.dut.create_eal_parameters(
            cores="1S/5C/1T", socket=self.ports_socket
        )

        self.dut.send_expect(
            r"./%s %s -- -i --disable-rss --rxq=%d --txq=%d --nb-cores=4 --portmask=%s"
            % (self.path, self.eal_para, MAX_TXQ_RXQ, MAX_TXQ_RXQ, self.portMask),
            "testpmd>",
            30,
        )

        self.dut.send_expect("set fwd rxonly", "testpmd>", 10)
        self.dut.send_expect("set verbose 1", "testpmd>", 10)
        self.enable_vxlan(self.dut_port)
        self.enable_vxlan(self.recv_port)
        self.pmdout.wait_link_status_up(self.dut_port)
        config = VxlanTestConfig(self)
        config_vlan = VxlanTestConfig(self, inner_vlan=1)
        config.outer_mac_dst = self.dut_port_mac
        config_vlan.outer_mac_dst = self.dut_port_mac
        expect_queue = randint(1, MAX_TXQ_RXQ - 1)

        rule_list = [
            # check inner mac + inner vlan filter can work
            "flow create {} ingress pattern eth / ipv4 / udp / vxlan / eth dst is {} / vlan tci is {} / end actions pf "
            "/ queue index {} / end".format(
                self.dut_port,
                config_vlan.inner_mac_dst,
                config_vlan.inner_vlan,
                expect_queue,
            ),
            # check inner mac + inner vlan + tunnel id filter can work
            "flow create {} ingress pattern eth / ipv4 / udp / vxlan vni is {} / eth dst is {} "
            "/ vlan tci is {} / end actions pf / queue index {} / end".format(
                self.dut_port,
                config_vlan.vni,
                config_vlan.inner_mac_dst,
                config_vlan.inner_vlan,
                expect_queue,
            ),
            # check inner mac + tunnel id filter can work
            "flow create {} ingress pattern eth / ipv4 / udp / vxlan vni is {} / eth dst is {} / end actions pf "
            "/ queue index {} / end".format(
                self.dut_port, config.vni, config.inner_mac_dst, expect_queue
            ),
            # check inner mac filter can work
            "flow create {} ingress pattern eth / ipv4 / udp / vxlan / eth dst is {} / end actions pf / queue index {} "
            "/ end".format(self.dut_port, config.inner_mac_dst, expect_queue),
            # check outer mac + inner mac + tunnel id filter can work
            "flow create {} ingress pattern eth dst is {} / ipv4 / udp / vxlan vni is {} / eth dst is {} "
            "/ end actions pf / queue index {} / end".format(
                self.dut_port,
                config.outer_mac_dst,
                config.vni,
                config.inner_mac_dst,
                expect_queue,
            )
            # iip not supported by now
            # 'flow create {} ingress pattern eth / ipv4 / udp / vxlan / eth / ipv4 dst is {} / end actions pf '
            # '/ queue index {} / end'.format(self.dut_port,
            #                                 config.inner_ip_dst,
            #                                 queue)
        ]

        for rule in rule_list:
            if "vlan tci is" in rule:
                self.filter_and_check(rule, config_vlan, expect_queue)
            else:
                self.filter_and_check(rule, config, expect_queue)

        self.dut.send_expect("quit", "#", 10)

    def test_tunnel_filter_invalid(self):
        """
        verify tunnel filter parameter check function
        """
        # invalid parameter
        queue_id = 3

        config = VxlanTestConfig(self)
        config.outer_mac_dst = self.dut_port_mac

        self.eal_para = self.dut.create_eal_parameters(
            cores="1S/5C/1T", socket=self.ports_socket
        )

        self.dut.send_expect(
            r"./%s %s -- -i --disable-rss --rxq=4 --txq=4 --nb-cores=4 --portmask=%s"
            % (self.path, self.eal_para, self.portMask),
            "testpmd>",
            30,
        )

        self.enable_vxlan(self.dut_port)
        self.enable_vxlan(self.recv_port)
        self.pmdout.wait_link_status_up(self.dut_port)
        rule = (
            "flow create {} ingress pattern eth / ipv4 / udp / vxlan vni is {} / eth dst is {} / end actions pf "
            "/ queue index {} / end".format(
                self.dut_port, config.vni, self.invalid_mac, queue_id
            )
        )
        out = self.tunnel_filter_add_nocheck(rule)
        self.verify("Bad arguments" in out, "Failed to detect invalid mac")

        rule = (
            "flow create {} ingress pattern eth / ipv4 / udp / vxlan vni is {} / eth / ipv4 dst is {} "
            "/ end actions pf / queue index {} / end".format(
                self.dut_port, config.vni, self.invalid_ip, queue_id
            )
        )
        out = self.tunnel_filter_add_nocheck(rule)
        self.verify("Bad arguments" in out, "Failed to detect invalid ip")

        # testpmd is not support
        # rule = 'flow create {} ingress pattern eth / ipv4 / udp / vxlan vni is {} / eth dst is {} / vlan vid is {} ' \
        #        '/ end actions pf / queue index {} / end'.format(self.dut_port,
        #                                                         config.vni,
        #                                                         config.inner_mac_dst,
        #                                                         self.invalid_vlan,
        #                                                         queue_id)
        # out = self.tunnel_filter_add_nocheck(rule)
        # self.verify("Invalid argument" in out, "Failed to detect invalid vlan")

        rule = (
            "flow create {} ingress pattern eth / ipv4 / udp / vxlan vni is {} / eth dst is {} / end actions pf "
            "/ queue index {} / end".format(
                self.dut_port, config.vni, config.inner_mac_dst, self.invalid_queue
            )
        )
        out = self.tunnel_filter_add_nocheck(rule)
        self.verify("Invalid queue ID" in out, "Failed to detect invalid queue")

        self.dut.send_expect("stop", "testpmd>", 10)
        self.dut.send_expect("quit", "#", 10)

    def config_tunnelfilter(self, dut_port, recv_port, perf_config, pcapfile):
        pkts = []
        config = VxlanTestConfig(self, payload_size=self.vxlan_payload - 4)
        config.inner_vlan = self.default_vlan
        config.outer_mac_dst = self.dut.get_mac_address(dut_port)
        config.pcap_file = pcapfile

        tun_filter = perf_config["tunnel_filter"]
        recv_queue = perf_config["recvqueue"]
        # there's known bug that if enable vxlan, rss will be disabled
        if tun_filter == "None" and recv_queue == "Multi":
            print((utils.RED("RSS and Tunel filter can't enable in the same time")))
        else:
            self.enable_vxlan(dut_port)

        if tun_filter != "None":
            rule_config = {
                "dut_port": dut_port,
                "outer_mac_dst": config.outer_mac_dst,
                "inner_mac_dst": config.inner_mac_dst,
                "inner_ip_dst": config.inner_ip_dst,
                "inner_vlan": config.inner_vlan,
                "tun_filter": tun_filter,
                "vni": config.vni,
                "queue": 0,
            }
            self.perf_tunnel_filter_set_rule(rule_config)

        if perf_config["Packet"] == "Normal":
            config.outer_udp_dst = 63
            config.outer_mac_dst = self.dut.get_mac_address(dut_port)
            config.payload_size = (
                PACKET_LEN - HEADER_SIZE["eth"] - HEADER_SIZE["ip"] - HEADER_SIZE["udp"]
            )

        # add default pkt into pkt list
        pkt = config.create_pcap()
        pkts.append(pkt)

        # add other pkts into pkt list when enable multi receive queues
        if recv_queue == "Multi" and tun_filter != "None":
            for queue in range(self.tunnel_multiqueue - 1):
                if "imac" in tun_filter:
                    config.inner_mac_dst = "00:00:20:00:00:0%d" % (queue + 2)
                if "ivlan" in tun_filter:
                    config.inner_vlan = queue + 2
                if "tenid" in tun_filter:
                    config.vni = queue + 2

                # add tunnel filter the same as pkt
                pkt = config.create_pcap()
                pkts.append(pkt)

                rule_config = {
                    "dut_port": dut_port,
                    "outer_mac_dst": config.outer_mac_dst,
                    "inner_mac_dst": config.inner_mac_dst,
                    "inner_ip_dst": config.inner_ip_dst,
                    "inner_vlan": config.inner_vlan,
                    "tun_filter": tun_filter,
                    "vni": config.vni,
                    "queue": (queue + 1),
                }
                self.perf_tunnel_filter_set_rule(rule_config)

        # save pkt list into pcap file
        wrpcap(config.pcap_file, pkts)
        self.tester.session.copy_file_to(config.pcap_file)

    def combine_pcap(self, dest_pcap, src_pcap):
        pkts = rdpcap(dest_pcap)
        if len(pkts) != 1:
            return

        pkts_src = rdpcap(src_pcap)
        pkts += pkts_src

        wrpcap(dest_pcap, pkts)

    def test_perf_vxlan_tunnelfilter_performance_2ports(self):
        self.result_table_create(self.tunnel_header)
        core_list = self.dut.get_core_list(
            "1S/%dC/1T" % (self.tunnel_multiqueue * 2 + 1), socket=self.ports_socket
        )

        pmd_temp = (
            "./%s %s -- -i --disable-rss --rxq=2 --txq=2 --nb-cores=4 --portmask=%s"
        )

        for perf_config in self.tunnel_perf:
            tun_filter = perf_config["tunnel_filter"]
            recv_queue = perf_config["recvqueue"]
            print(
                (
                    utils.GREEN(
                        "Measure tunnel performance of [%s %s %s]"
                        % (perf_config["Packet"], tun_filter, recv_queue)
                    )
                )
            )

            if tun_filter == "None" and recv_queue == "Multi":
                pmd_temp = (
                    "./%s %s -- -i --rss-udp --rxq=2 --txq=2 --nb-cores=4 --portmask=%s"
                )

            self.eal_para = self.dut.create_eal_parameters(cores=core_list)
            pmd_cmd = pmd_temp % (self.path, self.eal_para, self.portMask)
            self.dut.send_expect(pmd_cmd, "testpmd> ", 100)

            # config flow
            self.config_tunnelfilter(
                self.dut_port, self.recv_port, perf_config, "flow1.pcap"
            )
            # config the flows
            tgen_input = []
            tgen_input.append(
                (
                    self.tester.get_local_port(self.dut_port),
                    self.tester.get_local_port(self.recv_port),
                    "flow1.pcap",
                )
            )

            if BIDIRECT:
                self.config_tunnelfilter(
                    self.recv_port, self.dut_port, perf_config, "flow2.pcap"
                )
                tgen_input.append(
                    (
                        self.tester.get_local_port(self.recv_port),
                        self.tester.get_local_port(self.dut_port),
                        "flow2.pcap",
                    )
                )

            self.dut.send_expect("set fwd io", "testpmd>", 10)
            self.dut.send_expect("start", "testpmd>", 10)
            self.pmdout.wait_link_status_up(self.dut_port)
            if BIDIRECT:
                wirespeed = self.wirespeed(self.nic, PACKET_LEN, 2)
            else:
                wirespeed = self.wirespeed(self.nic, PACKET_LEN, 1)

            # run traffic generator
            use_vm = True if recv_queue == "Multi" and tun_filter == "None" else False
            _, pps = self.suite_measure_throughput(tgen_input, use_vm=use_vm)

            pps /= 1000000.0
            perf_config["Mpps"] = pps
            perf_config["pct"] = pps * 100 / wirespeed

            out = self.dut.send_expect("stop", "testpmd>", 10)
            self.dut.send_expect("quit", "# ", 10)

            # verify every queue work fine
            check_queue = 0
            if recv_queue == "Multi":
                for queue in range(check_queue):
                    self.verify(
                        "Queue= %d -> TX Port" % (queue) in out,
                        "Queue %d no traffic" % queue,
                    )

            table_row = [
                perf_config["Packet"],
                tun_filter,
                recv_queue,
                perf_config["Mpps"],
                perf_config["pct"],
            ]

            self.result_table_add(table_row)

        self.result_table_print()

    def test_perf_vxlan_checksum_performance_2ports(self):
        self.result_table_create(self.chksum_header)
        vxlan = VxlanTestConfig(self, payload_size=self.vxlan_payload)
        vxlan.outer_mac_dst = self.dut.get_mac_address(self.dut_port)
        vxlan.pcap_file = "vxlan1.pcap"
        vxlan.inner_mac_dst = "00:00:20:00:00:01"
        vxlan.create_pcap()

        vxlan_queue = VxlanTestConfig(self, payload_size=self.vxlan_payload)
        vxlan_queue.outer_mac_dst = self.dut.get_mac_address(self.dut_port)
        vxlan_queue.pcap_file = "vxlan1_1.pcap"
        vxlan_queue.inner_mac_dst = "00:00:20:00:00:02"
        vxlan_queue.create_pcap()

        # socket/core/thread
        core_list = self.dut.get_core_list(
            "1S/%dC/1T" % (self.tunnel_multiqueue * 2 + 1), socket=self.ports_socket
        )
        core_mask = utils.create_mask(core_list)

        self.dut_ports = self.dut.get_ports_performance(force_different_nic=False)
        tx_port = self.tester.get_local_port(self.dut_ports[0])
        rx_port = self.tester.get_local_port(self.dut_ports[1])

        for cal in self.cal_type:
            recv_queue = cal["recvqueue"]
            print(
                (
                    utils.GREEN(
                        "Measure checksum performance of [%s %s %s]"
                        % (cal["Type"], recv_queue, cal["csum"])
                    )
                )
            )

            # configure flows
            tgen_input = []
            if recv_queue == "Multi":
                tgen_input.append((tx_port, rx_port, "vxlan1.pcap"))
                tgen_input.append((tx_port, rx_port, "vxlan1_1.pcap"))
            else:
                tgen_input.append((tx_port, rx_port, "vxlan1.pcap"))

            # multi queue and signle queue commands
            if recv_queue == "Multi":
                pmd_temp = "./%s %s -- -i --disable-rss --rxq=2 --txq=2 --nb-cores=4 --portmask=%s"
            else:
                pmd_temp = "./%s %s -- -i --nb-cores=2 --portmask=%s"

            self.eal_para = self.dut.create_eal_parameters(cores=core_list)
            pmd_cmd = pmd_temp % (self.path, self.eal_para, self.portMask)

            self.dut.send_expect(pmd_cmd, "testpmd> ", 100)
            self.dut.send_expect("set fwd csum", "testpmd>", 10)
            self.enable_vxlan(self.dut_port)
            self.enable_vxlan(self.recv_port)
            self.pmdout.wait_link_status_up(self.dut_port)

            # redirect flow to another queue by tunnel filter
            rule_config = {
                "dut_port": self.dut_port,
                "outer_mac_dst": vxlan.outer_mac_dst,
                "inner_mac_dst": vxlan.inner_mac_dst,
                "inner_ip_dst": vxlan.inner_ip_dst,
                "inner_vlan": 0,
                "tun_filter": "imac",
                "vni": vxlan.vni,
                "queue": 0,
            }
            self.perf_tunnel_filter_set_rule(rule_config)

            if recv_queue == "Multi":
                rule_config = {
                    "dut_port": self.dut_port,
                    "outer_mac_dst": vxlan_queue.outer_mac_dst,
                    "inner_mac_dst": vxlan_queue.inner_mac_dst,
                    "inner_ip_dst": vxlan_queue.inner_ip_dst,
                    "inner_vlan": 0,
                    "tun_filter": "imac",
                    "vni": vxlan.vni,
                    "queue": 1,
                }
                self.perf_tunnel_filter_set_rule(rule_config)

            for pro in cal["csum"]:
                self.csum_set_type(pro, self.dut_port)
                self.csum_set_type(pro, self.recv_port)

            self.dut.send_expect("start", "testpmd>", 10)

            wirespeed = self.wirespeed(self.nic, PACKET_LEN, 1)

            # run traffic generator
            _, pps = self.suite_measure_throughput(tgen_input)

            pps /= 1000000.0
            cal["Mpps"] = pps
            cal["pct"] = pps * 100 / wirespeed

            out = self.dut.send_expect("stop", "testpmd>", 10)
            self.dut.send_expect("quit", "# ", 10)

            # verify every queue work fine
            check_queue = 1
            if recv_queue == "Multi":
                for queue in range(check_queue):
                    self.verify(
                        "Queue= %d -> TX Port" % (queue) in out,
                        "Queue %d no traffic" % queue,
                    )

            table_row = [cal["Type"], recv_queue, cal["Mpps"], cal["pct"]]
            self.result_table_add(table_row)

        self.result_table_print()

    def enable_vxlan(self, port):
        self.dut.send_expect(
            "rx_vxlan_port add %d %d" % (VXLAN_PORT, port), "testpmd>", 10
        )

    def csum_set_type(self, proto, port):
        self.dut.send_expect("port stop all", "testpmd>")
        out = self.dut.send_expect("csum set %s hw %d" % (proto, port), "testpmd>", 10)
        self.dut.send_expect("port start all", "testpmd>")
        self.verify("Bad arguments" not in out, "Failed to set vxlan csum")
        self.verify("error" not in out, "Failed to set vxlan csum")

    def csum_set_sw(self, proto, port):
        self.dut.send_expect("port stop all", "testpmd>")
        out = self.dut.send_expect("csum set %s sw %d" % (proto, port), "testpmd>", 10)
        self.dut.send_expect("port start all", "testpmd>")
        self.verify("Bad arguments" not in out, "Failed to set vxlan csum")
        self.verify("error" not in out, "Failed to set vxlan csum")

    def tunnel_filter_add(self, rule):
        out = self.dut.send_expect(rule, "testpmd>", 3)
        self.verify("Flow rule #0 created" in out, "Flow rule create failed")
        return out

    def tunnel_filter_add_nocheck(self, rule):
        out = self.dut.send_expect(rule, "testpmd>", 3)
        return out

    def tunnel_filter_del(self, *args):
        out = self.dut.send_expect("flow flush 0", "testpmd>", 10)
        return out

    def set_up(self):
        """
        Run before each test case.
        """
        pass

    def tear_down(self):
        """
        Run after each test case.
        """
        self.dut.kill_all()

    def tear_down_all(self):
        """
        Run after each test suite.
        """
        pass


import os
import pickle
import numpy as np

with open("results/unity_to_trinity_UDA_semi_10/evaluation_trinity_test/confidence_map_UDA.pkl", "rb") as f:
    data = pickle.load(f)

filename = data[0]
score = np.array(data[1])

sorted_index = np.argsort(-score)
images_to_label = []

for i in range(200):
    images_to_label.append(
        os.path.join("/home/yirus/Datasets/Active_Learning/trinity/train/images/",
                     filename[sorted_index[i]])
    )
    print(score[sorted_index[i]])

with open("dataloaders/eye/trinity_top_200.pkl", "wb") as f:
    pickle.dump(images_to_label, f)


import matplotlib.pyplot as plt
plt.style.use('classic') # 其它style可以试试seaborn, bmh, ggplot, grayscale
import numpy as np

fig = plt.figure()  # 创建图形
ax = plt.axes()  # 创建维度

x = np.linspace(0, 10, 1000)  # 在0到10之间生成1000个等间隔的数

# 已有x, 画出sin(x)的数据, 颜色设置为红,加入label用于制作图例
ax.plot(x, np.sin(x), color='red', label='sin(x)')

# 在同一幅图形中绘制多根线条，只需要多次调用plot函数即可
# 颜色设置为16进制金色，16进制需要前面打井号
# linestyle 可以是 -  --  -.  :
ax.plot(x, np.cos(x), color='#ffd700', linestyle='--', label='cos(x)')

plt.xlim(0, 11)  # 调整x轴的范围
plt.ylim(-1.5, 1.5)  # 调整y轴的范围
# 同样功能的函数：plt.axis([xmin, xmax, ymin, ymax])
# 如： plt.axis([-10, 10, -2, 2])
# plt.axis('tight')  # 可以将坐标轴压缩到刚好足够绘制折线图像的大小
# plt.axis('equal')  # 设置x轴与y轴使用相同的长度单位

plt.title("A Sine Curve")  # 设置标题
plt.xlabel("x")  # 设置x轴标签
plt.ylabel("sin(x)")  # 设置y轴标签
plt.legend()  # 召唤标签
plt.show()  # 显示图像


def longest_common_subsequence(str1: str, str2: str) -> str:
    """
    :param str1: represented as columns in opt
    :param str2: represented as rows in opt
    :return: the longest common subsequence
    """
    if '' in [str1, str2]:
        return ''

    # create opt table
    opt = [[0] * len(str1) for _i in range(len(str2))]
    for i in range(len(str2)):
        for j in range(len(str1)):
            if str2[i] == str1[j]:
                opt[i][j] = 1 if 0 in [i, j] else opt[i - 1][j-1] + 1
            else:
                opt[i][j] = 0 if 0 in [i, j] else max(opt[i-1][j], opt[i][j-1])

    # find path from opt table
    sequence = ''
    offset = 0
    for i in range(len(str2)-1, -1, -1):
        for j in range(len(str1)-1-offset, -1, -1):
            offset += 1
            if 0 in [i, j] or opt[i][j] != opt[i][j-1]:
                sequence = str2[i] + sequence
                break
    return sequence

print(longest_common_subsequence('ABCD', 'A'))


"""
Given n non-negative integers representing the histogram’s bar height where the width of each bar is 1,
find the area of largest rectangle in the histogram.

Largest Rectangle in Histogram: Example 1

Above is a histogram where width of each bar is 1, given height = [2,1,5,6,2,3].

Largest Rectangle in Histogram: Example 2

The largest rectangle is shown in the shaded area, which has area = 10 unit.

For example,
Given height = [2,1,5,6,2,3],
return 10.
"""

class Solution:

    def largestArea(self, arr):
        n = len(arr)
        i = 0
        curr = 0
        largest_area = 0
        stack = []
        while i < n:

            if not stack or arr[stack[-1]] <= arr[i]:
                stack.append(i)
                i+= 1
            else:
                top = stack[-1]
                stack.pop()

                if not stack:
                    curr = arr[top]*i
                else:
                    curr = arr[top]*(i-stack[-1]-1)
                if curr > largest_area:
                    largest_area = curr
        while stack:
            top = stack[-1]
            stack.pop()
            if not stack:
                curr = arr[top]*i
            else:
                curr = arr[top]*(i-stack[-1]-1)
            if curr > largest_area:
                largest_area = curr
        return largest_area

s= Solution()
a = [6,2,5,3,12,7,8,6,9]
print(s.largestArea(a))

Python/StacksAndQueues/CleverStack/Largest Rectangle in Histogram.py


import logging
import multiprocessing
import queue
import time

from gi.repository import GLib

from meld.matchers import myers

log = logging.getLogger(__name__)

class MatcherWorker(multiprocessing.Process):

    END_TASK = -1

    matcher_class = myers.InlineMyersSequenceMatcher

    def __init__(self, tasks, results):
        super().__init__()
        self.tasks = tasks
        self.results = results
        self.daemon = True

    def run(self):
        while True:
            task_id, (text1, textn) = self.tasks.get()
            if task_id == self.END_TASK:
                break

            try:
                matcher = self.matcher_class(None, text1, textn)
                self.results.put((task_id, matcher.get_opcodes()))
            except Exception as e:
                log.error("Exception while running diff: %s", e)
            time.sleep(0)

class CachedSequenceMatcher:
    """Simple class for caching diff results, with LRU-based eviction

    Results from the SequenceMatcher are cached and timestamped, and
    subsequently evicted based on least-recent generation/usage. The LRU-based
    eviction is overly simplistic, but is okay for our usage pattern.
    """

    TASK_GRACE_PERIOD = 1

    def __init__(self, scheduler):
        """Create a new caching sequence matcher

        :param scheduler: a `meld.task.SchedulerBase` used to schedule
            sequence comparison result checks
        """
        self.scheduler = scheduler
        self.cache = {}
        self.tasks = multiprocessing.Queue()
        self.tasks.cancel_join_thread()
        # Limiting the result queue here has the effect of giving us
        # much better interactivity. Without this limit, the
        # result-checker tends to get starved and all highlights get
        # delayed until we're almost completely finished.
        self.results = multiprocessing.Queue(5)
        self.results.cancel_join_thread()
        self.thread = MatcherWorker(self.tasks, self.results)
        self.task_id = 1
        self.queued_matches = {}
        GLib.idle_add(self.thread.start)

    def stop(self) -> None:
        self.tasks.put((MatcherWorker.END_TASK, ('', '')))
        if self.thread.is_alive():
            self.thread.join(self.TASK_GRACE_PERIOD)
            if self.thread.exitcode is None:
                self.thread.terminate()
        self.cache = {}
        self.queued_matches = {}

    def match(self, text1, textn, cb):
        texts = (text1, textn)
        try:
            self.cache[texts][1] = time.time()
            opcodes = self.cache[texts][0]
            GLib.idle_add(lambda: cb(opcodes))
        except KeyError:
            GLib.idle_add(lambda: self.enqueue_task(texts, cb))

    def enqueue_task(self, texts, cb):
        if not bool(self.queued_matches):
            self.scheduler.add_task(self.check_results)
        self.queued_matches[self.task_id] = (texts, cb)
        self.tasks.put((self.task_id, texts))
        self.task_id += 1

    def check_results(self):
        try:
            task_id, opcodes = self.results.get(block=True, timeout=0.01)
            texts, cb = self.queued_matches.pop(task_id)
            self.cache[texts] = [opcodes, time.time()]
            GLib.idle_add(lambda: cb(opcodes))
        except queue.Empty:
            pass

        return bool(self.queued_matches)

    def clean(self, size_hint):
        """Clean the cache if necessary

        @param size_hint: the recommended minimum number of cache entries
        """
        if len(self.cache) <= size_hint * 3:
            return
        items = list(self.cache.items())
        items.sort(key=lambda it: it[1][1])
        for item in items[:-size_hint * 2]:
            del self.cache[item[0]]


import os

dirIcpc = r'/Users/pino/Documents/The-Empire-Strikes-Back/ICPC/Codes/'
cppJson = "/Users/pino/Documents/Competitive Programming/.vscode/snippets.code-snippets"

def solve(file, dir, out):
  # print(file, dir)
  cppFile = open(dir, "r")

  name = file[:-4]
  # print(name)

  out.write("  \"{}\": {{\n".format(name))
  out.write("    \"prefix\": \"{}\",\n".format(name))

  def getBody(cppFile):
    codeByLines = list()
    for line in cppFile:
      printableLine = str("")

      for c in line:
        if c == '\\' or c == '\$' or c == '\"':
          printableLine += '\\' # special dummy characters
        printableLine += c

      if line.endswith("\n"):
        printableLine = printableLine[:-1]
      codeByLines.append(printableLine)

    return codeByLines

  out.write("    \"body\": [\n")
  for line in getBody(cppFile):
    out.write("    \"{}\",\n".format(line))
  out.write("    ],\n")

  out.write("    \"description:\" \"{}\"\n".format(file))
  out.write("  },\n")

def main():
  print("Snippets up to date.")
  rootdir = dirIcpc

  out = open(cppJson, "w")
  out.write("{\n")

  for subdir, dirs, files in os.walk(rootdir):
    if subdir.endswith("GeometryIDK") or subdir.endswith("GeometryIDK/InsideStruct"):
      continue
    for file in files:
      #print os.path.join(subdir, file)
      filepath = subdir + os.sep + file

      if filepath.endswith(".cpp"):
        solve(file, filepath, out)

  out.write("}\n")
  out.close()

if __name__ == "__main__":
    main()


from math import *
from main import calcul_rayon_Gauss as Rg
import Conversion
from PyQt5 import QtWidgets
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from math import *
import Conversion
import main
import mpmath as mp

class Ui_Form11(object):
    def function(self):
        if self.comboBox_2.currentText() == "WGS 84":
            a=6378137
            b=6356752.314
            E = 4 * ((b * b) *pi* 1.00449)
        elif self.comboBox_2.currentText() == "Clarck 1880":
            a=6378249.20
            b=6356515.0
            E = 4 * ((b * b) *pi* 1.00456)

        if self.comboBox_3.currentText() == "calcul_rayon_v":
            R=main.calcul_rayon_v(a,b)

        elif self.comboBox_3.currentText() == "calcul_rayon_Gauss":
            R=main.calcul_rayon_Gauss(a,b)

        elif self.comboBox_3.currentText() == "calcul_rayon_a":
            R=main.calcul_rayon_a(E)

        elif self.comboBox_3.currentText() == "calcul_rayon_moyen":
            R=main.calcul_rayon_moyen(a,b)

        phi1d = float(self.lineEdit.text())
        phi1m = float(self.lineEdit_2.text())
        phi1s = float(self.lineEdit_3.text())
        phi2d = float(self.lineEdit_14.text())
        phi2m = float(self.lineEdit_7.text())
        phi2s = float(self.lineEdit_15.text())

        lambda1d = float(self.lineEdit_4.text())
        lambda1m = float(self.lineEdit_5.text())
        lambda1s = float(self.lineEdit_6.text())
        lambda2d = float(self.lineEdit_23.text())
        lambda2m = float(self.lineEdit_22.text())
        lambda2s = float(self.lineEdit_24.text())
        phi1=Conversion.dmstorad(phi1d,phi1m,phi1s)
        phi2 = Conversion.dmstorad(phi2d, phi2m, phi2s)
        lambda1= Conversion.dmstorad(lambda1d,lambda1m,lambda1s)
        lambda2 = Conversion.dmstorad(lambda2d, lambda2m, lambda2s)
        delta_lambda=lambda1-lambda2
        #DETERMINATION DE SIGMA12
        #sigma12=acos(sin(phi1)*sin(phi2)+cos(phi1)*cos(phi2)*cos(delta_lambda))*(180/pi)
        sigma12 = acos((sin(phi1) * sin(phi2) + cos(phi1) * cos(phi2) * cos(delta_lambda)))
        dis_ang_metre = sigma12 *R
        #print(dis_ang_metre)
        #DETRMINATION DE A12
        cot1= (tan(phi2)*cos(phi1)/sin(delta_lambda)) - (sin(phi1)*mp.cot(delta_lambda))
        A12=mp.acot(cot1)
        (A12_d, A12_m, A12_s) = Conversion.radtodms(A12)
        #DETERMINATION D'AZIMUT DE RETOUR
        A21=(mp.acot(-((tan(phi1)*cos(phi2)/sin(delta_lambda))-sin(phi2)*mp.cot(delta_lambda))))
        (A21_d, A21_m, A21_s) = Conversion.radtodms(A21)

        self.lineEdit_20.setText(str(A12_d))
        self.lineEdit_19.setText(str(A12_m))
        self.lineEdit_21.setText(str(A12_s))

        self.lineEdit_17.setText(str(A21_d))
        self.lineEdit_18.setText(str(A21_m))
        self.lineEdit_16.setText(str(A21_s))

        self.lineEdit_25.setText(str(dis_ang_metre))
    def setupUi(self, Form):
        if not Form.objectName():
            Form.setObjectName(u"Form")
        Form.resize(840, 377)
        Form.setStyleSheet(u"background-color: rgb(171, 180, 255);")
        self.label_2 = QLabel(Form)
        self.label_2.setObjectName(u"label_2")
        self.label_2.setGeometry(QRect(8, 134, 91, 21))
        font = QFont()
        font.setPointSize(9)
        font.setBold(False)
        self.label_2.setFont(font)
        self.label_3 = QLabel(Form)
        self.label_3.setObjectName(u"label_3")
        self.label_3.setGeometry(QRect(176, 135, 16, 16))
        font1 = QFont()
        font1.setPointSize(10)
        font1.setBold(True)
        self.label_3.setFont(font1)
        self.label_4 = QLabel(Form)
        self.label_4.setObjectName(u"label_4")
        self.label_4.setGeometry(QRect(251, 134, 16, 16))
        self.label_4.setFont(font1)
        self.label_5 = QLabel(Form)
        self.label_5.setObjectName(u"label_5")
        self.label_5.setGeometry(QRect(330, 134, 16, 16))
        self.label_5.setFont(font1)
        self.pushButton = QPushButton(Form)
        self.pushButton.setObjectName(u"pushButton")
        self.pushButton.setGeometry(QRect(370, 310, 101, 41))
        font2 = QFont()
        font2.setPointSize(9)
        font2.setBold(True)
        self.pushButton.setFont(font2)
        self.pushButton.setStyleSheet(u"\n"
"background-color: rgb(255, 255, 255);")
        self.label_17 = QLabel(Form)
        self.label_17.setObjectName(u"label_17")
        self.label_17.setGeometry(QRect(80, 20, 271, 51))
        font3 = QFont()
        font3.setPointSize(11)
        font3.setBold(True)
        self.label_17.setFont(font3)
        self.label_17.setStyleSheet(u"color: rgb(170, 0, 0);\n"
"color: rgb(0, 0, 127);")
        self.label_24 = QLabel(Form)
        self.label_24.setObjectName(u"label_24")
        self.label_24.setGeometry(QRect(675, 133, 16, 16))
        self.label_11 = QLabel(Form)
        self.label_11.setObjectName(u"label_11")
        self.label_11.setGeometry(QRect(177, 176, 16, 16))
        self.label_11.setFont(font1)
        self.label_13 = QLabel(Form)
        self.label_13.setObjectName(u"label_13")
        self.label_13.setGeometry(QRect(332, 175, 16, 16))
        self.label_13.setFont(font1)
        self.label_12 = QLabel(Form)
        self.label_12.setObjectName(u"label_12")
        self.label_12.setGeometry(QRect(253, 176, 16, 16))
        self.label_12.setFont(font1)
        self.lineEdit_6 = QLineEdit(Form)
        self.lineEdit_6.setObjectName(u"lineEdit_6")
        self.lineEdit_6.setGeometry(QRect(269, 175, 57, 19))
        self.lineEdit_6.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.lineEdit_4 = QLineEdit(Form)
        self.lineEdit_4.setObjectName(u"lineEdit_4")
        self.lineEdit_4.setGeometry(QRect(114, 176, 57, 19))
        self.lineEdit_4.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.lineEdit_2 = QLineEdit(Form)
        self.lineEdit_2.setObjectName(u"lineEdit_2")
        self.lineEdit_2.setGeometry(QRect(193, 134, 56, 19))
        self.lineEdit_2.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.label_10 = QLabel(Form)
        self.label_10.setObjectName(u"label_10")
        self.label_10.setGeometry(QRect(8, 179, 101, 16))
        self.label_10.setFont(font)
        self.lineEdit_3 = QLineEdit(Form)
        self.lineEdit_3.setObjectName(u"lineEdit_3")
        self.lineEdit_3.setGeometry(QRect(269, 134, 57, 19))
        self.lineEdit_3.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.lineEdit = QLineEdit(Form)
        self.lineEdit.setObjectName(u"lineEdit")
        self.lineEdit.setGeometry(QRect(114, 133, 57, 20))
        self.lineEdit.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.lineEdit_5 = QLineEdit(Form)
        self.lineEdit_5.setObjectName(u"lineEdit_5")
        self.lineEdit_5.setGeometry(QRect(193, 176, 56, 19))
        self.lineEdit_5.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.lineEdit_16 = QLineEdit(Form)
        self.lineEdit_16.setObjectName(u"lineEdit_16")
        self.lineEdit_16.setGeometry(QRect(720, 228, 91, 20))
        self.lineEdit_16.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.label_16 = QLabel(Form)
        self.label_16.setObjectName(u"label_16")
        self.label_16.setGeometry(QRect(704, 228, 16, 16))
        self.label_16.setFont(font1)
        self.label_27 = QLabel(Form)
        self.label_27.setObjectName(u"label_27")
        self.label_27.setGeometry(QRect(433, 230, 71, 16))
        self.label_27.setFont(font)
        self.lineEdit_17 = QLineEdit(Form)
        self.lineEdit_17.setObjectName(u"lineEdit_17")
        self.lineEdit_17.setGeometry(QRect(531, 228, 71, 20))
        self.lineEdit_17.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.lineEdit_18 = QLineEdit(Form)
        self.lineEdit_18.setObjectName(u"lineEdit_18")
        self.lineEdit_18.setGeometry(QRect(630, 228, 71, 19))
        self.lineEdit_18.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.label_25 = QLabel(Form)
        self.label_25.setObjectName(u"label_25")
        self.label_25.setGeometry(QRect(610, 230, 16, 16))
        self.label_26 = QLabel(Form)
        self.label_26.setObjectName(u"label_26")
        self.label_26.setGeometry(QRect(817, 230, 16, 16))
        font4 = QFont()
        font4.setPointSize(10)
        self.label_26.setFont(font4)
        self.label_28 = QLabel(Form)
        self.label_28.setObjectName(u"label_28")
        self.label_28.setGeometry(QRect(816, 182, 16, 16))
        self.label_28.setFont(font4)
        self.lineEdit_19 = QLineEdit(Form)
        self.lineEdit_19.setObjectName(u"lineEdit_19")
        self.lineEdit_19.setGeometry(QRect(629, 180, 71, 20))
        self.lineEdit_19.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.lineEdit_20 = QLineEdit(Form)
        self.lineEdit_20.setObjectName(u"lineEdit_20")
        self.lineEdit_20.setGeometry(QRect(530, 180, 71, 20))
        self.lineEdit_20.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.label_29 = QLabel(Form)
        self.label_29.setObjectName(u"label_29")
        self.label_29.setGeometry(QRect(609, 182, 16, 16))
        self.label_30 = QLabel(Form)
        self.label_30.setObjectName(u"label_30")
        self.label_30.setGeometry(QRect(703, 180, 16, 16))
        self.label_30.setFont(font1)
        self.lineEdit_21 = QLineEdit(Form)
        self.lineEdit_21.setObjectName(u"lineEdit_21")
        self.lineEdit_21.setGeometry(QRect(719, 180, 91, 20))
        self.lineEdit_21.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.label_31 = QLabel(Form)
        self.label_31.setObjectName(u"label_31")
        self.label_31.setGeometry(QRect(432, 182, 71, 16))
        self.label_31.setFont(font)
        self.label_14 = QLabel(Form)
        self.label_14.setObjectName(u"label_14")
        self.label_14.setGeometry(QRect(9, 220, 91, 21))
        self.label_14.setFont(font)
        self.label_15 = QLabel(Form)
        self.label_15.setObjectName(u"label_15")
        self.label_15.setGeometry(QRect(177, 221, 16, 16))
        self.label_15.setFont(font1)
        self.label_22 = QLabel(Form)
        self.label_22.setObjectName(u"label_22")
        self.label_22.setGeometry(QRect(331, 220, 16, 16))
        self.label_22.setFont(font1)
        self.lineEdit_7 = QLineEdit(Form)
        self.lineEdit_7.setObjectName(u"lineEdit_7")
        self.lineEdit_7.setGeometry(QRect(194, 220, 56, 19))
        self.lineEdit_7.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.label_23 = QLabel(Form)
        self.label_23.setObjectName(u"label_23")
        self.label_23.setGeometry(QRect(252, 220, 16, 16))
        self.label_23.setFont(font1)
        self.lineEdit_14 = QLineEdit(Form)
        self.lineEdit_14.setObjectName(u"lineEdit_14")
        self.lineEdit_14.setGeometry(QRect(115, 219, 57, 20))
        self.lineEdit_14.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.lineEdit_15 = QLineEdit(Form)
        self.lineEdit_15.setObjectName(u"lineEdit_15")
        self.lineEdit_15.setGeometry(QRect(270, 220, 57, 19))
        self.lineEdit_15.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.label_32 = QLabel(Form)
        self.label_32.setObjectName(u"label_32")
        self.label_32.setGeometry(QRect(9, 260, 91, 21))
        self.label_32.setFont(font)
        self.label_33 = QLabel(Form)
        self.label_33.setObjectName(u"label_33")
        self.label_33.setGeometry(QRect(177, 261, 16, 16))
        self.label_33.setFont(font1)
        self.label_34 = QLabel(Form)
        self.label_34.setObjectName(u"label_34")
        self.label_34.setGeometry(QRect(331, 260, 16, 16))
        self.label_34.setFont(font1)
        self.lineEdit_22 = QLineEdit(Form)
        self.lineEdit_22.setObjectName(u"lineEdit_22")
        self.lineEdit_22.setGeometry(QRect(194, 260, 56, 19))
        self.lineEdit_22.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.label_35 = QLabel(Form)
        self.label_35.setObjectName(u"label_35")
        self.label_35.setGeometry(QRect(252, 260, 16, 16))
        self.label_35.setFont(font1)
        self.lineEdit_23 = QLineEdit(Form)
        self.lineEdit_23.setObjectName(u"lineEdit_23")
        self.lineEdit_23.setGeometry(QRect(115, 259, 57, 20))
        self.lineEdit_23.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.lineEdit_24 = QLineEdit(Form)
        self.lineEdit_24.setObjectName(u"lineEdit_24")
        self.lineEdit_24.setGeometry(QRect(270, 260, 57, 19))
        self.lineEdit_24.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.label_36 = QLabel(Form)
        self.label_36.setObjectName(u"label_36")
        self.label_36.setGeometry(QRect(432, 133, 141, 20))
        self.label_36.setFont(font)

        self.lineEdit_25 = QLineEdit(Form)
        self.lineEdit_25.setObjectName(u"lineEdit_25")
        self.lineEdit_25.setGeometry(QRect(577, 133, 91, 20))
        self.lineEdit_25.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.comboBox_2 = QComboBox(Form)
        self.comboBox_2.addItem("")
        self.comboBox_2.addItem("")
        self.comboBox_2.setObjectName(u"comboBox_2")
        self.comboBox_2.setGeometry(QRect(400, 30, 141, 31))
        font5 = QFont()
        font5.setKerning(False)
        self.comboBox_2.setFont(font5)
        self.comboBox_2.setAcceptDrops(True)
        self.comboBox_2.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.comboBox_3 = QComboBox(Form)
        self.comboBox_3.addItem(u"calcul_rayon_v")
        self.comboBox_3.addItem("")
        self.comboBox_3.addItem("")
        self.comboBox_3.addItem("")
        self.comboBox_3.setObjectName(u"comboBox_3")
        self.comboBox_3.setGeometry(QRect(570, 30, 211, 31))
        self.comboBox_3.setFont(font5)
        self.comboBox_3.setStyleSheet(u"background-color: rgb(255, 255, 255);")
        self.pushButton.clicked.connect(self.function)
        #self.pushButton.clicked.connect(self.function)
        self.retranslateUi(Form)

        QMetaObject.connectSlotsByName(Form)
    # setupUi

    def retranslateUi(self, Form):
        Form.setWindowTitle(QCoreApplication.translate("Form", u"Form", None))
        self.label_2.setText(QCoreApplication.translate("Form", u"Latitude de P1", None))
        self.label_3.setText(QCoreApplication.translate("Form", u"\u00b0", None))
        self.label_4.setText(QCoreApplication.translate("Form", u"'", None))
        self.label_5.setText(QCoreApplication.translate("Form", u"\"", None))
        self.pushButton.setText(QCoreApplication.translate("Form", u"Convertir", None))
        self.label_17.setText(QCoreApplication.translate("Form", u"Probl\u00e8me Inverse Sur La Sph\u00e8re", None))
        self.label_24.setText(QCoreApplication.translate("Form", u"m", None))
        self.label_11.setText(QCoreApplication.translate("Form", u"\u00b0", None))
        self.label_13.setText(QCoreApplication.translate("Form", u"\"", None))
        self.label_12.setText(QCoreApplication.translate("Form", u"'", None))
        self.label_10.setText(QCoreApplication.translate("Form", u"Longitude de P1", None))
        self.label_16.setText(QCoreApplication.translate("Form", u"'", None))
        self.label_27.setText(QCoreApplication.translate("Form", u"Azimut A21", None))
        self.label_25.setText(QCoreApplication.translate("Form", u"\u00b0", None))
        self.label_26.setText(QCoreApplication.translate("Form", u"\"", None))
        self.label_28.setText(QCoreApplication.translate("Form", u"\"", None))
        self.label_29.setText(QCoreApplication.translate("Form", u"\u00b0", None))
        self.label_30.setText(QCoreApplication.translate("Form", u"'", None))
        self.label_31.setText(QCoreApplication.translate("Form", u"Azimut A12", None))
        self.label_14.setText(QCoreApplication.translate("Form", u"Latitude de P2", None))
        self.label_15.setText(QCoreApplication.translate("Form", u"\u00b0", None))
        self.label_22.setText(QCoreApplication.translate("Form", u"\"", None))
        self.label_23.setText(QCoreApplication.translate("Form", u"'", None))
        self.label_32.setText(QCoreApplication.translate("Form", u"Longitude de P2", None))
        self.label_33.setText(QCoreApplication.translate("Form", u"\u00b0", None))
        self.label_34.setText(QCoreApplication.translate("Form", u"\"", None))
        self.label_35.setText(QCoreApplication.translate("Form", u"'", None))
        self.label_36.setText(QCoreApplication.translate("Form", u"Distance entre P1 et P2", None))
        self.comboBox_2.setItemText(0, QCoreApplication.translate("Form", u"Clarck 1880", None))
        self.comboBox_2.setItemText(1, QCoreApplication.translate("Form", u"WGS 84", None))

        self.comboBox_3.setItemText(1, QCoreApplication.translate("Form", u"calcul_rayon_Gauss", None))
        self.comboBox_3.setItemText(2, QCoreApplication.translate("Form", u"calcul_rayon_moyen", None))
        self.comboBox_3.setItemText(3, QCoreApplication.translate("Form", u"calcul_rayon_a", None))

    # retranslateUi

if __name__ == "__main__":
    import sys
    app = QtWidgets.QApplication(sys.argv)
    Form = QtWidgets.QWidget()
    ui = Ui_Form()
    ui.setupUi(Form)
    Form.show()
    sys.exit(app.exec_())


"""
https://docs.python.org/3/library/xml.etree.elementtree.html
"""

# Parsing XML
# We can import this data by reading from a file:
import xml.etree.ElementTree as ET
tree = ET.parse('country_data.xml')
root = tree.getroot()
# Or directly from a string:
root = ET.fromstring(country_data_as_string)
# As an Element, root has a tag and a dictionary of attributes:
print(root.tag, root.attrib)
# It also has children nodes over which we can iterate:
for child in root:
    print(child.tag, child.attrib)
# Children are nested, and we can access specific child nodes by index:
print(root[0][1].text)

# Pull API for non-blocking parsing
# parse XML incrementally, without blocking operations
parser = ET.XMLPullParser(['start', 'end'])
parser.feed('<mytag>sometext')
list(parser.read_events())
parser.feed(' more text</mytag>')
for event, elem in parser.read_events():
    print(event)
    print(elem.tag, 'text=', elem.text)

# Finding interesting elements
# iterate recursively over all the sub-tree below it (its children, their children, and so on)
for neighbor in root.iter('neighbor'):
    print(neighbor.attrib)
# finall() finds only elements with a tag which are direct children of the current element
# find() finds the first child with a particular tag
# text accesses the element’s text content
# get() accesses the element’s attributes
for country in root.findall('country'):
    rank = country.find('rank').text
    name = country.get('name')
    print(name, rank)

# Modifying an XML File