# import logging
import math
import random
from ctypes import POINTER

import numpy as np
import numpy.linalg
import cv2
import gym.wrappers.monitoring.video_recorder
import gym_duckietown.objmesh
import gym_duckietown.objects
import gym_duckietown.simulator
import gym_duckietown.envs
import gym_duckietown.wrappers
import pyglet
from pyglet import gl, window, image

WINDOW_WIDTH = 800
WINDOW_HEIGHT = 600

FULL_VIEW_MODE = 0
TOP_DOWN_VIEW_MODE = 1
FRONT_VIEW_MODE = 2
N_VIEW_MODES = 3

from .Evaluator import Evaluator

class BaseEnvironment(gym_duckietown.envs.DuckietownEnv):
	def __init__(self,
			 top_down = False,
			 cam_height = 5,
			 video_path: str = None,
			 **kwargs):
		super().__init__(**kwargs)
		self.horizon_color = self._perturb(self.color_sky)
		self.cam_fov_y = gym_duckietown.simulator.CAMERA_FOV_Y
		self.top_down = top_down

		self.eval = Evaluator(self)

		self._view_mode = 0
		self.top_cam_height = cam_height

		self.actions = [0, 0]

		if video_path is not None:
			self.rec = gym.wrappers.monitoring.video_recorder.VideoRecorder(self, path=video_path, enabled = True)
		else:
			self.rec = None

		self.renderables = None

	def add_renderable(self, r):
		if self.renderables is None: self.renderables = [r]
		else: self.renderables.append(r)
		return self.renderables

	def next_view(self):
		self._view_mode = (self._view_mode + 1) % N_VIEW_MODES

	def set_view(self, view):
		self._view_mode = view % N_VIEW_MODES

	def toggle_single_view(self):
		if self._view_mode == TOP_DOWN_VIEW_MODE:
			self._view_mode = FRONT_VIEW_MODE
		else:
			self._view_mode = TOP_DOWN_VIEW_MODE

	# Returns a list with all road tiles in the current map. Each item of the list contains the
	# tile indices (not positional coordinates), and the tile type (curve, straight, intersection).
	def roads(self):
		return self._roads

	def current_tile(self):
		return self.get_grid_coords(self.cur_pos)

	def tile_center(self, i, j=None):
		if j is None:
			i, j = i[0], i[1]
		return (np.array([i, j])+0.5)*self.road_tile_size

	def nearest_drivable(self, x: float, y: float = None) -> (float, float):
		if y is None: x, y = x[0], x[1]
		min_d, min_p = math.inf, None
		for t in self.drivable_tiles:
			a, b = t["coords"]
			cx, cy = (a+0.5)*self.road_tile_size, (b+0.5)*self.road_tile_size
			d = (cx-x)*(cx-x)+(cy-y)*(cy-y)
			if min_d > d: min_d, min_p = d, (cx, cy)
		return min_p

	def get_position(self):
		return np.delete(self.cur_pos, 1)

	def top_down_obs(self, segment = False):
		return self._render_img(
			WINDOW_WIDTH,
			WINDOW_HEIGHT,
			self.multi_fbo_human,
			self.final_fbo_human,
			self.img_array_human,
			top_down = True,
			segment=segment,
			# callback = (lambda: self.mailbox.render()) if self.mailbox is not None else None,
		)

	def front(self, segment = False):
		return self._render_img(
			WINDOW_WIDTH,
			WINDOW_HEIGHT,
			self.multi_fbo_human,
			self.final_fbo_human,
			self.img_array_human,
			top_down = False,
			segment=segment,
			# callback = (lambda: self.mailbox.render()) if self.mailbox is not None else None,
		)

	def render(self, mode: str = "human", close: bool = False, segment: bool = False, text: str = ""):
		"""
		Render the environment for human viewing

		mode: "human", "top_down", "free_cam", "rgb_array"

		"""
		assert mode in ["human", "top_down", "free_cam", "rgb_array"]

		if close:
			if self.window:
				self.window.close()
			return

		top_down = mode == 'top_down'
		# Render the image
		top = self.top_down_obs(segment)
		bot = self.front(segment)

		if self.distortion and not self.undistort and mode != "free_cam":
			bot = self.camera_model.distort(bot)

		win_width = WINDOW_WIDTH
		if self._view_mode == FULL_VIEW_MODE:
			img = np.concatenate((top, bot), axis=1)
			win_width = 2*WINDOW_WIDTH
		elif self._view_mode == TOP_DOWN_VIEW_MODE:
			img = top
		else:
			img = bot

		if self.window is not None:
			self.window.set_size(win_width, WINDOW_HEIGHT)

		if mode == 'rgb_array':
			return img

		if self.window is None:
			config = gl.Config(double_buffer=False)
			self.window = window.Window(
				width=win_width,
				height=WINDOW_HEIGHT,
				resizable=False,
				config=config
			)

		self.window.clear()
		self.window.switch_to()
		self.window.dispatch_events()

		# Bind the default frame buffer
		gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, 0)

		# Setup orghogonal projection
		gl.glMatrixMode(gl.GL_PROJECTION)
		gl.glLoadIdentity()
		gl.glMatrixMode(gl.GL_MODELVIEW)
		gl.glLoadIdentity()
		gl.glOrtho(0, WINDOW_WIDTH, 0, WINDOW_HEIGHT, 0, 10)

		# Draw the image to the rendering window
		width = img.shape[1]
		height = img.shape[0]
		img = np.ascontiguousarray(np.flip(img, axis=0))
		img_data = image.ImageData(
			width,
			height,
			'RGB',
			img.ctypes.data_as(POINTER(gl.GLubyte)),
			pitch=width * 3,
		)
		img_data.blit(
			0,
			0,
			0,
			width=WINDOW_WIDTH,
			height=WINDOW_HEIGHT
		)

		# Display position/state information
		if mode != "free_cam":
			x, y, z = self.cur_pos
			self.text_label.text = (
				f"pos: ({x:.2f}, {y:.2f}, {z:.2f}), angle: "
				f"{np.rad2deg(self.cur_angle):.1f} deg, steps: {self.step_count}, "
				f"speed: {self.speed:.2f} m/s"
			)
			if len(text) > 0: self.text_label.text += text
			self.text_label.draw()

		# Force execution of queued commands
		gl.glFlush()

		if self.rec is not None:
			self.rec.capture_frame()

		if self.renderables is not None:
			for r in self.renderables: r.render()

		return img

	def close(self):
		if self.rec is not None:
			self.rec.close()
		super().close()

	def reset(self, segment: bool = False, force = True):
		if force:
			self.force_reset(segment)

	def step(self, vel: float, angle: float):
		self.actions[0], self.actions[1] = vel, angle
		obs, reward, done, info = super().step(self.actions)
		return obs, reward, done, info

	def pointing_direction(self) -> tuple:
		q = math.pi*0.25
		theta = self.cur_angle
		# East
		if q >= theta > -q:
			return 1, 0
		t = math.pi*0.75
		# North
		if t >= theta > q:
			return 0, -1
		# South
		if -q >= theta > -t:
			return 0, 1
		# West
		return -1, 0

	def get_dir_vec(self):
		return gym_duckietown.simulator.get_dir_vec(self.cur_angle)

	def force_reset(self, segment: bool = False):
		super().reset(segment)

	# We have to convert from window positions to actual Duckietown coordinates.
	def convert_coords(self, x: int, y: int) -> (float, float):
		w = self.grid_width+3
		h = self.grid_height+2
		dw = WINDOW_WIDTH/w
		dh = WINDOW_HEIGHT/h
		x -= 1.5*dw
		y += dh
		x, y = x/dw, (WINDOW_HEIGHT-y) / dh
		return (x+0.5)*self.road_tile_size, (y+0.5)*self.road_tile_size

	# The inverse transformation of the above.
	def unconvert_coords(self, x: float, y: float = None) -> (int, int):
		if y is None:
			x, y = x[0], x[1]
		x, y = x/self.road_tile_size-0.5, y/self.road_tile_size-0.5
		w = self.grid_width+3
		h = self.grid_height+2
		dw = WINDOW_WIDTH/w
		dh = WINDOW_HEIGHT/h
		x, y = dw*x, dh*y-WINDOW_HEIGHT
		return int(x + 1.5*dw), int(y - dh)

	def add_duckie(self, x, y = None, static = True, optional = False):
		if y is None:
			x, y = x[0], x[1]
		obj = _get_obj_props('duckie', x, y, static, optional = optional)
		self.objects.append(
			gym_duckietown.objects.DuckieObj(
				obj,
				False,
				gym_duckietown.simulator.SAFETY_RAD_MULT,
				self.road_tile_size
			)
		)

	def add_big_duckie(self, x, y = None, static = True, optional = False):
		if y is None:
			x, y = x[0], x[1]
		obj = _get_obj_props('duckie', x, y, static, rescale = 3.0, optional = optional)
		self.objects.append(
			gym_duckietown.objects.DuckieObj(
				obj,
				False,
				gym_duckietown.simulator.SAFETY_RAD_MULT,
				self.road_tile_size
			)
		)

	def add_static_duckie(self, x, y = None, angle = None):
		if y is None: x, y = x[0], x[1]
		if angle is None: angle = self.np_random.random()*math.pi
		obj = _get_obj_props('duckie', x, y, True, rescale = 3.0, angle = angle)
		self.objects.append(gym_duckietown.objects.WorldObj(obj, False, gym_duckietown.simulator.SAFETY_RAD_MULT))

	def add_static_big_duckie(self, x, y = None, angle = None):
		if y is None: x, y = x[0], x[1]
		if angle is None: angle = self.np_random.random()*math.pi
		obj = _get_obj_props('duckie', x, y, True, angle = angle)
		self.objects.append(gym_duckietown.objects.WorldObj(obj, False, gym_duckietown.simulator.SAFETY_RAD_MULT))

	def add_static_duckiebot(self, x, y = None, angle = None):
		if y is None: x, y = x[0], x[1]
		if angle is None: angle = self.np_random.random()*math.pi
		obj = _get_obj_props('duckiebot', x, y , True, rescale = 2.0, angle = angle)
		self.objects.append(gym_duckietown.objects.WorldObj(obj, False, gym_duckietown.simulator.SAFETY_RAD_MULT))

	def add_cone(self, x, y = None, scale = 1.0):
		if y is None:
			x, y = x[0], x[1]
		obj = _get_obj_props('cone', x, y, rescale = scale)
		c = gym_duckietown.objects.WorldObj(obj, False, gym_duckietown.simulator.SAFETY_RAD_MULT)
		self.objects.append(c)
		return c

	def add_walking_duckie(self, x, y = None):
		if y is None:
			x, y = x[0], x[1]
		obj = _get_obj_props('duckie', x, y, False)
		obj['kind'] = 'duckiebot'
		d = gym_duckietown.objects.DuckiebotObj(obj, False, gym_duckietown.simulator.SAFETY_RAD_MULT,
											  gym_duckietown.simulator.WHEEL_DIST,
											  gym_duckietown.simulator.ROBOT_WIDTH,
											  gym_duckietown.simulator.ROBOT_LENGTH)
		self.objects.append(d)
		return d

	def add_light(self, x, y):
		li = gl.GL_LIGHT0 + 1

		li_pos = [x, 0.5, y, 1.0]
		diffuse = [0.5, 0.5, 0.5, 0.5]
		ambient = [0.5, 0.5, 0.5, 0.5]
		specular = [0.5, 0.5, 0.5, 1.0]
		spot_direction = [0.5, -0.5, 0.5]
		gl.glLightfv(li, gl.GL_POSITION, (gl.GLfloat * 4)(*li_pos))
		gl.glLightfv(li, gl.GL_AMBIENT, (gl.GLfloat * 4)(*ambient))
		gl.glLightfv(li, gl.GL_DIFFUSE, (gl.GLfloat * 4)(*diffuse))
		gl.glLightfv(li, gl.GL_SPECULAR, (gl.GLfloat * 4)(*specular))
		gl.glLightfv(li, gl.GL_SPOT_DIRECTION, (gl.GLfloat * 3)(*spot_direction))
		# gl.glLightfv(li, gl.GL_SPOT_EXPONENT, (gl.GLfloat * 1)(64.0))
		gl.glLightf(li, gl.GL_SPOT_CUTOFF, 60)

		gl.glLightfv(li, gl.GL_CONSTANT_ATTENUATION, (gl.GLfloat * 1)(1.0))
		# gl.glLightfv(li, gl.GL_LINEAR_ATTENUATION, (gl.GLfloat * 1)(0.1))
		gl.glLightfv(li, gl.GL_QUADRATIC_ATTENUATION, (gl.GLfloat * 1)(0.2))
		gl.glEnable(li)

	def sine_target(self, t: np.ndarray, s: np.ndarray = None):
		if s is None:
			s = np.delete(self.get_dir_vec(), 1)
		u, v = s/np.linalg.norm(s), t/np.linalg.norm(t)
		return np.cross(u, v)/(np.linalg.norm(u, ord=1)*np.linalg.norm(v, ord=1))

	def lf_target(self):
		_, ct, c0 = self.closest_curve_point(self.cur_pos, self.cur_angle, delta = 0.2)
		u, v = np.delete(c0, 1), np.delete(ct, 1)
		p = u-self.get_position()
		t = math.asin(self.sine_target(v))
		d = np.linalg.norm(p)*np.sign(self.sine_target(p, v))
		return d, t

	def tile_position(self, x, y, centered: bool = False) -> (float, float):
		s = self.road_tile_size
		if centered: return x*s+s/2, y*s+s/2
		return x*s, y*s

	road_tiles = set(["curve_left", "curve_right", "straight", "4way", "3way_left", "3way_right"])

	def random_road_pose(self) -> (np.ndarray, float):
		R = [np.array(t["coords"])*self.road_tile_size+self.road_tile_size/2 for t in self.grid if t["kind"] in DuckievillageEnv.road_tiles]
		return np.insert(random.choice(R), 1, 0), math.pi*self.np_random.random()


import csv

class Question:
    """Documentation for the Question class."""

    def __init__(self, file_name) -> None:
        self.questions_fieldnames = [
            "no",
            "question",
            "answer",
            "language",
        ]
        self.file_name = file_name
        self.initialize_file()

    def add_questions(self, questions):
        """
        Adds questions to self.file_name csv

        Parameters:
            questions: (list of (str, str))

        Returns:
            None
        """
        for ques in questions:
            self.add_question(*ques)

    def add_question(self, question, answer, language="python"):
        """
        Adds a question to self.file_name csv

        Parameters:
            question (str): A string containing the question
            answer (str): The answer to the question

        Returns:
            None
        """
        last_no = self.next_question_no

        new_question = {
            "no": last_no,
            "question": question,
            "answer": answer,
            "language": language,
        }

        headers = self.first_row

        with open(self.file_name, "a", newline="") as questions_file:
            writer = csv.DictWriter(
                questions_file, fieldnames=self.questions_fieldnames
            )

            if not headers:
                writer.writeheader()

            writer.writerow(new_question)

    def get_questions(self):
        """
        Get all questions from csv file

        Parameters:

        Returns:
            questions (list of dicts): List of dicts with questions and answers
        """
        questions = []

        with open(self.file_name, newline="") as questions_file:
            file_reader = csv.DictReader(questions_file)

            for question_row in file_reader:
                questions.append(question_row)

        return questions

    def filter_questions_by_no(self, question_nos):
        """
        Finds and returns a question from csv file

        Parameter:
            no (str or int): The unique identifier of the question

        Returns:
            question (dict): Dict containing no, question and answer fields
        """
        questions = self.get_questions()

        if len(question_nos) == 1 and question_nos[0] == 0:
            return questions

        return [ques for ques in questions if int(ques.get("no")) in question_nos]

    @property
    def first_row(self):
        """
        Retrieve the first row of the csv

        Returns:
            field_names (list): List of items in first row
        """
        with open(self.file_name, newline="") as questions_file:
            file_reader = csv.DictReader(questions_file)
            return file_reader.fieldnames

    @property
    def last_question_no(self):
        """
        Retrieve the no of the last question

        Returns:
            no (str or int): Number for the next question
        """
        questions = self.get_questions()

        try:
            return int(questions[-1]["no"])
        except IndexError:
            return 0

    @property
    def next_question_no(self):
        """
        Retrieve the index for the next question

        Returns:
            no (str or int): Number for the next question
        """
        return self.last_question_no + 1

    def delete_questions(self, numbers):
        """
        Remove question from csv file

        Parameters:
            row_no (str or int): Value of question id in csv file

        Returns:
            None
        """
        questions = self.get_questions()

        filtered_questions = [
            question for question in questions if int(question["no"]) not in numbers
        ]

        with open(self.file_name, "w", newline="") as questions_file:
            writer = csv.DictWriter(
                questions_file, fieldnames=self.questions_fieldnames
            )
            writer.writeheader()

            for i, question in enumerate(filtered_questions):
                question["no"] = i + 1
                writer.writerow(question)

    def initialize_file(self):
        try:
            with open(self.file_name, newline=""):
                ...
        except FileNotFoundError:
            with open(self.file_name, "w"):
                ...

    def clear_file(self):
        # opening the file with w+ mode truncates the file
        f = open(self.file_name, "w+")
        f.close()


"""The :mod:`~virtual_rainforest.core.axes` module handles the validation of data being
loaded into the core data storage of the virtual rainforest simulation. The main
functionality in this module is ensuring that any loaded data is congruent with the core
axes of the simulation and the configuration of a given simulation.

The AxisValidator class
=======================

The :class:`~virtual_rainforest.core.axes.AxisValidator` abstract base class provides an
extensible framework for validating data arrays. Each subclass of the base class defines
a particular core axis along with the name or names of dimensions in the input array
that are expected to map onto that axis. So, for example, a validator can support the
``spatial`` axis using the ``x`` and ``y`` dimensions. Each individual subclass provides
bespoke methods to test whether that validator can be applied to an input data array and
then a validation routine to apply when it can.

When new :class:`~virtual_rainforest.core.axes.AxisValidator` subclasses are defined,
they are automatically added to the
:attr:`~virtual_rainforest.core.axes.AXIS_VALIDATORS` registry. This maintains a list of
the validators defined for each core axis.

Note that the set of validators defined for a specific core axis should be mutually
exclusive: only one should be applicable to any given dataset being tested on that axis.

DataArray validation
====================

The :func:`~virtual_rainforest.core.axes.validate_dataarray` function takes an input
Data Array and applies validation where applicable across all the core axes. The
function returns the validated input (possibly altered to align with the core axes)
along with a dictionary using the set of core axes as names: the value associated with
each axis name is the name of the AxisValidator applied or None if the input did not
match a validator on that axis.

Core axes
=========

The 'spatial' axis
------------------

The :class:`~virtual_rainforest.core.axes.AxisValidator` subclasses defined for the
'spatial' axis  standardise the spatial structure of the input data to use a single
``cell_id`` spatial axis, which maps data onto the cell IDs used for indexing in the
:class:`~virtual_rainforest.core.grid.Grid` instance for the simulation. `x`
"""  # noqa: D205, D415

from abc import ABC, abstractmethod
from typing import Any, Optional, Type

import numpy as np
from xarray import DataArray

from virtual_rainforest.core.grid import Grid
from virtual_rainforest.core.logger import LOGGER

class AxisValidator(ABC):
    """The AxisValidator abstract base class.

    This abstract base class provides the structure for axis validators. These are used
    to check that a ``DataArray`` to be added to a ``Data`` instance is congruent with
    the configuration of a virtual rainforest simulation. The base class provides
    abstract methods that provide the following functionality:

    * :meth:`~virtual_rainforest.core.axes.AxisValidator.can_validate`: test that a
      given ``AxisValidator`` subclass can be applied to the inputs.
    * :meth:`~virtual_rainforest.core.axes.AxisValidator.run_validation`: run
      appropriate validation and standardisation on the input ``DataArray``.

    The :meth:`~virtual_rainforest.core.axes.AxisValidator.can_validate` method should
    be used first to check that a particular ``DataArray`` can be validated, and then
    the :meth:`~virtual_rainforest.core.axes.AxisValidator.run_validation` method can be
    used to validate that input if appropriate.
    """

    core_axis: str
    """Class attribute giving the name of the core axis for an AxisValidator."""

    dim_names: set[str]
    """Class attribute giving the dimension names for an AxisValidator."""

    def __init__(self) -> None:
        super().__init__()

    @classmethod
    def __init_subclass__(cls) -> None:
        """Adds new subclasses to the AxisValidator registry.

        When new subclasses are created this method automatically extends the
        :attr:`~virtual_rainforest.core.axes.AXIS_VALIDATORS` registry. AxisValidators
        are arranged in the registry dictionary as lists keyed under core axis names,
        and the core axis name for a given subclass is set in the  subclass
        :attr:`~virtual_rainforest.core.axes.AxisValidator.core_axis`
        class attribute.

        Raises:
            ValueError: if the subclass attributes are invalid.
        """

        if not hasattr(cls, "core_axis"):
            raise ValueError("Class attribute core_axis not set.")

        if not isinstance(cls.core_axis, str) or cls.core_axis == "":
            raise ValueError(
                "Class attribute core_axis is not a string or is an empty string."
            )

        if not hasattr(cls, "dim_names"):
            raise ValueError("Class attribute dim_names not set.")

        if not isinstance(cls.dim_names, set) or any(
            [not isinstance(x, str) for x in cls.dim_names]
        ):
            raise ValueError("Class attribute dim_names is not a set of strings.")

        if cls.core_axis in AXIS_VALIDATORS:
            AXIS_VALIDATORS[cls.core_axis].append(cls)
        else:
            AXIS_VALIDATORS[cls.core_axis] = [cls]

        LOGGER.debug("Adding '%s' AxisValidator: %s", cls.core_axis, cls.__name__)

    @abstractmethod
    def can_validate(self, value: DataArray, grid: Grid, **kwargs: Any) -> bool:
        """Check if an AxisValidator subclass applies to inputs.

        A given AxisValidator subclass must provide a ``run_validation`` method that
        defines data validation that should be applied to the inputs. However, the
        validation for a particular subclass will only apply to inputs with particular
        features, such as an array with a given dimension name or a set grid type.

        In a subclass, the implementation of this method should check whether the
        validation implemented in ``run_validation`` **can** be applied to the inputs.

        Args:
            value: An input DataArray to check
            grid: A Grid object giving the spatial configuration of the simulation.
            kwargs: Other configuration details to be used.

        Returns:
            A boolean showing if the `run_validation` method of the subclass can be
            applied to the inputs.
        """

    @abstractmethod
    def run_validation(self, value: DataArray, grid: Grid, **kwargs: Any) -> DataArray:
        """Validate the input DataArray.

        The implementation for an AxisValidator subclass should define a set of checks
        on the inputs that are used to validate that the input DataArray value is
        congruent with the simulation configuration. The method can also perform
        standardisation and return a modified input value that has been aligned to the
        simulation structure.

        Args:
            value: An input DataArray to check
            grid: A Grid object giving the spatial configuration of the simulation.
            kwargs: Other configuration details to be used.

        Returns:
            A DataArray that passes validation, possibly modified to align with internal
            data structures.
        """

AXIS_VALIDATORS: dict[str, list[Type[AxisValidator]]] = {}
"""A registry for different axis validators subclasses

This registry contains a dictionary of lists of AxisValidator subclasses. Each list
contains all of AxisValidator subclasses that apply to a particular core axis, and the
core axis names are used as the key to these lists.

Users defined AxisValidator subclasses will be automatically added to this registry by
the ``__subclass_init__`` method.
"""

def validate_dataarray(
    value: DataArray, grid: Grid, **kwargs: Any
) -> tuple[DataArray, dict]:
    """Validate a DataArray across the core axes.

    The AXIS_VALIDATORS registry provides a list of AxisValidators subclasses for each
    core axis. This function loops over the core axes, and checks whether to apply
    validation for that axis to the input DataArray.

    For each axis in turn, the function first checks if the dimension names of the input
    DataArray overlap with the set of dimension names used across the AxisValidators for
    that axis. If not, then the next axis is checked without altering the input array.
    If the dimension names do match the axis then the appropriate AxisValidator is then
    used to validate the input array and the validated array is passed on to the next
    axis for further validation.

    Args:
        value: An input DataArray for validation
        grid: A Grid object giving the spatial configuration.
        kwargs: Further configuration details to be passed to AxisValidators

    Returns:
        The function returns the validated data array and a dictionary recording which
        AxisValidator classes were applied to each of the core axes.

    Raises:
        ValueError: If the input data array uses dimension names required for an axis
            without matching a registered validator.
        RuntimeError: If more than one validator reports that it can validate an input.
    """

    validation_dict: dict[str, Optional[str]] = {}
    to_raise: Exception

    # Get the validators applying to each axis
    for axis in AXIS_VALIDATORS:
        validators: list[Type[AxisValidator]] = AXIS_VALIDATORS[axis]

        # Get the set of dim names across all of the validators for this axis
        validator_dims = set.union(*[v.dim_names for v in validators])

        # If the dataarray includes any of those dimension names, one of the validators
        # for that axis must be able to validate the array, otherwise we can skip
        # validation on this axis and return the input array.
        matching_dims = validator_dims.intersection(value.dims)
        if matching_dims:
            # There should be one and only validator that can validate for this axis.
            validator_found = [v for v in validators if v().can_validate(value, grid)]

            if len(validator_found) == 0:
                to_raise = ValueError(
                    f"DataArray uses '{axis}' axis dimension names but does "
                    f"not match a validator: {','.join(matching_dims)}"
                )
                LOGGER.critical(to_raise)
                raise to_raise

            if len(validator_found) > 1:
                to_raise = RuntimeError(
                    f"Validators on '{axis}' axis not mutually exclusive"
                )
                LOGGER.critical(to_raise)
                raise to_raise

            # Get the appropriate Validator class and then use it to update the data
            # array
            this_validator = validator_found[0]
            try:
                value = this_validator().run_validation(value, grid, **kwargs)
            except Exception as excep:
                LOGGER.critical(excep)
                raise

            validation_dict[axis] = this_validator.__name__

        else:
            validation_dict[axis] = None

    return value, validation_dict

class Spat_CellId_Coord_Any(AxisValidator):
    """Validate *cell_id* coordinates on the *spatial* core axis.

    Applies to:
        An input DataArray that provides coordinate values along a ``cell_id`` dimension
        is assumed to map data onto the cells values defined in the
        :class:`~virtual_rainforest.core.grid.Grid` configured for the simulation. The
        coordinate values are tested to check they provide a one-to-one map onto the
        ``cell_id`` values defined for each cell in the ``Grid``. Because ``cell_id``
        values are defined for cells in all grid types, this validator does not require
        a particular grid geometry.
    """

    core_axis = "spatial"
    dim_names = {"cell_id"}

    def can_validate(self, value: DataArray, grid: Grid, **kwargs: Any) -> bool:
        """Check the validator applies to the inputs.

        Args:
            value: An input DataArray to check
            grid: A Grid object giving the spatial configuration of the simulation.
            kwargs: Other configuration details to be used.

        Returns:
            A boolean showing if this subclass can be applied to the inputs.
        """
        return self.dim_names.issubset(value.dims) and self.dim_names.issubset(
            value.coords
        )

    def run_validation(self, value: DataArray, grid: Grid, **kwargs: Any) -> DataArray:
        """Run validation on the inputs.

        Validation will fail if the ``cell_id`` coordinate values:

        * are not unique, or
        * do not provide a one-to-one mapping onto the set of ``cell_id`` values defined
          in the configured ``Grid`` object.

        Args:
            value: An input DataArray to check
            grid: A Grid object giving the spatial configuration of the simulation.
            kwargs: Other configuration details to be used.

        Raises:
            ValueError: when ``cell_id`` values are not congruent with the ``Grid``.

        Returns:
            A DataArray standardised to match the ``cell_id`` values in the ``Grid``
            object.
        """
        da_cell_ids = value["cell_id"].values

        if len(np.unique(da_cell_ids)) != len(da_cell_ids):
            raise ValueError("The data cell ids contain duplicate values.")

        if not set(grid.cell_id) == set(da_cell_ids):
            raise ValueError(
                "The data cell ids do not provide a one-to-one map onto grid cell ids."
            )

        # Now ensure sorting and any subsetting:
        # https://stackoverflow.com/questions/8251541
        da_sortorder = np.argsort(da_cell_ids)
        gridid_pos = np.searchsorted(da_cell_ids[da_sortorder], grid.cell_id)
        da_indices = da_sortorder[gridid_pos]

        return value.isel(cell_id=da_indices)

class Spat_CellId_Dim_Any(AxisValidator):
    """Validate *cell_id* dimension on the *spatial* core axis.

    Applies to:
        The input DataArray for this validator has a ``cell_id`` dimension but there are
        no ``cell_id`` values provided as coordinates along that dimension. The
        ``cell_id`` dimension name means that the array is assumed to provide data for
        the cells defined in the :class:`~virtual_rainforest.core.grid.Grid` configured
        for the simulation. The data are then assumed to be in the same order as the
        cells in the ``Grid``.  As a one-dimensional array of cells is defined for all
        grid configurations, this validator does not require a particular grid geometry.
    """

    core_axis = "spatial"
    dim_names = {"cell_id"}

    def can_validate(self, value: DataArray, grid: Grid, **kwargs: Any) -> bool:
        """Check the validator applies to the inputs.

        Args:
            value: An input DataArray to check
            grid: A Grid object giving the spatial configuration of the simulation.
            kwargs: Other configuration details to be used.

        Returns:
            A boolean showing if this subclass can be applied to the inputs.
        """

        return self.dim_names.issubset(value.dims) and not self.dim_names.issubset(
            value.coords
        )

    def run_validation(self, value: DataArray, grid: Grid, **kwargs: Any) -> DataArray:
        """Run validation on the inputs.

        Validation will fail when the ``cell_id`` dimension:

        * is not of exactly the same length as the list of cells defined in the
          configured ``Grid`` object.

        Args:
            value: An input DataArray to check
            grid: A Grid object giving the spatial configuration of the simulation.
            kwargs: Other configuration details to be used.

        Raises:
            ValueError: when ``cell_id`` values are not congruent with the ``Grid``.

        Returns:
            A DataArray standardised to match the ``cell_id`` values in the ``Grid``
            object.
        """
        # Cell ID is only a dimenson with a give length - assume the order correct and
        # check the right number of cells found
        n_found = value["cell_id"].size
        if grid.n_cells != n_found:
            raise ValueError(
                f"Grid defines {grid.n_cells} cells, data provides {n_found}"
            )

        return value

class Spat_XY_Coord_Square(AxisValidator):
    """Validate *x* and *y* coordinates on the *spatial* core axis for a square grid.

    Applies to:
        An input DataArray that provides coordinates along ``x`` and ``y`` dimensions is
        assumed to map data onto the cells defined in a
        :class:`~virtual_rainforest.core.grid.Grid` configured for the simulation with a
        ``square`` cell geometry. The pairwise combinations of ``x`` and ``y``
        coordinates in the data are expected to provide a one-to-one mapping onto  grid
        cell geometries.

    This validator also remaps ``x`` and ``y`` dimensions onto the internal ``cell_id``
    dimension used in the :mod:`~virtual_rainforest.core.grid` module.
    """

    core_axis = "spatial"
    dim_names = {"x", "y"}

    def can_validate(self, value: DataArray, grid: Grid, **kwargs: Any) -> bool:
        """Check the validator applies to the inputs.

        Args:
            value: An input DataArray to check
            grid: A Grid object giving the spatial configuration of the simulation.
            kwargs: Other configuration details to be used.

        Returns:
            A boolean showing if this subclass can be applied to the inputs.
        """
        return self.dim_names.issubset(value.dims) and self.dim_names.issubset(
            value.coords
        )

    def run_validation(self, value: DataArray, grid: Grid, **kwargs: Any) -> DataArray:
        """Run validation on the inputs.

        Validation will fail when the ``x`` and ``y`` coordinates:

        * Fall on cell geometry boundaries: unambiguous coordinates within the geometry,
          such as cell centroids, should be used.
        * The coordinate pairs do not provide a one-to-one mapping onto the cells:
          multiple coordinates map onto a single cell, there is not a coordinate pair
          for each cell, or some coordinate pairs do not map onto a cell.

        Args:
            value: An input DataArray to check
            grid: A Grid object giving the spatial configuration of the simulation.
            kwargs: Other configuration details to be used.

        Raises:
            ValueError: when ``x`` and ``y`` values are not congruent with the ``Grid``.

        Returns:
            A DataArray with the ``x`` and ``y`` dimensions remapped onto the internal
            ``cell_id`` dimension used in the :mod:`~virtual_rainforest.core.grid`
            module.
        """
        # Get x and y coords to check the extents and cell coverage.
        #
        # TODO - Note that mapping all the cells here is a bit extreme with a square
        # grid - could just map one row and column to confirm the indexing into the
        # data, but this is a more general solution.

        # Use .stack() to convert the axis into stacked pairs of values
        xy_pairs = value.stack(cell=("y", "x")).coords
        idx_pairs = value.drop_vars(("x", "y")).stack(cell=("y", "x")).coords

        # Get the mapping of points onto the grid
        idx_x, idx_y = grid.map_xy_to_cell_indexing(
            x_coords=xy_pairs["x"].values,
            y_coords=xy_pairs["y"].values,
            x_idx=idx_pairs["x"].values,
            y_idx=idx_pairs["y"].values,
        )

        # Now remap the grids from xy to cell_id - this uses the rather under described
        # vectorized indexing feature in xarray:
        #
        #   https://docs.xarray.dev/en/stable/user-guide/indexing.html
        #
        # Specifically using DataArray.isel to select indexed dimensions by name, which
        # avoids issues with different permutations of the axes - and map XY onto a
        # common cell_id

        return value.isel(
            x=DataArray(idx_x, dims=["cell_id"]), y=DataArray(idx_y, dims=["cell_id"])
        )

class Spat_XY_Dim_Square(AxisValidator):
    """Validate *x* and *y* dimensions on the *spatial* core axis on a square grid.

    Applies to:
        An input DataArray with ``x`` and ``y`` dimensions specifies the size of the
        array along those dimensions but does not provide coordinates for the cells. The
        input is then  assumed to provide an array with the same shape as a the cell
        grid of a :class:`~virtual_rainforest.core.grid.Grid` configured for the
        simulation with a ``square`` cell geometries.

    This validator also remaps ``x`` and ``y`` dimensions onto the internal ``cell_id``
    dimension used in the :mod:`~virtual_rainforest.core.grid` module.
    """

    core_axis = "spatial"
    dim_names = {"x", "y"}

    def can_validate(self, value: DataArray, grid: Grid, **kwargs: Any) -> bool:
        """Check the validator applies to the inputs.

        Args:
            value: An input DataArray to check
            grid: A Grid object giving the spatial configuration of the simulation.
            kwargs: Other configuration details to be used.

        Returns:
            A boolean showing if this subclass can be applied to the inputs.
        """
        return self.dim_names.issubset(value.dims) and not self.dim_names.issubset(
            value.coords
        )

    def run_validation(self, value: DataArray, grid: Grid, **kwargs: Any) -> DataArray:
        """Run validation on the inputs.

        Validation will fail when the ``x`` and ``y`` dimensions:

        * do not have exactly the same shape - numbers of rows (``y``) and columns
          (``x``) as the configured square ``Grid``.

        Args:
            value: An input DataArray to check
            grid: A Grid object giving the spatial configuration of the simulation.
            kwargs: Other configuration details to be used.

        Raises:
            ValueError: when ``x`` and ``y`` values are not congruent with the ``Grid``.

        Returns:
            A DataArray with the ``x`` and ``y`` dimensions remapped onto the internal
            ``cell_id`` dimension used in the :mod:`~virtual_rainforest.core.grid`
            module.
        """
        # Otherwise the data array must be the same shape as the grid
        if grid.cell_nx != value.sizes["x"] or grid.cell_ny != value.sizes["y"]:
            raise ValueError("Data XY dimensions do not match square grid")

        # Use DataArray.stack to combine the x and y into a multiindex called cell_id,
        # with x varying fastest (cell_id goes from top left to top right, then down by
        # rows), and then use these stacked indices to map the 2D onto grid cell order,
        # using isel() to avoid issues with dimension ordering.
        darray_stack = value.stack(cell_id=("y", "x"))

        return value.isel(
            x=DataArray(darray_stack.coords["x"].values, dims=["cell_id"]),
            y=DataArray(darray_stack.coords["y"].values, dims=["cell_id"]),
        )


# -*- coding: UTF-8 -*-
# vim: set et ai ci sm tw=78 si sw=4 ru filetype=python fileencoding=utf-8 :

import readline
from funcs import *
from common import *

# ------------------------------------------------------------------------
# specify addresses manually
def manual(cache):
    if C.number:
        print(lookup(cache, C.number))
    else:
        question = ""
        while question != "quit":
            question = input("key to lookup ('quit' to stop): ")
            if question == "quit":
                break
            print(lookup(cache, question))


# -*- coding: utf-8 -*-
"""
Created on Sat Jul 14 23:09:23 2018

@author: Akshay Jagadale
"""

class Solution:
    def transpose(self, A):
        """
        :type A: List[List[int]]
        :rtype: List[List[int]]
        """

        res = []
        for i in range(0,len(A[0])):
            temp = []
            for j in range(0,len(A)):
                temp.append(A[j][i])
            res.append(temp)
        return(res)

myobj =  Solution()
print(myobj.transpose([[1,2,3],[4,5,6]]))

Leet Code Problems/867. Transpose Matrix.py


import sys
import pygame

from pygame.locals import *

from local import *

def terminal():
    """
    定义终止函数，退出pygame，退出程序
    :return:
    """
    pygame.quit()  # 退出pygame
    sys.exit()

def show_splash_page():
    """
    首界面显示，不停循环
    :return:
    """
    # 加载snake文字
    snake_surface = bigFont.render('Snake', True, GREEN)
    # 获取矩形
    snake_rect = snake_surface.get_rect()
    print(snake_rect)  # <rect(0, 0, 276, 129)> 左上角的坐标为 0，0
    # 背景界面
    window.fill(BGCOLOR)
    # 展示snake, blit 传图
    window.blit(snake_surface, (100, 100))  # 表示左上角的坐标
    # 刷新界面
    pygame.display.flip()
    while True:
        # get events from the queue，获取事件列表(获取事件的两种写法)
        # event_list = pygame.event.get()
        # for event in event_list:
        #     if event.type == QUIT:  # pygame.locals
        #         terminal()
        # 获取退出事件（第二种写法）
        event = pygame.event.get(QUIT)
        if event:
            terminal()

window = pygame.display.set_mode((WINDOWWIDTH, WINDOWHEIGHT))
def start():
    # 全局变量区, 但是必须首先调用此函数，首界面窗口可以不用刷新显示，global表示可以修改
    global window, bigFont, smallFont
    # 初始化pygame
    pygame.init()
    # 创建窗口，This function will create a display Surface.
    window = pygame.display.set_mode((WINDOWWIDTH, WINDOWHEIGHT))
    # 定义大文字
    bigFont = pygame.font.Font('resources/ARBERKLEY.ttf', 100)
    # 中号字体
    centerFont = pygame.font.Font('resources/ARBERKLEY.ttf', 50)
    # 定义小文字
    smallFont = pygame.font.Font('resources/ARBERKLEY.ttf', 18)
    # 定义小的文字
    # 游戏引擎，显示首界面
    show_splash_page()
    # 显示游戏界面
    # 显示游戏结束界面

if __name__ == '__main__':
    start()


from django.contrib import admin
from django.urls import path, include

urlpatterns = [
    path('admin/', admin.site.urls),
    path('', include('QueryService.urls'))
]

admin.site.site_header = "SCVM Admin"
admin.site.site_title = "Administration"
admin.site.index_title = "SCVM Administration Page"


from adapter import urllib2_oauth
import compat
import util

class Client(object):
    request_token_url = None
    authorize_url = None
    access_token_url = None

    def __init__(self, client, token=None,
                 adapter=urllib2_oauth.Adapter, **fetch_parameters):
        self.adapter = adapter
        self.fetch_parameters = dict(client=client, token=token,
                                     **fetch_parameters)

    def set_token(self, token):
        self.fetch_parameters['token'] = token

    def fetch(self, *args, **kwargs):
        raise_errs = kwargs.pop('raise_http_errors', False)
        kwargs = dict(self.fetch_parameters, **kwargs)
        resp = self.adapter.fetch(*args, **kwargs)
        if str(resp.code).startswith('2') or not raise_errs:
            return resp
        else:
            raise ClientError('HTTP error %s: %r' % (resp.code, str(resp)))

    def get_authorize_url(self, authorize_url=None, request_token_url=None,
                          **kwargs):
        """Return a redirect-ready authorize url and temporary credentials"""

        url = request_token_url or self.request_token_url
        if not url:
            raise ValueError('request_token_url')

        auth_url = authorize_url or self.authorize_url
        if not auth_url:
            raise ValueError('authorize_url')

        kwargs.setdefault('method', 'POST')
        if 'oauth_callback' not in self.fetch_parameters:
            kwargs.setdefault('oauth_callback', 'oob')

        resp = self.fetch(url, **kwargs)

        temp = util.Credentials.build(str(resp))

        auth_url = util.url_extend(
            auth_url, dict(oauth_token=temp.identifier))

        return auth_url, temp

    def get_access_token(self, temporary_credentials, verifier,
                         access_token_url=None, **kwargs):

        url = access_token_url or self.access_token_url
        if not url:
            raise ValueError('access_token_url')

        kwargs['token'] = util.Credentials.build(temporary_credentials)

        if 'oauth_verifier=' in verifier:
            verifier = dict(
                compat.parse_qsl(verifier.split('?')[-1]))['oauth_verifier']
        elif hasattr(verifier, 'get'):
            verifier = verifier.get('oauth_verifier')
        kwargs['oauth_verifier'] = verifier

        resp = self.fetch(url, **kwargs)

        return util.Credentials.build(str(resp))

class ClientError(Exception):
    pass


# -*- coding: utf-8 -*-
"""
Created on Tue Feb  1 17:28:00 2022

Context features: Features related to the group of points being clustered.
Includes information like the number of points being clustered, measures of
dispersion, and relative size of each point within the group of points

Peritem features: Features related to our question, which is 'which set of
clustering algorithms and indexes will perform best on this group of points'.
See HYPERPARAMETER_LIST which lists a chosen set of clustering algorithms
(called 'hyperparameters' below) which are candidates for performing clustering
on a group of points

# Legacy information
_MODEL4_DIRECTORY = './final_model/Run_20191024002109model4/1572051525'
# I do not know what this file is for, or what it does
HYPERPARAMETER_SERVING_FILEPATH = r'../data/serving_hyperparameters.dat'

@author: jvorsten
"""

# Python imports
import configparser
import pickle
from typing import Dict, Union, Iterable, List
import base64

# Third party imports
import tensorflow as tf
import requests
from pydantic import BaseModel
from fastapi import FastAPI
import uvicorn
import numpy as np

# Local imports

# Declarations
"""Peritem clusterer hyperparameters used for prediction
# Return these with the output prediction. User will be able
# To rank clusterer hyperparameters based on prediction"""
DEFAULT_PERITEM_FEATURES_FILE = r'./default_serving_peritem_features.dat'
with open(DEFAULT_PERITEM_FEATURES_FILE, 'rb') as f:
    # Import from file
    HYPERPARAMETER_LIST = pickle.load(f)

LIST_SIZE_MODEL4 = 200

# Serving configuration
config = configparser.ConfigParser()
config.read(r'./serving_config.ini')
RANKING_MODEL_URL = config['ranking']['RANKING_MODEL_URL']

# FastAPI application
app = FastAPI(title=config['FastAPI']['title'],
              description=config['FastAPI']['description'],
              version=config['FastAPI']['version'],
              )

#%%

class DatabaseFeatures:
    n_instance:int
    n_features:int
    len_var:float
    uniq_ratio:float
    n_len1:float
    n_len2:float
    n_len3:float
    n_len4:float
    n_len5:float
    n_len6:float
    n_len7:float

def _bytes_feature(value):
  """Returns a bytes_list from a string / byte."""
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))

def _float_feature(value):
  """Returns a float_list from a float / double."""
  return tf.train.Feature(float_list=tf.train.FloatList(value=value))

def _int64_feature(value):
  """Returns an int64_list from a bool / enum / int / uint."""
  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))

def tf_feature_mapper(value, dtype=None):
    # Float -> tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
    # int -> tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
    # Create a dictionary mapper to map the function to data type
    dict_mapper = {int:_int64_feature,
                   float:_float_feature,
                   np.float64:_float_feature,
                   bytes:_bytes_feature,
                   np.bytes_:_bytes_feature,
                   bool:_int64_feature}

    # Make sure all list items are same dtype
    if hasattr(value, '__iter__'):
        if dtype:
            expected_type = dtype
        else:
            expected_type = type(value[0])
        all_same = all(isinstance(item, expected_type) for item in value)

        if not all_same: # Cast as the first type in iterable
            try:
                value = [expected_type(item) for item in value]
            except ValueError:
                raise ValueError('Cannot cast all items in passed list to \
                                 same type')

        return dict_mapper[expected_type](value)

    # Handle single values (not list)
    else:
        if dtype:
            expected_type = dtype
        else:
            expected_type = type(value)

        return dict_mapper[expected_type]([value])

    return None

def serialize_context_from_dictionary(context_features: Dict[str, Union[int,float]]):
    """Create a serialized tf.train.Example proto from a dictionary of context
        features
    input
    -------
    context_features: (dict) Must contain the keys
        ['n_instance', 'n_features', 'len_var', 'uniq_ratio',
         'n_len1', 'n_len2', 'n_len3', 'n_len4', 'n_len5',
         'n_len6', 'n_len7']
    output
    ------
    context_proto_str: (bytes) serialized context features in tf.train.Example
        proto

    the context_feature spec is of the form
    {'n_instance': FixedLenFeature(shape=(1,), dtype=tf.float32, default_value=(0.0,)),
     'n_features': FixedLenFeature(shape=(1,), dtype=tf.float32, default_value=(0.0,)),
     'len_var': FixedLenFeature(shape=(1,), dtype=tf.float32, default_value=(0.0,)),
     'uniq_ratio': FixedLenFeature(shape=(1,), dtype=tf.float32, default_value=(0.0,)),
     'n_len1': FixedLenFeature(shape=(1,), dtype=tf.float32, default_value=(0.0,)),
     'n_len2': FixedLenFeature(shape=(1,), dtype=tf.float32, default_value=(0.0,)),
     'n_len3': FixedLenFeature(shape=(1,), dtype=tf.float32, default_value=(0.0,)),
     'n_len4': FixedLenFeature(shape=(1,), dtype=tf.float32, default_value=(0.0,)),
     'n_len5': FixedLenFeature(shape=(1,), dtype=tf.float32, default_value=(0.0,)),
     'n_len6': FixedLenFeature(shape=(1,), dtype=tf.float32, default_value=(0.0,)),
     'n_len7': FixedLenFeature(shape=(1,), dtype=tf.float32, default_value=(0.0,))
     }"""

    # Create dictionary of context features
    context_dict = {}
    enforce_order = {'n_instance', 'n_features', 'len_var', 'uniq_ratio',
                     'n_len1', 'n_len2', 'n_len3', 'n_len4', 'n_len5',
                     'n_len6', 'n_len7'}

    # Ensure integrity of context_features
    if enforce_order != set(context_features.keys()):
        msg="Invalid key {} included in context_features"\
            .format(enforce_order.difference(context_features.keys()))
        raise KeyError(msg)

    # tf.train.Feature for each value passed in context_features
    for key in enforce_order:
        # Serialize context features
        val = context_features[key] # Enforce float
        context_dict[key] = tf_feature_mapper(float(val), dtype=float)

    # Create serialized context tf.train.Example
    context_proto = tf.train.Example(
            features=tf.train.Features(feature=context_dict))
    context_proto_str = context_proto.SerializeToString() # Serialized

    return context_proto_str

def serialize_examples_model4(hyperparameter_list: Iterable[Dict[str,str]],
                              list_size: int):
    """
    Create a list of serialized tf.train.Example proto.
    Use this for serving & predicting models. Not used for training; the
    "relevance" label is not included.
    Per-item example features are chosen as a fixed-set of clustering
    hyperparameters based on list_size; they are not based on how each document
    was actually clustered.
    peritem features are generated from a constant list of clusterer
    hyperparameters.

    input
    -------
    hyperparameter_list: (iterable of dictionaries) each dictionary defining a
    set of clustering 'hyperparameters'. Each dictionary of hyperparameters
    defines a group of inputs used to cluster a group of objects. Example
        [{'by_size': 'False',
          'n_components': '8',
          'reduce': 'MDS',
          'clusterer': 'kmeans',
          'index': 'optk_MDS_gap_Tib'},
         {'by_size': 'True',
          'n_components': '0',
          'reduce': 'False',
          'clusterer': 'kmeans',
          'index': 'optk_X_gap*_max'}, {...}]
    list_size: (int) Number of examples to include. This must be determined
        from the input size of the ranking model. Optionally, pass False
        if you do not want peritem examples to be padded

    output
    -------
    peritem_list: (list) a list of serialized per-item (exmpale) featuers
      in the form of tf.train.Example protos
    """

    peritem_features = [] # For iterating through and saving
    for hyperparameters in hyperparameter_list:
        single_item_feature_dict = {} # Append to list

        # Append all features to peritem features
        by_size = str(hyperparameters['by_size'])
        clusterer = str(hyperparameters['clusterer'])
        index = str(hyperparameters['index'])
        n_components = str(hyperparameters['n_components'])
        reduce = str(hyperparameters['reduce'])

        single_item_feature_dict['by_size'] = by_size.encode()
        single_item_feature_dict['n_components'] = n_components.encode()
        single_item_feature_dict['reduce'] = reduce.encode()
        single_item_feature_dict['clusterer'] = clusterer.encode()
        single_item_feature_dict['index'] = index.encode()
        peritem_features.append(single_item_feature_dict)

    def _pad(peritem_features, pad_value=b''):
        # Pad with empty string
        cur_size = len(peritem_features)
        new_size = list_size

        example_keys = list(peritem_features[0].keys())
        blank_exmaple_dict = {}
        for key in example_keys:
            blank_exmaple_dict[key] = pad_value

        for i in range(new_size-cur_size):
            peritem_features.append(blank_exmaple_dict)

        return peritem_features

    def _trim(peritem_features):
        # Return the first _list_size of peritem_features
        new_size = list_size
        trimmed_peritem_features = peritem_features[:new_size]
        return trimmed_peritem_features

    if bool(list_size): # Only pad if list_size is defined
        if (list_size > len(peritem_features)):
            peritem_features = _pad(peritem_features)
        elif (list_size < len(peritem_features)):
            peritem_features = _trim(peritem_features)

    # Create a list of serialized per-item features
    peritem_list = []

    for single_item_feature_dict in peritem_features:
        peritem_dict = {}

        # Add peritem features to dictionary
        peritem_dict['by_size'] = tf_feature_mapper(
                [single_item_feature_dict['by_size']])
        peritem_dict['n_components'] = tf_feature_mapper(
                [single_item_feature_dict['n_components']])
        peritem_dict['reduce'] = tf_feature_mapper(
                [single_item_feature_dict['reduce']])
        peritem_dict['clusterer'] = tf_feature_mapper(
                [single_item_feature_dict['clusterer']])
        peritem_dict['index'] = tf_feature_mapper(
                [single_item_feature_dict['index']])

        # Create EIE and append to serialized peritem_list
        peritem_proto = tf.train.Example(
                features=tf.train.Features(feature=peritem_dict))
        peritem_proto_str = peritem_proto.SerializeToString() # Serialized

        # List of serialized tf.train.Example
        peritem_list.append(peritem_proto_str)

    return peritem_list

def serialize_example_in_example(database_features: Dict[str, float],
                                 hyperparameter_list: Iterable[Dict[str,str]],
                                 list_size: int):

    #1. Context features (bytes object)
    serialized_context = serialize_context_from_dictionary(database_features)

    #2. Peritem features (bytes object)
    serialized_peritem = serialize_examples_model4(
        hyperparameter_list,
        list_size=list_size)

    # Prepare serialized feature spec for EIE format
    serialized_dict = {'serialized_context':_bytes_feature([serialized_context]),
                       'serialized_examples':_bytes_feature(serialized_peritem)
                       }

    # Convert to tf.train.Example object
    serialized_proto = tf.train.Example(
            features=tf.train.Features(feature=serialized_dict))
    serialized_example_in_example = serialized_proto.SerializeToString()

    return serialized_example_in_example

def cached_serialize_example_in_example(database_features: Dict[str, float],
                                        serialized_peritem: bytes):
    """Unfortunately, the HYPERPARAMETER_LIST object is not hashable
    unless I convert each dictionary to a static dictionary, and the list
    to a vector (lru caching requires hashable objects)"""

    #1. Context features (bytes object)
    serialized_context = serialize_context_from_dictionary(database_features)

    #2. Peritem features (bytes object)
    # Should be constant - cache the results for faster execution

    # Prepare serialized feature spec for EIE format
    serialized_dict = {'serialized_context':_bytes_feature([serialized_context]),
                       'serialized_examples':_bytes_feature(serialized_peritem)
                       }

    # Convert to tf.train.Example object
    serialized_proto = tf.train.Example(
            features=tf.train.Features(feature=serialized_dict))
    serialized_example_in_example = serialized_proto.SerializeToString()

    return serialized_example_in_example

def get_top_n_hyperparameters(n: int,
                              hyperparameter_list: List[Dict[str,str]],
                              scores: List[float]):
    """Return the top 5 hyperparameter sets based on a list of scores
    inputs
    -------
    hyperparameter_list: (list of dictionary) each entry """
    idx = sorted(range(len(scores)), key=scores.__getitem__)

    top_n_hyperparameters = []
    worst_n_hyperparameters = []
    for i in range(n):
        # Top n predictions
        top_n_hyperparameters.append(HYPERPARAMETER_LIST[idx[-i]])
        # Worst n predictions
        worst_n_hyperparameters.append(HYPERPARAMETER_LIST[idx[i]])

    return top_n_hyperparameters, worst_n_hyperparameters

#%% FastAPI declaration

SERIALIZED_PERITEM = serialize_examples_model4(
    HYPERPARAMETER_LIST,
    list_size=LIST_SIZE_MODEL4)

class RawInputData(BaseModel):
    """Raw input data from HTTP Web form
    inputs description
    -------
    n_instance: number of instances within data
    n_features: number of features per instance
    len_var: variance of lenght of categorical text features
    uniq_ratio: ratio of (number of instances / number of features)
    n_len1: number features with a 1-length word after tokenization of its text feature
    n_len2: number features with a 2-length word after tokenization of its text feature
    n_len3: ...
    n_len4:
    n_len5:
    n_len6:
    n_len7:
    """
    n_instance:float
    n_features:float
    len_var:float
    uniq_ratio:float
    n_len1:float
    n_len2:float
    n_len3:float
    n_len4:float
    n_len5:float
    n_len6:float
    n_len7:float

class ClusteringHyperparameters(BaseModel):
    """Contents of HYPERPARAMETER_LIST
    """
    by_size:str
    n_components:str
    reduce:str
    clusterer:str
    index:str

class PredictorOutput(BaseModel):
    """We will choose to output the top 5, and worst clustering hyperparameters
    predicted based on your database context features
    each output dictionary contains the keys
    {'by_size':str,'n_components':str,'reduce':str,'clusterer':str,'index':str}
        """
    top_n: List[ClusteringHyperparameters]
    worst_n: List[ClusteringHyperparameters]

@app.on_event("startup")
async def startup():
    """How could I use this to add objects to the application state?
    app.state.PredictionObject = load_all_of_my_predictors()
    Then use app.state.PredictionObject in all of the endpoints?
    Access the app object by importing Request from fastapi, and adding
    it as a parameter in each endpoint like
    async def endpoint(request: Request, data_model:DataModel):"""
    pass

@app.get("/")
async def root():
    msg=("See API endpoints at /clustering-ranking/model4predict")
    return {"message":msg}

@app.post("/clustering-ranking/model4predict/", response_model=PredictorOutput)
async def clustering_ranking_model4(database_features: RawInputData):
    """Serve predictions from the CompNBPredictor"""

    # Serialize input context features into an example-in-example
    serialized_example_in_example = cached_serialize_example_in_example(
        dict(database_features),
        SERIALIZED_PERITEM)

    # Tensorflow serving requires
    json_data = {
      "signature_name":"serving_default",
      "instances":[
          {"b64":base64.b64encode(serialized_example_in_example)\
                       .decode('utf-8')}
              ]
          }

    # Response is a list of predicitons {'predictions': [[float,float,...]]}
    resp = requests.post(RANKING_MODEL_URL, json=json_data)
    relevancies = resp.json()['predictions'][0]
    top_5_hyperparameters, worst_5_hyperparameter = \
        get_top_n_hyperparameters(5, HYPERPARAMETER_LIST, relevancies)

    return PredictorOutput(top_n=top_5_hyperparameters,
                           worst_n=worst_5_hyperparameter)

#%%
if __name__ == "__main__":
    uvicorn.run(
        app,
        host="127.0.0.1",
        port=config['FastAPI']['SERVE_PORT']
        )


"""
This file is built to handle inventories and their movement

The Inventory class will handle items, extending a list essentially,
allowing items (another object, to be pulled out of the inventory and between inventories
"""
import GamePiece as gp

class Item:
    """
    Base item class, mostly to handle moving items around for inventories
    should be inheirited for specific types of items? most likely
    """
    def __init__(self,name, base_identifier, amount=1,allow_stack=True):
        self.name = name
        self.amount = amount
        self.base_identifier = base_identifier
        self.allow_stack = allow_stack

    def rename(self,name: str):
        """
        rename an item
        :param name: new name for item
        :return: updates item.name
        """
        self.name = name

class Inventory:
    """
    This class will contain a list of items that can be moved
    """
    def __init__(self):
        self.inventory = []

    def get_inventory(self,named=True):
        """
        Returns the list of items in it's inventory
        :param:named: if return should return a list of item objects, or names
        :return: either list of item objects, or list of names of the objects
        """
        # iterate through all items, append to out variable
        out = []
        for item in self.inventory:
            if named:
                value = item.name
            else:
                value = item

            out.append(value)

        return out

    def get_inv_ids(self):
        """
        returns a list of inventory unique id's
        :return: list of unique ids
        """
        out = []
        for item in self.inventory:
            out.append(item.base_identifier)

        return out

    def add_item(self,items: list):
        """
        add item to inventory, stack if possible
        :param items: list of items to add to inventory
        :return: nothing, updates self.inventory
        """
        # get list of id's
        item_ids = self.get_inv_ids()

        for item in items:
            # first check if similar item is in inventory
            if item.base_identifier in item_ids:
                # check for stacking

                # get index of match
                item_index = item_ids.index(item.base_identifier)

                #get item from index
                matched_item = self.inventory[item_index]

                can_stack = matched_item.allow_stack
                if can_stack:
                    # add amount to stack
                    matched_item.amount += item.amount
                    if matched_item.amount <= 0:
                        self.inventory.remove(matched_item)
                # for the case where the id is there, but cannot stack, no item is added

            else:
                # new item, add to inventory and add it's base identifier to the list, if copies
                # will be added to, like multiple
                self.inventory.append(item)
                item_ids.append(item.base_identifier)
                if item.amount <= 0:
                    self.inventory.remove(item)

class Pile(gp.Piece):
    """
    Pile of items gamepiece, will have a location and items contained within it
    """
    def __init__(self,map, point, color, char = '/',inventory=[]):
        """
        init inventory and super init game piece
        :param map: map piece is on
        :param point: tuple where piece is at
        :param color: color of character
        :param char: character to represent pile
        :param inventory: list of Items in the pile
        """
        # create inventory and add items to it
        self.inventory = Inventory()
        self.inventory.add_item(inventory)
        super().__init__(map,point,color=color,char=char)
        # check if inventory is empty
        # if empty, mark as false, otherwise true
        self.is_empty = bool(self.inventory)

def get_test_items():
    """
    build a list of test items
    :return: list of items for testing
    """
    out = []
    out.append(Item('potion','potion_base'))
    out.append(Item('gold','gold',))
    out.append(Item('sword','sword_a',allow_stack=False))
    return out[0],out[1],out[2]


'''Author: Shawon Ashraf'''

import re
import os

import numpy as np

"""
Clean punctuation marks, special characters and short forms from a sentence
param s - a sentences as a string

Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
"""

def clean_str(s):
    """Clean sentence"""
    s = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", s)
    s = re.sub(r"\'s", " \'s", s)
    s = re.sub(r"\'ve", " \'ve", s)
    s = re.sub(r"n\'t", " n\'t", s)
    s = re.sub(r"\'re", " \'re", s)
    s = re.sub(r"\'d", " \'d", s)
    s = re.sub(r"\'ll", " \'ll", s)
    s = re.sub(r",", " , ", s)
    s = re.sub(r"!", " ! ", s)
    s = re.sub(r"\(", " \( ", s)
    s = re.sub(r"\)", " \) ", s)
    s = re.sub(r"\?", " \? ", s)
    s = re.sub(r"\s{2,}", " ", s)
    s = re.sub(r'\S*(x{2,}|X{2,})\S*', "xxx", s)
    s = re.sub(r'[^\x00-\x7F]+', "", s)
    return s.strip().lower()

"""
load embedding weights from a pre-trained embedding set
We are using Glove embeddings here.

Takes the model name, the vocabulary dictionary, max word length and the embedding dimensions
as params

returns word -> weight mappings as a matrix (max word length, embedding dim)

allowed embedding dims = 50, 100, 200, 300

https://github.com/lukas/ml-class/blob/master/videos/cnn-text/imdb-embedding.py
"""

def load_embedding_weights_as_matrix(vocabulary, max_len_words, embedding_dims):
    model_dir = 'models'
    model_name = f"retrofitted.txt"
    model_path = os.path.join(model_dir, model_name)

    # word -> embedding coefficient mapping
    # so that we can get weights by words
    embedding_dict = dict()

    # load from file
    if os.path.exists(model_path):
        print(f"Loading model -> {model_name}")

        with open(model_path, encoding='utf-8') as f:
            for line in f:
                values = line.split()
                word = values[0]
                coefs = np.asarray(values[1:], dtype='float32')
                embedding_dict[word] = coefs

    # if the file doesn't exist, download it
    else:
        print("Model doesn't exist, downloading .....")
        # TODO - implement a function to download the model

    # weight matrix for words
    # from word -> weight lookup
    embedding_matrix = np.zeros((max_len_words, embedding_dims))
    for word, index in vocabulary.items():
        if index > max_len_words - 1:
            break
        else:
            embedding_vector = embedding_dict.get(word)
            if embedding_vector is not None:
                embedding_matrix[index] = embedding_vector

    return embedding_matrix

"""
Get text and labels separated from a list of EmoteInstance
From the corpus
"""

def get_text_and_labels(data):
    texts = [emo.text for emo in data]
    labels = [emo.label for emo in data]

    cleaned_texts = [clean_str(s) for s in texts]

    return cleaned_texts, labels


'''
1) Question check if number is a prime and it's first bit is set

for checking the bit position bin() function is used this is more effective than dividing
by 2 for the number of bit postions because each bit position is known by just passing
position as argument.

'''

def is_prime(integer):
    for number in range(2,integer//2+1):
        if integer % number == 0:
            return False
    return True

def is_bit_position_set(integer,position):
    bin_val = str(bin(integer))
    if bin_val[-(position+1)] == '1':
        return True
    return False

list1 = [1,7,13,17]
for element in list1:
    if is_prime(element) and is_bit_position_set(element,1):
        print('yes')
    else:
        print('no')


"""Core functionalities of the Eddington platform."""
from eddington.exceptions import (
    EddingtonException,
    FittingDataColumnExistenceError,
    FittingDataColumnIndexError,
    FittingDataColumnsLengthError,
    FittingDataError,
    FittingDataInvalidFile,
    FittingDataRecordsSelectionError,
    FittingFunctionLoadError,
    FittingFunctionRuntimeError,
)
from eddington.fitting import fit
from eddington.fitting_data import FittingData
from eddington.fitting_function_class import FittingFunction, fitting_function
from eddington.fitting_functions_list import (
    constant,
    cos,
    exponential,
    hyperbolic,
    inverse_power,
    linear,
    normal,
    parabolic,
    poisson,
    polynomial,
    sin,
    straight_power,
)
from eddington.fitting_functions_registry import FittingFunctionsRegistry
from eddington.fitting_result import FittingResult
from eddington.plot.figure_builder import FigureBuilder
from eddington.plot.plot_legacy import (
    add_errorbar,
    add_grid,
    add_legend,
    add_plot,
    get_figure,
    horizontal_line,
    label_axes,
    plot_data,
    plot_fitting,
    plot_residuals,
    title,
)
from eddington.plot.plot_util import build_repr_string, show_or_export
from eddington.print_util import to_relevant_precision_string
from eddington.random_util import random_data

__version__ = "0.0.24.dev2"

__all__ = [
    "__version__",
    # Fitting functions infrastructure
    "FittingFunction",
    "fitting_function",
    "FittingFunctionsRegistry",
    # Fitting functions
    "constant",
    "exponential",
    "hyperbolic",
    "linear",
    "parabolic",
    "polynomial",
    "cos",
    "sin",
    "straight_power",
    "inverse_power",
    "normal",
    "poisson",
    # Fitting algorithm
    "fit",
    # Exceptions
    "EddingtonException",
    "FittingFunctionRuntimeError",
    "FittingFunctionLoadError",
    "FittingDataError",
    "FittingDataColumnExistenceError",
    "FittingDataColumnIndexError",
    "FittingDataInvalidFile",
    "FittingDataColumnsLengthError",
    "FittingDataRecordsSelectionError",
    # Data structures
    "FittingData",
    "FittingResult",
    # Plot
    "FigureBuilder",
    "show_or_export",
    "build_repr_string",
    # Plot legacy
    "plot_data",
    "plot_fitting",
    "plot_residuals",
    "get_figure",
    "add_errorbar",
    "add_legend",
    "add_grid",
    "add_plot",
    "horizontal_line",
    "label_axes",
    "title",
    # Print
    "to_relevant_precision_string",
    # Miscellaneous
    "random_data",
]


"""code from Pearson Education, Inc p104 """

class BinaryTree:
    def __init__(self):
        self.root = None
        self.size = 0

    # Return True if the element is in the tree
    def search(self, e):
        current = self.root # Start from the root

        while current != None:
            if e < current.element:
                current = current.left
            elif e > current.element:
                current = current.right
            else: # element matches current.element
                return True # Element is found

        return False

    # Insert element e into the binary search tree
    # Return True if the element is inserted successfully
    def insert(self, e):
        if self.root == None:
          self.root = self.createNewNode(e) # Create a new root
        else:
          # Locate the parent node
          parent = None
          current = self.root
          while current != None:
            if e < current.element:
              parent = current
              current = current.left
            elif e > current.element:
              parent = current
              current = current.right
            else:
              return False # Duplicate node not inserted

          # Create the new node and attach it to the parent node
          if e < parent.element:
            parent.left = self.createNewNode(e)
          else:
            parent.right = self.createNewNode(e)

        self.size += 1 # Increase tree size
        return True # Element inserted

    # Create a new TreeNode for element e
    def createNewNode(self, e):
      return TreeNode(e)
    """
    # Return the size of the tree
    def getSize(self):
      return self.size"""

    # Inorder traversal from the root
    def inorder(self):
      self.inorderHelper(self.root)

    # Inorder traversal from a subtree
    def inorderHelper(self, r):
      if r != None:
        self.inorderHelper(r.left)
        print(r.element, end = " ")
        self.inorderHelper(r.right)

    # Postorder traversal from the root
    def postorder(self):
      self.postorderHelper(self.root)

    # Postorder traversal from a subtree
    def postorderHelper(self, root):
      if root != None:
        self.postorderHelper(root.left)
        self.postorderHelper(root.right)
        print(root.element, end = " ")

    # Preorder traversal from the root
    def preorder(self):
      self.preorderHelper(self.root)

    # Preorder traversal from a subtree
    def preorderHelper(self, root):
      if root != None:
        print(root.element, end = " ")
        self.preorderHelper(root.left)
        self.preorderHelper(root.right)

    # Return true if the tree is empty
    def isEmpty(self):
      return self.size == 0

    # Remove all elements from the tree
    def clear(self):
      self.root == None
      self.size == 0

    # Return the root of the tree
    def getRoot(self):
      return self.root

    def PrintLeafLeftToRight(self):
        root = BinaryTree.getRoot(self)
        s1 = []
        # Stack to store all the
        # leaf nodes
        s2 = []
    # Push the root node
        s1.append(root)
        while len(s1) != 0:
            curr = s1.pop()
        # If current node has a left child
        # push it onto the first stack
            if curr.left:
                s1.append(curr.left)
        # If current node has a right child
        # push it onto the first stack
            if curr.right:
                s1.append(curr.right)
        # If current node is a leaf node
        # push it onto the second stack
            elif not curr.left and not curr.right:
                s2.append(curr)
    # Print all the leaf nodes
        while len(s2) != 0:
            print(s2.pop().element, end = " ")

class TreeNode:
    def __init__(self, e):
      self.element = e
      self.left = None # Point to the left node, default None
      self.right = None # Point to the right node, default None

# Python code t get difference of two lists
# Not using set()
def Diff(li1, li2):
    return (list(set(li1) - set(li2)))

# Driver Code
    ####################### Main test binary tree

def main(size = 7):
    tree = BinaryTree()
    tree.insert("George")
    tree.insert("Michael")
    tree.insert("Tom")
    tree.insert("Adam")
    tree.insert("Jones")
    tree.insert("Peter")
    tree.insert("Daniel")

    # Traverse tree
    print("Inorder (sorted): ", end = "")
    tree.inorder()
    print("\nPostorder: ", end = "")
    tree.postorder()
    print("\nPreorder: ", end = "")
    tree.preorder()

    numbers = input("Enter family members separated by comma ")
    li1  = numbers.split(",")

    print ("\n\nInserting the following values:")
    for i in numbers:
        print(i, end=" ")
    print()
    intTree = BinaryTree()

    for e in numbers:
      intTree.insert(e)
    print("\nPreorder traversal:")
    intTree.preorder()
    print("\n\nInorder traversal:")
    intTree.inorder()
    print("\n\nPostorder traversal:")
    intTree.postorder()
    print("\n\nLeaf Nodes Only:")
    intTree.PrintLeafLeftToRight()
    print("\n\nNonLeafNodes:")

    #print("\n\nNon-Leaf Nodes:")
    #intTree.findFullNode()

if __name__ == "__main__":
    main()


import os
import sys
import xml.dom.minidom
from data_structure import ResultStructure, Table, AdjRelation
from os.path import join as osj

class eval:
    STR = "-str"
    REG = "-reg"
    DEFAULT_ENCODING = "UTF-8"

    def __init__(self, res_path, gt_path, cell_ious=None):
        if cell_ious is None:
            cell_ious = [0.5, 0.6]
        self.cell_ious = cell_ious

        self.return_result = None
        self.reg = False
        self.str = False

        self.resultFile = res_path
        self.inPrefix = os.path.split(res_path)[-1].split(".")[0]

        self.str = True
        self.GTFile = osj(gt_path, self.inPrefix + ".xml")

        self.gene_ret_lst()

    @property
    def result(self):
        return self.return_result

    def gene_ret_lst(self):
        ret_lst = []
        for iou in self.cell_ious:
            temp = self.compute_retVal(iou)
            ret_lst.append(temp)

        ret_lst.append(self.inPrefix + ".xml")
        # print("Done processing {}\n".format(self.resultFile))
        self.return_result = ret_lst

    def compute_retVal(self, iou):
        gt_dom = xml.dom.minidom.parse(self.GTFile)
        # incorrect submission format handling
        try:
            result_dom = xml.dom.minidom.parse(self.resultFile)
        except Exception:
            # result_dom = xml.dom.minidom.parse(dummyDom)
            gt_tables = eval.get_table_list(gt_dom)
            retVal = ResultStructure(truePos=0, gtTotal=len(gt_tables), resTotal=0)
            return retVal

        # result_dom = xml.dom.minidom.parse(self.resultFile)
        if self.reg:
            ret = self.evaluate_result_reg(gt_dom, result_dom, iou)
            return ret
        if self.str:
            ret = self.evaluate_result_str(gt_dom, result_dom, iou)
            return ret

    @staticmethod
    def get_table_list(dom):
        """
        return a list of Table objects corresponding to the table element of the DOM.
        """
        return [Table(_nd) for _nd in dom.documentElement.getElementsByTagName("table")]

    @staticmethod
    def evaluate_result_reg(gt_dom, result_dom, iou_value):
        # parse the tables in input elements
        gt_tables = eval.get_table_list(gt_dom)
        result_tables = eval.get_table_list(result_dom)

        # duplicate result table list
        remaining_tables = result_tables.copy()

        # map the tables in gt and result file
        table_matches = []  # @param: table_matches - list of mapping of tables in gt and res file, in order (gt, res)
        for gtt in gt_tables:
            for rest in remaining_tables:
                if gtt.compute_table_iou(rest) >= iou_value:
                    remaining_tables.remove(rest)
                    table_matches.append((gtt, rest))
                    break

        assert len(table_matches) <= len(gt_tables)
        assert len(table_matches) <= len(result_tables)

        retVal = ResultStructure(truePos=len(table_matches), gtTotal=len(gt_tables), resTotal=len(result_tables))
        return retVal

    @staticmethod
    def evaluate_result_str(gt_dom, result_dom, iou_value, table_iou_value=0.8):
        # parse the tables in input elements
        gt_tables = eval.get_table_list(gt_dom)
        result_tables = eval.get_table_list(result_dom)

        gt_match_status = [False for table in gt_tables]
        rest_match_status = [False for table in result_tables]
        table_matches = []  # @param: table_matches - list of mapping of tables in gt and res file, in order (gt, res)
        for i in range(len(gt_tables)):
            gtt = gt_tables[i]
            for j in range(len(result_tables)):
                if rest_match_status[j]:
                    continue
                rest = result_tables[j]
                # note: for structural analysis, use 0.8 for table mapping
                if gtt.compute_table_iou(rest) >= table_iou_value:
                    table_matches.append((gtt, rest))
                    gt_match_status[i] = True
                    rest_match_status[j] = True
                    break

        gt_remaining, remaining_tables = [], []
        for i in range(len(gt_match_status)):
            if not gt_match_status[i]:
                gt_remaining.append(gt_tables[i])

        for i in range(len(rest_match_status)):
            if not rest_match_status[i]:
                remaining_tables.append(result_tables[i])

        total_gt_relation, total_res_relation, total_correct_relation = 0, 0, 0
        for gt_table, ress_table in table_matches:
            # set up the cell mapping for matching tables
            cell_mapping = gt_table.find_cell_mapping(ress_table, iou_value)
            # set up the adj relations, convert the one for result table to a dictionary for faster searching
            gt_AR = gt_table.find_adj_relations()
            total_gt_relation += len(gt_AR)

            res_AR = ress_table.find_adj_relations()
            total_res_relation += len(res_AR)

            # Now map GT adjacency relations to result
            lMappedAR = []
            for ar in gt_AR:
                try:
                    resFromCell = cell_mapping[ar.fromText]
                    resToCell = cell_mapping[ar.toText]
                    # make a mapped adjacency relation
                    lMappedAR.append(AdjRelation(resFromCell, resToCell, ar.direction))
                except Exception:
                    # no mapping is possible
                    pass

            # compare two list of adjacency relation
            correct_dect = 0
            for ar1 in res_AR:
                for ar2 in lMappedAR:
                    if ar1.isEqual(ar2):
                        correct_dect += 1
                        break

            total_correct_relation += correct_dect

        # handle gt_relations in unmatched gt table
        for gtt_remain in gt_remaining:
            total_gt_relation += len(gtt_remain.find_adj_relations())

        # handle gt_relation in unmatched res table
        for res_remain in remaining_tables:
            total_res_relation += len(res_remain.find_adj_relations())

        retVal = ResultStructure(truePos=total_correct_relation, gtTotal=total_gt_relation, resTotal=total_res_relation)
        return retVal

if __name__ == "__main__":
    cur_path = sys.argv[1]
    # print(cur_path)
    eval("-trackB2", cur_path)


from glad.lang.common.loader import BaseLoader

_EGL_LOADER = '''
int gladLoadEGL(void) {
    return gladLoadEGLLoader((GLADloadproc)eglGetProcAddress);
}
'''

_EGL_HEADER = '''
#ifndef __glad_egl_h_

#ifdef __egl_h_
#error EGL header already included, remove this include, glad already provides it
#endif

#define __glad_egl_h_
#define __egl_h_

#if defined(_WIN32) && !defined(APIENTRY) && !defined(__CYGWIN__) && !defined(__SCITECH_SNAP__)
#define APIENTRY __stdcall
#endif

#ifndef APIENTRY
#define APIENTRY
#endif
#ifndef APIENTRYP
#define APIENTRYP APIENTRY *
#endif
#ifndef GLAPI
#define GLAPI extern
#endif

#ifdef __cplusplus
extern "C" {
#endif

typedef void* (* GLADloadproc)(const char *name);
'''

_EGL_HEADER_LOADER = '''
GLAPI int gladLoadEGL(void);
'''

_EGL_HEADER_END = '''
#ifdef __cplusplus
}
#endif

#endif
'''

_EGL_HAS_EXT = '''
'''

class EGLCLoader(BaseLoader):
    def write(self, fobj):
        if not self.disabled:
            fobj.write(_EGL_LOADER)

    def write_begin_load(self, fobj):
        # suppress unused warnings
        fobj.write('\t(void) load;\n')

    def write_end_load(self, fobj):
        fobj.write('\treturn 1;\n')

    def write_find_core(self, fobj):
        pass

    def write_has_ext(self, fobj):
        fobj.write(_EGL_HAS_EXT)

    def write_header(self, fobj):
        fobj.write(_EGL_HEADER)
        if not self.disabled:
            fobj.write(_EGL_HEADER_LOADER)

    def write_header_end(self, fobj):
        fobj.write(_EGL_HEADER_END)


#!/usr/bin/python3
#
import sys, os, os.path, stat
import shutil, hashlib
import subprocess

import manipkg

def hash_file(pofile):
    md5 = hashlib.md5()
    sha256 = hashlib.sha256()
    with open(pofile, 'rb') as po:
        bytes = po.read()
    md5.update(bytes)
    sha256.update(bytes)
    return (md5.digest(), sha256.digest())

def add_deb(deb, isopool):
    pofile = isopool + '/' + os.path.basename(deb)
    shutil.copyfile(deb, pofile)
    debsiz = os.path.getsize(pofile)
    debdgst = hash_file(pofile)

    debdir = '/tmp/debpot'
    if os.path.exists(debdir):
        if os.path.isfile(debdir):
            os.remove(debdir)
        else:
            shutil.rmtree(debdir)

    extract = ['dpkg-deb', '--raw-extract', pofile, debdir]
    comp = subprocess.run(extract, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    if comp.returncode != 0:
        print("Cannot extract the deb: {}".format(pofile))
        return

    ctlfile = debdir + '/DEBIAN/control'
    if not os.path.isfile(ctlfile):
        print("Invalid deb file: {}")
        return

    with open(ctlfile, "r") as ctl:
        ctl_lines = ctl.readlines()
    shutil.rmtree(debdir)

    sort_ctl = []
    for ln in ctl_lines:
        if ln.find('Package: ') == 0:
            pkgname = ln.split()[1].rstrip('\n')
            sort_ctl.append(ln)
            continue
        if ln.find('Version: ') == 0:
            pkgver = ln.split()[1].rstrip('\n')
            sort_ctl.append(ln)
            continue
        if ln.find('Source: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Priority: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Section: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Maintainer: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Architecture: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Breaks: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Replaces: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Recommends: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Conflicts: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Homepage: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Pre-Depends: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Tag: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Multi-Arch: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Depends:') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Installed-Size: ') == 0:
            sort_ctl.append(ln)
            continue
        if ln.find('Description: ') == 0:
            sort_ctl.append(ln)
            continue
    sort_ctl.append('FileName: pool/lenvdi/' + os.path.basename(deb) + '\n')
    sort_ctl.append('Size: ' + str(debsiz) + '\n')
    sort_ctl.append('MD5sum: ' + debdgst[0].hex() + '\n')
    sort_ctl.append('SHA256: ' + debdgst[1].hex() + '\n')

    print("Process Package: {}".format(pkgname))
    with open('/tmp/Package', 'w') as pkgw:
        for ln in sort_ctl:
            pkgw.write(ln)
    return (pkgname, pkgver, sort_ctl)

if len(sys.argv) < 3:
    print("Usage: {} debdir isodir".format(sys.argv[0]))
    sys.exit(1)

debdir = sys.argv[1]
isotop = sys.argv[2]
arch = 'amd64'
if len(sys.argv) > 3:
    arch = sys.argv[3]

if not os.path.isdir(debdir):
    print("{} is not a directory!".format(debdir))
    sys.exit(2)
if not os.path.isdir(isotop):
    print("{} is not a directory!".format(isotop))
    sys.exit(2)
#diskinfo = isotop + '/.disk/info'
#if not os.path.isfile(diskinfo):
#    print("'{}' is not a top directory for Debian ISO".format(isotop))
#    sys.exit(3)

#with open(diskinfo, 'r') as info:
#    ln = info.readlines()[0]
#if ln.find('Debian GNU/Linux') == -1:
#    print("{} is not a top directory for Debian 10 ISO")
#    sys.exit(4)

fmode = stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH
isopool = isotop + '/pool/lenvdi'
if not os.path.isdir(isopool):
    os.chmod(isotop+'/pool', fmode)
    os.makedirs(isopool)
isodist = isotop + '/dists/lenvdi/main/binary-' + arch
if not os.path.isdir(isodist):
    os.chmod(isotop+'/dists', fmode)
    os.makedirs(isodist)
pkgfile = isodist + '/Packages'
if not os.path.isfile(pkgfile):
    open(pkgfile, 'w').close()

tmpf = "/tmp/Packages.tmp"
entries = os.listdir(debdir)
for entry in entries:
    debfile = debdir + '/' + entry
    if not os.path.isfile(debfile):
        continue
    pkginfo = add_deb(debfile, isopool)
    if pkginfo:
        manipkg.pack_remove(pkgfile, tmpf, pkginfo[0], pkginfo[1])
        pout = open(tmpf, "a")
        for ln in pkginfo[2]:
            pout.write(ln)
        pout.write('\n')
        pout.close()
        shutil.copyfile(tmpf, pkgfile)

sys.exit(0)


from django.shortcuts import render
# Create your views here.
from django.shortcuts import render
from django.http import HttpResponse
from pcapfile import savefile
from pcapfile.protocols.linklayer import ethernet
from pcapfile.protocols.network import ip
from pcapfile.protocols.transport import udp

import binascii

testcap = open("/home/manish/PycharmProjects/pcap/webapp/Test_1.pcap")
capfile = savefile.load_savefile(testcap, verbose=True)

def index(request):
                answer = []
                for packet in capfile.packets:
                    try:
                        udp_packet = None
                        ptype = " "
                        mac_packet = ethernet.Ethernet(packet.raw());
                        if mac_packet.type == 2048:
                            ip_packet = ip.IP(binascii.unhexlify(ethernet.Ethernet(packet.raw()).payload))
                            ptype = "IPv4"
                            if ip_packet.p == 17:
                                udp_packet = udp.UDP(binascii.unhexlify(ethernet.Ethernet(packet.raw()).payload))
                                ptype = "UDP"
                        else:
                                ptype = "unknown"
                        answer.extend([

                             str(packet.timestamp),   #timestamp
                             str(mac_packet.src),   #Sender Mac Address
                             str(mac_packet.dst),   #Receiver Mac Address
                             str(ip_packet.src),    #Sender IP
                             str(ip_packet.dst),    #Receiver IP
                             str(ip_packet.len)+ " " +"byte",    #Payload lenght with IP header
                           #  str(ip_packet),        #Print whole packet information

                             str(mac_packet.type),  #Protocol Type

                             str(len(capfile.packets)) + " " + "Packets",  # Lenght of Pcacp file
                             str(ptype),
                            ])

                        if udp_packet is not None:
                            answer.extend([
                                str(udp_packet.dst_port),
                                str(udp_packet.src_port)
                            ])
                        else:
                            answer.extend([
                                str(ptype)   #add port details of TCP packets
                            ])

                        answer.append("")

                    except:
                        import traceback
                        traceback.print_exc()

                return HttpResponse("<br>" . join(answer))


#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
#                                  Giangi Sacco
#                        NASA Jet Propulsion Laboratory
#                      California Institute of Technology
#                        (C) 2009  All Rights Reserved
#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

#!/usr/bin/env python
import os

Import('envformimage')
install = envformimage['PRJ_LIB_DIR']
listFiles = ['acpatch.F','rmpatch.F','rcpatch.F','intp_coef.F']
lib = envformimage.Library(target = 'formimage', source = listFiles)
envformimage.Install(install,lib)
envformimage.Alias('install',install)


#This class is used to store response objects
import apsw
from lib.resultObject import ResultObject
from colored import fg, bg, attr

class Database:

    #Initialize the database
    def __init__(self, dbfile):
        self.dbfile = dbfile
        self.conn = apsw.Connection(':memory:')
        self.cursor = self.conn.cursor()
        self.rs = attr('reset')

        #Create Database tables
        create = 'CREATE TABLE IF NOT EXISTS responses('
        create += 'timestamp datetime default current_timestamp,'
        create += 'respID text NOT NULL,'
        create += 'responseSize text NOT NULL,'
        create += 'statusCode text NOT NULL,'
        create += 'time text NOT NULL,'
        create += 'numHeaders text NOT NULL,'
        create += 'numTokens text NOT NULL)'
        self.cursor.execute(create)

        create = 'CREATE TABLE IF NOT EXISTS requests('
        create += 'reqID text NOT NULL,'
        create += 'method text NOT NULL,'
        create += 'proxy text NOT NULL,'
        create += 'headers text NOT NULL,'
        create += 'cookies text NOT NULL,'
        create += 'url text NOT NULL,'
        create += 'data text NOT NULL,'
        create += 'module text NOT NULL)'
        self.cursor.execute(create)

    #Insert response Object into database
    def insert_result(self, resObj):
        data = {
                    "respID": str(resObj.respID),
                    "responseSize": str(resObj.responseSize),
                    "statusCode": str(resObj.statusCode),
                    "time": str(resObj.time),
                    "numHeaders": str(resObj.numHeaders),
                    "numTokens": str(resObj.numTokens),
                }
        insert = 'INSERT INTO responses(respID,responseSize,statusCode,time,'
        insert += 'numHeaders,numTokens) VALUES (:respID,:responseSize,'
        insert += ':statusCode,:time,:numHeaders,:numTokens)'
        try:
            self.cursor.execute(insert, data)
        except apsw.OperationalError:
            print('SQL Error: insert_result')

    #Insert request Object
    def insert_request(self, reqObj):
        insert = 'INSERT INTO requests(reqID,method,proxy,headers,cookies,'
        insert += 'url,data,module) VALUES (:reqID,:method,:proxy,:headers,'
        insert += ':cookies,:url,:data,:module)'
        try:
            self.cursor.execute(insert, reqObj)
        except apsw.OperationalError:
            print('SQL Error: insert request')

    #Get number of entries
    def get_count(self):
        select = 'SELECT COUNT(*) FROM responses'
        try:
            self.cursor.execute(select)
            results = self.cursor.fetchone()
            return results[0]
        except apsw.OperationalError:
            print('SQL Error: get_count')
            pass

    #Get response IDs
    def get_responses(self):
        respdata = []
        select = 'SELECT respID FROM responses'
        try:
            self.cursor.execute(select)
            resps = self.cursor.fetchall()
            for i in resps:
                respdata.append(i[0])
            return respdata
        except apsw.OperationalError:
            print('SQL Error: get responses')

    def return_all(self):
        self.conn = apsw.Connection(self.dbfile)
        self.cursor = self.conn.cursor()
        select = 'SELECT requests.module,requests.url,requests.headers,'
        select += 'responses.responseSize,responses.statusCode,responses.time,'
        select += 'responses.numHeaders,responses.numTokens FROM requests '
        select += 'INNER JOIN responses ON responses.respID == requests.reqID'
        try:
            self.cursor.execute(select)
            resps = self.cursor.fetchall()
            forprint = fg(1)+'Module: '+self.rs+'URL '+fg(4)+'status '+fg(10)
            forprint += 'size '+fg(3)+'time '+fg(13)+'numHeaders '+fg(14)
            forprint += 'numTokens'+fg(8)+' headers'+self.rs
            print(forprint)
            for i in resps:
                response = fg(1)+i[0]+self.rs+': '+self.rs+i[1]+' '+fg(4)+i[4]
                response += ' '+fg(10)+i[3]+' '+fg(3)+i[5]+' '+fg(13)+i[6]+' '
                response += fg(14)+i[7]+'  '+fg(8)+i[2]+self.rs
                print(response)
        except apsw.OperationalError:
            print('SQL Error: return all')
        except Exception as e:
            print(e)
        return

    #Close the database connection
    def close(self):
        self.conn.close()