import json
from typing import List

none = "d3043820717d74d9a17694c176d39733"

class Persistence:
    """
    # Arguments
    persist_root_device: bool
    persist_block_devices: bool
    block_devices_mode: str
    persist_private_ip: bool
    """
    def __init__(self, persist_root_device: bool = none, persist_block_devices: bool = none,
                 block_devices_mode: str = none, persist_private_ip: bool = none):
        self.persist_root_device = persist_root_device
        self.persist_block_devices = persist_block_devices
        self.block_devices_mode = block_devices_mode
        self.persist_private_ip = persist_private_ip

class HealthCheck:
    """
    # Arguments
    type: bool
    auto_healing: bool
    grace_period: int
    unhealthy_duration: int
    """
    def __init__(self, type: str = none, auto_healing: bool = none, grace_period: int = none,
                 unhealthy_duration: int = none):
        self.type = type
        self.auto_healing = auto_healing
        self.grace_period = grace_period
        self.unhealthy_duration = unhealthy_duration

class Task:
    """
    # Arguments
    task_type: str
    start_time: str
    cron_expression: str
    is_enabled: bool
    frequency: str
    """
    def __init__(self, task_type: str = none, start_time: str = none, cron_expression: str = none,
                 is_enabled: bool = none, frequency: str = none):
        self.is_enabled = is_enabled
        self.frequency = frequency
        self.start_time = start_time
        self.cron_expression = cron_expression
        self.task_type = task_type

class Scheduling:
    """
    # Arguments
    tasks: list[Task]
    """
    def __init__(self, tasks: List[Task] = none):
        self.tasks = tasks

class RevertToSpot:
    """
    # Arguments
    perform_at: str
    """
    def __init__(self, perform_at: str = none):
        self.perform_at = perform_at

class Strategy:
    """
    # Arguments
    life_cycle: str
    orientation: str
    draining_timeout: int
    fallback_to_od: bool
    utilize_reserved_instances: bool
    utilize_commitments: bool
    optimization_windows: list[str]
    revert_to_spot: RevertToSpot
    minimum_instance_lifetime: int
    """
    def __init__(self, life_cycle: str = none, orientation: str = none, draining_timeout: int = none,
                 fallback_to_od: bool = none,
                 utilize_reserved_instances: bool = none, utilize_commitments: bool = none,
                 optimization_windows: List[str] = none,
                 revert_to_spot: RevertToSpot = none, minimum_instance_lifetime: int = none):
        self.life_cycle = life_cycle
        self.orientation = orientation
        self.draining_timeout = draining_timeout
        self.fallback_to_od = fallback_to_od
        self.utilize_reserved_instances = utilize_reserved_instances
        self.utilize_commitments = utilize_commitments
        self.optimization_windows = optimization_windows
        self.revert_to_spot = revert_to_spot
        self.minimum_instance_lifetime = minimum_instance_lifetime

class EBS:
    """
    # Arguments
    delete_on_termination: bool
    encrypted: bool
    iops: int
    throughput: float
    volume_size: int
    volume_type: str
    kms_key_id: str
    snapshot_id: str
    """
    def __init__(self, delete_on_termination: bool = none, encrypted: bool = none, iops: int = none,
                 throughput: float = none,
                 kms_key_id: str = none,
                 snapshot_id: str = none,
                 volume_size: int = none,
                 volume_type: str = none):
        self.delete_on_termination = delete_on_termination
        self.encrypted = encrypted
        self.iops = iops
        self.throughput = throughput
        self.volume_size = volume_size
        self.volume_type = volume_type
        self.kms_key_id = kms_key_id
        self.snapshot_id = snapshot_id

class BlockDeviceMapping:
    """
    # Arguments
    device_name: str
    ebs: EBS
    no_device: str
    virtual_name: str
    """
    def __init__(self, device_name: str = none, ebs: EBS = none, no_device: str = none, virtual_name: str = none):
        self.device_name = device_name
        self.ebs = ebs
        self.no_device = no_device
        self.virtual_name = virtual_name

class CreditSpecification:
    """
    # Arguments
    cpu_credits: str
    """
    def __init__(self, cpu_credits: str = none):
        self.cpu_credits = cpu_credits

class IamRole:
    """
    # Arguments
    name: str
    arn: str
    """
    def __init__(self, name: str = none, arn: str = none):
        self.name = name
        self.arn = arn

class InstanceTypes:
    """
    # Arguments
    preferred_type: str
    types: list[str]
    """
    def __init__(self, preferred_type: str = none, types: List[str] = none):
        self.preferred_type = preferred_type
        self.types = types

class NetworkInterface:
    """
    # Arguments
    device_index: int
    associate_public_ip_address: bool
    associate_ipv6_address: bool
    """
    def __init__(self, device_index: int = none, associate_public_ip_address: bool = none,
                 associate_ipv6_address: bool = none):
        self.device_index = device_index
        self.associate_public_ip_address = associate_public_ip_address
        self.associate_ipv6_address = associate_ipv6_address

class TagSpecification:
    """
    # Arguments
    should_tag: bool
    """
    def __init__(self, should_tag: bool = none):
        self.should_tag = should_tag

class ResourceTagSpecification:
    """
    # Arguments
    volumes: TagSpecification
    snapshots: TagSpecification
    enis: TagSpecification
    amis: TagSpecification
    """
    def __init__(self, volumes: TagSpecification = none, snapshots: TagSpecification = none, enis: TagSpecification = none,
                 amis: TagSpecification = none):
        self.volumes = volumes
        self.snapshots = snapshots
        self.enis = enis
        self.amis = amis

class Tag:
    """
    # Arguments
    tag_key: str
    tag_value: str
    """
    def __init__(self, tag_key: str = none, tag_value: str = none):
        self.tag_key = tag_key
        self.tag_value = tag_value

class LaunchSpecification:
    """
    # Arguments
    instance_types: InstanceTypes
    ebs_optimized: bool
    monitoring: bool
    tenancy: str
    iam_role: IamRole
    security_group_ids: list[str]
    image_id: str
    key_pair: str
    tags: list[Tag]
    resource_tag_specification: ResourceTagSpecification
    user_data: str
    shutdown_script: str
    credit_specification: CreditSpecification
    network_interfaces: list[NetworkInterface]
    block_device_mappings: list[BlockDeviceMapping]
    """
    def __init__(self, instance_types: InstanceTypes = none, ebs_optimized: bool = none, monitoring: bool = none,
                 tenancy: str = none,
                 iam_role: IamRole = none, security_group_ids: List[str] = none, image_id: str = none,
                 key_pair: str = none, tags: List[Tag] = none,
                 resource_tag_specification: ResourceTagSpecification = none, user_data: str = none,
                 shutdown_script: str = none,
                 credit_specification: CreditSpecification = none, network_interfaces: List[NetworkInterface] = none,
                 block_device_mappings: List[BlockDeviceMapping] = none):
        self.instance_types = instance_types
        self.ebs_optimized = ebs_optimized
        self.monitoring = monitoring
        self.tenancy = tenancy
        self.iam_role = iam_role
        self.security_group_ids = security_group_ids
        self.image_id = image_id
        self.key_pair = key_pair
        self.tags = tags
        self.resource_tag_specification = resource_tag_specification
        self.user_data = user_data
        self.shutdown_script = shutdown_script
        self.block_device_mappings = block_device_mappings
        self.credit_specification = credit_specification
        self.network_interfaces = network_interfaces

class Compute:
    """
    # Arguments
    subnet_ids: list[str]
    vpc_id: str
    elastic_ip: str
    private_ip: str
    product: str
    launch_specification: LaunchSpecification
    """
    def __init__(self, subnet_ids: List[str] = none, vpc_id: str = none, elastic_ip: str = none, private_ip: str = none,
                 product: str = none,
                 launch_specification: LaunchSpecification = none):
        self.subnet_ids = subnet_ids
        self.vpc_id = vpc_id
        self.elastic_ip = elastic_ip
        self.private_ip = private_ip
        self.product = product
        self.launch_specification = launch_specification

class Route53RecordSetConfiguration:
    """
    # Arguments
    name: str
    use_public_ip: bool
    use_public_dns: bool
    """
    def __init__(self, name: str = none, use_public_ip: bool = none, use_public_dns: bool = none):
        self.name = name
        self.use_public_ip = use_public_ip
        self.use_public_dns = use_public_dns

class Route53DomainConfiguration:
    """
    # Arguments
    hosted_zone_id: str
    spotinst_account_id: str
    record_set_type: str
    record_sets: list[Route53RecordSetConfiguration]
    """
    def __init__(self, hosted_zone_id: str = none, spotinst_account_id: str = none, record_set_type: str = none,
                 record_sets: List[Route53RecordSetConfiguration] = none):
        self.hosted_zone_id = hosted_zone_id
        self.spotinst_account_id = spotinst_account_id
        self.record_set_type = record_set_type
        self.record_sets = record_sets

class Route53Configuration:
    """
    # Arguments
    domains: list[Route53DomainsConfiguration]
    """
    def __init__(self, domains: List[Route53DomainConfiguration] = none):
        self.domains = domains

class LoadBalancer:
    """
    # Arguments
    name: str
    arn: str
    type: str
    balancer_id: str
    target_set_id: str
    az_awareness: bool
    auto_weight: bool
    """
    def __init__(self, name: str = none, arn: str = none, type: str = none, balancer_id: str = none,
                 target_set_id: str = none, az_awareness: bool = none,
                 auto_weight: bool = none):
        self.name = name
        self.arn = arn
        self.type = type
        self.balancer_id = balancer_id
        self.target_set_id = target_set_id
        self.az_awareness = az_awareness
        self.auto_weight = auto_weight

class LoadBalancersConfiguration:
    """
    # Arguments
    load_balancers: list[LoadBalancer]
    """

    def __init__(self, load_balancers: List[LoadBalancer] = none):
        self.load_balancers = load_balancers

class IntegrationsConfig:
    """
    # Arguments
    route53: Route53Configuration
    load_balancers_config: LoadBalancersConfiguration
    """

    def __init__(self, route53: Route53Configuration = none, load_balancers_config: LoadBalancersConfiguration = none):
        self.route53 = route53
        self.load_balancers_config = load_balancers_config

class ManagedInstance:
    """
    # Arguments
    name: str
    description: str
    region: str
    strategy: Strategy
    persistence: Persistence
    health_check: HealthCheck
    scheduling: Scheduling
    compute: Compute
    integrations: IntegrationsConfig
    """

    def __init__(self, name: str = none, description: str = none, region: str = none,
                 strategy: Strategy = none,
                 persistence: Persistence = none, health_check: HealthCheck = none, scheduling: Scheduling = none,
                 compute: Compute = none, integrations: IntegrationsConfig = none):
        self.name = name
        self.description = description
        self.region = region
        self.strategy = strategy
        self.persistence = persistence
        self.health_check = health_check
        self.compute = compute
        self.scheduling = scheduling
        self.integrations = integrations

# region Client Requests
class ManagedInstanceCreationRequest:
    def __init__(self, managed_instance: ManagedInstance):
        self.managed_instance = managed_instance

    def toJSON(self):
        return json.dumps(self, default=lambda o: o.__dict__,
                          sort_keys=True, indent=4)

class DeallocationConfig:
    def __init__(self, should_delete_network_interfaces: bool = none,
                 should_delete_volumes: bool = none, should_delete_snapshots: bool = none, should_delete_images: bool = none,
                 should_terminate_instance: bool = none):
        self.should_delete_network_interfaces = should_delete_network_interfaces
        self.should_delete_volumes = should_delete_volumes
        self.should_delete_snapshots = should_delete_snapshots
        self.should_delete_images = should_delete_images
        self.should_terminate_instance = should_terminate_instance

    def toJSON(self):
        return json.dumps(self, default=lambda o: o.__dict__,
                          sort_keys=True, indent=4)

class AmiBackup:
    def __init__(self, should_delete_images: bool = none):
        self.should_delete_images = should_delete_images

    def toJSON(self):
        return json.dumps(self, default=lambda o: o.__dict__,
                          sort_keys=True, indent=4)

class ManagedInstanceDeletionRequest:
    def __init__(self, deallocation_config: DeallocationConfig = none, ami_backup: AmiBackup = none):
        self.deallocation_config = deallocation_config
        self.ami_backup = ami_backup

    def toJSON(self):
        return json.dumps(self, default=lambda o: o.__dict__,
                          sort_keys=True, indent=4)

class ManagedInstanceUpdateRequest:
    def __init__(self, managed_instance: ManagedInstance):
        self.managed_instance = managed_instance

    def toJSON(self):
        return json.dumps(self, default=lambda o: o.__dict__,
                          sort_keys=True, indent=4)

class ManagedInstanceState:
    def __init__(self, id: str = none, state: str = none):
        self.id = id
        self.state = state

class ManagedInstanceUpdateStatesRequest:
    def __init__(self, managed_instance_states: List[ManagedInstanceState] = none):
        self.managed_instance_states = managed_instance_states

    def toJSON(self):
        return json.dumps(self, default=lambda o: o.__dict__,
                        sort_keys=True, indent=4)

class AvailabilityZones:
    def __init__(self, name: str = None, subnet_ids: List[str] = None):
        self.name = name
        self.subnet_ids = subnet_ids

class ManagedInstanceMigration:
    def __init__(self, should_keep_private_ip: bool = none, original_instance_id: str = none,
                region: str = none, should_terminate_instance: bool = none, managed_instance_name: str =none,
                product: str = none, spot_instances_types: List[str] = none, availability_zones: List[AvailabilityZones] = none):
        self.should_keep_private_ip = should_keep_private_ip
        self.original_instance_id = original_instance_id
        self.region = region
        self.should_terminate_instance = should_terminate_instance
        self.managed_instance_name = managed_instance_name
        self.product = product
        self.spot_instance_types = spot_instances_types
        self.availability_zones = availability_zones

class ManagedInstanceBulkMigrationRequest:
    def __init__(self, migrations: List[ManagedInstanceMigration] = none):
        self.migrations = migrations

    def toJSON(self):
        return json.dumps(self, default=lambda o: o.__dict__,
                        sort_keys=True, indent=4)

class ManagedInstanceBulkMigrationStatusRequest:
    def __init__(self, migration_ids: List[str] = none):
        self.migration_ids = migration_ids

    def toJSON(self):
        return json.dumps(self, default=lambda o: o.__dict__,
                        sort_keys=True, indent=4)
# endregion


# Configuration file for the Sphinx documentation builder.
#
# For the full list of built-in configuration values, see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

import sys
import os
from datetime import date

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.

sys.path.append(os.path.abspath('./_extension'))

# -- Project information -----------------------------------------------------

# The documented project’s name.
project = '読書ノート'

# The author name(s) of the document.
author = 'プレハブ小屋'

# A copyright statement in the style '2008, Author Name'.
copyright = f'1999-{date.today().year}, {author} All rights reserved'

# The short X.Y version.
# The major project version, used as the replacement for |version|.
version = '1.5'

# The full project version, used as the replacement for |release| and e.g. in
# the HTML templates.
release = '1.5dev'

# -- General configuration -----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
    'disablesearchindex',
    'IPython.sphinxext.ipython_console_highlighting',
    'IPython.sphinxext.ipython_directive',
    'japanesesupport',
    'sphinx.ext.githubpages',
    'sphinx.ext.mathjax',
    'sphinx.ext.todo',
    'sphinxcontrib.mermaid',]

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The master toctree document.
root_doc = 'index'

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# A list of warning types to suppress arbitrary warning messages.
suppress_warnings = ['autosectionlabel.*']

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes:
# https://www.sphinx-doc.org/en/master/usage/theming.html
#
# Alabaster and Scrolls themes are mobile-optimated.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {
    'body_text': 'black',
    'caption_font_family': 'initial',
    'description': '個人的な読書、学習、研究ノート。',
    'description_font_style': 'initial',
    'font_family': 'initial',
    'font_size': 'initial',
    'footer_text': 'black',
    'head_font_family': 'initial',
    'github_button': False,
    'github_repo': 'notebook',
    'github_user': 'showa-yojyo',
    'link': 'deeppink',
    'link_hover': 'deeppink',
    'narrow_sidebar_link': 'deeppink',
    'nosidebar': True,
    'page_width': '1200px',
    'show_powered_by': False,
    'show_relbars': True,
    'sidebar_link': 'deeppink',
    'sidebar_link_underscore': 'deeppink',
    'sidebar_text': 'initial',
}

# A list of JavaScript filename.
html_js_files = [
    #'mathjax-v3.js', # move to mathjax_path
    'mermaid.js',
]

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%Y-%m-%d (%a) %H:%M:%S (%Z)'

# Custom sidebar templates, maps document names to template names.
# Alabaster wants explicit settings for sidebars:
html_sidebars = {
    '**': [
        'about.html',
        'navigation.html',
        'relations.html',
        #'searchbox.html',
        'donate.html',
    ]
}

# If false, no index is generated.
html_use_index = False

# If true, the reST sources are included in the HTML build as _sources/name. The
# default is True.
html_copy_source = False

# If true (and html_copy_source is true as well), links to the reST sources will
# be added to the sidebar.
html_show_sourcelink = False

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, the text around the keyword is shown as summary of each search
# result. Default is True.
html_show_search_summary = False

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
html_show_sphinx = False

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# -- Options for MathJax -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/extensions/math.html

# Use customized version
mathjax_path = "mathjax-v3.js"

# -- Options for sphinx.ext.todo
# https://www.sphinx-doc.org/en/master/usage/extensions/todo.html

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for Mermaid -------------------------------------------------
# https://github.com/mgaitan/sphinxcontrib-mermaid

# See html_js_files
mermaid_version = ""
mermaid_init_js = ""


#!/usr/bin/env python

# =============================================================================
# MODULE DOCSTRING
# =============================================================================

"""
Test classes and functions in tfep.nn.dynamics.egnn.
"""

# =============================================================================
# GLOBAL IMPORTS
# =============================================================================

import numpy as np
import pytest
import scipy.spatial.transform
import torch

from tfep.nn.dynamics.egnn import EGNNDynamics
from tfep.utils.geometry import batchwise_rotate
from tfep.utils.misc import atom_to_flattened, flattened_to_atom

# =============================================================================
# TEST MODULE CONFIGURATION
# =============================================================================

_old_default_dtype = None

def setup_module(module):
    global _old_default_dtype
    _old_default_dtype = torch.get_default_dtype()
    torch.set_default_dtype(torch.double)

def teardown_module(module):
    torch.set_default_dtype(_old_default_dtype)

# =============================================================================
# TESTS EQUIVARIANT GRAPH NEURAL NETWORK DYNAMICS
# =============================================================================

@pytest.mark.parametrize('batch_size', [1, 2])
def test_egnn_dynamics_node_embedding(batch_size):
    """Test the embedding of the atom types and time into the node features."""
    particle_types = torch.tensor([0, 0, 1])  # Three particles.
    time_feat_dim = 2
    node_feat_dim = 4

    egnn_dynamics = EGNNDynamics(
        particle_types=particle_types,
        r_cutoff=6.0,
        time_feat_dim=time_feat_dim,
        node_feat_dim=node_feat_dim,
    )

    h = egnn_dynamics._create_node_embedding(t=torch.tensor(1.0), batch_size=batch_size)

    # The features result from the concatenation of the atom type and time expansions.
    assert h.shape == (batch_size*len(particle_types), node_feat_dim)

    # Identical atom types must have identical embeddings.
    if batch_size == 1:
        zero_type = [1]
        one_type = []
    else:
        zero_type = [1, 3, 4]
        one_type = [5]

    for i in zero_type:
        assert torch.allclose(h[0], h[i])
    for i in one_type:
        assert torch.allclose(h[2], h[i])
    assert not torch.allclose(h[0], h[2])

@pytest.mark.parametrize('batch_size', [1, 4])
@pytest.mark.parametrize('r_cutoff', [1.5, 10.0])
@pytest.mark.parametrize('seed', [0, 5, 10])
def test_egnn_dynamics_equivariance_property(batch_size, r_cutoff, seed):
    """Test that the output of the EGNN has the correct equivariance properties."""
    particle_types = torch.tensor([0, 0, 1])
    n_particles = len(particle_types)

    # Create the equivariant graph.
    egnn_dynamics = EGNNDynamics(
        particle_types=particle_types,
        r_cutoff=r_cutoff,
        time_feat_dim=2,
        node_feat_dim=5,
        distance_feat_dim=4,
        n_layers=2,
        initialize_identity=False,
    )

    # Create a random input.
    generator = torch.Generator()
    generator.manual_seed(seed)
    t = torch.rand(1, generator=generator)
    x = torch.randn(batch_size, n_particles*3, generator=generator)

    # Run the dynamics.
    vel = egnn_dynamics(t, x)

    # Now re-run the dynamics with a randomly rotated input.
    random_state = np.random.RandomState(seed)
    rotation_matrices = scipy.spatial.transform.Rotation.random(
        num=batch_size, random_state=random_state)
    rotation_matrices = torch.tensor(rotation_matrices.as_matrix())

    # Rotate the input.
    x_rotated = batchwise_rotate(flattened_to_atom(x), rotation_matrices)
    x_rotated = atom_to_flattened(x_rotated)

    # The velocity should rotate accordingly.
    vel_rotated = egnn_dynamics(t, x_rotated)
    ref_vel = atom_to_flattened(batchwise_rotate(flattened_to_atom(vel), rotation_matrices))
    assert torch.allclose(ref_vel, vel_rotated)

    # Translate the input with shape (batch_size, n_particles*3).
    translation = torch.randn(batch_size, 3)
    translation = translation.repeat(1, n_particles)
    x_translated = x + translation

    # The velocity should be left invariant.
    vel_translated = egnn_dynamics(t, x_translated)
    assert torch.allclose(vel, vel_translated)

    # Velocity is equivariant w.r.t. permuting two particles of the same type.
    def permute(_x, idx1, idx2):
        _x_permuted = _x.clone()
        _x_permuted[:,idx1*3:(idx1+1)*3] = _x[:,idx2*3:(idx2+1)*3]
        _x_permuted[:,idx2*3:(idx2+1)*3] = _x[:,idx1*3:(idx1+1)*3]
        return _x_permuted

    x_permuted = permute(x, 0, 1)
    vel_permuted = egnn_dynamics(t, x_permuted)
    assert torch.allclose(permute(vel, 0, 1), vel_permuted)

    # Permuting two particles with different type changes the velocity.
    x_permuted = permute(x, 0, 2)
    vel_permuted = egnn_dynamics(t, x_permuted)
    assert not torch.allclose(permute(vel, 0, 2), vel_permuted)

def test_egnn_dynamics_identity():
    """Test that EGNNDynamics can be initialized to output zero velocity."""
    batch_size = 10
    particle_types = torch.tensor([0, 0, 1])
    n_particles = len(particle_types)

    # Create the equivariant graph.
    egnn_dynamics = EGNNDynamics(
        particle_types=particle_types,
        r_cutoff=10.0,
        time_feat_dim=2,
        node_feat_dim=5,
        distance_feat_dim=4,
        n_layers=2,
        initialize_identity=True,
    )

    # Create a random input.
    generator = torch.Generator()
    generator.manual_seed(0)
    t = torch.rand(1, generator=generator)
    x = torch.randn(batch_size, n_particles*3, generator=generator)

    # Run the dynamics.
    vel = egnn_dynamics(t, x)

    # The output is identically 0.0.
    assert torch.allclose(vel, torch.zeros_like(vel))


import pygame
import time
import threading
import math
try:
   import pymunk
   import pymunk.pygame_util
except:
   pass
pygame.init()

screen = None
image = None

print('asdf')
print('I like trains')
screen = pygame.display.set_mode((600,600),pygame.RESIZABLE)
pygame.display.set_caption('Hello Class')
image = pygame.image.load ('vangogh.jpg').convert()
screen.blit (image,(((300,300))[0]-image.get_size()[0]//2,((300,300))[1]-image.get_size()[1]//2))
pygame.display.update()
_text = pygame.font.Font('freesansbold.ttf',26).render ('Hello Again',True,pygame.Color('#cc33cc'))
_rect = _text.get_rect()
_rect.x = 100
_rect.y = 100
screen.blit (_text,_rect)
pygame.display.update()
_quitEvent = False
while not _quitEvent:
   for event in (pygame.event.get()):
      if (event.type) == (pygame.QUIT):
         _quitEvent = True
         break


import csv, cv2
import pandas as pd
from sklearn import svm, metrics
from sklearn.model_selection import train_test_split

# 알파벳 사진 잘라내기
gray_img = cv2.imread('alph/alphabets.png', 0)
al = 'abcdefghijklmnopqrstuvwxyz'
h, w = gray_img.shape
al_w = w/26
ww = 0
imgs = []

for i in range(0, 26):
    img = gray_img[0:h, int(i*al_w):int(ww+al_w)]
    ww += al_w
    imgs.append(img)

# 알파벳 바이너리 파일을 csv파일로 만들어서 저장하기.
# idx = 0
# csv_f = open('alph/al.csv', 'w', encoding='utf-8')
# for x in imgs:
#     fdata = x.ravel()
#     img_data = list(map(lambda n: str(n), fdata))
#     csv_f.write(al[idx] + ',')
#     csv_f.write(','.join(img_data) + '\r\n')
#     idx+=1

path = 'alph/'
train_label = []
train_data = []

with open(path+'al.csv', 'r') as f:
    for line in f:
        if len(line)==1:
            continue
        l = line.strip().split(',')
        train_label.append(l[0])
        train_data.append([int(a) for a in l[1:]])

# train_data, test_data, train_label, test_label = train_test_split(train_data, train_label)

clf = svm.SVC()
clf.fit(train_data, train_label)
# t = cv2.imread('alph/test1.PNG', 0)
# t = t.reshape(1,  h, int(w/26))
pre = clf.predict([train_data[0]])
# score = metrics.accuracy_score(test_label, pre)
print(pre)
# print('score:', score)


import serial
import time

# Open a serial connection to Roomba
ser = serial.Serial(port='/dev/serial0', baudrate=115200)

print("connected to: " + ser.portstr)

#this will store the line
line = []

while True:
    for c in ser.read():
        line.append(c)
        if c == '\n':
            print("Line: " + ''.join(line))
            line = []
            break

ser.close()


from typing import List

class Solution:
    def sortedSquares(self, nums: List[int]) -> List[int]:
        if len(nums) == 1:
            return [nums[0] ** 2]

        result = []
        index = 0

        while index < len(nums) and nums[index] < 0:
            index += 1

        negatives = nums[:index]
        positives = nums[index:]
        negatives = negatives[::-1]

        pos_index = 0
        neg_index = 0

        while pos_index < len(positives) or neg_index < len(negatives):
            if pos_index == len(positives):
                result.append(negatives[neg_index] ** 2)
                neg_index += 1
            elif neg_index == len(negatives):
                result.append(positives[pos_index] ** 2)
                pos_index += 1
            else:
                if abs(negatives[neg_index]) > positives[pos_index]:
                    result.append(positives[pos_index] ** 2)
                    pos_index += 1
                else:
                    result.append(negatives[neg_index] ** 2)
                    neg_index += 1

        return result

def main():
    sol = Solution()
    print(sol.sortedSquares([-4,-1,0,3,10]))
    print(sol.sortedSquares([-7,-3,2,3,11]))
    print(sol.sortedSquares([2]))
    print(sol.sortedSquares([-4, -4, -3]))

if __name__ == '__main__':
    main()


from numpy import *

class Rs:
   def __init__(self, str):
     self.str=str

   def concentration(self, str):
   	 a1=(float)(self.str)*0.52918*power(10.0,-8)
   	 result=3.0/(4.0*pi*power(a1,3.0))
   	 return float32(result)

   def bandwidth(self, str):
     k=power(9.0*pi/4.0,1.0/3.0)/float(self.str)
     result=k*k/2*27.211
     return float32(result)

if __name__ == '__main__':
   s2=""
   s3=""
   e = Rs(2.07)
   s2=e.concentration(2.07)
   s3=e.bandwidth(2.07)
   print(s2,s3)


#!/usr/bin/env python3
import argparse
import os.path
import subprocess
from io import open
from aktoro.compiler import compile_ak, AK_GRAMMAR
from aktoro.parser import PipelineRewriter, VariantPatternRewriter

__path__ = os.path.dirname(__file__)

cli = argparse.ArgumentParser()
subparsers = cli.add_subparsers(dest="subcommand")

def sub_command(args=None, parent=subparsers):
    if args is None:
        args = []

    def decorator(func):
        parser = parent.add_parser(func.__name__, description=func.__doc__)
        for arg in args:
            parser.add_argument(*arg[0], **arg[1])
        parser.set_defaults(func=func)

    return decorator

def argument(*name_or_flags, **kwargs):
    return [*name_or_flags], kwargs

@sub_command([argument('filename', type=str, help="filename"),
              argument('-o', type=str, help="output")])
def build(args):
    input_filename = os.path.join(__path__, args.filename)
    output_filename = os.path.join(__path__, args.o)
    with open(input_filename) as ak:
        program = ak.read()

    generated = compile_ak(program)

    input_filename_no_extension = input_filename.split(".ak", 1)[0]
    input_path = input_filename_no_extension.split("/")
    input_path = "/".join(input_path[:len(input_path) - 1])
    temp_go_filename = f"{input_filename_no_extension}_aktoro_generated.go"
    with open(temp_go_filename, "w") as go_file:
        go_file.write(generated)
    build_str = f"cd {input_path} && go build"
    if args.o:
        build_str += f" -o {output_filename}"
    subprocess.check_output(build_str, shell=True, )
    os.remove(temp_go_filename)

@sub_command([argument('filename', type=str, help="filename")])
def run(args):
    input_filename = os.path.join(__path__, args.filename)
    with open(input_filename) as ak:
        program = ak.read()

    generated = compile_ak(program)
    input_filename_no_extension = input_filename.split(".ak", 1)[0]
    temp_go_filename = f"{input_filename_no_extension}_aktoro_generated.go"
    with open(temp_go_filename, "w") as go_file:
        go_file.write(generated)
    output = subprocess.check_output(f"go run {temp_go_filename}", shell=True)
    output = output.decode("utf-8")
    os.remove(temp_go_filename)
    print(output)

@sub_command([argument('filename', type=str, help="filename")])
def parse(args):
    input_filename = os.path.join(__path__, args.filename)
    with open(input_filename) as ak:
        program = ak.read()

    parse_tree = AK_GRAMMAR.parse(program)
    parse_tree = PipelineRewriter().visit(parse_tree)
    parse_tree = VariantPatternRewriter().visit(parse_tree)
    print(parse_tree.pretty())

@sub_command([argument('filename', type=str, help="filename")])
def generate(args):
    input_filename = os.path.join(__path__, args.filename)
    with open(input_filename) as ak:
        program = ak.read()

    generated = compile_ak(program)
    print(generated)

if __name__ == "__main__":
    args = cli.parse_args()
    if args.subcommand is None:
        cli.print_help()
    else:
        args.func(args)


# -*- coding:utf-8 -*-
# editor: zzh
# date: 2021/3/1
#根据提取的信息进行数据标注

import re_extract
import os
import re

patterns = re_extract.pattern_dict
target_tag = [tag for tag in patterns.keys()]
classifications = re_extract.classifications
def has_tag(line,type):  #句子是不是与该模式匹配
    pattern = patterns[type]
    return pattern.search(line) != None

def get_line_replace(line,type):   #匹配出关键信息
    print('\n',line)
    pattern = patterns[type]
    find = pattern.search(line)
    if find:
        print("     >"+find.group())
        return find.group()
    else:
        print("     >"+line)
        return line

def tag_paragraph(content):
    content_taged = content
    lines = re.split('[，。！？：；,!;、?()（）??]',content)
    visited = []
    for line in lines:
        if(line.strip() in visited):
            continue
        line = line.strip()

        for tag in target_tag:
            if has_tag(line,tag):
                target_line = get_line_replace(line, tag)
                content_taged = content_taged.replace(target_line, ' '+target_line + '/' + tag + ' ').replace('/' + tag + ' ' +'/' + tag , '/' + tag)
    # print(content_taged)
    return content_taged

def tag_dir(path,savepath):
    files = os.listdir(path)
    for file in files:
        print("**"*30)
        print(file)
        filepath = path + "\\" + file
        savefilepath = savepath + "\\" + file
        with open(filepath, "r", encoding="utf8") as f:
            with open(savefilepath, "w", encoding="utf8") as g:
                title = f.readline()
                time = f.readline()
                source = f.readline()
                contents = f.readlines()
                g.write(title+time+source)
                for content in contents:
                    content_taged = tag_paragraph(content)
                    g.write(content_taged)

if __name__ == '__main__':
    for classification in classifications:
        tag_dir("D:\MyProject\天气灾害分类及信息提取\新闻文本\范野学长毕设数据\原标注数据\\"+classification,"D:\MyProject\天气灾害分类及信息提取\新闻文本\范野学长毕设数据\统合数据1\\"+classification)
    # 将一个文件夹中的所有新闻文件标注后放到另一文件夹
    # content = """ 中新网惠州八月二十五日电　(康孝娟 白剑涛 严初)广东惠州惠城区陈江镇水围村梧村水库，四名青年二十三日在岛上钓鱼时，突然遭到雷击，一人当场死亡，三人昏倒在地。而在当天，惠州市两个镇出现了雷击事故，共造成一死四伤。  　　笔者在惠州市第一人民医院住院部见到了被雷击中的伤者。在十楼烧伤科病房，伤者黄世波静静躺在病床上，无法说话，胸部和左手上臂都有灼伤痕迹。  　　据了解，二十三日是处暑，当天下午，一声秋雷响起，惠州市区下起了暴雨，暑气悄然褪去。惠州市惠城区陈江街道办事处、三栋镇都出现雷击事故，已造成一人死亡，四人受伤。  　　惠州市气象局防雷所陈所长说，受副热带高压控制，每年七、八月为雷雨高发季节，全国百分之八十的雷区都集中在广东。而陈江水围村梧村水库属空旷地，易引雷，加之小岛在高处，成为典型的雷区。  　　黄世波是惠城区三栋镇某工地建筑工人。据其妻子告诉记者，二十三日下午四时许，黄世波骑乘摩托车到镇隆青草窝朋友家闲聚，在路上忽然听到一声雷响，他便被雷击，呆坐在地上，路人马上报警，将他送院治疗。 据悉，梧村水库一带经常发生雷击事件，去年水库旁几个种菜妇女和建筑工也被雷击致死。  　　针对已发生的两起雷击事故，惠州市气象局防雷所陈所长希望借助媒体告知市民在行雷时切勿在空旷地上跑步、拨打手机等易引雷的行为，避免再发生类似事故。"""
    # tag_paragraph(content)


#!/usr/bin/env python
'''code description'''
# pylint: disable = I0011, E0401, C0103

class Solution(object):
    '''Solution description'''
    def func(self, t, s):
        '''Solution function description'''
        t = iter(t)
        return all(c in t for c in s)

def main():
    '''main function'''
    _solution = Solution()
    inp = [('jsofidjoajsodfijfoisajdisdfiaoisjdfijaposdjifoaisjgmsdflmx;lcvkpoeiwroqiwherdnxinxjnvlz;xcvkzx;lckfjowairjoewjr[qji', 'ojji'),\
           ('jsafoji', '')]
    for i in inp:
        print(_solution.func(i[0], i[1]))

if __name__ == "__main__":
    main()


from scipy import stats
import numpy as np

def load_scores(url_scores):
    Y = []
    f = open(url_scores, 'r')
    line = f.readline()
    while line:
        arch = line.split(" ")
        if len(arch)>1:
            Y.append(float(arch[1]))
        line = f.readline()
    f.close()

    return Y

mnist = load_scores("report1000Products_mnist.txt")
cifar = load_scores("report1000Products_cifar.txt")

def get_correlations(mnist, cifar):
    tau, p_value = stats.kendalltau(mnist, cifar)
    (pearson,p_p) = stats.pearsonr(mnist, cifar)
    (spearman,p_s) = stats.spearmanr(mnist, cifar)
    print(tau, pearson, spearman)

#get_correlations(mnist, cifar)

def split_sets(accuracies, threshold):
    accuracies = np.array(accuracies)
    s = np.where( accuracies > threshold )

    return s[0].size

mnist_split = [split_sets(mnist, 0.3), split_sets(mnist, 0.5), split_sets(mnist, 0.9), split_sets(mnist, 0.95)]
cifar_split = [split_sets(cifar, 0.3), split_sets(cifar, 0.5), split_sets(cifar, 0.9), split_sets(cifar, 0.95)]
print(cifar_split)


from django.urls import path
from . import views

app_name = 'main'

urlpatterns = [
    path('', views.search_by_id, name='home'),
    path('detail/<str:pk>/', views.details, name='detail'),
    path('csv/<str:pk>', views.csv_file, name='csv_file'),
    path('pdf/<str:pk>', views.pdf_file, name='pdf_file'),
]


from django.core.paginator import Paginator, InvalidPage
from rest_framework import viewsets
from rest_framework import status
from rest_framework.decorators import action
from rest_framework.response import Response

from tasks.models import Task
from tasks.serializers import TaskSerializer
from todo_app.settings import TASKS_ON_PAGE

class TasksViewSet(viewsets.ViewSet):
    def list(self, request):
        category = request.GET['task_category']
        if category == 'active':
            queryset = Task.objects.filter(status=False)
        elif category == 'completed':
            queryset = Task.objects.filter(status=True)
        else:
            queryset = Task.objects.all()
        queryset = queryset.order_by('id')

        request_page = request.GET.get('number_page')

        paginator = Paginator(queryset, TASKS_ON_PAGE)

        next_page = 1
        previous_page = 1

        try:
            final_data = paginator.page(request_page)
        except InvalidPage:
            final_data = paginator.page(paginator.num_pages)

        serializer = TaskSerializer(final_data, many=True)

        if final_data.has_next():
            next_page = final_data.next_page_number()
        if final_data.has_previous():
            previous_page = final_data.previous_page_number()

        paginaton = {
            'count_pages': paginator.num_pages,
            'count_elements': paginator.count,
            'page': final_data.number,
            'has_next': final_data.has_next(),
            'next_page': next_page,
            'prev_page': previous_page,
        }

        tasks_data = {
            'all_tasks_count': Task.objects.all().count(),
            'active_tasks_count': Task.objects.filter(status=False).count(),
            'completed_tasks_count': Task.objects.filter(status=True).count()
        }
        checkbox_all_status = (tasks_data['all_tasks_count'] == tasks_data['completed_tasks_count']
                                    if tasks_data['all_tasks_count'] else False)

        return Response({
            'data': serializer.data,
            'tasks_data': tasks_data,
            'checkbox_all_status': checkbox_all_status,
            'pagination': paginaton,
            },
            status=status.HTTP_200_OK)

    def create(self, request):
        serializer = TaskSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    @action(methods=['patch'], detail=False, url_path=r'check_all')
    def partial_update_common(self, request):
        Task.objects.exclude(status=request.data['status']).update(status=request.data['status'])
        return Response(status=status.HTTP_200_OK)

    def partial_update(self, request, pk=None):
        task = Task.objects.get(id=pk)
        serializer = TaskSerializer(task, data=request.data, partial=True)
        if serializer.is_valid():
            serializer.save()
            return Response(status=status.HTTP_200_OK)
        return Response(status=status.HTTP_400_BAD_REQUEST)

    @action(methods=['delete'], detail=False, url_path=r'delete_all_checked')
    def destroy_completed(self, request):
        Task.objects.filter(status=True).delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

    def destroy(self, request, pk=None):
        Task.objects.get(id=pk).delete()
        return Response(status=status.HTTP_204_NO_CONTENT)


# This dataset contains employment and the number of employees, broken down:
#   - by economic activities (1); by professional status (2)

# Motivation: OECD data shows us the international employment trends (change in values of employment over years),
# which enables comparison of employment trends between Singapore and selected overseas countries (those that were picked out as tech hubs).

# This script is used to extract data by year for employment for the selected list of OECD Countries
# for the sector of Information and Communication (ISIC Rev 4).

# If the list of countries needs to be changed, check for the country code to pull the data and
# update C_API, inserting the correct value to get the correct column of data

import requests
import json
import pandas as pd
import datetime as dt
import pycountry

# Location in returned dictionary, likely will not cause errors but may return wrong data, will need to check again
C_API = {'AUS': '0:15:1:0',
'ISR': '0:13:0:0',
'JPN': '0:13:2:0',
'KOR': '0:13:2:0',
'GBR': '0:10:2:0',
'EST': '0:11:2:0',
'FRA': '0:13:0:0',
'DEU': '0:13:2:0',
'IRL': '0:13:0:0',
'ITA': '0:13:2:0',
'NLD': '0:13:2:0',
'POL': '0:13:2:0',
'PRT': '0:13:2:0',
'ESP': '0:13:2:0',
'SWE': '0:13:2:0'}

# Country code used, it can be referenced from pycountry, but workato does not provide access to pycountry so this is used in the script on workato instead
COUNTRY_CODE = {'AUS': 'Australia',
          'ISR': 'Israel',
          'JPN': 'Japan',
          'KOR': 'Korea, Republic of',
          'GBR': 'United Kingdom',
          'EST': 'Estonia',
          'FRA': 'France',
          'DEU': 'Germany',
          'IRL': 'Ireland',
          'ITA': 'Italy',
          'NLD': 'Netherlands',
          'POL': 'Poland',
          'PRT': 'Portugal',
          'ESP': 'Spain',
          'SWE': 'Sweden'}

# Used to ensure that the data is pulled until the current year - limitation is OECD may not provide all years until now
# Start year is from 2008 is because that is how far back the data is provided by OECD
start_year = 2008; end_year = dt.datetime.now().year
# Base link used for API call, with year updated till current year
base = f'''https://stats.oecd.org/SDMX-JSON/data/ALFS_EMP/COUNTRY_CODE.\
YAREV4+YA994TL1_ST+YA994AL1_ST+YA99A4L1_ST+YA994IL1_ST+YA99B4L1_ST+YA99C4L1_ST+YA99D4L1_ST+YA99E4L1_ST+YA99F4L1_ST+YA994SL1_ST+YA99G4L1_ST+YA99H4L1_ST+YA99I4L1_ST+YA99J4L1_ST+YA99K4L1_ST+YA99L4L1_ST+YA99M4L1_ST+YA99N4L1_ST+YA99O4L1_ST+YA99P4L1_ST+YA99Q4L1_ST+YA99X4L1_ST+YA99S4L1_ST+YA99T4L1_ST+YA99U4L1_ST.MA+FE+TT.A/\
all?startTime={start_year}&endTime={end_year}'''

# column_names is the structure for columns for the dataframe
column_names = ["Country Code", "Country"]
YEARS = [f'{year}' for year in range(start_year, end_year + 1)]
column_names.extend(YEARS)

df = pd.DataFrame(columns=column_names)
empty_item = "na" #used as the default value for empty cells of data, can also be np.nan

# base_row is an empty row with all empty values
base_row = {column_name:empty_item for column_name in column_names}

# Iterating through the countries
for c_code in C_API.keys():
    new_row = base_row.copy()
    # Update Country Code and Country for easier reference when viewing
    new_row["Country Code"] = c_code;
    # pycountry.countries.get(alpha_3=C_API).name is the same as
    # C_CODE[c_code]
    new_row["Country"] = COUNTRY_CODE[c_code] #pycountry.countries.get(alpha_3=C_API).name

    # Replaces the country code from the base link to get API extraction for each country
    link = base.replace("COUNTRY_CODE", c_code)
    req = requests.get(link)

    # Getting the data from the API call
    JS = json.loads(req.text)
    test = JS['dataSets'][0]['series']
    data = test[C_API[c_code]]['observations']

    years_dct = JS['structure']['dimensions']['observation'][0]['values']
    years = [year['id'] for year in years_dct] # years where current data is present

    # Original data
    o_keys = [data[key][0] for key in data]
    # Assigning data to the new_row dictionary to insert into the dataframe
    for index in range(len(years)):
        real_year = years[index]
        o_index = o_keys[index]

        new_row[real_year] = o_index

    # Inserting new_row as a new row into the dataframe
    df.loc[len(df)] = new_row

# Writing the dataframe into a csv file
df.to_csv('OECD.csv', index=False, float_format='%.2f')


from enum import Enum
from PyQt5 import QtCore
import threading
from acquisition.dm6000b.packet import Packet
from acquisition.device import Device, DeviceException, DeviceTimeoutException

class FunctionUnit(Device):
    class State(Enum):
        Ready = 0
        Busy = 1
        Bad = 2

    stateChanged = QtCore.pyqtSignal()
    _packetReceivedSignal = QtCore.pyqtSignal(Packet)

    def __init__(self, dm6000b, deviceName, funitCode):
        super().__init__(dm6000b, deviceName)
        self._state = self.State.Ready
        self._outstanding = {}
        self._funitCode = funitCode
        # Using self.parent() in any case where a reference to dm6000 is needed yields the same results, but required
        # 50.5 microseconds during testing, whereas self.dm6000b took only 83.6 nanoseconds
        self.dm6000b = dm6000b
        if self._funitCode in self.dm6000b._funitSubdevices:
            raise DeviceException(self, 'Another subdevice is already registered to receive responses for function unit {}.'.format(self._funitCode))
        self.dm6000b._funitSubdevices[self._funitCode] = self
        self._packetReceivedSignal.connect(self._packetReceivedSlot, QtCore.Qt.QueuedConnection)

    def _packetReceivedSlot(self, packet):
        txPacket = None
        # NB: an event notification is sent unprompted and therefore has no associated request
        if not packet.isEventNotification and packet.cmdCode in self._outstanding:
            txPackets = self._outstanding[packet.cmdCode]
            # Presumably the DM6000B reponds to requests for the same function unit and command in FIFO order, so if no
            # request is found matching the response, the oldest request is assumed to have provoked the response
            if packet.statusCode != 0:
                txPacket = txPackets[0]
                del txPackets[0]
            else:
                for i, tp in enumerate(txPackets):
                    if tp.parameter == packet.parameter:
                        txPacket = tp
                        del txPackets[i]
                        break
                if txPacket is None:
                    txPacket = txPackets[0]
                    del txPackets[0]
            if len(txPackets) == 0:
                del self._outstanding[packet.cmdCode]
#       if self._funitCode == 85:
#           print('got response for "{}":  funitCode: {}, statusCode: {}, cmdCode: {}, parameter: {}, outstanding: {}'.format(self.deviceName, packet.funitCode, packet.statusCode, packet.cmdCode, packet.parameter, len(self._outstanding)))
        self._processReceivedPacket(txPacket, packet)
        self._updateState()

    def _processReceivedPacket(self, txPacket, rxPacket):
        raise NotImplementedError('Pure virtual function called.')

    def _updateState(self):
        if len(self._outstanding) == 0:
            if self._state == self.State.Busy:
                self._state = self.State.Ready
                self.stateChanged.emit()
        else:
            if self._state == self.State.Ready:
                self._state = self.State.Busy
                self.stateChanged.emit()

    def _transmit(self, packet):
        if packet.cmdCode not in self._outstanding:
            self._outstanding[packet.cmdCode] = [packet]
        else:
            self._outstanding[packet.cmdCode].append(packet)
#       if self._funitCode == 85:
#           print('sent to "{}":  funitCode: {}, cmdCode: {}, parameter: {}, outstanding: {}'.format(self.deviceName, packet.funitCode, packet.cmdCode, packet.parameter, len(self._outstanding)))
        self.dm6000b._workerSendPacketSignal.emit(packet)
        self._updateState()

    @QtCore.pyqtProperty(State, notify=stateChanged)
    def state(self):
        return self._state

    def waitForReady(self, timeout=None):
        '''Block until Device State changes from Busy.  If timeout seconds elapse while Device State remains Busy, a
        DeviceTimeoutException is thrown.  If Device State was Bad when waitForReady(..) was called, or if Device State
        changes from Busy to Bad during the wait, a DeviceException is thrown.  Note that this function relies on being called
        from the thread owning the Device (normally the main thread or GUI thread).  Calling it from another thread creates
        a race condition: the main thread's event loop may be running and, if it is, it will eat a stateChanged signal
        emitted before we can start executing our local event loop.'''
        if self._state == self.State.Busy:
            self._waitForSignal(self.stateChanged, timeout)
        if self._state == self.State.Bad:
            raise DeviceException(self, 'Device is in bad state.')


# 1)

def strLength(myString):
    return len(myString)

print('String length is', strLength("Praveen"))

# 2)

def countChar(myString):
    # countDict ={}
    keys =[]
    values =[]
    for i in myString:
        keys.append(i)
        values.append(myString.count(i))

    countDict = dict(zip(keys, values))
    return countDict

print(countChar("Praveen"))

# 3)
from parse import *

name = "Praveen"

print("My name is {fname}".format(fname = name))

print(parse("My name is {}", "My name is Praveen"))


import json
import requests
import time
from bs4 import BeautifulSoup

headers = {
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'accept-encoding': 'gzip, deflate, br',
    'accept-language': 'uk,en-US;q=0.9,en;q=0.8,ru;q=0.7',
    'sec-ch-ua': '"Google Chrome";v="89", "Chromium";v="89", ";Not A Brand";v="99"',
    'sec-ch-ua-mobile': '?0',
    'sec-fetch-dest': 'document',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-site': 'none',
    'sec-fetch-user': '?1',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'
}

def find_uik_18(address: [], addresses_json: json):
    for adr in addresses_json['addresses']:
        data = adr.split(',')
        if len(data) >= 4:
            if data[2].upper().rfind(address[0].upper()) == 0 and data[3] == address[1]:
                if len(address) == 3:
                    if data[4] == address[2]:
                        return addresses_json["uik #"]
                    else:
                        # TODO: mind about return
                        return None
                else:
                    return addresses_json["uik #"]
    return None

def is_vrn_in_vrns(vrn, vrns):
    for v in vrns:
        if vrn in v.keys():
            return 1
    return 0

def get_address_info(address: str, quantity_return=2):
    format_address = f"%20".join(address.split(" "))
    url = f"http://www.cikrf.ru/iservices/voter-services/address/search/{format_address}"
    addresses_info = json.loads(requests.get(url, headers=headers).text)
    vrns = set()
    probably_addresses = {}
    id = 0
    for address in addresses_info:
        try:
            uik = get_uik_by_add_id(address['id'])
        except json.decoder.JSONDecodeError:
            continue
        vrn = uik["vrn"]
        vrns.add(vrn)
        if quantity_return == 1:
            return vrns
        #if address["leaf"]:
        suff = address['name'].rfind(',')
        pref = address['name'][:suff]
        pref = pref[pref.find(', ') + 1:]
        pref = pref[pref.find(', ') + 1:]
        pref = pref[pref.find(', ') + 2:]
        probably_addresses[pref[:suff]] = {'id': id, 'uik': {'name': uik['name'], 'address': uik["votingAddress"]["address"], "vrn": uik["vrn"], "lat": uik["votingAddress"]["lat"], "lon": uik["votingAddress"]["lon"]}}
        id += 1
    if quantity_return == 1:
        return vrns
    return vrns, probably_addresses

def get_uik_by_add_id(id):
    url = f"http://www.cikrf.ru/iservices/voter-services/committee/address/{str(id)}"
    response = requests.get(url, headers=headers).text
    return json.loads(response)

def get_uik_info(uik_num):
    url = "http://cikrf.ru/iservices/voter-services/committee/subjcode/77/num/" + str(uik_num)
    return json.loads(requests.get(url, headers=headers).text)

def get_list_of_candidates(UIK_VRN):
    list_of_candidates = []
    CAMPAIGN_VRN = json.loads(requests.get(f"http://cikrf.ru/iservices/voter-services/vibory/committee/{str(UIK_VRN)}",
                                           headers=headers).text)[0]["vrn"]
    mandates = get_nums_districts(CAMPAIGN_VRN)

    # requests.get("http://cikrf.ru/iservices/sgo-visual-rest/vibory/{CAMPAIGN_VRN}/tvd", headers=headers)
    PAGE_NUM = 0
    total_pages = 1
    while PAGE_NUM != total_pages:
        PAGE_NUM += 1
        candidates_json = json.loads(
            requests.get(f"http://cikrf.ru/iservices/sgo-visual-rest/vibory/{CAMPAIGN_VRN}/candidates/?page={PAGE_NUM}",
                         headers=headers).text)
        total_pages = candidates_json["page"]["totalPages"]
        for candidate in candidates_json["_embedded"]["candidateDtoList"]:
            list_of_candidates.append(candidate)
    return list_of_candidates, mandates

def find_uik(id):
    url = f"http://www.cikrf.ru/iservices/voter-services/committee/address/{str(id)}"

def get_mandates(url):
    response = requests.get(url, headers=headers)
    code = response.text
    code = code[code.find("tvdTreeJson = {"):]
    code = code[:code.find("\n")]
    code = code[len("tvdTreeJson = "):]
    mandates_info = json.loads(code)
    return mandates_info["children"]

def get_nums_districts(campaign_vrn):
    url = f"http://www.vybory.izbirkom.ru/region/izbirkom?action=show&root=1&vrn={campaign_vrn}&prver=0&pronetvd=null&region=77&sub_region=77"
    mandates = get_mandates(url)
    uiks = {}
    if mandates == []:
        #TODO
        response = requests.get(url, headers=headers)
        code = response.text
        code = code[code.find("tvdTreeJson = {"):]
        code = code[:code.find("\n")]
        code = code[len("tvdTreeJson = "):]
        mandates_info = json.loads(code)
    else:
        for m in mandates:
            m_name = m["text"]
            m_num = m_name[m_name.rfind(" ") + 1:]
            if m_num.find("№") != -1:
               m_num = m_num[m_num.find("№") + 1:]
            uiks[m_num] = get_list_uiks(m["href"])
    return uiks

def get_list_uiks(url_suffix):
    url = "http://www.vybory.izbirkom.ru/" + url_suffix
    mandates = get_mandates(url)
    list_num_uiks = []
    for m in mandates:
        if m["href"] == url_suffix:
            uiks = m["children"]
            break
    for u in uiks:
        uik_name = u["text"]
        list_num_uiks.append(uik_name[uik_name.rfind(" ") + 2:])
    return list_num_uiks

def is_add_exist(address):
    vrns = get_address_info(address, 1)
    return len(vrns) > 0

if __name__ == "__main__":
    # Город%20Москва%20Восточный%20административный%20округ%20район%20Ивановское%20КУПАВЕНСКИЙ%20Б.%20ПР.%202/
    # "http://www.cikrf.ru/iservices/voter-services/address/search/%D0%93%D0%BE%D1%80%D0%BE%D0%B4%20%D0%9C%D0%BE%D1%81%D0%BA%D0%B2%D0%B0%20%D0%A1%D0%B5%D0%B2%D0%B5%D1%80%D0%BE%D0%92%D0%BE%D1%81%D1%82%D0%BE%D1%87%D0%BD%D1%8B%D0%B9%20%D0%B0%D0%B4%D0%BC%D0%B8%D0%BD%D0%B8%D1%81%D1%82%D1%80%D0%B0%D1%82%D0%B8%D0%B2%D0%BD%D1%8B%D0%B9%20%D0%BE%D0%BA%D1%80%D1%83%D0%B3%20%D1%80%D0%B0%D0%B9%D0%BE%D0%BD%20%D0%9C%D0%B0%D1%80%D1%8C%D0%B8%D0%BD%D0%B0%20%D1%80%D0%BE%D1%89%D0%B0%20%D0%94%D0%9E%D0%A1%D0%A2%D0%9E%D0%95%D0%92%D0%A1%D0%9A%D0%9E%D0%93%D0%9E%20%D0%A3%D0%9B%20%D0%B4%204%20%D0%BA%202/"
    # Город%20Москва%20Восточный%20административный%20округ%20район%20Гольяново%20БАЙКАЛЬСКАЯ%20УЛ%2044%20д%203/
    # f"http://www.cikrf.ru/iservices/voter-services/committee/address/141333300347676160000378975"

    #print(get_address_info("Город Москва Восточный административный округ район Ивановское КУПАВЕНСКИЙ Б. ПР. 2"))

    #print(get_list_of_candidates(4774012116710))
    print(get_nums_districts(4774004309365))
    # TODO: сделать словарь {vrn:[num, list_of_candidates]}
    # TODO: пройти по всем uik(4000 + проверка) и выкачать все данные

    # get_list_areas("http://ginfo.ru/ulicy/" + '?okrug=4', "", 1)
    # with open("addresses.json", "w") as f:
    #    f.write(str(get_list_streets("http://ginfo.ru/ulicy/")))


import chainer
import chainer.links as L
import chainer.functions as F
from chainer import cuda, optimizers, serializers
import numpy as np
import argparse
import pylab
import cv2 as cv
import os
from model import Generator,Discriminator
from prepare import prepare_dataset, prepare_mask
import requests

xp=cuda.cupy
cuda.get_device(0).use()

def set_optimizer(model,alpha=0.0002,beta1=0.5, beta2=0.999):
    optimizer=optimizers.Adam(alpha=alpha,beta1=beta1, beta2=beta2)
    optimizer.setup(model)

    return optimizer

parser=argparse.ArgumentParser(description="HairStyleTransfer")
parser.add_argument("--epochs",default=1000,type=int,help="the number of epochs")
parser.add_argument("--iterations",default=2000,type=int,help="the number of iteration")
parser.add_argument("--batchsize",default=16,type=int,help="batch size")
parser.add_argument("--testsize",default=3,type=int,help="test size")
parser.add_argument("--cw",default=10.0,type=float,help="the weight of cycle loss")
parser.add_argument("--iw",default=10.0,type=float,help="the weigt of identity loss")
parser.add_argument("--cpl",default=10.0,type=float,help="the weight of context preserving loss")
parser.add_argument("--hpl",default=10.0,type=float,help="the weigt of haircolor preserving loss")
parser.add_argument("--Ntrain",default=850,type=int,help="the number of training images")
parser.add_argument("--interval",default=5,type=int,help="the interval of snapshot")
parser.add_argument("--size",default=128,type=int,help="image width")
parser.add_argument("--cluster",default=2,type=int,help="the number of clusters")

args=parser.parse_args()
epochs=args.epochs
iterations=args.iterations
batchsize=args.batchsize
testsize=args.testsize
cycle_weight=args.cw
identity_weight=args.iw
cp_weight=args.cpl
hp_weight=args.hpl
Ntrain=args.Ntrain
interval=args.interval
size=args.size
cluster=args.cluster

x_path="/medium/"
y_path="/twin/"
x_tag_path="/medium_mask/"
y_tag_path="/twin_mask/"
x_list=os.listdir(x_tag_path)
y_list=os.listdir(y_tag_path)
Nx = len(x_list) - 50
Ny = len(y_list)

test_box=[]
binary_box=[]
for _ in range(testsize):
    rnd=np.random.randint(Nx,Nx+50)
    filename=x_tag_path+x_list[rnd]
    binary=prepare_mask(filename,size,cluster)
    filename=x_path+x_list[rnd]
    image=prepare_dataset(filename,size,cluster)
    test_box.append(image)
    binary_box.append(binary)

test_img=chainer.as_variable(xp.array(test_box).astype(xp.float32))
test_mask=chainer.as_variable(xp.array(binary_box).astype(xp.float32))

outdir="./output"
if not os.path.exists(outdir):
    os.mkdir(outdir)

generator_xy = Generator()
generator_xy.to_gpu()
gen_xy_opt = set_optimizer(generator_xy)

generator_yx = Generator()
generator_yx.to_gpu()
gen_yx_opt = set_optimizer(generator_yx)

discriminator_xy = Discriminator()
discriminator_xy.to_gpu()
dis_xy_opt = set_optimizer(discriminator_xy, alpha=0.0001)

discriminator_yx = Discriminator()
discriminator_yx.to_gpu()
dis_yx_opt = set_optimizer(discriminator_yx,alpha=0.0001)

for epoch in range(epochs):
    sum_dis_loss=0
    sum_gen_loss=0
    for batch in range(0,iterations,batchsize):
        x_box=[]
        x_binary_box=[]
        y_box=[]
        y_binary_box=[]
        for index in range(batchsize):
            rnd=np.random.randint(1,Nx)
            aug=np.random.randint(2)
            filename=x_tag_path+x_list[rnd]
            binary=prepare_mask(filename,size,aug)
            filename=x_path+x_list[rnd]
            image=prepare_dataset(filename,size,aug)

            x_box.append(image)
            x_binary_box.append(binary)
            rnd=np.random.randint(1,Ny)
            aug=np.random.randint(2)
            filename=y_tag_path+y_list[rnd]
            binary=prepare_mask(filename,size,aug)
            filename=y_path+y_list[rnd]
            image=prepare_dataset(filename,size,aug)
            y_box.append(image)
            y_binary_box.append(binary)

        x_img=chainer.as_variable(xp.array(x_box).astype(xp.float32))
        x_mask=chainer.as_variable(xp.array(x_binary_box).astype(xp.float32))
        y_img=chainer.as_variable(xp.array(y_box).astype(xp.float32))
        y_mask=chainer.as_variable(xp.array(y_binary_box).astype(xp.float32))

        x_y_img, x_y_mask=generator_xy(x_img, x_mask)
        y_x_img, y_x_mask=generator_yx(y_img, y_mask)

        y_real=discriminator_xy(y_img, y_mask)
        y_fake=discriminator_xy(x_y_img, x_y_mask)
        x_real=discriminator_yx(x_img, x_mask)
        x_fake=discriminator_yx(y_x_img, y_x_mask)

        dis_loss_y=F.mean(F.softplus(-y_real)) + F.mean(F.softplus(y_fake))
        #dis_loss_y = 0.5 * (F.mean((y_real-1.0)**2) + F.mean(y_fake**2))
        dis_loss_x=F.mean(F.softplus(-x_real)) + F.mean(F.softplus(x_fake))
        #dis_loss_x = 0.5 * (F.mean((x_real-1.0)**2) + F.mean(x_fake**2))

        x_y_img.unchain_backward()
        x_y_mask.unchain_backward()
        y_x_img.unchain_backward()
        y_x_mask.unchain_backward()

        discriminator_xy.cleargrads()
        dis_loss_y.backward()
        dis_xy_opt.update()
        dis_loss_y.unchain_backward()

        discriminator_yx.cleargrads()
        dis_loss_x.backward()
        dis_yx_opt.update()
        dis_loss_x.unchain_backward()

        x_y_img, x_y_mask=generator_xy(x_img, x_mask)
        x_y_x_img, x_y_x_mask=generator_yx(x_y_img, x_y_mask)

        y_x_img, y_x_mask=generator_yx(y_img, y_mask)
        y_x_y_img, y_x_y_mask=generator_xy(y_x_img, y_x_mask)

        y_fake=discriminator_xy(x_y_img, x_y_mask)
        x_fake=discriminator_yx(y_x_img, y_x_mask)

        #gen_loss=F.mean(F.softplus(-y_fake))
        gen_loss = 0.5 * F.mean((y_fake-1.0)**2)
        #gen_loss+=F.mean(F.softplus(-x_fake))
        gen_loss += 0.5 * F.mean((x_fake-1.0)**2)

        cycle_loss_x=F.mean_absolute_error(x_img,x_y_x_img) + F.mean_absolute_error(x_mask, x_y_x_mask)
        cycle_loss_y=F.mean_absolute_error(y_img,y_x_y_img) + F.mean_absolute_error(y_mask, y_x_y_mask)

        x_idt_img, x_idt_mask = generator_yx(x_img, x_mask)
        y_idt_img, y_idt_mask = generator_xy(y_img, y_mask)

        identity_loss_x=F.mean_absolute_error(x_idt_img,x_img) + F.mean_absolute_error(x_idt_mask, x_mask)
        identity_loss_y=F.mean_absolute_error(y_idt_img,y_img) + F.mean_absolute_error(y_idt_mask, y_mask)

        ctx_weight_x=F.tile(xp.ones_like(x_mask) - x_mask*x_y_mask, (1,3,1,1))
        context_preserve_loss_x = F.mean_absolute_error(ctx_weight_x * x_img, ctx_weight_x * x_y_img)
        ctx_weight_y=F.tile(xp.ones_like(y_mask) - y_mask*y_x_mask, (1,3,1,1))
        context_preserve_loss_y = F.mean_absolute_error(ctx_weight_y * y_img, ctx_weight_y * y_x_img)

        #haircolor_preserve_loss_x = F.mean_absolute_error(F.average_pooling_2d(x_img*F.tile(x_mask,(1,3,1,1)),(128,128)), F.average_pooling_2d(x_y_img*F.tile(x_y_mask,(1,3,1,1)),(128,128)))
        #haircolor_preserve_loss_y = F.mean_absolute_error(F.average_pooling_2d(y_img*F.tile(y_mask,(1,3,1,1)),(128,128)), F.average_pooling_2d(y_x_img*F.tile(y_x_mask,(1,3,1,1)),(128,128)))

        gen_loss+=cycle_weight*(cycle_loss_x+cycle_loss_y) + identity_weight*(identity_loss_x+identity_loss_y)
        gen_loss+=cp_weight*(context_preserve_loss_x+context_preserve_loss_y)

        generator_xy.cleargrads()
        generator_yx.cleargrads()
        gen_loss.backward()
        gen_xy_opt.update()
        gen_yx_opt.update()
        gen_loss.unchain_backward()

        sum_dis_loss=dis_loss_x.data.get() + dis_loss_y.data.get()
        sum_gen_loss=gen_loss.data.get()

        if epoch%interval==0 and batch==0:
            pylab.rcParams['figure.figsize'] = (16.0,16.0)
            pylab.clf()

            serializers.save_npz("generator_xy_lsgan.model",generator_xy)
            serializers.save_npz("generator_yx_lsgan.model",generator_yx)

            with chainer.using_config("train",False):
                y_img, y_mask=generator_xy(test_img, test_mask)
                y_img_t=y_img.data.get()
                y_mask_t=y_mask.data.get()
                t_img=test_img.data.get()
                t_mask=test_mask.data.get()

                y_img.unchain_backward()
                y_mask.unchain_backward()

                for i in range(testsize):
                    tmp=(np.clip(t_img[i]*127.5+127.5,0,255)).transpose(1,2,0).astype(np.uint8)
                    pylab.subplot(testsize,4,4*i+1)
                    pylab.imshow(tmp)
                    pylab.axis("off")
                    pylab.savefig("%s/visualize_%d.png"%(outdir,epoch))
                    tmp=(np.clip(t_mask[i]*255.0,0,255)).reshape(128,128).astype(np.uint8)
                    pylab.subplot(testsize,4,4*i+2)
                    pylab.imshow(tmp, cmap='gray')
                    pylab.axis("off")
                    pylab.savefig("%s/visualize_%d.png"%(outdir,epoch))
                    tmp=np.clip(y_img_t[i]*127.5+127.5,0,255).transpose(1,2,0).astype(np.uint8)
                    pylab.subplot(testsize,4,4*i+3)
                    pylab.imshow(tmp)
                    pylab.axis("off")
                    pylab.savefig("%s/visualize_%d.png"%(outdir,epoch))
                    tmp=np.clip(y_mask_t[i]*255.0,0,255).reshape(128,128).astype(np.uint8)
                    pylab.subplot(testsize,4,4*i+4)
                    pylab.imshow(tmp, cmap='gray')
                    pylab.axis("off")
                    pylab.savefig("%s/visualize_%d.png"%(outdir,epoch))

    print("epoch:{}".format(epoch))
    print("Discriminator loss:{}".format(sum_dis_loss/iterations))
    print("Generator loss:{}".format(sum_gen_loss/iterations))


import cv2
import numpy as np
import scipy.io as sio
from scipy.linalg import null_space
import matplotlib.pyplot as plt
import sys
from sklearn.neighbors import NearestNeighbors
from mpl_toolkits.mplot3d import Axes3D

def find_match(img1, img2):
    x1 = np.zeros((0,2))
    x2 = np.zeros((0,2))
    # key points and descriptors
    sift = cv2.xfeatures2d.SIFT_create()
    kp1, des1 = sift.detectAndCompute(img1,None)
    kp2, des2 = sift.detectAndCompute(img2,None)
    # translate key points to actual x, y coordinates
    pts1 = np.array([kp1[idx].pt for idx in range(0, len(kp1))]).reshape(-1, 2)
    pts2 = np.array([kp2[idx].pt for idx in range(0, len(kp2))]).reshape(-1, 2)
    # nearest neighbors fitting larger image
    neigh = NearestNeighbors(n_neighbors=2)
    neigh.fit(des2)
    distances, indices = neigh.kneighbors(des1)
    for i in range(len(distances)):
        # ratio test
        if distances[i, 0]/distances[i, 1] < 0.7:
            x1 = np.append(x1, [pts1[i]], axis = 0)
            x2 = np.append(x2, [pts2[indices[i,0]]], axis = 0)

    return x1, x2

def compute_F(pts1, pts2):
    ransac_iter = 1000
    threshold = 0.05
    max_inliers = -1
    F = None
    for i in range(ransac_iter):
        eight_point = np.random.choice(pts1.shape[0], 8, replace=False)
        pts1_eight = pts1[eight_point, :]
        pts2_eight = pts2[eight_point, :]
        A = np.zeros((8, 9))
        for j in range(8):
            u = pts1_eight[j]
            v = pts2_eight[j]
            A[j, 0] = u[0] * v[0]
            A[j, 1] = u[1] * v[0]
            A[j, 2] = v[0]
            A[j, 3] = u[0] * v[1]
            A[j, 4] = u[1] * v[1]
            A[j, 5] = v[1]
            A[j, 6] = u[0]
            A[j, 7] = u[1]
            A[j, 8] = 1

        F_1 = null_space(A)
        F_1 = F_1[:, 0].reshape(3, 3)

        u, s, vh = np.linalg.svd(F_1)
        s[2] = 0
        F_clean = np.matmul( np.matmul(u, np.diag(s)) , vh)

        num_inlier = 0
        for j in range(len(pts1)):
            u = np.array([pts1[j,0], pts1[j,1], 1])
            v = np.array([pts2[j,0], pts2[j,1], 1])
            # err = np.matmul( np.matmul(v.T, F_clean) , u)
            err = np.matmul(v.T, np.matmul(F_clean, u))
            if np.abs(err) < threshold:
                num_inlier += 1

        if num_inlier > max_inliers:
            max_inliers = num_inlier
            F = F_clean

    # F = np.asarray(F)
    print(num_inlier)
    return F

def skew(vector):
    a, b, c = vector
    skew = np.asarray([
        [0, -c, b],
        [c, 0, -a],
        [-b, a, 0],
    ])
    return skew

def triangulation(P1, P2, pts1, pts2):
    # TO DO
    pts3D = np.zeros((len(pts1), 3))
    for i in range(len(pts1)):
        u = skew(np.append(pts1[i,:], 1))
        v = skew(np.append(pts2[i,:], 1))

        u_P1 = np.matmul(u, P1)
        v_P2 = np.matmul(v, P2)

        A = np.concatenate((u_P1, v_P2), axis=0)
        x = null_space(A, rcond=0.1)
        x = x[:,0]
        x = x/x[3]
        pts3D[i,:] = x[:3]

    return pts3D

def disambiguate_pose(Rs, Cs, pts3Ds):
    # TO DO
    best_idx = -1
    max_pts = -1

    for i in range(len(Rs)):
        nValid = 0
        c = Cs[i].reshape((3,))
        r = Rs[i][2,:]

        for p in pts3Ds[i]:
            cam_1 = (p-c)[2]
            cam_2 = np.matmul(p-c, r)
            if cam_1 > 0 and cam_2 > 0:
                nValid += 1
        if nValid > max_pts:
            max_pts = nValid
            best_idx = i

    R = Rs[best_idx]
    C = Cs[best_idx]
    pts3D = pts3Ds[best_idx]
    return R, C, pts3D

def compute_rectification(K, R, C):
    # TO DO
    # C = C.reshape(3,)
    C = C.reshape(-1)

    R_x = -C / np.linalg.norm(C)
    z_tilde = np.array([0,0,1])
    R_z = z_tilde - R_x * np.dot(z_tilde, R_x)
    R_z = R_z / np.linalg.norm(R_z)
    R_y = np.cross(R_z, R_x)

    R_rect = np.asarray([R_x, R_y, R_z])

    H1 = K @ R_rect @ np.linalg.inv(K)
    H2 = K @ R_rect @ R.T @ np.linalg.inv(K)
    return H1, H2

def dense_match(img1, img2):
    # TO DO
    sift = cv2.xfeatures2d.SIFT_create()
    size = 10
    kp = []
    for i in range(img1.shape[0]):
        for j in range(img1.shape[1]):
            kp.append(cv2.KeyPoint(j, i, size))

    kp_1, des_1 = sift.compute(img1, kp)

    kp = []
    for i in range(img2.shape[0]):
        for j in range(img2.shape[1]):
            kp.append(cv2.KeyPoint(j, i, size))

    kp_2, des_2 = sift.compute(img2, kp)

    des_1 = des_1.reshape((img1.shape[0], img1.shape[1], 128))
    des_2 = des_2.reshape((img2.shape[0], img2.shape[1], 128))
    disparity = np.zeros(img1.shape)

    for u in range(img1.shape[0]):
        for i in range(img1.shape[1]):
            if img1[u, i] == 0:
                continue
            d_1 = des_1[u, i]
            dist = []
            for j in range(i+1):
                d_2 = des_2[u, j]
                dist.append(np.linalg.norm(d_1 - d_2)**2)
            disparity[u, i] = np.abs(np.argmin(dist) - i)

    return disparity

# PROVIDED functions
def compute_camera_pose(F, K):
    E = K.T @ F @ K
    R_1, R_2, t = cv2.decomposeEssentialMat(E)
    # 4 cases
    R1, t1 = R_1, t
    R2, t2 = R_1, -t
    R3, t3 = R_2, t
    R4, t4 = R_2, -t

    Rs = [R1, R2, R3, R4]
    ts = [t1, t2, t3, t4]
    Cs = []
    for i in range(4):
        Cs.append(-Rs[i].T @ ts[i])
    return Rs, Cs

def visualize_img_pair(img1, img2):
    img = np.hstack((img1, img2))
    if img1.ndim == 3:
        plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
    else:
        plt.imshow(img, cmap='gray')
    plt.axis('off')
    plt.show()

def visualize_find_match(img1, img2, pts1, pts2):
    assert pts1.shape == pts2.shape, 'x1 and x2 should have same shape!'
    img_h = img1.shape[0]
    scale_factor1 = img_h/img1.shape[0]
    scale_factor2 = img_h/img2.shape[0]
    img1_resized = cv2.resize(img1, None, fx=scale_factor1, fy=scale_factor1)
    img2_resized = cv2.resize(img2, None, fx=scale_factor2, fy=scale_factor2)
    pts1 = pts1 * scale_factor1
    pts2 = pts2 * scale_factor2
    pts2[:, 0] += img1_resized.shape[1]
    img = np.hstack((img1_resized, img2_resized))
    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
    for i in range(pts1.shape[0]):
        plt.plot([pts1[i, 0], pts2[i, 0]], [pts1[i, 1], pts2[i, 1]], 'b.-', linewidth=0.5, markersize=5)
    plt.axis('off')
    plt.show()

def visualize_epipolar_lines(F, pts1, pts2, img1, img2):
    assert pts1.shape == pts2.shape, 'x1 and x2 should have same shape!'
    ax1 = plt.subplot(121)
    ax2 = plt.subplot(122)
    ax1.imshow(cv2.cvtColor(img1, cv2.COLOR_BGR2RGB))
    ax2.imshow(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB))

    for i in range(pts1.shape[0]):
        x1, y1 = int(pts1[i][0] + 0.5), int(pts1[i][1] + 0.5)
        ax1.scatter(x1, y1, s=5)
        p1, p2 = find_epipolar_line_end_points(img2, F, (x1, y1))
        ax2.plot([p1[0], p2[0]], [p1[1], p2[1]], linewidth=0.5)

    for i in range(pts2.shape[0]):
        x2, y2 = int(pts2[i][0] + 0.5), int(pts2[i][1] + 0.5)
        ax2.scatter(x2, y2, s=5)
        p1, p2 = find_epipolar_line_end_points(img1, F.T, (x2, y2))
        ax1.plot([p1[0], p2[0]], [p1[1], p2[1]], linewidth=0.5)

    ax1.axis('off')
    ax2.axis('off')
    plt.show()

def find_epipolar_line_end_points(img, F, p):
    img_width = img.shape[1]
    el = np.dot(F, np.array([p[0], p[1], 1]).reshape(3, 1))
    p1, p2 = (0, -el[2] / el[1]), (img.shape[1], (-img_width * el[0] - el[2]) / el[1])
    _, p1, p2 = cv2.clipLine((0, 0, img.shape[1], img.shape[0]), p1, p2)
    return p1, p2

def visualize_camera_poses(Rs, Cs):
    assert(len(Rs) == len(Cs) == 4)
    fig = plt.figure()
    R1, C1 = np.eye(3), np.zeros((3, 1))
    for i in range(4):
        R2, C2 = Rs[i], Cs[i]
        ax = fig.add_subplot(2, 2, i+1, projection='3d')
        draw_camera(ax, R1, C1)
        draw_camera(ax, R2, C2)
        set_axes_equal(ax)
        ax.set_xlabel('x axis')
        ax.set_ylabel('y axis')
        ax.set_zlabel('z axis')
        ax.view_init(azim=-90, elev=0)
    fig.tight_layout()
    plt.show()

def visualize_camera_poses_with_pts(Rs, Cs, pts3Ds):
    assert(len(Rs) == len(Cs) == 4)
    fig = plt.figure()
    R1, C1 = np.eye(3), np.zeros((3, 1))
    for i in range(4):
        R2, C2, pts3D = Rs[i], Cs[i], pts3Ds[i]
        ax = fig.add_subplot(2, 2, i+1, projection='3d')
        draw_camera(ax, R1, C1, 5)
        draw_camera(ax, R2, C2, 5)
        ax.plot(pts3D[:, 0], pts3D[:, 1], pts3D[:, 2], 'b.')
        set_axes_equal(ax)
        ax.set_xlabel('x axis')
        ax.set_ylabel('y axis')
        ax.set_zlabel('z axis')
        ax.view_init(azim=-90, elev=0)
    fig.tight_layout()
    plt.show()

def draw_camera(ax, R, C, scale=0.2):
    axis_end_points = C + scale * R.T  # (3, 3)
    vertices = C + scale * R.T @ np.array([[1, 1, 1], [-1, 1, 1], [-1, -1, 1], [1, -1, 1]]).T  # (3, 4)
    vertices_ = np.hstack((vertices, vertices[:, :1]))  # (3, 5)

    # draw coordinate system of camera
    ax.plot([C[0], axis_end_points[0, 0]], [C[1], axis_end_points[1, 0]], [C[2], axis_end_points[2, 0]], 'r-')
    ax.plot([C[0], axis_end_points[0, 1]], [C[1], axis_end_points[1, 1]], [C[2], axis_end_points[2, 1]], 'g-')
    ax.plot([C[0], axis_end_points[0, 2]], [C[1], axis_end_points[1, 2]], [C[2], axis_end_points[2, 2]], 'b-')

    # draw square window and lines connecting it to camera center
    ax.plot(vertices_[0, :], vertices_[1, :], vertices_[2, :], 'k-')
    ax.plot([C[0], vertices[0, 0]], [C[1], vertices[1, 0]], [C[2], vertices[2, 0]], 'k-')
    ax.plot([C[0], vertices[0, 1]], [C[1], vertices[1, 1]], [C[2], vertices[2, 1]], 'k-')
    ax.plot([C[0], vertices[0, 2]], [C[1], vertices[1, 2]], [C[2], vertices[2, 2]], 'k-')
    ax.plot([C[0], vertices[0, 3]], [C[1], vertices[1, 3]], [C[2], vertices[2, 3]], 'k-')

def set_axes_equal(ax):
    x_limits = ax.get_xlim3d()
    y_limits = ax.get_ylim3d()
    z_limits = ax.get_zlim3d()

    x_range, x_middle = abs(x_limits[1] - x_limits[0]), np.mean(x_limits)
    y_range, y_middle = abs(y_limits[1] - y_limits[0]), np.mean(y_limits)
    z_range, z_middle = abs(z_limits[1] - z_limits[0]), np.mean(z_limits)

    plot_radius = 0.5*max([x_range, y_range, z_range])

    ax.set_xlim3d([x_middle - plot_radius, x_middle + plot_radius])
    ax.set_ylim3d([y_middle - plot_radius, y_middle + plot_radius])
    ax.set_zlim3d([z_middle - plot_radius, z_middle + plot_radius])

def visualize_disparity_map(disparity):
    plt.imshow(disparity, cmap='jet')
    plt.show()

if __name__ == '__main__':
    # read in left and right images as RGB images
    img_left = cv2.imread('./left.bmp', 1)
    img_right = cv2.imread('./right.bmp', 1)
    # visualize_img_pair(img_left, img_right)

    # Step 1: find correspondences between image pair
    pts1, pts2 = find_match(img_left, img_right)
    # visualize_find_match(img_left, img_right, pts1, pts2)

    # Step 2: compute fundamental matrix
    F = compute_F(pts1, pts2)
    # visualize_epipolar_lines(F, pts1, pts2, img_left, img_right)

    # Step 3: computes four sets of camera poses
    K = np.array([[350, 0, 960/2], [0, 350, 540/2], [0, 0, 1]])
    Rs, Cs = compute_camera_pose(F, K)
    # visualize_camera_poses(Rs, Cs)

    # Step 4: triangulation
    pts3Ds = []
    P1 = K @ np.hstack((np.eye(3), np.zeros((3, 1))))
    for i in range(len(Rs)):
        P2 = K @ np.hstack((Rs[i], -Rs[i] @ Cs[i]))
        pts3D = triangulation(P1, P2, pts1, pts2)
        pts3Ds.append(pts3D)
    # visualize_camera_poses_with_pts(Rs, Cs, pts3Ds)

    # Step 5: disambiguate camera poses
    R, C, pts3D = disambiguate_pose(Rs, Cs, pts3Ds)

    # Step 6: rectification
    H1, H2 = compute_rectification(K, R, C)
    img_left_w = cv2.warpPerspective(img_left, H1, (img_left.shape[1], img_left.shape[0]))
    img_right_w = cv2.warpPerspective(img_right, H2, (img_right.shape[1], img_right.shape[0]))
    # visualize_img_pair(img_left_w, img_right_w)

    # Step 7: generate disparity map
    img_left_w = cv2.resize(img_left_w, (int(img_left_w.shape[1] / 2), int(img_left_w.shape[0] / 2)))  # resize image for speed
    img_right_w = cv2.resize(img_right_w, (int(img_right_w.shape[1] / 2), int(img_right_w.shape[0] / 2)))
    img_left_w = cv2.cvtColor(img_left_w, cv2.COLOR_BGR2GRAY)  # convert to gray scale
    img_right_w = cv2.cvtColor(img_right_w, cv2.COLOR_BGR2GRAY)
    disparity = dense_match(img_left_w, img_right_w)
    # visualize_disparity_map(disparity)

    # # save to mat
    sio.savemat('stereo.mat', mdict={'pts1': pts1, 'pts2': pts2, 'F': F, 'pts3D': pts3D, 'H1': H1, 'H2': H2,
                                     'img_left_w': img_left_w, 'img_right_w': img_right_w, 'disparity': disparity})


#!/usr/bin/env python
from runtime_context import LOGGER
import boto3
from lib import ddb

DDB_CLIENT = boto3.client('dynamodb')

class FileModel(ddb.Model):
    """file model"""

    _TABLE_NAME = 'Files'
    _FIELDS = [
        ('id', ddb.StringAttribute),
        ('uploaded_at', ddb.DateTimeAttribute),
        ('stored_at', ddb.DateTimeAttribute),
        ('deleted_at', ddb.DateTimeAttribute),
        ('name', ddb.StringAttribute),
        ('type', ddb.StringAttribute),
        ('size', ddb.IntegerAttribute),
        ('meta', ddb.JsonAttribute)
    ]

    @classmethod
    def create(cls, data, **kwargs):
        """create file item"""
        put_data = cls.serialize(data)
        response = DDB_CLIENT.put_item(
            TableName=cls._TABLE_NAME,
            Item=put_data
        )
        return response

    @classmethod
    def get_by_id(cls, id):
        """get item with id"""
        response = DDB_CLIENT.get_item(
            TableName=cls._TABLE_NAME,
            Key={
                'id': {
                    'S': id
                }
            }
        )
        get_data = cls.deserialize(response['Item'])
        return get_data

    @classmethod
    def get_by_ids(cls, id_list):
        """get list of files by ids
        supports up to 20 ids
        """
        id_list = id_list[:20]  # shorten the list to 20
        request_items = {
            'Files': {
                'Keys': []
            }
        }
        for id_item in id_list:
            request_items['Files']['Keys'].append({
                'id': {
                    'S': id_item
                }
            })
        response = DDB_CLIENT.batch_get_item(RequestItems=request_items)
        get_data = []
        for file in response['Responses']['Files']:
            get_data.append(cls.deserialize(file))
        return get_data

    @classmethod
    def update(cls, data):
        """update an item, it only supports setting attribute (not removing)"""
        id = data.pop('id')
        ref = 65
        attr_names = {}
        attr_values = {}
        expressions = []
        for field_name, (field_serializer, field_deserializer) in cls._FIELDS:
            value = data.get(field_name)
            if value is not None:
                ref_chr = chr(ref)
                attr_names['#' + ref_chr] = field_name
                attr_values[':' + ref_chr] = field_serializer(value)
                expressions.append('#{} = :{}'.format(ref_chr, ref_chr))
                ref += 1
        response = DDB_CLIENT.update_item(
            TableName=cls._TABLE_NAME,
            Key={
                'id': {
                    'S': id
                }
            },
            ExpressionAttributeNames=attr_names,
            ExpressionAttributeValues=attr_values,
            UpdateExpression='SET ' + ','.join(expressions)
        )
        return response

    @classmethod
    def list_expired(cls, expiry_at):
        """list expired files"""
        response = DDB_CLIENT.scan(
            TableName=cls._TABLE_NAME,
            ExpressionAttributeNames={
                '#ID': 'id'
            },
            ProjectionExpression='#ID',
            ExpressionAttributeValues={
                ':expiry_at': {
                    'S': expiry_at.isoformat()
                }
            },
            FilterExpression='uploaded_at < :expiry_at and attribute_not_exists(stored_at)'
        )
        return response.get('Items')

    @classmethod
    def delete_by_id(cls, id):
        """delete item by id"""
        response = DDB_CLIENT.delete_item(
            TableName=cls._TABLE_NAME,
            Key={
                'id': {
                    'S': id
                }
            }
        )
        return response


import pandas as pd
import numpy as np
import hashlib
import heapq as hq
from datetime import datetime
from scipy.sparse import dok_matrix
from collections import Counter, defaultdict
import math

from nltk.stem import PorterStemmer
from nltk.tokenize import RegexpTokenizer
import nltk
nltk.download('stopwords', quiet=True)
from nltk.corpus import stopwords

wd_tokens = RegexpTokenizer(r'\w+')
stop_words = set(stopwords.words('english'))
ps = PorterStemmer()

class Engine:
    '''Class that represents a Simple Search Engine'''
    SIZE = 2**63 - 1 # max size of the hash
    def __init__(self, domain:pd.DataFrame, out_columns:list):
        self.domain = domain
        self.corpus = None
        self.out_columns = out_columns
        self.matrix = dok_matrix((Engine.SIZE,domain.index.max()+1), dtype=np.float32)

    def fit(self, corpus_columns:list):
        '''fits the engine to a corpus of documents, by creating a term indicator matrix'''
        self.corpus = self.domain[corpus_columns]
        for _,search_field in self.corpus.items():
            search_field.to_frame()\
                .apply(
                    lambda doc: self.document_process(doc.name, doc.iloc[0]),
                    axis=1)
        return self

    def document_process(self, doc_id, doc):
        '''Processes each document'''
        for term in Engine.get_terms(doc):
            self.add_term(term, doc_id)

    def add_term(self, term, doc, value=1):
        self.matrix[term, doc] += value

    def search(self, query, k=15):
        '''Returns all the documents that contain all the terms in the query'''
        terms = Engine.get_terms(query)
        docs_score = self.matrix[terms].sum(axis=0).A1
        docs = np.where(docs_score == len(terms))[0]
        return self.domain[self.out_columns].iloc[docs].iloc[:k]

    @staticmethod
    def clean_str(s, hash=False, tokenizer = wd_tokens.tokenize, stemmer = ps.stem):
        '''Cleans, stems, and eventually hashes a query string into a list of terms'''
        s = s.replace('\n', '')
        terms = [stemmer(word.lower()) for word in tokenizer(s) if word not in stop_words]
        if hash:
            return [Engine.hash_term(term) for term in terms]
        return terms

    @staticmethod
    def get_terms(q, hash=True):
        '''Returns the terms in the query without repetitions'''
        return list(set(Engine.clean_str(q, hash)))

    @staticmethod
    def hash_term(s):
        '''Hash of a string in int, with max size SIZE'''
        return int(hashlib.sha1(s.encode("utf-8")).hexdigest(), 16) % (Engine.SIZE)

class SimilarityEngine(Engine):
    '''This class represents an improved version of the orginal SimpleEngine, that obtains documents by similarity'''
    IDF = lambda N, df: np.log(N / df) + 1

    def __init__(self, domain:pd.DataFrame, out_columns:list):
        super().__init__(domain, out_columns)
        self.idf = None
        self.norm = dict()

    def document_process(self, doc_id, doc):
        v_doc = SimilarityEngine.term_frequency(doc)
        for term, tf in v_doc.items():
            self.add_term(term, doc_id, value=tf)

    def fit(self, corpus_columns:list):
        super().fit(corpus_columns)

        # Inverse Document Frequency
        self.idf = Counter([ i for i,_ in self.matrix.keys() ])
        for t, fr in self.idf.items():
            self.idf[t] = SimilarityEngine.IDF(self.matrix.shape[1], fr)

        for i,j in self.matrix.keys():
            self.matrix[i,j] *= self.idf[i]

        csc = self.matrix.tocsc()
        for doc_id in self.domain.index:
            self.norm[doc_id] = np.sqrt(np.sum([v**2 for v in csc.getcol(doc_id).data]))
        return self

    def retrieve_doc_vec(self, terms):
        '''Retrieves vector rappresentation of documents given some terms'''
        doc_vec = {}
        m = self.matrix[sorted(terms.keys()),:].tocsc()
        for i in m.nonzero()[1]:
            doc_vec[i] = m.getcol(i).toarray()[:,0]
        return doc_vec

    def search(self, query, k=10, all_columns=False):
        '''returns the k most similar documents to the query'''
        docs_score, _, _ = self.raw_search(query, k)
        docs = [doc_id for _, doc_id in hq.nlargest(k, docs_score)]

        res = self.domain.iloc[docs].copy()
        res['similarity'] = [sim for sim, _ in hq.nlargest(k, docs_score)]
        if all_columns:
            return res[res.similarity != 0]
        return res[self.out_columns + ['similarity']][res.similarity != 0]

    def similar_vectors(self, query, k=10):
        '''Returns the k most similar vectors to the query'''
        docs_score, query_vec, doc_vec = self.raw_search(query, k)
        vecs = {self.domain.placeName.iloc[id]:doc_vec[id]/np.linalg.norm(doc_vec[id]) for _, id in hq.nlargest(k, docs_score)}
        vecs['QUERY'] = query_vec / np.linalg.norm(query_vec)
        return vecs

    def raw_search(self, query, k=7200):
        '''Returns an heap with the similarity scores for the query'''
        terms = SimilarityEngine.term_frequency(query)
        query_vec = np.array([self.idf[t] * tf for t, tf in terms.items() ])

        doc_vec = self.retrieve_doc_vec(terms)

        docs_score = []
        for doc_id, d in doc_vec.items():
            similarity = SimilarityEngine.cosine_sim(d,query_vec)
            # keep only the first k values in the heap
            if k >= 0:
                k -= 1
                hq.heappush(docs_score, (similarity, doc_id))
            else:
                hq.heappushpop(docs_score, (similarity, doc_id))
        return docs_score, query_vec, doc_vec

    @staticmethod
    def term_frequency(d, hash=True):
        '''Returns the term frequency of a document in term:value pairs'''
        terms = Engine.clean_str(d, hash)
        v = Counter(terms)
        return {t:cnt/sum(v.values()) for t, cnt in v.items()}

    @staticmethod
    def vectorize(s):
        '''returns an array of the term frequencies for a given query'''
        tf = SimilarityEngine.term_frequency(s)
        return np.array(list(tf.values()))

    @staticmethod
    def cosine_sim(v1, v2, norm1=None):
        '''computes the cosine similarity between vectors'''
        n1 = np.linalg.norm(v1) if norm1 is None else norm1
        n2 = np.linalg.norm(v2)
        return v1.dot(v2) / n1 / n2 if n1 * n2 != 0  else 0

class RankEngine(SimilarityEngine):
    '''Searches and then ranks'''
    def __init__(self, domain:pd.DataFrame, out_columns:list,
            weights = {
                'editors_score' : 2,
                'wants_score' : 1,
                'visited_score' : 1,
                'related_score' : 0
            },
            ratio = 0.5,
            lat='placeAlt',
            lng='placeLng',
            time='placePubDate'):
        super().__init__(domain, out_columns)
        self.ratio = ratio
        self.set_weights(weights)
        self.time_column = time
        self.lng_column = lng
        self.lat_column = lat

    def set_weights(self, weights):
        '''updates a weight value'''
        self.weights = weights
        norm = sum(self.weights.values())
        for col, w in self.weights.items():
            self.domain[col] = self.normalize(self.domain[col])
            self.weights[col] /= norm
        self.domain['importance'] = self.domain.apply(self.importance, axis=1)

    def importance(self,row):
        '''computes the rank score of the document'''
        score = self.weights.copy()
        for col in self.weights.keys():
            score[col] *= row[col]
        return sum(score.values())

    def location_search(self, query, lat, lng, k =10):
        '''Searches the query and ranks by distance from the coordinates'''
        docs_sim, _, _ = super().raw_search(query)
        distance = self.domain[[self.lat_column, self.lng_column]].apply(
            lambda r: RankEngine.haversine_distance(r[0], r[1], lat, lng),
            axis=1)
        distance = 1 - RankEngine.normalize(distance)
        return self.present(docs_sim, distance, k, [self.lat_column,self.lng_column])

    def time_search(self, query, date=datetime.now(), k=10):
        '''Searches the query and ranks by distance from the given timestamp'''
        docs_sim, _, _ = super().raw_search(query)
        distance = np.abs((self.domain[self.time_column] - date).dt.total_seconds())
        distance = 1 - RankEngine.normalize(distance)
        return self.present(docs_sim, distance, k, [self.time_column])

    def search(self, query, k=10, columns = []):
        '''Searches the query and ranks by importance'''
        docs_sim, _, _ = super().raw_search(query)
        importance = self.domain.importance
        return self.present(docs_sim, importance, k , columns)

    def present(self, docs_sim, ranking, k, columns=[]):
        '''Takes an heap of similarity values as input and a ranking score and outputs a pandas dataframe'''
        docs_score = []
        for sim, doc_id in hq.nlargest(k*100, docs_sim):
            score = self.ratio*sim + (1-self.ratio) * ranking.iloc[doc_id]
            hq.heappush(docs_score, (score ,doc_id))

        docs = [doc_id for _, doc_id in hq.nlargest(k, docs_score)]
        res = self.domain.iloc[docs].copy()
        res['score'] = [score for score, _ in hq.nlargest(k, docs_score)]
        if columns == True:
            return res
        return res[self.out_columns + columns + ['score']].iloc[:k]

    @staticmethod
    def haversine_distance(lat_1, lng_1, lat_2, lng_2):
        '''Calculates distance between coordinates'''
        # https://stackoverflow.com/a/44743104/9419492
        lng_1, lat_1, lng_2, lat_2 = map(np.radians, [lng_1, lat_1, lng_2, lat_2])
        d_lat = lat_2 - lat_1
        d_lng = lng_2 - lng_1

        temp = math.sin(d_lat / 2) ** 2 + math.cos(lat_1) * math.cos(lat_2) * math.sin(d_lng / 2) ** 2
        return 6373.0 * (2 * math.atan2(math.sqrt(temp), math.sqrt(1 - temp)))

    @staticmethod
    def normalize(column):
        '''returns min-max normalization of an array or pandas series'''
        column -= column.min()
        return column / column.max()

class FilterEngine():
    '''Searches across multiple corpora of documents and then filters them '''
    def __init__(self, domain):
        self.domain = domain
        self.corpora = []
        self.searches = dict()
        self.corpora_weights = dict()

    def fit(self, corpora):
        self.corpora = corpora
        for col in corpora:
            self.searches[col] = SimilarityEngine(self.domain,['placeName'])
            self.searches[col].fit([col])
            self.corpora_weights[col] = 1/np.sqrt(len(self.searches[col].idf))
        self.corpora_weights = {k:v/sum(self.corpora_weights.values()) for k,v in self.corpora_weights.items()}

    def filter(self, row,
        editors = set(),
        tags = set(),
        min_visited = 0, max_visited = math.inf,
        lists = set()):
        '''returns True if all the constraints are respected'''
        rowEditors, rowTags, rowLists = map(lambda x: eval(x) if x is not np.nan else set(), [row.placeEditors, row.placeTags, row.placeRelatedLists])
        return editors.issubset(rowEditors)  \
            and tags.issubset(rowTags) \
            and lists.issubset(rowLists) \
            and row.numPeopleVisited > min_visited and row.numPeopleVisited < max_visited

    def similarities(self, queries):
        sim = pd.DataFrame()
        if len(queries) == 0:
            sim = sim.reindex(index=data.index)
            sim['similarity'] = 0
            return sim

        for col, query in queries.items():
            col_sim = self.searches[col].search(query, k=self.domain.shape[0])['similarity']
            col_sim.rename(col+'_sim', inplace=True)
            sim = pd.concat([sim,col_sim], axis='columns')
        sim['similarity'] = 0
        for col in queries.keys():
            sim['similarity'] = sim.similarity.add(self.corpora_weights[col]*sim[col+'_sim'], fill_value=0)
        return sim

    def search(
        self, queries = dict(),
        editors = [],
        tags = [],
        min_visited = 0, max_visited = math.inf,
        lists = [] ,
        columns=[],
        k = 15):
        '''Searches the across multiple corpuses and filters the results on the constraints'''
        sim = self.similarities({k:v for k,v in queries.items() if k in self.corpora})

        docs = pd.concat([ self.domain, sim ], axis='columns', join='inner')
        filtered = docs.apply(
            lambda r : self.filter(r, set(editors), set(tags), min_visited, max_visited, set(lists)),
            axis=1)
        res = docs[filtered].nlargest(k, 'similarity')
        if columns==True:
            return res
        return res[list(set(['placeName', 'placeUrl', 'similarity'] +list( queries.keys() )))]